sm
/
DepperDan
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Initial commit of revamped build scripts. All set to Apache License

main
Stephan Menzel 2025-06-23 15:39:05 +02:00
commit d62319e14b
52 changed files with 2661 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
build.log
.idea/
*.pyc

231
build_deps.py Normal file
View File

@ -0,0 +1,231 @@
#!/usr/bin/env python3
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import argparse
import json
import os
import datetime
import logging as log
import importlib
from build_functions.build_abseil import build_abseil
from build_functions.build_asio import build_asio
from build_functions.build_boost import build_boost
from build_functions.build_cares import build_cares
from build_functions.build_ceres import build_ceres
from build_functions.build_curl import build_curl
from build_functions.build_cppzmq import build_cppzmq
from build_functions.build_eigen import build_eigen
from build_functions.build_fineftp import build_fineftp
from build_functions.build_glog import build_glog
from build_functions.build_googletest import build_googletest
from build_functions.build_grpc import build_grpc
from build_functions.build_hdf5 import build_hdf5
from build_functions.build_jpeg import build_jpeg
from build_functions.build_libzmq import build_libzmq
from build_functions.build_matplot import build_matplot
from build_functions.build_onetbb import build_onetbb
from build_functions.build_opencv import build_opencv
from build_functions.build_openssl import build_openssl
from build_functions.build_opensubdiv import build_opensubdiv
from build_functions.build_openusd import build_openusd
from build_functions.build_protobuf import build_protobuf
from build_functions.build_qt5 import build_qt5
from build_functions.build_qt6 import build_qt6
from build_functions.build_qwt import build_qwt
from build_functions.build_recycle import build_recycle
from build_functions.build_spdlog import build_spdlog
from build_functions.build_tclap import build_tclap
from build_functions.build_re2 import build_re2
from build_functions.build_tcp_pubsub import build_tcp_pubsub
from build_functions.build_termcolor import build_termcolor
from build_functions.build_tinyxml2 import build_tinyxml2
from build_functions.build_yaml_cpp import build_yaml_cpp
from build_functions.build_zlib import build_zlib
import common.settings
from package.package_info import PackageInfo, get_package_info, build_package_tree
import sys
from pathlib import Path
from common.errors import DependencyInfoError
from build_functions.build_utils import print_banner, file_and_console_log, logfile
# Iterate over all Python files in the `build_functions` folder and import them
module_path = Path(__file__).parent / "build_functions"
for file in module_path.glob("*.py"):
module_name = f"build_functions.{file.stem}"
module = importlib.import_module(module_name)
# If the module has a `build` function, call it
if hasattr(module, "build"):
module.build()
# Plenty of warnings raising out of protobuf when using c++23
os.environ["CXXFLAGS"] = "-D_SILENCE_ALL_CXX23_DEPRECATION_WARNINGS"
def build_all(prefix: Path | str) -> dict:
"""Build all known dependencies
:return The merged (partial) SBOM
:raise DependencyInfoError
"""
sbom = {}
file_and_console_log(f"Build script running in python {sys.version}")
file_and_console_log(f"Installing to: {prefix}")
# First those without any dependencies
build_asio(prefix, sbom)
build_tclap(prefix, sbom)
build_spdlog(prefix, sbom)
build_boost(prefix, sbom)
build_zlib(prefix, sbom)
build_glog(prefix, sbom)
build_abseil(prefix, sbom)
build_eigen(prefix, sbom)
build_libzmq(prefix, sbom)
build_jpeg(prefix, sbom)
build_onetbb(prefix, sbom)
build_cares(prefix, sbom)
build_recycle(prefix, sbom)
build_termcolor(prefix, sbom)
build_tinyxml2(prefix, sbom)
build_yaml_cpp(prefix, sbom)
# Conditional. This could depend on abseil but I'll try to build without
build_googletest(prefix, sbom)
# Then the ones that depend on any of those above
# Note that this doesn't constitute a dependency tree yet
build_curl(prefix, sbom)
build_fineftp(prefix, sbom)
build_re2(prefix, sbom)
build_cppzmq(prefix, sbom)
build_protobuf(prefix, sbom)
build_ceres(prefix, sbom)
build_opencv(prefix, sbom)
build_matplot(prefix, sbom)
build_openssl(prefix, sbom)
build_opensubdiv(prefix, sbom)
build_openusd(prefix, sbom)
build_hdf5(prefix, sbom)
build_tcp_pubsub(prefix, sbom)
# This one gets a special place as it depends on many of the above
build_grpc(prefix, sbom)
build_qt5(prefix, sbom)
build_qt6(prefix, sbom)
build_qwt(prefix, sbom)
return sbom
build_functions = {
"abseil-cpp": build_abseil,
"asio": build_asio,
"boost": build_boost,
"curl": build_curl,
"glog": build_glog,
"hdf5": build_hdf5,
"protobuf": build_protobuf,
"eigen": build_eigen,
"ceres-solver": build_ceres,
"libzmq": build_libzmq,
"cppzmq": build_cppzmq,
"fineftp": build_fineftp,
"googletest": build_googletest,
"libjpeg-turbo": build_jpeg,
"opencv": build_opencv,
"matplotplusplus": build_matplot,
"openssl": build_openssl,
"opensubdiv": build_opensubdiv,
"openusd": build_openusd,
"onetbb": build_onetbb,
"re2": build_re2,
"recycle": build_recycle,
"spdlog": build_spdlog,
"tclap": build_tclap,
"c-ares": build_cares,
"grpc": build_grpc,
"qt5": build_qt5,
"qt6": build_qt6,
"qwt": build_qwt,
"termcolor": build_termcolor,
"tcp_pubsub": build_tcp_pubsub,
"tinyxml2": build_tinyxml2,
"yaml-cpp": build_yaml_cpp,
"zlib": build_zlib,
}
if __name__ == '__main__':
try:
with open(logfile, 'w') as f:
f.write(str(datetime.datetime.now()) + f' - started {__file__}\n')
parser = argparse.ArgumentParser()
parser.add_argument("--install-dir", "-i", type=str, required=True, help="install into this directory")
parser.add_argument("--package", "-p", type=str, default="all", help="build specific package or 'all'")
parser.add_argument("--rebuild", action="store_true", help="always delete build dir")
parser.add_argument("--sbom", "-s", type=str, default="sbom.json",
help="Append to this SBOM in CycloneDX format. If all packages are being built (-p=all) "
"a previously existing sbom file gets deleted and overwritten")
parser.add_argument("--build-dir", "-b", type=Path, default=Path("raw"), help="use this directory to build in")
args = parser.parse_args()
global_prefix = Path(args.install_dir)
if not global_prefix.is_absolute():
script_dir = Path(__file__).parent.resolve().absolute()
global_prefix = script_dir / global_prefix
common.settings.set_global_build_dir(args.build_dir)
common.settings.set_global_install_prefix(args.install_dir)
common.settings.set_global_rebuild(args.rebuild)
package = args.package
sbomfile = args.sbom
build_package_tree()
if package == "all":
# Build all packages
sbom = build_all(prefix=global_prefix)
# (Over)write CycloneDX SBOM
with open(sbomfile, "w") as f:
json.dump(sbom, f)
else:
if package not in build_functions:
raise DependencyInfoError(f"Don't know how to build {package}")
sbom = {}
# Build the package, amend SBOM with package info
build_functions[package](prefix=global_prefix, sbom=sbom)
# The partial SBOM gets written into the install dir so we can later combine it.
# We cannot merge them into one file right here as in the pipeline there may be multiple tasks in parallel
package_info = get_package_info(package)
partial_sbom_filename = package_info.install_location() / "partial_sbom.json"
with open(partial_sbom_filename, "w") as f:
json.dump(sbom, f)
print_banner("Done")
exit(0)
except ValueError as ve:
print(f"Cannot read or write SBOM: {str(ve)}")
except DependencyInfoError as die:
print(f"Dependency info error: {str(die)}")
exit(-1)

0
build_functions/__init__.py Normal file
View File

42
build_functions/build_abseil.py Normal file
View File

@ -0,0 +1,42 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import subprocess
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.directory_helpers import pushd
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_abseil(prefix: Path | str, sbom: dict):
print_banner("Building Abseil (Protobuf Dependency)")
package_info = get_package_info("abseil-cpp")
package_info.add_to_sbom(sbom)
abseil_cmake_args = [
("BUILD_SHARED_LIBS:BOOL", "OFF"),
("BUILD_TESTING:BOOL", "OFF"),
("ABSL_PROPAGATE_CXX_STD:BOOL", "ON"),
("ABSL_BUILD_TESTING:BOOL", "OFF"),
("ABSL_BUILD_TEST_HELPERS", "OFF"),
("ABSL_ENABLE_INSTALL:BOOL", "ON"),
("ABSL_RUN_TESTS:BOOL", "OFF"),
]
abseil_dir = clone_git_tag(package_info, recursive=True)
# Abseil LTS doesn't build with CMake>=3.30 due to some imported GTest target.
# Gotta patch that shit until this situation is resolved. https://github.com/abseil/abseil-cpp/issues/690
with pushd(abseil_dir):
subprocess.run(["git", "apply", "..\\..\\gtest_fix.patch"])
install_dir = cmake_build_install(abseil_dir, package_info, cmake_args=abseil_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

38
build_functions/build_asio.py Normal file
View File

@ -0,0 +1,38 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import shutil
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.git_helpers import clone_git_tag
from common.headers_only import headers_install
from package.package_info import get_package_info
def build_asio(prefix: Path | str, sbom: dict):
print_banner("Building Asio")
package_info = get_package_info("asio")
package_info.add_to_sbom(sbom)
# Asio has no build steps. It also has no CMake support.
# It assumes to be included header only. How we will go about the missing
# CMake finder remains to be seen
asio_dir = clone_git_tag(package_info, recursive=True)
install_dir = headers_install(asio_dir, package_info, subdir=Path("asio") / "include")
write_package_version_batch(package_info.version)
# Asio doesn't come with CMake support. Depending modules such as fineftp seemed to
# entirely ignore this unless you use the bundled version. To make this happen anyway,
# we copy a little cmake module finder to where we just installed it.
# This should enable depending modules to set asio_DIR to that install dir
shutil.copy(Path(__file__).parent.parent / "patches" / "asio" / "asioConfig.cmake", install_dir)
return install_dir

71
build_functions/build_boost.py Normal file
View File

@ -0,0 +1,71 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import os
from pathlib import Path
from build_functions.build_utils import run_in_shell, print_banner, file_and_console_log
from common.azure import write_package_version_batch
from common.directory_helpers import pushd, get_local_prefix
from common.git_helpers import clone_git_tag
import common.settings
from package.package_info import get_package_info
def build_boost(prefix: Path | str, sbom: dict):
print_banner("Building Boost")
package_info = get_package_info("boost")
package_info.add_to_sbom(sbom)
boost_dir = clone_git_tag(package_info, recursive=True)
with pushd(boost_dir):
prefix = package_info.install_location()
if not common.settings.rebuild and os.path.exists("built_and_installed.txt"):
file_and_console_log("already built, exiting")
return prefix
if os.name == 'nt':
bootstrap_suffix = "bat"
b2_suffix = ".exe"
cmdprefix = ""
flags = "define=BOOST_USE_WINAPI_VERSION=0x0A00 define=_WIN32_WINNT=0x0A00"
else:
cmdprefix = "./"
bootstrap_suffix = "sh"
b2_suffix = ""
flags = "cxxflags=-fPIC cflags=-fPIC"
if os.name != 'nt':
run_in_shell('chmod +x bootstrap.sh')
run_in_shell('chmod +x ./tools/build/src/engine/build.sh')
run_in_shell(
f"{cmdprefix}bootstrap.{bootstrap_suffix} {common.settings.boost_bootstrap_toolset} --prefix={prefix}")
# Normally I wouldn't exclude this many (or indeed any) libs from the build
# but I have to save time to get the pipeline run below 1h
run_in_shell(
f"{cmdprefix}b2{b2_suffix} {flags} {common.settings.boost_toolset} cxxstd={common.settings.cpp_standard} -j {common.settings.num_cores} "
f"--prefix={prefix} "
"--without-mpi "
"--without-graph_parallel "
# "--without-python "
"address-model=64 architecture=x86 link=static runtime-link=shared "
"variant=release threading=multi install")
with open("built_and_installed.txt", "w") as lockfile:
lockfile.write("built")
write_package_version_batch(package_info.version)
# This stopped working somehow...
# if run_in_buildpipeline:
# shutil.rmtree(local_directory)
return prefix

34
build_functions/build_cares.py Normal file
View File

@ -0,0 +1,34 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_cares(prefix: Path | str, sbom: dict):
print_banner("Building C-Ares (gRPC Dependency)")
package_info = get_package_info("c-ares")
package_info.add_to_sbom(sbom)
cares_cmake_args = [
("BUILD_SHARED_LIBS:BOOL", "OFF"),
("CARES_BUILD_TESTS:BOOL", "OFF"),
("CARES_BUILD_CONTAINER_TESTS:BOOL", "OFF"),
("CARES_BUILD_TOOLS:BOOL", "OFF"),
("CARES_SHARED:BOOL", "OFF"),
("CARES_STATIC:BOOL", "ON"),
("CARES_STATIC_PIC:BOOL", "ON"),
("CARES_MSVC_STATIC_RUNTIME:BOOL", "OFF")
]
cares_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(cares_dir, package_info, cmake_args=cares_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

34
build_functions/build_ceres.py Normal file
View File

@ -0,0 +1,34 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_ceres(prefix: Path | str, sbom: dict):
print_banner("Building Ceres")
package_info = get_package_info("ceres-solver")
package_info.add_to_sbom(sbom)
eigen_install_path = package_info.dependency_path("eigen")
glog_install_path = package_info.dependency_path("glog")
ceres_cmake_args = [
("EIGENSPARSE:BOOL", "ON"),
("BUILD_EXAMPLES:BOOL", "OFF"),
("BUILD_TESTING:BOOL", "OFF"),
("SCHUR_SPECIALIZATIONS:BOOL", "OFF"),
("USE_CUDA:BOOL", "OFF"),
("Eigen3_DIR:PATH", str(eigen_install_path / "share" / "eigen3" / "cmake")),
("glog_DIR:PATH", str(glog_install_path / "lib" / "cmake" / "glog"))
]
eigen_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(eigen_dir, package_info, cmake_args=ceres_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

29
build_functions/build_cppzmq.py Normal file
View File

@ -0,0 +1,29 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_cppzmq(prefix: Path | str, sbom: dict):
print_banner("Building Lib cppzmq")
package_info = get_package_info("cppzmq")
package_info.add_to_sbom(sbom)
libzmq_install_path = package_info.dependency_path("libzmq")
cppzmq_cmake_args = [
("CPPZMQ_BUILD_TESTS:BOOL", "OFF"),
("ZeroMQ_DIR:PATH", str(libzmq_install_path / "CMake"))
]
cppzmq_dir = clone_git_tag(package_info, recursive=True)
install_dir = cmake_build_install(cppzmq_dir, package_info, cmake_args=cppzmq_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

60
build_functions/build_curl.py Normal file
View File

@ -0,0 +1,60 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
import common.settings
from package.package_info import get_package_info
def build_curl(prefix: Path | str, sbom: dict):
print_banner("Building CURL")
package_info = get_package_info("curl")
package_info.add_to_sbom(sbom)
zlib_install_path = package_info.dependency_path("zlib")
# It only appears to depend on gtest when I enable tests, which I don't
# googletest_install_path = package_info.dependency_path(prefix, "googletest")
curl_cmake_args = [
("BUILD_LIBCURL_DOCS:BOOL", "OFF"),
("BUILD_MISC_DOCS:BOOL", "OFF"),
("BUILD_CURL_EXE:BOOL", "OFF"),
("BUILD_EXAMPLES:BOOL", "OFF"),
("BUILD_STATIC_LIBS:BOOL", "ON"),
# This is only optional until 8.11, which is why this is not using a more recent version
# Libpsl has no Windows build support. https://github.com/curl/curl/issues/16486
# This means, you're probably not able to upgrade beyond 8.11.x
("CURL_USE_LIBPSL:BOOL", "OFF"),
("CURL_USE_LIBIDN2:BOOL", "OFF"),
("CURL_USE_LIBSSH2:BOOL", "OFF"),
("ENABLE_CURL_MANUAL:BOOL", "OFF"),
# Many optional dependencies. I switch off as many as I can get away with.
# Once we know for sure which ones we need we can always bring them in
("USE_HTTPSRR:BOOL", "OFF"),
("USE_ECH:BOOL", "OFF"),
("USE_LIBIDN2:BOOL", "OFF"),
("USE_MSH3:BOOL", "OFF"),
("USE_NGHTTP2:BOOL", "OFF"),
("USE_NGTCP2:BOOL", "OFF"),
("ZLIB_ROOT:PATH", str(zlib_install_path)),
("ZLIB_USE_STATIC_LIBS:BOOL", "ON"),
("ZLIB_LIBRARY_RELEASE:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name)),
("ZLIB_LIBRARY_DEBUG:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name))
]
curl_dir = clone_git_tag(package_info, recursive=True)
install_dir = cmake_build_install(curl_dir, package_info, cmake_args=curl_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

40
build_functions/build_eigen.py Normal file
View File

@ -0,0 +1,40 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
import common.settings
from package.package_info import get_package_info
def build_eigen(prefix: Path | str, sbom: dict):
print_banner("Building Eigen")
package_info = get_package_info("eigen")
# Eigen's build system has weird bugs
# https://stackoverflow.com/questions/71876437/cmake-error-compiler-does-not-support-c11-when-configuring-eigen-with-visual
# which make it fail with anything but explicitly set C++11. Sad, really. It appears to be fixed in master
# but I can't switch to master for a medical device.
global cpp_standard
prev_cpp_standard = common.settings.cpp_standard
cpp_standard = "11"
package_info.add_to_sbom(sbom)
eigen_cmake_args = [
("CMAKE_Fortran_COMPILER_NOT_FOUND:BOOL", "ON")
]
eigen_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(eigen_dir, package_info, cmake_args=eigen_cmake_args)
write_package_version_batch(package_info.version)
cpp_standard = prev_cpp_standard
return install_dir

37
build_functions/build_fineftp.py Normal file
View File

@ -0,0 +1,37 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_fineftp(prefix: Path | str, sbom: dict):
print_banner("Building fineftp")
package_info = get_package_info("fineftp")
package_info.add_to_sbom(sbom)
asio_install_path = package_info.dependency_path("asio")
# It only appears to depend on gtest when I enable tests, which I don't
# googletest_install_path = package_info.dependency_path(prefix, "googletest")
fineftp_cmake_args = [
("FINEFTP_SERVER_BUILD_SAMPLES:BOOL", "OFF"),
("FINEFTP_SERVER_BUILD_TESTS:BOOL", "OFF"),
("FINEFTP_SERVER_USE_BUILTIN_ASIO:BOOL", "OFF"),
("asio_INCLUDE_DIR:PATH", (asio_install_path / "include").as_posix()),
("asio_DIR:PATH", asio_install_path.as_posix()),
]
fineftp_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(fineftp_dir, package_info, cmake_args=fineftp_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

30
build_functions/build_glog.py Normal file
View File

@ -0,0 +1,30 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_glog(prefix: Path | str, sbom: dict):
print_banner("Building Glog")
package_info = get_package_info("glog")
package_info.add_to_sbom(sbom)
cmake_args = [
("WITH_GFLAGS:BOOL", "OFF"),
("WITH_GMOCK:BOOL", "OFF"),
("WITH_GTEST:BOOL", "OFF"),
("WITH_UNWIND:BOOL", "OFF")
]
src_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(src_dir, package_info, cmake_args=cmake_args)
write_package_version_batch(package_info.version)
return install_dir

29
build_functions/build_googletest.py Normal file
View File

@ -0,0 +1,29 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_googletest(prefix: Path | str, sbom: dict):
print_banner("Building googletest")
package_info = get_package_info("googletest")
package_info.add_to_sbom(sbom)
googletest_cmake_args = [
# Debatable
("GTEST_HAS_ABSL:BOOL", "OFF"),
]
googletest_dir = clone_git_tag(package_info, recursive=True)
install_dir = cmake_build_install(googletest_dir, package_info, cmake_args=googletest_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

71
build_functions/build_grpc.py Normal file
View File

@ -0,0 +1,71 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
import common.settings
from package.package_info import get_package_info
def build_grpc(prefix: Path | str, sbom: dict):
print_banner("Building gRPC")
package_info = get_package_info("grpc")
# Gotta reduce this for gRPC. The build agent keeps running out of mem
global num_cores
num_cores = 1 if common.settings.run_in_buildpipeline else 6
package_info.add_to_sbom(sbom)
zlib_install_path = package_info.dependency_path("zlib")
re2_install_path = package_info.dependency_path("re2")
openssl_install_path = package_info.dependency_path("openssl")
abseil_install_path = package_info.dependency_path("abseil-cpp")
cares_install_path = package_info.dependency_path("c-ares")
protobuf_install_path = package_info.dependency_path("protobuf")
grpc_cmake_args = [
("BUILD_SHARED_LIBS:BOOL", "OFF"),
("BUILD_TESTING:BOOL", "OFF"),
("ABSL_PROPAGATE_CXX_STD:BOOL", "ON"),
("gRPC_BUILD_GRPC_CPP_PLUGIN:BOOL", "ON"),
("gRPC_BUILD_GRPC_CSHARP_PLUGIN:BOOL", "ON"),
("gRPC_BUILD_GRPC_PYTHON_PLUGIN:BOOL", "ON"),
("gRPC_BUILD_GRPC_NODE_PLUGIN:BOOL", "OFF"),
("gRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN:BOOL", "OFF"),
("gRPC_BUILD_GRPC_PHP_PLUGIN:BOOL", "OFF"),
("gRPC_BUILD_GRPC_RUBY_PLUGIN:BOOL", "OFF"),
# ("gRPC_BUILD_MSVC_MP_COUNT:STRING", "1"),
# ("gRPC_DOWNLOAD_ARCHIVES:BOOL", "ON"),
("gRPC_ABSL_PROVIDER:STRING", "package"),
("absl_DIR:PATH", str(abseil_install_path / "lib" / "cmake" / "absl")),
("gRPC_CARES_PROVIDER:STRING", "package"),
("c-ares_DIR:PATH", str(cares_install_path / "lib" / "cmake" / "c-ares")),
("gRPC_PROTOBUF_PROVIDER:STRING", "package"),
("Protobuf_DIR:PATH", str(protobuf_install_path / "cmake")),
("utf8_range_DIR:PATH", str(protobuf_install_path / "lib" / "cmake" / "utf8_range")),
("gRPC_SSL_PROVIDER", "package"),
("OPENSSL_ROOT_DIR:PATH", openssl_install_path),
("OPENSSL_USE_STATIC_LIBS:BOOL", "ON"),
("gRPC_RE2_PROVIDER", "package"),
("re2_DIR:PATH", str(re2_install_path / "lib" / "cmake" / "re2")),
("gRPC_ZLIB_PROVIDER:STRING", "package"),
("ZLIB_ROOT:PATH", str(zlib_install_path)),
("ZLIB_USE_STATIC_LIBS:BOOL", "ON"), # doesn't appear to do its job
("ZLIB_LIBRARY_RELEASE:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name)),
("ZLIB_LIBRARY_DEBUG:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name))
]
grpc_dir = clone_git_tag(package_info, recursive=True)
install_dir = cmake_build_install(grpc_dir, package_info, cmake_args=grpc_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

43
build_functions/build_hdf5.py Normal file
View File

@ -0,0 +1,43 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
import common.settings
from package.package_info import get_package_info
def build_hdf5(prefix: Path | str, sbom: dict):
print_banner("Building hdf5")
package_info = get_package_info("hdf5")
package_info.add_to_sbom(sbom)
zlib_install_path = package_info.dependency_path("zlib")
hdf5_cmake_args = [
("H5EX_BUILD_EXAMPLES:BOOL", "OFF"),
("H5EX_BUILD_HL:BOOL", "OFF"),
("HDF5_BUILD_CPP_LIB:BOOL", "ON"),
("HDF5_BUILD_EXAMPLES:BOOL", "OFF"),
("HDF5_BUILD_TOOLS:BOOL", "OFF"),
("HDF5_BUILD_UTILS:BOOL", "OFF"),
# This might come in handy
("HDF5_ENABLE_SZIP_ENCODING:BOOL", "OFF"),
("ZLIB_ROOT:PATH", str(zlib_install_path)),
("ZLIB_USE_STATIC_LIBS:BOOL", "ON"),
("ZLIB_LIBRARY_RELEASE:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name)),
("ZLIB_LIBRARY_DEBUG:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name))
]
hdf5_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(hdf5_dir, package_info, cmake_args=hdf5_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

27
build_functions/build_jpeg.py Normal file
View File

@ -0,0 +1,27 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_jpeg(prefix: Path | str, sbom: dict):
print_banner("Building Jpeg-Turbo")
package_info = get_package_info("libjpeg-turbo")
package_info.add_to_sbom(sbom)
jpeg_cmake_args = [
("ENABLE_SHARED:BOOL", "OFF"),
("WITH_CRT_DLL:BOOL", "ON")
]
jpeg_dir = clone_git_tag(package_info, recursive=True)
install_dir = cmake_build_install(jpeg_dir, package_info, cmake_args=jpeg_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

30
build_functions/build_libzmq.py Normal file
View File

@ -0,0 +1,30 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_libzmq(prefix: Path | str, sbom: dict):
print_banner("Building Lib ZeroMQ")
package_info = get_package_info("libzmq")
package_info.add_to_sbom(sbom)
libzmq_cmake_args = [
("BUILD_SHARED:BOOL", "OFF"), # yes, they seem to have their own
("BUILD_TESTS:BOOL", "OFF"), # yes, they seem to have their own
("ZMQ_BUILD_TESTS:BOOL", "OFF"), # correction! It would appear as if they have _two_
("WITH_PERF_TOOL:BOOL", "OFF")
]
libzmq_dir = clone_git_tag(package_info, recursive=True)
install_dir = cmake_build_install(libzmq_dir, package_info, cmake_args=libzmq_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

40
build_functions/build_matplot.py Normal file
View File

@ -0,0 +1,40 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
import common.settings
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_matplot(prefix: Path | str, sbom: dict):
print_banner("Building for Matplot++")
package_info = get_package_info("matplotplusplus")
package_info.add_to_sbom(sbom)
jpeg_install_path = package_info.dependency_path("libjpeg-turbo")
# Matplot won't build with C++23
prev_cpp_standard = common.settings.cpp_standard
common.settings.cpp_standard = "17"
matplot_cmake_args = [
("MATPLOTPP_BUILD_TESTS:BOOL", "OFF"),
("MATPLOTPP_BUILD_EXAMPLES:BOOL", "OFF"),
("JPEG_INCLUDE_DIR:PATH", str(jpeg_install_path / "include")),
("JPEG_LIBRARY_RELEASE:FILEPATH", str(jpeg_install_path / "lib" / "jpeg-static.lib")),
("JPEG_LIBRARY_DEBUG:FILEPATH", str(jpeg_install_path / "lib" / "jpeg-static.lib"))
]
matplot_dir = clone_git_tag(package_info, recursive=True)
install_dir = cmake_build_install(matplot_dir, package_info, cmake_args=matplot_cmake_args)
write_package_version_batch(package_info.version)
common.settings.cpp_standard = prev_cpp_standard
return install_dir

32
build_functions/build_onetbb.py Normal file
View File

@ -0,0 +1,32 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from common.settings import temporarily_set_shared
from package.package_info import get_package_info
def build_onetbb(prefix: Path | str, sbom: dict):
print_banner("Building oneTBB")
package_info = get_package_info("onetbb")
package_info.add_to_sbom(sbom)
# While we build everything static, we cannot do that with oneTBB. They issue a very strong
# warning if we try. Since we have to live with a number of DLL's anyway (possibly Qt)
# I think we just have to take one for the team, considering how low level tbb is.
with temporarily_set_shared():
cmake_args = [
("TBB_TEST:BOOL", "OFF")
]
src_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(src_dir, package_info, cmake_args=cmake_args)
write_package_version_batch(package_info.version)
return install_dir

110
build_functions/build_opencv.py Normal file
View File

@ -0,0 +1,110 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
import common.settings
from package.package_info import get_package_info
def build_opencv(prefix: Path | str, sbom: dict):
print_banner("Building OpenCV")
package_info = get_package_info("opencv")
package_info.add_to_sbom(sbom)
eigen_install_path = package_info.dependency_path("eigen")
zlib_install_path = package_info.dependency_path("zlib")
opencv_cmake_args = [
# Those two undocumented lines are necessary to make the install target use the correct
# subdir for the libraries
# ("OpenCV_ARCH:STRING", "x64"),
# ("OpenCV_RUNTIME:STRING", "vc17"),
("BUILD_SHARED_LIBS:BOOL", "OFF"),
("BUILD_TESTS:BOOL", "OFF"),
("BUILD_PERF_TESTS:BOOL", "OFF"),
("BUILD_WITH_STATIC_CRT:BOOL", "OFF"),
("BUILD_OPENJPEG:BOOL", "OFF"),
("BUILD_IPP_IW:BOOL", "OFF"),
("BUILD_ITT:BOOL", "OFF"),
("BUILD_JASPER:BOOL", "OFF"),
("BUILD_JPEG:BOOL", "OFF"),
("BUILD_ITT:BOOL", "OFF"),
("BUILD_JAVA:BOOL", "OFF"),
("BUILD_PNG:BOOL", "OFF"),
("BUILD_PROTOBUF:BOOL", "OFF"),
("BUILD_TIFF:BOOL", "OFF"),
("BUILD_WEBP:BOOL", "OFF"),
("BUILD_VTK:BOOL", "OFF"),
("BUILD_ZLIB:BOOL", "OFF"),
("BUILD_opencv_apps:BOOL", "OFF"),
("BUILD_opencv_calib3d:BOOL", "OFF"),
("BUILD_opencv_dnn:BOOL", "OFF"),
("BUILD_opencv_features2d:BOOL", "OFF"),
("BUILD_opencv_flann:BOOL", "OFF"),
("BUILD_opencv_imgcodecs:BOOL", "ON"),
("BUILD_opencv_java_bindings_generator:BOOL", "OFF"),
("BUILD_opencv_js_bindings_generator:BOOL", "OFF"),
("BUILD_opencv_objc_bindings_generator:BOOL", "OFF"),
("BUILD_opencv_objdetect:BOOL", "OFF"),
("BUILD_opencv_ml:BOOL", "OFF"),
("BUILD_opencv_photo:BOOL", "OFF"),
("BUILD_opencv_python3:BOOL", "OFF"),
("BUILD_opencv_python_bindings_generator:BOOL", "OFF"),
("BUILD_opencv_python_tests:BOOL", "OFF"),
("DNN_ENABLE_PLUGINS:BOOL", "OFF"),
("HIGHGUI_ENABLE_PLUGINS:BOOL", "OFF"),
("WITH_ADE:BOOL", "OFF"),
("WITH_DIRECTML:BOOL", "OFF"),
("WITH_DIRECTX:BOOL", "OFF"),
("WITH_DSHOW:BOOL", "OFF"),
("WITH_EIGEN:BOOL", "ON"),
("Eigen3_DIR:PATH", str(eigen_install_path / "share" / "eigen3" / "cmake")),
("WITH_FLATBUFFERS:BOOL", "OFF"),
("WITH_WITH_GSTREAMER", "OFF"),
("WITH_IMGCODEC_HDR:BOOL", "OFF"),
("WITH_IMGCODEC_PFM:BOOL", "OFF"),
("WITH_IMGCODEC_PXM:BOOL", "OFF"),
("WITH_IMGCODEC_SUNRASTER:BOOL", "OFF"),
("WITH_IPP:BOOL", "OFF"),
("WITH_ITT:BOOL", "OFF"),
("WITH_JASPER:BOOL", "OFF"),
("WITH_JPEG:BOOL", "OFF"),
("WITH_LAPACK:BOOL", "OFF"),
("WITH_MSMF:BOOL", "OFF"),
("WITH_MSMF_DXVA:BOOL", "OFF"),
("WITH_OBSENSOR:BOOL", "OFF"),
("WITH_OPENCL:BOOL", "OFF"),
("WITH_OPENCLAMDBLAS:BOOL", "OFF"),
("WITH_OPENCLAMDFFT:BOOL", "OFF"),
("WITH_OPENEXR:BOOL", "OFF"),
("WITH_OPENJPEG:BOOL", "OFF"),
("WITH_PNG:BOOL", "OFF"),
("WITH_PROTOBUF:BOOL", "OFF"),
("WITH_TIFF:BOOL", "OFF"),
("WITH_WEBP:BOOL", "OFF"),
("WITH_WIN32UI:BOOL", "OFF"),
("WITH_VTK:BOOL", "OFF"),
("ZLIB_ROOT:PATH", str(zlib_install_path)),
("ZLIB_USE_STATIC_LIBS:BOOL", "ON"), # doesn't appear to do its job
("ZLIB_LIBRARY_RELEASE:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name)),
("ZLIB_LIBRARY_DEBUG:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name))
]
opencv_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(opencv_dir, package_info, cmake_args=opencv_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

114
build_functions/build_openssl.py Normal file
View File

@ -0,0 +1,114 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import os
import zipfile
from pathlib import Path
# we need requests to download perl package but I don't want to install a venv just for that
# with this trick we can use requests that comes bundled with pip
try:
import requests
except ImportError:
import pip._vendor.requests as requests
from build_functions.build_utils import run_in_shell, print_banner, file_and_console_log
from common.azure import write_package_version_batch
from common.directory_helpers import pushd, get_local_prefix
from common.git_helpers import clone_git_tag
import common.settings
from package.package_info import get_package_info
def build_openssl(prefix: Path | str, sbom: dict):
print_banner("Building OpenSSL")
package_info = get_package_info("openssl")
package_info.add_to_sbom(sbom)
zlib_install_path = package_info.dependency_path("zlib")
zlib_include_path = zlib_install_path / "include"
zlib_library_path = zlib_install_path / "lib" / common.settings.zlib_static_lib_name
openssl_dir = clone_git_tag(package_info, recursive=False)
with pushd(openssl_dir):
install_prefix = get_local_prefix(prefix)
if not common.settings.rebuild and Path("built_and_installed.txt").exists():
file_and_console_log("already built, exiting")
return install_prefix
# In the azure devops pipeline there's no Perl available, yet we need it to compile OpenSSL
# So I download the portable package and use this
perl_package_filename = "strawberry-perl-5.38.2.2-64bit-portable.zip"
url = f"https://github.com/StrawberryPerl/Perl-Dist-Strawberry/releases/download/SP_53822_64bit/{perl_package_filename}"
r = requests.get(url, allow_redirects=True)
with open(perl_package_filename, "wb") as package_file:
package_file.write(r.content)
with zipfile.ZipFile(perl_package_filename, "r") as zip_ref:
os.mkdir("perl_portable")
zip_ref.extractall("perl_portable")
path_amend = Path("./perl_portable/c/bin").resolve()
os.environ["PATH"] += os.pathsep + str(path_amend)
# This requires not only perl but also assumes we are in a "Developer Shell" on Windows,
# meaning we have nmake in the path
if os.name == "nt":
run_in_shell("perl.exe .\\Configure no-shared no-legacy zlib no-zlib-dynamic "
"threads no-unit-test no-egd "
f"--with-zlib-include={str(zlib_include_path)} "
f"--with-zlib-lib={str(zlib_library_path)} "
f"--prefix={str(install_prefix)} --openssldir={str(install_prefix)} "
"VC-WIN64A")
# OpenSSL always assumes /MT when building statically. Looks like this cannot be overridden
# So we change the makefile by replacing the occurrences
#
# Update: This trick used to work but no longer does with 3.2.1 / recent MSVC
# I have failed to find a solution but found this source: https://github.com/openssl/openssl/discussions/22668
# arguing this is not necessary (anymore?) because of the /Zl switch which defers the selection
# of the runtime to executable link time. I'm not really buying it but there's little
# I can do about that right now and I will try if it really works when we use OpenSSL with static
# runtime selection. We we only relly know once the egm links against this warning free and works.
# If such is not the case and you end up here looking at this, try again to modify the selected
# runtime like I do below.
#
with open('makefile', 'r') as file:
filedata = file.read()
# Replace the MT flags
filedata = filedata.replace("/MT", "/MD")
# Write the file out again
with open('makefile', 'w') as lockfile:
lockfile.write(filedata)
# If you are here debugging why this doesn't work, you are probably not starting
# this in a x64 native tools command prompt shell
run_in_shell("nmake install")
# I know, dirty, but building openssl takes for evah
with open("built_and_installed.txt", "w") as lockfile:
lockfile.write("built")
elif os.name == "posix":
run_in_shell("perl ./Configure no-shared zlib no-zlib-dynamic threads no-unit-test "
f"--with-zlib-include={str(zlib_include_path)} "
f"--with-zlib-lib={str(zlib_library_path)} "
f"--prefix={str(install_prefix)} --openssldir={str(install_prefix)} "
"")
run_in_shell(f"make -j{common.settings.num_cores} install")
with open("built_and_installed.txt", "w") as lockfile:
lockfile.write("built")
write_package_version_batch(package_info.version)
return install_prefix

43
build_functions/build_opensubdiv.py Normal file
View File

@ -0,0 +1,43 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
import common.settings
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_opensubdiv(prefix: Path | str, sbom: dict):
print_banner("Building OpenSubdiv")
package_info = get_package_info("opensubdiv")
package_info.add_to_sbom(sbom)
onetbb_install_path = package_info.dependency_path("onetbb")
zlib_install_path = package_info.dependency_path("zlib")
# This library is quite a beast. It has a lot of additional acceleration dependencies
# such as OpenCL, CUDA, you name it. This is meant to provide a mininmal build to get OpenUSD
# to work. Once it does, we may look into accelerating things
cmake_args = [
("NO_EXAMPLES:BOOL", "ON"),
("NO_TUTORIALS:BOOL", "ON"),
("NO_REGRESSION:BOOL", "ON"),
("NO_CUDA:BOOL", "ON"),
("NO_OPENCL:BOOL", "ON"),
("TBB_DIR:PATH", str(onetbb_install_path / "lib" / "cmake" / "TBB")),
("ZLIB_ROOT:PATH", str(zlib_install_path)),
("ZLIB_USE_STATIC_LIBS:BOOL", "ON"), # doesn't appear to do its job
("ZLIB_LIBRARY_RELEASE:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name)),
("ZLIB_LIBRARY_DEBUG:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name))
]
src_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(src_dir, package_info, cmake_args=cmake_args)
write_package_version_batch(package_info.version)
return install_dir

46
build_functions/build_openusd.py Normal file
View File

@ -0,0 +1,46 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
import common.settings
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_openusd(prefix: Path | str, sbom: dict):
print_banner("Building OpenUSD")
package_info = get_package_info("openusd")
package_info.add_to_sbom(sbom)
onetbb_install_path = package_info.dependency_path("onetbb")
opensubdiv_install_path = package_info.dependency_path("opensubdiv")
zlib_install_path = package_info.dependency_path("zlib")
cmake_args = [
# We're gonna need Python but I had trouble compiling the bindings.
# Disabled for the first tests
("PXR_ENABLE_PYTHON_SUPPORT:BOOL", "OFF"),
("PXR_BUILD_EXAMPLES:BOOL", "OFF"),
("PXR_BUILD_TESTS:BOOL", "OFF"),
("PXR_BUILD_TUTORIALS:BOOL", "ON"),
("PXR_BUILD_HTML_DOCUMENTATION:BOOL", "OFF"),
("TBB_DIR:PATH", str(onetbb_install_path / "lib" / "cmake" / "TBB")),
("OpenSubdiv_DIR:PATH", str(opensubdiv_install_path / "lib" / "cmake" / "OpenSubdiv")),
("ZLIB_ROOT:PATH", str(zlib_install_path)),
("ZLIB_USE_STATIC_LIBS:BOOL", "ON"), # doesn't appear to do its job
("ZLIB_LIBRARY_RELEASE:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name)),
("ZLIB_LIBRARY_DEBUG:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name))
]
src_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(src_dir, package_info, cmake_args=cmake_args)
write_package_version_batch(package_info.version)
return install_dir

44
build_functions/build_protobuf.py Normal file
View File

@ -0,0 +1,44 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
import common.settings
from package.package_info import get_package_info
def build_protobuf(prefix: Path | str, sbom: dict):
print_banner("Building Protobuf")
package_info = get_package_info("protobuf")
package_info.add_to_sbom(sbom)
abseil_install_path = package_info.dependency_path("abseil-cpp")
zlib_install_path = package_info.dependency_path("zlib")
protobuf_cmake_args = [
("BUILD_SHARED_LIBS:BOOL", "OFF"),
("BUILD_TESTING:BOOL", "OFF"),
("protobuf_BUILD_TESTS:BOOL", "OFF"),
("protobuf_WITH_ZLIB:BOOL", "ON"),
("ABSL_PROPAGATE_CXX_STD:BOOL", "ON"),
("protobuf_MSVC_STATIC_RUNTIME:BOOL", "OFF"),
("protobuf_ABSL_PROVIDER:STRING", "package"),
("absl_DIR:PATH", str(abseil_install_path / "lib" / "cmake" / "absl")),
("protobuf_INSTALL:BOOL", "ON"),
("protobuf_ZLIB_PROVIDER:STRING", "package"),
("ZLIB_ROOT:PATH", str(zlib_install_path)),
("ZLIB_USE_STATIC_LIBS:BOOL", "ON"), # doesn't appear to do its job
("ZLIB_LIBRARY_RELEASE:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name)),
("ZLIB_LIBRARY_DEBUG:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name))
# We are on fake debug
]
protobuf_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(protobuf_dir, package_info, cmake_args=protobuf_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

87
build_functions/build_qt5.py Normal file
View File

@ -0,0 +1,87 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import os
from pathlib import Path
from build_functions.build_utils import run_in_shell, print_banner, file_and_console_log
from common.azure import write_package_version_batch
from common.directory_helpers import pushd, get_local_prefix
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_qt5(prefix: Path | str, sbom: dict):
print_banner("Building Qt5")
package_info = get_package_info("qt5")
package_info.add_to_sbom(sbom)
zlib_install_path = package_info.dependency_path("zlib")
# Protobuf may not be such a strong dependency
protobuf_install_path = package_info.dependency_path("protobuf")
qt_src_dir = clone_git_tag(package_info, recursive=True)
with pushd(qt_src_dir):
install_dir = package_info.install_location()
build_folder = "_build"
if not os.path.isdir(build_folder):
os.makedirs(build_folder)
with pushd(build_folder):
configure = Path("..") / "configure.bat"
zlib_include = zlib_install_path / 'include'
zlib_lib = zlib_install_path / 'lib' / 'zlibstatic.lib'
run_in_shell(
f"{configure} "
f"-prefix {install_dir} "
"-release "
"-mp "
"-confirm-license "
"-c++std c++2b "
"-opensource "
# Specify exactly which modules we need. This is way better than excluding
# what we don't need, assuming this is a lot
# "-submodules qtbase,qtsvg "
"-gui -widgets -no-dbus "
"-D QT_BUILD_TESTS_BY_DEFAULT=OFF "
# f"-system-zlib -I {str((zlib_install_path / 'include').as_posix())} -L {str(zlib_lib.as_posix())} "
"-nomake examples "
"-nomake tests "
"-skip qtlottie "
"-skip qtmultimedia "
"-skip qtpurchasing "
"-skip qtwebengine "
"-skip qtwebchannel "
"-skip qtwayland "
"-skip qtspeech "
"-skip qtdatavis3d "
"-skip qtlocation "
"-skip qtdeclarative "
"-skip qtconnectivity "
"-skip qtsystems "
"-skip qtxmlpatterns "
"-skip qtscript "
# "-- "
# This should not be necessary but the -system-zlib parameter above appears bugged.
# It should help CMake to find ZLib
# f"-DZLIB_INCLUDE_DIR=\"{zlib_include.as_posix()}\" "
# f"-DZLIB_LIBRARY=\"{zlib_lib.as_posix()}\" "
)
run_in_shell(f'nmake')
run_in_shell(f'nmake install')
with open("built_and_installed.txt", "w") as lockfile:
lockfile.write(f"built release")
write_package_version_batch(package_info.version)
return install_dir

73
build_functions/build_qt6.py Normal file
View File

@ -0,0 +1,73 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import os
from pathlib import Path
from build_functions.build_utils import run_in_shell, print_banner, file_and_console_log
from common.azure import write_package_version_batch
from common.directory_helpers import pushd, get_local_prefix
from common.git_helpers import clone_git_tag
import common.settings
from package.package_info import get_package_info
def build_qt6(prefix: Path | str, sbom: dict):
print_banner("Building Qt6")
package_info = get_package_info("qt6")
package_info.add_to_sbom(sbom)
zlib_install_path = package_info.dependency_path("zlib")
# Protobuf may not be such a strong dependency
protobuf_install_path = package_info.dependency_path("protobuf")
qt_src_dir = clone_git_tag(package_info, recursive=True)
with pushd(qt_src_dir):
install_dir = package_info.install_location()
build_folder = "_build"
if not os.path.isdir(build_folder):
os.makedirs(build_folder)
with pushd(build_folder):
configure = Path("..") / "configure.bat"
zlib_include = zlib_install_path / 'include'
zlib_lib = zlib_install_path / 'lib' / 'zlibstatic.lib'
run_in_shell(
f"{configure} "
f"-prefix {install_dir} "
"-release "
"-confirm-license "
"-c++std c++2b "
"-opensource "
# Specify exactly which modules we need. This is way better than excluding
# what we don't need, assuming this is a lot
"-submodules qtbase,qtsvg "
"-D QT_BUILD_TESTS_BY_DEFAULT=OFF "
f"-system-zlib -I {str((zlib_install_path / 'include').as_posix())} -L {str(zlib_lib.as_posix())} "
"-nomake examples "
"-nomake tests "
"-- "
# This should not be necessary but the -system-zlib parameter above appears bugged.
# It should help CMake to find ZLib
f"-DZLIB_INCLUDE_DIR=\"{zlib_include.as_posix()}\" "
f"-DZLIB_LIBRARY=\"{zlib_lib.as_posix()}\" "
)
# Qt6 can use Cmake, Qt5 cannot
run_in_shell(f'cmake --build . --parallel {common.settings.num_cores}')
run_in_shell(f'cmake --install .')
with open("built_and_installed.txt", "w") as lockfile:
lockfile.write(f"built release")
write_package_version_batch(package_info.version)
return install_dir

94
build_functions/build_qwt.py Normal file
View File

@ -0,0 +1,94 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import os
import re
from pathlib import Path
from build_functions.build_utils import run_in_shell, print_banner, file_and_console_log
from common.azure import write_package_version_batch
from common.directory_helpers import pushd
from common.errors import BuildError
from common.git_helpers import clone_git_tag
import common.settings
from package.package_info import get_package_info
def build_qwt(prefix: Path | str, sbom: dict):
"""This gave me a lot of headaches. Very unusual build system, if that term is applicable at all.
Also, they seemed to have moved _from_ github _to_ Sourceforge prior to the 6.3.0 tag.
They have also changed the build config files a bit, so this will only work with 6.3.0
"""
print_banner("Building Qwt")
if not os.name == "nt":
raise BuildError(f"qwt is only implemented on windows. Please implement me for your system")
package_info = get_package_info("qwt")
package_info.add_to_sbom(sbom)
qt_install_path = package_info.dependency_path("qt5")
qmake_path = qt_install_path / "bin" / "qmake.exe" if os.name == "nt" else "qmake"
if not qmake_path.is_file():
raise BuildError(f"Qmake executable at {qmake_path} doesn't seem to be present")
src_dir = clone_git_tag(package_info, recursive=False)
with pushd(src_dir):
if not common.settings.rebuild and os.path.exists("built_and_installed.txt"):
file_and_console_log("already built, exiting")
return prefix
install_dir = package_info.install_location()
# I don't think this old hand-rolled build system supports out-of-src builds.
# I will try to build it right here.
# It looks like as if configuring this involves two modifications to local files.
buildfile = Path("qwtbuild.pri")
buildfile_bak = Path("qwtbuild.pri.bak")
buildfile_bak.unlink(missing_ok=True)
buildfile.rename(buildfile_bak)
# I'm trying to set the build config to release here but that doesn't work.
# It still builds debug as well. No idea how to fix this
with open(buildfile_bak, 'r') as fi, open(buildfile, 'w') as fo:
found = False
regex = re.compile(R"^\s+CONFIG\s+\+=\s(debug_and_release)$")
for line in fi:
if not found:
if m := regex.match(line):
line = line.replace(m.group(1),"release")
found = True
fo.write(line)
else:
fo.write(line)
configfile = Path("qwtconfig.pri")
configfile_bak = Path("qwtconfig.pri.bak")
configfile_bak.unlink(missing_ok=True)
configfile.rename(configfile_bak)
with open(configfile_bak, 'r') as fi, open(configfile, 'w') as fo:
found = False
regex = re.compile(R"^\s*QWT_INSTALL_PREFIX\s+=\s+(C:/Qwt-\$\$QWT_VERSION-dev)$")
for line in fi:
if not found:
if m := regex.match(line):
line = line.replace(m.group(1), str(install_dir.as_posix()))
found = True
fo.write(line)
else:
fo.write(line)
run_in_shell(f'{qmake_path} qwt.pro')
run_in_shell("nmake")
run_in_shell("nmake install")
with open("built_and_installed.txt", "w") as lockfile:
lockfile.write(f"built release")
write_package_version_batch(package_info.version)
return install_dir

30
build_functions/build_re2.py Normal file
View File

@ -0,0 +1,30 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_re2(prefix: Path | str, sbom: dict):
print_banner("Building re2")
package_info = get_package_info("re2")
package_info.add_to_sbom(sbom)
abseil_install_path = package_info.dependency_path("abseil-cpp")
re2_cmake_args = [
("RE2_BUILD_TESTING", "OFF"),
("absl_DIR:PATH", str(abseil_install_path / "lib" / "cmake" / "absl")),
]
re2_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(re2_dir, package_info, cmake_args=re2_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

27
build_functions/build_recycle.py Normal file
View File

@ -0,0 +1,27 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_recycle(prefix: Path | str, sbom: dict):
print_banner("Building recycle")
package_info = get_package_info("recycle")
package_info.add_to_sbom(sbom)
cmake_args = [
("BUILD_TESTING", "OFF"),
]
src_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(src_dir, package_info, cmake_args=cmake_args)
write_package_version_batch(package_info.version)
return install_dir

30
build_functions/build_spdlog.py Normal file
View File

@ -0,0 +1,30 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_spdlog(prefix: Path | str, sbom: dict):
print_banner("Building spdlog")
package_info = get_package_info("spdlog")
package_info.add_to_sbom(sbom)
spdlog_cmake_args = [
("SPDLOG_BUILD_PIC:BOOL", "ON"),
("SPDLOG_BUILD_SHARED:BOOL", "OFF"),
("SPDLOG_BUILD_BENCH:BOOL", "OFF"),
("SPDLOG_BUILD_TESTS:BOOL", "OFF"),
("SPDLOG_BUILD_EXAMPLE:BOOL", "OFF"),
]
spdlog_dir = clone_git_tag(package_info, recursive=True)
install_dir = cmake_build_install(spdlog_dir, package_info, cmake_args=spdlog_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

31
build_functions/build_tclap.py Normal file
View File

@ -0,0 +1,31 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.git_helpers import clone_git_tag
from common.headers_only import headers_install
from package.package_info import get_package_info
def build_tclap(prefix: Path | str, sbom: dict):
print_banner("Building Tclap")
package_info = get_package_info("tclap")
package_info.add_to_sbom(sbom)
# This is what happens when you don't include boost. You end up with tons of small
# and eventually unmaintained libraries for every little thing even though
# Boost.ProgramOptions exists.
# This doesn't seem to be needing build, judging by its Conanfile
tclap_dir = clone_git_tag(package_info, recursive=True)
install_dir = headers_install(tclap_dir, package_info, subdir=Path("include"))
write_package_version_batch(package_info.version)
return install_dir

38
build_functions/build_tcp_pubsub.py Normal file
View File

@ -0,0 +1,38 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_tcp_pubsub(prefix: Path | str, sbom: dict):
print_banner("Building tcp_pubsub")
package_info = get_package_info("tcp_pubsub")
package_info.add_to_sbom(sbom)
asio_install_path = package_info.dependency_path("asio")
recycle_install_path = package_info.dependency_path("recycle")
cmake_args = [
("BUILD_TESTING:BOOL", "OFF"),
("asio_DIR:PATH", str(asio_install_path)),
("asio_INCLUDE_DIR:PATH", str(asio_install_path / "include")),
# Try to use recycle from here but it's fetched as submodule and these are disregarded.
("recycle_DIR:PATH", str(recycle_install_path)),
("recycle_INCLUDE_DIR:PATH", str(recycle_install_path / "include")),
]
# This fetches recycle as submodule but doesn't allow recycle to be used from install
# As recycle is header only I guess that may be okayish
src_dir = clone_git_tag(package_info, recursive=True)
install_dir = cmake_build_install(src_dir, package_info, cmake_args=cmake_args)
write_package_version_batch(package_info.version)
return install_dir

28
build_functions/build_termcolor.py Normal file
View File

@ -0,0 +1,28 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_termcolor(prefix: Path | str, sbom: dict):
print_banner("Building termcolor")
package_info = get_package_info("termcolor")
package_info.add_to_sbom(sbom)
termcolor_cmake_args = [
("TERMCOLOR_TESTS:BOOL", "OFF"),
]
termcolor_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(termcolor_dir, package_info, cmake_args=termcolor_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

28
build_functions/build_tinyxml2.py Normal file
View File

@ -0,0 +1,28 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_tinyxml2(prefix: Path | str, sbom: dict):
print_banner("Building tinyxml2")
package_info = get_package_info("tinyxml2")
package_info.add_to_sbom(sbom)
tinyxml2_cmake_args = [
# No additional args known
]
tinyxml2_dir = clone_git_tag(package_info, recursive=True)
install_dir = cmake_build_install(tinyxml2_dir, package_info, cmake_args=tinyxml2_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

37
build_functions/build_utils.py Normal file
View File

@ -0,0 +1,37 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import datetime
import os
import subprocess
logfile = 'build.log'
def file_and_console_log(*args, **kwargs):
print(*args, **kwargs)
with open(logfile, 'a') as f:
f.write(*args)
f.write('\n')
def print_banner(text: str):
file_and_console_log("#########################################################")
file_and_console_log(" " + text)
file_and_console_log("#########################################################")
def run_in_shell(*args, **kwargs):
cmd = ' '.join(args)
print("> " + cmd)
with open(logfile, 'a') as f:
f.write('[cwd] ' + os.getcwd() + '\n')
f.write(cmd + '\n')
try:
start = datetime.datetime.now()
subprocess.run(*args, **kwargs, shell=True, check=True)
with open(logfile, 'a') as f:
f.write(f' took {(datetime.datetime.now() - start).total_seconds()} seconds.\n\n')
except Exception as e:
file_and_console_log("Error: " + str(e))
raise e

27
build_functions/build_yaml_cpp.py Normal file
View File

@ -0,0 +1,27 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_yaml_cpp(prefix: Path | str, sbom: dict):
print_banner("Building yaml-cpp")
package_info = get_package_info("yaml-cpp")
package_info.add_to_sbom(sbom)
cmake_args = [
("YAML_BUILD_SHARED_LIBS:BOOL", "OFF")
]
src_dir = clone_git_tag(package_info, recursive=False)
install_dir = cmake_build_install(src_dir, package_info, cmake_args=cmake_args)
write_package_version_batch(package_info.version)
return install_dir

30
build_functions/build_zlib.py Normal file
View File

@ -0,0 +1,30 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
from pathlib import Path
from build_functions.build_utils import print_banner
from common.azure import write_package_version_batch
from common.cmake import cmake_build_install
from common.git_helpers import clone_git_tag
from package.package_info import get_package_info
def build_zlib(prefix: Path | str, sbom: dict):
print_banner("Building Zlib")
package_info = get_package_info("zlib")
package_info.add_to_sbom(sbom)
zlib_cmake_args = [
("BUILD_SHARED_LIBS:BOOL", "OFF"),
("BUILD_TESTING:BOOL", "OFF"),
("ZLIB_BUILD_EXAMPLES:BOOL", "OFF"),
]
zlib_dir = clone_git_tag(package_info, recursive=True)
install_dir = cmake_build_install(zlib_dir, package_info, cmake_args=zlib_cmake_args)
write_package_version_batch(package_info.version)
return install_dir

0
common/__init__.py Normal file
View File

16
common/azure.py Normal file
View File

@ -0,0 +1,16 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
def write_package_version_batch(version: str):
"""In order to propagate the version we have installed to subsequent Azure tasks,
there seems to be only one choice: to set a task variable.
This writes a small script to do that.
Each build operation writes that script and the pipeline task calls it after the build,
removing the need of the pipeline to really know the version.
"""
version_cmd = f"echo ##vso[task.setvariable variable=installed_version]{version}\r\n"
output_version_cmd = f"echo ##vso[task.setvariable variable=out_installed_version;isOutput=true]{version}\r\n"
with open("version_setter.bat", "w") as batch_file:
batch_file.write(version_cmd)
batch_file.write(output_version_cmd)

52
common/cmake.py Normal file
View File

@ -0,0 +1,52 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import os
from pathlib import Path
from build_functions.build_utils import file_and_console_log, run_in_shell
from common.directory_helpers import pushd
import common.settings
from package.package_info import PackageInfo
def cmake_build_install(local_directory, package_info: PackageInfo, cmake_args: list[tuple[str, str]] = []):
argstr = ""
# Create flags string out of args tuples
for key, value in common.settings.cmake_global_flags:
slash_value = value.replace("\\", "/")
argstr += f"-D{key}={slash_value} "
for key, value in cmake_args:
slash_value = value.replace("\\", "/")
argstr += f"-D{key}={slash_value} "
argstr += f"-DCMAKE_CXX_STANDARD:STRING={common.settings.cpp_standard} "
build_folder = '_build'
with pushd(local_directory):
install_prefix = package_info.install_location()
if not os.path.isdir(build_folder):
os.makedirs(build_folder)
with pushd(build_folder):
if not common.settings.rebuild and Path("built_and_installed.txt").exists():
file_and_console_log("already built, exiting")
return install_prefix
run_in_shell(f"cmake .. {common.settings.cmake_toolset} -DCMAKE_CONFIGURATION_TYPES:STRING=Release "
f"{argstr} -DCMAKE_INSTALL_PREFIX={install_prefix}")
run_in_shell(f'cmake --build . --config Release --target INSTALL --parallel {common.settings.num_cores}')
with open("built_and_installed.txt", "w") as lockfile:
lockfile.write(f"built release")
# This stopped working somehow...
# if run_in_buildpipeline:
# shutil.rmtree(local_directory)
return install_prefix

32
common/directory_helpers.py Normal file
View File

@ -0,0 +1,32 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import contextlib
import os
from pathlib import Path
from build_functions.build_utils import file_and_console_log
@contextlib.contextmanager
def pushd(new_dir):
"""temporarily go into subdir - https://stackoverflow.com/questions/6194499/pushd-through-os-system
"""
previous_dir = os.getcwd()
os.chdir(new_dir)
try:
yield
finally:
os.chdir(previous_dir)
def get_local_prefix(prefix: Path | str) -> Path:
p = Path(prefix) / Path.cwd().name
if os.name == 'posix':
if not "PKG_CONFIG_PATH" in os.environ:
os.environ["PKG_CONFIG_PATH"] = ''
os.environ["PKG_CONFIG_PATH"] += p / 'lib' / 'pkg_config'
file_and_console_log("PKG_CONFIG_PATH = " + os.environ["PKG_CONFIG_PATH"])
return p

17
common/errors.py Normal file
View File

@ -0,0 +1,17 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
class DependencyInfoError(Exception):
"""Thrown whenever some information about dependencies is missing."""
def __init__(self, message):
"""Construct this exception type."""
super().__init__(message)
class BuildError(Exception):
"""Thrown when build related things at runtime don't work out."""
def __init__(self, message):
"""Construct this exception type."""
super().__init__(message)

42
common/git_helpers.py Normal file
View File

@ -0,0 +1,42 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import datetime
import shutil
from pathlib import Path
import logging as log
from build_functions.build_utils import run_in_shell
import common.settings
from common.tags import sanitize_tag
from package.package_info import PackageInfo
def clone_git_tag(package_info: PackageInfo, backup_existing=False, delete_existing=False, recursive=False) -> Path:
"""I'm not using gitlib for this as I don't want dependencies that would require a venv or something
"""
# git_repo_name = Path(package_info.repo).stem
common.settings.build_dir.mkdir(parents=True, exist_ok=True)
# local_dir = Path("raw") / Path(git_repo_name.lower() + '-' + sanitize_tag(package_info.tag).replace('_', '.'))
# local_dir = Path("raw") / Path(git_repo_name.lower() + '-' + sanitize_tag(package_info.tag))
src_dir = package_info.src_dir()
recursive_flag = "--recursive --shallow-submodules" if recursive else ""
if src_dir.exists():
if delete_existing:
shutil.rmtree(src_dir)
elif common.settings.rebuild:
shutil.rmtree(src_dir / "_build", ignore_errors=True)
elif src_dir.exists() and backup_existing:
date_str = datetime.datetime.now().strftime("%Y%m%d")
src_dir.rename(Path(date_str) / package_info.name)
else:
log.info(f"Cloning {package_info.repo} ...")
run_in_shell(
f"git clone -c advice.detachedHead=false {recursive_flag} --depth 1 -b {package_info.tag} {package_info.repo} {src_dir}")
return src_dir if src_dir.is_dir() else None

23
common/headers_only.py Normal file
View File

@ -0,0 +1,23 @@
import shutil
from pathlib import Path
from common.directory_helpers import pushd, get_local_prefix
from package.package_info import PackageInfo
def headers_install(src_directory: Path | str, package_info: PackageInfo, subdir: Path | str):
"""To keep the callers below looking same-y, this is like cmake_build_install()
for header only libraries
:param src_directory: I believe this is where the src copy resides
:param package_info: package info
:param subdir: Everything under that subdir will be copied
"""
with pushd(src_directory):
install_prefix = package_info.install_location()
# header only libs are structured any way they want, so I have to treat them
# individually by having the subtree handed in here
shutil.copytree(subdir, install_prefix / "include", dirs_exist_ok=True)
return install_prefix

90
common/settings.py Normal file
View File

@ -0,0 +1,90 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import math
import multiprocessing
import os
from contextlib import contextmanager
from pathlib import Path
cmake_config_flag = ""
cmake_toolset = ""
boost_toolset = ""
boost_bootstrap_toolset = ""
build_dir = Path("raw")
install_prefix = Path(R"C:\devel\3rd_party")
rebuild = False
# Hard wire this. According to Azure docs here:
# https://github.com/MicrosoftDocs/azure-devops-docs/blob/main/docs/pipelines/agents/hosted.md
# they run with 2 cores, which gives us a suitable parallel count of 3
# num_cores = 3 if run_in_buildpipeline else 6
# doesn't seem to work
run_in_buildpipeline = "BUILD_ARTIFACTSTAGINGDIRECTORY" in os.environ
num_cores = 3 if run_in_buildpipeline else 6
if os.name == "posix":
cmake_config_flag = "CMAKE_BUILD_TYPE=Release"
cmake_toolset = ""
lib_wildcard = "*.a"
dll_wildcard = "*.so"
zlib_static_lib_name = "libz.a"
elif os.name == "nt": # Windows
cmake_config_flag = "CMAKE_CONFIGURATION_TYPES:STRING=Release"
cmake_toolset = "-T v143"
boost_toolset = "msvc-" + cmake_toolset[-3:-1] + "." + cmake_toolset[-1]
boost_bootstrap_toolset = "vc" + cmake_toolset[-3:-1] + cmake_toolset[-1]
lib_wildcard = "*.lib"
dll_wildcard = "*.dll"
zlib_static_lib_name = "zlibstatic.lib"
else:
print("Unsupported OS")
exit(1)
cpp_standard = "23"
number_of_jobs = max(1, math.floor(multiprocessing.cpu_count()))
cmake_global_flags = [
("BUILD_SHARED_LIBS:BOOL", "OFF"),
("BUILD_TESTING:BOOL", "OFF"),
("CMAKE_CXX_STANDARD_REQUIRED:BOOL", "ON"),
("CMAKE_MSVC_RUNTIME_LIBRARY:STRING", "\"MultiThreadedDLL\""),
("CMAKE_POSITION_INDEPENDENT_CODE:BOOL", "ON"),
("CMAKE_BUILD_TYPE:STRING", "Release")
]
def set_global_rebuild(new_rebuild: bool = False) -> None:
global rebuild
rebuild = new_rebuild
def set_global_install_prefix(new_prefix: Path) -> None:
global install_prefix
install_prefix = new_prefix
def set_global_build_dir(new_build_dir: Path) -> None:
global build_dir
build_dir = new_build_dir
def switch_shared_libs(shared: bool) -> None:
global cmake_global_flags
for i, (k, v) in enumerate(cmake_global_flags):
if k == "BUILD_SHARED_LIBS:BOOL":
cmake_global_flags[i] = ("BUILD_SHARED_LIBS:BOOL", "ON") if shared else ("BUILD_SHARED_LIBS:BOOL", "OFF")
@contextmanager
def temporarily_set_shared():
"""Create a scope in which we build dynamic"""
global cmake_global_flags
idx = 0
for idx, (k, v) in enumerate(cmake_global_flags):
if k == "BUILD_SHARED_LIBS:BOOL":
cmake_global_flags[idx] = ("BUILD_SHARED_LIBS:BOOL", "ON")
break
try:
yield # control goes to the inner `with` block
finally:
cmake_global_flags[idx] = ("BUILD_SHARED_LIBS:BOOL", "OFF")

47
common/tags.py Normal file
View File

@ -0,0 +1,47 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import re
def sanitize_tag(git_tag: str) -> str:
"""Remove common recurring prefixes of git tags to get to a semver like version we want
This also sanitizes other tags. Input is any of the tag formats the dependencies may have.
:return: major.minor.patch variant of the input
"""
if git_tag.startswith('openssl'):
return git_tag[8:]
# Qt uses this for lts releases
if x := re.match(R"^v(\d+\.\d+\.\d+)-lts-lgpl$", git_tag):
return x.group(1)
if x := re.match(R"^boost-(\d+\.\d+\.\d+)$", git_tag):
return x.group(1)
# Something like asio-1-34-2. Chris does his own thing
if x := re.match(R"^asio-(\d+)-(\d+)-(\d+)$", git_tag):
return f"{x.group(1)}.{x.group(2)}.{x.group(3)}"
if x := re.match(R"^curl-(\d+)_(\d+)_(\d+)$", git_tag):
return f"{x.group(1)}.{x.group(2)}.{x.group(3)}"
if x := re.match(R"^hdf5-(\d+\.\d+\.\d+)$", git_tag):
return x.group(1)
if x := re.match(R"^yaml-cpp-(\d+\.\d+\.\d+)$", git_tag):
return x.group(1)
if git_tag.startswith('cares-'):
return git_tag[6:]
if git_tag.startswith('curl-'):
return git_tag[5:]
if x := re.match(R"^v(\d+[._]\d+[._]\d+)$", git_tag):
return x.group(1)
return git_tag

0
package/__init__.py Normal file
View File

156
package/package_info.py Normal file
View File

@ -0,0 +1,156 @@
# (c) 2025 by Stephan Menzel
# Licensed under the Apache License, Version 2.0.
# See attached file LICENSE for full details
import json
from pathlib import Path
from common.errors import DependencyInfoError
import common.settings
from common.tags import sanitize_tag
# will be populated by build_package_tree()
packages = {}
class PackageInfo:
"""Parsed from packages.json, wraps info about each dependency"""
version = ""
repo = ""
tag = ""
dependencies = {}
# sbom relevant info
name = ""
description = None
license_id = None
license_name = None
license_url = ""
def __init__(self, package_name: str):
pckjson = {}
with open("packages.json", "r") as jsonfile:
pckjson = json.load(jsonfile)
if package_name not in pckjson:
raise DependencyInfoError(f"Cannot find version info for {package_name} in packages.json")
pckdict = pckjson[package_name]
self.repo = pckdict["repo"]
self.tag = pckdict["tag"]
self.version = pckdict["version"]
self.dependencies = {}
if "depends" in pckdict:
for dependency in pckdict["depends"]:
if dependency not in packages:
packages[dependency] = PackageInfo(dependency)
self.dependencies[dependency] = packages[dependency]
# Save some data to later generate the SBOM with
self.name = package_name
self.description = pckdict["description"] if "description" in pckdict else None
if "license_id" in pckdict:
self.license_id = pckdict["license_id"]
else:
self.license_name = pckdict["license_name"]
self.license_url = pckdict["license_url"]
def dependency_path(self, package_name: str) -> Path:
"""Give the path underneath which the package, according to its version, is installed"""
if package_name not in self.dependencies:
raise DependencyInfoError(f"Cannot find dependency {package_name} in {self.name}.")
return self.dependencies[package_name].install_location()
def bom_ref(self) -> str:
"""Return the CycloneDX bom-ref for this package"""
return f"{self.name}=={self.version}"
def purl(self) -> str:
"""Return the CycloneDX purl for this package"""
return f"pkg:cmake/{self.name}@{self.version}"
def add_to_sbom(self, sbom: dict) -> None:
"""Add all information we have about this package to a CycloneDX SBOM"""
if "components" not in sbom:
sbom["components"] = []
license = {
"acknowledgement": "concluded",
"url": self.license_url
}
if self.license_id:
license["id"] = self.license_id
else:
license["name"] = self.license_name
sbom["components"].append({
"name": self.name,
"purl": self.purl(),
"type": "library",
"version": self.version,
"bom-ref": self.bom_ref(),
"description": self.description,
"externalReferences": [
{
"comment": "from packaging metadata Project-URL: Source Code",
"type": "other",
"url": self.repo,
},
],
"licenses": [
{
"license": license
},
]
})
if "dependencies" not in sbom:
sbom["dependencies"] = []
dependencies_entry = {"ref": self.purl()}
for name, dep_info in self.dependencies.items():
if "dependsOn" not in dependencies_entry:
dependencies_entry["dependsOn"] = []
dependencies_entry["dependsOn"].append(dep_info.purl())
sbom["dependencies"].append(dependencies_entry)
def install_location(self) -> Path:
"""Get the directory where this package out to be installed in
"""
return common.settings.install_prefix / Path(f"{self.name}-{self.version}")
def src_dir(self) -> Path:
"""Get us the directory where the source is going to be checked out in (a subdir under "raw" normally)
"""
return common.settings.build_dir / Path(f"{self.name}-{sanitize_tag(self.tag)}")
def build_package_tree() -> None:
"""Assemble the global dependency tree of packages known to that system
"""
global packages
pckjson = {}
with open("packages.json", "r") as jsonfile:
pckjson = json.load(jsonfile)
for package_name in pckjson.keys():
if package_name not in packages:
packages[package_name] = PackageInfo(package_name)
def get_package_info(name: str) -> PackageInfo:
"""Get the PackageInfo struct of that name or
:throws DependencyInfoError
"""
global packages
if name not in packages:
raise DependencyInfoError(f"No build info for {name}")
return packages[name]

340
packages.json Normal file
View File

@ -0,0 +1,340 @@
{
"abseil-cpp": {
"repo": "https://github.com/abseil/abseil-cpp.git",
"tag": "20240116.2",
"version": "24.1.16",
"description": "Collection of peer reviewed C++ libraries by Google",
"license_id": "Apache-2.0",
"license_url": "https://github.com/abseil/abseil-cpp/blob/master/LICENSE"
},
"asio": {
"repo": "https://github.com/chriskohlhoff/asio.git",
"tag": "asio-1-34-2",
"version": "1.34.2",
"description": "The one and only, in non-Boost variant",
"license_name": "Boost License 1.0",
"license_url": "https://github.com/chriskohlhoff/asio/blob/master/asio/LICENSE_1_0.txt"
},
"boost": {
"repo": "https://github.com/boostorg/boost.git",
"tag": "boost-1.88.0",
"version": "1.88.0",
"description": "Collection of peer reviewed C++ libraries",
"license_name": "Boost License 2.0",
"license_url": "https://www.boost.org/users/license.html"
},
"c-ares": {
"repo": "https://github.com/c-ares/c-ares.git",
"tag": "v1.32.2",
"version": "1.32.2",
"description": "Google DNS name resolution",
"license_id": "MIT",
"license_url": "https://c-ares.org/license.html"
},
"curl": {
"repo": "https://github.com/curl/curl.git",
"tag": "curl-8_11_1",
"version": "8.11.1",
"depends": [
"zlib"
],
"description": "A command line tool and library for transferring data with URL syntax",
"license_name": "CURL license",
"license_url": "https://github.com/curl/curl?tab=License-1-ov-file#readme"
},
"fineftp": {
"repo": "https://github.com/eclipse-ecal/fineftp-server.git",
"tag": "v1.5.1",
"version": "1.5.1",
"depends": [
"asio"
],
"description": "eCAL fineftp",
"license_id": "MIT",
"license_url": "https://github.com/eclipse-ecal/fineftp-server/blob/master/LICENSE"
},
"glog": {
"repo": "https://github.com/google/glog.git",
"tag": "v0.6.0",
"version": "0.6.0",
"description": "Google logging library",
"license_id": "Apache-2.0",
"license_url": "https://github.com/golang/glog/blob/master/LICENSE"
},
"googletest": {
"repo": "https://github.com/google/googletest.git",
"tag": "v1.17.0",
"version": "1.17.0",
"description": "Unit test library",
"license_name": "BSD-3-Clause",
"license_url": "https://github.com/google/googletest/blob/main/LICENSE"
},
"hdf5": {
"repo": "https://github.com/HDFGroup/hdf5.git",
"tag": "hdf5-1.14.6",
"version": "1.14.6",
"depends": [
"zlib"
],
"description": "Hierarchical Data Format version 5",
"license_name": "HDF5 license",
"license_url": "https://github.com/HDFGroup/hdf5/blob/develop/LICENSE"
},
"protobuf": {
"repo": "https://github.com/protocolbuffers/protobuf.git",
"tag": "v27.2",
"version": "27.2.0",
"depends": [
"zlib",
"abseil-cpp"
],
"description": "Protocol buffers binary serialization",
"license_name": "Proprietary Google License",
"license_url": "https://github.com/protocolbuffers/protobuf/blob/main/LICENSE"
},
"eigen": {
"repo": "https://gitlab.com/libeigen/eigen.git",
"tag": "3.4.0",
"version": "3.4.0",
"description": "Eigen is a C++ template library for linear algebra",
"license_id": "MPL-2.0",
"license_url": "https://www.mozilla.org/en-US/MPL/2.0/"
},
"tclap": {
"repo": "https://github.com/xguerin/tclap.git",
"tag": "v1.2.5",
"version": "1.2.5",
"description": "Command line argument parser",
"license_id": "MIT",
"license_url": "https://github.com/xguerin/tclap?tab=MIT-1-ov-file#readme"
},
"termcolor": {
"repo": "https://github.com/ikalnytskyi/termcolor.git",
"tag": "v2.1.0",
"version": "2.1.0",
"description": "Command line argument parser",
"license_id": "BSD-3-Clause",
"license_url": "https://github.com/ikalnytskyi/termcolor/blob/master/LICENSE"
},
"spdlog": {
"repo": "https://github.com/gabime/spdlog.git",
"tag": "v1.15.3",
"version": "1.15.3",
"description": "Logging library",
"license_id": "MIT",
"license_url": "https://github.com/gabime/spdlog/blob/main/LICENSE"
},
"ceres-solver": {
"repo": "https://github.com/ceres-solver/ceres-solver.git",
"tag": "2.2.0",
"version": "2.2.0",
"depends": [
"eigen",
"glog"
],
"description": "C++ library for modeling and solving large, complicated optimization problems",
"license_name": "Proprietary Google 2023",
"license_url": "http://ceres-solver.org/license.html"
},
"libzmq": {
"repo": "https://github.com/zeromq/libzmq.git",
"tag": "v4.3.5",
"version": "4.3.5",
"description": "ZeroMQ network communication patterns base library",
"license_id": "MPL-2.0",
"license_url": "https://github.com/abseil/abseil-cpp/blob/master/LICENSE"
},
"cppzmq": {
"repo": "https://github.com/zeromq/cppzmq",
"tag": "v4.10.0",
"version": "4.10.0",
"depends": [
"libzmq"
],
"description": "C++ wrapper for ZeroMQ",
"license_id": "MIT",
"license_url": "https://github.com/zeromq/cppzmq/blob/master/LICENSE"
},
"libjpeg-turbo": {
"repo": "https://github.com/libjpeg-turbo/libjpeg-turbo.git",
"tag": "3.0.2",
"version": "3.0.2",
"description": "SIMD optimized jpeg image compression",
"license_name": "BSD-style",
"license_url": "https://github.com/libjpeg-turbo/libjpeg-turbo/blob/main/LICENSE.md"
},
"onetbb": {
"repo": "https://github.com/uxlfoundation/oneTBB.git",
"tag": "v2022.1.0",
"version": "2022.1.0",
"description": "oneAPI Threading Building Blocks",
"license_id": "Apache-2.0",
"license_url": "https://github.com/uxlfoundation/oneTBB/blob/master/LICENSE.txt"
},
"opencv": {
"repo": "https://github.com/opencv/opencv.git",
"tag": "4.11.0",
"version": "4.11.0",
"depends": [
"eigen",
"zlib"
],
"description": "Multi-purpose graphics library",
"license_id": "Apache-2.0",
"license_url": "https://opencv.org/license/"
},
"opensubdiv": {
"repo": "https://github.com/PixarAnimationStudios/OpenSubdiv.git",
"tag": "v3_6_0",
"version": "3.6.0",
"depends": [
"onetbb",
"zlib"
],
"description": "Universal Scene Description",
"license_id": "Apache-2.0",
"license_url": "https://github.com/PixarAnimationStudios/OpenUSD?tab=License-1-ov-file#readme"
},
"openusd": {
"repo": "https://github.com/PixarAnimationStudios/OpenUSD.git",
"tag": "v25.05.01",
"version": "25.5.1",
"depends": [
"boost",
"onetbb",
"opensubdiv",
"zlib"
],
"description": "Universal Scene Description",
"license_id": "Apache-2.0",
"license_url": "https://github.com/PixarAnimationStudios/OpenUSD?tab=License-1-ov-file#readme"
},
"matplotplusplus": {
"repo": "https://github.com/alandefreitas/matplotplusplus.git",
"tag": "v1.2.0",
"version": "1.2.0",
"depends": [
"libjpeg-turbo"
],
"description": "C++ implementation of the matplot graph plotting library",
"license_id": "MIT",
"license_url": "https://github.com/alandefreitas/matplotplusplus/blob/master/LICENSE"
},
"openssl": {
"repo": "https://github.com/openssl/openssl.git",
"tag": "openssl-3.2.4",
"version": "3.2.4",
"depends": [
"zlib"
],
"description": "Standard encryption layer library",
"license_id": "Apache-2.0",
"license_url": "https://openssl-library.org/source/license/index.html"
},
"re2": {
"repo": "https://github.com/google/re2.git",
"tag": "2024-07-02",
"version": "24.7.2",
"depends": [
"abseil-cpp"
],
"description": "Googles take on regular expressions",
"license_id": "BSD-3-Clause",
"license_url": "https://github.com/google/re2/blob/main/LICENSE"
},
"recycle": {
"repo": "https://github.com/steinwurf/recycle.git",
"tag": "7.0.0",
"version": "7.0.0",
"description": "Simple resource pool for recycling resources in C++",
"license_id": "BSD-3-Clause",
"license_url": "https://raw.githubusercontent.com/steinwurf/recycle/refs/heads/master/LICENSE.rst"
},
"grpc": {
"repo": "https://github.com/grpc/grpc.git",
"tag": "v1.65.1",
"version": "1.65.1",
"depends": [
"zlib",
"re2",
"openssl",
"abseil-cpp",
"c-ares",
"protobuf"
],
"description": "Protobuf based RPC abstraction by Google",
"license_id": "Apache-2.0",
"license_url": "https://github.com/grpc/grpc/blob/master/LICENSE"
},
"qt5": {
"repo": "https://github.com/qt/qt5.git",
"tag": "v5.15.16-lts-lgpl",
"version": "5.15.16",
"depends": [
"zlib",
"protobuf"
],
"description": "Qt",
"license_id": "Apache-2.0",
"license_url": "https://github.com/qt/qt5/tree/dev/LICENSES"
},
"qt6": {
"repo": "https://github.com/qt/qt5.git",
"tag": "v6.9.1",
"version": "6.9.1",
"depends": [
"zlib",
"protobuf"
],
"description": "Qt",
"license_id": "Apache-2.0",
"license_url": "https://github.com/qt/qt5/tree/dev/LICENSES"
},
"qwt": {
"repo": "https://git.code.sf.net/p/qwt/git",
"tag": "v6.3.0",
"version": "6.3.0",
"depends": [
"qt5"
],
"description": "Qt Widgets for Technical Applications",
"license_name": "Qwt license",
"license_url": "https://github.com/opencor/qwt?tab=License-1-ov-file"
},
"tcp_pubsub": {
"repo": "https://github.com/eclipse-ecal/tcp_pubsub.git",
"tag": "v2.0.1",
"version": "2.0.1",
"depends": [
"asio",
"recycle"
],
"description": "TCP based publish-subscribe library for C++",
"license_id": "MIT",
"license_url": "https://github.com/eclipse-ecal/tcp_pubsub/blob/master/LICENSE"
},
"tinyxml2": {
"repo": "https://github.com/leethomason/tinyxml2",
"tag": "11.0.0",
"version": "11.0.0",
"description": "A lightweight XML parser",
"license_id": "Zlib",
"license_url": "https://github.com/leethomason/tinyxml2/blob/master/LICENSE.txt"
},
"yaml-cpp": {
"repo": "https://github.com/jbeder/yaml-cpp.git",
"tag": "0.8.0",
"version": "0.8.0",
"description": "A YAML parser and emitter in C++",
"license_id": "MIT",
"license_url": "https://github.com/jbeder/yaml-cpp/blob/master/LICENSE"
},
"zlib": {
"repo": "https://github.com/madler/zlib.git",
"tag": "v1.3.1",
"version": "1.3.1",
"description": "Standard compression algorithm",
"license_id": "Zlib",
"license_url": "https://www.zlib.net/zlib_license.html"
}
}

6
patches/asio/asioConfig.cmake Normal file
View File

@ -0,0 +1,6 @@
add_library(asio INTERFACE EXCLUDE_FROM_ALL)
target_include_directories(asio INTERFACE
$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include>
)
target_compile_definitions(asio INTERFACE ASIO_STANDALONE)
add_library(asio::asio ALIAS asio)

2
version_setter.bat Normal file
View File

@ -0,0 +1,2 @@
echo ##vso[task.setvariable variable=installed_version]3.6.0
echo ##vso[task.setvariable variable=out_installed_version;isOutput=true]3.6.0