Initial commit of revamped build scripts. All set to Apache License

common/azure.py

@@ -0,0 +1,16 @@
+# (c) 2025 by Stephan Menzel
+# Licensed under the Apache License, Version 2.0.
+# See attached file LICENSE for full details
+
+def write_package_version_batch(version: str):
+    """In order to propagate the version we have installed to subsequent Azure tasks,
+        there seems to be only one choice: to set a task variable.
+        This writes a small script to do that.
+        Each build operation writes that script and the pipeline task calls it after the build,
+        removing the need of the pipeline to really know the version.
+    """
+    version_cmd = f"echo ##vso[task.setvariable variable=installed_version]{version}\r\n"
+    output_version_cmd = f"echo ##vso[task.setvariable variable=out_installed_version;isOutput=true]{version}\r\n"
+    with open("version_setter.bat", "w") as batch_file:
+        batch_file.write(version_cmd)
+        batch_file.write(output_version_cmd)

common/cmake.py

@@ -0,0 +1,52 @@
+# (c) 2025 by Stephan Menzel
+# Licensed under the Apache License, Version 2.0.
+# See attached file LICENSE for full details
+
+import os
+from pathlib import Path
+
+from build_functions.build_utils import file_and_console_log, run_in_shell
+from common.directory_helpers import pushd
+import common.settings
+from package.package_info import PackageInfo
+
+
+def cmake_build_install(local_directory, package_info: PackageInfo, cmake_args: list[tuple[str, str]] = []):
+    argstr = ""
+
+    # Create flags string out of args tuples
+    for key, value in common.settings.cmake_global_flags:
+        slash_value = value.replace("\\", "/")
+        argstr += f"-D{key}={slash_value} "
+
+    for key, value in cmake_args:
+        slash_value = value.replace("\\", "/")
+        argstr += f"-D{key}={slash_value} "
+
+    argstr += f"-DCMAKE_CXX_STANDARD:STRING={common.settings.cpp_standard} "
+
+    build_folder = '_build'
+    with pushd(local_directory):
+        install_prefix = package_info.install_location()
+
+        if not os.path.isdir(build_folder):
+            os.makedirs(build_folder)
+        with pushd(build_folder):
+            if not common.settings.rebuild and Path("built_and_installed.txt").exists():
+                file_and_console_log("already built, exiting")
+                return install_prefix
+
+            run_in_shell(f"cmake .. {common.settings.cmake_toolset} -DCMAKE_CONFIGURATION_TYPES:STRING=Release "
+                         f"{argstr} -DCMAKE_INSTALL_PREFIX={install_prefix}")
+
+            run_in_shell(f'cmake --build . --config Release --target INSTALL --parallel {common.settings.num_cores}')
+
+            with open("built_and_installed.txt", "w") as lockfile:
+                lockfile.write(f"built release")
+
+    # This stopped working somehow...
+    # if run_in_buildpipeline:
+    #     shutil.rmtree(local_directory)
+
+    return install_prefix
+

common/directory_helpers.py

@@ -0,0 +1,32 @@
+# (c) 2025 by Stephan Menzel
+# Licensed under the Apache License, Version 2.0.
+# See attached file LICENSE for full details
+
+import contextlib
+import os
+from pathlib import Path
+
+from build_functions.build_utils import file_and_console_log
+
+
+@contextlib.contextmanager
+def pushd(new_dir):
+    """temporarily go into subdir - https://stackoverflow.com/questions/6194499/pushd-through-os-system
+    """
+
+    previous_dir = os.getcwd()
+    os.chdir(new_dir)
+    try:
+        yield
+    finally:
+        os.chdir(previous_dir)
+
+
+def get_local_prefix(prefix: Path | str) -> Path:
+    p = Path(prefix) / Path.cwd().name
+    if os.name == 'posix':
+        if not "PKG_CONFIG_PATH" in os.environ:
+            os.environ["PKG_CONFIG_PATH"] = ''
+        os.environ["PKG_CONFIG_PATH"] += p / 'lib' / 'pkg_config'
+        file_and_console_log("PKG_CONFIG_PATH = " + os.environ["PKG_CONFIG_PATH"])
+    return p

common/errors.py

@@ -0,0 +1,17 @@
+# (c) 2025 by Stephan Menzel
+# Licensed under the Apache License, Version 2.0.
+# See attached file LICENSE for full details
+
+class DependencyInfoError(Exception):
+    """Thrown whenever some information about dependencies is missing."""
+
+    def __init__(self, message):
+        """Construct this exception type."""
+        super().__init__(message)
+
+class BuildError(Exception):
+    """Thrown when build related things at runtime don't work out."""
+
+    def __init__(self, message):
+        """Construct this exception type."""
+        super().__init__(message)

common/git_helpers.py

@@ -0,0 +1,42 @@
+# (c) 2025 by Stephan Menzel
+# Licensed under the Apache License, Version 2.0.
+# See attached file LICENSE for full details
+
+import datetime
+import shutil
+from pathlib import Path
+import logging as log
+from build_functions.build_utils import run_in_shell
+import common.settings
+from common.tags import sanitize_tag
+from package.package_info import PackageInfo
+
+
+def clone_git_tag(package_info: PackageInfo, backup_existing=False, delete_existing=False, recursive=False) -> Path:
+    """I'm not using gitlib for this as I don't want dependencies that would require a venv or something
+    """
+    # git_repo_name = Path(package_info.repo).stem
+
+    common.settings.build_dir.mkdir(parents=True, exist_ok=True)
+    # local_dir = Path("raw") / Path(git_repo_name.lower() + '-' + sanitize_tag(package_info.tag).replace('_', '.'))
+    # local_dir = Path("raw") / Path(git_repo_name.lower() + '-' + sanitize_tag(package_info.tag))
+
+    src_dir = package_info.src_dir()
+
+    recursive_flag = "--recursive --shallow-submodules" if recursive else ""
+
+    if src_dir.exists():
+        if delete_existing:
+            shutil.rmtree(src_dir)
+        elif common.settings.rebuild:
+            shutil.rmtree(src_dir / "_build", ignore_errors=True)
+    elif src_dir.exists() and backup_existing:
+        date_str = datetime.datetime.now().strftime("%Y%m%d")
+        src_dir.rename(Path(date_str) / package_info.name)
+    else:
+        log.info(f"Cloning {package_info.repo} ...")
+        run_in_shell(
+            f"git clone -c advice.detachedHead=false {recursive_flag} --depth 1 -b {package_info.tag} {package_info.repo} {src_dir}")
+
+    return src_dir if src_dir.is_dir() else None
+

common/headers_only.py

@@ -0,0 +1,23 @@
+import shutil
+from pathlib import Path
+
+from common.directory_helpers import pushd, get_local_prefix
+from package.package_info import PackageInfo
+
+
+def headers_install(src_directory: Path | str, package_info: PackageInfo, subdir: Path | str):
+    """To keep the callers below looking same-y, this is like cmake_build_install()
+     for header only libraries
+     :param src_directory: I believe this is where the src copy resides
+     :param package_info: package info
+     :param subdir: Everything under that subdir will be copied
+    """
+    with pushd(src_directory):
+        install_prefix = package_info.install_location()
+
+        # header only libs are structured any way they want, so I have to treat them
+        # individually by having the subtree handed in here
+        shutil.copytree(subdir, install_prefix / "include", dirs_exist_ok=True)
+
+    return install_prefix
+

common/settings.py

@@ -0,0 +1,90 @@
+# (c) 2025 by Stephan Menzel
+# Licensed under the Apache License, Version 2.0.
+# See attached file LICENSE for full details
+
+import math
+import multiprocessing
+import os
+from contextlib import contextmanager
+from pathlib import Path
+
+cmake_config_flag = ""
+cmake_toolset = ""
+boost_toolset = ""
+boost_bootstrap_toolset = ""
+
+build_dir = Path("raw")
+install_prefix = Path(R"C:\devel\3rd_party")
+rebuild = False
+
+# Hard wire this. According to Azure docs here:
+# https://github.com/MicrosoftDocs/azure-devops-docs/blob/main/docs/pipelines/agents/hosted.md
+# they run with 2 cores, which gives us a suitable parallel count of 3
+# num_cores = 3 if run_in_buildpipeline else 6
+# doesn't seem to work
+run_in_buildpipeline = "BUILD_ARTIFACTSTAGINGDIRECTORY" in os.environ
+num_cores = 3 if run_in_buildpipeline else 6
+
+if os.name == "posix":
+    cmake_config_flag = "CMAKE_BUILD_TYPE=Release"
+    cmake_toolset = ""
+    lib_wildcard = "*.a"
+    dll_wildcard = "*.so"
+    zlib_static_lib_name = "libz.a"
+elif os.name == "nt":  # Windows
+    cmake_config_flag = "CMAKE_CONFIGURATION_TYPES:STRING=Release"
+    cmake_toolset = "-T v143"
+    boost_toolset = "msvc-" + cmake_toolset[-3:-1] + "." + cmake_toolset[-1]
+    boost_bootstrap_toolset = "vc" + cmake_toolset[-3:-1] + cmake_toolset[-1]
+    lib_wildcard = "*.lib"
+    dll_wildcard = "*.dll"
+    zlib_static_lib_name = "zlibstatic.lib"
+else:
+    print("Unsupported OS")
+    exit(1)
+
+cpp_standard = "23"
+
+number_of_jobs = max(1, math.floor(multiprocessing.cpu_count()))
+
+cmake_global_flags = [
+    ("BUILD_SHARED_LIBS:BOOL", "OFF"),
+    ("BUILD_TESTING:BOOL", "OFF"),
+    ("CMAKE_CXX_STANDARD_REQUIRED:BOOL", "ON"),
+    ("CMAKE_MSVC_RUNTIME_LIBRARY:STRING", "\"MultiThreadedDLL\""),
+    ("CMAKE_POSITION_INDEPENDENT_CODE:BOOL", "ON"),
+    ("CMAKE_BUILD_TYPE:STRING", "Release")
+]
+
+def set_global_rebuild(new_rebuild: bool = False) -> None:
+    global rebuild
+    rebuild = new_rebuild
+
+def set_global_install_prefix(new_prefix: Path) -> None:
+    global install_prefix
+    install_prefix = new_prefix
+
+def set_global_build_dir(new_build_dir: Path) -> None:
+    global build_dir
+    build_dir = new_build_dir
+
+def switch_shared_libs(shared: bool) -> None:
+    global cmake_global_flags
+    for i, (k, v) in enumerate(cmake_global_flags):
+        if k == "BUILD_SHARED_LIBS:BOOL":
+            cmake_global_flags[i] = ("BUILD_SHARED_LIBS:BOOL", "ON") if shared else ("BUILD_SHARED_LIBS:BOOL", "OFF")
+
+
+@contextmanager
+def temporarily_set_shared():
+    """Create a scope in which we build dynamic"""
+    global cmake_global_flags
+    idx = 0
+    for idx, (k, v) in enumerate(cmake_global_flags):
+        if k == "BUILD_SHARED_LIBS:BOOL":
+            cmake_global_flags[idx] = ("BUILD_SHARED_LIBS:BOOL", "ON")
+            break
+    try:
+        yield  # control goes to the inner `with` block
+    finally:
+        cmake_global_flags[idx] = ("BUILD_SHARED_LIBS:BOOL", "OFF")

common/tags.py

@@ -0,0 +1,47 @@
+# (c) 2025 by Stephan Menzel
+# Licensed under the Apache License, Version 2.0.
+# See attached file LICENSE for full details
+
+import re
+
+
+def sanitize_tag(git_tag: str) -> str:
+    """Remove common recurring prefixes of git tags to get to a semver like version we want
+    This also sanitizes other tags. Input is any of the tag formats the dependencies may have.
+
+    :return: major.minor.patch variant of the input
+    """
+
+    if git_tag.startswith('openssl'):
+        return git_tag[8:]
+
+    # Qt uses this for lts releases
+    if x := re.match(R"^v(\d+\.\d+\.\d+)-lts-lgpl$", git_tag):
+        return x.group(1)
+
+    if x := re.match(R"^boost-(\d+\.\d+\.\d+)$", git_tag):
+        return x.group(1)
+
+    # Something like asio-1-34-2. Chris does his own thing
+    if x := re.match(R"^asio-(\d+)-(\d+)-(\d+)$", git_tag):
+        return f"{x.group(1)}.{x.group(2)}.{x.group(3)}"
+
+    if x := re.match(R"^curl-(\d+)_(\d+)_(\d+)$", git_tag):
+        return f"{x.group(1)}.{x.group(2)}.{x.group(3)}"
+
+    if x := re.match(R"^hdf5-(\d+\.\d+\.\d+)$", git_tag):
+        return x.group(1)
+
+    if x := re.match(R"^yaml-cpp-(\d+\.\d+\.\d+)$", git_tag):
+        return x.group(1)
+
+    if git_tag.startswith('cares-'):
+        return git_tag[6:]
+
+    if git_tag.startswith('curl-'):
+        return git_tag[5:]
+
+    if x := re.match(R"^v(\d+[._]\d+[._]\d+)$", git_tag):
+        return x.group(1)
+
+    return git_tag