diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..a1721bd --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,13 @@ +Copyright 2025 Stephan Menzel + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..770fe81 --- /dev/null +++ b/README.md @@ -0,0 +1,76 @@ +# Depper Dan +#### Windows Library Builds made easi(er) + +## About + +Depper Dan is a collection of scripts to enable (CMake based) C++ projects +with a set of open source dependencies to build them +automatically. + +## Motivation + +Unlike Linuxoid systems, C++ projects that use Open Source dependencies +traditionally have a hard time building those dependencies, +especially automatic, with custom build settings or when they +depend on one another. + +This script realizes this in a re-usable and extendable manner +but only for a limited set of libraries. It is meant to be used +with Visual Studio 2022 for build tools and modern CMake. + +## Usage + +### Prerequisites + +* Visual Studio (2022 was what this was developed for but others might work) +* git (available in the Path) +* CMake (available in the Path) +* Python (3.11, 3.12, etc) +* This repository, checked out and with temporary disk space + +### Available packages + +Each package that can be built with this system must be +given in `packages.jon`. For every package therein, you will +find a build script in the folder `build_functions`. + +### Build + +To use the system, you must specify an installation folder. +It should have as short a name as possible (on windows) to avoid MAX_PATH issues +which may arise out of any build. +In the examples below we will use `C:\devel\3rd_party`. +This directory will be used to install each package built +with Depper Dan and it should be considered stable and system +wide. + +As a shell, use "x64 Native Tools Command Prompt" to ensure +the correct build tools and environment being set. + +Create a venv unless it's already present and activate it +```PowerShell +> cd /depper/dan/repo +> python -m venv .venv +> .venv\Scripts\Activate +``` + +Build a single package into your installation dir +```PowerShell +> python .\depper_dan.py -i C:\devel\3rd_party -p zlib +``` + +## Extension + +This script is meant to be hacked on. It is not meant +to be a complete extensible package manager like conan or [vcpkg](https://vcpkg.io). + +Extending it by more packages generally involves: + +* Add package information in `packages.jon`. +* Add a build function in `build_functions`. + + +## License + +Depper Dan is being published under the +Apache License 2.0 \ No newline at end of file diff --git a/build_functions/build_abseil.py b/build_functions/build_abseil.py index 90be636..e70f62a 100644 --- a/build_functions/build_abseil.py +++ b/build_functions/build_abseil.py @@ -13,7 +13,7 @@ from common.git_helpers import clone_git_tag from package.package_info import get_package_info -def build_abseil(prefix: Path | str, sbom: dict): +def build_abseil_cpp(prefix: Path | str, sbom: dict) -> Path: print_banner("Building Abseil") @@ -33,8 +33,8 @@ def build_abseil(prefix: Path | str, sbom: dict): # Abseil LTS doesn't build with CMake>=3.30 due to some imported GTest target. # Gotta patch that shit until this situation is resolved. https://github.com/abseil/abseil-cpp/issues/690 - with pushd(abseil_dir): - subprocess.run(["git", "apply", "..\\..\\gtest_fix.patch"]) + # with pushd(abseil_dir): + # subprocess.run(["git", "apply", "..\\..\\gtest_fix.patch"]) install_dir = cmake_build_install(abseil_dir, package_info, cmake_args=abseil_cmake_args) write_package_version_batch(package_info.version) diff --git a/build_functions/build_cares.py b/build_functions/build_cares.py index 69a8673..e7b3091 100644 --- a/build_functions/build_cares.py +++ b/build_functions/build_cares.py @@ -11,7 +11,7 @@ from common.git_helpers import clone_git_tag from package.package_info import get_package_info -def build_cares(prefix: Path | str, sbom: dict): +def build_c_ares(prefix: Path, sbom: dict) -> Path: print_banner("Building C-Ares (gRPC Dependency)") diff --git a/build_functions/build_grpc.py b/build_functions/build_grpc.py index b0c535b..80dc57a 100644 --- a/build_functions/build_grpc.py +++ b/build_functions/build_grpc.py @@ -6,13 +6,13 @@ from pathlib import Path from build_functions.build_utils import print_banner from common.azure import write_package_version_batch -from common.cmake import cmake_build_install +from common.cmake import cmake_build_install, assemble_prefix_path from common.git_helpers import clone_git_tag import common.settings from package.package_info import get_package_info -def build_grpc(prefix: Path | str, sbom: dict): +def build_grpc(prefix: Path, sbom: dict) -> Path: print_banner("Building gRPC") @@ -24,12 +24,16 @@ def build_grpc(prefix: Path | str, sbom: dict): package_info.add_to_sbom(sbom) + deps = [ + package_info.dependency("zlib"), + package_info.dependency("re2"), + package_info.dependency("openssl"), + package_info.dependency("abseil-cpp"), + package_info.dependency("c-ares"), + package_info.dependency("protobuf") + ] + zlib_install_path = package_info.dependency_path("zlib") - re2_install_path = package_info.dependency_path("re2") - openssl_install_path = package_info.dependency_path("openssl") - abseil_install_path = package_info.dependency_path("abseil-cpp") - cares_install_path = package_info.dependency_path("c-ares") - protobuf_install_path = package_info.dependency_path("protobuf") grpc_cmake_args = [ ("BUILD_SHARED_LIBS:BOOL", "OFF"), @@ -45,27 +49,23 @@ def build_grpc(prefix: Path | str, sbom: dict): # ("gRPC_BUILD_MSVC_MP_COUNT:STRING", "1"), # ("gRPC_DOWNLOAD_ARCHIVES:BOOL", "ON"), ("gRPC_ABSL_PROVIDER:STRING", "package"), - ("absl_DIR:PATH", str(abseil_install_path / "lib" / "cmake" / "absl")), ("gRPC_CARES_PROVIDER:STRING", "package"), - ("c-ares_DIR:PATH", str(cares_install_path / "lib" / "cmake" / "c-ares")), ("gRPC_PROTOBUF_PROVIDER:STRING", "package"), - ("Protobuf_DIR:PATH", str(protobuf_install_path / "cmake")), - ("utf8_range_DIR:PATH", str(protobuf_install_path / "lib" / "cmake" / "utf8_range")), - ("gRPC_SSL_PROVIDER", "package"), - ("OPENSSL_ROOT_DIR:PATH", openssl_install_path), + ("gRPC_RE2_PROVIDER", "package"), + ("gRPC_ZLIB_PROVIDER:STRING", "package"), + + ("CMAKE_PREFIX_PATH:STRING", assemble_prefix_path(deps)), + + # ("OPENSSL_ROOT_DIR:PATH", openssl_install_path), ("OPENSSL_USE_STATIC_LIBS:BOOL", "ON"), - ("gRPC_RE2_PROVIDER", "package"), - ("re2_DIR:PATH", str(re2_install_path / "lib" / "cmake" / "re2")), - - ("gRPC_ZLIB_PROVIDER:STRING", "package"), ("ZLIB_ROOT:PATH", str(zlib_install_path)), ("ZLIB_USE_STATIC_LIBS:BOOL", "ON"), # doesn't appear to do its job ("ZLIB_LIBRARY_RELEASE:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name)), ("ZLIB_LIBRARY_DEBUG:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name)) ] - grpc_dir = clone_git_tag(package_info, recursive=True) + grpc_dir = clone_git_tag(package_info, recursive=False) install_dir = cmake_build_install(grpc_dir, package_info, cmake_args=grpc_cmake_args) write_package_version_batch(package_info.version) return install_dir diff --git a/build_functions/build_openssl.py b/build_functions/build_openssl.py index 72186cf..98a8c81 100644 --- a/build_functions/build_openssl.py +++ b/build_functions/build_openssl.py @@ -14,7 +14,7 @@ import common.settings from package.package_info import get_package_info -def build_openssl(prefix: Path | str, sbom: dict): +def build_openssl(prefix: Path, sbom: dict): print_banner("Building OpenSSL") @@ -30,7 +30,7 @@ def build_openssl(prefix: Path | str, sbom: dict): with pushd(openssl_dir): - install_prefix = get_local_prefix(prefix) + install_prefix = package_info.install_location() if not common.settings.rebuild and Path("built_and_installed.txt").exists(): file_and_console_log("already built, exiting") diff --git a/build_functions/build_openusd.py b/build_functions/build_openusd.py index 8b31756..177133d 100644 --- a/build_functions/build_openusd.py +++ b/build_functions/build_openusd.py @@ -6,12 +6,13 @@ from pathlib import Path from build_functions.build_utils import print_banner from common.azure import write_package_version_batch -from common.cmake import cmake_build_install, assemble_prefix_path +from common.cmake import cmake_build_install, assemble_prefix_path, CMakeBuildType import common.settings from common.git_helpers import clone_git_tag from common.settings import temporarily_set_shared from package.package_info import get_package_info + def build_openusd(prefix: Path | str, sbom: dict): print_banner("Building OpenUSD") @@ -47,11 +48,11 @@ def build_openusd(prefix: Path | str, sbom: dict): ("CMAKE_PREFIX_PATH:STRING", assemble_prefix_path(deps)), ("ZLIB_ROOT:PATH", str(zlib_install_path.as_posix())), - ("ZLIB_USE_STATIC_LIBS:BOOL", "ON"), # doesn't appear to do its job + ("ZLIB_USE_STATIC_LIBS:BOOL", "ON"), ("ZLIB_LIBRARY_RELEASE:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name)), ("ZLIB_LIBRARY_DEBUG:FILEPATH", str(zlib_install_path / "lib" / common.settings.zlib_static_lib_name)) ] src_dir = clone_git_tag(package_info, recursive=False) - install_dir = cmake_build_install(src_dir, package_info, cmake_args=cmake_args) + install_dir = cmake_build_install(src_dir, package_info, cmake_args=cmake_args, build_type=CMakeBuildType.RELWITHDEBINFO) write_package_version_batch(package_info.version) return install_dir diff --git a/build_functions/build_re2.py b/build_functions/build_re2.py index 2b743c3..1cd07bb 100644 --- a/build_functions/build_re2.py +++ b/build_functions/build_re2.py @@ -11,7 +11,7 @@ from common.git_helpers import clone_git_tag from package.package_info import get_package_info -def build_re2(prefix: Path | str, sbom: dict): +def build_re2(prefix: Path | str, sbom: dict) -> None: print_banner("Building re2") diff --git a/common/cmake.py b/common/cmake.py index 269b650..f6da137 100644 --- a/common/cmake.py +++ b/common/cmake.py @@ -3,6 +3,7 @@ # See attached file LICENSE for full details import os +from enum import Enum from pathlib import Path from build_functions.build_utils import file_and_console_log, run_in_shell @@ -11,7 +12,17 @@ import common.settings from package.package_info import PackageInfo -def cmake_build_install(local_directory, package_info: PackageInfo, cmake_args: list[tuple[str, str]] = []): +class CMakeBuildType(Enum): + RELEASE = (1, "Release") + RELWITHDEBINFO = (2, "RelWithDebInfo") + DEBUG = (3, "Debug") + + def __init__(self, num, label): + self._num = num + self.label = label + +def cmake_build_install(local_directory, package_info: PackageInfo, cmake_args: list[tuple[str, str]] = [], + build_type: CMakeBuildType = CMakeBuildType.RELEASE) -> Path: argstr = "" # Create flags string out of args tuples @@ -39,7 +50,7 @@ def cmake_build_install(local_directory, package_info: PackageInfo, cmake_args: run_in_shell(f"cmake .. {common.settings.cmake_toolset} -DCMAKE_CONFIGURATION_TYPES:STRING=Release " f"{argstr} -DCMAKE_INSTALL_PREFIX={install_prefix}") - run_in_shell(f'cmake --build . --config Release --target INSTALL --parallel {common.settings.num_cores}') + run_in_shell(f'cmake --build . --config {build_type.label} --target INSTALL --parallel {common.settings.num_cores}') with open("built_and_installed.txt", "w") as lockfile: lockfile.write(f"built release") @@ -51,5 +62,5 @@ def cmake_build_install(local_directory, package_info: PackageInfo, cmake_args: return install_prefix def assemble_prefix_path(packages: list[PackageInfo]) -> str: - paths = [str(pkg.install_location()) for pkg in packages] - return ";".join(paths) \ No newline at end of file + paths = [str(pkg.install_location().as_posix()) for pkg in packages] + return ";".join(paths) diff --git a/common/settings.py b/common/settings.py index fd229ca..93a535a 100644 --- a/common/settings.py +++ b/common/settings.py @@ -26,13 +26,11 @@ run_in_buildpipeline = "BUILD_ARTIFACTSTAGINGDIRECTORY" in os.environ num_cores = 3 if run_in_buildpipeline else 6 if os.name == "posix": - cmake_config_flag = "CMAKE_BUILD_TYPE=Release" cmake_toolset = "" lib_wildcard = "*.a" dll_wildcard = "*.so" zlib_static_lib_name = "libz.a" elif os.name == "nt": # Windows - cmake_config_flag = "CMAKE_CONFIGURATION_TYPES:STRING=Release" cmake_toolset = "-T v143" boost_toolset = "msvc-" + cmake_toolset[-3:-1] + "." + cmake_toolset[-1] boost_bootstrap_toolset = "vc" + cmake_toolset[-3:-1] + cmake_toolset[-1] @@ -53,7 +51,8 @@ cmake_global_flags = [ ("CMAKE_CXX_STANDARD_REQUIRED:BOOL", "ON"), ("CMAKE_MSVC_RUNTIME_LIBRARY:STRING", "\"MultiThreadedDLL\""), ("CMAKE_POSITION_INDEPENDENT_CODE:BOOL", "ON"), - ("CMAKE_BUILD_TYPE:STRING", "Release") + ("CMAKE_CONFIGURATION_TYPES:STRING", "Release;RelWithDebInfo"), + # ("CMAKE_BUILD_TYPE:STRING", "Release") ] def set_global_rebuild(new_rebuild: bool = False) -> None: diff --git a/build_deps.py b/depper_dan.py similarity index 82% rename from build_deps.py rename to depper_dan.py index fc5abe0..3355ab1 100644 --- a/build_deps.py +++ b/depper_dan.py @@ -15,10 +15,10 @@ import importlib from build_functions.build_ecal import build_ecal from build_functions.build_ecaludp import build_ecaludp from build_functions.build_ftxui import build_ftxui -from build_functions.build_abseil import build_abseil +from build_functions.build_abseil import build_abseil_cpp from build_functions.build_asio import build_asio from build_functions.build_boost import build_boost -from build_functions.build_cares import build_cares +from build_functions.build_cares import build_c_ares from build_functions.build_ceres import build_ceres from build_functions.build_curl import build_curl from build_functions.build_cppzmq import build_cppzmq @@ -58,7 +58,7 @@ from package.package_info import PackageInfo, get_package_info, build_package_tr import sys from pathlib import Path -from common.errors import DependencyInfoError +from common.errors import DependencyInfoError, BuildError from build_functions.build_utils import print_banner, file_and_console_log, logfile @@ -89,10 +89,10 @@ def build_all(prefix: Path | str) -> dict: file_and_console_log(f"Installing to: {prefix}") # First those without any dependencies - build_abseil(prefix, sbom) + build_abseil_cpp(prefix, sbom) build_asio(prefix, sbom) build_boost(prefix, sbom) - build_cares(prefix, sbom) + build_c_ares(prefix, sbom) build_ftxui(prefix, sbom) build_glog(prefix, sbom) build_eigen(prefix, sbom) @@ -140,7 +140,7 @@ def build_all(prefix: Path | str) -> dict: build_functions = { - "abseil-cpp": build_abseil, + "abseil-cpp": build_abseil_cpp, "asio": build_asio, "boost": build_boost, "ceres-solver": build_ceres, @@ -168,7 +168,7 @@ build_functions = { "recycle": build_recycle, "spdlog": build_spdlog, "tclap": build_tclap, - "c-ares": build_cares, + "c-ares": build_c_ares, "grpc": build_grpc, "qt5": build_qt5, "qt6": build_qt6, @@ -181,13 +181,50 @@ build_functions = { "zlib": build_zlib, } +def call_builder(package_name: str) -> None: + """Call a build function by name or throw if it doesn't exist + """ + func_name = f"build_{package_name.lower().replace('-', '_')}" + func = globals().get(func_name) + if callable(func): + + sbom = {} + + # Build the package, amend SBOM with package info + func(prefix=Path(), sbom=sbom) + + # The partial SBOM gets written into the install dir so we can later combine it. + # We cannot merge them into one file right here as in the pipeline there may be multiple tasks in parallel + package_info = get_package_info(package_name) + partial_sbom_filename = package_info.install_location() / "partial_sbom.json" + with open(partial_sbom_filename, "w") as f: + json.dump(sbom, f) + + else: + raise DependencyInfoError(f"Don't know how to build {package_name}") + + +def build_package(package_name: str) -> None: + """Build the named package including all of its dependencies + """ + package_info = get_package_info(package_name) + deps = package_info.get_dependencies() + + # Build the dependencies first + for d in deps: + call_builder(d) + + # And then the actual package + call_builder(package_name) + + if __name__ == '__main__': try: with open(logfile, 'w') as f: f.write(str(datetime.datetime.now()) + f' - started {__file__}\n') - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser("Depper Dan") parser.add_argument("--install-dir", "-i", type=str, required=True, help="install into this directory") parser.add_argument("--package", "-p", type=str, default="all", help="build specific package or 'all'") parser.add_argument("--rebuild", action="store_true", help="always delete build dir") @@ -220,20 +257,7 @@ if __name__ == '__main__': with open(sbomfile, "w") as f: json.dump(sbom, f) else: - if package not in build_functions: - raise DependencyInfoError(f"Don't know how to build {package}") - - sbom = {} - - # Build the package, amend SBOM with package info - build_functions[package](prefix=global_prefix, sbom=sbom) - - # The partial SBOM gets written into the install dir so we can later combine it. - # We cannot merge them into one file right here as in the pipeline there may be multiple tasks in parallel - package_info = get_package_info(package) - partial_sbom_filename = package_info.install_location() / "partial_sbom.json" - with open(partial_sbom_filename, "w") as f: - json.dump(sbom, f) + build_package(package) print_banner("Done") exit(0) diff --git a/package/package_info.py b/package/package_info.py index 6e6a979..c7a79d7 100644 --- a/package/package_info.py +++ b/package/package_info.py @@ -64,6 +64,22 @@ class PackageInfo: raise DependencyInfoError(f"Cannot find dependency {package_name} in {self.name}.") return self.dependencies[package_name] + def get_dependencies(self) -> list[str]: + """Hand out all the dependencies in order of suitable build + Those without any dependencies will be first, then the ones depending on the above + """ + + deps_0 = set[str]() + deps_1 = set[str]() + + # I should be recursive here but right now I'm too lazy for this + for name, info in self.dependencies.items(): + if len(info.dependencies) == 0: + deps_0.add(name) + else: + deps_1.add(name) + return list(deps_0) + list(deps_1) + def dependency_path(self, package_name: str) -> Path: """Give the path underneath which the package, according to its version, is installed""" if package_name not in self.dependencies: diff --git a/packages.json b/packages.json index d7ef8a5..074daf9 100644 --- a/packages.json +++ b/packages.json @@ -1,8 +1,8 @@ { "abseil-cpp": { "repo": "https://github.com/abseil/abseil-cpp.git", - "tag": "20240116.2", - "version": "24.1.16", + "tag": "20250127.1", + "version": "25.1.27", "description": "Collection of peer reviewed C++ libraries by Google", "license_id": "Apache-2.0", "license_url": "https://github.com/abseil/abseil-cpp/blob/master/LICENSE" @@ -252,8 +252,8 @@ }, "openssl": { "repo": "https://github.com/openssl/openssl.git", - "tag": "openssl-3.2.4", - "version": "3.2.4", + "tag": "openssl-3.2.5", + "version": "3.2.5", "depends": [ "zlib" ], @@ -317,8 +317,8 @@ }, "grpc": { "repo": "https://github.com/grpc/grpc.git", - "tag": "v1.65.1", - "version": "1.65.1", + "tag": "v1.72.2", + "version": "1.72.2", "depends": [ "zlib", "re2",