From fa652448f4865f9bcffba21a085f5fcda6e03f1b Mon Sep 17 00:00:00 2001 From: Konrad Breitsprecher Date: Tue, 27 Oct 2026 16:30:30 +0100 Subject: [PATCH 1/4] scripts: add relocatable tarball builder (pkgformat tgz) SilKitTGZ builds SIL Kit with cmake/ctest like the deb (RelWithDebInfo, SILKIT_BUILD_LINUX_PACKAGE=ON, lib/ as libdir) and packs two archives: - libsilkit-..x86_64.tar.gz with bin/, lib/ (cmake config), include/, man pages, the registry systemd unit and share/doc/libsilkit/ (CHANGELOG.md of the version, LICENSE, README.rst) - libsilkit-..x86_64.debug.tar.gz with the split debug info under lib/debug/.build-id/ (dwz, objcopy --only-keep-debug, gnu-debuglink) Platforms: ubuntu2004 (gcc), el7 (devtoolset-11), el8 (gcc-toolset-12). The archives are reproducible: SOURCE_DATE_EPOCH (default: commit time of the sil-kit sources) for all mtimes, sorted entries, root ownership, normalized modes, no name or time in the gzip header, -ffile-prefix-map for the build paths. Packed with Python tarfile, GNU tar 1.26 on el7 lacks --sort. Co-Authored-By: Claude Opus 5.5 --- scripts/silkit_linux_packaging.py | 3 + scripts/silkit_tgz.py | 276 ++++++++++++++++++++++++++++++ 2 files changed, 279 insertions(+) create mode 100644 scripts/silkit_tgz.py diff --git a/scripts/silkit_linux_packaging.py b/scripts/silkit_linux_packaging.py index 51c4636..f44a8cc 100644 --- a/scripts/silkit_linux_packaging.py +++ b/scripts/silkit_linux_packaging.py @@ -16,6 +16,7 @@ from silkit_pkg_interface import SilKitPKG from silkit_deb import SilKitDEB from silkit_rpm import SilKitRPM +from silkit_tgz import SilKitTGZ logger = logging.getLogger("SilKit Packaging") @@ -25,6 +26,8 @@ def PkgClassFactory(build_info: BuildInfo) -> SilKitPKG: return SilKitDEB(build_info) elif build_info.pkgformat.lower() == "rpm": return SilKitRPM(build_info) + elif build_info.pkgformat.lower() == "tgz": + return SilKitTGZ(build_info) else: raise NotImplementedError(f"PKG format {build_info.pkgformat} is not implemented yet!") diff --git a/scripts/silkit_tgz.py b/scripts/silkit_tgz.py new file mode 100644 index 0000000..7865cf6 --- /dev/null +++ b/scripts/silkit_tgz.py @@ -0,0 +1,276 @@ +import gzip +import logging +import os +import re +import shutil +import subprocess +import tarfile + +from pathlib import Path + +from silkit_pkg_interface import SilKitPKG +from silkit_pkg_utils import BuildInfo, BuildFlags +from silkit_pkg_utils import get_global_loglevel, get_global_formatting + +############################################################################### +logger = logging.getLogger("TGZBuilder") +console_handler = logging.StreamHandler() +console_handler.setLevel(get_global_loglevel()) +console_handler.setFormatter(logging.Formatter(get_global_formatting())) +logger.addHandler(console_handler) +logger.propagate = False + + +class SilKitTGZ(SilKitPKG): + """Relocatable binary tarball plus a separate debug symbol tarball. + + Layout of libsilkit-..x86_64.tar.gz: + bin/, lib/ (libSilKit.so*, cmake/SilKit), include/, share/man/man1/, + share/sil-kit/sil-kit-registry.service, share/doc/libsilkit/ + Layout of libsilkit-..x86_64.debug.tar.gz: + lib/debug/.build-id/xx/yyyy.debug + """ + + def __init__(self, build_info: BuildInfo): + g_loglevel = get_global_loglevel() + if logger.level != g_loglevel: + # Update loglevel, not perfect but works + logger.setLevel(g_loglevel) + console_handler.setLevel(g_loglevel) + logger.debug("Creating TGZBuilder Instance") + self.build_info = build_info + + def get_buildinfo(self): + return self.build_info + + def source_dir_name(self): + silkit_version = self.build_info.version + return f"libsilkit{silkit_version.major}-{str(silkit_version)}" + + def package_name(self): + # '~' is not kept by GitHub release assets, use '-' for suffixes like rc1 + version = str(self.build_info.version).replace("~", "-") + return f"libsilkit-{version}.{self.build_info.platform}.x86_64" + + def setup_build_env(self): + self.build_flags = SilKitTGZ.__get_build_flags(self.build_info.platform) + logger.info(f"Got build_flags: {self.build_flags}") + + work_dir = self.build_info.work_dir.resolve() + self.source_dir = work_dir / self.source_dir_name() + self.build_dir = work_dir / "_build_tgz" + self.stage_dir = work_dir / "_stage_tgz" / self.package_name() + self.debug_stage_dir = work_dir / "_stage_tgz_debug" / (self.package_name() + ".debug") + + for d in (self.build_dir, self.stage_dir.parent, self.debug_stage_dir.parent): + shutil.rmtree(d, ignore_errors=True) + d.mkdir(parents=True) + + self.source_date_epoch = self.__source_date_epoch() + logger.info(f"SOURCE_DATE_EPOCH={self.source_date_epoch}") + + def build(self): + try: + self.__configure() + self.__run(["cmake", "--build", str(self.build_dir)]) + self.__run(["ctest", "--output-on-failure", "-R", "^(I|T)", "--test-dir", str(self.build_dir)]) + self.__run(["cmake", "--install", str(self.build_dir), "--prefix", str(self.stage_dir)]) + self.__split_debug_info() + self.__add_extra_files() + self.__pack(self.stage_dir, self.build_info.work_dir / (self.package_name() + ".tar.gz")) + self.__pack(self.debug_stage_dir, self.build_info.work_dir / (self.package_name() + ".debug.tar.gz")) + except RuntimeError: + raise + except Exception as ex: + raise RuntimeError("Error while building the tarball") from ex + + def copy_artifacts(self): + try: + Path.mkdir(self.build_info.output_dir, parents=True, exist_ok=True) + except Exception as ex: + logger.error(f"Cannot create output dir {self.build_info.output_dir}") + logger.error(f"mkdir: {str(ex)}") + return + + for suffix in (".tar.gz", ".debug.tar.gz"): + shutil.copy2(self.build_info.work_dir / (self.package_name() + suffix), self.build_info.output_dir) + + ########################################################################### + + def __run(self, cmd, **kwargs): + logger.debug(f"Calling: {' '.join(cmd)}") + try: + return subprocess.run(cmd, check=True, **kwargs) + except subprocess.CalledProcessError as ex: + raise RuntimeError(f"Command failed: {' '.join(cmd)}") from ex + + def __source_date_epoch(self) -> int: + epoch = os.environ.get("SOURCE_DATE_EPOCH") + if epoch: + return int(epoch) + # Fall back to the commit time of the sil-kit sources + result = subprocess.run( + ["git", "-c", "safe.directory=*", "-C", str(self.source_dir), "log", "-1", "--format=%ct"], + check=False, + stdout=subprocess.PIPE, + universal_newlines=True, + ) + if result.returncode != 0 or not result.stdout.strip(): + raise RuntimeError("SOURCE_DATE_EPOCH is not set and the sil-kit sources have no git history") + return int(result.stdout.strip()) + + def __configure(self): + # Map the build paths away so they do not end up in the binaries + prefix_map = f"-ffile-prefix-map={self.source_dir}=. -ffile-prefix-map={self.build_dir}=." + flags = f"-gdwarf-4 {prefix_map} {self.build_flags.add_platform_flags}".strip() + + env = os.environ.copy() + env["CFLAGS"] = flags + env["CXXFLAGS"] = flags + env["SOURCE_DATE_EPOCH"] = str(self.source_date_epoch) + + self.__run( + [ + "cmake", + "-S", str(self.source_dir), + "-B", str(self.build_dir), + "-GNinja", + "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + f"-DCMAKE_C_COMPILER={self.build_flags.c_compiler}", + f"-DCMAKE_CXX_COMPILER={self.build_flags.cxx_compiler}", + "-DCMAKE_INSTALL_LIBDIR=lib", + "-DCMAKE_POLICY_VERSION_MINIMUM=3.5", + "-DSILKIT_BUILD_LINUX_PACKAGE=ON", + "-DSILKIT_BUILD_DEMOS=OFF", + "-DSILKIT_INSTALL_SOURCE=OFF", + ], + env=env, + ) + + def __elf_files(self): + for path in sorted(self.stage_dir.rglob("*")): + if path.is_symlink() or not path.is_file(): + continue + with open(path, "rb") as f: + if f.read(4) == b"\x7fELF": + yield path + + def __build_id(self, path: Path) -> str: + notes = subprocess.run( + ["readelf", "-n", str(path)], check=True, stdout=subprocess.PIPE, universal_newlines=True + ).stdout + match = re.search(r"Build ID:\s*([0-9a-f]+)", notes) + if not match: + raise RuntimeError(f"{path} has no build ID, cannot place its debug info") + return match.group(1) + + def __split_debug_info(self): + for elf in self.__elf_files(): + logger.info(f"Splitting debug info off {elf.relative_to(self.stage_dir)}") + # dwz is an optimization only; it refuses some inputs, that is fine + subprocess.run(["dwz", "-q", str(elf)], check=False) + + build_id = self.__build_id(elf) + debug_file = ( + self.debug_stage_dir / "lib" / "debug" / ".build-id" / build_id[:2] / (build_id[2:] + ".debug") + ) + debug_file.parent.mkdir(parents=True, exist_ok=True) + + self.__run(["objcopy", "--only-keep-debug", "--compress-debug-sections=zlib", str(elf), str(debug_file)]) + debug_file.chmod(0o644) + self.__run( + [ + "objcopy", + "--strip-debug", + "--strip-unneeded", + f"--add-gnu-debuglink={debug_file}", + str(elf), + ] + ) + + def __find_changelog(self) -> Path: + version = self.build_info.version + versions_dir = self.source_dir / "docs" / "changelog" / "versions" + plain_version = f"{version.major}.{version.minor}.{version.patch}" + + changelog = versions_dir / f"{plain_version}.md" + if not changelog.exists(): + # During release preparation the entry still lives in latest.md + changelog = versions_dir / "latest.md" + + with open(changelog, "r", encoding="utf-8") as f: + header = f.readline() + match = re.match(r"^#\s*\[([0-9]+\.[0-9]+\.[0-9]+)\]", header) + if not match or match.group(1) != plain_version: + raise RuntimeError(f"{changelog} does not describe version {plain_version}: {header.strip()}") + return changelog + + def __add_extra_files(self): + pkg_path = self.build_info.silkit_pkg_path.expanduser() + + service_dir = self.stage_dir / "share" / "sil-kit" + service_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(pkg_path / "rpm" / "sil-kit-registry.service", service_dir / "sil-kit-registry.service") + + # cmake installs LICENSE and README only with SILKIT_INSTALL_SOURCE=ON + doc_dir = self.stage_dir / "share" / "doc" / "libsilkit" + doc_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(self.source_dir / "LICENSE", doc_dir / "LICENSE") + shutil.copy2(self.source_dir / "README.rst", doc_dir / "README.rst") + shutil.copy2(self.__find_changelog(), doc_dir / "CHANGELOG.md") + + def __pack(self, stage_dir: Path, archive: Path): + """Reproducible tar.gz: sorted entries, root ownership, normalized modes, + mtime = SOURCE_DATE_EPOCH, and no name or time in the gzip header.""" + + epoch = self.source_date_epoch + + def normalize(info: tarfile.TarInfo) -> tarfile.TarInfo: + info.uid = info.gid = 0 + info.uname = info.gname = "" + info.mtime = epoch + if info.isdir(): + info.mode = 0o755 + elif info.isfile(): + info.mode = 0o755 if info.mode & 0o111 else 0o644 + return info + + logger.info(f"Packing {archive.name}") + with open(archive, "wb") as raw: + with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as gz: + with tarfile.open(fileobj=gz, mode="w", format=tarfile.GNU_FORMAT) as tar: + # add() recurses in sorted order (Python >= 3.7) + tar.add(str(stage_dir), arcname=stage_dir.name, filter=normalize) + + @staticmethod + def __get_build_flags(platform: str) -> BuildFlags: + logger.info(f"Building for platform: {platform}") + + def env_or(name, default): + return os.environ.get(name, default) + + if platform == "ubuntu2004": + return BuildFlags( + add_platform_flags="", + add_debuild_flags=[], + c_compiler=env_or("CC", "gcc"), + cxx_compiler=env_or("CXX", "g++"), + ) + + if platform == "el7": + return BuildFlags( + add_platform_flags="", + add_debuild_flags=[], + c_compiler=env_or("CC", "/opt/rh/devtoolset-11/root/usr/bin/gcc"), + cxx_compiler=env_or("CXX", "/opt/rh/devtoolset-11/root/usr/bin/g++"), + ) + + if platform == "el8": + return BuildFlags( + add_platform_flags="", + add_debuild_flags=[], + c_compiler=env_or("CC", "/opt/rh/gcc-toolset-12/root/usr/bin/gcc"), + cxx_compiler=env_or("CXX", "/opt/rh/gcc-toolset-12/root/usr/bin/g++"), + ) + + raise RuntimeError(f"Unknown tarball platform: {platform}") From 2b97fcc6eeaf5de3a1bbed99ef59f756f1333919 Mon Sep 17 00:00:00 2001 From: Konrad Breitsprecher Date: Tue, 27 Oct 2026 16:30:30 +0100 Subject: [PATCH 2/4] debian: map build paths for reproducible packages dpkg 1.19 on Ubuntu 20.04 does not enable reproducible=+fixfilepath by default. Also use the real option name SILKIT_INSTALL_SOURCE. Co-Authored-By: Claude Opus 5.5 --- debian/rules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/rules b/debian/rules index 252aa7e..8ab5b56 100755 --- a/debian/rules +++ b/debian/rules @@ -6,9 +6,9 @@ export DEB_CXXFLAGS_MAINT_APPEND = "${PLATFORM_BUILD_FLAGS}" %: dh $@ --buildsystem=cmake+ninja --builddirectory=_build -export DEB_BUILD_MAINT_OPTIONS=optimize=-lto +export DEB_BUILD_MAINT_OPTIONS=optimize=-lto reproducible=+fixfilepath override_dh_auto_configure: - dh_auto_configure -- -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_SOURCE=Off -DSILKIT_BUILD_LINUX_PACKAGE=On -DSILKIT_BUILD_DEMOS=Off -DCMAKE_POLICY_VERSION_MINIMUM=3.5 + dh_auto_configure -- -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DSILKIT_INSTALL_SOURCE=Off -DSILKIT_BUILD_LINUX_PACKAGE=On -DSILKIT_BUILD_DEMOS=Off -DCMAKE_POLICY_VERSION_MINIMUM=3.5 override_dh_auto_test: ctest --output-on-failure -R '^(I|T)' --test-dir _build From 534a69c459ce6366089c9ecf50d4ba7c09b31133 Mon Sep 17 00:00:00 2001 From: Konrad Breitsprecher Date: Tue, 27 Oct 2026 16:30:31 +0100 Subject: [PATCH 3/4] ci: build all Linux packages from release/packages.json release/packages.json is the single source of truth for the Linux packages: every target names its build image, format, platform, test image, whether it is released and its asset globs. Released are the Ubuntu 20.04 debs (with the dbgsym ddeb and buildinfo) and the tarballs plus debug tarballs for Ubuntu 20.04, CentOS 7 and AlmaLinux 8; the arm64 deb and the el9 rpm are built but not released. package-silkit.yml is rewritten around it: - inputs silkit_ref (pinned sha or tag), pkg_sha, expected_version (checked against debian/changelog, rpm spec and the sil-kit sources), correlation_id (run name); maintainer defaults to debian/control - builds with docker run on the host, the actions do not run in CentOS 7 - tests every package in a plain distribution container (tests/run_package_test.sh, test_deb.py learned --package-format tgz) - uploads the artifact sil-kit-pkg (30 days) with all assets, pkg-manifest.json (sil-kit sha and submodules, pkg sha, image digests, sha256 and size per file) and SHA256 - no release job: sil-kit-delivery publishes the released assets to the vectorgrp/sil-kit and vectorgrp/sil-kit-pkg releases rebuild-check.yml builds everything twice and compares the sha256 of all files, with a diffoscope report on differences. The legacy .github/actions (debian:sid docker action) is removed. Co-Authored-By: Claude Opus 5.5 --- .github/actions/Dockerfile | 31 -- .github/actions/action.yml | 9 - .github/actions/build_deb.sh | 140 ------- .github/actions/entrypoint.sh | 11 - .github/actions/fetch_cmake.sh | 37 -- .github/workflows/package-silkit.yml | 540 ++++++++++----------------- .github/workflows/rebuild-check.yml | 85 +++++ README.rst | 106 ++++-- release/packages.json | 110 ++++++ scripts/check_version.py | 93 +++++ scripts/collect_packages.py | 129 +++++++ tests/run_package_test.sh | 74 ++++ tests/test_deb.py | 72 +++- 13 files changed, 821 insertions(+), 616 deletions(-) delete mode 100644 .github/actions/Dockerfile delete mode 100644 .github/actions/action.yml delete mode 100755 .github/actions/build_deb.sh delete mode 100755 .github/actions/entrypoint.sh delete mode 100644 .github/actions/fetch_cmake.sh create mode 100644 .github/workflows/rebuild-check.yml create mode 100644 release/packages.json create mode 100644 scripts/check_version.py create mode 100644 scripts/collect_packages.py create mode 100755 tests/run_package_test.sh diff --git a/.github/actions/Dockerfile b/.github/actions/Dockerfile deleted file mode 100644 index f4d5eb3..0000000 --- a/.github/actions/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -FROM debian:sid - -ENV TZ "Europe/Berlin" -ENV DEBIAN_FRONTEND "noninteractive" -ENV LC_ALL C.UTF-8 -ENV LANG C.UTF-8 - -RUN apt update \ - && apt -y install \ - sudo \ - build-essential \ - debhelper \ - devscripts \ - dh-make \ - dh-cmake \ - \ - ninja-build \ - \ - libfmt-dev \ - libspdlog-dev \ - libasio-dev \ - libyaml-cpp-dev \ - wget \ - libgtest-dev \ - libgmock-dev \ - && true - -COPY entrypoint.sh /entrypoint.sh - -ENTRYPOINT ["/entrypoint.sh"] - diff --git a/.github/actions/action.yml b/.github/actions/action.yml deleted file mode 100644 index 71248c4..0000000 --- a/.github/actions/action.yml +++ /dev/null @@ -1,9 +0,0 @@ -# action.yml -name: 'Package SIL Kit for Debian' -description: 'Package SIL Kit for Debian' -runs: - using: 'docker' - image: 'Dockerfile' - args: - - .github/actions/build_deb.sh - diff --git a/.github/actions/build_deb.sh b/.github/actions/build_deb.sh deleted file mode 100755 index c23c8e9..0000000 --- a/.github/actions/build_deb.sh +++ /dev/null @@ -1,140 +0,0 @@ -#! /bin/sh - -echo "Building the SIL Kit packages" - -if [ -z $SILKIT_PKG_URL ] ; then - SILKIT_PKG_URL="." -fi -debian_path="${SILKIT_PKG_URL}/debian" - -platform=$1 - -echo "SILKIT-PKG: Building for platform $platform" - -silkit_pkg_found=false -# Check if the provided path has a debian directory -if [ ! -d ${debian_path} ] ; then - echo "SILKIT-PKG: Could not find local debian directory, trying to clone the sil-kit-pkg git repo!" - git -c http.sslVerify=false clone --depth=1 ${SILKIT_PKG_URL} -b ${SILKIT_PKG_REF} sil-kit-pkg - ret_val=$? - if [ "$ret_val" != '0' ] ; then - echo "SILKIT-PKG: Could not clone $SILKIT_PKG_URL : $SILKIT_PKG_REF" - exit 64 - fi - debian_path="./sil-kit-pkg/debian" -fi - - -SILKIT_FULL_VERSION=`sed -En "s/libsilkit \(([0-9]+\.[0-9]+\.[0-9]+)-*([0-9]+).*/\1 \2/p" $debian_path/changelog` -SILKIT_VERSION=`echo $SILKIT_FULL_VERSION | cut -d ' ' -f1` -SILKIT_DEBIAN_REVISION=`echo $SILKIT_FULL_VERSION | cut -d ' ' -f2` -echo "Packaging SILKIT Version $SILKIT_VERSION" -echo "Packaging DEBIAN Version $SILKIT_DEBIAN_REVISION" - -# Check if running in CI -if [ -n "${CI_RUN+1}" ] ; then - - echo "Saving VERSION and REVISION to Github step outputs!" - echo "silkit_version=${SILKIT_VERSION}" >> "$GITHUB_OUTPUT" - echo "silkit_debian_revision=${SILKIT_DEBIAN_REVISION}" >> "$GITHUB_OUTPUT" -fi - -if [ -d ${SILKIT_SOURCE_URL} ] ; then - echo "SILKIT-PKG: Using local checkout!" - silkit_path="${SILKIT_SOURCE_URL}" - -else - ## Set GIT Options - if [ -n SILKIT_VENDORED_PACKAGES ] ; then - echo "Vendoring Packages!" - SUBMODULE_CMD="--recurse-submodules --shallow-submodules" - else - SUBMODULE_CMD="" - fi - - # Since we want to allow building SIL KIT from VERSIONS different from the current CHANGELOG Version - # We need to distinguish how to checkout and reset the local git repo - if [ -n SILKIT_REVISION ] ; then - CLONE_VERSION="main" - else - CLONE_VERSION="sil-kit/v${SILKIT_VERSION}" - fi - - git -c http.sslVerify=false clone ${SUBMODULE_CMD} --depth=1 ${SILKIT_SOURCE_URL} -b ${CLONE_VERSION} libsilkit-${SILKIT_VERSION} - ret_val=$? - if [ "$ret_val" != '0' ] ; then - echo "SILKIT-PKG: Could get SIL Kit sources at ${SILKIT_SOURCE_URL}:${SILKIT_REVISION}" - echo "Trying local directory at ${PWD}/libsilkit-${SILKIT_VERSION}" - fi - - if [ ! -d ./libsilkit-${SILKIT_VERSION} ]; then - echo "No SIL Kit sources found!" - exit 64 - fi - - silkit_path="./libsilkit-${SILKIT_VERSION}" - - # Get the correct git ref of the package - if [ -n $SILKIT_REVISION ] ; then - echo "WARNING: SILKIT GIT REF specified! MAKE SURE THAT THIS VERSION IS COMPATIBLE WITH THE VERSION SPECIFIED IN THE DEBIAN/UBUNTU CHANGELOG!" - echo "SILKIT REVISION: ${SILKIT_REVISION}" - echo "TRY GETTING REF AS TAGS" - git -c http.sslVerify=false -C ./libsilkit-${SILKIT_VERSION} fetch --depth=1 origin refs/tags/${SILKIT_REVISION}:refs/tags/${SILKIT_REVISION} --no-tags - ret_val=$? - if [ "$ret_val" != '0' ] ; then - echo "TRY GETTING REF AS COMMIT SHA" - git -c http.sslVerify=false -C ./libsilkit-${SILKIT_VERSION} fetch --depth=1 origin ${SILKIT_REVISION} - ret_val=$? - fi - - if [ "$ret_val" != '0' ] ; then - echo "Could not find REF $SILKIT_REVISION in $SILKIT_SOURCE_URL\nExiting" - exit 64 - fi - echo "RESETTING TO REF: $SILKIT_REVISION" - git -C ./libsilkit-${SILKIT_VERSION} reset --hard ${SILKIT_REVISION} - fi -fi - -tar --exclude='.git' -czf ${silkit_path}/../libsilkit_${SILKIT_VERSION}.orig.tar.gz -C ${silkit_path} . -echo "Running tar on ${silkit_path}:" -ret_val=$? -if [ "$ret_val" != '0' ] ; then - echo "SILKIT-PKG: Could create orig tarball $orig_tarball" - exit 64 -fi - -cp -r ${debian_path} ${silkit_path} -cd ${silkit_path} - -# Set build environment for the different platforms -case $platform in - 20.04 ) platform_additional_build_flags="" - debuild_additional_flags="-d --prepend-path=/opt/vector/bin/" - C_COMPILER="clang-10" - CXX_COMPILER="clang++-10" ;; - - 22.04 ) platform_additional_build_flags=-gdwarf-4 - debuild_additional_flags="" - C_COMPILER="clang" - CXX_COMPILER="clang++" ;; - - * ) platform_additional_build_flags="" - debuild_additional_flags="" - C_COMPILER="clang" - CXX_COMPILER="clang++" ;; -esac - -echo "Additional build flags: $platform_additional_build_flags" -echo "Additional debuild flags: $debuild_additional_flags" -echo "C COMPILER: $C_COMPILER" -echo "CXX COMPILER: $CXX_COMPILER" - -echo "Running debuild:" -debuild $debuild_additional_flags --set-envvar=PLATFORM_BUILD_FLAGS=$platform_additional_build_flags --set-envvar=CC=$C_COMPILER --set-envvar=CXX=$CXX_COMPILER -us -uc --lintian-opts -E --pedantic - -ret_val=$? -if [ "$ret_val" != '0' ] ; then - echo "SILKIT-PKG: \"debuild -us -uc\" exit code $ret_val" - exit 64 -fi diff --git a/.github/actions/entrypoint.sh b/.github/actions/entrypoint.sh deleted file mode 100755 index bfd191e..0000000 --- a/.github/actions/entrypoint.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -runner_uid=$(ls -lnd . | awk '{ print $3; }') -runner_gid=$(ls -lnd . | awk '{ print $4; }') -echo "Runner UID: ${runner_uid}" -echo "Runner GID: ${runner_gid}" - -useradd --uid "${runner_uid}" --no-user-group --home-dir "$(pwd)" --no-create-home worker - -sudo -E -u worker -- "$@" - diff --git a/.github/actions/fetch_cmake.sh b/.github/actions/fetch_cmake.sh deleted file mode 100644 index c74cab2..0000000 --- a/.github/actions/fetch_cmake.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# fetch an appropriate cmake if necessary -# we need at least cmake 3.11, ubuntu 18.04 LTS has 3.10 -set -u #bail on undefined var -CMAKE_VERSION=${CMAKE_VERSION:-3.25.1} -dist="https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" -dest="/usr" -sha256="3a5008b613eeb0724edeb3c15bf91d6ce518eb8eebc6ee758f89a0f4ff5d1fd6" -die() { - echo -e -- "ERRROR $@" - exit 2 -} -#check that cmake has a minor number that is larger than 10 (e.g. 3.11.X) -if which cmake >/dev/null -then - vers="$(cmake --version| awk '/^cmake v/{print $3}')" - minor="$(echo $vers | sed -e 's;[0-9]\+\.\([0-9]\+\)\.[0-9]\+;\1;g')" - if [ "${minor}" -gt "11" ] - then - echo "Suitable cmake found: version ${vers}" - exit 0 - fi -fi -( - set -e - mkdir -p ${dest} 2> /dev/null - cd /tmp - tgz=$(basename ${dist}) - wget --no-check-certificate ${dist} - _sum=$(sha256sum ${tgz} | awk '{print $1}') - if [ "${sha256}" != "${_sum}" ] - then - rm -f ${tgz} - die "Downloaded cmake checksum mismatch:\nwant: ${sha256}\nhave: ${_sum}" - fi - tar -xzf ${tgz} --strip=1 -C ${dest} -) || die cmake fetch failed diff --git a/.github/workflows/package-silkit.yml b/.github/workflows/package-silkit.yml index 18fb4a6..219d892 100644 --- a/.github/workflows/package-silkit.yml +++ b/.github/workflows/package-silkit.yml @@ -1,85 +1,86 @@ name: "SilKit Packaging Workflow" +run-name: "package ${{ inputs.silkit_ref }} ${{ inputs.correlation_id }}" + +# Builds every target in release/packages.json from a pinned sil-kit ref, tests each +# package in a plain distribution container and publishes everything as the artifact +# "sil-kit-pkg" (packages, pkg-manifest.json, SHA256). +# +# Started manually, or by sil-kit-delivery (pr.yaml) with +# gh workflow run package-silkit.yml --ref -f silkit_ref= ... +# Releases are created by sil-kit-delivery promote.yaml from this artifact. + on: workflow_dispatch: inputs: - silkit_source_repo: - description: 'URL to the sil-kit repository in the form namespace/repo.git' + silkit_ref: + description: "sil-kit commit SHA or tag (vectorgrp/sil-kit)" required: true type: string - silkit_source_ref: - description: 'Ref of the SIL Kit sources repo.' + pkg_sha: + description: "Expected sil-kit-pkg commit; the run fails if the dispatched ref is at another commit" + required: false + type: string + expected_version: + description: "Expected SIL Kit version (major.minor.patch); fails if changelog, spec or sources disagree" required: false type: string - default: 'main' - silkit_pkg_ref: - description: 'sil-kit-pkg ref. Creates a release if not empty' + correlation_id: + description: "Free text shown in the run name, e.g. the triggering delivery run" required: false type: string debian_fullname: - description: 'Name of the maintainer creating the current package' - required: true + description: "Maintainer name for the Debian build (default: Maintainer in debian/control)" + required: false type: string debian_email: - description: 'Email of the maintainer creating the current package' - required: true + description: "Maintainer email for the Debian build (default: Maintainer in debian/control)" + required: false type: string - debian_arch: - description: 'Ubuntu CPU arch the package should be build for' - required: true - default: 'amd64' + workflow_call: + inputs: + silkit_ref: { required: true, type: string } + pkg_sha: { required: false, type: string } + expected_version: { required: false, type: string } + correlation_id: { required: false, type: string } + debian_fullname: { required: false, type: string } + debian_email: { required: false, type: string } + artifact_suffix: + description: "Appended to all artifact names, so the workflow can run twice in one run" + required: false type: string +permissions: + contents: read + packages: read + env: PKG_WORKDIR: _i - PKG_OUTDIR: _o + PKG_OUTDIR: _o + ARTIFACT_SUFFIX: ${{ inputs.artifact_suffix }} jobs: - get_version: + targets: + name: Targets and version runs-on: ubuntu-latest - name: Job to get the current SIL KIT Version to package outputs: - silkit_major: ${{ steps.get_version.outputs.silkit_major }} - silkit_minor: ${{ steps.get_version.outputs.silkit_minor }} - silkit_patch: ${{ steps.get_version.outputs.silkit_patch }} - silkit_suffix: ${{ steps.get_version.outputs.silkit_suffix }} + matrix: ${{ steps.matrix.outputs.matrix }} + major: ${{ steps.version.outputs.major }} + minor: ${{ steps.version.outputs.minor }} + patch: ${{ steps.version.outputs.patch }} + suffix: ${{ steps.version.outputs.suffix }} + version: ${{ steps.version.outputs.version }} + silkit_sha: ${{ steps.silkit.outputs.sha }} + silkit_submodules: ${{ steps.silkit.outputs.submodules }} + source_date_epoch: ${{ steps.silkit.outputs.source_date_epoch }} steps: - - name: Checkout (sil-kit-pkg) - uses: actions/checkout@v6 - with: - path: sil-kit-pkg - - - name: Get Version - id: get_version - shell: python + - name: Check sil-kit-pkg commit + if: inputs.pkg_sha != '' run: | - import os - import re - - with open('sil-kit-pkg/debian/changelog', 'r') as cl: - line = cl.readline() - reg = r'libsilkit\s+\(([0-9]+)\.([0-9]+)\.([0-9]+)\~*([A-Za-z]+[0-9]+)?\-[0-9]ubuntu[0-9]\).*' - reg_match = re.match(reg, line) - - with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: - fh.write(f"silkit_major={reg_match.group(1)}\n") - fh.write(f"silkit_minor={reg_match.group(2)}\n") - fh.write(f"silkit_patch={reg_match.group(3)}\n") - - suffix = "" if reg_match.group(4) is None else reg_match.group(4) - fh.write(f"silkit_suffix={suffix}\n") - - - package_ubuntu: - runs-on: ubuntu-latest - name: Job to build .deb for Ubuntu 20.04+ packages - needs: get_version - container: - image: ghcr.io/vectorgrp/sil-kit-docker-build/sil-kit-ci-packaging-ubuntu-20.04:main - env: - ARTIFACT_NAME: silkit-ubuntu-20.04-x86_64-deb - - steps: + if [ "${{ inputs.pkg_sha }}" != "${{ github.sha }}" ]; then + echo "::error::dispatched ref is at ${{ github.sha }}, expected ${{ inputs.pkg_sha }}" + exit 1 + fi - name: Checkout (sil-kit-pkg) uses: actions/checkout@v6 @@ -89,94 +90,42 @@ jobs: - name: Checkout (sil-kit) uses: actions/checkout@v6 with: - repository: ${{ inputs.silkit_source_repo }} - submodules: recursive + repository: vectorgrp/sil-kit + ref: ${{ inputs.silkit_ref }} path: sil-kit - ref: ${{ inputs.silkit_source_ref }} - github-server-url: https://github.com - - name: Install cmake + - name: sil-kit commit + id: silkit + working-directory: sil-kit run: | - sh sil-kit-pkg/.github/actions/fetch_cmake.sh - - - name: Build - id: build - env: - DEBFULLNAME: ${{ inputs.debian_fullname }} - DEBEMAIL: ${{ inputs.debian_fullname }} - CI_RUN: "1" - BUILD_SETTINGS: | - { - - "SilKitInfo": { - "url": "sil-kit", - "ref": "", - "recursive": true, - "is_local": true - }, - "package_repo_path": "sil-kit-pkg", - "version": { - "major": ${{ needs.get_version.outputs.silkit_major }}, - "minor": ${{ needs.get_version.outputs.silkit_minor }}, - "patch": ${{ needs.get_version.outputs.silkit_patch }}, - "suffix": "${{ needs.get_version.outputs.silkit_suffix }}" - }, - "pkgformat": "deb", - "work_dir": "${{ env.PKG_WORKDIR }}", - "keep_temp": true, - "output_dir": "${{ env.PKG_OUTDIR }}", - "platform": "Ubuntu-20.04", - "debuild": { - "args": ["-d", "--prepend-path=/opt/vector/bin"] - } - } - + echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + echo "source_date_epoch=$(git log -1 --format=%ct)" >> "$GITHUB_OUTPUT" + # top-level submodule commits as recorded in the sil-kit tree + echo "submodules=$(git ls-tree -r HEAD | awk '$2 == "commit" { print $4 "\t" $3 }' \ + | jq -R -s -c 'split("\n") | map(select(length > 0) | split("\t") | {(.[0]): .[1]}) | add // {}')" >> "$GITHUB_OUTPUT" + + - name: Version + id: version run: | - mkdir -p $PKG_WORKDIR - mkdir -p $PKG_OUTDIR - touch $PKG_WORKDIR/build_cfg.json - printf %s $BUILD_SETTINGS > $PKG_WORKDIR/build_cfg.json - python3 sil-kit-pkg/scripts/silkit_linux_packaging.py \ - --build-cfg $PKG_WORKDIR/build_cfg.json \ - --verbose - - - name: Test Package - id: test - env: - PKG_DIR: ./${{ env.PKG_OUTDIR }} - TEST_DIR: ./sil-kit-pkg/tests/ + python3 sil-kit-pkg/scripts/check_version.py \ + --pkg-dir sil-kit-pkg \ + --silkit-dir sil-kit \ + --expected-version "${{ inputs.expected_version }}" >> "$GITHUB_OUTPUT" + cat "$GITHUB_OUTPUT" + + - name: Matrix + id: matrix run: | - python3 sil-kit-pkg/tests/test_deb.py \ - --package-directory "$PKG_DIR" \ - --test-directory "$TEST_DIR" \ - --distro ubuntu \ - --distro-version 20 - - - name: Artifact - id: artifact - uses: actions/upload-artifact@v7 - with: - name: ${{ env.ARTIFACT_NAME }} - path: | - ./${{ env.PKG_OUTDIR }}/*.dsc - ./${{ env.PKG_OUTDIR }}/*.build* - ./${{ env.PKG_OUTDIR }}/*.changes - ./${{ env.PKG_OUTDIR }}/*.*deb - retention-days: 1 - - - package_almalinux: - runs-on: ubuntu-latest - name: Job to build Alma Linux 9 packages - needs: get_version - container: - image: ghcr.io/vectorgrp/sil-kit-docker-build/sil-kit-ci-packaging-fedora-40:main - options: --privileged - env: - ARTIFACT_NAME: silkit-almalinux-9-x86_64-rpm - + echo "matrix=$(jq -c '{include: [.targets[] | .debuild_args = (.debuild_args // [])]}' sil-kit-pkg/release/packages.json)" >> "$GITHUB_OUTPUT" + + package: + name: Package ${{ matrix.id }} + needs: targets + runs-on: ${{ matrix.runs_on }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.targets.outputs.matrix) }} steps: - - name: Checkout (sil-kit-pkg) uses: actions/checkout@v6 with: @@ -185,236 +134,143 @@ jobs: - name: Checkout (sil-kit) uses: actions/checkout@v6 with: - repository: ${{ inputs.silkit_source_repo }} + repository: vectorgrp/sil-kit + ref: ${{ needs.targets.outputs.silkit_sha }} submodules: recursive path: sil-kit - ref: ${{ inputs.silkit_source_ref }} - github-server-url: https://github.com - - - name: Build - id: build - env: - CI_RUN: "1" - BUILD_SETTINGS: | - { - "SilKitInfo": { - "url": "sil-kit", - "ref": "", - "recursive": true, - "is_local": true - }, - "package_repo_path": "sil-kit-pkg", - "version": { - "major": ${{ needs.get_version.outputs.silkit_major }}, - "minor": ${{ needs.get_version.outputs.silkit_minor }}, - "patch": ${{ needs.get_version.outputs.silkit_patch }}, - "suffix": "${{ needs.get_version.outputs.silkit_suffix }}" - }, - "pkgformat": "rpm", - "work_dir": "${{ env.PKG_WORKDIR }}", - "keep_temp": true, - "output_dir": "${{ env.PKG_OUTDIR }}", - "platform": "epel9" - } + - name: Log in to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Pull image + id: image run: | - mkdir -p $PKG_WORKDIR - mkdir -p $PKG_OUTDIR - touch $PKG_WORKDIR/build_cfg.json - printf %s $BUILD_SETTINGS > $PKG_WORKDIR/build_cfg.json - usermod -a -G mock root - python3 sil-kit-pkg/scripts/silkit_linux_packaging.py \ - --build-cfg $PKG_WORKDIR/build_cfg.json \ - --verbose + docker pull "${{ matrix.image }}" + echo "digest=$(docker image inspect --format '{{index .RepoDigests 0}}' "${{ matrix.image }}")" >> "$GITHUB_OUTPUT" - - name: Test Package - id: test + - name: Build env: - PKG_DIR: ./${{ env.PKG_OUTDIR }} - TEST_DIR: ./sil-kit-pkg/tests + TARGET: ${{ toJson(matrix) }} + SOURCE_DATE_EPOCH: ${{ needs.targets.outputs.source_date_epoch }} + DEBFULLNAME: ${{ inputs.debian_fullname }} + DEBEMAIL: ${{ inputs.debian_email }} run: | - dnf install -y cmake clang ninja-build - python3 sil-kit-pkg/tests/test_deb.py \ - --package-directory "$PKG_DIR" \ - --test-directory "$TEST_DIR" \ - --distro fedora \ - --distro-version 40 + maintainer="$(sed -n 's/^Maintainer: //p' sil-kit-pkg/debian/control)" + export DEBFULLNAME="${DEBFULLNAME:-${maintainer% <*}}" + export DEBEMAIL="${DEBEMAIL:-$(echo "$maintainer" | sed 's/.*<\(.*\)>.*/\1/')}" + + mkdir -p "$PKG_WORKDIR" "$PKG_OUTDIR" + jq -n \ + --argjson target "$TARGET" \ + --arg major "${{ needs.targets.outputs.major }}" \ + --arg minor "${{ needs.targets.outputs.minor }}" \ + --arg patch "${{ needs.targets.outputs.patch }}" \ + --arg suffix "${{ needs.targets.outputs.suffix }}" \ + --arg work "$PKG_WORKDIR" --arg out "$PKG_OUTDIR" '{ + SilKitInfo: { url: "sil-kit", ref: "", recursive: true, is_local: true }, + package_repo_path: "sil-kit-pkg", + version: { major: $major, minor: $minor, patch: $patch, suffix: $suffix }, + pkgformat: $target.pkgformat, + platform: $target.platform, + work_dir: $work, + keep_temp: true, + output_dir: $out, + debuild: { args: $target.debuild_args } + }' > "$PKG_WORKDIR/build_cfg.json" + cat "$PKG_WORKDIR/build_cfg.json" + + # docker run instead of a job container: the GitHub actions (node 20) + # do not run inside the CentOS 7 image + docker run --rm ${{ matrix.container_options }} \ + -v "$PWD:/w" -w /w \ + -e SOURCE_DATE_EPOCH -e DEBFULLNAME -e DEBEMAIL -e CI_RUN=1 \ + "${{ matrix.image }}" \ + bash -c ' + set -e + if [ "${{ matrix.pkgformat }}" = rpm ]; then usermod -a -G mock root; fi + python3 sil-kit-pkg/scripts/silkit_linux_packaging.py --build-cfg "$0/build_cfg.json" --verbose + ' "$PKG_WORKDIR" + + - name: Select assets + env: + TARGET: ${{ toJson(matrix) }} + run: | + python3 sil-kit-pkg/scripts/collect_packages.py select \ + --target "$TARGET" \ + --source "$PKG_OUTDIR" \ + --dest "dist/${{ matrix.id }}" \ + --image-digest "${{ steps.image.outputs.digest }}" - - name: Artifact - id: artifact + - name: Upload uses: actions/upload-artifact@v7 with: - name: ${{ env.ARTIFACT_NAME }} - path: | - ./${{ env.PKG_OUTDIR }}/*.rpm + name: pkg-${{ matrix.id }}${{ env.ARTIFACT_SUFFIX }} + path: dist/ + if-no-files-found: error retention-days: 1 - package_ubuntu_arm: - runs-on: ubuntu-22.04-arm - name: Job to build .deb for Ubuntu 22.04+ ARM64 packages - needs: get_version - env: - ARTIFACT_NAME: silkit-ubuntu-22.04-aarch64-deb - + test: + name: Test ${{ matrix.id }} + needs: [targets, package] + runs-on: ${{ matrix.runs_on }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.targets.outputs.matrix) }} steps: - - name: Checkout (sil-kit-pkg) uses: actions/checkout@v6 with: path: sil-kit-pkg - - name: Checkout (sil-kit) - uses: actions/checkout@v6 + - name: Download + uses: actions/download-artifact@v4 with: - repository: ${{ inputs.silkit_source_repo }} - submodules: recursive - path: sil-kit - ref: ${{ inputs.silkit_source_ref }} - github-server-url: https://github.com - - - name: Install debhelper - run: | - sudo apt update -y &&\ - sudo apt-get install -y \ - debhelper \ - dpkg-dev \ - devscripts \ - dh-make \ - zip \ - cmake \ - ninja-build - - - name: Build - id: build - env: - DEBFULLNAME: ${{ inputs.debian_fullname }} - DEBEMAIL: ${{ inputs.debian_fullname }} - CI_RUN: "1" - BUILD_SETTINGS: | - { - - "SilKitInfo": { - "url": "sil-kit", - "ref": "", - "recursive": true, - "is_local": true - }, - "package_repo_path": "sil-kit-pkg", - "version": { - "major": ${{ needs.get_version.outputs.silkit_major }}, - "minor": ${{ needs.get_version.outputs.silkit_minor }}, - "patch": ${{ needs.get_version.outputs.silkit_patch }}, - "suffix": "${{ needs.get_version.outputs.silkit_suffix }}" - }, - "pkgformat": "deb", - "work_dir": "${{ env.PKG_WORKDIR }}", - "keep_temp": true, - "output_dir": "${{ env.PKG_OUTDIR }}", - "platform": "Ubuntu-22.04", - "debuild": { - "args": ["-d", "--prepend-path=/opt/vector/bin"] - } - } - - run: | - mkdir -p $PKG_WORKDIR - mkdir -p $PKG_OUTDIR - touch $PKG_WORKDIR/build_cfg.json - printf %s $BUILD_SETTINGS > $PKG_WORKDIR/build_cfg.json - python3 sil-kit-pkg/scripts/silkit_linux_packaging.py \ - --build-cfg $PKG_WORKDIR/build_cfg.json \ - --verbose + name: pkg-${{ matrix.id }}${{ env.ARTIFACT_SUFFIX }} + path: dist - - name: Test Package - id: test - env: - PKG_DIR: ./${{ env.PKG_OUTDIR }} - TEST_DIR: ./sil-kit-pkg/tests/ + - name: Test in ${{ matrix.test_image }} run: | - python3 sil-kit-pkg/tests/test_deb.py \ - --package-directory "$PKG_DIR" \ - --test-directory "$TEST_DIR" \ - --distro ubuntu \ - --distro-version 22 - - - name: Artifact - id: artifact - uses: actions/upload-artifact@v7 - with: - name: ${{ env.ARTIFACT_NAME }} - path: | - ./${{ env.PKG_OUTDIR }}/*.dsc - ./${{ env.PKG_OUTDIR }}/*.build* - ./${{ env.PKG_OUTDIR }}/*.changes - ./${{ env.PKG_OUTDIR }}/*.*deb - retention-days: 1 + docker run --rm -v "$PWD:/w" -w /w "${{ matrix.test_image }}" \ + sh sil-kit-pkg/tests/run_package_test.sh "${{ matrix.pkgformat }}" "/w/dist/${{ matrix.id }}" - - - make-release: + collect: + name: Collect + needs: [targets, test] runs-on: ubuntu-latest - needs: [package_ubuntu, package_almalinux, package_ubuntu_arm] - if: inputs.silkit_pkg_ref != '' - permissions: - contents: write - steps: + - name: Checkout (sil-kit-pkg) + uses: actions/checkout@v6 + with: + path: sil-kit-pkg - - name: download_artifact + - name: Download uses: actions/download-artifact@v4 - - - name: Prepare artifact - run: | - sha256sum silkit-almalinux-*-rpm/* >> ./SHA256 - sha256sum silkit-ubuntu-*-deb/* >> ./SHA256 - - - name: Prepare GIT tag - id: git_tag - run: | - echo ${{ inputs.silkit_pkg_ref }} - echo ${{ inputs.silkit_pkg_ref }} | sed --expression="s/~rc/-rc/g" - echo "git_tag=$(echo ${{ inputs.silkit_pkg_ref }} | sed --expression="s/~rc/-rc/g")" >> $GITHUB_OUTPUT - case "${{ inputs.silkit_pkg_ref }}" in - *rc* ) - echo 'prerelease=--prerelease' >> $GITHUB_OUTPUT - ;; - *) - echo 'prerelease=' >> $GITHUB_OUTPUT - ;; - esac - - - name: Create git tag - uses: actions/github-script@v7 with: - script: | - try { - await github.rest.git.createRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: 'refs/tags/${{ steps.git_tag.outputs.git_tag }}', - sha: context.sha, - }) - } catch(error) { - core.warning("Could not create the specified ref, continuing without it!") - } - + pattern: pkg-*${{ env.ARTIFACT_SUFFIX }} + path: targets + merge-multiple: true - - name: Release artifact - env: - GH_TOKEN: ${{ github.token }} + - name: Manifest run: | - echo ${{ steps.git_tag.outputs.git_tag }} - gh release create \ - ${{ steps.git_tag.outputs.git_tag }} \ - ${{ github.workspace }}/silkit-ubuntu-*-deb/*.*deb \ - ${{ github.workspace }}/silkit-ubuntu-*-deb/*.build* \ - ${{ github.workspace }}/silkit-ubuntu-*-deb/*.changes \ - ${{ github.workspace }}/silkit-ubuntu-*-x86_64-deb/*.dsc \ - ${{ github.workspace }}/silkit-almalinux-*-rpm/* \ - SHA256 \ - --verify-tag \ - ${{ steps.git_tag.outputs.prerelease }} \ - --repo "${{ github.repository }}" \ - --title "${{ inputs.silkit_pkg_ref }}" \ - --notes "Ubuntu and Fedora Packages for SIL Kit ${{ steps.git_tag.outputs.git_tag }}" + python3 sil-kit-pkg/scripts/collect_packages.py manifest \ + --source targets \ + --dest sil-kit-pkg-out \ + --version "${{ needs.targets.outputs.version }}" \ + --silkit-ref "${{ inputs.silkit_ref }}" \ + --silkit-sha "${{ needs.targets.outputs.silkit_sha }}" \ + --silkit-submodules '${{ needs.targets.outputs.silkit_submodules }}' \ + --pkg-sha "${{ github.sha }}" \ + --source-date-epoch "${{ needs.targets.outputs.source_date_epoch }}" \ + --run-url "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + - name: Upload + uses: actions/upload-artifact@v7 + with: + name: sil-kit-pkg${{ env.ARTIFACT_SUFFIX }} + path: sil-kit-pkg-out/ + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/rebuild-check.yml b/.github/workflows/rebuild-check.yml new file mode 100644 index 0000000..c427e10 --- /dev/null +++ b/.github/workflows/rebuild-check.yml @@ -0,0 +1,85 @@ +name: "Rebuild Check" + +# Builds all packages twice, one build after the other, and compares the sha256 of every +# file. A mismatch means the build is not reproducible; diffoscope shows where. + +on: + workflow_dispatch: + inputs: + silkit_ref: + description: "sil-kit commit SHA or tag (vectorgrp/sil-kit)" + required: true + type: string + default: main + schedule: + - cron: "0 3 * * 1" + +permissions: + contents: read + packages: read + +jobs: + first: + uses: ./.github/workflows/package-silkit.yml + with: + silkit_ref: ${{ inputs.silkit_ref || 'main' }} + artifact_suffix: -first + + second: + needs: first + uses: ./.github/workflows/package-silkit.yml + with: + silkit_ref: ${{ inputs.silkit_ref || 'main' }} + artifact_suffix: -second + + compare: + needs: [first, second] + runs-on: ubuntu-latest + steps: + - name: Download first build + uses: actions/download-artifact@v4 + with: + name: sil-kit-pkg-first + path: first + + - name: Download second build + uses: actions/download-artifact@v4 + with: + name: sil-kit-pkg-second + path: second + + - name: Compare + id: compare + run: | + jq -r '.files[] | "\(.name) \(.sha256)"' first/pkg-manifest.json | sort > first.sha + jq -r '.files[] | "\(.name) \(.sha256)"' second/pkg-manifest.json | sort > second.sha + if diff -u first.sha second.sha; then + echo "All $(wc -l < first.sha) files are identical." >> "$GITHUB_STEP_SUMMARY" + else + join first.sha second.sha | awk '$2 != $3 { print $1 }' > differing.txt + echo "Not reproducible:" >> "$GITHUB_STEP_SUMMARY" + sed 's/^/- /' differing.txt >> "$GITHUB_STEP_SUMMARY" + echo "failed=true" >> "$GITHUB_OUTPUT" + fi + + - name: diffoscope + if: steps.compare.outputs.failed == 'true' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends diffoscope + mkdir -p diffoscope + while read -r name; do + diffoscope --html "diffoscope/${name}.html" "first/${name}" "second/${name}" || true + done < differing.txt + + - name: Upload diffoscope reports + if: steps.compare.outputs.failed == 'true' + uses: actions/upload-artifact@v7 + with: + name: diffoscope + path: diffoscope/ + retention-days: 30 + + - name: Fail + if: steps.compare.outputs.failed == 'true' + run: exit 1 diff --git a/README.rst b/README.rst index 4bd7d41..d014ed8 100644 --- a/README.rst +++ b/README.rst @@ -48,6 +48,63 @@ Currently we support building the packages on the following platforms: * Fedora * 40+ +In addition, relocatable binary tarballs (``pkgformat: tgz``) are built for: + +* Ubuntu 20.04 (``ubuntu2004``) +* CentOS 7 (``el7``) +* AlmaLinux 8 (``el8``) + +``el7``/``el8``/``el9`` stand for Enterprise Linux 7/8/9, i.e. RHEL and its binary compatible +rebuilds (CentOS 7 = el7, AlmaLinux 8 = el8, AlmaLinux 9 = el9). + +Release definition +================== + +``release/packages.json`` is the single source of truth for the Linux packages. Every entry +(target) names the build image, the package format and platform, the plain distribution image +used for the smoke test, whether it is released (``release``) and the files that make up the +package (``assets``, glob patterns). + +The packaging workflow builds every target. sil-kit-delivery publishes the assets of all targets +with ``"release": true`` to the GitHub releases of ``vectorgrp/sil-kit`` and ``vectorgrp/sil-kit-pkg``. + +Containers +========== + +The build images live in ``containers/`` and are published to +``ghcr.io/vectorgrp/sil-kit-pkg/`` by the ``Build Images`` workflow +(``.github/workflows/build-images.yml``) on every change on ``main``: + +* ``ubuntu-2004``: deb and tarball for Ubuntu 20.04 +* ``ubuntu-2204``: arm64 deb (built on an arm64 runner) +* ``centos-7``: el7 tarball, devtoolset-11 +* ``alma-8``: el8 tarball, gcc-toolset-12 +* ``fedora-40``: el9 rpm via mock + +Only public sources are used: base images are pinned by digest, CMake comes from the official +Kitware release and ninja from the official PyPI wheel, both verified by sha256. Once the images +are published, pin them by digest in ``release/packages.json`` +(``ghcr.io/vectorgrp/sil-kit-pkg/@sha256:...``, the digest is in the job summary of the +``Build Images`` run). + +Tarballs +======== + +``libsilkit-$VERSION..x86_64.tar.gz`` contains ``bin/``, ``lib/`` (with the CMake +package config in ``lib/cmake/SilKit``), ``include/``, ``share/man/man1/``, +``share/sil-kit/sil-kit-registry.service`` and ``share/doc/libsilkit/`` (``CHANGELOG.md``, +``LICENSE``, ``README.rst``). It can be extracted anywhere:: + + tar -xzf libsilkit-$VERSION.el7.x86_64.tar.gz + cmake -DCMAKE_PREFIX_PATH=$PWD/libsilkit-$VERSION.el7.x86_64 ... + +The debug symbols are in ``libsilkit-$VERSION..x86_64.debug.tar.gz`` under +``lib/debug/.build-id/``. Point gdb at them with ``set debug-file-directory /lib/debug``. + +The tarballs are reproducible: the same sil-kit commit, sil-kit-pkg commit and image give +identical files (``SOURCE_DATE_EPOCH`` is the commit time of the sil-kit sources). The +``Rebuild Check`` workflow verifies this by building everything twice. + Setting up the build environment ================================ @@ -115,7 +172,7 @@ Let's examine an example config to build a SIL Kit package: :: * Additional version suffix of the package * pkgformat - * Determines the package type to build. ``deb`` for Debian/Ubuntu builds. ``rpm`` for RedHat/Fedora builds. + * Determines the package type to build. ``deb`` for Debian/Ubuntu builds. ``rpm`` for RedHat/Fedora builds. ``tgz`` for the relocatable tarballs. * work_dir * The directory where the build is happening * keep_temp @@ -131,6 +188,7 @@ Let's examine an example config to build a SIL Kit package: :: * epel8 * epel9 * fc40 + * ubuntu2004, el7, el8 (``tgz``) Fedora Extra Packages for Enterprise Linux ------------------------------------------- @@ -162,15 +220,6 @@ Debian requires the Maintainer environment variables to be set correctly: export DEBFULLNAME="Awesome Dev" export DEBEMAIL="awesome_dev@your-domain.something" -Additional Parameters for the CI -================================ - -In addition to the environment variables (used by build_deb.sh), the Github Action uses some more inputs to identify/store the correct artifacts - -* debian_arch - - The cpu architecture the debian package is build for. Currently we only support/test on **amd64** - Building the package ==================== @@ -200,30 +249,19 @@ The rest of the config can use the template values. Then all you need to run is: Github CI --------- -If you have forked sil-kit-pkg and you can use Github Actions, this is how you can build a libsilkit4 package - -How to get to the Action -************************ - -* Click on the ``Actions`` tab in your Github repo -* Click on the ``Silkit Packaging Workflow`` tab -* Click on the ``Run Workflow`` tab - -Setup for the Workflow -********************** +The ``SilKit Packaging Workflow`` (``.github/workflows/package-silkit.yml``) builds all targets of +``release/packages.json``, tests every package in the plain ``test_image`` of its target and +uploads the result as the artifact ``sil-kit-pkg`` (kept for 30 days): all assets, +``pkg-manifest.json`` (sil-kit commit and submodules, sil-kit-pkg commit, image digests, +sha256 and size of every file) and ``SHA256``. -* SIL Kit Repository URL - Set to the sil-kit github repo -* SIL Kit Source Repo ref - Set to the branch/tag/commit id of the sil-kit repo you want to use -* sil-kit-pkg ref - If you want to build a release you can set the this to a release tag -* Maintainer name (only used on Debian/Ubuntu) - Name of the maintainer creating the package -* Maintainer email (only used on Debian/Ubuntu) - Email of the maintainer creating the package +sil-kit-delivery starts it for every delivery build. To run it manually: -* DEBIAN_ARCH - amd64 +* Click on the ``Actions`` tab, then ``SilKit Packaging Workflow``, then ``Run workflow`` +* Choose the sil-kit-pkg branch or tag to build from +* ``silkit_ref``: the sil-kit commit SHA or tag, e.g. ``v5.0.8`` +* optional: ``expected_version`` fails the run if ``debian/changelog``, ``rpm/libsilkit5.spec`` + and the sil-kit sources do not all have this version +* optional: maintainer name and email for the Debian build (default: ``Maintainer`` in ``debian/control``) -Click the ``Run Workflow`` Button. Packages for Ubuntu 20.04 and Alma Linux 9 will be in the artifacts of this Workflow run. +GitHub releases are created by sil-kit-delivery from this artifact, not by this repository. diff --git a/release/packages.json b/release/packages.json new file mode 100644 index 0000000..70bd90a --- /dev/null +++ b/release/packages.json @@ -0,0 +1,110 @@ +{ + "comment": [ + "Single source of truth for the Linux packages of SIL Kit.", + "package-silkit.yml builds every target; sil-kit-delivery publishes the assets of targets with release: true", + "to the vectorgrp/sil-kit and vectorgrp/sil-kit-pkg releases.", + "Images are pinned by digest once build-images.yml has published them (see README.rst)." + ], + "targets": [ + { + "id": "ubuntu-20.04-x86_64-deb", + "runs_on": "ubuntu-latest", + "image": "ghcr.io/vectorgrp/sil-kit-pkg/ubuntu-2004:main", + "container_options": "", + "pkgformat": "deb", + "platform": "Ubuntu-20.04", + "debuild_args": [ + "-d", + "--prepend-path=/opt/cmake/bin" + ], + "test_image": "ubuntu:20.04", + "release": true, + "assets": [ + "libsilkit5_*_amd64.deb", + "libsilkit-dev_*_amd64.deb", + "silkit-utils_*_amd64.deb", + "libsilkit5-dbgsym_*_amd64.ddeb", + "libsilkit_*_amd64.buildinfo" + ] + }, + { + "id": "ubuntu-20.04-x86_64-tgz", + "runs_on": "ubuntu-latest", + "image": "ghcr.io/vectorgrp/sil-kit-pkg/ubuntu-2004:main", + "container_options": "", + "pkgformat": "tgz", + "platform": "ubuntu2004", + "test_image": "ubuntu:20.04", + "release": true, + "assets": [ + "libsilkit-*.ubuntu2004.x86_64.tar.gz", + "libsilkit-*.ubuntu2004.x86_64.debug.tar.gz" + ] + }, + { + "id": "centos-7-x86_64-tgz", + "runs_on": "ubuntu-latest", + "image": "ghcr.io/vectorgrp/sil-kit-pkg/centos-7:main", + "container_options": "", + "pkgformat": "tgz", + "platform": "el7", + "test_image": "centos:7", + "release": true, + "assets": [ + "libsilkit-*.el7.x86_64.tar.gz", + "libsilkit-*.el7.x86_64.debug.tar.gz" + ] + }, + { + "id": "alma-8-x86_64-tgz", + "runs_on": "ubuntu-latest", + "image": "ghcr.io/vectorgrp/sil-kit-pkg/alma-8:main", + "container_options": "", + "pkgformat": "tgz", + "platform": "el8", + "test_image": "almalinux:8", + "release": true, + "assets": [ + "libsilkit-*.el8.x86_64.tar.gz", + "libsilkit-*.el8.x86_64.debug.tar.gz" + ] + }, + { + "id": "ubuntu-22.04-aarch64-deb", + "runs_on": "ubuntu-22.04-arm", + "image": "ghcr.io/vectorgrp/sil-kit-pkg/ubuntu-2204:main", + "container_options": "", + "pkgformat": "deb", + "platform": "Ubuntu-22.04", + "debuild_args": [ + "-d", + "--prepend-path=/opt/cmake/bin" + ], + "test_image": "ubuntu:22.04", + "release": false, + "assets": [ + "libsilkit5_*_arm64.deb", + "libsilkit-dev_*_arm64.deb", + "silkit-utils_*_arm64.deb", + "libsilkit5-dbgsym_*_arm64.ddeb", + "libsilkit_*_arm64.buildinfo" + ] + }, + { + "id": "alma-9-x86_64-rpm", + "runs_on": "ubuntu-latest", + "image": "ghcr.io/vectorgrp/sil-kit-pkg/fedora-40:main", + "container_options": "--privileged", + "pkgformat": "rpm", + "platform": "epel9", + "test_image": "almalinux:9", + "release": false, + "assets": [ + "libsilkit5-*.el9.x86_64.rpm", + "libsilkit5-devel-*.el9.x86_64.rpm", + "libsilkit5-utils-*.el9.x86_64.rpm", + "libsilkit5-debuginfo-*.el9.x86_64.rpm" + ] + } + ] +} diff --git a/scripts/check_version.py b/scripts/check_version.py new file mode 100644 index 0000000..2647f4f --- /dev/null +++ b/scripts/check_version.py @@ -0,0 +1,93 @@ +#! /bin/env python3 +# +# SPDX-FileCopyrightText: 2026 Vector Informatik GmbH +# +# SPDX-License-Identifier: MIT +# +"""Check that debian/changelog, rpm/libsilkit5.spec and the sil-kit sources agree on the +version, and optionally that it matches the version delivery expects. + +Prints major, minor, patch, suffix and version as key=value lines (GITHUB_OUTPUT format). +""" +import argparse +import re +import sys + +from pathlib import Path + + +def read(path: Path) -> str: + with open(path, "r", encoding="utf-8") as f: + return f.read() + + +def deb_version(pkg_dir: Path): + first_line = read(pkg_dir / "debian" / "changelog").splitlines()[0] + match = re.match(r"libsilkit\s+\(([0-9]+)\.([0-9]+)\.([0-9]+)~*([A-Za-z]+[0-9]+)?-[0-9]+ubuntu[0-9]+\)", first_line) + if not match: + sys.exit(f"cannot parse the version from debian/changelog: {first_line}") + return match.group(1), match.group(2), match.group(3), match.group(4) or "" + + +def rpm_version(pkg_dir: Path): + spec = read(pkg_dir / "rpm" / "libsilkit5.spec") + parts = [] + for component in ("major", "minor", "patch"): + match = re.search(rf"^%define version_{component} ([0-9]+)\s*$", spec, re.MULTILINE) + if not match: + sys.exit(f"no version_{component} in rpm/libsilkit5.spec") + parts.append(match.group(1)) + return tuple(parts) + + +def silkit_version(silkit_dir: Path): + header = silkit_dir / "SilKit" / "include" / "silkit" / "capi" / "SilKitVersionMacros.h" + if header.exists(): + text = read(header) + pattern = r"^#define SILKIT_VERSION_{} ([0-9]+)\s*$" + else: + # sil-kit refs from before the version moved into SilKitVersionMacros.h + text = read(silkit_dir / "SilKit" / "cmake" / "SilKitVersion.cmake") + pattern = r"^\s*set\s*\(\s*SILKIT_VERSION_{}\s*([0-9]+)\s*\)\s*$" + parts = [] + for component in ("MAJOR", "MINOR", "PATCH"): + match = re.search(pattern.format(component), text, re.MULTILINE) + if not match: + sys.exit(f"no SILKIT_VERSION_{component} in the sil-kit sources") + parts.append(match.group(1)) + return tuple(parts) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--pkg-dir", type=Path, required=True) + ap.add_argument("--silkit-dir", type=Path, required=True) + ap.add_argument("--expected-version", default="", help="major.minor.patch, empty to skip") + args = ap.parse_args() + + major, minor, patch, suffix = deb_version(args.pkg_dir) + deb = (major, minor, patch) + checks = { + "rpm/libsilkit5.spec": rpm_version(args.pkg_dir), + "sil-kit sources": silkit_version(args.silkit_dir), + } + if args.expected_version: + checks["expected_version"] = tuple(args.expected_version.split("-")[0].split(".")) + + errors = [ + f"{name} has {'.'.join(version)}, debian/changelog has {'.'.join(deb)}" + for name, version in checks.items() + if version != deb + ] + if errors: + sys.exit("version mismatch (is the changelog PR merged?):\n " + "\n ".join(errors)) + + print(f"major={major}") + print(f"minor={minor}") + print(f"patch={patch}") + print(f"suffix={suffix}") + print(f"version={major}.{minor}.{patch}") + + +if __name__ == "__main__": + main() diff --git a/scripts/collect_packages.py b/scripts/collect_packages.py new file mode 100644 index 0000000..e695403 --- /dev/null +++ b/scripts/collect_packages.py @@ -0,0 +1,129 @@ +#! /bin/env python3 +# +# SPDX-FileCopyrightText: 2026 Vector Informatik GmbH +# +# SPDX-License-Identifier: MIT +# +"""Helpers for package-silkit.yml. + + select copy the files of one target that match its asset globs into a directory + and write .target.json next to them + manifest merge the per-target directories into one and write pkg-manifest.json and SHA256 +""" +import argparse +import hashlib +import json +import shutil +import sys + +from pathlib import Path + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def select(args): + target = json.loads(args.target) + args.dest.mkdir(parents=True, exist_ok=True) + + files = [] + for pattern in target["assets"]: + matches = sorted(p for p in args.source.glob(pattern) if p.is_file()) + if not matches: + sys.exit(f"{target['id']}: no file matches {pattern} in {args.source}") + for match in matches: + if match.name not in files: + shutil.copy2(match, args.dest / match.name) + files.append(match.name) + + info = { + "id": target["id"], + "release": target["release"], + "image": target["image"], + "image_digest": args.image_digest, + "files": files, + } + with open(args.dest / f"{target['id']}.target.json", "w", encoding="utf-8") as f: + json.dump(info, f, indent=2) + print(json.dumps(info, indent=2)) + + +def manifest(args): + args.dest.mkdir(parents=True, exist_ok=True) + + targets = [] + files = [] + for target_json in sorted(args.source.glob("*/*.target.json")): + with open(target_json, "r", encoding="utf-8") as f: + target = json.load(f) + for name in target["files"]: + path = target_json.parent / name + if (args.dest / name).exists(): + sys.exit(f"{name} is produced by more than one target") + shutil.copy2(path, args.dest / name) + files.append({ + "name": name, + "target": target["id"], + "release": target["release"], + "sha256": sha256(path), + "size": path.stat().st_size, + }) + targets.append({k: target[k] for k in ("id", "release", "image", "image_digest")}) + + if not files: + sys.exit("no packages found") + + pkg_manifest = { + "version": args.version, + "silkit_ref": args.silkit_ref, + "silkit_sha": args.silkit_sha, + "silkit_submodules": json.loads(args.silkit_submodules), + "pkg_sha": args.pkg_sha, + "source_date_epoch": int(args.source_date_epoch), + "run_url": args.run_url, + "targets": targets, + "files": files, + } + with open(args.dest / "pkg-manifest.json", "w", encoding="utf-8") as f: + json.dump(pkg_manifest, f, indent=2) + f.write("\n") + with open(args.dest / "SHA256", "w", encoding="utf-8") as f: + for entry in files: + f.write(f"{entry['sha256']} {entry['name']}\n") + print(json.dumps(pkg_manifest, indent=2)) + + +def main(): + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="command", required=True) + + s = sub.add_parser("select") + s.add_argument("--target", required=True, help="target object from release/packages.json as JSON") + s.add_argument("--source", type=Path, required=True) + s.add_argument("--dest", type=Path, required=True) + s.add_argument("--image-digest", default="") + s.set_defaults(func=select) + + m = sub.add_parser("manifest") + m.add_argument("--source", type=Path, required=True, help="directory with one subdirectory per target") + m.add_argument("--dest", type=Path, required=True) + m.add_argument("--version", required=True) + m.add_argument("--silkit-ref", required=True) + m.add_argument("--silkit-sha", required=True) + m.add_argument("--silkit-submodules", default="{}", help="JSON object path -> sha") + m.add_argument("--pkg-sha", required=True) + m.add_argument("--source-date-epoch", required=True) + m.add_argument("--run-url", required=True) + m.set_defaults(func=manifest) + + args = ap.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/tests/run_package_test.sh b/tests/run_package_test.sh new file mode 100755 index 0000000..10ac937 --- /dev/null +++ b/tests/run_package_test.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# SPDX-FileCopyrightText: 2026 Vector Informatik GmbH +# +# SPDX-License-Identifier: MIT +# +# Smoke test a package inside a plain distribution container (no build image). +# Installs a compiler, cmake and ninja, then runs test_deb.py. +# +# Usage (from the sil-kit-pkg checkout): tests/run_package_test.sh +set -eu + +format="$1" +package_dir="$2" +here="$(cd "$(dirname "$0")" && pwd)" +utils="${here}/../containers/utils" + +. /etc/os-release +major="${VERSION_ID%%.*}" + +case "${ID}-${major}" in + ubuntu-*) + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y --no-install-recommends ca-certificates curl g++ ninja-build python3 + sh "${utils}/install-cmake.sh" + export CC=gcc CXX=g++ + ;; + centos-7) + sed -i -e 's/^mirrorlist=/#mirrorlist=/' \ + -e 's|^#\?baseurl=http://mirror.centos.org/centos/\$releasever|baseurl=https://vault.centos.org/7.9.2009|' \ + /etc/yum.repos.d/CentOS-*.repo + yum install -y centos-release-scl + sed -i -e 's/^mirrorlist=/#mirrorlist=/' \ + -e 's|^#\?\s*baseurl=http://mirror.centos.org/centos/7/sclo/|baseurl=https://vault.centos.org/7.9.2009/sclo/|' \ + /etc/yum.repos.d/CentOS-SCLo-*.repo + yum install -y curl devtoolset-11-gcc-c++ python3 unzip + sh "${utils}/install-cmake.sh" + sh "${utils}/install-ninja.sh" + export CC=/opt/rh/devtoolset-11/root/usr/bin/gcc CXX=/opt/rh/devtoolset-11/root/usr/bin/g++ + ;; + almalinux-8) + dnf install -y dnf-plugins-core + dnf config-manager --set-enabled powertools + dnf install -y gcc-c++ ninja-build python3 tar + sh "${utils}/install-cmake.sh" + export CC=gcc CXX=g++ + ;; + almalinux-9|fedora-*) + dnf install -y dnf-plugins-core + dnf config-manager --set-enabled crb || true + dnf install -y clang ninja-build python3 tar + sh "${utils}/install-cmake.sh" + export CC=clang CXX=clang++ + ;; + *) + echo "unsupported test distribution ${ID} ${VERSION_ID}" >&2 + exit 64 + ;; +esac + +export PATH="/opt/cmake/bin:${PATH}" + +case "${ID}" in + ubuntu) distro=ubuntu ;; + fedora|almalinux) [ "${format}" = rpm ] && distro=fedora || distro="${ID}" ;; + *) distro="${ID}" ;; +esac + +python3 "${here}/test_deb.py" \ + --package-format "${format}" \ + --package-directory "${package_dir}" \ + --test-directory "${here}" \ + --distro "${distro}" \ + --distro-version "${major}" diff --git a/tests/test_deb.py b/tests/test_deb.py index 6bf0e21..9adf4e8 100644 --- a/tests/test_deb.py +++ b/tests/test_deb.py @@ -10,6 +10,7 @@ import shlex import subprocess import signal +import tarfile from argparse import ArgumentParser from pathlib import Path @@ -25,15 +26,20 @@ def create_arg_parser() -> ArgumentParser: ap.add_argument("--test-directory", type=Path, required=True) ap.add_argument("--distro", type=str, required=True) ap.add_argument("--distro-version", type=int, required=True) + ap.add_argument("--package-format", type=str, choices=["deb", "rpm", "tgz"], required=False, + help="Defaults to deb on ubuntu and rpm on fedora") return ap +def as_root(args): + return args if os.geteuid() == 0 else ['sudo'] + args + def install_debs(deb_directory): print("\nInstall SilKit Ubuntu packages") print("--------------------------------\n", flush=True) debs = deb_directory.glob('*.deb') str_args = [str(deb.resolve()) for deb in debs] - args = ['sudo', 'apt', 'install', '-y'] + str_args + args = as_root(['apt', 'install', '-y'] + str_args) try: subprocess.run(args, check=True) @@ -54,12 +60,39 @@ def install_rpms(rpm_directory): print("Could not install the RPM files. Exiting with ERROR!") exit(64) -def run_registry(): +def extract_tarball(tgz_directory, test_dir) -> Path: + """Extract the binary tarball (not the .debug one) and return its root directory.""" + + print("\nExtract SilKit tarball") + print("--------------------------------\n", flush=True) + tarballs = [t for t in tgz_directory.glob('libsilkit-*.tar.gz') if not t.name.endswith('.debug.tar.gz')] + debug_tarballs = list(tgz_directory.glob('libsilkit-*.debug.tar.gz')) + if len(tarballs) != 1 or len(debug_tarballs) != 1: + print(f"Expected one tarball and one debug tarball in {tgz_directory}, found {tarballs} and {debug_tarballs}") + exit(64) + + extract_dir = (test_dir / '_pkg').resolve() + subprocess.run(['rm', '-rf', str(extract_dir)], check=True) + extract_dir.mkdir() + for tarball in tarballs + debug_tarballs: + with tarfile.open(tarball) as tar: + tar.extractall(extract_dir) + + root = extract_dir / tarballs[0].name[:-len('.tar.gz')] + debug_root = extract_dir / debug_tarballs[0].name[:-len('.tar.gz')] + if not (root / 'bin' / 'sil-kit-registry').exists(): + print(f"{root} does not contain bin/sil-kit-registry") + exit(64) + if not list((debug_root / 'lib' / 'debug' / '.build-id').glob('*/*.debug')): + print(f"{debug_root} does not contain any debug files") + exit(64) + return root + +def run_registry(registry_cmd='sil-kit-registry'): print("\nRun sil-kit-registry") print("--------------------------------\n", flush=True) - cmd = 'sil-kit-registry' - registry_proc = subprocess.Popen(cmd, shell=False) + registry_proc = subprocess.Popen(registry_cmd, shell=False) sleep(2) if registry_proc.poll() != None: @@ -82,10 +115,12 @@ def close_registry(reg_proc: subprocess.Popen): else: print("All is fine, exiting gracefully!") -def build_test(test_dir, distro: str, distro_version: int): +def build_test(test_dir, distro: str, distro_version: int, prefix_path: Optional[Path] = None): cc = "clang-10" if distro.lower() == "ubuntu" and distro_version < 22 else "clang" cxx = "clang++-10" if distro.lower() == "ubuntu" and distro_version < 22 else "clang++" + cc = os.environ.get("CC", cc) + cxx = os.environ.get("CXX", cxx) print("\nBuild the SilKit Test program") print(f"Using {cc} and {cxx}") @@ -100,7 +135,10 @@ def build_test(test_dir, distro: str, distro_version: int): current_env["CC"]=cc current_env["CXX"]=cxx build_dir = test_dir / '_build' - subprocess.run(['cmake', '-GNinja', '../'], env=current_env, cwd=build_dir, check=True) + cmake_args = ['cmake', '-GNinja', '../'] + if prefix_path is not None: + cmake_args.append(f'-DCMAKE_PREFIX_PATH={prefix_path}') + subprocess.run(cmake_args, env=current_env, cwd=build_dir, check=True) # Build the test print(f"Build dir {build_dir}") @@ -119,21 +157,31 @@ def main(): argument_parser = create_arg_parser() args =argument_parser.parse_args() - # Check deb install - if args.distro.lower() == "ubuntu": + package_format = args.package_format + if package_format is None: + package_format = {"ubuntu": "deb", "fedora": "rpm"}.get(args.distro.lower()) + + test_dir = args.test_directory + prefix_path = None + registry_cmd = 'sil-kit-registry' + + # Check package install + if package_format == "deb": install_debs(args.package_directory) - elif args.distro.lower() == "fedora": + elif package_format == "rpm": install_rpms(args.package_directory) + elif package_format == "tgz": + prefix_path = extract_tarball(args.package_directory, test_dir) + registry_cmd = str(prefix_path / 'bin' / 'sil-kit-registry') else: print(f"{args.distro} is currently not supported. Exiting!") exit(64) # Check demo program (compilation/link/execution) - test_dir = args.test_directory build_dir = test_dir / '_build' - build_test(test_dir, args.distro, args.distro_version) - registry = run_registry() + build_test(test_dir, args.distro, args.distro_version, prefix_path) + registry = run_registry(registry_cmd) ret_code = 0 try: run_test(build_dir) From 09447369485124c2bf9a4d1bebc405d8f43eda00 Mon Sep 17 00:00:00 2001 From: Konrad Breitsprecher Date: Tue, 10 Nov 2026 11:20:17 +0100 Subject: [PATCH 4/4] tgz: rename tarballs to silkit-.. The tarballs contain more than the library (registry, utilities, headers, CMake config), so the lib prefix is misleading. Co-Authored-By: Claude Opus 5.5 --- README.rst | 8 ++++---- release/packages.json | 12 ++++++------ scripts/silkit_tgz.py | 6 +++--- tests/test_deb.py | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/README.rst b/README.rst index d014ed8..744af4c 100644 --- a/README.rst +++ b/README.rst @@ -90,15 +90,15 @@ are published, pin them by digest in ``release/packages.json`` Tarballs ======== -``libsilkit-$VERSION..x86_64.tar.gz`` contains ``bin/``, ``lib/`` (with the CMake +``silkit-$VERSION..x86_64.tar.gz`` contains ``bin/``, ``lib/`` (with the CMake package config in ``lib/cmake/SilKit``), ``include/``, ``share/man/man1/``, ``share/sil-kit/sil-kit-registry.service`` and ``share/doc/libsilkit/`` (``CHANGELOG.md``, ``LICENSE``, ``README.rst``). It can be extracted anywhere:: - tar -xzf libsilkit-$VERSION.el7.x86_64.tar.gz - cmake -DCMAKE_PREFIX_PATH=$PWD/libsilkit-$VERSION.el7.x86_64 ... + tar -xzf silkit-$VERSION.el7.x86_64.tar.gz + cmake -DCMAKE_PREFIX_PATH=$PWD/silkit-$VERSION.el7.x86_64 ... -The debug symbols are in ``libsilkit-$VERSION..x86_64.debug.tar.gz`` under +The debug symbols are in ``silkit-$VERSION..x86_64.debug.tar.gz`` under ``lib/debug/.build-id/``. Point gdb at them with ``set debug-file-directory /lib/debug``. The tarballs are reproducible: the same sil-kit commit, sil-kit-pkg commit and image give diff --git a/release/packages.json b/release/packages.json index 70bd90a..f1bca56 100644 --- a/release/packages.json +++ b/release/packages.json @@ -37,8 +37,8 @@ "test_image": "ubuntu:20.04", "release": true, "assets": [ - "libsilkit-*.ubuntu2004.x86_64.tar.gz", - "libsilkit-*.ubuntu2004.x86_64.debug.tar.gz" + "silkit-*.ubuntu2004.x86_64.tar.gz", + "silkit-*.ubuntu2004.x86_64.debug.tar.gz" ] }, { @@ -51,8 +51,8 @@ "test_image": "centos:7", "release": true, "assets": [ - "libsilkit-*.el7.x86_64.tar.gz", - "libsilkit-*.el7.x86_64.debug.tar.gz" + "silkit-*.el7.x86_64.tar.gz", + "silkit-*.el7.x86_64.debug.tar.gz" ] }, { @@ -65,8 +65,8 @@ "test_image": "almalinux:8", "release": true, "assets": [ - "libsilkit-*.el8.x86_64.tar.gz", - "libsilkit-*.el8.x86_64.debug.tar.gz" + "silkit-*.el8.x86_64.tar.gz", + "silkit-*.el8.x86_64.debug.tar.gz" ] }, { diff --git a/scripts/silkit_tgz.py b/scripts/silkit_tgz.py index 7865cf6..0176019 100644 --- a/scripts/silkit_tgz.py +++ b/scripts/silkit_tgz.py @@ -24,10 +24,10 @@ class SilKitTGZ(SilKitPKG): """Relocatable binary tarball plus a separate debug symbol tarball. - Layout of libsilkit-..x86_64.tar.gz: + Layout of silkit-..x86_64.tar.gz: bin/, lib/ (libSilKit.so*, cmake/SilKit), include/, share/man/man1/, share/sil-kit/sil-kit-registry.service, share/doc/libsilkit/ - Layout of libsilkit-..x86_64.debug.tar.gz: + Layout of silkit-..x86_64.debug.tar.gz: lib/debug/.build-id/xx/yyyy.debug """ @@ -50,7 +50,7 @@ def source_dir_name(self): def package_name(self): # '~' is not kept by GitHub release assets, use '-' for suffixes like rc1 version = str(self.build_info.version).replace("~", "-") - return f"libsilkit-{version}.{self.build_info.platform}.x86_64" + return f"silkit-{version}.{self.build_info.platform}.x86_64" def setup_build_env(self): self.build_flags = SilKitTGZ.__get_build_flags(self.build_info.platform) diff --git a/tests/test_deb.py b/tests/test_deb.py index 9adf4e8..1fd291d 100644 --- a/tests/test_deb.py +++ b/tests/test_deb.py @@ -65,8 +65,8 @@ def extract_tarball(tgz_directory, test_dir) -> Path: print("\nExtract SilKit tarball") print("--------------------------------\n", flush=True) - tarballs = [t for t in tgz_directory.glob('libsilkit-*.tar.gz') if not t.name.endswith('.debug.tar.gz')] - debug_tarballs = list(tgz_directory.glob('libsilkit-*.debug.tar.gz')) + tarballs = [t for t in tgz_directory.glob('silkit-*.tar.gz') if not t.name.endswith('.debug.tar.gz')] + debug_tarballs = list(tgz_directory.glob('silkit-*.debug.tar.gz')) if len(tarballs) != 1 or len(debug_tarballs) != 1: print(f"Expected one tarball and one debug tarball in {tgz_directory}, found {tarballs} and {debug_tarballs}") exit(64)