From c0298f08679694960adf14f2041e436d081c2210 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:48:06 +0200 Subject: [PATCH 1/9] feat: python standalone images from python-build-standalone New python/ package: distroless Python images from python-build-standalone (PBS) prebuilt tarballs instead of the Debian python packages used by python3/. PBS statically links the native libraries (OpenSSL, zlib, sqlite3, libffi, bz2, lzma, ncurses, expat) into the interpreter, so the image adds no python runtime debs - only libc-bin, whose ldconfig plus a generated per-arch ld.so.cache keeps ctypes.util.find_library() working (python3/ parity). Matrix: 3.13.15 / 3.14.7 x debian13 x amd64/arm64/s390x/riscv64 x root/nonroot x normal/debug - more Python versions than Debian ships on any single release (trixie has 3.13 only) and an arch python3/ never had (s390x); 3.15 is added by the updater once PBS releases it stable. debian12 and armv7/ppc64le are excluded (PBS publishes no usable builds). Pinned to PBS release 20260814. --- .github/workflows/check-ldconfig.yaml | 2 +- .../update-deb-package-snapshots.yml | 2 +- .gitignore | 8 +- BUILD | 36 +++ MODULE.bazel | 5 + MODULE.bazel.lock | 164 +++++++++++ private/extensions/python.bzl | 259 ++++++++++++++++++ python/BUILD | 50 ++++ python/README.md | 68 +++++ python/config.bzl | 21 ++ python/control | 14 + python/ldconfig.bzl | 76 +++++ python/ldconfig/ld.so.cache.amd64 | Bin 0 -> 1783 bytes python/ldconfig/ld.so.cache.arm64 | Bin 0 -> 1863 bytes python/ldconfig/ld.so.cache.riscv64 | Bin 0 -> 1815 bytes python/ldconfig/ld.so.cache.s390x | Bin 0 -> 1727 bytes python/ldconfig/ldconfig.sh | 56 ++++ python/python.bzl | 120 ++++++++ python/testdata/example.yaml | 6 + python/testdata/python3.13.yaml | 10 + python/testdata/python3.14.yaml | 10 + python/testdata/python3.yaml | 120 ++++++++ python/testdata/stdlib_check.py | 131 +++++++++ 23 files changed, 1155 insertions(+), 3 deletions(-) create mode 100644 private/extensions/python.bzl create mode 100644 python/BUILD create mode 100644 python/README.md create mode 100644 python/config.bzl create mode 100644 python/control create mode 100644 python/ldconfig.bzl create mode 100644 python/ldconfig/ld.so.cache.amd64 create mode 100644 python/ldconfig/ld.so.cache.arm64 create mode 100644 python/ldconfig/ld.so.cache.riscv64 create mode 100644 python/ldconfig/ld.so.cache.s390x create mode 100755 python/ldconfig/ldconfig.sh create mode 100644 python/python.bzl create mode 100644 python/testdata/example.yaml create mode 100644 python/testdata/python3.13.yaml create mode 100644 python/testdata/python3.14.yaml create mode 100644 python/testdata/python3.yaml create mode 100644 python/testdata/stdlib_check.py diff --git a/.github/workflows/check-ldconfig.yaml b/.github/workflows/check-ldconfig.yaml index 2dede145b..e946f708b 100644 --- a/.github/workflows/check-ldconfig.yaml +++ b/.github/workflows/check-ldconfig.yaml @@ -23,4 +23,4 @@ jobs: uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 - name: Check ldconfig caches - run: bazel test //python3:check_ldconfig_tests + run: bazel test //python3:check_ldconfig_tests //python:check_ldconfig_tests diff --git a/.github/workflows/update-deb-package-snapshots.yml b/.github/workflows/update-deb-package-snapshots.yml index c1077b60e..1bdf6acb0 100644 --- a/.github/workflows/update-deb-package-snapshots.yml +++ b/.github/workflows/update-deb-package-snapshots.yml @@ -33,7 +33,7 @@ jobs: - name: Update ldconfig if: env.DISTROLESS_DIFF - run: bazel run //python3:update_ldconfig + run: bazel run //python3:update_ldconfig //python:update_ldconfig - name: Run update sboms script if: env.DISTROLESS_DIFF diff --git a/.gitignore b/.gitignore index 3718e7ab6..54fd319ec 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,10 @@ /output/ # ignore user bazelrc .bazelrc.user -*.log \ No newline at end of file +*.log +# local bazel caches +.gocache/ +/.gotmp + +# rendered image reconstruction (regenerable via bazel) +/python/Dockerfile diff --git a/BUILD b/BUILD index 9063b4f90..1ea2a9b84 100644 --- a/BUILD +++ b/BUILD @@ -4,6 +4,7 @@ load("//java:config.bzl", "JAVA_ARCHITECTURES", "JAVA_DISTROS", "JAVA_MAJOR_VERS load("//nodejs:config.bzl", "NODEJS_ARCHITECTURES", "NODEJS_DISTROS", "NODEJS_MAJOR_VERSIONS") load("//private/oci:defs.bzl", "sign_and_push_all") load("//private/tools/lifecycle:defs.bzl", "attach_lifecycle_tags") +load("//python:config.bzl", PBS_PYTHON_ARCHITECTURES = "PYTHON_ARCHITECTURES", PBS_PYTHON_DISTROS = "PYTHON_DISTROS", PBS_PYTHON_MAJOR_VERSIONS = "PYTHON_MAJOR_VERSIONS") load("//python3:config.bzl", "PYTHON_ARCHITECTURES", "PYTHON_DISTROS") load("//static:config.bzl", "STATIC_ARCHITECTURES", "STATIC_DISTROS") @@ -196,6 +197,39 @@ NODEJS |= { for (tag_base, debug_mode, user) in VARIANTS } +############### +# PYTHON (PBS) # +############### +PYTHON = { + "{REGISTRY}/{PROJECT_ID}/python" + version + "-" + distro + ":" + tag_base + "-" + arch: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + arch + "_" + distro + for version in PBS_PYTHON_MAJOR_VERSIONS + for distro in PBS_PYTHON_DISTROS + for arch in PBS_PYTHON_ARCHITECTURES[distro][version] + for (tag_base, debug_mode, user) in VARIANTS +} + +# oci_image_index +PYTHON |= { + "{REGISTRY}/{PROJECT_ID}/python" + version + "-" + distro + ":" + tag_base: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + distro + for version in PBS_PYTHON_MAJOR_VERSIONS + for distro in PBS_PYTHON_DISTROS + for (tag_base, debug_mode, user) in VARIANTS +} + +PYTHON |= { + "{REGISTRY}/{PROJECT_ID}/python" + version + ":" + tag_base + "-" + arch: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + arch + "_" + DEFAULT_DISTRO + for version in PBS_PYTHON_MAJOR_VERSIONS + for arch in PBS_PYTHON_ARCHITECTURES[DEFAULT_DISTRO][version] + for (tag_base, debug_mode, user) in VARIANTS +} + +# oci_image_index +PYTHON |= { + "{REGISTRY}/{PROJECT_ID}/python" + version + ":" + tag_base: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + DEFAULT_DISTRO + for version in PBS_PYTHON_MAJOR_VERSIONS + for (tag_base, debug_mode, user) in VARIANTS +} + ############### # JAVA_BASE # ############### @@ -270,6 +304,8 @@ ALL |= PYTHON3 ALL |= NODEJS +ALL |= PYTHON + ALL |= JAVA_BASE ALL |= JAVA diff --git a/MODULE.bazel b/MODULE.bazel index ecf50d358..9c8d2affe 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -50,5 +50,10 @@ node = use_extension("//private/extensions:node.bzl", "node") node.archive() use_repo(node, "node_versions", "nodejs22_amd64", "nodejs22_arm", "nodejs22_arm64", "nodejs22_ppc64le", "nodejs22_s390x", "nodejs24_amd64", "nodejs24_arm64", "nodejs24_ppc64le", "nodejs24_s390x", "nodejs26_amd64", "nodejs26_arm64", "nodejs26_ppc64le", "nodejs26_s390x") +### PYTHON ### +py = use_extension("//private/extensions:python.bzl", "python") +py.archive() +use_repo(py, "python313_amd64", "python313_arm64", "python313_riscv64", "python313_s390x", "python314_amd64", "python314_arm64", "python314_riscv64", "python314_s390x", "python_versions") + ### DEBIAN ### include("//private/repos/deb:deb.MODULE.bazel") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index f9940075f..638bc8fce 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -580,6 +580,170 @@ "recordedRepoMappingEntries": [] } }, + "//private/extensions:python.bzl%python": { + "general": { + "bzlTransitiveDigest": "rGMb7dM985BnsjzKJj2GhFhvyQVKKJqD/RBTmboWUDU=", + "usagesDigest": "tG689EIwVPWG0ZSvYu3XHygIO8sACWdZ24C/r1Ck6Ng=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "python313_amd64": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "45816a2653b47a6cc48d8ada4ea1185758a4c2db389d012b31e0205e5ccb548b", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.13.15+20260814", + "python_version": "3.13", + "architecture": "amd64", + "control": "@@//python:control" + } + }, + "python313_arm64": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "303efcce34b86fd8b0d8a260327dbf8d0d4fba6d2d77b2bca311e8bbd19265e1", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.13.15+20260814", + "python_version": "3.13", + "architecture": "arm64", + "control": "@@//python:control" + } + }, + "python313_s390x": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "41b878867844e56a2757bef4dfdb906596ffc43c0a9a816e5b3f38dc09c97d8e", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-s390x-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.13.15+20260814", + "python_version": "3.13", + "architecture": "s390x", + "control": "@@//python:control" + } + }, + "python313_riscv64": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "f6c666d439a46d6729e211e465362233d2b3f94d4cc9cf117b1d1045fca62b9c", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-riscv64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.13.15+20260814", + "python_version": "3.13", + "architecture": "riscv64", + "control": "@@//python:control" + } + }, + "python314_amd64": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "490953e4a7e97a6ed64d0f0bd04f2d4d9a7bb45368890d3fa32ba05d0ff5c48f", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.14.7+20260814", + "python_version": "3.14", + "architecture": "amd64", + "control": "@@//python:control" + } + }, + "python314_arm64": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "5bad659dc6d686b430f60104d53abfecb1cd93de37b8aaa112f10f0cea2cb655", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.14.7+20260814", + "python_version": "3.14", + "architecture": "arm64", + "control": "@@//python:control" + } + }, + "python314_s390x": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "8ceecd7af302de3f6e6c4564dd671e4d1fa3a2a69f7d40075c75a2747d6ceacf", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-s390x-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.14.7+20260814", + "python_version": "3.14", + "architecture": "s390x", + "control": "@@//python:control" + } + }, + "python314_riscv64": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "9a85c88447db6ff01dfb6e6dfb27b2c856d48776d9ce8503ffd9ab2fa47960e6", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-riscv64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.14.7+20260814", + "python_version": "3.14", + "architecture": "riscv64", + "control": "@@//python:control" + } + }, + "python_versions": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_versions_repo", + "attributes": { + "versions": { + "3.13_amd64": "3.13.15", + "3.13_arm64": "3.13.15", + "3.13_s390x": "3.13.15", + "3.13_riscv64": "3.13.15", + "3.14_amd64": "3.14.7", + "3.14_arm64": "3.14.7", + "3.14_s390x": "3.14.7", + "3.14_riscv64": "3.14.7" + } + } + } + }, + "moduleExtensionMetadata": { + "explicitRootModuleDirectDeps": [ + "python_versions", + "python313_amd64", + "python313_arm64", + "python313_s390x", + "python313_riscv64", + "python314_amd64", + "python314_arm64", + "python314_s390x", + "python314_riscv64" + ], + "explicitRootModuleDirectDevDeps": [], + "useAllRepos": "NO", + "reproducible": false + }, + "recordedRepoMappingEntries": [] + } + }, "//private/extensions:version.bzl%version": { "general": { "bzlTransitiveDigest": "a4FYxHYUQ822kcAcKD5tzEjksar43fkO3cC2oeyx6zA=", diff --git a/private/extensions/python.bzl b/private/extensions/python.bzl new file mode 100644 index 000000000..7efce295e --- /dev/null +++ b/private/extensions/python.bzl @@ -0,0 +1,259 @@ +"python" + +BUILD_TMPL = """\ +# GENERATED BY python_archive.bzl +load("@distroless//private/pkg:debian_spdx.bzl", "debian_spdx") +load("@distroless//private/util:merge_providers.bzl", "merge_providers") +load("@distroless//private/util:tar.bzl", "tar") + +tar( + name = "data", + extension = "tar.gz", + srcs = glob( + [ + "output/bin/python{pyver}", + "output/lib/python{pyver}/**", + "output/lib/libpython{pyver}*", + ], + exclude = [ + "output/lib/python{pyver}/site-packages/pip/**", + "output/lib/python{pyver}/site-packages/pip-*.dist-info/**", + ], + ), + symlinks = {{ + "/python/bin/python": "python{pyver}", + "/python/bin/python3": "python{pyver}", + }}, + package_dir = "/python", + strip_prefix = "external/{canonical_name}/output" +) + +tar( + name = "_control", + extension = "tar.gz", + srcs = ["control"] +) + +debian_spdx( + name = "spdx", + control = ":_control", + data = ":data", + package_name = "{package_name}", + spdx_id = "{spdx_id}", + sha256 = "{sha256}", + urls = [{urls}] +) + +merge_providers( + name = "{name}", + srcs = [":data", ":spdx"], + visibility = ["//visibility:public"], +) +""" + +def _impl(rctx): + rctx.report_progress("Fetching {}".format(rctx.attr.package_name)) + rctx.download_and_extract( + url = rctx.attr.urls, + sha256 = rctx.attr.sha256, + type = rctx.attr.type, + stripPrefix = rctx.attr.strip_prefix, + output = "output", + ) + rctx.template( + "control", + rctx.attr.control, + substitutions = { + "{{VERSION}}": rctx.attr.version, + "{{ARCHITECTURE}}": rctx.attr.architecture, + "{{SHA256}}": rctx.attr.sha256, + }, + ) + rctx.file( + "BUILD.bazel", + content = BUILD_TMPL.format( + canonical_name = rctx.attr.name, + name = rctx.attr.name.split("~")[-1], + package_name = rctx.attr.package_name, + spdx_id = rctx.attr.name, + pyver = rctx.attr.python_version, + urls = ",".join(['"%s"' % url for url in rctx.attr.urls]), + sha256 = rctx.attr.sha256, + ), + ) + +python_archive = repository_rule( + implementation = _impl, + attrs = { + "urls": attr.string_list(mandatory = True), + "sha256": attr.string(mandatory = True), + "type": attr.string(default = ".tar.gz"), + "strip_prefix": attr.string(), + "package_name": attr.string(default = "python"), + "version": attr.string(mandatory = True), + # the x.y version used for bin/lib paths, e.g. "3.14" + "python_version": attr.string(mandatory = True), + "architecture": attr.string(mandatory = True), + # control is only used to populate the sbom, see https://github.com/GoogleContainerTools/distroless/issues/1373 + # for why writing debian control files to the image is incompatible with scanners. + "control": attr.label(), + }, +) + +_PYTHON_VERSIONS_TMPL = """\ +"python versions" + +# AUTO GENERATED. DO NOT EDIT. +PYTHON_VERSIONS = {versions} +""" + +def _python_versions_repo_impl(rctx): + rctx.file("versions.bzl", _PYTHON_VERSIONS_TMPL.format(versions = str(rctx.attr.versions))) + rctx.file("BUILD.bazel", 'exports_files(["versions.bzl"])') + +python_versions_repo = repository_rule( + implementation = _python_versions_repo_impl, + attrs = { + "versions": attr.string_dict(), + }, +) + +def _python_impl(module_ctx): + mod = module_ctx.modules[0] + + if len(module_ctx.modules) > 1: + fail("python.archive should be called only once") + if not mod.is_root: + fail("python.archive should be called from root module only.") + + # Python from python-build-standalone (https://github.com/astral-sh/python-build-standalone) + # Release 20260814. Linux targets only (distroless images). + # Versions 3.13, 3.14 (new minors added by update-python-archives when stable). + # NOTE: armv7 is intentionally absent: PBS publishes soft-float gnueabi builds + # (interpreter /lib/ld-linux.so.3) which cannot run on the distroless armhf base + # (loader /usr/lib/ld-linux-armhf.so.3); ppc64le is not published by PBS. + python_archive( + name = "python313_amd64", + sha256 = "45816a2653b47a6cc48d8ada4ea1185758a4c2db389d012b31e0205e5ccb548b", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260814", + python_version = "3.13", + architecture = "amd64", + control = "//python:control", + ) + + python_archive( + name = "python313_arm64", + sha256 = "303efcce34b86fd8b0d8a260327dbf8d0d4fba6d2d77b2bca311e8bbd19265e1", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260814", + python_version = "3.13", + architecture = "arm64", + control = "//python:control", + ) + + python_archive( + name = "python313_s390x", + sha256 = "41b878867844e56a2757bef4dfdb906596ffc43c0a9a816e5b3f38dc09c97d8e", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-s390x-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260814", + python_version = "3.13", + architecture = "s390x", + control = "//python:control", + ) + + python_archive( + name = "python313_riscv64", + sha256 = "f6c666d439a46d6729e211e465362233d2b3f94d4cc9cf117b1d1045fca62b9c", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-riscv64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260814", + python_version = "3.13", + architecture = "riscv64", + control = "//python:control", + ) + + python_archive( + name = "python314_amd64", + sha256 = "490953e4a7e97a6ed64d0f0bd04f2d4d9a7bb45368890d3fa32ba05d0ff5c48f", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260814", + python_version = "3.14", + architecture = "amd64", + control = "//python:control", + ) + + python_archive( + name = "python314_arm64", + sha256 = "5bad659dc6d686b430f60104d53abfecb1cd93de37b8aaa112f10f0cea2cb655", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260814", + python_version = "3.14", + architecture = "arm64", + control = "//python:control", + ) + + python_archive( + name = "python314_s390x", + sha256 = "8ceecd7af302de3f6e6c4564dd671e4d1fa3a2a69f7d40075c75a2747d6ceacf", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-s390x-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260814", + python_version = "3.14", + architecture = "s390x", + control = "//python:control", + ) + + python_archive( + name = "python314_riscv64", + sha256 = "9a85c88447db6ff01dfb6e6dfb27b2c856d48776d9ce8503ffd9ab2fa47960e6", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-riscv64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260814", + python_version = "3.14", + architecture = "riscv64", + control = "//python:control", + ) + + python_versions_repo( + name = "python_versions", + versions = { + "3.13_amd64": "3.13.15", + "3.13_arm64": "3.13.15", + "3.13_s390x": "3.13.15", + "3.13_riscv64": "3.13.15", + "3.14_amd64": "3.14.7", + "3.14_arm64": "3.14.7", + "3.14_s390x": "3.14.7", + "3.14_riscv64": "3.14.7", + }, + ) + + return module_ctx.extension_metadata( + root_module_direct_deps = [ + "python_versions", + "python313_amd64", + "python313_arm64", + "python313_s390x", + "python313_riscv64", + "python314_amd64", + "python314_arm64", + "python314_s390x", + "python314_riscv64", + ], + root_module_direct_dev_deps = [], + ) + +_archive = tag_class(attrs = {}) + +python = module_extension( + implementation = _python_impl, + tag_classes = { + "archive": _archive, + }, +) diff --git a/python/BUILD b/python/BUILD new file mode 100644 index 000000000..1cdca39ee --- /dev/null +++ b/python/BUILD @@ -0,0 +1,50 @@ +load("//private/util:tar.bzl", "tar") +load(":config.bzl", "PYTHON_ARCHITECTURES", "PYTHON_DISTROS", "PYTHON_MAJOR_VERSIONS", "PYTHON_PACKAGES") +load(":ldconfig.bzl", "python_ldconfig") +load(":python.bzl", "python_image", "python_image_index") + +package(default_visibility = ["//visibility:public"]) + +[ + python_image( + arch = arch, + distro = distro, + major_version = major_version, + packages = PYTHON_PACKAGES[distro], + ) + for distro in PYTHON_DISTROS + for major_version in PYTHON_MAJOR_VERSIONS + for arch in PYTHON_ARCHITECTURES[distro][major_version] +] + +[ + python_image_index( + architectures = PYTHON_ARCHITECTURES[distro][major_version], + distro = distro, + major_version = major_version, + ) + for distro in PYTHON_DISTROS + for major_version in PYTHON_MAJOR_VERSIONS +] + +# ld.so.cache per arch, generated by running ldconfig inside the image +# (bazel run //python:update_ldconfig), shipped at /etc/ld.so.cache so +# ctypes.util.find_library() works. +[ + tar( + name = "ldconfig_cache_{}".format(arch), + srcs = ["ldconfig/ld.so.cache.{}".format(arch)], + args = [ + "--format", + "gnutar", + ], + extension = "tar.gz", + mtree = ["etc/ld.so.cache uid=0 gid=0 uname=root gname=root mode=0644 time=0 type=file content=$(location ldconfig/ld.so.cache.{})".format(arch)], + ) + for arch in PYTHON_ARCHITECTURES["debian13"]["3.14"] +] + +python_ldconfig( + architectures = PYTHON_ARCHITECTURES["debian13"]["3.14"], + distro = "debian13", +) diff --git a/python/README.md b/python/README.md new file mode 100644 index 000000000..e7b91e11a --- /dev/null +++ b/python/README.md @@ -0,0 +1,68 @@ +# Documentation for `python/` (python-build-standalone) + +## Image Contents + +These images contain a minimal Linux, Python-based runtime built from a +[python-build-standalone](https://github.com/astral-sh/python-build-standalone) +prebuilt tarball instead of the Debian python package. + +Supported versions (matching the official CPython support windows): + +- Python 3.13 (`python3.13-debian13`) +- Python 3.14 (`python3.14-debian13`) + +Each image contains everything in the [base image](../base/README.md), +plus a standalone CPython install under `/python`: + +- `/python/bin/python3.X` (with `/python/bin/python` and `/python/bin/python3` symlinks) +- `/python/lib/python3.X/` (standard library) +- `/python/lib/libpython3.X.so.1.0` + +No shell, no pip (install dependencies into `site-packages` in a builder stage), +no include files. python-build-standalone builds are relocatable and reference +only glibc at runtime, so no extra Debian packages are needed. + +## Usage + +The entrypoint of these images is set to `python3.X`, so they expect users to +supply a path to a .py file in the CMD. + +See the Python [Hello World](../examples/python3/) directory for an example. + +### Compatibility + +When using these images in a multi-stage build, use a build image with the same +Python version to avoid ABI-related errors. For example, when targeting +`python3.14-debian13`, use a `python:3.14-slim-trixie` (or python-build-standalone) +build stage. + +When building virtual environments, note that the distroless Python path is +`/python/bin/python3.14`. To ensure your virtual environment's internal links +are correct, your build environment should match this path. If your build image +uses a different path, create a symlink during the build stage, e.g.: + +```Dockerfile +RUN ln -s /usr/local/bin/python3.14 /python/bin/python3.14 +``` + +### ctypes.util.find_library + +`ctypes.util.find_library()` works for system libraries (e.g. `libc`, `libssl`, `libz`): +the image ships `/sbin/ldconfig` (from `libc-bin`) and a pre-generated `ld.so.cache` at +`/etc/ld.so.cache`. The cache is generated per arch by running ldconfig inside the image: + + bazel run //python:update_ldconfig + +and verified by `bazel test //python:check_ldconfig_tests` (also in CI). + +Libraries that python-build-standalone links statically into the interpreter +(sqlite3, ncurses, bz2, lzma, expat, zlib, ...) have no shared object to find — +the corresponding stdlib modules work out of the box. `python -m venv` works and +bootstraps pip from the bundled `ensurepip` wheels. + +## Provenance + +Python comes from a [python-build-standalone](https://github.com/astral-sh/python-build-standalone) +prebuilt tarball (`cpython-3.X.Y+--unknown-linux-gnu-install_only.tar.gz`). +The exact release, versions and shas are pinned in `//private/extensions:python.bzl` and +updated via `knife update-python-archives` (see `knife.d/update_python_archives.sh`). diff --git a/python/config.bzl b/python/config.bzl new file mode 100644 index 000000000..5c5dcc16b --- /dev/null +++ b/python/config.bzl @@ -0,0 +1,21 @@ +"python configurations" + +PYTHON_DISTROS = ["debian13"] + +# The image tag segment, e.g. python3.14-debian13. +# Full CPython versions live in //private/extensions:python.bzl (python_versions repo). +# debian12 is intentionally absent: new debian12 builds are being deprecated upstream. +PYTHON_MAJOR_VERSIONS = ["3.13", "3.14"] + +PYTHON_ARCHITECTURES = { + "debian13": { + "3.13": ["amd64", "arm64", "s390x", "riscv64"], + "3.14": ["amd64", "arm64", "s390x", "riscv64"], + }, +} + +# deb packages added to the image: libc-bin provides /sbin/ldconfig so +# ctypes.util.find_library() works via a generated ld.so.cache (python3/ parity). +PYTHON_PACKAGES = { + "debian13": ["libc-bin"], +} diff --git a/python/control b/python/control new file mode 100644 index 000000000..f40da6999 --- /dev/null +++ b/python/control @@ -0,0 +1,14 @@ +Package: python3 +Version: {{VERSION}} +Architecture: {{ARCHITECTURE}} +Maintainer: python-build-standalone +Homepage: https://github.com/astral-sh/python-build-standalone +SHA256: {{SHA256}} +Description: Python from python-build-standalone + Standalone, redistributable build of CPython, see + https://github.com/astral-sh/python-build-standalone. + . + Installed under /python. Entrypoint is /python/bin/python3. + . + pip is not included in the image (matches the nodejs image stripping npm); + install dependencies into site-packages in a builder stage instead. diff --git a/python/ldconfig.bzl b/python/ldconfig.bzl new file mode 100644 index 000000000..7a6e6bf45 --- /dev/null +++ b/python/ldconfig.bzl @@ -0,0 +1,76 @@ +"Macros for generating ldconfig cache for python images" + +load("@rules_oci//oci:defs.bzl", "oci_load") + +def python_ldconfig(architectures, distro): + """Generates ldconfig cache targets for python images. + + Args: + architectures: list of architectures to support + distro: the distribution name (e.g. debian13) + """ + + # 1. Create the oci_load targets (the scripts that load images into docker) + for arch in architectures: + oci_load( + name = "load_python314_root_{}_{}".format(arch, distro), + image = ":python314_root_{}_{}".format(arch, distro), + repo_tags = ["bazel/python:python314_root_{}_{}".format(arch, distro)], + ) + + # 2. Create the genrules that actually trigger the loading during 'bazel build' + for arch in architectures: + native.genrule( + name = "do_load_{}".format(arch), + outs = ["do_load_{}.done".format(arch)], + cmd = "$(location :load_python314_root_{arch}_{distro}) && touch $@".format( + arch = arch, + distro = distro, + ), + tags = [ + "local", + "no-sandbox", + ], + tools = [":load_python314_root_{}_{}".format(arch, distro)], + ) + + # 3. Create the update_ldconfig binary + native.sh_binary( + name = "update_ldconfig", + srcs = ["ldconfig/ldconfig.sh"], + args = ["update"] + architectures, + data = ["ldconfig/ldconfig.sh"] + [":do_load_{}".format(arch) for arch in architectures], + tags = ["local"], + ) + + # 4. Create the architecture-specific tests + for arch in architectures: + native.sh_test( + name = "check_ldconfig_{}_test".format(arch), + srcs = ["ldconfig/ldconfig.sh"], + args = [ + "check", + arch, + ], + data = [ + "ldconfig/ld.so.cache.{}".format(arch), + ":do_load_{}".format(arch), + ], + tags = [ + arch, + "local", + "manual", + "external", + "no-cache", + ], + ) + + # 5. Create the test suite + native.test_suite( + name = "check_ldconfig_tests", + tests = [ + ":check_ldconfig_{}_test".format(arch) + for arch in architectures + ], + tags = ["manual"], + ) diff --git a/python/ldconfig/ld.so.cache.amd64 b/python/ldconfig/ld.so.cache.amd64 new file mode 100644 index 0000000000000000000000000000000000000000..e50ae2e2096f0feee8f8b8793f56141838fb19af GIT binary patch literal 1783 zcma)6F>ljA7`;;94|G6?#i&9mik#vkvZ&Y^B#P)CV5xOJUme5w9Qo|1bU>&pFtRZL z8$%ZcMwYHjh=Bniv2|iVY?XNDFj(lbub*^@{QUCXyYIeSsICu zjQw0?>;k@DU&cqPR)PPz#Mn2ic3J%EMaDj1waemfzw5%kMd)tK@#PYc~w}_xZBAX$Qlc}1Ok|$9TJETOEbEOIqB{Hs4OTN{8DD$Q2 zlEtd)?@m`*m)Qs0F;hINL=<{IOTCS|GS0a6?(N>+o_eFg%;M9TG)xUYAb?RsbD4eP4MPDTYYX*Xgi3|tf z3C7s*MaK5y`^Q0iwAR8Q#(o`O>lUBuJ&?1 zZ=k&(^q;8tnW*~}?0bU$gqm-Oy5G?72iJ6Kxw_Bc{fO|t4ectSuRuFR=!c>GCi2|D z-drR0=a|b#!rV0HCytfJem&o{-$axFwWrBZz zx*3t@9kf@3ejna934f>0{?nH+&&!;bVzN5!t4!DT`l+s+T8N@Fo0*arb+$W}%e=Nc z?V{XyS@qb6F2Gia==HiNQEnBLan#6D%v#ljl+qeCb=1&fq^*sF$qQcXg-o>F3+DE` z6HJ;8y0T5A%}n#I@{u!_D)h8jHtqR{REVg_?1ivXU8VH=(fDk*ZhH42b(M88Yo)m> z^@b6=8FB9h%rJdj$@8gfhS-;@nv{|!Q4;&jlAV*db1qx3+Wjx`5Q`}1N*NI)GOkl+ zbiwnSi+R2|##nXw+4p^|cl+WJW9#^>5@GaamImjp$T;IVxV~{?Z9Nzc#$i7k_E0Ju Z)uZ!)t$3_tP)a4amBCb&m`2@j{{WxjsLTKW literal 0 HcmV?d00001 diff --git a/python/ldconfig/ld.so.cache.riscv64 b/python/ldconfig/ld.so.cache.riscv64 new file mode 100644 index 0000000000000000000000000000000000000000..b21da562c0006dbb1d8b9c40ff815443f3295e60 GIT binary patch literal 1815 zcma)-KWNoJ9LG;tY!A)0jaZ&f;Lqfj>ZH3T(A(yX2K6lZZkm%2E zt{yo=Ku_cfejJ_!)A!Pvr9w+A*OA z@cwazzWQI>b?n>qDz{=joeWj3o4sMC8>g0{tjun%B(^j=8!KedSf2G!?p#bgHlh!( zbt(pfK1x(i%Gx+;>O(4NjhZ`ZfASz*`dmir3~p`%Iw literal 0 HcmV?d00001 diff --git a/python/ldconfig/ld.so.cache.s390x b/python/ldconfig/ld.so.cache.s390x new file mode 100644 index 0000000000000000000000000000000000000000..73d4ee69511d7b1e249c05d0dc1bfbc7fd08ce20 GIT binary patch literal 1727 zcmai!y^GX97>CDAHX_*A2x60KA#@!f z?}G;&gaX=q&R;&@;YVn%&r#QDx&Id0E1eGs=pPvP3GmN4_a1@&y9WLS_)DGpV8cJw zx%UP1?{ywLguL$PeDEEi!6%*j4cLET;4$!P27U?r{?WPj6Zkis57%HFF6(^o8{+w) zbMGqHxuA3ZFT`_Z;HSX98u(-2_jK+pz&`9*|7rI(xB=}-pTd0L-YsaM&WBUTD>CpG zz)PJ67lHpW^I*{D8&J0!YI1vX5%EGaizpXO%SuvK@}S@h3L4o&b2e`j&fS{!L~L74 z5cjaEDH)C28d5zgYuRchRyRssiGxO=;$m(e16?F#rG8aIi=^$DB)&QkFG&J9X2#E9EkRr3j(3cWX7A z-QIl_Jcs`sPKgu;MINs2vaG-&+}poD-3iC>RuU!S5jaXF&E#gNYMgNvR*W;OSa`@P Js2kDg{sAoccyj;% literal 0 HcmV?d00001 diff --git a/python/ldconfig/ldconfig.sh b/python/ldconfig/ldconfig.sh new file mode 100755 index 000000000..bc4926dea --- /dev/null +++ b/python/ldconfig/ldconfig.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -eo pipefail + +MODE=$1 +shift + +if [[ "${MODE}" != "update" && "${MODE}" != "check" ]]; then + echo "Usage: $0 ..." >&2 + exit 1 +fi + +# Root to find the reference cache files +ROOT="python/ldconfig" +if [[ -n "${BUILD_WORKSPACE_DIRECTORY}" ]]; then + cd "${BUILD_WORKSPACE_DIRECTORY}" +elif [[ ! -d "${ROOT}" ]]; then + # Fallback to finding it in the same dir as the script + ROOT=$(dirname "$0") +fi + +CID="" +TEMP_DIR="" +cleanup() { + local exit_status=$? + if [[ -n "${CID}" ]]; then docker rm -f "${CID}"; fi + if [[ -n "${TEMP_DIR}" ]]; then rm -rf "${TEMP_DIR}"; fi + exit "${exit_status}" +} +trap cleanup EXIT + +for arch in "$@"; do + if [ "$MODE" = "update" ]; then echo "===> Updating ldconfig cache for ${arch}..."; else echo "===> Checking ldconfig cache for ${arch}..."; fi + + # Run ldconfig and extract the cache. + # We assume the image is ALREADY loaded (handled by Bazel build dependencies) + CID=$(docker create --platform "linux/${arch}" --entrypoint /sbin/ldconfig "bazel/python:python314_root_${arch}_debian13") + docker start -a "${CID}" + + if [[ "${MODE}" == "update" ]]; then + docker cp "${CID}:/etc/ld.so.cache" "${ROOT}/ld.so.cache.${arch}" + echo "✅ Updated ${arch} cache." + else + TEMP_DIR=$(mktemp -d) + docker cp "${CID}:/etc/ld.so.cache" "${TEMP_DIR}/ld.so.cache.${arch}" + if ! cmp -s "${TEMP_DIR}/ld.so.cache.${arch}" "${ROOT}/ld.so.cache.${arch}"; then + echo "❌ ERROR: ldconfig cache for ${arch} is out of date!" >&2 + echo "Run 'bazel run //python:update_ldconfig' to regenerate." >&2 + exit 1 + fi + rm -rf "${TEMP_DIR}" + TEMP_DIR="" + echo "✅ ldconfig cache for ${arch} is up to date." + fi + docker rm -f "${CID}" + CID="" +done diff --git a/python/python.bzl b/python/python.bzl new file mode 100644 index 000000000..e22411d6c --- /dev/null +++ b/python/python.bzl @@ -0,0 +1,120 @@ +"python image definitions" + +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("@python_versions//:versions.bzl", "PYTHON_VERSIONS") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_image_index") +load("//common:variables.bzl", "DEBUG_MODE", "OS_RELEASE", "USERS") +load("//private/util:deb.bzl", "deb") +load("//private/util:tar.bzl", "tar") + +def _bin_version(major_version): + """bin/lib version, e.g. \"3.15rc1\" -> \"3.15\".""" + return major_version.split("rc")[0] + +def python_image_index(distro, major_version, architectures): + """python image index for a distro. + + Args: + distro: name of distribution + major_version: version of python, e.g. "3.14" + architectures: all architectures included in index + """ + for mode in DEBUG_MODE: + for user in USERS: + oci_image_index( + name = "python" + major_version.replace(".", "") + mode + "_" + user + "_" + distro, + images = [ + "python" + major_version.replace(".", "") + mode + "_" + user + "_" + arch + "_" + distro + for arch in architectures + ], + ) + +def _example_tar(): + # only create once + if native.existing_rule("example"): + return + + tar( + name = "example", + extension = "tar.gz", + srcs = ["testdata/stdlib_check.py"], + ) + +def python_image(distro, major_version, arch, packages): + """python and debug image with tests. + + Python comes from a python-build-standalone prebuilt tarball + (//private/extensions:python.bzl) instead of the Debian python package. + + Args: + distro: name of distribution + major_version: version of python, e.g. "3.14" + arch: the target arch + packages: any deb packages to add to the image + """ + + _version_key = major_version + "_" + arch + if _version_key not in PYTHON_VERSIONS: + fail("No version found for Python version/arch: " + _version_key) + _annotations = { + "org.opencontainers.image.source": OS_RELEASE["HOME_URL"], + "com.google.distroless.python.version": PYTHON_VERSIONS[_version_key], + } + + for mode in DEBUG_MODE: + for user in USERS: + oci_image( + name = "python" + major_version.replace(".", "") + mode + "_" + user + "_" + arch + "_" + distro, + base = "//cc:cc" + mode + "_" + user + "_" + arch + "_" + distro, + entrypoint = ["/python/bin/python" + _bin_version(major_version)], + env = {"LANG": "C.UTF-8"}, + tars = [ + deb.package(arch, distro, pkg) + for pkg in packages + ] + [ + "@python" + major_version.replace(".", "") + "_" + arch, + ] + ([":ldconfig_cache_" + arch] if distro == "debian13" else []), + annotations = _annotations, + ) + + _example_tar() + + # Structure tests run on the non-debug images only: testdata/python3.yaml + # asserts there is no shell, and debug images ship busybox /bin/sh + # (matches python3/python.bzl). + for user in USERS: + container_structure_test( + name = "python" + major_version.replace(".", "") + "_" + user + "_" + arch + "_" + distro + "_test", + size = "medium", + configs = [ + "testdata/python3.yaml", + "testdata/python3." + _bin_version(major_version).split(".")[1] + ".yaml", + ], + image = ":python" + major_version.replace(".", "") + "_" + user + "_" + arch + "_" + distro, + tags = [ + arch, + "manual", + ], + ) + + for mode in DEBUG_MODE: + for user in USERS: + oci_image( + name = "example_python" + major_version.replace(".", "") + mode + "_" + user + "_" + arch + "_" + distro, + base = ":python" + major_version.replace(".", "") + mode + "_" + user + "_" + arch + "_" + distro, + tars = [ + ":example", + ], + ) + + for user in USERS: + container_structure_test( + name = "example_python" + major_version.replace(".", "") + "_" + user + "_" + arch + "_" + distro + "_test", + size = "medium", + configs = ["testdata/example.yaml"], + image = ":example_python" + major_version.replace(".", "") + "_" + user + "_" + arch + "_" + distro, + tags = [ + arch, + "manual", + ], + ) diff --git a/python/testdata/example.yaml b/python/testdata/example.yaml new file mode 100644 index 000000000..664c96348 --- /dev/null +++ b/python/testdata/example.yaml @@ -0,0 +1,6 @@ +schemaVersion: "2.0.0" +commandTests: + - name: stdlib_check + command: "/python/bin/python3" + args: ["/stdlib_check.py"] + expectedOutput: ["ALL GOOD"] diff --git a/python/testdata/python3.13.yaml b/python/testdata/python3.13.yaml new file mode 100644 index 000000000..dbfe8ad2a --- /dev/null +++ b/python/testdata/python3.13.yaml @@ -0,0 +1,10 @@ +schemaVersion: "2.0.0" +commandTests: + - name: version + command: "/python/bin/python3.13" + args: ["--version"] + expectedOutput: ["Python 3.13.15"] + - name: symlink + command: "/python/bin/python3" + args: ["--version"] + expectedOutput: ["Python 3.13.15"] diff --git a/python/testdata/python3.14.yaml b/python/testdata/python3.14.yaml new file mode 100644 index 000000000..6fc2e4b38 --- /dev/null +++ b/python/testdata/python3.14.yaml @@ -0,0 +1,10 @@ +schemaVersion: "2.0.0" +commandTests: + - name: version + command: "/python/bin/python3.14" + args: ["--version"] + expectedOutput: ["Python 3.14.7"] + - name: symlink + command: "/python/bin/python3" + args: ["--version"] + expectedOutput: ["Python 3.14.7"] diff --git a/python/testdata/python3.yaml b/python/testdata/python3.yaml new file mode 100644 index 000000000..ef567c8de --- /dev/null +++ b/python/testdata/python3.yaml @@ -0,0 +1,120 @@ +schemaVersion: "2.0.0" +commandTests: + - name: hello + command: "/python/bin/python3" + args: ["-c", "print('Hello World')"] + expectedOutput: ['Hello World'] + + # ensure there is no shell + - name: no_shell + command: "/python/bin/python3" + args: ["-c", + "import subprocess, sys; subprocess.check_call(sys.executable + ' -h', shell=True)"] + exitCode: 1 + + # file names are UTF-8: default for modern Linux systems + # The \xe9 backslash must be double-escaped to avoid YAML string parsing weirdness + - name: filesystem_utf8 + command: "/python/bin/python3" + args: ["-c", "open(u'h\\xe9llo', 'w'); import sys; print(sys.getfilesystemencoding())"] + expectedOutput: ['utf-8'] + + # the print function should output UTF-8 + - name: print_utf8 + command: "/python/bin/python3" + args: ["-c", "print(u'h\\xe9llo.txt')"] + expectedOutput: ['h\xe9llo'] + + # find_library works via /sbin/ldconfig (libc-bin) + the generated ld.so.cache + - name: find_library + command: "/python/bin/python3" + args: ["-c", "import ctypes.util; assert ctypes.util.find_library('c')"] + + # import every module installed with the Python package + - name: import_everything + exitCode: 0 + expectedOutput: ['FINISHED ENTIRE SCRIPT'] + command: "/python/bin/python3" + args: + - "-c" + # multi-line YAML string with Python script that imports all modules that are installed. + # This ensures we have the right native library dependencies. + - | + import importlib, os, pkgutil, sys + + skip_modules = frozenset(( + # Windows-specific modules + 'asyncio.windows_events', + 'asyncio.windows_utils', + 'ctypes.wintypes', + 'encodings.cp65001', + 'encodings.mbcs', + 'encodings.oem', + 'multiprocessing.popen_spawn_win32', + 'winreg', + + # Python regression tests "for internal use by Python only" + 'test', + + # calls sys.exit + 'unittest.__main__', + 'venv.__main__', + + # python-build-standalone builds without _gdbm + 'dbm.gnu', + + # iOS-only support module + '_ios_support', + + # GUI: no Tcl/Tk in the image (never_import below covers the + # tkinter/idlelib/turtle packages; _tkinter is the C extension) + '_tkinter', + + # new REPL; _minimal_curses needs libncurses not shipped in the image + '_pyrepl', + + # prints the Zen of Python on import + 'this', + )) + + # packages we never import: GUI modules (no Tcl/Tk in the image, + # idlelib raises SystemExit on import without Tk) and removed distutils + never_import = ('tkinter', 'idlelib', 'turtle', 'distutils', 'test', 'venv') + + failed = [] + + def walk(path, prefix=''): + try: + infos = list(pkgutil.iter_modules(path)) + except Exception as e: + failed.append('walk {}: {}'.format(path, e)) + return + for info in infos: + name = prefix + info.name + if name in skip_modules or name.startswith(never_import): + continue + if info.ispkg: + try: + mod = importlib.import_module(name) + if hasattr(mod, '__path__'): + walk(mod.__path__, name + '.') + except BaseException as e: + failed.append('{}: {}: {}'.format(name, type(e).__name__, e)) + continue + try: + importlib.import_module(name) + except BaseException as e: + failed.append('{}: {}: {}'.format(name, type(e).__name__, e)) + + stdlib_root = os.path.join(os.path.dirname(sys.executable), os.pardir, 'lib', 'python%d.%d' % sys.version_info[:2]) + for p in sys.path: + if os.path.abspath(p).startswith(os.path.abspath(stdlib_root)) and 'site-packages' not in p: + walk([p]) + + if failed: + for f in failed[:20]: + print('FAIL {}'.format(f)) + raise SystemExit('failed to import {} modules'.format(len(failed))) + + # ensures some module does not exit early (e.g unittest.__main__) + print('FINISHED ENTIRE SCRIPT') diff --git a/python/testdata/stdlib_check.py b/python/testdata/stdlib_check.py new file mode 100644 index 000000000..c0b7d18a7 --- /dev/null +++ b/python/testdata/stdlib_check.py @@ -0,0 +1,131 @@ +#!/python/bin/python3 +"""Import every stdlib module available in the distroless python image and +exercise a representative set with one call each.""" + +import importlib +import pkgutil +import sys + +skip_modules = frozenset(( + # Windows-specific modules + "asyncio.windows_events", + "asyncio.windows_utils", + "ctypes.wintypes", + "encodings.cp65001", + "encodings.mbcs", + "encodings.oem", + "multiprocessing.popen_spawn_win32", + "winreg", + # Python regression tests + "test", + # calls sys.exit + "unittest.__main__", + "venv.__main__", + # python-build-standalone builds without _gdbm + "dbm.gnu", + # platform-specific / GUI extensions not relevant in a server image + "_ios_support", + "_tkinter", + # new REPL; _minimal_curses needs libncurses not shipped in the image + "_pyrepl", + # prints the Zen of Python on import + "this", +)) + +# packages we never import: GUI modules (no Tcl/Tk in the image, and +# idlelib raises SystemExit on import without Tk) and distutils (removed) +never_import = ("tkinter", "idlelib", "turtle", "distutils", "test", "venv") + +failed = [] + +def walk(path, prefix=""): + try: + infos = list(pkgutil.iter_modules(path)) + except Exception as e: + failed.append("walk %s: %s" % (path, e)) + return + for info in infos: + name = prefix + info.name + if name in skip_modules or name.startswith(never_import): + continue + if info.ispkg: + try: + mod = importlib.import_module(name) + if hasattr(mod, "__path__"): + walk(mod.__path__, name + ".") + except BaseException as e: + failed.append("%s: %s: %s" % (name, type(e).__name__, e)) + continue + try: + importlib.import_module(name) + imported[0] += 1 + except BaseException as e: + failed.append("%s: %s: %s" % (name, type(e).__name__, e)) + +import os + +stdlib_root = os.path.join(os.path.dirname(sys.executable), os.pardir, "lib", "python%d.%d" % sys.version_info[:2]) +imported = [0] +for p in sys.path: + # only the stdlib tree; skip the script dir (sys.path[0]) and site-packages + if os.path.abspath(p).startswith(os.path.abspath(stdlib_root)) and "site-packages" not in p: + walk([p]) + +print("imported %d modules" % imported[0]) +if failed: + for f in failed[:30]: + print("FAIL " + f) + print("FAILED %d" % len(failed)) + sys.exit(1) + +# one representative call each; any exception fails the run +calls = [ + lambda: importlib.import_module("hashlib").sha256(b"distroless").hexdigest(), + lambda: importlib.import_module("json").dumps({"a": [1, 2.5, None, "x"]}), + lambda: importlib.import_module("sqlite3").connect(":memory:").execute("select 1").fetchone(), + lambda: importlib.import_module("zlib").compress(b"x" * 1000), + lambda: importlib.import_module("bz2").compress(b"x" * 1000), + lambda: importlib.import_module("lzma").compress(b"x" * 1000), + lambda: importlib.import_module("ctypes").CDLL("libc.so.6"), + lambda: importlib.import_module("base64").b64encode(b"hello"), + lambda: importlib.import_module("uuid").uuid4().hex, + lambda: importlib.import_module("datetime").datetime.now().isoformat(), + lambda: importlib.import_module("collections").Counter("abracadabra")["a"], + lambda: importlib.import_module("io").BytesIO(b"data").read(), + lambda: importlib.import_module("struct").pack(">I", 42), + lambda: importlib.import_module("csv").reader(["a,b"]).__next__(), + lambda: importlib.import_module("re").match(r"\d+", "123x").group(), + lambda: importlib.import_module("string").ascii_letters.__len__(), + lambda: importlib.import_module("ssl").create_default_context(), + lambda: importlib.import_module("ssl").get_default_verify_paths(), + lambda: importlib.import_module("socket").getaddrinfo("localhost", 80), + lambda: importlib.import_module("urllib.request").urlopen("https://example.com", timeout=10).status, + lambda: importlib.import_module("subprocess").check_output([sys.executable, "-c", "print(1)"]).strip(), + lambda: importlib.import_module("os").getpid(), + lambda: importlib.import_module("math").sqrt(2), + lambda: importlib.import_module("statistics").mean([1, 2, 3, 4]), + lambda: importlib.import_module("fractions").Fraction(1, 3) * 3, + lambda: importlib.import_module("calendar").isleap(2024), + lambda: importlib.import_module("html").escape("&"), + lambda: importlib.import_module("glob").glob("/python/bin/*").__len__(), + lambda: importlib.import_module("shutil").which(os.path.basename(sys.executable)), + lambda: importlib.import_module("tempfile").gettempdir(), + lambda: importlib.import_module("threading").current_thread().name, + lambda: importlib.import_module("queue").Queue().empty(), + lambda: importlib.import_module("concurrent.futures").ThreadPoolExecutor(max_workers=1).submit(lambda: 1).result(), +] + +for i, call in enumerate(calls): + try: + call() + except Exception as e: + failed.append("call %d: %s: %s" % (i, type(e).__name__, e)) + +if failed: + for f in failed: + print("FAIL " + f) + print("FAILED %d" % len(failed)) + sys.exit(1) + +print("calls OK: %d" % len(calls)) +print("ALL GOOD on Python %s (%s)" % (sys.version.split()[0], sys.platform)) From b7f412d41389123a8c48f8acce02bcf847fe02eb Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:48:34 +0200 Subject: [PATCH 2/9] feat: python-build-standalone updater + tests (knife update-python-archives) knife update-python-archives keeps the PBS pins in //private/extensions:python.bzl current: reads the matrix (minors + archs) from python/config.bzl, queries the latest PBS release, rewrites every archive (url/sha/version), and - when a new stable minor appears - extends the config.bzl matrix, MODULE.bazel use_repo and the version testdata. An update triggers on a patch move or on a release-tag move with unchanged patches (PBS rebuilds its bundled native libraries, e.g. expat, under the same CPython version). Idempotent; --verbose/--dry-run for inspection. Tests: hermetic smoke test //python:update_python_archives_test (parser consistency) + black-box contract test //python:update_python_archives_contract_test running the updater offline against fixture releases - tag-only bump, patch bump, new-minor fill, convergence. --- BUILD | 3 + knife | 65 +++- knife.d/BUILD.bazel | 1 + knife.d/update_python_archives.sh | 362 ++++++++++++++++++ private/extensions/BUILD.bazel | 1 + python/BUILD | 28 ++ .../update_python_archives_contract_test.sh | 110 ++++++ python/update_python_archives_test.sh | 30 ++ 8 files changed, 599 insertions(+), 1 deletion(-) create mode 100644 knife.d/BUILD.bazel create mode 100644 knife.d/update_python_archives.sh create mode 100755 python/update_python_archives_contract_test.sh create mode 100755 python/update_python_archives_test.sh diff --git a/BUILD b/BUILD index 1ea2a9b84..2d944c2af 100644 --- a/BUILD +++ b/BUILD @@ -10,6 +10,9 @@ load("//static:config.bzl", "STATIC_ARCHITECTURES", "STATIC_DISTROS") package(default_visibility = ["//visibility:public"]) +# allow updater tests to read MODULE.bazel +exports_files(["MODULE.bazel"]) + DEFAULT_DISTRO = "debian13" VARIANTS = [ diff --git a/knife b/knife index a348013a5..72bf4cbec 100755 --- a/knife +++ b/knife @@ -155,6 +155,66 @@ function cmd_lint () { buildifier -mode=$mode $(find . -type f \( -name 'BUILD*' -o -name 'WORKSPACE*' -o -name '*.bzl' \)) } +function cmd_update_python_archives () { + if ! which jq > /dev/null; then + echo "🧱 No jq executable was found" + exit 1 + fi + if ! which curl > /dev/null; then + echo "🧱 No curl executable was found" + exit 1 + fi + + VERBOSE=0 + DRY_RUN=0 + for arg in "$@"; do + case "$arg" in + --verbose) + VERBOSE=1 + ;; + --dry-run) + DRY_RUN=1 + ;; + *) + echo "unknown option: $arg" + exit 1 + ;; + esac + done + export VERBOSE DRY_RUN + + source knife.d/update_python_archives.sh + old_versions=$(get_python_versions) + if ! out=$(generate_python_archives); then + case "$out" in + *MUTATED_PARTIAL*) + echo "update failed mid-mutation; files may be partially updated." >&2 + echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh MODULE.bazel.lock" >&2 + ;; + *) + echo "update failed; no files were mutated (pre-flight validation)." >&2 + ;; + esac + return 1 + fi + if [ "$out" = "NO_CHANGE" ]; then + echo "python archives are up to date" + return 0 + fi + if [ "$DRY_RUN" = 1 ]; then + echo "dry-run: no files written" + return 0 + fi + echo "updated python archives to release $out" + if ! update_test_versions_python "$old_versions"; then + echo "testdata update failed; extension/config.bzl were already updated." >&2 + echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh MODULE.bazel.lock" >&2 + return 1 + fi + # the extension changed: the lockfile digest is stale until bazel re-syncs it + echo "sync the lockfile: bazel mod deps --lockfile_mode=update" >&2 +} + function cmd_update_node_archives () { if ! which jq > /dev/null; then echo "🧱 No jq executable was found" @@ -278,8 +338,11 @@ deb-versions) update-node-archives) cmd_update_node_archives ;; +update-python-archives) + cmd_update_python_archives "${@:2}" + ;; ~~nocmd) # no command provided - echo "provide a command: lock, update-snapshots, github-update-snapshots, update-non-snapshots, test, deb-versions, update-node-archives" + echo "provide a command: lock, update-snapshots, github-update-snapshots, update-non-snapshots, test, deb-versions, update-node-archives, update-python-archives" exit 1 ;; *) # unknown command diff --git a/knife.d/BUILD.bazel b/knife.d/BUILD.bazel new file mode 100644 index 000000000..6f14faccf --- /dev/null +++ b/knife.d/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["update_python_archives.sh"]) diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh new file mode 100644 index 000000000..820117f8b --- /dev/null +++ b/knife.d/update_python_archives.sh @@ -0,0 +1,362 @@ +set -o pipefail -o errexit -o nounset + +# Copyright 2026 Google Inc. All rights reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Functions for updating python-build-standalone archives from the knife utility. +# +# Reads the python matrix (minors + archs) from //python:config.bzl and the +# current versions from //private/extensions:python.bzl, queries the latest PBS +# release, then rewrites the archive/version data in the extension (plus +# config.bzl and testdata when versions move). +# +# An update triggers when the CPython patch moves OR when the PBS release tag +# moves under an unchanged patch (a tag-only bump means the release rebuilt its +# bundled native libraries, e.g. OpenSSL/expat, for the same CPython version). + +PYTHON_TRIPLES=( + "amd64=x86_64-unknown-linux-gnu" + "arm64=aarch64-unknown-linux-gnu" + "s390x=s390x-unknown-linux-gnu" + "riscv64=riscv64-unknown-linux-gnu" +) + +# prints "_ " per matrix entry, one per line +function get_python_versions() { + sed -n '/python_versions_repo(/,/^ )$/p' private/extensions/python.bzl \ + | grep -oE '"[0-9]+\.[0-9]+_[a-z0-9]+": "[0-9]+\.[0-9]+\.[0-9]+"' \ + | sed -E 's/"([^"]+)": "([0-9.]+)"/\1 \2/' +} + +# prints the minors from the build matrix, one per line (e.g. 3.13, 3.14) +function get_python_minors() { + sed -n 's/^PYTHON_MAJOR_VERSIONS = \[\(.*\)\]$/\1/p' python/config.bzl \ + | grep -oE '"[0-9]+\.[0-9]+"' | tr -d '"' +} + +# prints archs for a minor from the build matrix, one per line +function get_python_archs() { + local minor="$1" + grep "\"${minor}\": \[" python/config.bzl \ + | grep -oE '"[a-z0-9]+"' | tr -d '"' +} + +function triple_for_arch() { + local entry + for entry in "${PYTHON_TRIPLES[@]}"; do + [[ "$entry" == "$1="* ]] && echo "${entry#*=}" && return 0 + done + return 1 +} + +# prints the current version for a minor+arch from the extension, if any +function current_version() { + get_python_versions | awk -v key="$1_$2" '$1 == key { print $2 }' +} + +# prints the full pinned version (patch + PBS release tag, e.g. "3.13.15+20260814") +# for a minor+arch from the extension's python_archive blocks, if any. +# A tag-only bump (same patch, rebuilt native libs) must trigger an update. +function pinned_version() { + local minor="$1" arch="$2" short + short=$(echo "$minor" | tr -d '.') + awk -v name="python${short}_${arch}" ' + $0 ~ ("name = \"" name "\"") { found = 1 } + found && /version = / { + sub(/.*version = "/, ""); sub(/".*/, ""); print; exit + } + ' private/extensions/python.bzl +} + +# rewrites the _python_impl data section of the extension (archives + versions + metadata) +# and, when a new stable minor appeared, extends the matrix in config.bzl. +# prints the new release tag on success, or "NO_CHANGE" when everything is current. +function generate_python_archives() { + local latest_release sha256sums + local minors minor arch triple version sha python_short arch_anchor matrix_min published latest_minor + PYTHON_MUTATED=0 + local -a archive_blocks versions_entries metadata_deps changes repos + local changed=0 verbose=${VERBOSE:-0} dry_run=${DRY_RUN:-0} current + + # PBS release data source: hermetic tests inject local fixtures via PBS_RELEASE_FILE + # + PBS_SHA256SUMS_FILE; otherwise the live latest-release.json / SHA256SUMS are used. + if [ -n "${PBS_RELEASE_FILE:-}" ] && [ -n "${PBS_SHA256SUMS_FILE:-}" ]; then + latest_release=$(sed -n 's/.*"tag"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$PBS_RELEASE_FILE" | head -1) + sha256sums=$(cat "$PBS_SHA256SUMS_FILE") + else + # PBS publishes a machine-readable latest-release.json (avoids GitHub API rate limits); + # fall back to the API when the raw file is unavailable. + latest_release=$(curl -sSL https://raw.githubusercontent.com/astral-sh/python-build-standalone/latest-release/latest-release.json 2>/dev/null | jq -r '.tag') + if [ -z "$latest_release" ] || [ "$latest_release" = "null" ]; then + latest_release=$(curl -sSL https://api.github.com/repos/astral-sh/python-build-standalone/releases/latest 2>/dev/null | jq -r '.tag_name') + fi + sha256sums=$(curl -sSL "https://github.com/astral-sh/python-build-standalone/releases/download/${latest_release}/SHA256SUMS") + fi + [ -n "$latest_release" ] && [ "$latest_release" != "null" ] || { echo "no latest PBS release found" >&2; exit 1; } + [ -n "$sha256sums" ] || { echo "no SHA256SUMS for ${latest_release}" >&2; exit 1; } + + local -a minors + minors=() + local minors_tmp archs_tmp + minors_tmp=$(mktemp) + get_python_minors > "$minors_tmp" + while IFS= read -r minor; do minors+=("$minor"); done < "$minors_tmp" + rm -f "$minors_tmp" + [ ${#minors[@]} -gt 0 ] || { echo "no minors parsed from python/config.bzl" >&2; exit 1; } + + # sort + dedupe internally: config.bzl order/typos must not leak through + minors=($(printf '%s\n' "${minors[@]}" | sort -uV)) + arch_anchor=${minors[${#minors[@]} - 1]} + matrix_min=${minors[0]} + + # all stable minors published by PBS (on amd64), sorted; rc/a/b excluded by + # requiring 3.X.Y. The matrix is a contiguous support window: the maintainer + # drops the tail (oldest minor goes EOL); the updater fills every published + # minor above the oldest one. + published=$(echo "$sha256sums" \ + | grep -oE 'cpython-3\.[0-9]+\.[0-9]+\+[0-9]+-x86_64-unknown-linux-gnu-install_only\.tar\.gz' \ + | sed -E 's/cpython-(3\.[0-9]+)\.[0-9]+.*/\1/' | sort -uV) + local fill fill_m + fill=() + for fill_m in $published; do + if [[ "$fill_m" > "$matrix_min" ]] && [[ " ${minors[*]} " != *" $fill_m "* ]]; then + fill+=("$fill_m") + fi + done + if [ ${#fill[@]} -gt 0 ]; then + echo "new stable minor(s) detected: ${fill[*]}" >&2 + for fill_m in "${fill[@]}"; do + minors+=("$fill_m") + changes+=("add minor ${fill_m}") + done + minors=($(printf '%s\n' "${minors[@]}" | sort -uV)) + # the matrix (config.bzl) is missing these minors; treat as a change even + # when the extension still carries the version data (e.g. removed from the + # matrix only, or a middle minor restored) + changed=1 + fi + latest_minor=$arch_anchor + + for minor in "${minors[@]}"; do + python_short=$(echo "$minor" | tr -d '.') + # a newly detected minor is not in config.bzl yet: reuse the previous minor's archs + archs_tmp=$(mktemp) + get_python_archs "$minor" > "$archs_tmp" + if [ ! -s "$archs_tmp" ]; then + get_python_archs "$latest_minor" > "$archs_tmp" + fi + [ -s "$archs_tmp" ] || { echo "no archs for ${minor} in python/config.bzl" >&2; exit 1; } + while IFS= read -r arch; do + triple=$(triple_for_arch "$arch") || { echo "no triple for ${arch}" >&2; exit 1; } + # latest stable patch for this minor+arch (3.X.Y; excludes rc/a/b) + version=$(echo "$sha256sums" \ + | grep -oE "cpython-${minor}\.[0-9]+\+${latest_release}-${triple}-install_only\.tar\.gz" \ + | sed -E "s/cpython-(${minor}\.[0-9]+)\+.*/\1/" | sort -V | tail -1) + [ -n "$version" ] || { echo "no ${minor} ${triple} install_only asset in ${latest_release}" >&2; exit 1; } + sha=$(echo "$sha256sums" | grep "cpython-${version}+${latest_release}-${triple}-install_only.tar.gz" | cut -d' ' -f1) + [ -n "$sha" ] || { echo "no sha for ${version} ${triple}" >&2; exit 1; } + + current=$(current_version "$minor" "$arch") + pinned=$(pinned_version "$minor" "$arch") + new_full="${version}+${latest_release}" + if [ "$verbose" = 1 ]; then + if [ "$current" = "$version" ] && [ "$pinned" = "$new_full" ]; then + echo " ${minor} ${arch}: ${pinned} (unchanged)" >&2 + else + echo " ${minor} ${arch}: ${pinned:-} -> ${new_full} (update)" >&2 + fi + fi + # an update is needed when the patch moves OR when the PBS release tag + # moves under an unchanged patch (native library rebuild) + if [ "$current" != "$version" ] || [ "$pinned" != "$new_full" ]; then + changed=1 + changes+=("update ${minor} ${arch}: ${pinned:-} -> ${new_full}") + fi + + archive_blocks+=(" python_archive( + name = \"python${python_short}_${arch}\", + sha256 = \"${sha}\", + strip_prefix = \"python/\", + urls = [\"https://github.com/astral-sh/python-build-standalone/releases/download/${latest_release}/cpython-${version}+${latest_release}-${triple}-install_only.tar.gz\"], + version = \"${version}+${latest_release}\", + python_version = \"${minor}\", + architecture = \"${arch}\", + control = \"//python:control\", + )") + versions_entries+=(" \"${minor}_${arch}\": \"${version}\",") + metadata_deps+=(" \"python${python_short}_${arch}\",") + repos+=("python${python_short}_${arch}") + done < "$archs_tmp" + rm -f "$archs_tmp" + done + + if [ "$dry_run" = 1 ]; then + if [ "$changed" = 0 ]; then + echo "NO_CHANGE" + else + printf 'would %s\n' "${changes[@]}" >&2 + echo "DRY_RUN" + fi + return 0 + fi + + if [ "$changed" = 0 ]; then + echo "NO_CHANGE" + return 0 + fi + + printf '%s\n' "${changes[@]}" >&2 + + local start end section tmp + start=$(grep -n '# Python from python-build-standalone' private/extensions/python.bzl | head -1 | cut -d: -f1) + end=$(grep -n '^_archive = tag_class' private/extensions/python.bzl | head -1 | cut -d: -f1) + local minors_list + minors_list=$(echo "${minors[*]}" | sed 's/ /, /g') + section=" # Python from python-build-standalone (https://github.com/astral-sh/python-build-standalone) + # Release ${latest_release}. Linux targets only (distroless images). + # Versions ${minors_list} (new minors added by update-python-archives when stable). + # NOTE: armv7 is intentionally absent: PBS publishes soft-float gnueabi builds + # (interpreter /lib/ld-linux.so.3) which cannot run on the distroless armhf base + # (loader /usr/lib/ld-linux-armhf.so.3); ppc64le is not published by PBS. +$(printf '%s\n\n' "${archive_blocks[@]}") + + python_versions_repo( + name = \"python_versions\", + versions = { +$(printf '%s\n' "${versions_entries[@]}") + }, + ) + + return module_ctx.extension_metadata( + root_module_direct_deps = [ + \"python_versions\", +$(printf '%s\n' "${metadata_deps[@]}") + ], + root_module_direct_dev_deps = [], + ) + +" + tmp=$(mktemp) + head -n $((start - 1)) private/extensions/python.bzl > "$tmp" + printf '%s' "$section" >> "$tmp" + tail -n +"$end" private/extensions/python.bzl >> "$tmp" + # pre-flight: the generated extension must be buildifier-clean BEFORE it replaces + # the committed file; a broken generator aborts here with nothing mutated. + if which buildifier >/dev/null 2>&1; then + if ! buildifier -mode=fix "$tmp"; then + echo "generated extension failed buildifier; aborting without changes" >&2 + rm -f "$tmp" + return 1 + fi + fi + # config.bzl: apply the seds to a temp copy and verify it BEFORE any real + # mutation, mirroring the extension; only verified content is mv'd in place. + local config_tmp="" + if [ ${#fill[@]} -gt 0 ]; then + local arch_list fill_entries fill_m2 + grep -q '^PYTHON_MAJOR_VERSIONS = ' python/config.bzl \ + || { echo "PYTHON_MAJOR_VERSIONS line not found in python/config.bzl" >&2; rm -f "$tmp"; return 1; } + grep -q "\"${latest_minor}\": \[" python/config.bzl \ + || { echo "arch map for ${latest_minor} not found in python/config.bzl" >&2; rm -f "$tmp"; return 1; } + arch_list=$(get_python_archs "$latest_minor" | sed 's/^/"/; s/$/",/' | tr '\n' ' ' | sed 's/ $//') + minors_quoted=$(printf '\"%s\", ' "${minors[@]}" | sed 's/, $//') + fill_entries="" + for fill_m2 in "${fill[@]}"; do + fill_entries="${fill_entries}\n \"${fill_m2}\": [${arch_list}]," + done + config_tmp=$(mktemp) + cp python/config.bzl "$config_tmp" + sed -i '' -e "s/^PYTHON_MAJOR_VERSIONS = .*/PYTHON_MAJOR_VERSIONS = [${minors_quoted}]/" "$config_tmp" + sed -i '' -e "s|\"${latest_minor}\": \[[^]]*\],|\"${latest_minor}\": [${arch_list}],${fill_entries}|" "$config_tmp" + for fill_m2 in "${fill[@]}"; do + grep -qE "PYTHON_MAJOR_VERSIONS = .*\"${fill_m2}\"" "$config_tmp" \ + || { echo "config.bzl update for ${fill_m2} did not land (format drift?)" >&2; rm -f "$config_tmp" "$tmp"; return 1; } + grep -q "\"${fill_m2}\": \[" "$config_tmp" \ + || { echo "config.bzl arch map for ${fill_m2} did not land" >&2; rm -f "$config_tmp" "$tmp"; return 1; } + done + # MODULE.bazel: the new minor's repos must be visible to the root module + local module_tmp repos_sorted + module_tmp=$(mktemp) + cp MODULE.bazel "$module_tmp" + repos_sorted=$(printf '"%s", ' $(printf '%s\n' "${repos[@]}" | sort) | sed 's/, $//') + sed -i '' -e "s/^use_repo(py, .*/use_repo(py, ${repos_sorted}, \"python_versions\")/" "$module_tmp" + for fill_m2 in "${fill[@]}"; do + grep -q "\"python$(echo "$fill_m2" | tr -d '.').*_" "$module_tmp" \ + || { echo "use_repo update for ${fill_m2} did not land" >&2; rm -f "$config_tmp" "$tmp" "$module_tmp"; return 1; } + done + fi + + # both mutations are now verified content; apply them. + PYTHON_MUTATED=1 + mv "$tmp" private/extensions/python.bzl || { echo "extension update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + if [ -n "$config_tmp" ]; then + mv "$config_tmp" python/config.bzl || { echo "config.bzl update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + mv "$module_tmp" MODULE.bazel || { echo "MODULE.bazel update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + fi + + # normalize formatting (repo convention; no-op when buildifier is missing). + # This runs post-mutation, so a failure must be labeled mid-mutation. + if which buildifier >/dev/null 2>&1; then + if ! buildifier -mode=fix private/extensions/python.bzl python/config.bzl; then + echo "buildifier failed on the updated files" >&2 + echo "MUTATED_PARTIAL" + return 1 + fi + fi + + echo "$latest_release" +} + +# All fallible steps run against temp files and are verified before anything is +# replaced: a failed run mutates nothing (RED). A successful run leaves +# MODULE.bazel.lock stale until refreshed (bazel mod deps --lockfile_mode=update; +# CI enforces --lockfile_mode=error). + +# bumps the version strings in the version-specific testdata yamls (python3.13.yaml etc.) +# and creates the yaml for a newly added minor. $1: a snapshot of get_python_versions +# taken before generate_python_archives. +function update_test_versions_python() { + local old_snapshot minor file old new expected + old_snapshot=$1 + # keep the hermetic smoke test's expected minors in sync with the matrix + expected=$(get_python_minors | tr '\n' ' ') + sed -i '' -e "s/\"\$minors\" = \"[^\"]*\"/\"\$minors\" = \"${expected}\"/" python/update_python_archives_test.sh + for minor in $(get_python_minors); do + file="python/testdata/python3.$(echo "$minor" | cut -d. -f2).yaml" + new=$(current_version "$minor" "amd64") + if [ ! -f "$file" ]; then + [ -n "$new" ] || continue + cat > "$file" <&2; return 1; } + echo "bumped $file to ${new}" + fi + done +} diff --git a/private/extensions/BUILD.bazel b/private/extensions/BUILD.bazel index e69de29bb..648b61304 100644 --- a/private/extensions/BUILD.bazel +++ b/private/extensions/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["python.bzl"]) diff --git a/python/BUILD b/python/BUILD index 1cdca39ee..76a111481 100644 --- a/python/BUILD +++ b/python/BUILD @@ -5,6 +5,34 @@ load(":python.bzl", "python_image", "python_image_index") package(default_visibility = ["//visibility:public"]) +sh_test( + name = "update_python_archives_test", + srcs = ["update_python_archives_test.sh"], + data = [ + ":config.bzl", + "//:MODULE.bazel", + "//knife.d:update_python_archives.sh", + "//private/extensions:python.bzl", + ], +) + +# Black-box contract test: after a new PBS release every file needed to build the +# new images must update consistently (extension, config.bzl, MODULE.bazel, +# testdata). Offline via PBS_RELEASE_FILE/PBS_SHA256SUMS_FILE fixtures. +sh_test( + name = "update_python_archives_contract_test", + srcs = ["update_python_archives_contract_test.sh"], + data = [ + ":config.bzl", + ":update_python_archives_test.sh", + "testdata/python3.13.yaml", + "testdata/python3.14.yaml", + "//:MODULE.bazel", + "//knife.d:update_python_archives.sh", + "//private/extensions:python.bzl", + ], +) + [ python_image( arch = arch, diff --git a/python/update_python_archives_contract_test.sh b/python/update_python_archives_contract_test.sh new file mode 100755 index 000000000..9e87d4b90 --- /dev/null +++ b/python/update_python_archives_contract_test.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Black-box contract test for the python updater (knife.d/update_python_archives.sh). +# +# Contract: after a NEW PBS release, the updater must rewrite ALL files needed to +# build the new images consistently and converge: +# - private/extensions/python.bzl: archive blocks (url/sha/version incl. release tag) +# AND the python_versions_repo dict (patch bumps) +# - python/config.bzl: matrix extension when a new stable minor appears +# - MODULE.bazel: use_repo gains the new minor's archives +# - python/testdata/python3.X.yaml: version strings (patch bumps) / new yaml (new minor) +# - a second run is a NO_CHANGE no-op (the state is the updater's fixed point) +# Runs fully offline; fixture release data is injected via PBS_RELEASE_FILE / +# PBS_SHA256SUMS_FILE. +set -euo pipefail + +cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" + +FIX=$(mktemp -d) +trap 'rm -rf "$FIX"' EXIT + +# fixture workspace: every file the updater mutates, copied verbatim from the tree +mkdir -p "$FIX/private/extensions" "$FIX/python/testdata" +cp knife.d/update_python_archives.sh "$FIX/" +cp private/extensions/python.bzl "$FIX/private/extensions/" +cp python/config.bzl "$FIX/python/" +cp MODULE.bazel "$FIX/" +cp python/update_python_archives_test.sh "$FIX/python/" +cp python/testdata/python3.13.yaml python/testdata/python3.14.yaml "$FIX/python/testdata/" + +cd "$FIX" +source update_python_archives.sh + +# --- fixtures --------------------------------------------------------------- +# 64-hex fake shas: the updater only copies them, nothing validates them here. +SHA_A=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +SHA_B=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +SHA_C=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +SHA_D=dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd +TRIPLES=(x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu s390x-unknown-linux-gnu riscv64-unknown-linux-gnu) + +make_sha256sums() { # $1=release $2=patch313 $3=patch314 $4=patch315 ("" = no 3.15 yet) + local release=$1 p313=$2 p314=$3 p315=${4:-} + : > SHA256SUMS + for t in "${TRIPLES[@]}"; do + echo "$SHA_A cpython-${p313}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + echo "$SHA_B cpython-${p314}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + if [ -n "$p315" ]; then + echo "$SHA_C cpython-${p315}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + fi + done +} + +run_updater() { # prints stdout; fails the test on a non-zero exit + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + generate_python_archives 2>"$FIX/updater.err" +} + +# --- phase A: tag-only bump (new release, same patches) ----------------------- +# the maintainer-reported gap: 20260814 -> 20260815 with unchanged CPython versions +echo '{"tag": "20990101"}' > release.json +make_sha256sums 20990101 3.13.15 3.14.7 +[ "$(run_updater)" = "20990101" ] || { echo "phase A: expected update to 20990101"; cat "$FIX/updater.err"; exit 1; } +grep -q 'releases/download/20990101/' private/extensions/python.bzl || { echo "phase A: archives not re-pointed at new release"; exit 1; } +! grep -q '20260807' private/extensions/python.bzl || { echo "phase A: old release still referenced"; exit 1; } +grep -q '3.13.15+20990101' private/extensions/python.bzl || { echo "phase A: archive version lacks new release tag"; exit 1; } +grep -q '"3.13_amd64": "3.13.15"' private/extensions/python.bzl || { echo "phase A: versions dict must not change on a tag-only bump"; exit 1; } +grep -q 'Python 3.13.15' python/testdata/python3.13.yaml || { echo "phase A: testdata must not change on a tag-only bump"; exit 1; } +[ "$(run_updater)" = "NO_CHANGE" ] || { echo "phase A: second run must be NO_CHANGE"; cat "$FIX/updater.err"; exit 1; } + +# --- phase B: patch bump (new release, new patches) --------------------------- +snap_b=$(get_python_versions) +echo '{"tag": "20990102"}' > release.json +make_sha256sums 20990102 3.13.16 3.14.8 +[ "$(run_updater)" = "20990102" ] || { echo "phase B: expected update to 20990102"; cat "$FIX/updater.err"; exit 1; } +grep -q '3.13.16+20990102' private/extensions/python.bzl || { echo "phase B: archive version not bumped"; exit 1; } +grep -q '"3.13_amd64": "3.13.16"' private/extensions/python.bzl || { echo "phase B: versions dict not bumped"; exit 1; } +update_test_versions_python "$snap_b" +grep -q 'Python 3.13.16' python/testdata/python3.13.yaml || { echo "phase B: testdata 3.13 not bumped"; exit 1; } +grep -q 'Python 3.14.8' python/testdata/python3.14.yaml || { echo "phase B: testdata 3.14 not bumped"; exit 1; } + +# --- phase C: new stable minor fill (3.15 appears upstream) ------------------- +snap_c=$(get_python_versions) +echo '{"tag": "20990103"}' > release.json +make_sha256sums 20990103 3.13.16 3.14.8 3.15.0 +[ "$(run_updater)" = "20990103" ] || { echo "phase C: expected update to 20990103"; cat "$FIX/updater.err"; exit 1; } +grep -q '"3.15"' python/config.bzl || { echo "phase C: 3.15 missing from PYTHON_MAJOR_VERSIONS"; exit 1; } +grep -q '"3.15": \[' python/config.bzl || { echo "phase C: 3.15 arch map missing from config.bzl"; exit 1; } +grep -q 'python315_amd64' MODULE.bazel || { echo "phase C: use_repo not extended with python315 repos"; exit 1; } +grep -q 'python315_amd64' private/extensions/python.bzl || { echo "phase C: 3.15 archive missing from extension"; exit 1; } +grep -q '3.15.0+20990103' private/extensions/python.bzl || { echo "phase C: 3.15 archive version wrong"; exit 1; } +update_test_versions_python "$snap_c" +[ -f python/testdata/python3.15.yaml ] || { echo "phase C: python3.15.yaml not created"; exit 1; } +grep -q 'Python 3.15.0' python/testdata/python3.15.yaml || { echo "phase C: python3.15.yaml version wrong"; exit 1; } + +# --- convergence + structural consistency ------------------------------------- +[ "$(run_updater)" = "NO_CHANGE" ] || { echo "final: expected NO_CHANGE"; cat "$FIX/updater.err"; exit 1; } +for minor in $(get_python_minors); do + for arch in $(get_python_archs "$minor"); do + v=$(current_version "$minor" "$arch") + [ -n "$v" ] || { echo "final: no versions-dict entry for ${minor}_${arch}"; exit 1; } + p=$(pinned_version "$minor" "$arch") + [ -n "$p" ] || { echo "final: no archive pin for ${minor}_${arch}"; exit 1; } + case "$p" in + "$v"+[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]) ;; + *) echo "final: pin for ${minor}_${arch} must be '+', got [$p]"; exit 1 ;; + esac + done +done + +echo "update_python_archives contract OK (tag-only bump, patch bump, new minor, convergence)" diff --git a/python/update_python_archives_test.sh b/python/update_python_archives_test.sh new file mode 100755 index 000000000..d9e05128a --- /dev/null +++ b/python/update_python_archives_test.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Hermetic smoke test for the python updater's parser functions. +# Verifies the committed matrix (python/config.bzl) and the extension +# (private/extensions/python.bzl) stay consistent - no network, no fixtures. +set -euo pipefail + +cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" + +source knife.d/update_python_archives.sh + +minors=$(get_python_minors | tr '\n' ' ') +[ "$minors" = "3.13 3.14 " ] || { echo "unexpected minors: [$minors]"; exit 1; } + +for minor in $(get_python_minors); do + for arch in $(get_python_archs "$minor"); do + version=$(current_version "$minor" "$arch") + [ -n "$version" ] || { echo "missing version for ${minor}_${arch}"; exit 1; } + # the archive pin must carry the PBS release tag: the updater keys change + # detection on it, so a tag-only bump (native library rebuild under the same + # CPython patch) must never be silently dropped. + pinned=$(pinned_version "$minor" "$arch") + [ -n "$pinned" ] || { echo "missing pinned version for ${minor}_${arch}"; exit 1; } + case "$pinned" in + "$version"+[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]) ;; + *) echo "pinned version for ${minor}_${arch} must be '+', got: [$pinned]"; exit 1 ;; + esac + done +done + +echo "update_python_archives parsing OK" From bb9d71c630597d080b65761d284cb920b0db5f8b Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:04:04 +0200 Subject: [PATCH 3/9] feat: updater generates PBS SBOM (python/pbs-sbom.spdx.json) The update now also regenerates an SPDX 2.3 SBOM from the release's component manifest (pythonbuild/downloads.py): cpython versions, bundled pip/setuptools and the statically embedded native libraries, each with version, license, sha256 and download URL. Build-time-only tools are excluded. Covered by the contract test (SBOM regenerated on tag/patch/new-minor updates, stable on no-op runs). --- knife | 4 +- knife.d/update_python_archives.sh | 26 + python/BUILD | 1 + python/README.md | 2 + python/gen_pbs_sbom.py | 93 +++ python/pbs-sbom.spdx.json | 647 ++++++++++++++++++ .../update_python_archives_contract_test.sh | 29 + 7 files changed, 800 insertions(+), 2 deletions(-) create mode 100644 python/gen_pbs_sbom.py create mode 100644 python/pbs-sbom.spdx.json diff --git a/knife b/knife index 72bf4cbec..ccecf4ead 100755 --- a/knife +++ b/knife @@ -189,7 +189,7 @@ function cmd_update_python_archives () { case "$out" in *MUTATED_PARTIAL*) echo "update failed mid-mutation; files may be partially updated." >&2 - echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh MODULE.bazel.lock" >&2 + echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh python/pbs-sbom.spdx.json MODULE.bazel.lock" >&2 ;; *) echo "update failed; no files were mutated (pre-flight validation)." >&2 @@ -208,7 +208,7 @@ function cmd_update_python_archives () { echo "updated python archives to release $out" if ! update_test_versions_python "$old_versions"; then echo "testdata update failed; extension/config.bzl were already updated." >&2 - echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh MODULE.bazel.lock" >&2 + echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh python/pbs-sbom.spdx.json MODULE.bazel.lock" >&2 return 1 fi # the extension changed: the lockfile digest is stale until bazel re-syncs it diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh index 820117f8b..c34fb33d9 100644 --- a/knife.d/update_python_archives.sh +++ b/knife.d/update_python_archives.sh @@ -216,6 +216,31 @@ function generate_python_archives() { return 0 fi + # PBS SBOM: regenerate python/pbs-sbom.spdx.json from the release's component + # manifest (pythonbuild/downloads.py at the release tag); hermetic tests inject + # a local copy via PBS_DOWNLOADS_FILE. + local downloads_file sbom_tmp downloads_tmp + if [ -n "${PBS_DOWNLOADS_FILE:-}" ]; then + downloads_file="$PBS_DOWNLOADS_FILE" + else + downloads_tmp=$(mktemp) + if ! curl -sSL "https://raw.githubusercontent.com/astral-sh/python-build-standalone/${latest_release}/pythonbuild/downloads.py" -o "$downloads_tmp"; then + echo "no PBS downloads manifest for ${latest_release}" >&2 + rm -f "$downloads_tmp" + exit 1 + fi + downloads_file="$downloads_tmp" + fi + [ -s "$downloads_file" ] || { echo "empty PBS downloads manifest" >&2; exit 1; } + sbom_tmp=$(mktemp) + if ! python3 python/gen_pbs_sbom.py "$downloads_file" "$latest_release" "$sbom_tmp"; then + echo "PBS SBOM generation failed" >&2 + rm -f "$sbom_tmp" "$downloads_tmp" + exit 1 + fi + grep -q "$latest_release" "$sbom_tmp" || { echo "PBS SBOM does not mention ${latest_release}" >&2; rm -f "$sbom_tmp" "$downloads_tmp"; exit 1; } + [ -n "${PBS_DOWNLOADS_FILE:-}" ] || rm -f "$downloads_tmp" + printf '%s\n' "${changes[@]}" >&2 local start end section tmp @@ -300,6 +325,7 @@ $(printf '%s\n' "${metadata_deps[@]}") # both mutations are now verified content; apply them. PYTHON_MUTATED=1 mv "$tmp" private/extensions/python.bzl || { echo "extension update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + mv "$sbom_tmp" python/pbs-sbom.spdx.json || { echo "SBOM update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } if [ -n "$config_tmp" ]; then mv "$config_tmp" python/config.bzl || { echo "config.bzl update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } mv "$module_tmp" MODULE.bazel || { echo "MODULE.bazel update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } diff --git a/python/BUILD b/python/BUILD index 76a111481..54f4e5765 100644 --- a/python/BUILD +++ b/python/BUILD @@ -24,6 +24,7 @@ sh_test( srcs = ["update_python_archives_contract_test.sh"], data = [ ":config.bzl", + ":gen_pbs_sbom.py", ":update_python_archives_test.sh", "testdata/python3.13.yaml", "testdata/python3.14.yaml", diff --git a/python/README.md b/python/README.md index e7b91e11a..454892494 100644 --- a/python/README.md +++ b/python/README.md @@ -66,3 +66,5 @@ Python comes from a [python-build-standalone](https://github.com/astral-sh/pytho prebuilt tarball (`cpython-3.X.Y+--unknown-linux-gnu-install_only.tar.gz`). The exact release, versions and shas are pinned in `//private/extensions:python.bzl` and updated via `knife update-python-archives` (see `knife.d/update_python_archives.sh`). +A generated SPDX SBOM for the pinned release is committed at `python/pbs-sbom.spdx.json` +(regenerated by the updater; lists the statically embedded native libraries with versions). diff --git a/python/gen_pbs_sbom.py b/python/gen_pbs_sbom.py new file mode 100644 index 000000000..44f42ed53 --- /dev/null +++ b/python/gen_pbs_sbom.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Emit an SPDX 2.3 SBOM for a python-build-standalone release. + +Reads the release's component manifest (pythonbuild/downloads.py) and lists +the runtime components: the CPython versions, bundled pip/setuptools, and the +native libraries statically linked into the interpreters. Build-time-only +tools (autoconf, binutils, musl, llvm, ...) are excluded. + +Usage: gen_pbs_sbom.py +""" +import datetime +import importlib.util +import json +import sys + +LICENSE_REF = {"bzip2-1.0.6": "LicenseRef-bzip2-1.0.6"} + + +def declared(entry): + ids = entry.get("licenses") or [] + out = [LICENSE_REF.get(i) or i for i in ids] + if not ids and entry.get("library_names"): + out = ["LicenseRef-Public-Domain"] # sqlite: no SPDX id for public domain + return " AND ".join(out) if out else "NOASSERTION" + + +def is_runtime(name, entry): + return name.startswith("cpython") or name in ("pip", "setuptools") or bool(entry.get("library_names")) + + +def main(): + manifest, release, output = sys.argv[1], sys.argv[2], sys.argv[3] + spec = importlib.util.spec_from_file_location("pbs_downloads", manifest) + if spec is None or spec.loader is None: + sys.exit("cannot load manifest: " + manifest) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + downloads = module.DOWNLOADS + + doc = { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "python-build-standalone-{}-linux".format(release), + "documentNamespace": "https://github.com/astral-sh/python-build-standalone/releases/tag/{}/spdx.json".format(release), + "creationInfo": { + "created": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "creators": ["Tool: distroless-pbs-sbom (python/gen_pbs_sbom.py)"], + }, + "packages": [ + { + "name": "python-build-standalone", + "SPDXID": "SPDXRef-PBS", + "versionInfo": release, + "downloadLocation": "https://github.com/astral-sh/python-build-standalone/releases/tag/{}".format(release), + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "supplier": "Organization: astral-sh", + "filesAnalyzed": False, + } + ], + "relationships": [], + } + for name, entry in sorted(downloads.items()): + if not is_runtime(name, entry): + continue + pid = "SPDXRef-" + name.replace("-", "_").replace(".", "_") + doc["packages"].append( + { + "name": name, + "SPDXID": pid, + "versionInfo": str(entry.get("version", "")), + "downloadLocation": entry.get("url", "NOASSERTION"), + "licenseConcluded": "NOASSERTION", + "licenseDeclared": declared(entry), + "checksums": [{"algorithm": "SHA256", "checksumValue": entry.get("sha256", "")}], + "filesAnalyzed": False, + } + ) + doc["relationships"].append( + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": pid, + } + ) + with open(output, "w") as fh: + json.dump(doc, fh, indent=1) + fh.write("\n") + + +if __name__ == "__main__": + main() diff --git a/python/pbs-sbom.spdx.json b/python/pbs-sbom.spdx.json new file mode 100644 index 000000000..95ec61873 --- /dev/null +++ b/python/pbs-sbom.spdx.json @@ -0,0 +1,647 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "python-build-standalone-20260814-linux", + "documentNamespace": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260814/spdx.json", + "creationInfo": { + "created": "2026-08-19T10:03:07Z", + "creators": [ + "Tool: distroless-pbs-sbom (python/gen_pbs_sbom.py)" + ] + }, + "packages": [ + { + "name": "python-build-standalone", + "SPDXID": "SPDXRef-PBS", + "versionInfo": "20260814", + "downloadLocation": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260814", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "supplier": "Organization: astral-sh", + "filesAnalyzed": false + }, + { + "name": "bdb", + "SPDXID": "SPDXRef-bdb", + "versionInfo": "6.0.19", + "downloadLocation": "https://ftp.osuosl.org/pub/blfs/conglomeration/db/db-6.0.19.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Sleepycat", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "2917c28f60903908c2ca4587ded1363b812c4e830a5326aaa77c9879d13ae18e" + } + ], + "filesAnalyzed": false + }, + { + "name": "bzip2", + "SPDXID": "SPDXRef-bzip2", + "versionInfo": "1.0.8", + "downloadLocation": "https://astral-sh.github.io/mirror/files/bzip2-1.0.8.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "LicenseRef-bzip2-1.0.6", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.10", + "SPDXID": "SPDXRef-cpython_3_10", + "versionInfo": "3.10.21", + "downloadLocation": "https://www.python.org/ftp/python/3.10.21/Python-3.10.21.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a0da1e72132e950154eca0f6f47d5db828454700de20e5113667940d81e0db04" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.11", + "SPDXID": "SPDXRef-cpython_3_11", + "versionInfo": "3.11.16", + "downloadLocation": "https://www.python.org/ftp/python/3.11.16/Python-3.11.16.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "91bcdebfdde239a003ae93738a7fce0f9230fee5c4bc2b86f6e6e8c6f98aabe8" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.12", + "SPDXID": "SPDXRef-cpython_3_12", + "versionInfo": "3.12.14", + "downloadLocation": "https://www.python.org/ftp/python/3.12.14/Python-3.12.14.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "5c8462af5790baf43a321a1559dbe0db06d1be4300fb85fb53c40060668e548a" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.13", + "SPDXID": "SPDXRef-cpython_3_13", + "versionInfo": "3.13.15", + "downloadLocation": "https://www.python.org/ftp/python/3.13.15/Python-3.13.15.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "1e66a7945a48390ee4c2a4268a0e4185884059a13c4aab6d148aa208deea4a76" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.14", + "SPDXID": "SPDXRef-cpython_3_14", + "versionInfo": "3.14.7", + "downloadLocation": "https://www.python.org/ftp/python/3.14.7/Python-3.14.7.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "3b48dac8fb59f62eaa67ac83c1eb12bda1b7a08406dd286e252c11a66be27f81" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.15", + "SPDXID": "SPDXRef-cpython_3_15", + "versionInfo": "3.15.0rc1", + "downloadLocation": "https://www.python.org/ftp/python/3.15.0/Python-3.15.0rc1.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "f84dad680ab2147417d2739355c2678f0f9acffe4ae8ef77895de1454b384b07" + } + ], + "filesAnalyzed": false + }, + { + "name": "expat", + "SPDXID": "SPDXRef-expat", + "versionInfo": "2.8.3", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_3/expat-2.8.3.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "f6256df90c906773d344da084402b7d3e4f22ed41b1a59c989098a83d3ea0c85" + } + ], + "filesAnalyzed": false + }, + { + "name": "libX11", + "SPDXID": "SPDXRef-libX11", + "versionInfo": "1.8.13", + "downloadLocation": "https://www.x.org/releases/individual/lib/libX11-1.8.13.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT AND X11", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "acf0e7cd7541110e6330ecb539441a2d53061f386ec7be6906dfde0de2598470" + } + ], + "filesAnalyzed": false + }, + { + "name": "libXau", + "SPDXID": "SPDXRef-libXau", + "versionInfo": "1.0.12", + "downloadLocation": "https://www.x.org/releases/individual/lib/libXau-1.0.12.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "2402dd938da4d0a332349ab3d3586606175e19cb32cb9fe013c19f1dc922dcee" + } + ], + "filesAnalyzed": false + }, + { + "name": "libedit", + "SPDXID": "SPDXRef-libedit", + "versionInfo": "20240808-3.1", + "downloadLocation": "https://thrysoee.dk/editline/libedit-20240808-3.1.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "BSD-3-Clause", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "5f0573349d77c4a48967191cdd6634dd7aa5f6398c6a57fe037cc02696d6099f" + } + ], + "filesAnalyzed": false + }, + { + "name": "libffi", + "SPDXID": "SPDXRef-libffi", + "versionInfo": "3.4.6", + "downloadLocation": "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e" + } + ], + "filesAnalyzed": false + }, + { + "name": "libffi-3.3", + "SPDXID": "SPDXRef-libffi_3_3", + "versionInfo": "3.3", + "downloadLocation": "https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "72fba7922703ddfa7a028d513ac15a85c8d54c8d67f55fa5a4802885dc652056" + } + ], + "filesAnalyzed": false + }, + { + "name": "libxcb", + "SPDXID": "SPDXRef-libxcb", + "versionInfo": "1.17.0", + "downloadLocation": "https://xcb.freedesktop.org/dist/libxcb-1.17.0.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "2c69287424c9e2128cb47ffe92171e10417041ec2963bceafb65cb3fcf8f0b85" + } + ], + "filesAnalyzed": false + }, + { + "name": "mpdecimal", + "SPDXID": "SPDXRef-mpdecimal", + "versionInfo": "4.0.0", + "downloadLocation": "https://astral-sh.github.io/mirror/files/mpdecimal-4.0.0.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "BSD-2-Clause", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c" + } + ], + "filesAnalyzed": false + }, + { + "name": "ncurses", + "SPDXID": "SPDXRef-ncurses", + "versionInfo": "6.5", + "downloadLocation": "https://astral-sh.github.io/mirror/files/ncurses-6.5.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "X11", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "136d91bc269a9a5785e5f9e980bc76ab57428f604ce3e5a5a90cebc767971cc6" + } + ], + "filesAnalyzed": false + }, + { + "name": "openssl-1.1", + "SPDXID": "SPDXRef-openssl_1_1", + "versionInfo": "1.1.1w", + "downloadLocation": "https://www.openssl.org/source/openssl-1.1.1w.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "OpenSSL", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8" + } + ], + "filesAnalyzed": false + }, + { + "name": "openssl-3.5", + "SPDXID": "SPDXRef-openssl_3_5", + "versionInfo": "3.5.7", + "downloadLocation": "https://github.com/openssl/openssl/releases/download/openssl-3.5.7/openssl-3.5.7.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Apache-2.0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a8c0d28a529ca480f9f36cf5792e2cd21984552a3c8e4aa11a24aa31aeac98e8" + } + ], + "filesAnalyzed": false + }, + { + "name": "pip", + "SPDXID": "SPDXRef-pip", + "versionInfo": "26.2.1", + "downloadLocation": "https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e" + } + ], + "filesAnalyzed": false + }, + { + "name": "setuptools", + "SPDXID": "SPDXRef-setuptools", + "versionInfo": "82.0.1", + "downloadLocation": "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb" + } + ], + "filesAnalyzed": false + }, + { + "name": "sqlite", + "SPDXID": "SPDXRef-sqlite", + "versionInfo": "3530100", + "downloadLocation": "https://www.sqlite.org/2026/sqlite-autoconf-3530100.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "LicenseRef-Public-Domain", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "83e6b2020a034e9a7ad4a72feea59e1ad52f162e09cbd26735a3ffb98359fc4f" + } + ], + "filesAnalyzed": false + }, + { + "name": "tcl", + "SPDXID": "SPDXRef-tcl", + "versionInfo": "9.0.4", + "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tcl9.0.4-src.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "TCL", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d0aed49230bc02a65c1e0229e65f34590a4b037ec40d546f32573b467f7551ea" + } + ], + "filesAnalyzed": false + }, + { + "name": "tcl-8612", + "SPDXID": "SPDXRef-tcl_8612", + "versionInfo": "8.6.12", + "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tcl8.6.12-src.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "TCL", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "26c995dd0f167e48b11961d891ee555f680c175f7173ff8cb829f4ebcde4c1a6" + } + ], + "filesAnalyzed": false + }, + { + "name": "tk", + "SPDXID": "SPDXRef-tk", + "versionInfo": "9.0.4", + "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tk9.0.4-src.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "TCL", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d7a146d2917eb8b5cc95276dbf0e3d03c7464d2b19c1675357857c989301dbb4" + } + ], + "filesAnalyzed": false + }, + { + "name": "tk-8612", + "SPDXID": "SPDXRef-tk_8612", + "versionInfo": "8.6.12", + "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tk8.6.12-src.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "TCL", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "12395c1f3fcb6bed2938689f797ea3cdf41ed5cb6c4766eec8ac949560310630" + } + ], + "filesAnalyzed": false + }, + { + "name": "uuid", + "SPDXID": "SPDXRef-uuid", + "versionInfo": "1.0.3", + "downloadLocation": "https://sourceforge.net/projects/libuuid/files/libuuid-1.0.3.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "BSD-3-Clause", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "46af3275291091009ad7f1b899de3d0cea0252737550e7919d17237997db5644" + } + ], + "filesAnalyzed": false + }, + { + "name": "xz", + "SPDXID": "SPDXRef-xz", + "versionInfo": "5.8.3", + "downloadLocation": "https://github.com/tukaani-project/xz/releases/download/v5.8.3/xz-5.8.3.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "0BSD", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "3d3a1b973af218114f4f889bbaa2f4c037deaae0c8e815eec381c3d546b974a0" + } + ], + "filesAnalyzed": false + }, + { + "name": "zlib", + "SPDXID": "SPDXRef-zlib", + "versionInfo": "1.3.2", + "downloadLocation": "https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Zlib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "bb329a0a2cd0274d05519d61c667c062e06990d72e125ee2dfa8de64f0119d16" + } + ], + "filesAnalyzed": false + }, + { + "name": "zlib-ng", + "SPDXID": "SPDXRef-zlib_ng", + "versionInfo": "2.2.4", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Zlib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004" + } + ], + "filesAnalyzed": false + }, + { + "name": "zstd", + "SPDXID": "SPDXRef-zstd", + "versionInfo": "1.5.7", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "BSD-3-Clause", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e" + } + ], + "filesAnalyzed": false + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-bdb" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-bzip2" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_10" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_11" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_12" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_13" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_14" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_15" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-expat" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libX11" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libXau" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libedit" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libffi" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libffi_3_3" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libxcb" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-mpdecimal" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-ncurses" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-openssl_1_1" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-openssl_3_5" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-pip" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-setuptools" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-sqlite" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-tcl" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-tcl_8612" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-tk" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-tk_8612" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-uuid" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-xz" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-zlib" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-zlib_ng" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-zstd" + } + ] +} diff --git a/python/update_python_archives_contract_test.sh b/python/update_python_archives_contract_test.sh index 9e87d4b90..ef6a42df8 100755 --- a/python/update_python_archives_contract_test.sh +++ b/python/update_python_archives_contract_test.sh @@ -25,8 +25,32 @@ cp private/extensions/python.bzl "$FIX/private/extensions/" cp python/config.bzl "$FIX/python/" cp MODULE.bazel "$FIX/" cp python/update_python_archives_test.sh "$FIX/python/" +cp python/gen_pbs_sbom.py "$FIX/python/" cp python/testdata/python3.13.yaml python/testdata/python3.14.yaml "$FIX/python/testdata/" +# fake PBS component manifest (pythonbuild/downloads.py) for the SBOM step +cat > "$FIX/downloads.py" <<'EOF' +DOWNLOADS = { + "cpython-3.14": { + "url": "https://www.python.org/ftp/python/3.14.7/Python-3.14.7.tar.xz", + "version": "3.14.7", + "licenses": ["Python-2.0"], + }, + "expat": { + "url": "https://example.invalid/expat.tar.gz", + "version": "2.8.3", + "licenses": ["MIT"], + "library_names": ["expat"], + }, + "zlib": { + "url": "https://example.invalid/zlib.tar.gz", + "version": "1.3.2", + "licenses": ["Zlib"], + "library_names": ["z"], + }, +} +EOF + cd "$FIX" source update_python_archives.sh @@ -52,6 +76,7 @@ make_sha256sums() { # $1=release $2=patch313 $3=patch314 $4=patch315 ("" = no 3. run_updater() { # prints stdout; fails the test on a non-zero exit PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_DOWNLOADS_FILE="$FIX/downloads.py" \ generate_python_archives 2>"$FIX/updater.err" } @@ -66,6 +91,8 @@ grep -q '3.13.15+20990101' private/extensions/python.bzl || { echo "phase A: arc grep -q '"3.13_amd64": "3.13.15"' private/extensions/python.bzl || { echo "phase A: versions dict must not change on a tag-only bump"; exit 1; } grep -q 'Python 3.13.15' python/testdata/python3.13.yaml || { echo "phase A: testdata must not change on a tag-only bump"; exit 1; } [ "$(run_updater)" = "NO_CHANGE" ] || { echo "phase A: second run must be NO_CHANGE"; cat "$FIX/updater.err"; exit 1; } +grep -q '20990101' python/pbs-sbom.spdx.json || { echo "phase A: SBOM not regenerated for the new release"; exit 1; } +grep -q '"expat"' python/pbs-sbom.spdx.json || { echo "phase A: SBOM missing bundled component"; exit 1; } # --- phase B: patch bump (new release, new patches) --------------------------- snap_b=$(get_python_versions) @@ -77,6 +104,7 @@ grep -q '"3.13_amd64": "3.13.16"' private/extensions/python.bzl || { echo "phase update_test_versions_python "$snap_b" grep -q 'Python 3.13.16' python/testdata/python3.13.yaml || { echo "phase B: testdata 3.13 not bumped"; exit 1; } grep -q 'Python 3.14.8' python/testdata/python3.14.yaml || { echo "phase B: testdata 3.14 not bumped"; exit 1; } +grep -q '20990102' python/pbs-sbom.spdx.json || { echo "phase B: SBOM not regenerated"; exit 1; } # --- phase C: new stable minor fill (3.15 appears upstream) ------------------- snap_c=$(get_python_versions) @@ -94,6 +122,7 @@ grep -q 'Python 3.15.0' python/testdata/python3.15.yaml || { echo "phase C: pyth # --- convergence + structural consistency ------------------------------------- [ "$(run_updater)" = "NO_CHANGE" ] || { echo "final: expected NO_CHANGE"; cat "$FIX/updater.err"; exit 1; } +grep -q '20990103' python/pbs-sbom.spdx.json || { echo "final: SBOM must stay on the last release"; exit 1; } for minor in $(get_python_minors); do for arch in $(get_python_archs "$minor"); do v=$(current_version "$minor" "$arch") From cba59e315ae50849af8d506820a512223726f95d Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:19:37 +0200 Subject: [PATCH 4/9] feat: PBS SBOM gets purls, pip trivy-scannable The SBOM entries now carry purls: pkg:pypi for pip (scannable by trivy, license MIT from the pypi fallback map) and pkg:generic for the native libraries. Versions prefer the manifest's actual_version (sqlite shipped its SQLITE_VERSION_NUMBER, 3530100, instead of 3.53.1.0), and setuptools is no longer listed: install_only tarballs do not ship it, it is a PBS build-time tool. --- python/README.md | 5 + python/gen_pbs_sbom.py | 33 +++++- python/pbs-sbom.spdx.json | 232 ++++++++++++++++++++++++++++++++++---- 3 files changed, 245 insertions(+), 25 deletions(-) diff --git a/python/README.md b/python/README.md index 454892494..69649e612 100644 --- a/python/README.md +++ b/python/README.md @@ -68,3 +68,8 @@ The exact release, versions and shas are pinned in `//private/extensions:python. updated via `knife update-python-archives` (see `knife.d/update_python_archives.sh`). A generated SPDX SBOM for the pinned release is committed at `python/pbs-sbom.spdx.json` (regenerated by the updater; lists the statically embedded native libraries with versions). +The pip entry carries a pypi purl and is trivy-scannable; the native libraries are +source pins (`pkg:generic`) that SBOM scanners skip by design (no advisory feed exists +for source-pinned C libraries). Check it with: + + trivy sbom --scanners vuln,license --severity HIGH,CRITICAL --ignore-unfixed --exit-code 1 python/pbs-sbom.spdx.json diff --git a/python/gen_pbs_sbom.py b/python/gen_pbs_sbom.py index 44f42ed53..fda6a565f 100644 --- a/python/gen_pbs_sbom.py +++ b/python/gen_pbs_sbom.py @@ -14,18 +14,33 @@ import sys LICENSE_REF = {"bzip2-1.0.6": "LicenseRef-bzip2-1.0.6"} +# Bundled pypi components whose license the PBS manifest does not carry; +# stable, well-known values (used only as fallback when licenses=[]). +PYPI_LICENSE = {"pip": "MIT"} -def declared(entry): +def declared(name, entry): ids = entry.get("licenses") or [] + if not ids and name in PYPI_LICENSE: + ids = [PYPI_LICENSE[name]] out = [LICENSE_REF.get(i) or i for i in ids] if not ids and entry.get("library_names"): out = ["LicenseRef-Public-Domain"] # sqlite: no SPDX id for public domain return " AND ".join(out) if out else "NOASSERTION" +def purl(name, version): + if name in PYPI_LICENSE: + return "pkg:pypi/{}@{}".format(name, version) + # C libraries built from source tarballs: generic purl (trivy skips these + # for vuln/license scanning by design; the purl is provenance metadata). + return "pkg:generic/{}@{}".format(name, version) + + def is_runtime(name, entry): - return name.startswith("cpython") or name in ("pip", "setuptools") or bool(entry.get("library_names")) + # pip is bundled in site-packages; setuptools is a PBS build-time tool and + # is NOT shipped in install_only tarballs (verified against 3.13.15+20260814). + return name.startswith("cpython") or name == "pip" or bool(entry.get("library_names")) def main(): @@ -65,14 +80,24 @@ def main(): if not is_runtime(name, entry): continue pid = "SPDXRef-" + name.replace("-", "_").replace(".", "_") + # prefer actual_version (sqlite ships its SQLITE_VERSION_NUMBER, e.g. + # 3530100, in `version` alongside actual_version 3.53.1.0) + version = str(entry.get("actual_version") or entry.get("version", "")) doc["packages"].append( { "name": name, "SPDXID": pid, - "versionInfo": str(entry.get("version", "")), + "versionInfo": version, "downloadLocation": entry.get("url", "NOASSERTION"), "licenseConcluded": "NOASSERTION", - "licenseDeclared": declared(entry), + "licenseDeclared": declared(name, entry), + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": purl(name, version), + } + ], "checksums": [{"algorithm": "SHA256", "checksumValue": entry.get("sha256", "")}], "filesAnalyzed": False, } diff --git a/python/pbs-sbom.spdx.json b/python/pbs-sbom.spdx.json index 95ec61873..f21a58b7c 100644 --- a/python/pbs-sbom.spdx.json +++ b/python/pbs-sbom.spdx.json @@ -5,7 +5,7 @@ "name": "python-build-standalone-20260814-linux", "documentNamespace": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260814/spdx.json", "creationInfo": { - "created": "2026-08-19T10:03:07Z", + "created": "2026-08-19T10:30:39Z", "creators": [ "Tool: distroless-pbs-sbom (python/gen_pbs_sbom.py)" ] @@ -28,6 +28,13 @@ "downloadLocation": "https://ftp.osuosl.org/pub/blfs/conglomeration/db/db-6.0.19.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Sleepycat", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/bdb@6.0.19" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -43,6 +50,13 @@ "downloadLocation": "https://astral-sh.github.io/mirror/files/bzip2-1.0.8.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "LicenseRef-bzip2-1.0.6", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/bzip2@1.0.8" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -58,6 +72,13 @@ "downloadLocation": "https://www.python.org/ftp/python/3.10.21/Python-3.10.21.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.10@3.10.21" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -73,6 +94,13 @@ "downloadLocation": "https://www.python.org/ftp/python/3.11.16/Python-3.11.16.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.11@3.11.16" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -88,6 +116,13 @@ "downloadLocation": "https://www.python.org/ftp/python/3.12.14/Python-3.12.14.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.12@3.12.14" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -103,6 +138,13 @@ "downloadLocation": "https://www.python.org/ftp/python/3.13.15/Python-3.13.15.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.13@3.13.15" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -118,6 +160,13 @@ "downloadLocation": "https://www.python.org/ftp/python/3.14.7/Python-3.14.7.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.14@3.14.7" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -133,6 +182,13 @@ "downloadLocation": "https://www.python.org/ftp/python/3.15.0/Python-3.15.0rc1.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.15@3.15.0rc1" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -148,6 +204,13 @@ "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_3/expat-2.8.3.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/expat@2.8.3" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -163,6 +226,13 @@ "downloadLocation": "https://www.x.org/releases/individual/lib/libX11-1.8.13.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT AND X11", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libX11@1.8.13" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -178,6 +248,13 @@ "downloadLocation": "https://www.x.org/releases/individual/lib/libXau-1.0.12.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libXau@1.0.12" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -193,6 +270,13 @@ "downloadLocation": "https://thrysoee.dk/editline/libedit-20240808-3.1.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "BSD-3-Clause", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libedit@20240808-3.1" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -208,6 +292,13 @@ "downloadLocation": "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libffi@3.4.6" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -223,6 +314,13 @@ "downloadLocation": "https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libffi-3.3@3.3" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -238,6 +336,13 @@ "downloadLocation": "https://xcb.freedesktop.org/dist/libxcb-1.17.0.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libxcb@1.17.0" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -253,6 +358,13 @@ "downloadLocation": "https://astral-sh.github.io/mirror/files/mpdecimal-4.0.0.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "BSD-2-Clause", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/mpdecimal@4.0.0" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -268,6 +380,13 @@ "downloadLocation": "https://astral-sh.github.io/mirror/files/ncurses-6.5.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "X11", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/ncurses@6.5" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -283,6 +402,13 @@ "downloadLocation": "https://www.openssl.org/source/openssl-1.1.1w.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "OpenSSL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/openssl-1.1@1.1.1w" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -298,6 +424,13 @@ "downloadLocation": "https://github.com/openssl/openssl/releases/download/openssl-3.5.7/openssl-3.5.7.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Apache-2.0", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/openssl-3.5@3.5.7" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -312,26 +445,18 @@ "versionInfo": "26.2.1", "downloadLocation": "https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl", "licenseConcluded": "NOASSERTION", - "licenseDeclared": "NOASSERTION", - "checksums": [ + "licenseDeclared": "MIT", + "externalRefs": [ { - "algorithm": "SHA256", - "checksumValue": "71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e" + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:pypi/pip@26.2.1" } ], - "filesAnalyzed": false - }, - { - "name": "setuptools", - "SPDXID": "SPDXRef-setuptools", - "versionInfo": "82.0.1", - "downloadLocation": "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", - "licenseConcluded": "NOASSERTION", - "licenseDeclared": "NOASSERTION", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb" + "checksumValue": "71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e" } ], "filesAnalyzed": false @@ -339,10 +464,17 @@ { "name": "sqlite", "SPDXID": "SPDXRef-sqlite", - "versionInfo": "3530100", + "versionInfo": "3.53.1.0", "downloadLocation": "https://www.sqlite.org/2026/sqlite-autoconf-3530100.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "LicenseRef-Public-Domain", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/sqlite@3.53.1.0" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -358,6 +490,13 @@ "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tcl9.0.4-src.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "TCL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/tcl@9.0.4" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -373,6 +512,13 @@ "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tcl8.6.12-src.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "TCL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/tcl-8612@8.6.12" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -388,6 +534,13 @@ "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tk9.0.4-src.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "TCL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/tk@9.0.4" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -403,6 +556,13 @@ "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tk8.6.12-src.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "TCL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/tk-8612@8.6.12" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -418,6 +578,13 @@ "downloadLocation": "https://sourceforge.net/projects/libuuid/files/libuuid-1.0.3.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "BSD-3-Clause", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/uuid@1.0.3" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -433,6 +600,13 @@ "downloadLocation": "https://github.com/tukaani-project/xz/releases/download/v5.8.3/xz-5.8.3.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "0BSD", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/xz@5.8.3" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -448,6 +622,13 @@ "downloadLocation": "https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Zlib", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/zlib@1.3.2" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -463,6 +644,13 @@ "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Zlib", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/zlib-ng@2.2.4" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -478,6 +666,13 @@ "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "BSD-3-Clause", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/zstd@1.5.7" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -588,11 +783,6 @@ "relationshipType": "CONTAINS", "relatedSpdxElement": "SPDXRef-pip" }, - { - "spdxElementId": "SPDXRef-PBS", - "relationshipType": "CONTAINS", - "relatedSpdxElement": "SPDXRef-setuptools" - }, { "spdxElementId": "SPDXRef-PBS", "relationshipType": "CONTAINS", From cfe6310e823be93e82b7aaeeb18f41a34ab47abe Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:19:52 +0200 Subject: [PATCH 5/9] feat: verify PBS embedded native libs against the manifest (drift => RED) A PBS release can bump its statically linked native libraries while the CPython versions stay unchanged (maintainer-reported gap: python pin alone would not notice). The updater now downloads the x86_64 install tarball (sha-verified against SHA256SUMS), dissects libpython3*.so and compares the embedded openssl/zlib/expat/ncurses/bzip2/sqlite/xz version markers against the release manifest; a mismatch aborts the update. trivy cannot see these libraries in any mode, so the dissection is the only drift check possible. The contract test gained a hermetic tarball fixture (PBS_TARBALL_FILE) and a drift phase: a tarball whose markers no longer match the manifest must fail the updater without mutating the workspace. get_python_archs now returns an empty result for a minor missing from config.bzl instead of aborting under errexit+pipefail (the new-minor fallback relies on it). --- knife.d/update_python_archives.sh | 61 +++++++++- python/BUILD | 1 + python/README.md | 7 ++ python/pbs_embedded_versions.py | 115 ++++++++++++++++++ .../update_python_archives_contract_test.sh | 64 +++++++++- 5 files changed, 245 insertions(+), 3 deletions(-) create mode 100644 python/pbs_embedded_versions.py diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh index c34fb33d9..b7548ca8c 100644 --- a/knife.d/update_python_archives.sh +++ b/knife.d/update_python_archives.sh @@ -48,8 +48,11 @@ function get_python_minors() { # prints archs for a minor from the build matrix, one per line function get_python_archs() { local minor="$1" + # a missing minor must yield an empty result (the caller falls back to the + # previous minor's archs for a newly detected one), NOT kill the updater: + # errexit+pipefail would otherwise abort on the failed grep. grep "\"${minor}\": \[" python/config.bzl \ - | grep -oE '"[a-z0-9]+"' | tr -d '"' + | grep -oE '"[a-z0-9]+"' | tr -d '"' || true } function triple_for_arch() { @@ -241,6 +244,62 @@ function generate_python_archives() { grep -q "$latest_release" "$sbom_tmp" || { echo "PBS SBOM does not mention ${latest_release}" >&2; rm -f "$sbom_tmp" "$downloads_tmp"; exit 1; } [ -n "${PBS_DOWNLOADS_FILE:-}" ] || rm -f "$downloads_tmp" + # PBS embedded native libraries: dissect libpython3*.so from the x86_64 install + # tarball and verify the statically linked C libraries against the manifest. + # A release can bump the embedded libs while the CPython versions stay the same + # (maintainer-reported gap); this turns that drift into a hard error. The tarball + # is sha-verified against SHA256SUMS. Hermetic tests inject a fake tarball via + # PBS_TARBALL_FILE. + local tarball tarball_tmp so_dir so_path + if [ -n "${PBS_TARBALL_FILE:-}" ]; then + tarball="$PBS_TARBALL_FILE" + else + tarball_tmp=$(mktemp) + local fname tarball_sha + fname=$(printf '%s\n' "$sha256sums" | awk '/x86_64-unknown-linux-gnu-install_only\.tar\.gz$/ {print $2; exit}') + tarball_sha=$(printf '%s\n' "$sha256sums" | awk -v f="$fname" '$2 == f {print $1; exit}') + if [ -z "$fname" ] || [ -z "$tarball_sha" ]; then + echo "no x86_64 install tarball in SHA256SUMS" >&2 + rm -f "$tarball_tmp" + exit 1 + fi + if ! curl -sSL "https://github.com/astral-sh/python-build-standalone/releases/download/${latest_release}/${fname}" -o "$tarball_tmp"; then + echo "cannot download ${fname}" >&2 + rm -f "$tarball_tmp" + exit 1 + fi + local got_sha + got_sha=$(python3 -c "import hashlib,sys; print(hashlib.sha256(open(sys.argv[1],'rb').read()).hexdigest())" "$tarball_tmp") + if [ "$got_sha" != "$tarball_sha" ]; then + echo "sha256 mismatch for ${fname}: expected ${tarball_sha}, got ${got_sha}" >&2 + rm -f "$tarball_tmp" + exit 1 + fi + tarball="$tarball_tmp" + fi + so_dir=$(mktemp -d) + if ! python3 - "$tarball" "$so_dir" <<'PYEOF'; then +import sys, tarfile +tar = tarfile.open(sys.argv[1]) +for member in tar.getmembers(): + if "/lib/libpython3." in member.name and member.name.endswith(".so") and member.isfile(): + tar.extract(member, sys.argv[2]) + sys.exit(0) +sys.exit("no libpython3*.so in tarball") +PYEOF + rm -rf "$so_dir" "$tarball_tmp" + exit 1 + fi + so_path=$(find "$so_dir" -name 'libpython3.*.so' | head -1) + # stdout is the updater's machine contract (the release tag); diagnostics to stderr. + if ! python3 python/pbs_embedded_versions.py "$so_path" "$downloads_file" >&2; then + echo "PBS embedded native libraries drift detected" >&2 + rm -rf "$so_dir" "$tarball_tmp" + exit 1 + fi + rm -rf "$so_dir" + [ -n "${PBS_TARBALL_FILE:-}" ] || rm -f "$tarball_tmp" + printf '%s\n' "${changes[@]}" >&2 local start end section tmp diff --git a/python/BUILD b/python/BUILD index 54f4e5765..872574bc9 100644 --- a/python/BUILD +++ b/python/BUILD @@ -25,6 +25,7 @@ sh_test( data = [ ":config.bzl", ":gen_pbs_sbom.py", + ":pbs_embedded_versions.py", ":update_python_archives_test.sh", "testdata/python3.13.yaml", "testdata/python3.14.yaml", diff --git a/python/README.md b/python/README.md index 69649e612..5b8765362 100644 --- a/python/README.md +++ b/python/README.md @@ -73,3 +73,10 @@ source pins (`pkg:generic`) that SBOM scanners skip by design (no advisory feed for source-pinned C libraries). Check it with: trivy sbom --scanners vuln,license --severity HIGH,CRITICAL --ignore-unfixed --exit-code 1 python/pbs-sbom.spdx.json + +The updater additionally dissects the x86_64 install tarball's `libpython3*.so` +(`python/pbs_embedded_versions.py`) and verifies the embedded openssl/zlib/expat/ +ncurses/bzip2/sqlite/xz versions against the manifest — a release that bumps the +native libraries while the CPython version stays the same fails the update +(`trivy image` cannot see statically embedded libraries; the dissection is the +only check that can). diff --git a/python/pbs_embedded_versions.py b/python/pbs_embedded_versions.py new file mode 100644 index 000000000..22e7f7fb6 --- /dev/null +++ b/python/pbs_embedded_versions.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Verify the native libraries statically linked into a PBS libpython .so. + +python-build-standalone compiles the C libraries (openssl, sqlite, zlib, ...) +into libpython3.x.so at release build time. No SBOM scanner can see them there +(trivy's binary detection does not cover statically embedded libs), and a +release can bump them while the CPython version stays unchanged - invisible +when only the python version pin is tracked. This dissects the binary for the +version markers those libraries leave behind and compares them against the +release manifest (pythonbuild/downloads.py). + +Marker patterns were empirically verified against cpython-3.13.15+20260814: + strong openssl "OpenSSL 3.5.7 9 Jun 2026" + zlib "deflate 1.3.2 Copyright 1995-2026 Jean-loup Gailly" + expat "expat_2.8.3" + ncurses "ncurses 6.5.20240427" (manifest keeps the 6.5 prefix) + bzip2 "1.0.8, 13-Jul-2019" + weak sqlite bare "3.53.1" literal (manifest actual_version) + xz bare "5.8.3" literal + absent libffi, readline, gdbm, tcl, uuid, libedit, libX11, libxcb: no + version string is embedded -> reported unverifiable, never fails. + +Usage: pbs_embedded_versions.py +Exit code 0 = every extractable manifest library matches the binary; +1 = a library's embedded version differs from the manifest (drift). +""" +import importlib.util +import re +import sys + +# manifest key -> (regex with one capture group for the version) +STRONG = [ + ("openssl-3.5", re.compile(rb"OpenSSL (\d+\.\d+\.\d+[a-z]?)\s+\d{1,2} [A-Z][a-z]{2} \d{4}")), + ("openssl-1.1", re.compile(rb"OpenSSL (1\.1\.1[a-z]?)\s+\d{1,2} [A-Z][a-z]{2} \d{4}")), + ("zlib", re.compile(rb"deflate (\d+\.\d+\.\d+) Copyright")), + ("expat", re.compile(rb"expat_(\d+\.\d+\.\d+)")), + ("ncurses", re.compile(rb"ncurses (\d+\.\d+\.\d+(?:\.\d+)?)")), + ("bzip2", re.compile(rb"(\d+\.\d+\.\d+), \d{1,2}-[A-Z][a-z]{2}-\d{4}")), +] +# manifest keys with only a bare version literal: presence check (weak) +WEAK = ("sqlite", "xz") + + +def load_manifest(path): + spec = importlib.util.spec_from_file_location("pbs_downloads", path) + if spec is None or spec.loader is None: + sys.exit("cannot load manifest: " + path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.DOWNLOADS + + +def expected_version(entry): + """Human version for the manifest entry: actual_version when present + (sqlite ships its SQLITE_VERSION_NUMBER in `version`, e.g. 3530100, + alongside actual_version 3.53.1.0), else `version` as-is.""" + v = entry.get("actual_version") or entry.get("version") or "" + return re.sub(r"\.0$", "", v) + + +def marker_version(blob, regex, name, entry): + match = regex.search(blob) + if not match: + return None # marker absent (library not linked into this .so) + return match.group(1).decode() + + +def check_weak(blob, name, entry): + expected = expected_version(entry) + if not expected: + return None + # bare literal with non-digit boundaries: "3.53.1" must not match "3.53.10" + pattern = re.compile(rb"(? "$FIX/downloads.py" <<'EOF' @@ -42,6 +43,12 @@ DOWNLOADS = { "licenses": ["MIT"], "library_names": ["expat"], }, + "sqlite": { + "url": "https://example.invalid/sqlite.tar.gz", + "version": "3530100", + "actual_version": "3.53.1.0", + "library_names": ["sqlite3"], + }, "zlib": { "url": "https://example.invalid/zlib.tar.gz", "version": "1.3.2", @@ -51,6 +58,27 @@ DOWNLOADS = { } EOF +# fake PBS install tarball: python/lib/libpython3.13.so with the same embedded +# library markers the real one carries (see python/pbs_embedded_versions.py) +make_tarball() { # $1 = output path; markers must match the fixture manifest + python3 - "$1" <<'PYEOF' +import sys, tarfile, io +blob = ( + b"OpenSSL 3.5.7 9 Jun 2026\n" + b"deflate 1.3.2 Copyright 1995-2026 Jean-loup Gailly and Mark Adler\n" + b"expat_2.8.3\n" + b"ncurses 6.5.20240427\n" + b"1.0.8, 13-Jul-2019\n" + b"3.53.1\n5.8.3\n" +) +with tarfile.open(sys.argv[1], "w:gz") as tar: + info = tarfile.TarInfo("python/lib/libpython3.13.so") + info.size = len(blob) + tar.addfile(info, io.BytesIO(blob)) +PYEOF +} +make_tarball "$FIX/tarball.tar.gz" + cd "$FIX" source update_python_archives.sh @@ -75,9 +103,17 @@ make_sha256sums() { # $1=release $2=patch313 $3=patch314 $4=patch315 ("" = no 3. } run_updater() { # prints stdout; fails the test on a non-zero exit + # bash -c: the updater aborts with exit 1 on fatal errors (knife contract), + # which must not kill the test script. + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_FILE="$FIX/tarball.tar.gz" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" +} + +run_updater_expect_fail() { # non-zero exit is the expectation (drift => RED) PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ - PBS_DOWNLOADS_FILE="$FIX/downloads.py" \ - generate_python_archives 2>"$FIX/updater.err" + PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_FILE="$FIX/drift.tar.gz" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" && return 1 || return 0 } # --- phase A: tag-only bump (new release, same patches) ----------------------- @@ -94,6 +130,30 @@ grep -q 'Python 3.13.15' python/testdata/python3.13.yaml || { echo "phase A: tes grep -q '20990101' python/pbs-sbom.spdx.json || { echo "phase A: SBOM not regenerated for the new release"; exit 1; } grep -q '"expat"' python/pbs-sbom.spdx.json || { echo "phase A: SBOM missing bundled component"; exit 1; } +# --- phase DRIFT: embedded library bumped without a manifest change ---------- +# the maintainer-reported gap made a hard error: a release whose binary embeds +# e.g. zlib 1.3.3 while the manifest still pins 1.3.2 must fail the updater. +python3 - "$FIX/drift.tar.gz" <<'PYEOF' +import sys, tarfile, io +blob = ( + b"OpenSSL 3.5.7 9 Jun 2026\n" + b"deflate 1.3.3 Copyright 1995-2026 Jean-loup Gailly and Mark Adler\n" + b"expat_2.8.3\n" + b"ncurses 6.5.20240427\n" + b"1.0.8, 13-Jul-2019\n" + b"3.53.1\n5.8.3\n" +) +with tarfile.open(sys.argv[1], "w:gz") as tar: + info = tarfile.TarInfo("python/lib/libpython3.13.so") + info.size = len(blob) + tar.addfile(info, io.BytesIO(blob)) +PYEOF +echo '{"tag": "20990102"}' > release.json +make_sha256sums 20990102 3.13.15 3.14.7 +run_updater_expect_fail || { echo "phase DRIFT: expected the updater to fail"; cat "$FIX/updater.err"; exit 1; } +grep -qi 'drift' "$FIX/updater.err" || { echo "phase DRIFT: missing drift error message"; cat "$FIX/updater.err"; exit 1; } +! grep -q '20990102' private/extensions/python.bzl || { echo "phase DRIFT: workspace must be untouched after a RED"; exit 1; } + # --- phase B: patch bump (new release, new patches) --------------------------- snap_b=$(get_python_versions) echo '{"tag": "20990102"}' > release.json From f205dd40f821ff1a4ddf44affcb8b0ae93b92d7d Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:20:06 +0200 Subject: [PATCH 6/9] feat: NVD CPE check for pinned native libs (RED on HIGH/CRITICAL) pbs_cve_check.py queries the NVD API for the dissection-verified versions of the seven embedded libraries (openssl/sqlite/zlib/expat/bzip2/ncurses/xz) and aborts the update on HIGH/CRITICAL CVEs; dependency-reference false positives (mutt/OpenLDAP-style CPE matches with vulnerable=false) are filtered out. trivy has no advisory feed for pkg:generic, so this is the only real CVE signal for those libraries. NVD_API_KEY removes the rate-limit sleeps, PBS_SKIP_CVE_CHECK=1 bypasses. Hermetic test with fixture NVD responses covers the clean path, the RED path and the false-positive filter. Live result for the 20260814 pin: sqlite 3.53.1 carries CVE-2026-11822/CVE-2026-11824 (fixed in 3.53.2); the rest are clean. --- knife.d/update_python_archives.sh | 12 +++ python/BUILD | 11 +++ python/README.md | 7 ++ python/pbs_cve_check.py | 129 +++++++++++++++++++++++++ python/pbs_cve_check_test.sh | 23 +++++ python/testdata/cve_fixture_clean.json | 11 +++ python/testdata/cve_fixture_high.json | 120 +++++++++++++++++++++++ python/testdata/cve_sbom_mini.json | 13 +++ 8 files changed, 326 insertions(+) create mode 100644 python/pbs_cve_check.py create mode 100755 python/pbs_cve_check_test.sh create mode 100644 python/testdata/cve_fixture_clean.json create mode 100644 python/testdata/cve_fixture_high.json create mode 100644 python/testdata/cve_sbom_mini.json diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh index b7548ca8c..2fad7be14 100644 --- a/knife.d/update_python_archives.sh +++ b/knife.d/update_python_archives.sh @@ -300,6 +300,18 @@ PYEOF rm -rf "$so_dir" [ -n "${PBS_TARBALL_FILE:-}" ] || rm -f "$tarball_tmp" + # NVD CVE check: the pinned native libraries are invisible to trivy + # (pkg:generic has no advisory feed), so query NVD CPE data for the exact + # verified versions and RED on HIGH/CRITICAL. Hermetic tests inject fixtures + # directly into pbs_cve_check.py and skip this block (no network). + if [ -z "${PBS_TARBALL_FILE:-}" ] && [ -z "${PBS_SKIP_CVE_CHECK:-}" ]; then + if ! python3 python/pbs_cve_check.py "$sbom_tmp"; then + echo "PBS pinned native libraries have HIGH/CRITICAL CVEs; update blocked" >&2 + rm -f "$sbom_tmp" + exit 1 + fi + fi + printf '%s\n' "${changes[@]}" >&2 local start end section tmp diff --git a/python/BUILD b/python/BUILD index 872574bc9..3b221a94a 100644 --- a/python/BUILD +++ b/python/BUILD @@ -35,6 +35,17 @@ sh_test( ], ) +sh_test( + name = "pbs_cve_check_test", + srcs = ["pbs_cve_check_test.sh"], + data = [ + ":pbs_cve_check.py", + "testdata/cve_fixture_clean.json", + "testdata/cve_fixture_high.json", + "testdata/cve_sbom_mini.json", + ], +) + [ python_image( arch = arch, diff --git a/python/README.md b/python/README.md index 5b8765362..b17617b04 100644 --- a/python/README.md +++ b/python/README.md @@ -80,3 +80,10 @@ ncurses/bzip2/sqlite/xz versions against the manifest — a release that bumps t native libraries while the CPython version stays the same fails the update (`trivy image` cannot see statically embedded libraries; the dissection is the only check that can). + +The dissection-verified versions are then checked against NVD CPE data +(`python/pbs_cve_check.py`, one of trivy's own CVE sources — trivy itself has +no advisory feed for source-pinned C libraries): the update is blocked on +HIGH/CRITICAL CVEs. The 20260814 pin e.g. embeds sqlite 3.53.1 with +CVE-2026-11822 / CVE-2026-11824 (fixed in sqlite 3.53.2); set +`PBS_SKIP_CVE_CHECK=1` to bypass in an emergency. diff --git a/python/pbs_cve_check.py b/python/pbs_cve_check.py new file mode 100644 index 000000000..2032d1208 --- /dev/null +++ b/python/pbs_cve_check.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Check the pinned PBS native libraries against NVD CVE data. + +The native libraries embedded in python-build-standalone releases (openssl, +sqlite, zlib, expat, bzip2, ncurses, xz) are source pins (`pkg:generic`) that +trivy and other SBOM scanners cannot match against advisory feeds. NVD tracks +them as CPE products with per-version CVE data, so this queries the NVD API 2.0 +with the exact version pinned in the SBOM (which the updater has already +verified against the binary) and fails on HIGH/CRITICAL findings. + +False-positive filtering: the NVD `cpeName` query also returns CVEs of other +products (mutt, OpenLDAP, httpd, ...) whose configurations merely reference the +library. A CVE is reported only when the library's own CPE appears as +vulnerable=true in its configuration. + +Usage: + pbs_cve_check.py + pbs_cve_check.py --nvd-fixture # hermetic tests + +Env: NVD_API_KEY (optional; removes the unauthenticated rate-limit sleeps). +Exit 0: no CVEs or none HIGH/CRITICAL on pinned versions. Exit 1: findings. +""" +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +# SBOM component name -> NVD CPE vendor/product (all verified extractable from +# the binary by python/pbs_embedded_versions.py). +CPES = { + "openssl-3.5": ("openssl", "openssl"), + "sqlite": ("sqlite", "sqlite"), + "zlib": ("zlib", "zlib"), + "expat": ("libexpat", "expat"), + "bzip2": ("bzip2", "bzip2"), + "ncurses": ("gnu", "ncurses"), + "xz": ("tukaani", "xz"), +} +GATE = {"HIGH", "CRITICAL"} + + +def cpe_version(name, version): + # sqlite's actual_version is "3.53.1.0"; NVD CPEs use "3.53.1". + if name == "sqlite" and version.endswith(".0"): + return version[:-2] + return version + + +def fetch_nvd(cpe, api_key, fixture): + if fixture is not None: + return fixture.get(cpe, {"vulnerabilities": []}) + url = "https://services.nvd.nist.gov/rest/json/cves/2.0?cpeName=" + urllib.parse.quote(cpe, safe=":") + req = urllib.request.Request(url, headers={"User-Agent": "distroless-pbs-cve-check"}) + if api_key: + req.add_header("apiKey", api_key) + for attempt in range(4): + try: + with urllib.request.urlopen(req, timeout=60) as resp: + return json.load(resp) + except urllib.error.HTTPError as err: + if err.code == 429: + time.sleep(10 * (attempt + 1)) # unauth limit: ~5 req/30s + continue + raise + sys.exit("NVD API rate limited for " + cpe) + + +def affects(cve, vendor, product): + # walk every configuration, recursing into children nodes (NVD nests + # dependency/AND-OR groups); keep only vulnerable matches on the product. + def walk(nodes): + for node in nodes: + for match in node.get("cpeMatch", []): + parts = match["criteria"].split(":") + if len(parts) > 5 and parts[3] == vendor and parts[4] == product and match.get("vulnerable"): + return True + if walk(node.get("children", [])): + return True + return False + for config in cve.get("configurations") or []: + if walk(config.get("nodes", [])): + return True + return False + + +def severity(cve): + metrics = cve.get("metrics", {}) + for key in ("cvssMetricV31", "cvssMetricV30", "cvssMetricV2"): + if metrics.get(key): + return metrics[key][0]["cvssData"].get("baseSeverity", "UNKNOWN") + return "UNKNOWN" + + +def main(): + sbom_path, rest = sys.argv[1], sys.argv[2:] + fixture = None + if rest and rest[0] == "--nvd-fixture": + fixture = json.load(open(rest[1])) + doc = json.load(open(sbom_path)) + versions = {p["name"]: p["versionInfo"] for p in doc["packages"]} + + findings = [] + for name, (vendor, product) in sorted(CPES.items()): + if name not in versions: + continue + cpe = "cpe:2.3:a:{}:{}:{}".format(vendor, product, cpe_version(name, versions[name])) + data = fetch_nvd(cpe, os.environ.get("NVD_API_KEY", ""), fixture) + for vuln in data.get("vulnerabilities", []): + cve = vuln["cve"] + if affects(cve, vendor, product): + desc = cve["descriptions"][0]["value"][:90] if cve.get("descriptions") else "" + findings.append((severity(cve), name, cve["id"], desc)) + + if not findings: + print("no CVEs found for pinned PBS native libraries") + return 0 + for sev, name, cid, desc in sorted(findings): + print("{} {} {} {}".format(sev.ljust(8), name.ljust(10), cid, desc)) + if any(sev in GATE for sev, _, _, _ in findings): + print("HIGH/CRITICAL CVEs on pinned versions - update blocked") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/pbs_cve_check_test.sh b/python/pbs_cve_check_test.sh new file mode 100755 index 000000000..3b5a52837 --- /dev/null +++ b/python/pbs_cve_check_test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Hermetic test for python/pbs_cve_check.py (NVD CPE check for the pinned +# native libraries): clean fixture exits 0, HIGH/CRITICAL fixture exits 1 and +# reports the library's own CVE while ignoring other-product false positives +# (mutt-style CPE references with vulnerable=false). +set -euo pipefail + +cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" + +out=$(python3 python/pbs_cve_check.py python/testdata/cve_sbom_mini.json --nvd-fixture python/testdata/cve_fixture_clean.json) +echo "$out" | grep -q 'no CVEs found' || { echo "clean: expected no findings"; exit 1; } + +set +e +out=$(python3 python/pbs_cve_check.py python/testdata/cve_sbom_mini.json --nvd-fixture python/testdata/cve_fixture_high.json 2>&1) +rc=$? +set -e +[ "$rc" = 1 ] || { echo "high: expected exit 1, got $rc"; exit 1; } +echo "$out" | grep -q 'CVE-2026-99999' || { echo "high: openssl CVE missing"; exit 1; } +echo "$out" | grep -q 'CVE-2026-00001' || { echo "high: MEDIUM zlib CVE should still be reported"; exit 1; } +! echo "$out" | grep -q 'CVE-2009-1390' || { echo "high: mutt false positive leaked"; exit 1; } +echo "$out" | grep -q 'update blocked' || { echo "high: gate message missing"; exit 1; } + +echo "pbs_cve_check OK" diff --git a/python/testdata/cve_fixture_clean.json b/python/testdata/cve_fixture_clean.json new file mode 100644 index 000000000..4a1a9a0a9 --- /dev/null +++ b/python/testdata/cve_fixture_clean.json @@ -0,0 +1,11 @@ +{ + "cpe:2.3:a:openssl:openssl:3.5.7": { + "vulnerabilities": [] + }, + "cpe:2.3:a:sqlite:sqlite:3.53.1": { + "vulnerabilities": [] + }, + "cpe:2.3:a:zlib:zlib:1.3.2": { + "vulnerabilities": [] + } +} \ No newline at end of file diff --git a/python/testdata/cve_fixture_high.json b/python/testdata/cve_fixture_high.json new file mode 100644 index 000000000..e85d0a1c0 --- /dev/null +++ b/python/testdata/cve_fixture_high.json @@ -0,0 +1,120 @@ +{ + "cpe:2.3:a:openssl:openssl:3.5.7": { + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2026-99999", + "descriptions": [ + { + "lang": "en", + "value": "Test: openssl 3.5.7 out-of-bounds read in the X.509 parser" + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "baseSeverity": "HIGH", + "baseScore": 8.1 + } + } + ] + }, + "configurations": [ + { + "nodes": [ + { + "cpeMatch": [ + { + "criteria": "cpe:2.3:a:openssl:openssl:3.5.7:*:*:*:*:*:*:*:*", + "vulnerable": true + } + ] + } + ] + } + ] + } + }, + { + "cve": { + "id": "CVE-2009-1390", + "descriptions": [ + { + "lang": "en", + "value": "mutt 1.5.19 when linked against OpenSSL: this CVE is about mutt, not openssl" + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "baseSeverity": "HIGH" + } + } + ] + }, + "configurations": [ + { + "nodes": [ + { + "cpeMatch": [ + { + "criteria": "cpe:2.3:a:mutt:mutt:1.5.19:*:*:*:*:*:*:*:*", + "vulnerable": true + }, + { + "criteria": "cpe:2.3:a:openssl:openssl:*:*:*:*:*:*:*:*", + "vulnerable": false + } + ] + } + ] + } + ] + } + } + ] + }, + "cpe:2.3:a:sqlite:sqlite:3.53.1": { + "vulnerabilities": [] + }, + "cpe:2.3:a:zlib:zlib:1.3.2": { + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2026-00001", + "descriptions": [ + { + "lang": "en", + "value": "Test: zlib 1.3.2 minor inflate issue" + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "baseSeverity": "MEDIUM" + } + } + ] + }, + "configurations": [ + { + "nodes": [ + { + "cpeMatch": [ + { + "criteria": "cpe:2.3:a:zlib:zlib:1.3.2:*:*:*:*:*:*:*:*", + "vulnerable": true + } + ] + } + ] + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/python/testdata/cve_sbom_mini.json b/python/testdata/cve_sbom_mini.json new file mode 100644 index 000000000..84635a053 --- /dev/null +++ b/python/testdata/cve_sbom_mini.json @@ -0,0 +1,13 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "pbs-mini", + "packages": [ + {"name": "python-build-standalone", "SPDXID": "SPDXRef-PBS", "versionInfo": "20260814"}, + {"name": "openssl-3.5", "SPDXID": "SPDXRef-openssl_3_5", "versionInfo": "3.5.7"}, + {"name": "sqlite", "SPDXID": "SPDXRef-sqlite", "versionInfo": "3.53.1.0"}, + {"name": "zlib", "SPDXID": "SPDXRef-zlib", "versionInfo": "1.3.2"} + ], + "relationships": [] +} From b8a69bbb2d5e2e93972a0757029ecc98a11197a7 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:47:25 +0200 Subject: [PATCH 7/9] fix: buildifier-sort python/BUILD data lists --- python/BUILD | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/BUILD b/python/BUILD index 3b221a94a..6fbca7e8a 100644 --- a/python/BUILD +++ b/python/BUILD @@ -23,12 +23,12 @@ sh_test( name = "update_python_archives_contract_test", srcs = ["update_python_archives_contract_test.sh"], data = [ + "testdata/python3.13.yaml", + "testdata/python3.14.yaml", ":config.bzl", ":gen_pbs_sbom.py", ":pbs_embedded_versions.py", ":update_python_archives_test.sh", - "testdata/python3.13.yaml", - "testdata/python3.14.yaml", "//:MODULE.bazel", "//knife.d:update_python_archives.sh", "//private/extensions:python.bzl", @@ -39,10 +39,10 @@ sh_test( name = "pbs_cve_check_test", srcs = ["pbs_cve_check_test.sh"], data = [ - ":pbs_cve_check.py", "testdata/cve_fixture_clean.json", "testdata/cve_fixture_high.json", "testdata/cve_sbom_mini.json", + ":pbs_cve_check.py", ], ) From 82c1eb0356f5b0d0c57e574015b5e6f77d7b3407 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:13:34 +0200 Subject: [PATCH 8/9] fix: stale and unclear comments and README pip claim in PBS tooling --- python/README.md | 5 +++-- python/gen_pbs_sbom.py | 2 +- python/pbs_cve_check.py | 2 +- python/update_python_archives_contract_test.sh | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/python/README.md b/python/README.md index b17617b04..7cd027d5c 100644 --- a/python/README.md +++ b/python/README.md @@ -18,8 +18,9 @@ plus a standalone CPython install under `/python`: - `/python/lib/python3.X/` (standard library) - `/python/lib/libpython3.X.so.1.0` -No shell, no pip (install dependencies into `site-packages` in a builder stage), -no include files. python-build-standalone builds are relocatable and reference +No shell, no include files. A bundled pip is on PATH; for reproducible +images, install dependencies into `site-packages` in a builder stage. +python-build-standalone builds are relocatable and reference only glibc at runtime, so no extra Debian packages are needed. ## Usage diff --git a/python/gen_pbs_sbom.py b/python/gen_pbs_sbom.py index fda6a565f..1a13c5be4 100644 --- a/python/gen_pbs_sbom.py +++ b/python/gen_pbs_sbom.py @@ -2,7 +2,7 @@ """Emit an SPDX 2.3 SBOM for a python-build-standalone release. Reads the release's component manifest (pythonbuild/downloads.py) and lists -the runtime components: the CPython versions, bundled pip/setuptools, and the +the runtime components: the CPython versions, bundled pip, and the native libraries statically linked into the interpreters. Build-time-only tools (autoconf, binutils, musl, llvm, ...) are excluded. diff --git a/python/pbs_cve_check.py b/python/pbs_cve_check.py index 2032d1208..03df8a890 100644 --- a/python/pbs_cve_check.py +++ b/python/pbs_cve_check.py @@ -62,7 +62,7 @@ def fetch_nvd(cpe, api_key, fixture): return json.load(resp) except urllib.error.HTTPError as err: if err.code == 429: - time.sleep(10 * (attempt + 1)) # unauth limit: ~5 req/30s + time.sleep(10 * (attempt + 1)) # unauthenticated rate limit: ~5 req/30s continue raise sys.exit("NVD API rate limited for " + cpe) diff --git a/python/update_python_archives_contract_test.sh b/python/update_python_archives_contract_test.sh index 26cb1e6af..c753ded81 100755 --- a/python/update_python_archives_contract_test.sh +++ b/python/update_python_archives_contract_test.sh @@ -9,8 +9,8 @@ # - MODULE.bazel: use_repo gains the new minor's archives # - python/testdata/python3.X.yaml: version strings (patch bumps) / new yaml (new minor) # - a second run is a NO_CHANGE no-op (the state is the updater's fixed point) -# Runs fully offline; fixture release data is injected via PBS_RELEASE_FILE / -# PBS_SHA256SUMS_FILE. +# Runs fully offline; fixture data is injected via PBS_RELEASE_FILE / +# PBS_SHA256SUMS_FILE / PBS_DOWNLOADS_FILE / PBS_TARBALL_FILE. set -euo pipefail cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" From 8e7141d72fe8e819b1c1200036fe6a9bba9f0599 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:34:22 +0200 Subject: [PATCH 9/9] fix: portable in-place sed in update_python_archives (GNU vs BSD -i "") --- knife.d/update_python_archives.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh index 2fad7be14..2f0c6c8fa 100644 --- a/knife.d/update_python_archives.sh +++ b/knife.d/update_python_archives.sh @@ -32,6 +32,15 @@ PYTHON_TRIPLES=( "riscv64=riscv64-unknown-linux-gnu" ) +# portable in-place sed: BSD sed needs `-i ''`, GNU sed reads the '' as an +# empty file name and errors. Write to a sibling temp and rename instead. +sed_inplace() { # $1 = sed expression, $2 = file + local tmp + tmp="${2}.tmp.$$" + sed -e "$1" "$2" > "$tmp" || { rm -f "$tmp"; return 1; } + mv "$tmp" "$2" +} + # prints "_ " per matrix entry, one per line function get_python_versions() { sed -n '/python_versions_repo(/,/^ )$/p' private/extensions/python.bzl \ @@ -373,8 +382,8 @@ $(printf '%s\n' "${metadata_deps[@]}") done config_tmp=$(mktemp) cp python/config.bzl "$config_tmp" - sed -i '' -e "s/^PYTHON_MAJOR_VERSIONS = .*/PYTHON_MAJOR_VERSIONS = [${minors_quoted}]/" "$config_tmp" - sed -i '' -e "s|\"${latest_minor}\": \[[^]]*\],|\"${latest_minor}\": [${arch_list}],${fill_entries}|" "$config_tmp" + sed_inplace "s/^PYTHON_MAJOR_VERSIONS = .*/PYTHON_MAJOR_VERSIONS = [${minors_quoted}]/" "$config_tmp" + sed_inplace "s|\"${latest_minor}\": \[[^]]*\],|\"${latest_minor}\": [${arch_list}],${fill_entries}|" "$config_tmp" for fill_m2 in "${fill[@]}"; do grep -qE "PYTHON_MAJOR_VERSIONS = .*\"${fill_m2}\"" "$config_tmp" \ || { echo "config.bzl update for ${fill_m2} did not land (format drift?)" >&2; rm -f "$config_tmp" "$tmp"; return 1; } @@ -386,7 +395,7 @@ $(printf '%s\n' "${metadata_deps[@]}") module_tmp=$(mktemp) cp MODULE.bazel "$module_tmp" repos_sorted=$(printf '"%s", ' $(printf '%s\n' "${repos[@]}" | sort) | sed 's/, $//') - sed -i '' -e "s/^use_repo(py, .*/use_repo(py, ${repos_sorted}, \"python_versions\")/" "$module_tmp" + sed_inplace "s/^use_repo(py, .*/use_repo(py, ${repos_sorted}, \"python_versions\")/" "$module_tmp" for fill_m2 in "${fill[@]}"; do grep -q "\"python$(echo "$fill_m2" | tr -d '.').*_" "$module_tmp" \ || { echo "use_repo update for ${fill_m2} did not land" >&2; rm -f "$config_tmp" "$tmp" "$module_tmp"; return 1; } @@ -428,7 +437,7 @@ function update_test_versions_python() { old_snapshot=$1 # keep the hermetic smoke test's expected minors in sync with the matrix expected=$(get_python_minors | tr '\n' ' ') - sed -i '' -e "s/\"\$minors\" = \"[^\"]*\"/\"\$minors\" = \"${expected}\"/" python/update_python_archives_test.sh + sed_inplace "s/\"\$minors\" = \"[^\"]*\"/\"\$minors\" = \"${expected}\"/" python/update_python_archives_test.sh for minor in $(get_python_minors); do file="python/testdata/python3.$(echo "$minor" | cut -d. -f2).yaml" new=$(current_version "$minor" "amd64") @@ -451,7 +460,7 @@ EOT fi old=$(echo "$old_snapshot" | awk -v key="${minor}_amd64" '$1 == key { print $2 }') if [ -n "$old" ] && [ -n "$new" ] && [ "$old" != "$new" ]; then - sed -i '' -e "s/Python ${old}/Python ${new}/g" "$file" + sed_inplace "s/Python ${old}/Python ${new}/g" "$file" grep -q "Python ${new}" "$file" || { echo "testdata bump for ${file} did not land" >&2; return 1; } echo "bumped $file to ${new}" fi