Build PyPI wheels for the CLI in release-build

.github/workflows/release-build.yml

@@ -185,3 +185,64 @@ jobs:
         with:
           name: wheel
           path: python/dist/*
+
+  # Wraps the signed binaries from the cli job into PyPI wheels (project
+  # "databricks"). Kept as a separate artifact from "wheel" (databricks-bundles)
+  # so each artifact maps to one PyPI project in the databricks-cli release workflow.
+  cli-wheels:
+    needs: cli
+    runs-on:
+      group: databricks-protected-runner-group-large
+      labels: linux-ubuntu-latest-large
+
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      # The script and LICENSE come from the workflow's own ref, like the
+      # local actions in the other jobs, so building an older tag still works.
+      - name: Checkout workflow ref
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          ref: ${{ github.sha }}
+          sparse-checkout: |
+            .github
+            tools/build_cli_wheels.py
+            LICENSE
+
+      - name: Setup JFrog
+        uses: ./.github/actions/setup-jfrog
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
+        with:
+          version: "0.6.5"
+
+      - name: Download signed CLI binaries
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: cli
+          path: cli-artifact
+
+      - name: Build wheels
+        run: python3 tools/build_cli_wheels.py cli-artifact cli-wheels
+
+      - name: Check wheel metadata
+        run: uvx twine@6.2.0 check --strict cli-wheels/*
+
+      - name: Smoke test linux-amd64 wheel
+        run: |
+          cli_version=$(basename cli-artifact/databricks_cli_*_SHA256SUMS)
+          cli_version=${cli_version#databricks_cli_}
+          cli_version=${cli_version%_SHA256SUMS}
+          uv venv "$RUNNER_TEMP/venv"
+          uv pip install --python "$RUNNER_TEMP/venv" cli-wheels/*manylinux*x86_64.whl
+          "$RUNNER_TEMP/venv/bin/databricks" --version | tee /dev/stderr | grep -F "$cli_version"
+
+      - name: Upload CLI wheels
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: cli-wheels
+          path: cli-wheels/*.whl
+          if-no-files-found: error

Taskfile.yml

@@ -265,6 +265,12 @@ tasks:
     cmds:
       - 'uv run -p ">=3.11" --no-project python tools/validate_codeowners.py'
 
+  test-cli-wheels:
+    desc: Unit-test the script that packages CLI binaries as PyPI wheels
+    dir: tools
+    cmds:
+      - 'uv run -p ">=3.11" --no-project python -m unittest -q test_build_cli_wheels'
+
   changelog-preview:
     desc: Print the CHANGELOG.md section the next release would add from .nextchanges/
     env:
@@ -314,6 +320,7 @@ tasks:
       - task: check-codeowners
       - task: check-lockfiles
       - task: check-agents-setup
+      - task: test-cli-wheels
 
   install-pythons:
     desc: Install Python 3.9-3.13 via uv

acceptance/cmd/pypi-wheel/make_archives.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+"""Fake goreleaser output: every platform archive holds the given CLI binary."""
+
+import sys
+import tarfile
+import zipfile
+from pathlib import Path
+
+cli, out = Path(sys.argv[1]), Path(sys.argv[2])
+out.mkdir()
+for goos in ("linux", "darwin", "windows"):
+    for goarch in ("amd64", "arm64"):
+        if goos == "windows":
+            path = out / f"databricks_cli_1.2.3_{goos}_{goarch}.zip"
+            with zipfile.ZipFile(path, "w") as z:
+                z.write(cli, "databricks.exe")
+        else:
+            path = out / f"databricks_cli_1.2.3_{goos}_{goarch}.tar.gz"
+            with tarfile.open(path, "w:gz") as t:
+                t.add(cli, "databricks")
+# Only the file name is used: it carries the version.
+(out / "databricks_cli_1.2.3_SHA256SUMS").touch()

acceptance/cmd/pypi-wheel/out.test.toml

@@ -0,0 +1 @@
+Cloud = false

acceptance/cmd/pypi-wheel/output.txt

@@ -0,0 +1,10 @@
+
+>>> python3 make_archives.py [CLI] archives
+
+>>> python3 [TESTROOT]/../tools/build_cli_wheels.py archives wheels [TESTROOT]/../LICENSE
+
+>>> ls wheels
+6
+
+>>> uvx -q --offline --no-index --find-links wheels --from databricks==1.2.3 databricks --version
+Databricks CLI v[CLI_VERSION]

acceptance/cmd/pypi-wheel/script

@@ -0,0 +1,10 @@
+# Package the CLI under test the way release-build does, then run it through uvx
+# to check that a wheel built by build_cli_wheels.py is usable by Python tooling.
+trace python3 make_archives.py $CLI archives
+trace python3 $TESTROOT/../tools/build_cli_wheels.py archives wheels $TESTROOT/../LICENSE > /dev/null
+trace ls wheels | grep -c '^databricks-1.2.3-py3-none-.*\.whl$'
+
+# uv picks the wheel matching this platform from the directory.
+trace uvx -q --offline --no-index --find-links wheels --from databricks==1.2.3 databricks --version
+
+rm -fr archives wheels

acceptance/cmd/pypi-wheel/test.toml

@@ -0,0 +1,5 @@
+Cloud = false
+
+# uvx may need to create a Python environment.
+Timeout = '180s'
+TimeoutWindows = '240s'

tools/build_cli_wheels.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""Package the released Databricks CLI binaries as PyPI wheels.
+
+Input is the directory of the `cli` release-build artifact: the goreleaser
+archives (databricks_cli_*_<os>_<arch>.{tar.gz,zip}) and the SHA256SUMS file, whose name carries the version.
+The binaries are taken from the archives as-is, so the signed Windows binaries
+are what ends up in the wheels.
+
+Each wheel is py3-none-<platform>: platform-specific but independent of the
+Python version. It contains only the binary, installed to the environment's
+scripts directory (bin/ or Scripts/) via the wheel `.data/scripts` mechanism,
+and no Python package, so `databricks` and `uvx databricks` run the binary
+directly without a Python bootstrapper.
+
+Usage: build_cli_wheels.py <cli-artifact-dir> <output-dir> [license-file]
+"""
+
+import base64
+import hashlib
+import io
+import re
+import sys
+import tarfile
+import zipfile
+from pathlib import Path
+
+DIST_NAME = "databricks"
+SUMMARY = "Databricks CLI"
+DESCRIPTION = """# Databricks CLI
+
+Command-line interface for Databricks. This package installs the prebuilt `databricks` binary.
+
+Run it with `uvx databricks` or `pipx run databricks`, or install it with `uv tool install databricks`.
+
+Documentation: https://docs.databricks.com/dev-tools/cli/
+Source: https://github.com/databricks/cli
+"""
+
+# (goreleaser os, goreleaser arch) -> wheel platform tag. Compressed tag sets
+# (dot-separated) make one wheel serve glibc and musl Linux: the binary is
+# statically linked (CGO_ENABLED=0).
+PLATFORMS = {
+    ("linux", "amd64"): "manylinux_2_17_x86_64.manylinux2014_x86_64.musllinux_1_1_x86_64",
+    ("linux", "arm64"): "manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64",
+    ("darwin", "amd64"): "macosx_11_0_x86_64",
+    ("darwin", "arm64"): "macosx_11_0_arm64",
+    ("windows", "amd64"): "win_amd64",
+    ("windows", "arm64"): "win_arm64",
+}
+
+# Fixed timestamp so rebuilding from the same artifact yields identical wheels.
+ZIP_DATE_TIME = (1980, 1, 1, 0, 0, 0)
+
+
+def read_version(artifact_dir: Path) -> str:
+    """Return the CLI version from the databricks_cli_<version>_SHA256SUMS file name."""
+    sums_files = list(artifact_dir.glob("databricks_cli_*_SHA256SUMS"))
+    if len(sums_files) != 1:
+        raise SystemExit(f"expected exactly one databricks_cli_*_SHA256SUMS in {artifact_dir}, found {sums_files}")
+    return sums_files[0].name[len("databricks_cli_") : -len("_SHA256SUMS")]
+
+
+def pep440(cli_version: str) -> str:
+    """Convert a goreleaser version to PEP 440.
+
+    Releases (1.23.0) are unchanged. Snapshots (1.23.1-dev+abc123) become
+    1.23.1.dev0+abc123.
+    """
+    m = re.fullmatch(r"(\d+\.\d+\.\d+)(?:-dev(?:\+([0-9A-Za-z.]+))?)?", cli_version)
+    if not m:
+        raise SystemExit(f"unsupported CLI version {cli_version!r}")
+    base, local = m.groups()
+    if cli_version == base:
+        return base
+    return f"{base}.dev0" + (f"+{local}" if local else "")
+
+
+def find_archive(artifact_dir: Path, goos: str, goarch: str) -> Path:
+    ext = "zip" if goos == "windows" else "tar.gz"
+    matches = list(artifact_dir.glob(f"databricks_cli_*{goos}_{goarch}.{ext}"))
+    if len(matches) != 1:
+        raise SystemExit(f"expected exactly one {goos}/{goarch} .{ext} archive in {artifact_dir}, found {matches}")
+    return matches[0]
+
+
+def extract_binary(archive: Path, binary_name: str) -> bytes:
+    if archive.name.endswith(".zip"):
+        with zipfile.ZipFile(archive) as z:
+            return z.read(binary_name)
+    with tarfile.open(archive) as t:
+        f = t.extractfile(binary_name)
+        if f is None:
+            raise SystemExit(f"{binary_name} is not a regular file in {archive}")
+        return f.read()
+
+
+def record_hash(data: bytes) -> str:
+    digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode()
+    return f"sha256={digest},{len(data)}"
+
+
+def build_wheel(
+    out_dir: Path,
+    version: str,
+    platform_tag: str,
+    binary_name: str,
+    binary: bytes,
+    license_text: bytes,
+) -> Path:
+    dist_info = f"{DIST_NAME}-{version}.dist-info"
+    data_dir = f"{DIST_NAME}-{version}.data"
+
+    metadata = (
+        "Metadata-Version: 2.1\n"
+        f"Name: {DIST_NAME}\n"
+        f"Version: {version}\n"
+        f"Summary: {SUMMARY}\n"
+        "Home-page: https://github.com/databricks/cli\n"
+        "License: Databricks License\n"
+        "License-File: LICENSE\n"
+        "Description-Content-Type: text/markdown\n"
+        f"\n{DESCRIPTION}"
+    ).encode()
+    wheel_meta = (
+        "Wheel-Version: 1.0\n"
+        "Generator: databricks-cli build_cli_wheels.py\n"
+        "Root-Is-Purelib: false\n" + "".join(f"Tag: py3-none-{t}\n" for t in platform_tag.split("."))
+    ).encode()
+
+    # name -> (content, unix mode)
+    files = {
+        f"{data_dir}/scripts/{binary_name}": (binary, 0o755),
+        f"{dist_info}/METADATA": (metadata, 0o644),
+        f"{dist_info}/WHEEL": (wheel_meta, 0o644),
+        f"{dist_info}/licenses/LICENSE": (license_text, 0o644),
+    }
+    record = "".join(f"{name},{record_hash(data)}\n" for name, (data, _) in files.items())
+    record += f"{dist_info}/RECORD,,\n"
+    files[f"{dist_info}/RECORD"] = (record.encode(), 0o644)
+
+    out = out_dir / f"{DIST_NAME}-{version}-py3-none-{platform_tag}.whl"
+    buf = io.BytesIO()
+    with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as z:
+        for name, (data, mode) in files.items():
+            info = zipfile.ZipInfo(name, ZIP_DATE_TIME)
+            info.external_attr = (0o100000 | mode) << 16
+            info.compress_type = zipfile.ZIP_DEFLATED
+            z.writestr(info, data)
+    out.write_bytes(buf.getvalue())
+    return out
+
+
+def main() -> None:
+    if len(sys.argv) not in (3, 4):
+        raise SystemExit(__doc__)
+    artifact_dir = Path(sys.argv[1])
+    out_dir = Path(sys.argv[2])
+    license_file = Path(sys.argv[3]) if len(sys.argv) == 4 else Path(__file__).resolve().parents[1] / "LICENSE"
+
+    cli_version = read_version(artifact_dir)
+    version = pep440(cli_version)
+    license_text = license_file.read_bytes()
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    for (goos, goarch), platform_tag in PLATFORMS.items():
+        archive = find_archive(artifact_dir, goos, goarch)
+        binary_name = "databricks.exe" if goos == "windows" else "databricks"
+        binary = extract_binary(archive, binary_name)
+        print(build_wheel(out_dir, version, platform_tag, binary_name, binary, license_text))
+
+
+if __name__ == "__main__":
+    main()

tools/test_build_cli_wheels.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+import io
+import tarfile
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+
+import build_cli_wheels as b
+
+
+class TestBuildCliWheels(unittest.TestCase):
+    def test_pep440(self):
+        self.assertEqual(b.pep440("1.23.0"), "1.23.0")
+        self.assertEqual(b.pep440("1.23.1-dev+abc123"), "1.23.1.dev0+abc123")
+        with self.assertRaises(SystemExit):
+            b.pep440("v1.23.0")
+
+    def test_end_to_end(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            art, out = Path(tmp, "art"), Path(tmp, "out")
+            art.mkdir()
+            for (goos, goarch), _ in b.PLATFORMS.items():
+                if goos == "windows":
+                    path = art / f"databricks_cli_1.23.0_{goos}_{goarch}.zip"
+                    with zipfile.ZipFile(path, "w") as z:
+                        z.writestr("databricks.exe", b"MZ")
+                else:
+                    path = art / f"databricks_cli_1.23.0_{goos}_{goarch}.tar.gz"
+                    with tarfile.open(path, "w:gz") as t:
+                        info = tarfile.TarInfo("databricks")
+                        info.size = 3
+                        t.addfile(info, io.BytesIO(b"ELF"))
+            (art / "databricks_cli_1.23.0_SHA256SUMS").touch()
+            lic = Path(tmp, "LICENSE")
+            lic.write_text("license")
+
+            b.sys.argv = ["x", str(art), str(out), str(lic)]
+            b.main()
+
+            wheels = sorted(p.name for p in out.glob("*.whl"))
+            self.assertEqual(len(wheels), len(b.PLATFORMS))
+            self.assertIn("databricks-1.23.0-py3-none-win_amd64.whl", wheels)
+            with zipfile.ZipFile(out / "databricks-1.23.0-py3-none-macosx_11_0_arm64.whl") as z:
+                script = z.getinfo("databricks-1.23.0.data/scripts/databricks")
+                self.assertEqual(script.external_attr >> 16, 0o100755)
+                self.assertIn("Tag: py3-none-macosx_11_0_arm64", z.read("databricks-1.23.0.dist-info/WHEEL").decode())
+
+
+if __name__ == "__main__":
+    unittest.main()