From 11beb1a1f06ab07ade4281ff321041a8d4d696cf Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 3 Sep 2026 15:51:17 -0700 Subject: [PATCH 1/3] devops(pipeline): resolve pip and npm packages from DevDiv_PublicPackages feed Route all pip installs through PipAuthenticate@1 and the playwright-core download through the DevDiv_PublicPackages npm registry so the release build no longer reaches pypi.org, files.pythonhosted.org or registry.npmjs.org (SFI-ES4.2.4). build_driver.py now reads the registry and credentials from .npmrc the way npm does, so no Node.js install is needed in the pipeline. Manual runs stop after Build so the build can be exercised without publishing. --- .azure-pipelines/publish.yml | 40 ++++++++++++--- .gitignore | 3 ++ scripts/build_driver.py | 95 ++++++++++++++++++++++++++++++++---- 3 files changed, 121 insertions(+), 17 deletions(-) diff --git a/.azure-pipelines/publish.yml b/.azure-pipelines/publish.yml index fbd916e46..8bb4bb589 100644 --- a/.azure-pipelines/publish.yml +++ b/.azure-pipelines/publish.yml @@ -36,25 +36,51 @@ extends: path: $(Build.ArtifactStagingDirectory)/esrp-build artifact: esrp-build steps: + - bash: | + if [[ ! "$CURRENT_BRANCH" =~ ^v1\..* ]]; then + echo "Can only publish from a release tag (v1.*)." + echo "Unexpected ref name: $CURRENT_BRANCH" + exit 1 + fi + env: + CURRENT_BRANCH: ${{ variables['Build.SourceBranchName'] }} + displayName: 'Check the ref is a release tag' + # Allow manual runs on any branch to exercise the build without publishing. + condition: ne(variables['Build.Reason'], 'Manual') - task: UsePythonVersion@0 inputs: versionSpec: '3.10' displayName: 'Use Python' - - task: NodeTool@0 + # Resolve every pip install (including the isolated build environments + # that `python -m build` and `pip install -e .` create) through the + # DevDiv_PublicPackages Azure Artifacts feed instead of pypi.org, as + # required by SFI-ES4.2.4. The task exports an authenticated PIP_INDEX_URL. + - task: PipAuthenticate@1 inputs: - versionSpec: '24.x' - displayName: 'Use Node.js' + artifactFeeds: DevDiv/DevDiv_PublicPackages + displayName: 'Authenticate pip to DevDiv_PublicPackages feed' + # scripts/build_driver.py downloads playwright-core from the npm registry + # configured in .npmrc, using the credentials npmAuthenticate@0 adds to it. + - script: echo "registry=https://devdiv.pkgs.visualstudio.com/DevDiv/_packaging/DevDiv_PublicPackages/npm/registry/" >> .npmrc + displayName: 'Point npm registry at DevDiv_PublicPackages feed' + - task: npmAuthenticate@0 + inputs: + workingFile: .npmrc + displayName: 'Authenticate npm to DevDiv_PublicPackages feed' - script: | - python -m pip install --upgrade pip - pip install -r local-requirements.txt - pip install -r requirements.txt - pip install -e . + python -m pip install --upgrade pip --disable-pip-version-check + pip install -r local-requirements.txt --disable-pip-version-check + pip install -r requirements.txt --disable-pip-version-check + pip install -e . --disable-pip-version-check for wheel in $(python setup.py --list-wheels); do PLAYWRIGHT_TARGET_WHEEL=$wheel python -m build --wheel --outdir $(Build.ArtifactStagingDirectory)/esrp-build done displayName: 'Install & Build' - job: Publish dependsOn: Build + # Only publish from release tags; manual runs on a branch stop after Build, + # which lets the build be exercised without publishing. + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v1.')) templateContext: type: releaseJob isProduction: true diff --git a/.gitignore b/.gitignore index 8424e9bfc..76cb0329a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ utils/docker/dist/ Pipfile Pipfile.lock .venv/ + +# Written by the release pipeline (npmAuthenticate@0); holds feed credentials. +.npmrc diff --git a/scripts/build_driver.py b/scripts/build_driver.py index 28714c8af..adce9940d 100755 --- a/scripts/build_driver.py +++ b/scripts/build_driver.py @@ -35,30 +35,35 @@ scripts/build_driver.py # assemble every platform bundle scripts/build_driver.py # assemble a single bundle, e.g. mac-arm64 -Set ``npm_config_registry`` to download ``playwright-core`` from an -alternative npm registry. +``playwright-core`` is downloaded from the npm registry configured the way npm +itself would resolve it: the ``npm_config_registry`` environment variable, else +``registry=`` in the repository-root ``.npmrc``, else ``~/.npmrc``, else the +public registry. Credentials for that registry (``_authToken``, or +``username``/``_password``) are read from the same ``.npmrc`` files, so the +release pipeline can point the build at an authenticated Azure Artifacts feed +with ``npmAuthenticate@0`` without needing npm installed. ``setup.py`` invokes the single-suffix form so a wheel build only downloads the one Node.js binary it needs. """ +import base64 import os import shutil import sys import tarfile import tempfile import time +import urllib.parse import urllib.request import zipfile from pathlib import Path -from typing import Iterable, List, NamedTuple, Set +from typing import Dict, Iterable, List, NamedTuple, Optional, Set REPO_ROOT = Path(__file__).resolve().parent.parent DRIVER_DIR = REPO_ROOT / "driver" -NPM_REGISTRY = os.environ.get( - "npm_config_registry", "https://registry.npmjs.org" -).rstrip("/") +DEFAULT_NPM_REGISTRY = "https://registry.npmjs.org" NODEJS_DIST = "https://nodejs.org/dist" @@ -86,12 +91,80 @@ def read_pin(name: str) -> str: return value -def download(url: str, destination: Path) -> None: +def _read_npmrc(path: Path) -> Dict[str, str]: + """Parse the ``key=value`` lines of an npmrc file (comments start with ; or #).""" + config: Dict[str, str] = {} + if not path.is_file(): + return config + for raw_line in path.read_text().splitlines(): + line = raw_line.strip() + if not line or line[0] in ";#": + continue + key, sep, value = line.partition("=") + if sep: + config[key.strip()] = value.strip() + return config + + +def npm_config() -> Dict[str, str]: + """Merge npmrc files with npm's precedence: project ``.npmrc`` over ``~/.npmrc``.""" + config = _read_npmrc(Path.home() / ".npmrc") + config.update(_read_npmrc(REPO_ROOT / ".npmrc")) + return config + + +def npm_registry(config: Dict[str, str]) -> str: + registry = ( + os.environ.get("npm_config_registry") + or os.environ.get("NPM_CONFIG_REGISTRY") + or config.get("registry") + or DEFAULT_NPM_REGISTRY + ) + return registry.rstrip("/") + + +def npm_auth_header(registry: str, config: Dict[str, str]) -> Optional[str]: + """Build the Authorization header npm would send to ``registry``, if any. + + npm scopes credentials to a "nerf dart" -- the registry URL without its + scheme, e.g. ``//host/org/_packaging/feed/npm/registry/`` -- and looks for + ``:_authToken=`` (bearer token) or ``:username=`` plus + ``:_password=`` (base64-encoded) entries, checking each parent + path in turn. This is the format ``npmAuthenticate@0`` and ``npm login`` + write. + """ + parts = urllib.parse.urlsplit(registry + "/") + path = parts.path + while True: + nerf_dart = f"//{parts.netloc}{path}" + token = config.get(f"{nerf_dart}:_authToken") + if token: + return f"Bearer {token}" + username = config.get(f"{nerf_dart}:username") + password = config.get(f"{nerf_dart}:_password") + if username and password: + decoded = base64.b64decode(password).decode() + credentials = base64.b64encode(f"{username}:{decoded}".encode()).decode() + return f"Basic {credentials}" + basic = config.get(f"{nerf_dart}:_auth") + if basic: + return f"Basic {basic}" + if path == "/": + return None + path = path[: path.rstrip("/").rfind("/") + 1] + + +def download(url: str, destination: Path, auth_header: Optional[str] = None) -> None: + request = urllib.request.Request(url) + if auth_header: + # Unredirected so the credential is not replayed to a different host + # if the registry redirects the tarball download to blob storage. + request.add_unredirected_header("Authorization", auth_header) last_error: Exception = RuntimeError("no attempt made") for attempt in range(1, 6): try: print(f"Downloading {url}") - with urllib.request.urlopen(url) as response: # noqa: S310 + with urllib.request.urlopen(request) as response: # noqa: S310 with open(destination, "wb") as out: shutil.copyfileobj(response, out) return @@ -136,9 +209,11 @@ def _extract_zip_file(archive: zipfile.ZipFile, name: str, destination: Path) -> def fetch_playwright_core(version: str, work_dir: Path) -> Path: """Download playwright-core@ and extract its package/ tree once.""" - url = f"{NPM_REGISTRY}/playwright-core/-/playwright-core-{version}.tgz" + config = npm_config() + registry = npm_registry(config) + url = f"{registry}/playwright-core/-/playwright-core-{version}.tgz" tgz = work_dir / f"playwright-core-{version}.tgz" - download(url, tgz) + download(url, tgz, npm_auth_header(registry, config)) with tarfile.open(tgz, "r:gz") as tar: # npm tarballs nest every file under a top-level "package/" directory, # which is exactly the bundle layout we want. From 59859aa92f46fb8a52f69cfbcf6ec5ff26283db5 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 3 Sep 2026 16:03:32 -0700 Subject: [PATCH 2/3] devops(driver): fetch playwright-core with npm pack Mirrors playwright-java: npm resolves the registry and credentials from the root .npmrc itself, so the hand-rolled .npmrc parsing goes away and the pipeline installs Node.js again. --- .azure-pipelines/publish.yml | 8 ++- CLAUDE.md | 4 +- CONTRIBUTING.md | 4 +- ROLLING.md | 2 +- scripts/build_driver.py | 118 +++++++++-------------------------- 5 files changed, 41 insertions(+), 95 deletions(-) diff --git a/.azure-pipelines/publish.yml b/.azure-pipelines/publish.yml index 8bb4bb589..522be3739 100644 --- a/.azure-pipelines/publish.yml +++ b/.azure-pipelines/publish.yml @@ -59,8 +59,12 @@ extends: inputs: artifactFeeds: DevDiv/DevDiv_PublicPackages displayName: 'Authenticate pip to DevDiv_PublicPackages feed' - # scripts/build_driver.py downloads playwright-core from the npm registry - # configured in .npmrc, using the credentials npmAuthenticate@0 adds to it. + - task: UseNode@1 + inputs: + version: '24.x' + displayName: 'Install Node.js' + # scripts/build_driver.py fetches playwright-core with `npm pack`, which + # picks up the registry and credentials from this .npmrc. - script: echo "registry=https://devdiv.pkgs.visualstudio.com/DevDiv/_packaging/DevDiv_PublicPackages/npm/registry/" >> .npmrc displayName: 'Point npm registry at DevDiv_PublicPackages feed' - task: npmAuthenticate@0 diff --git a/CLAUDE.md b/CLAUDE.md index e96c9c3a9..81a1adc4e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -15,13 +15,13 @@ Python bindings for [Playwright](https://playwright.dev). The Python client talk - `tests/async/`, `tests/sync/` — pytest suites. Most new tests are added to the async file with a sync mirror. - `DRIVER_VERSION` — the single source of truth for which Playwright release the driver is assembled from (one line, the `playwright-core` npm version, e.g. `1.61.0`, no `v` prefix). Read by `setup.py`, `scripts/build_driver.py`, and CI. The wheel build downloads `playwright-core` at this version from npm plus the matching Node.js binary and assembles the per-platform bundles — no source build. The version is baked into the staged bundle filenames (`driver/playwright--.zip`), so it doubles as the build cache key. - `NODE_VERSION` — the Node.js version bundled with the driver (one line, e.g. `24.16.0`). Maintained at roll time by `scripts/update_node_version.py` (latest LTS, mirroring upstream's `utils/build/update-playwright-node.mjs`). -- `scripts/build_driver.py` — assembles the per-platform driver bundles into `driver/` by downloading the `playwright-core` npm package (`DRIVER_VERSION`) and the official Node.js binaries (`NODE_VERSION`). Pure Python stdlib (no Node/npm/git); invoked from `setup.py`'s `bdist_wheel` with the target platform's suffix (no arg builds all six). +- `scripts/build_driver.py` — assembles the per-platform driver bundles into `driver/` by downloading the `playwright-core` npm package (`DRIVER_VERSION`) and the official Node.js binaries (`NODE_VERSION`). Fetches `playwright-core` with `npm pack` (needs Node.js/npm on PATH; honours a root `.npmrc`) and the Node.js binaries over plain HTTP; invoked from `setup.py`'s `bdist_wheel` with the target platform's suffix (no arg builds all six). - `api.json` is **not** shipped in the bundle and is never written into the driver — `scripts/update_api.sh` generates it from a nearby `microsoft/playwright` checkout (`$PW_SRC_DIR`) into a temp file and passes it to codegen via `PW_API_JSON` (read by `scripts/documentation_provider.py`). Needed only when regenerating the API, never at runtime. - `ROLLING.md`, `CONTRIBUTING.md` — human-facing setup and roll docs. ## Setup -`CONTRIBUTING.md` has the full sequence. The short version (needs Node.js, npm, git and bash for the driver build): +`CONTRIBUTING.md` has the full sequence. The short version (needs Node.js and npm for the driver build): ```sh python3 -m venv env && source env/bin/activate diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93baff37a..41d883dbd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,8 +23,8 @@ Build and install drivers: The driver is assembled from published artifacts — the `playwright-core` npm package (version pinned in `DRIVER_VERSION`) and the official Node.js binary -(pinned in `NODE_VERSION`). Building a wheel just downloads them; no Node/npm/git -toolchain is required. +(pinned in `NODE_VERSION`). Building a wheel downloads them with `npm pack` and +plain HTTP, so Node.js/npm must be installed; no git or source build is needed. ```sh pip install -e . diff --git a/ROLLING.md b/ROLLING.md index 78faf556b..191cae852 100644 --- a/ROLLING.md +++ b/ROLLING.md @@ -12,7 +12,7 @@ pre-commit install pip install -e . ``` * change the driver pin in `DRIVER_VERSION` (the `playwright-core` npm version, e.g. `1.61.0`) and refresh `NODE_VERSION`: `python scripts/update_node_version.py` -* download the new driver: `python -m build --wheel` (fetches `playwright-core` from npm + the matching Node.js binary and assembles the bundle; no source build). Set `npm_config_registry` to use a different npm registry. +* download the new driver: `python -m build --wheel` (fetches `playwright-core` via `npm pack` + the matching Node.js binary and assembles the bundle; no source build). Configure npm (e.g. `npm_config_registry`) to use a different npm registry. * generate API (needs a nearby `microsoft/playwright` checkout at `v`): `PW_SRC_DIR=../playwright ./scripts/update_api.sh` * commit changes & send PR * wait for bots to pass & merge the PR diff --git a/scripts/build_driver.py b/scripts/build_driver.py index adce9940d..0df5785e7 100755 --- a/scripts/build_driver.py +++ b/scripts/build_driver.py @@ -27,43 +27,39 @@ LICENSE - the Node.js license package/** - the playwright-core npm package -Unlike the old source build this needs no Node.js, npm, git or bash — only the -Python standard library. +``playwright-core`` is fetched with ``npm pack`` (run from the repository root, so +a root-level ``.npmrc`` -- e.g. the one the release pipeline writes to point at +the internal Azure Artifacts feed -- and its credentials are honoured). The +Node.js binaries are downloaded directly from nodejs.org. Apart from Node.js/npm +this needs only the Python standard library. Usage:: scripts/build_driver.py # assemble every platform bundle scripts/build_driver.py # assemble a single bundle, e.g. mac-arm64 -``playwright-core`` is downloaded from the npm registry configured the way npm -itself would resolve it: the ``npm_config_registry`` environment variable, else -``registry=`` in the repository-root ``.npmrc``, else ``~/.npmrc``, else the -public registry. Credentials for that registry (``_authToken``, or -``username``/``_password``) are read from the same ``.npmrc`` files, so the -release pipeline can point the build at an authenticated Azure Artifacts feed -with ``npmAuthenticate@0`` without needing npm installed. +Set ``npm_config_registry`` (or configure npm any other way) to download +``playwright-core`` from an alternative npm registry. ``setup.py`` invokes the single-suffix form so a wheel build only downloads the one Node.js binary it needs. """ -import base64 import os import shutil +import subprocess import sys import tarfile import tempfile import time -import urllib.parse import urllib.request import zipfile from pathlib import Path -from typing import Dict, Iterable, List, NamedTuple, Optional, Set +from typing import Iterable, List, NamedTuple, Set REPO_ROOT = Path(__file__).resolve().parent.parent DRIVER_DIR = REPO_ROOT / "driver" -DEFAULT_NPM_REGISTRY = "https://registry.npmjs.org" NODEJS_DIST = "https://nodejs.org/dist" @@ -91,80 +87,12 @@ def read_pin(name: str) -> str: return value -def _read_npmrc(path: Path) -> Dict[str, str]: - """Parse the ``key=value`` lines of an npmrc file (comments start with ; or #).""" - config: Dict[str, str] = {} - if not path.is_file(): - return config - for raw_line in path.read_text().splitlines(): - line = raw_line.strip() - if not line or line[0] in ";#": - continue - key, sep, value = line.partition("=") - if sep: - config[key.strip()] = value.strip() - return config - - -def npm_config() -> Dict[str, str]: - """Merge npmrc files with npm's precedence: project ``.npmrc`` over ``~/.npmrc``.""" - config = _read_npmrc(Path.home() / ".npmrc") - config.update(_read_npmrc(REPO_ROOT / ".npmrc")) - return config - - -def npm_registry(config: Dict[str, str]) -> str: - registry = ( - os.environ.get("npm_config_registry") - or os.environ.get("NPM_CONFIG_REGISTRY") - or config.get("registry") - or DEFAULT_NPM_REGISTRY - ) - return registry.rstrip("/") - - -def npm_auth_header(registry: str, config: Dict[str, str]) -> Optional[str]: - """Build the Authorization header npm would send to ``registry``, if any. - - npm scopes credentials to a "nerf dart" -- the registry URL without its - scheme, e.g. ``//host/org/_packaging/feed/npm/registry/`` -- and looks for - ``:_authToken=`` (bearer token) or ``:username=`` plus - ``:_password=`` (base64-encoded) entries, checking each parent - path in turn. This is the format ``npmAuthenticate@0`` and ``npm login`` - write. - """ - parts = urllib.parse.urlsplit(registry + "/") - path = parts.path - while True: - nerf_dart = f"//{parts.netloc}{path}" - token = config.get(f"{nerf_dart}:_authToken") - if token: - return f"Bearer {token}" - username = config.get(f"{nerf_dart}:username") - password = config.get(f"{nerf_dart}:_password") - if username and password: - decoded = base64.b64decode(password).decode() - credentials = base64.b64encode(f"{username}:{decoded}".encode()).decode() - return f"Basic {credentials}" - basic = config.get(f"{nerf_dart}:_auth") - if basic: - return f"Basic {basic}" - if path == "/": - return None - path = path[: path.rstrip("/").rfind("/") + 1] - - -def download(url: str, destination: Path, auth_header: Optional[str] = None) -> None: - request = urllib.request.Request(url) - if auth_header: - # Unredirected so the credential is not replayed to a different host - # if the registry redirects the tarball download to blob storage. - request.add_unredirected_header("Authorization", auth_header) +def download(url: str, destination: Path) -> None: last_error: Exception = RuntimeError("no attempt made") for attempt in range(1, 6): try: print(f"Downloading {url}") - with urllib.request.urlopen(request) as response: # noqa: S310 + with urllib.request.urlopen(url) as response: # noqa: S310 with open(destination, "wb") as out: shutil.copyfileobj(response, out) return @@ -209,11 +137,25 @@ def _extract_zip_file(archive: zipfile.ZipFile, name: str, destination: Path) -> def fetch_playwright_core(version: str, work_dir: Path) -> Path: """Download playwright-core@ and extract its package/ tree once.""" - config = npm_config() - registry = npm_registry(config) - url = f"{registry}/playwright-core/-/playwright-core-{version}.tgz" + npm = "npm.cmd" if sys.platform == "win32" else "npm" + spec = f"playwright-core@{version}" + # npm is run from the repository root so that a root-level .npmrc (registry + # and credentials) is honoured. `npm pack` writes -.tgz. + print(f"Downloading {spec} with npm pack", flush=True) + try: + subprocess.check_call( + [npm, "pack", spec, "--pack-destination", str(work_dir)], + cwd=REPO_ROOT, + ) + except FileNotFoundError: + raise SystemExit( + "npm was not found on PATH; Node.js/npm are required to assemble the driver." + ) + except subprocess.CalledProcessError as error: + raise SystemExit(f"npm pack {spec} failed with exit code {error.returncode}") tgz = work_dir / f"playwright-core-{version}.tgz" - download(url, tgz, npm_auth_header(registry, config)) + if not tgz.is_file(): + raise SystemExit(f"npm pack did not produce {tgz}") with tarfile.open(tgz, "r:gz") as tar: # npm tarballs nest every file under a top-level "package/" directory, # which is exactly the bundle layout we want. @@ -223,7 +165,7 @@ def fetch_playwright_core(version: str, work_dir: Path) -> Path: if m.name == "package" or m.name.startswith("package/") ] if not members: - raise SystemExit(f"No package/ entries found in {url}") + raise SystemExit(f"No package/ entries found in {tgz.name}") _extract_members(tar, work_dir, members) tgz.unlink() return work_dir / "package" From b3135cdb3dd74865675e4a640eaefc4d745fe355 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 3 Sep 2026 16:06:59 -0700 Subject: [PATCH 3/3] docs(driver): drop stale npm_config_registry mentions --- ROLLING.md | 2 +- scripts/build_driver.py | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/ROLLING.md b/ROLLING.md index 191cae852..63222c38c 100644 --- a/ROLLING.md +++ b/ROLLING.md @@ -12,7 +12,7 @@ pre-commit install pip install -e . ``` * change the driver pin in `DRIVER_VERSION` (the `playwright-core` npm version, e.g. `1.61.0`) and refresh `NODE_VERSION`: `python scripts/update_node_version.py` -* download the new driver: `python -m build --wheel` (fetches `playwright-core` via `npm pack` + the matching Node.js binary and assembles the bundle; no source build). Configure npm (e.g. `npm_config_registry`) to use a different npm registry. +* download the new driver: `python -m build --wheel` (fetches `playwright-core` via `npm pack` + the matching Node.js binary and assembles the bundle; no source build). * generate API (needs a nearby `microsoft/playwright` checkout at `v`): `PW_SRC_DIR=../playwright ./scripts/update_api.sh` * commit changes & send PR * wait for bots to pass & merge the PR diff --git a/scripts/build_driver.py b/scripts/build_driver.py index 0df5785e7..0583bca74 100755 --- a/scripts/build_driver.py +++ b/scripts/build_driver.py @@ -38,9 +38,6 @@ scripts/build_driver.py # assemble every platform bundle scripts/build_driver.py # assemble a single bundle, e.g. mac-arm64 -Set ``npm_config_registry`` (or configure npm any other way) to download -``playwright-core`` from an alternative npm registry. - ``setup.py`` invokes the single-suffix form so a wheel build only downloads the one Node.js binary it needs. """