From 35b27457a159f305d0a2cae7ae4a492f64dfc284 Mon Sep 17 00:00:00 2001 From: Joan Leon Date: Sat, 14 Nov 2026 17:24:02 +0100 Subject: [PATCH] ci: run the jobs side by side, and split the E2E tests of the CLI in four shards A run took about 11 minutes in one job, and the E2E tests of the CLI were 566 seconds of them. The checks that need neither a build nor a browser, the build, the E2E tests of the extension and the E2E tests of the CLI are jobs of their own, and the last one runs in four shards with vitest --shard. Each shard has a whole runner, as the suite had, so the tests that read timings keep their margin. .github/actions/setup installs Node, the dependencies and, when asked, the Chromium of Playwright for all of them. The decision to skip the slow jobs on a pull request that only changes documentation is a job the others depend on. --- .github/actions/setup/action.yml | 26 +++++++ .github/workflows/ci.yml | 130 +++++++++++++++++++++++-------- CLAUDE.md | 2 +- 3 files changed, 125 insertions(+), 33 deletions(-) create mode 100644 .github/actions/setup/action.yml diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..c72fc501 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,26 @@ +name: Set up the repository +description: Node.js at the version of .nvmrc, the dependencies of the lockfile and, when asked, the Chromium of Playwright. Run it after the checkout. + +inputs: + playwright: + description: Install the Chromium of Playwright with its system dependencies + default: "false" + +runs: + using: composite + steps: + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version-file: .nvmrc + cache: npm + + - name: Install dependencies + shell: bash + run: npm ci + + - name: Install Playwright Chromium + if: inputs.playwright == 'true' + shell: bash + working-directory: cli + run: npx playwright install --with-deps chromium diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b0e98f2..22b08092 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,14 +12,22 @@ concurrency: group: ci-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +permissions: + contents: read + +# The jobs run side by side, each on its own runner. The E2E tests of the CLI are most of the time +# of a run, so they are split in shards: a shard has a whole runner, as the suite had when it ran +# alone, and the tests that read timings keep their margin. jobs: - validate: + # A pull request that only changes documentation skips the jobs that build and launch Chromium + changes: runs-on: ubuntu-latest - # A run takes about 12 minutes. Without a limit, one that hangs holds the runner for 6 hours - timeout-minutes: 25 + timeout-minutes: 5 permissions: contents: read pull-requests: read + outputs: + skip: ${{ steps.slow.outputs.skip }} steps: - name: Checkout repository @@ -31,9 +39,8 @@ jobs: uses: actions/setup-node@v7 with: node-version-file: .nvmrc - cache: 'npm' - - name: Decide whether to skip the slow steps + - name: Decide whether to skip the slow jobs id: slow env: EVENT_NAME: ${{ github.event_name }} @@ -42,19 +49,22 @@ jobs: GH_TOKEN: ${{ github.token }} run: node scripts/ci-skip-slow-steps.js - - name: Install dependencies - run: npm ci + # What needs neither a build nor a browser + checks: + runs-on: ubuntu-latest + timeout-minutes: 10 - - name: Lint - run: npm run lint + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + persist-credentials: false - - name: Build - if: steps.slow.outputs.skip != 'true' - run: npm run build + - name: Set up the repository + uses: ./.github/actions/setup - - name: Check the built snippets - if: steps.slow.outputs.skip != 'true' - run: npm run check:built-snippets + - name: Lint + run: npm run lint - name: Check consistency run: npm run check:consistency @@ -68,12 +78,18 @@ jobs: if: github.event_name == 'push' run: node scripts/release-versions.js --untagged - - name: Run demo tests - run: npm run test:demos - - name: Check emojis have icons run: npm run check:emoji + - name: Check generated skills are up to date + run: npm run generate-skills:check + + - name: Check generated llms files are up to date + run: npm run generate-llms:check + + - name: Run demo tests + run: npm run test:demos + - name: Run icon plugin tests run: npm run test:icons @@ -89,12 +105,6 @@ jobs: - name: Run extension tests run: npm run test:extension - - name: Check generated skills are up to date - run: npm run generate-skills:check - - - name: Check generated llms files are up to date - run: npm run generate-llms:check - - name: Run skill script tests run: npm run test:skills @@ -110,21 +120,77 @@ jobs: - name: Run unit tests run: npm run test:unit --prefix cli - - name: Install Playwright Chromium - if: steps.slow.outputs.skip != 'true' - run: npx playwright install --with-deps chromium - working-directory: cli + build: + needs: changes + if: needs.changes.outputs.skip != 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up the repository + uses: ./.github/actions/setup + with: + playwright: "true" + + - name: Build + run: npm run build + + - name: Check the built snippets + run: npm run check:built-snippets # Reports to the job summary and does not block: a failing audit marks this step, not the job. - name: Audit agentic browsing with Lighthouse - if: steps.slow.outputs.skip != 'true' continue-on-error: true run: npm run audit:agentic + e2e-cli: + name: E2E of the CLI (${{ matrix.shard }}/4) + needs: changes + if: needs.changes.outputs.skip != 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + # A shard that fails does not stop the others: the run says everything that fails + fail-fast: false + matrix: + shard: [1, 2, 3, 4] + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up the repository + uses: ./.github/actions/setup + with: + playwright: "true" + - name: Run E2E tests - if: steps.slow.outputs.skip != 'true' - run: npm run test:e2e --prefix cli + run: npm run test:e2e --prefix cli -- --shard=${{ matrix.shard }}/4 + + e2e-extension: + name: E2E of the extension + needs: changes + if: needs.changes.outputs.skip != 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Set up the repository + uses: ./.github/actions/setup + with: + playwright: "true" - name: Run extension E2E tests - if: steps.slow.outputs.skip != 'true' run: npm run test:extension:e2e diff --git a/CLAUDE.md b/CLAUDE.md index 0c8708a4..7aee2cfe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -78,7 +78,7 @@ npm run smoke:cli-package # before a CLI release: the packed CLI installed i npm run test:e2e --prefix cli # Playwright against local fixtures (cli/tests/fixtures) ``` -CI (`.github/workflows/ci.yml`) runs lint, build, all the checks above except `eval:prompts`, `generate-skills:check` and the CLI unit and e2e tests. The pre-commit hook (`.githooks/pre-commit`) runs `check:consistency`, `check:emoji` and `lint`. +CI (`.github/workflows/ci.yml`) runs lint, build, all the checks above except `eval:prompts`, `generate-skills:check` and the CLI unit and e2e tests. Its jobs run side by side: `checks` (what needs neither a build nor a browser), `build`, the e2e tests of the extension, and the e2e tests of the CLI in four shards; `.github/actions/setup` installs Node, the dependencies and Chromium for all of them. The pre-commit hook (`.githooks/pre-commit`) runs `check:consistency`, `check:emoji` and `lint`. ## Snippets