From 18b8a243c24569bcee33c63df237b94196adc52b Mon Sep 17 00:00:00 2001 From: IMvision12 Date: Thu, 15 Oct 2026 19:22:55 -0700 Subject: [PATCH 1/2] Refactor Tests Run --- .github/TESTING.md | 36 ++++++++ .github/workflows/test_code.yml | 146 +++++++++++++++++++++++++++++++- 2 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 .github/TESTING.md diff --git a/.github/TESTING.md b/.github/TESTING.md new file mode 100644 index 00000000..bca84457 --- /dev/null +++ b/.github/TESTING.md @@ -0,0 +1,36 @@ +# Pull request test commands + +Lint & Format continues to run automatically on pull requests. Model, tokenizer, +image-processor, and processor tests run on a pull request only when **IMvision12** +posts a new conversation comment containing exactly: + +```text +/run-tests +``` + +Other users' commands, issue comments, edited comments, and other text do not +authorize the test suite. Re-running a comment-triggered test job also requires +IMvision12 to be the triggering account. + +The command starts the existing 16 test jobs: 9 model shards (Torch, TensorFlow, +and JAX), 1 tokenizer job, 3 image-processor jobs, and 3 processor jobs. They test +the open PR's head commit captured when the request is authorized. A later push +does not start another test run; post `/run-tests` again to test the new commit. + +Results appear on the tested PR commit as **Requested model, tokenizer & processor +tests**, with a link to the workflow's individual jobs. If these tests must block +merging, require this aggregate check in the repository's branch protection or +ruleset; automatically skipped test jobs are not a substitute for that check. + +The updated workflow must be present on the repository's default branch before +GitHub delivers comment-triggered runs. No additional personal access token or +repository secret is needed. + +Release-triggered tests still run automatically after lint succeeds. The release, +documentation, and standalone cross-backend parity workflows are unchanged. + +Authorization and result-reporting jobs have check-writing permission but never +check out PR code. Test jobs have read-only repository access and do not persist +checkout credentials. Comment-triggered tests do not use shared Actions caches +or publish Codecov coverage from the default-branch event context; release runs +retain their existing cache and coverage steps. diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index 405b5b11..1506f2ed 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -3,10 +3,13 @@ on: pull_request: release: types: [created] + issue_comment: + types: [created] jobs: lint-and-format: name: Lint & Format + if: github.event_name != 'issue_comment' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -17,10 +20,72 @@ jobs: - run: ruff check . - run: ruff format --check . + authorize-tests: + name: Authorize PR tests + if: >- + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + github.event.comment.user.login == 'IMvision12' && + github.triggering_actor == 'IMvision12' && + github.event.comment.body == '/run-tests' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + checks: write + outputs: + run_tests: ${{ steps.authorize.outputs.run_tests }} + head_sha: ${{ steps.authorize.outputs.head_sha }} + check_run_id: ${{ steps.authorize.outputs.check_run_id }} + steps: + # This privileged job never checks out or executes PR-controlled files. + - name: Validate owner command and pin the PR commit + id: authorize + uses: actions/github-script@v9 + with: + script: | + const { issue, comment, action } = context.payload; + if (action !== 'created' || !issue?.pull_request || + comment?.user?.login?.toLowerCase() !== 'imvision12' || + comment.body !== '/run-tests') { + throw new Error('Only IMvision12 can request PR tests with /run-tests.'); + } + const { data: pr } = await github.rest.pulls.get({ + ...context.repo, + pull_number: issue.number, + }); + if (pr.state !== 'open') { + throw new Error('Tests can only be requested on an open pull request.'); + } + if (!/^[0-9a-f]{40}$/.test(pr.head.sha)) { + throw new Error('GitHub returned an invalid PR head commit.'); + } + const { data: check } = await github.rest.checks.create({ + ...context.repo, + name: 'Requested model, tokenizer & processor tests', + head_sha: pr.head.sha, + status: 'in_progress', + details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + output: { + title: 'PR tests requested by IMvision12', + summary: `Testing PR #${issue.number} at ${pr.head.sha}. New pushes require a new /run-tests comment.`, + }, + }); + core.setOutput('head_sha', pr.head.sha); + core.setOutput('check_run_id', String(check.id)); + core.setOutput('run_tests', 'true'); + test-code: name: Test the code (${{ matrix.backend }} ${{ matrix.shard }}) - needs: lint-and-format + needs: [lint-and-format, authorize-tests] + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || + (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && + needs.authorize-tests.outputs.run_tests == 'true')) runs-on: ubuntu-latest + permissions: + contents: read strategy: fail-fast: false # Shard the model suite BY MODEL across parallel jobs (see the --shard @@ -42,10 +107,14 @@ jobs: KERAS_BACKEND: ${{ matrix.backend }} steps: - uses: actions/checkout@v4 + with: + ref: ${{ needs.authorize-tests.outputs.head_sha }} + persist-credentials: ${{ github.event_name != 'issue_comment' }} - uses: actions/setup-python@v5 with: python-version: '3.11' - name: Cache pip packages + if: github.event_name != 'issue_comment' uses: actions/cache@v4 with: path: ~/.cache/pip @@ -79,6 +148,7 @@ jobs: - name: Generate coverage run: coverage xml -o coverage.xml - name: Upload coverage + if: github.event_name != 'issue_comment' uses: codecov/codecov-action@v5 with: files: coverage.xml @@ -87,16 +157,27 @@ jobs: test-tokenizers: name: Test tokenizers (vs HF) - needs: lint-and-format + needs: [lint-and-format, authorize-tests] + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || + (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && + needs.authorize-tests.outputs.run_tests == 'true')) runs-on: ubuntu-latest + permissions: + contents: read env: KERAS_BACKEND: torch steps: - uses: actions/checkout@v4 + with: + ref: ${{ needs.authorize-tests.outputs.head_sha }} + persist-credentials: ${{ github.event_name != 'issue_comment' }} - uses: actions/setup-python@v5 with: python-version: '3.11' - name: Cache pip packages + if: github.event_name != 'issue_comment' uses: actions/cache@v4 with: path: ~/.cache/pip @@ -117,8 +198,15 @@ jobs: test-image-processors: name: Test image processors (vs HF, ${{ matrix.backend }}) - needs: lint-and-format + needs: [lint-and-format, authorize-tests] + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || + (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && + needs.authorize-tests.outputs.run_tests == 'true')) runs-on: ubuntu-latest + permissions: + contents: read strategy: fail-fast: false matrix: @@ -127,10 +215,14 @@ jobs: KERAS_BACKEND: ${{ matrix.backend }} steps: - uses: actions/checkout@v4 + with: + ref: ${{ needs.authorize-tests.outputs.head_sha }} + persist-credentials: ${{ github.event_name != 'issue_comment' }} - uses: actions/setup-python@v5 with: python-version: '3.11' - name: Cache pip packages + if: github.event_name != 'issue_comment' uses: actions/cache@v4 with: path: ~/.cache/pip @@ -154,8 +246,15 @@ jobs: test-processors: name: Test processors (vs HF, ${{ matrix.backend }}) - needs: lint-and-format + needs: [lint-and-format, authorize-tests] + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || + (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && + needs.authorize-tests.outputs.run_tests == 'true')) runs-on: ubuntu-latest + permissions: + contents: read strategy: fail-fast: false matrix: @@ -164,10 +263,14 @@ jobs: KERAS_BACKEND: ${{ matrix.backend }} steps: - uses: actions/checkout@v4 + with: + ref: ${{ needs.authorize-tests.outputs.head_sha }} + persist-credentials: ${{ github.event_name != 'issue_comment' }} - uses: actions/setup-python@v5 with: python-version: '3.11' - name: Cache pip packages + if: github.event_name != 'issue_comment' uses: actions/cache@v4 with: path: ~/.cache/pip @@ -188,3 +291,38 @@ jobs: run: | pytest tests/processors/test_keras_processors_vs_hf.py \ -v --durations=20 + + report-pr-tests: + name: Report requested PR tests + needs: [authorize-tests, test-code, test-tokenizers, test-image-processors, test-processors] + if: >- + always() && github.event_name == 'issue_comment' && + needs.authorize-tests.outputs.run_tests == 'true' + runs-on: ubuntu-latest + permissions: + checks: write + steps: + # Keep the write token isolated from the jobs that execute PR code. + - name: Publish the result on the tested PR commit + uses: actions/github-script@v9 + env: + TEST_RESULTS: ${{ toJSON(needs) }} + CHECK_RUN_ID: ${{ needs.authorize-tests.outputs.check_run_id }} + with: + script: | + const needs = JSON.parse(process.env.TEST_RESULTS); + const groups = ['test-code', 'test-tokenizers', 'test-image-processors', 'test-processors']; + const results = groups.map(name => needs[name].result); + const conclusion = results.every(result => result === 'success') + ? 'success' + : results.includes('cancelled') ? 'cancelled' : 'failure'; + await github.rest.checks.update({ + ...context.repo, + check_run_id: Number(process.env.CHECK_RUN_ID), + status: 'completed', + conclusion, + output: { + title: `Requested PR tests: ${conclusion}`, + summary: groups.map(name => `- ${name}: ${needs[name].result}`).join('\n'), + }, + }); From e7df46f01ba7fb2c4d62aaab692e9001017d0527 Mon Sep 17 00:00:00 2001 From: IMvision12 Date: Thu, 15 Oct 2026 19:33:58 -0700 Subject: [PATCH 2/2] Update --- .github/TESTING.md | 17 +- .github/workflows/model_tests.yml | 327 ++++++++++++++++++++++++++++++ .github/workflows/test_code.yml | 312 ---------------------------- 3 files changed, 338 insertions(+), 318 deletions(-) create mode 100644 .github/workflows/model_tests.yml diff --git a/.github/TESTING.md b/.github/TESTING.md index bca84457..a7e5b4f9 100644 --- a/.github/TESTING.md +++ b/.github/TESTING.md @@ -1,8 +1,12 @@ # Pull request test commands -Lint & Format continues to run automatically on pull requests. Model, tokenizer, -image-processor, and processor tests run on a pull request only when **IMvision12** -posts a new conversation comment containing exactly: +The automatic PR workflow (`test_code.yml`) contains only **Lint & Format**. +Model, tokenizer, image-processor, and processor jobs live in a separate workflow +(`model_tests.yml`) that does not subscribe to pull request events. They do not +appear as queued or skipped PR checks before a command is posted. + +Those tests run on a pull request only when **IMvision12** posts a new +conversation comment containing exactly: ```text /run-tests @@ -20,14 +24,15 @@ does not start another test run; post `/run-tests` again to test the new commit. Results appear on the tested PR commit as **Requested model, tokenizer & processor tests**, with a link to the workflow's individual jobs. If these tests must block merging, require this aggregate check in the repository's branch protection or -ruleset; automatically skipped test jobs are not a substitute for that check. +ruleset. The test check is created only after an authorized command. The updated workflow must be present on the repository's default branch before GitHub delivers comment-triggered runs. No additional personal access token or repository secret is needed. -Release-triggered tests still run automatically after lint succeeds. The release, -documentation, and standalone cross-backend parity workflows are unchanged. +Release events start the same lint and test jobs automatically in +`model_tests.yml`, with the existing commands and matrices unchanged. The release, +documentation, and standalone cross-backend parity workflow files are unchanged. Authorization and result-reporting jobs have check-writing permission but never check out PR code. Test jobs have read-only repository access and do not persist diff --git a/.github/workflows/model_tests.yml b/.github/workflows/model_tests.yml new file mode 100644 index 00000000..38ae3748 --- /dev/null +++ b/.github/workflows/model_tests.yml @@ -0,0 +1,327 @@ +name: Model tests +on: + release: + types: [created] + issue_comment: + types: [created] + +jobs: + lint-and-format: + name: Lint & Format + if: github.event_name == 'release' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: pip install ruff==0.16.1 + - run: ruff check . + - run: ruff format --check . + + authorize-tests: + name: Authorize PR tests + if: >- + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + github.event.comment.user.login == 'IMvision12' && + github.triggering_actor == 'IMvision12' && + github.event.comment.body == '/run-tests' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + checks: write + outputs: + run_tests: ${{ steps.authorize.outputs.run_tests }} + head_sha: ${{ steps.authorize.outputs.head_sha }} + check_run_id: ${{ steps.authorize.outputs.check_run_id }} + steps: + # This privileged job never checks out or executes PR-controlled files. + - name: Validate owner command and pin the PR commit + id: authorize + uses: actions/github-script@v9 + with: + script: | + const { issue, comment, action } = context.payload; + if (action !== 'created' || !issue?.pull_request || + comment?.user?.login?.toLowerCase() !== 'imvision12' || + comment.body !== '/run-tests') { + throw new Error('Only IMvision12 can request PR tests with /run-tests.'); + } + const { data: pr } = await github.rest.pulls.get({ + ...context.repo, + pull_number: issue.number, + }); + if (pr.state !== 'open') { + throw new Error('Tests can only be requested on an open pull request.'); + } + if (!/^[0-9a-f]{40}$/.test(pr.head.sha)) { + throw new Error('GitHub returned an invalid PR head commit.'); + } + const { data: check } = await github.rest.checks.create({ + ...context.repo, + name: 'Requested model, tokenizer & processor tests', + head_sha: pr.head.sha, + status: 'in_progress', + details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + output: { + title: 'PR tests requested by IMvision12', + summary: `Testing PR #${issue.number} at ${pr.head.sha}. New pushes require a new /run-tests comment.`, + }, + }); + core.setOutput('head_sha', pr.head.sha); + core.setOutput('check_run_id', String(check.id)); + core.setOutput('run_tests', 'true'); + + test-code: + name: Test the code (${{ matrix.backend }} ${{ matrix.shard }}) + needs: [lint-and-format, authorize-tests] + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || + (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && + needs.authorize-tests.outputs.run_tests == 'true')) + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + # Shard the model suite BY MODEL across parallel jobs (see the --shard + # option in tests/conftest.py). torch is fast enough unsharded; the JAX and + # TF legs trace + XLA-compile every model, so they are split to stay under + # the per-job time cap. Stacks with the per-model build-once reuse. + matrix: + include: + - { backend: torch, shard: "1/2" } + - { backend: torch, shard: "2/2" } + - { backend: tensorflow, shard: "1/3" } + - { backend: tensorflow, shard: "2/3" } + - { backend: tensorflow, shard: "3/3" } + - { backend: jax, shard: "1/4" } + - { backend: jax, shard: "2/4" } + - { backend: jax, shard: "3/4" } + - { backend: jax, shard: "4/4" } + env: + KERAS_BACKEND: ${{ matrix.backend }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.authorize-tests.outputs.head_sha }} + persist-credentials: ${{ github.event_name != 'issue_comment' }} + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Cache pip packages + if: github.event_name != 'issue_comment' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ matrix.backend }}-${{ hashFiles('pyproject.toml') }} + restore-keys: pip-${{ matrix.backend }}- + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -U "keras>=3.15.0" pytest pytest-cov + # CPU-only torch wheel: the default PyPI wheel bundles + # CUDA/Triton which segfaults during `import triton.knobs` + # on CUDA-less Linux runners (gets pulled in transitively by + # torchvision -> torch._dynamo). + pip install --index-url https://download.pytorch.org/whl/cpu \ + "torch>=2.5.0" torchvision + pip install "jax[cpu]" "tensorflow-cpu>=2.19.0" + pip install sentencepiece "matplotlib>=3.8.0" pillow + pip install huggingface_hub safetensors + pip install "transformers>=5.0.0" + pip install . + - name: Run all tests + run: | + pytest tests/integration/test_backend_compatibility.py \ + tests/integration/test_serialization.py \ + tests/integration/test_model_saving.py \ + tests/integration/test_data_formats.py \ + tests/integration/test_auto_registry.py \ + --shard ${{ matrix.shard }} \ + -v --durations=20 --cov=zeromodels --cov-append \ + -m "not slow and not gpu" + - name: Generate coverage + run: coverage xml -o coverage.xml + - name: Upload coverage + if: github.event_name != 'issue_comment' + uses: codecov/codecov-action@v5 + with: + files: coverage.xml + flags: ${{ matrix.backend }} + fail_ci_if_error: false + + test-tokenizers: + name: Test tokenizers (vs HF) + needs: [lint-and-format, authorize-tests] + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || + (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && + needs.authorize-tests.outputs.run_tests == 'true')) + runs-on: ubuntu-latest + permissions: + contents: read + env: + KERAS_BACKEND: torch + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.authorize-tests.outputs.head_sha }} + persist-credentials: ${{ github.event_name != 'issue_comment' }} + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Cache pip packages + if: github.event_name != 'issue_comment' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-tokenizers-${{ hashFiles('pyproject.toml') }} + restore-keys: pip-tokenizers- + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -U "keras>=3.15.0" pytest + pip install --index-url https://download.pytorch.org/whl/cpu "torch>=2.5.0" + pip install sentencepiece huggingface_hub "matplotlib>=3.8.0" pillow + pip install "transformers>=5.0.0" + pip install . + - name: Run tokenizer parity tests + run: | + pytest tests/processors/test_keras_tokenizers_vs_hf.py \ + -v --durations=20 + + test-image-processors: + name: Test image processors (vs HF, ${{ matrix.backend }}) + needs: [lint-and-format, authorize-tests] + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || + (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && + needs.authorize-tests.outputs.run_tests == 'true')) + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + backend: [torch, tensorflow, jax] + env: + KERAS_BACKEND: ${{ matrix.backend }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.authorize-tests.outputs.head_sha }} + persist-credentials: ${{ github.event_name != 'issue_comment' }} + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Cache pip packages + if: github.event_name != 'issue_comment' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ matrix.backend }}-${{ hashFiles('pyproject.toml') }} + restore-keys: pip-${{ matrix.backend }}- + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -U "keras>=3.15.0" pytest + pip install --index-url https://download.pytorch.org/whl/cpu \ + "torch>=2.5.0" torchvision + pip install "jax[cpu]" "tensorflow-cpu>=2.19.0" + pip install sentencepiece "matplotlib>=3.8.0" pillow + pip install huggingface_hub safetensors + pip install "transformers>=5.0.0" + pip install . + - name: Run image processor parity tests + run: | + pytest tests/processors/test_keras_image_processors_vs_hf.py \ + -v --durations=20 + + test-processors: + name: Test processors (vs HF, ${{ matrix.backend }}) + needs: [lint-and-format, authorize-tests] + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || + (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && + needs.authorize-tests.outputs.run_tests == 'true')) + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + backend: [torch, tensorflow, jax] + env: + KERAS_BACKEND: ${{ matrix.backend }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.authorize-tests.outputs.head_sha }} + persist-credentials: ${{ github.event_name != 'issue_comment' }} + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Cache pip packages + if: github.event_name != 'issue_comment' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ matrix.backend }}-${{ hashFiles('pyproject.toml') }} + restore-keys: pip-${{ matrix.backend }}- + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -U "keras>=3.15.0" pytest + pip install --index-url https://download.pytorch.org/whl/cpu \ + "torch>=2.5.0" torchvision + pip install "jax[cpu]" "tensorflow-cpu>=2.19.0" + pip install sentencepiece "matplotlib>=3.8.0" pillow + pip install huggingface_hub safetensors + pip install "transformers>=5.0.0" + pip install . + - name: Run processor parity tests + run: | + pytest tests/processors/test_keras_processors_vs_hf.py \ + -v --durations=20 + + report-pr-tests: + name: Report requested PR tests + needs: [authorize-tests, test-code, test-tokenizers, test-image-processors, test-processors] + if: >- + always() && github.event_name == 'issue_comment' && + needs.authorize-tests.outputs.run_tests == 'true' + runs-on: ubuntu-latest + permissions: + checks: write + steps: + # Keep the write token isolated from the jobs that execute PR code. + - name: Publish the result on the tested PR commit + uses: actions/github-script@v9 + env: + TEST_RESULTS: ${{ toJSON(needs) }} + CHECK_RUN_ID: ${{ needs.authorize-tests.outputs.check_run_id }} + with: + script: | + const needs = JSON.parse(process.env.TEST_RESULTS); + const groups = ['test-code', 'test-tokenizers', 'test-image-processors', 'test-processors']; + const results = groups.map(name => needs[name].result); + const conclusion = results.every(result => result === 'success') + ? 'success' + : results.includes('cancelled') ? 'cancelled' : 'failure'; + await github.rest.checks.update({ + ...context.repo, + check_run_id: Number(process.env.CHECK_RUN_ID), + status: 'completed', + conclusion, + output: { + title: `Requested PR tests: ${conclusion}`, + summary: groups.map(name => `- ${name}: ${needs[name].result}`).join('\n'), + }, + }); diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index 1506f2ed..865ea8b6 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -1,15 +1,10 @@ name: Tests on: pull_request: - release: - types: [created] - issue_comment: - types: [created] jobs: lint-and-format: name: Lint & Format - if: github.event_name != 'issue_comment' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -19,310 +14,3 @@ jobs: - run: pip install ruff==0.16.1 - run: ruff check . - run: ruff format --check . - - authorize-tests: - name: Authorize PR tests - if: >- - github.event_name == 'issue_comment' && - github.event.issue.pull_request && - github.event.comment.user.login == 'IMvision12' && - github.triggering_actor == 'IMvision12' && - github.event.comment.body == '/run-tests' - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - checks: write - outputs: - run_tests: ${{ steps.authorize.outputs.run_tests }} - head_sha: ${{ steps.authorize.outputs.head_sha }} - check_run_id: ${{ steps.authorize.outputs.check_run_id }} - steps: - # This privileged job never checks out or executes PR-controlled files. - - name: Validate owner command and pin the PR commit - id: authorize - uses: actions/github-script@v9 - with: - script: | - const { issue, comment, action } = context.payload; - if (action !== 'created' || !issue?.pull_request || - comment?.user?.login?.toLowerCase() !== 'imvision12' || - comment.body !== '/run-tests') { - throw new Error('Only IMvision12 can request PR tests with /run-tests.'); - } - const { data: pr } = await github.rest.pulls.get({ - ...context.repo, - pull_number: issue.number, - }); - if (pr.state !== 'open') { - throw new Error('Tests can only be requested on an open pull request.'); - } - if (!/^[0-9a-f]{40}$/.test(pr.head.sha)) { - throw new Error('GitHub returned an invalid PR head commit.'); - } - const { data: check } = await github.rest.checks.create({ - ...context.repo, - name: 'Requested model, tokenizer & processor tests', - head_sha: pr.head.sha, - status: 'in_progress', - details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, - output: { - title: 'PR tests requested by IMvision12', - summary: `Testing PR #${issue.number} at ${pr.head.sha}. New pushes require a new /run-tests comment.`, - }, - }); - core.setOutput('head_sha', pr.head.sha); - core.setOutput('check_run_id', String(check.id)); - core.setOutput('run_tests', 'true'); - - test-code: - name: Test the code (${{ matrix.backend }} ${{ matrix.shard }}) - needs: [lint-and-format, authorize-tests] - if: >- - always() && !cancelled() && - ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || - (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && - needs.authorize-tests.outputs.run_tests == 'true')) - runs-on: ubuntu-latest - permissions: - contents: read - strategy: - fail-fast: false - # Shard the model suite BY MODEL across parallel jobs (see the --shard - # option in tests/conftest.py). torch is fast enough unsharded; the JAX and - # TF legs trace + XLA-compile every model, so they are split to stay under - # the per-job time cap. Stacks with the per-model build-once reuse. - matrix: - include: - - { backend: torch, shard: "1/2" } - - { backend: torch, shard: "2/2" } - - { backend: tensorflow, shard: "1/3" } - - { backend: tensorflow, shard: "2/3" } - - { backend: tensorflow, shard: "3/3" } - - { backend: jax, shard: "1/4" } - - { backend: jax, shard: "2/4" } - - { backend: jax, shard: "3/4" } - - { backend: jax, shard: "4/4" } - env: - KERAS_BACKEND: ${{ matrix.backend }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.authorize-tests.outputs.head_sha }} - persist-credentials: ${{ github.event_name != 'issue_comment' }} - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Cache pip packages - if: github.event_name != 'issue_comment' - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ matrix.backend }}-${{ hashFiles('pyproject.toml') }} - restore-keys: pip-${{ matrix.backend }}- - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -U "keras>=3.15.0" pytest pytest-cov - # CPU-only torch wheel: the default PyPI wheel bundles - # CUDA/Triton which segfaults during `import triton.knobs` - # on CUDA-less Linux runners (gets pulled in transitively by - # torchvision -> torch._dynamo). - pip install --index-url https://download.pytorch.org/whl/cpu \ - "torch>=2.5.0" torchvision - pip install "jax[cpu]" "tensorflow-cpu>=2.19.0" - pip install sentencepiece "matplotlib>=3.8.0" pillow - pip install huggingface_hub safetensors - pip install "transformers>=5.0.0" - pip install . - - name: Run all tests - run: | - pytest tests/integration/test_backend_compatibility.py \ - tests/integration/test_serialization.py \ - tests/integration/test_model_saving.py \ - tests/integration/test_data_formats.py \ - tests/integration/test_auto_registry.py \ - --shard ${{ matrix.shard }} \ - -v --durations=20 --cov=zeromodels --cov-append \ - -m "not slow and not gpu" - - name: Generate coverage - run: coverage xml -o coverage.xml - - name: Upload coverage - if: github.event_name != 'issue_comment' - uses: codecov/codecov-action@v5 - with: - files: coverage.xml - flags: ${{ matrix.backend }} - fail_ci_if_error: false - - test-tokenizers: - name: Test tokenizers (vs HF) - needs: [lint-and-format, authorize-tests] - if: >- - always() && !cancelled() && - ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || - (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && - needs.authorize-tests.outputs.run_tests == 'true')) - runs-on: ubuntu-latest - permissions: - contents: read - env: - KERAS_BACKEND: torch - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.authorize-tests.outputs.head_sha }} - persist-credentials: ${{ github.event_name != 'issue_comment' }} - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Cache pip packages - if: github.event_name != 'issue_comment' - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-tokenizers-${{ hashFiles('pyproject.toml') }} - restore-keys: pip-tokenizers- - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -U "keras>=3.15.0" pytest - pip install --index-url https://download.pytorch.org/whl/cpu "torch>=2.5.0" - pip install sentencepiece huggingface_hub "matplotlib>=3.8.0" pillow - pip install "transformers>=5.0.0" - pip install . - - name: Run tokenizer parity tests - run: | - pytest tests/processors/test_keras_tokenizers_vs_hf.py \ - -v --durations=20 - - test-image-processors: - name: Test image processors (vs HF, ${{ matrix.backend }}) - needs: [lint-and-format, authorize-tests] - if: >- - always() && !cancelled() && - ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || - (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && - needs.authorize-tests.outputs.run_tests == 'true')) - runs-on: ubuntu-latest - permissions: - contents: read - strategy: - fail-fast: false - matrix: - backend: [torch, tensorflow, jax] - env: - KERAS_BACKEND: ${{ matrix.backend }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.authorize-tests.outputs.head_sha }} - persist-credentials: ${{ github.event_name != 'issue_comment' }} - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Cache pip packages - if: github.event_name != 'issue_comment' - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ matrix.backend }}-${{ hashFiles('pyproject.toml') }} - restore-keys: pip-${{ matrix.backend }}- - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -U "keras>=3.15.0" pytest - pip install --index-url https://download.pytorch.org/whl/cpu \ - "torch>=2.5.0" torchvision - pip install "jax[cpu]" "tensorflow-cpu>=2.19.0" - pip install sentencepiece "matplotlib>=3.8.0" pillow - pip install huggingface_hub safetensors - pip install "transformers>=5.0.0" - pip install . - - name: Run image processor parity tests - run: | - pytest tests/processors/test_keras_image_processors_vs_hf.py \ - -v --durations=20 - - test-processors: - name: Test processors (vs HF, ${{ matrix.backend }}) - needs: [lint-and-format, authorize-tests] - if: >- - always() && !cancelled() && - ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || - (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && - needs.authorize-tests.outputs.run_tests == 'true')) - runs-on: ubuntu-latest - permissions: - contents: read - strategy: - fail-fast: false - matrix: - backend: [torch, tensorflow, jax] - env: - KERAS_BACKEND: ${{ matrix.backend }} - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.authorize-tests.outputs.head_sha }} - persist-credentials: ${{ github.event_name != 'issue_comment' }} - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Cache pip packages - if: github.event_name != 'issue_comment' - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ matrix.backend }}-${{ hashFiles('pyproject.toml') }} - restore-keys: pip-${{ matrix.backend }}- - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -U "keras>=3.15.0" pytest - pip install --index-url https://download.pytorch.org/whl/cpu \ - "torch>=2.5.0" torchvision - pip install "jax[cpu]" "tensorflow-cpu>=2.19.0" - pip install sentencepiece "matplotlib>=3.8.0" pillow - pip install huggingface_hub safetensors - pip install "transformers>=5.0.0" - pip install . - - name: Run processor parity tests - run: | - pytest tests/processors/test_keras_processors_vs_hf.py \ - -v --durations=20 - - report-pr-tests: - name: Report requested PR tests - needs: [authorize-tests, test-code, test-tokenizers, test-image-processors, test-processors] - if: >- - always() && github.event_name == 'issue_comment' && - needs.authorize-tests.outputs.run_tests == 'true' - runs-on: ubuntu-latest - permissions: - checks: write - steps: - # Keep the write token isolated from the jobs that execute PR code. - - name: Publish the result on the tested PR commit - uses: actions/github-script@v9 - env: - TEST_RESULTS: ${{ toJSON(needs) }} - CHECK_RUN_ID: ${{ needs.authorize-tests.outputs.check_run_id }} - with: - script: | - const needs = JSON.parse(process.env.TEST_RESULTS); - const groups = ['test-code', 'test-tokenizers', 'test-image-processors', 'test-processors']; - const results = groups.map(name => needs[name].result); - const conclusion = results.every(result => result === 'success') - ? 'success' - : results.includes('cancelled') ? 'cancelled' : 'failure'; - await github.rest.checks.update({ - ...context.repo, - check_run_id: Number(process.env.CHECK_RUN_ID), - status: 'completed', - conclusion, - output: { - title: `Requested PR tests: ${conclusion}`, - summary: groups.map(name => `- ${name}: ${needs[name].result}`).join('\n'), - }, - });