diff --git a/.github/TESTING.md b/.github/TESTING.md new file mode 100644 index 00000000..a7e5b4f9 --- /dev/null +++ b/.github/TESTING.md @@ -0,0 +1,41 @@ +# Pull request test commands + +The automatic PR workflow (`test_code.yml`) contains only **Lint & Format**. +Model, tokenizer, image-processor, and processor jobs live in a separate workflow +(`model_tests.yml`) that does not subscribe to pull request events. They do not +appear as queued or skipped PR checks before a command is posted. + +Those tests run on a pull request only when **IMvision12** posts a new +conversation comment containing exactly: + +```text +/run-tests +``` + +Other users' commands, issue comments, edited comments, and other text do not +authorize the test suite. Re-running a comment-triggered test job also requires +IMvision12 to be the triggering account. + +The command starts the existing 16 test jobs: 9 model shards (Torch, TensorFlow, +and JAX), 1 tokenizer job, 3 image-processor jobs, and 3 processor jobs. They test +the open PR's head commit captured when the request is authorized. A later push +does not start another test run; post `/run-tests` again to test the new commit. + +Results appear on the tested PR commit as **Requested model, tokenizer & processor +tests**, with a link to the workflow's individual jobs. If these tests must block +merging, require this aggregate check in the repository's branch protection or +ruleset. The test check is created only after an authorized command. + +The updated workflow must be present on the repository's default branch before +GitHub delivers comment-triggered runs. No additional personal access token or +repository secret is needed. + +Release events start the same lint and test jobs automatically in +`model_tests.yml`, with the existing commands and matrices unchanged. The release, +documentation, and standalone cross-backend parity workflow files are unchanged. + +Authorization and result-reporting jobs have check-writing permission but never +check out PR code. Test jobs have read-only repository access and do not persist +checkout credentials. Comment-triggered tests do not use shared Actions caches +or publish Codecov coverage from the default-branch event context; release runs +retain their existing cache and coverage steps. diff --git a/.github/workflows/model_tests.yml b/.github/workflows/model_tests.yml new file mode 100644 index 00000000..38ae3748 --- /dev/null +++ b/.github/workflows/model_tests.yml @@ -0,0 +1,327 @@ +name: Model tests +on: + release: + types: [created] + issue_comment: + types: [created] + +jobs: + lint-and-format: + name: Lint & Format + if: github.event_name == 'release' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - run: pip install ruff==0.16.1 + - run: ruff check . + - run: ruff format --check . + + authorize-tests: + name: Authorize PR tests + if: >- + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + github.event.comment.user.login == 'IMvision12' && + github.triggering_actor == 'IMvision12' && + github.event.comment.body == '/run-tests' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + checks: write + outputs: + run_tests: ${{ steps.authorize.outputs.run_tests }} + head_sha: ${{ steps.authorize.outputs.head_sha }} + check_run_id: ${{ steps.authorize.outputs.check_run_id }} + steps: + # This privileged job never checks out or executes PR-controlled files. + - name: Validate owner command and pin the PR commit + id: authorize + uses: actions/github-script@v9 + with: + script: | + const { issue, comment, action } = context.payload; + if (action !== 'created' || !issue?.pull_request || + comment?.user?.login?.toLowerCase() !== 'imvision12' || + comment.body !== '/run-tests') { + throw new Error('Only IMvision12 can request PR tests with /run-tests.'); + } + const { data: pr } = await github.rest.pulls.get({ + ...context.repo, + pull_number: issue.number, + }); + if (pr.state !== 'open') { + throw new Error('Tests can only be requested on an open pull request.'); + } + if (!/^[0-9a-f]{40}$/.test(pr.head.sha)) { + throw new Error('GitHub returned an invalid PR head commit.'); + } + const { data: check } = await github.rest.checks.create({ + ...context.repo, + name: 'Requested model, tokenizer & processor tests', + head_sha: pr.head.sha, + status: 'in_progress', + details_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + output: { + title: 'PR tests requested by IMvision12', + summary: `Testing PR #${issue.number} at ${pr.head.sha}. New pushes require a new /run-tests comment.`, + }, + }); + core.setOutput('head_sha', pr.head.sha); + core.setOutput('check_run_id', String(check.id)); + core.setOutput('run_tests', 'true'); + + test-code: + name: Test the code (${{ matrix.backend }} ${{ matrix.shard }}) + needs: [lint-and-format, authorize-tests] + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || + (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && + needs.authorize-tests.outputs.run_tests == 'true')) + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + # Shard the model suite BY MODEL across parallel jobs (see the --shard + # option in tests/conftest.py). torch is fast enough unsharded; the JAX and + # TF legs trace + XLA-compile every model, so they are split to stay under + # the per-job time cap. Stacks with the per-model build-once reuse. + matrix: + include: + - { backend: torch, shard: "1/2" } + - { backend: torch, shard: "2/2" } + - { backend: tensorflow, shard: "1/3" } + - { backend: tensorflow, shard: "2/3" } + - { backend: tensorflow, shard: "3/3" } + - { backend: jax, shard: "1/4" } + - { backend: jax, shard: "2/4" } + - { backend: jax, shard: "3/4" } + - { backend: jax, shard: "4/4" } + env: + KERAS_BACKEND: ${{ matrix.backend }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.authorize-tests.outputs.head_sha }} + persist-credentials: ${{ github.event_name != 'issue_comment' }} + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Cache pip packages + if: github.event_name != 'issue_comment' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ matrix.backend }}-${{ hashFiles('pyproject.toml') }} + restore-keys: pip-${{ matrix.backend }}- + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -U "keras>=3.15.0" pytest pytest-cov + # CPU-only torch wheel: the default PyPI wheel bundles + # CUDA/Triton which segfaults during `import triton.knobs` + # on CUDA-less Linux runners (gets pulled in transitively by + # torchvision -> torch._dynamo). + pip install --index-url https://download.pytorch.org/whl/cpu \ + "torch>=2.5.0" torchvision + pip install "jax[cpu]" "tensorflow-cpu>=2.19.0" + pip install sentencepiece "matplotlib>=3.8.0" pillow + pip install huggingface_hub safetensors + pip install "transformers>=5.0.0" + pip install . + - name: Run all tests + run: | + pytest tests/integration/test_backend_compatibility.py \ + tests/integration/test_serialization.py \ + tests/integration/test_model_saving.py \ + tests/integration/test_data_formats.py \ + tests/integration/test_auto_registry.py \ + --shard ${{ matrix.shard }} \ + -v --durations=20 --cov=zeromodels --cov-append \ + -m "not slow and not gpu" + - name: Generate coverage + run: coverage xml -o coverage.xml + - name: Upload coverage + if: github.event_name != 'issue_comment' + uses: codecov/codecov-action@v5 + with: + files: coverage.xml + flags: ${{ matrix.backend }} + fail_ci_if_error: false + + test-tokenizers: + name: Test tokenizers (vs HF) + needs: [lint-and-format, authorize-tests] + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || + (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && + needs.authorize-tests.outputs.run_tests == 'true')) + runs-on: ubuntu-latest + permissions: + contents: read + env: + KERAS_BACKEND: torch + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.authorize-tests.outputs.head_sha }} + persist-credentials: ${{ github.event_name != 'issue_comment' }} + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Cache pip packages + if: github.event_name != 'issue_comment' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-tokenizers-${{ hashFiles('pyproject.toml') }} + restore-keys: pip-tokenizers- + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -U "keras>=3.15.0" pytest + pip install --index-url https://download.pytorch.org/whl/cpu "torch>=2.5.0" + pip install sentencepiece huggingface_hub "matplotlib>=3.8.0" pillow + pip install "transformers>=5.0.0" + pip install . + - name: Run tokenizer parity tests + run: | + pytest tests/processors/test_keras_tokenizers_vs_hf.py \ + -v --durations=20 + + test-image-processors: + name: Test image processors (vs HF, ${{ matrix.backend }}) + needs: [lint-and-format, authorize-tests] + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || + (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && + needs.authorize-tests.outputs.run_tests == 'true')) + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + backend: [torch, tensorflow, jax] + env: + KERAS_BACKEND: ${{ matrix.backend }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.authorize-tests.outputs.head_sha }} + persist-credentials: ${{ github.event_name != 'issue_comment' }} + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Cache pip packages + if: github.event_name != 'issue_comment' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ matrix.backend }}-${{ hashFiles('pyproject.toml') }} + restore-keys: pip-${{ matrix.backend }}- + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -U "keras>=3.15.0" pytest + pip install --index-url https://download.pytorch.org/whl/cpu \ + "torch>=2.5.0" torchvision + pip install "jax[cpu]" "tensorflow-cpu>=2.19.0" + pip install sentencepiece "matplotlib>=3.8.0" pillow + pip install huggingface_hub safetensors + pip install "transformers>=5.0.0" + pip install . + - name: Run image processor parity tests + run: | + pytest tests/processors/test_keras_image_processors_vs_hf.py \ + -v --durations=20 + + test-processors: + name: Test processors (vs HF, ${{ matrix.backend }}) + needs: [lint-and-format, authorize-tests] + if: >- + always() && !cancelled() && + ((github.event_name == 'release' && needs.lint-and-format.result == 'success') || + (github.event_name == 'issue_comment' && github.triggering_actor == 'IMvision12' && + needs.authorize-tests.outputs.run_tests == 'true')) + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + backend: [torch, tensorflow, jax] + env: + KERAS_BACKEND: ${{ matrix.backend }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.authorize-tests.outputs.head_sha }} + persist-credentials: ${{ github.event_name != 'issue_comment' }} + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Cache pip packages + if: github.event_name != 'issue_comment' + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: pip-${{ matrix.backend }}-${{ hashFiles('pyproject.toml') }} + restore-keys: pip-${{ matrix.backend }}- + - name: Install dependencies + run: | + python -m pip install --upgrade pip setuptools + pip install -U "keras>=3.15.0" pytest + pip install --index-url https://download.pytorch.org/whl/cpu \ + "torch>=2.5.0" torchvision + pip install "jax[cpu]" "tensorflow-cpu>=2.19.0" + pip install sentencepiece "matplotlib>=3.8.0" pillow + pip install huggingface_hub safetensors + pip install "transformers>=5.0.0" + pip install . + - name: Run processor parity tests + run: | + pytest tests/processors/test_keras_processors_vs_hf.py \ + -v --durations=20 + + report-pr-tests: + name: Report requested PR tests + needs: [authorize-tests, test-code, test-tokenizers, test-image-processors, test-processors] + if: >- + always() && github.event_name == 'issue_comment' && + needs.authorize-tests.outputs.run_tests == 'true' + runs-on: ubuntu-latest + permissions: + checks: write + steps: + # Keep the write token isolated from the jobs that execute PR code. + - name: Publish the result on the tested PR commit + uses: actions/github-script@v9 + env: + TEST_RESULTS: ${{ toJSON(needs) }} + CHECK_RUN_ID: ${{ needs.authorize-tests.outputs.check_run_id }} + with: + script: | + const needs = JSON.parse(process.env.TEST_RESULTS); + const groups = ['test-code', 'test-tokenizers', 'test-image-processors', 'test-processors']; + const results = groups.map(name => needs[name].result); + const conclusion = results.every(result => result === 'success') + ? 'success' + : results.includes('cancelled') ? 'cancelled' : 'failure'; + await github.rest.checks.update({ + ...context.repo, + check_run_id: Number(process.env.CHECK_RUN_ID), + status: 'completed', + conclusion, + output: { + title: `Requested PR tests: ${conclusion}`, + summary: groups.map(name => `- ${name}: ${needs[name].result}`).join('\n'), + }, + }); diff --git a/.github/workflows/test_code.yml b/.github/workflows/test_code.yml index 405b5b11..865ea8b6 100644 --- a/.github/workflows/test_code.yml +++ b/.github/workflows/test_code.yml @@ -1,8 +1,6 @@ name: Tests on: pull_request: - release: - types: [created] jobs: lint-and-format: @@ -16,175 +14,3 @@ jobs: - run: pip install ruff==0.16.1 - run: ruff check . - run: ruff format --check . - - test-code: - name: Test the code (${{ matrix.backend }} ${{ matrix.shard }}) - needs: lint-and-format - runs-on: ubuntu-latest - strategy: - fail-fast: false - # Shard the model suite BY MODEL across parallel jobs (see the --shard - # option in tests/conftest.py). torch is fast enough unsharded; the JAX and - # TF legs trace + XLA-compile every model, so they are split to stay under - # the per-job time cap. Stacks with the per-model build-once reuse. - matrix: - include: - - { backend: torch, shard: "1/2" } - - { backend: torch, shard: "2/2" } - - { backend: tensorflow, shard: "1/3" } - - { backend: tensorflow, shard: "2/3" } - - { backend: tensorflow, shard: "3/3" } - - { backend: jax, shard: "1/4" } - - { backend: jax, shard: "2/4" } - - { backend: jax, shard: "3/4" } - - { backend: jax, shard: "4/4" } - env: - KERAS_BACKEND: ${{ matrix.backend }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Cache pip packages - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ matrix.backend }}-${{ hashFiles('pyproject.toml') }} - restore-keys: pip-${{ matrix.backend }}- - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -U "keras>=3.15.0" pytest pytest-cov - # CPU-only torch wheel: the default PyPI wheel bundles - # CUDA/Triton which segfaults during `import triton.knobs` - # on CUDA-less Linux runners (gets pulled in transitively by - # torchvision -> torch._dynamo). - pip install --index-url https://download.pytorch.org/whl/cpu \ - "torch>=2.5.0" torchvision - pip install "jax[cpu]" "tensorflow-cpu>=2.19.0" - pip install sentencepiece "matplotlib>=3.8.0" pillow - pip install huggingface_hub safetensors - pip install "transformers>=5.0.0" - pip install . - - name: Run all tests - run: | - pytest tests/integration/test_backend_compatibility.py \ - tests/integration/test_serialization.py \ - tests/integration/test_model_saving.py \ - tests/integration/test_data_formats.py \ - tests/integration/test_auto_registry.py \ - --shard ${{ matrix.shard }} \ - -v --durations=20 --cov=zeromodels --cov-append \ - -m "not slow and not gpu" - - name: Generate coverage - run: coverage xml -o coverage.xml - - name: Upload coverage - uses: codecov/codecov-action@v5 - with: - files: coverage.xml - flags: ${{ matrix.backend }} - fail_ci_if_error: false - - test-tokenizers: - name: Test tokenizers (vs HF) - needs: lint-and-format - runs-on: ubuntu-latest - env: - KERAS_BACKEND: torch - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Cache pip packages - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-tokenizers-${{ hashFiles('pyproject.toml') }} - restore-keys: pip-tokenizers- - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -U "keras>=3.15.0" pytest - pip install --index-url https://download.pytorch.org/whl/cpu "torch>=2.5.0" - pip install sentencepiece huggingface_hub "matplotlib>=3.8.0" pillow - pip install "transformers>=5.0.0" - pip install . - - name: Run tokenizer parity tests - run: | - pytest tests/processors/test_keras_tokenizers_vs_hf.py \ - -v --durations=20 - - test-image-processors: - name: Test image processors (vs HF, ${{ matrix.backend }}) - needs: lint-and-format - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - backend: [torch, tensorflow, jax] - env: - KERAS_BACKEND: ${{ matrix.backend }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Cache pip packages - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ matrix.backend }}-${{ hashFiles('pyproject.toml') }} - restore-keys: pip-${{ matrix.backend }}- - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -U "keras>=3.15.0" pytest - pip install --index-url https://download.pytorch.org/whl/cpu \ - "torch>=2.5.0" torchvision - pip install "jax[cpu]" "tensorflow-cpu>=2.19.0" - pip install sentencepiece "matplotlib>=3.8.0" pillow - pip install huggingface_hub safetensors - pip install "transformers>=5.0.0" - pip install . - - name: Run image processor parity tests - run: | - pytest tests/processors/test_keras_image_processors_vs_hf.py \ - -v --durations=20 - - test-processors: - name: Test processors (vs HF, ${{ matrix.backend }}) - needs: lint-and-format - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - backend: [torch, tensorflow, jax] - env: - KERAS_BACKEND: ${{ matrix.backend }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Cache pip packages - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: pip-${{ matrix.backend }}-${{ hashFiles('pyproject.toml') }} - restore-keys: pip-${{ matrix.backend }}- - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools - pip install -U "keras>=3.15.0" pytest - pip install --index-url https://download.pytorch.org/whl/cpu \ - "torch>=2.5.0" torchvision - pip install "jax[cpu]" "tensorflow-cpu>=2.19.0" - pip install sentencepiece "matplotlib>=3.8.0" pillow - pip install huggingface_hub safetensors - pip install "transformers>=5.0.0" - pip install . - - name: Run processor parity tests - run: | - pytest tests/processors/test_keras_processors_vs_hf.py \ - -v --durations=20