diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d4a312f..5d73ba6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7 with: persist-credentials: false - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7 with: persist-credentials: false - name: Run native mock evaluation through the packaged Action diff --git a/.github/workflows/copilot-e2e.yml b/.github/workflows/copilot-e2e.yml index 9474fb9..ea17b85 100644 --- a/.github/workflows/copilot-e2e.yml +++ b/.github/workflows/copilot-e2e.yml @@ -28,7 +28,7 @@ jobs: group: evals-${{ github.workflow_ref }} cancel-in-progress: false steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7 with: persist-credentials: false - uses: ./ diff --git a/AGENTS.md b/AGENTS.md index 26805e2..6cad6cc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ - Keep this a thin GitHub adapter over the acquired Vally CLI. Translate supported commands and consume native JSONL; never duplicate selection, - configuration parsing, grading or execution. No Runtime/Core bundle is required. + configuration parsing, grading or execution. - Use Node 24, the pinned pnpm version, and one lockfile. Install with scripts disabled. Run focused tests during development and `pnpm check` before review. - Rebuild and commit `dist` after source changes. Never manually edit bundles. @@ -12,6 +12,7 @@ completion or use a report-only override for native failures. - Treat prompts, source, raw evidence and credentials as private. Default artifacts are allowlisted structured projections only. -- Use argument-array process calls and SHA-pinned third-party Actions. +- Use argument-array process calls. Use `actions/checkout@v7`; keep other + third-party Actions SHA-pinned. - Qualify native behavior with the built-in mock executor and deterministic graders. Do not substitute model-backed runs for model-free acceptance. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 30fbb6d..407a335 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,8 +21,9 @@ source-to-bundle comparison, and a clean consumer-directory packaging control. The clean consumer-directory packaging control deliberately lacks artifact credentials and verifies invalid-input failure. Linux CI separately exercises the packaged Action with real Vally acquisition, native mock-executor pass/fail -and lint, plus hosted artifact upload. No model credentials belong in CI or -test fixtures. +and lint, plus hosted artifact upload. Do not add model credentials to the +model-free workflow or unit-test fixtures. The separate Copilot E2E workflow +uses its automatic job token for the approved live example. PR check names map directly to these commands: @@ -33,14 +34,14 @@ PR check names map directly to these commands: | Unit tests | `pnpm test:coverage`: input, policy, reporting, mocked GitHub, tool reuse, orchestration and real model-free native CLI tests with coverage thresholds. | | Bundle verification | `pnpm check:package`: committed bundle freshness and a clean consumer-directory packaged entry-point test. | | Action smoke test | Executes `uses: ./` on Ubuntu with actual acquisition, native mock-executor pass/fail, lint, installation reuse and artifact upload. No models run. | -| Copilot E2E | For relevant same-repository PRs, lints the example and runs one real Copilot skill eval. This consumes organization Copilot AI credits. | +| evals (Copilot E2E workflow) | For relevant same-repository PRs, directly invokes the default Action on the example. This consumes organization Copilot AI credits. | Comment networking tests use synthetic GitHub responses, including serialized workflow guards, pagination, bot ownership, late runs/attempts, permission failures and readback. Installer tests use synthetic packages and exercise concurrent install/reuse and tampered files/receipts. `tests/native.test.ts` separately qualifies the real pinned Vally CLI with the built-in mock executor -and deterministic graders. No test claims live GitHub bot-token qualification. +and deterministic graders. Unit tests do not establish live GitHub token acceptance. The live `Copilot E2E` workflow directly exercises the default Action with one checkout and one `uses: ./` step. It deliberately uses combined Copilot and PR-write permissions in one job, just like the simplest consumer setup. @@ -54,11 +55,12 @@ without installing this repository. `pnpm check:dist` compares build bytes with the checked-in bundle. Never hand-edit generated files. Update the source, rebuild, and review both source and distribution changes. -Use strict TypeScript, argument-array process invocation, exact dependencies, -and SHA-pinned third-party Actions. Do not log raw evaluator/process output or +Use strict TypeScript, argument-array process invocation, and exact dependencies. +Use `actions/checkout@v7`; other third-party Actions remain SHA-pinned. +Do not log raw evaluator/process output or copy native runtime behavior into the Action. New native capabilities require verified CLI arguments/output contracts and model-free integration controls. -Only trusted own-repository code may eventually receive execution credentials; +Only trusted same-repository code may receive execution credentials; environment filtering is not an OS sandbox. The repository's `Copilot E2E` workflow is an explicitly approved live control @@ -72,6 +74,19 @@ capability through repository write access and branch governance. ## Release preparation +The first prerelease is [`v0.1.0`](https://github.com/github/evals-action/releases/tag/v0.1.0). +Keep published version tags unchanged. For future releases, verify the packaged +Action at the target commit and update consumer examples to the intended tag. +Action release versions are independent of the selected Vally version. + +The moving `v0` tag points to the latest approved 0.x release. After publishing +a new 0.x minor or patch release and verifying its checks, advance only `v0` +to that release's exact commit. Verify the tag target afterward. Do not move +version-specific tags, publish a separate GitHub release for the moving tag, +or use a blanket force-push of all tags. Release publication and advancing the +alias are separate steps; no automatic tag-moving workflow is configured. +Keep `v0` on the 0.x line when introducing a future `v1` release. + Before proposing a release, verify supported CLI versions and their native evidence contracts, run the Linux model-free controls, and qualify the optional reporting client with the supported hosted token. Do not advertise a plan diff --git a/README.md b/README.md index 6fc6281..ba34fc1 100644 --- a/README.md +++ b/README.md @@ -5,17 +5,21 @@ sticky PR comment** with the suite, eval names, verdicts, and evidence links. Vally owns selection, execution and grading; the Action handles installation and GitHub integration. -Supported host: **Linux x64 GitHub-hosted runners, Node 24**. Pin a reviewed -full Action commit SHA. This repository is private development code, not a -published Marketplace release; organization Actions access must permit your -consumer repository. +Use `github/evals-action@v0` for the latest 0.x pilot release +(currently [v0.1.0](https://github.com/github/evals-action/releases/tag/v0.1.0)). +Use `@v0.1.0` to stay on that specific Action release. The `v0` tag advances as +new 0.x releases are published; the interface may change during the pilot. +Supported host: **Linux x64 GitHub-hosted runners**; the Action uses Node 24 +without a Node setup step. The repository is private and shared with the GitHub +organization; your repository's Actions policy must allow it. ## Quick start One job, one checkout, one Action step. `command` defaults to `all`: validate configuration, lint the project, run the selected evals if lint succeeds, and post the result on PRs. Manual runs produce summaries and -artifacts without attempting to comment. +artifacts without attempting to comment. Save this in your repository as +`.github/workflows/evals-ci.yml`: ```yaml name: Evals @@ -23,7 +27,7 @@ on: pull_request: paths: - evals/** - - .github/workflows/evals.yml + - .github/workflows/evals-ci.yml workflow_dispatch: permissions: @@ -42,34 +46,39 @@ jobs: group: evals-${{ github.workflow_ref }} cancel-in-progress: false steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/checkout@v7 with: persist-credentials: false - - uses: github/evals-action@ + - uses: github/evals-action@v0 with: working-directory: evals suite: smoke ``` -Replace the SHA, project path and suite with your own. No reusable workflow, -custom verdict script, artifact plumbing, or manually supplied token is needed. -The Action's `github-token` input defaults to `${{ github.token }}`. The -non-cancelling concurrency group serializes updates to the sticky comment. +Replace `evals` with your Vally project directory and `smoke` with your suite. +Adjust the trigger paths to include your evals, configuration, skills and other +inputs. No command or token inputs are needed: the Action uses the job's +automatic `${{ github.token }}`. Keep the concurrency block as shown so +overlapping runs cannot create duplicate reports. + +Lint checks the entire project; the suite selects which evals run. Failed lint +skips evaluation. A failed eval still updates the comment, and the job remains +failed. The comment shows the suite, eval names, trial results and evidence links. Vally defaults to the registry's **latest stable release**. To select an exact -version instead, add `vally-version`: +version instead, add `vally-version`, for example: ```yaml - - uses: github/evals-action@ - with: - working-directory: evals - suite: smoke - vally-version: '0.20.0' +with: + working-directory: evals + suite: smoke + vally-version: '0.19.0' # Optional; omit for latest stable ``` -This requests that exact version once it is published; it does not require an -Action update. New versions work while the native CLI and evidence contracts -remain compatible. Unsupported behavior or malformed evidence fails explicitly; +The Action version (`@v0` or `@v0.1.0`) and Vally version are independent. Selecting a +published Vally release does not require an Action update. New versions work +while the native CLI and evidence contracts remain compatible. +Unsupported behavior or malformed evidence fails explicitly; there is no silent downgrade. Stable exact versions and `latest` are supported, not version ranges or prereleases. @@ -86,36 +95,7 @@ may access that token. This is convenient, not credential isolation. Use only trusted reviewed sources, grant no unnecessary permissions, and use separate commands/jobs below when you need a narrower model token. -## Included end-to-end example - -[`copilot-e2e.yml`](.github/workflows/copilot-e2e.yml) directly invokes this -repository's Action (`uses: ./`) with `command` omitted. It tests the same -simple interface consumers use against [`examples/copilot-smoke`](examples/copilot-smoke): -one real skill, one trial, a 60-second native timeout, and zero retries. -It runs for relevant same-repository PRs or manual dispatch, never forks. -The skill invocation and exact response are both graded. - -This is a test of the Action, not a reusable workflow for consumers to call. -There is no intermediate workflow wrapper. - -## Commands - -| `command` | Behavior | -| --- | --- | -| Omitted / `all` | Validate inputs, lint, run after successful lint, publish evidence, and update the PR comment. Lint failure skips the eval and reports the failure. | -| `lint` | Native `vally lint --eval-spec=`. Scans all evals recursively, needs no selector or model credentials. Publishes a summary and artifact, without commenting. | -| `run` | Native `vally eval --require-pass --output=jsonl ...`. Requires a suite, eval paths or tags. Publishes a summary and artifact, without commenting. | -| `report` | Download and verify structured artifacts, then update the same sticky comment. Never installs Vally or runs a model. PR events only. | -| `plan` | Explicit error: Vally 0.19.0 has no plan/dry-run CLI. Never falls back to a run. | - -Native arguments are separate argv values, never interpolated shell commands. -The Action does not parse `.vally.yaml`, implement grading, expand shell globs, -or install custom project dependencies. A suite cannot combine with paths/tags; -paths and tags can combine using Vally's native syntax. Put native glob patterns -in a suite rather than `eval-paths`. Lint ignores selectors with a warning; -explicit lint rejects execution overrides. - -## Optional permission isolation +## Advanced: separate jobs and permissions The **same Action** can be used in separate jobs. There is no custom verdict job: a failed lint or run job already makes the workflow fail, while @@ -127,18 +107,76 @@ job: a failed lint or run job already makes the workflow fail, while | Evaluate | `run` | `contents: read`, `copilot-requests: write` | | Comment | `report` | `contents: read`, `pull-requests: write` | -In the lint job, use `report-id: evals-lint`. In the run job use -`report-id: evals-run`, `copilot-token: ${{ github.token }}`, and make it depend -on successful lint. Each execution job checks out the consumer repository -because GitHub jobs have separate filesystems. Expose each step's `artifact-id` -and `artifact-digest` as job outputs. The reporting job needs **no checkout** -and can use the following complete job (assuming job IDs `lint` and `run`): +This complete alternative keeps PR-write permission out of the evaluator job. +Each execution job needs a checkout because jobs have separate filesystems; +the reporting job does not. All tokens below are GitHub-generated job tokens, +not PATs or stored secrets. + +
+Show the three-job workflow ```yaml +name: Evals +on: + pull_request: + paths: + - evals/** + - .github/workflows/evals-ci.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + lint: + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + artifact-id: ${{ steps.lint.outputs.artifact-id }} + artifact-digest: ${{ steps.lint.outputs.artifact-digest }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: github/evals-action@v0 + id: lint + with: + command: lint + working-directory: evals + report-id: evals-lint + + run: + needs: lint + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + copilot-requests: write + outputs: + artifact-id: ${{ steps.run.outputs.artifact-id }} + artifact-digest: ${{ steps.run.outputs.artifact-digest }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: github/evals-action@v0 + id: run + with: + command: run + working-directory: evals + suite: smoke + report-id: evals-run + copilot-token: ${{ github.token }} + max-retries: 0 + report: needs: [lint, run] if: always() && github.event_name == 'pull_request' && (needs.run.outputs.artifact-id != '' || needs.lint.outputs.artifact-id != '') runs-on: ubuntu-latest + timeout-minutes: 5 permissions: contents: read pull-requests: write @@ -146,7 +184,7 @@ and can use the following complete job (assuming job IDs `lint` and `run`): group: evals-${{ github.workflow_ref }} cancel-in-progress: false steps: - - uses: github/evals-action@ + - uses: github/evals-action@v0 with: command: report report-id: evals-run @@ -155,7 +193,7 @@ and can use the following complete job (assuming job IDs `lint` and `run`): lint-artifact-id: ${{ needs.lint.outputs.artifact-id }} lint-artifact-digest: ${{ needs.lint.outputs.artifact-digest }} if: always() && needs.run.outputs.artifact-id != '' - - uses: github/evals-action@ + - uses: github/evals-action@v0 with: command: report report-id: evals-lint @@ -164,13 +202,37 @@ and can use the following complete job (assuming job IDs `lint` and `run`): if: always() && needs.run.outputs.artifact-id == '' && needs.lint.outputs.artifact-id != '' ``` +
+ The two report steps are mutually exclusive: either report both results or -report only the lint failure. Both use the same comment identity `evals`. +report only the available lint evidence. Both use the same comment identity `evals`. Do not give the run job PR-write permission if you want isolation. Explicit `run` does not consume the automatic `github-token`; pass `copilot-token` or name advanced credentials explicitly. `report` uses its own automatic job token, or an explicit `report-token` override. +The explicit `run` step sets `max-retries: 0` to match simple mode; otherwise +explicit run retains Vally's retry default. If eval execution cannot produce an +artifact, only the available lint evidence is reported; the failed run job is +still visible in GitHub checks. + +## Commands + +| `command` | Behavior | +| --- | --- | +| Omitted / `all` | Validate inputs, lint, run after successful lint, publish evidence, and update the PR comment. | +| `lint` | Native whole-project lint. No selector or model credentials needed. Publishes a summary and artifact, without commenting. | +| `run` | Native `vally eval --require-pass --output=jsonl ...`. Requires a suite, eval paths or tags. Publishes a summary and artifact, without commenting. | +| `report` | Verify structured artifacts and update the sticky comment. Never installs Vally or runs a model. PR events only. | +| `plan` | Reserved and unsupported; fails explicitly, never runs an eval. | + +Native arguments are separate argv values, never interpolated shell commands. +The Action does not parse `.vally.yaml`, implement grading, expand shell globs, +or install custom project dependencies. A suite cannot combine with paths/tags; +paths and tags can combine using Vally's native syntax. Put native glob patterns +in a suite rather than `eval-paths`. Lint ignores selectors with a warning; +explicit lint rejects execution overrides. + ## Inputs | Input | Meaning | @@ -185,7 +247,7 @@ token, or an explicit `report-token` override. | `credential-env` | Advanced evaluator credential variable names, one per line, for BYOK/custom providers. Supplying these disables automatic model-token injection in all. Never put values here. Cannot list `COPILOT_GITHUB_TOKEN` with `copilot-token`. | | `report-token` | Optional bot/App comment token override for all/report. Human/PAT reporters are refused. Not valid on explicit lint/run. | | `report-id` | Stable report identity, default `evals`; letters/digits/underscore/hyphen. Maximum 64 characters (59-character prefix for all). Advanced `evals-run` / `evals-lint` share the `evals` comment. | -| `vally-version` | Default `latest`, resolved once per job to the npm latest stable release. Set an exact stable version, e.g. `0.20.0`, to pin the CLI. No per-version Action update is required. | +| `vally-version` | Default `latest`, resolved once per job to the npm latest stable release. Set an exact published stable version to pin the CLI. No per-version Action update is required. | | `model` | Native model override, including comma-separated native model lists. | | `runs`, `workers`, `timeout` | Positive integer native run overrides. Timeout is per trial in seconds. | | `max-retries` | Nonnegative run override. All mode defaults to 0; explicit run preserves native default. | @@ -276,6 +338,18 @@ repository/revision/run/attempt identity, and accepts only a bounded `report.json`. A companion must be successful lint from the same run/revision, but it can originate in a separate lint job. +## Example skill and eval + +[`examples/copilot-smoke`](examples/copilot-smoke) contains a real skill and an +eval that checks both skill invocation and exact output. This repository's +[`Copilot E2E` workflow](.github/workflows/copilot-e2e.yml) tests it with one +checkout and one `uses: ./` step, using the Action's defaults. In your repository, +use `github/evals-action@v0` instead. + +The Action prerelease is private and not listed on Marketplace. Organizations +that require full-SHA Action references can pin the release commit +`83fa12332c3e44ad899f347eaf153a0d9b6432dc` for `v0.1.0`. + ## Development and acquisition Acquisition resolves `latest` or the exact requested version from npm and diff --git a/SECURITY.md b/SECURITY.md index 9a596cc..3c9f4a6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -31,9 +31,8 @@ executors and graders. Fork jobs and `pull_request_target` are excluded. The checkout revision is checked before acquisition and again before execution. Workflow preparation can still intentionally change files in that checkout; the reported Git revision is not a claim that all working files are unchanged. -The default `all` step, or an explicit `run` step, is the team's execution authorization. Use a protected -GitHub environment when approval is required. No plan-preview approval token -or reusable authorization is invented. +The default `all` step, or an explicit `run` step, is the team's execution +authorization. Use a protected GitHub environment when approval is required. Only the run phase receives model credentials. In default `all` mode, the `github-token` input defaults to `${{ github.token }}` for both Copilot and @@ -63,12 +62,14 @@ run rejects `report-token`. Reporting downloads and digest-verifies the current run's structured artifacts and checks repository/revision/run/attempt identity. It never invokes Vally. Separate jobs reduce evaluator access to PR-write authority; they are optional, not a platform requirement. + PR comments show the selected suite and native eval names. These fields are untrusted and Markdown-escaped; do not put secrets in eval or suite names. Raw paths, prompts, transcripts and model output remain excluded from comments. Do not assume this credential routing prevents prompt injection in reviewed -evaluation content. Live Copilot authentication/billing requires a separately -approved pilot; synthetic transport tests do not establish server acceptance. +evaluation content. Copilot availability and billing depend on the consumer's +account policy and entitlements; synthetic transport tests do not establish +server acceptance. Vally defaults to npm's latest stable release, or an exact user-selected version. Acquisition validates registry package integrity, generates a fresh diff --git a/examples/copilot-smoke/README.md b/examples/copilot-smoke/README.md index 37be8d4..4e15a8f 100644 --- a/examples/copilot-smoke/README.md +++ b/examples/copilot-smoke/README.md @@ -12,13 +12,16 @@ consumer's simplest usage: one job, one checkout, and one direct Action call: `command` is omitted. The Action defaults to `all`: validate configuration, lint, run the real eval after successful lint, and update one sticky PR comment. -There is no reusable workflow, separate reporting job, or custom verdict script. -Consumers replace `./` with `github/evals-action@`. +Consumers replace `./` with `github/evals-action@v0` and point +`working-directory` and `suite` at their own evals. See the +[quick start](../../README.md#quick-start) for the full consumer workflow. The job grants contents read, Copilot access, and PR-comment write permission. The Action uses `${{ github.token }}` automatically. This combined-permission token is available to the evaluator; use trusted eval content. Advanced callers can use explicit `lint`, `run` and `report` commands in separate jobs instead. +The [advanced example](../../README.md#advanced-separate-jobs-and-permissions) +shows the complete isolated layout. This control consumes Copilot AI credits. It runs one stimulus and one trial, with a 60-second native timeout from the eval file and zero retries from the diff --git a/tests/example.test.ts b/tests/example.test.ts index af08a45..d343e1c 100644 --- a/tests/example.test.ts +++ b/tests/example.test.ts @@ -43,7 +43,7 @@ describe('real Copilot example contract', () => { expect(job.permissions).toEqual({ contents: 'read', 'copilot-requests': 'write', 'pull-requests': 'write' }); expect(job.steps).toHaveLength(2); expect(job.steps[0]).toEqual({ - uses: 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1', with: { 'persist-credentials': false }, + uses: 'actions/checkout@v7', with: { 'persist-credentials': false }, }); expect(job.steps[1]).toEqual({ uses: './', with: { 'working-directory': 'examples/copilot-smoke', suite: 'copilot-smoke' },