From 8cc906e3561b843c7b163e1a6ba25f48b18bb3c3 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 18 Oct 2026 22:01:19 -0700 Subject: [PATCH 01/20] ci: reusable repo-eval workflow for a repository's no-LLM checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit .github/workflows/repo-eval.yml (on: workflow_call) runs, on the caller's checkout, `repo validate` (a gate), `repo scan` (report only, job summary and artifact) and `repo tasks check` as a matrix over the selected tasks: `changed` (the tasks a PR touches, plus a smoke set when the harness, graders, manifest, calling workflow or `smoke-paths` change), `all`, or a list, minus `exclude-tasks`. dokimi is installed from a pinned commit SHA or tag (a branch is refused), with that commit's requirements.lock.txt as constraints. repo-eval-self.yml calls it on dokimi's own .agent-eval/ with the commit under test, when .agent-eval/, the repository mode or either workflow changes. The end of contract §9 documents the inputs and the caller snippet. Part of #41 (step D8). Signed-off-by: Nikolay Petrov --- .github/workflows/repo-eval-self.yml | 51 ++++ .github/workflows/repo-eval.yml | 389 ++++++++++++++++++++++++ dokimi/docs/repo-evaluation-contract.md | 50 +++ 3 files changed, 490 insertions(+) create mode 100644 .github/workflows/repo-eval-self.yml create mode 100644 .github/workflows/repo-eval.yml diff --git a/.github/workflows/repo-eval-self.yml b/.github/workflows/repo-eval-self.yml new file mode 100644 index 00000000..21bf4979 --- /dev/null +++ b/.github/workflows/repo-eval-self.yml @@ -0,0 +1,51 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: repo-eval-self + +# dokimi's own evals (.agent-eval/) through the reusable repo-eval.yml, as a +# repository calls it, with two differences: the workflow is this commit's +# (`./`), and so is the dokimi it installs, so a change to the repository +# mode is checked against these tasks before it merges. +# +# A change to a task checks that task. A change to the harness (run.py, the +# Dockerfile, the manifest), to the repository mode or to either workflow +# checks the smoke set: one fix task (hidden tests plus the whole suite) and +# the score-only review task. A dispatch checks all ten by default. +on: + pull_request: + paths: + - '.agent-eval/**' + - 'dokimi/dokimi/repos/**' + - '.github/workflows/repo-eval.yml' + - '.github/workflows/repo-eval-self.yml' + workflow_dispatch: + inputs: + tasks: + description: "`all`, or task ids separated by spaces or commas" + required: false + default: all + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + self: + permissions: + contents: read + uses: ./.github/workflows/repo-eval.yml + with: + # The commit under test: the merge commit on a pull request. + dokimi-ref: ${{ github.sha }} + dokimi-repository: ${{ github.repository }} + manifest: .agent-eval/repo-eval.yaml + tasks: ${{ inputs.tasks || 'changed' }} + smoke-tasks: fix-degrade-pre review-pr-conventions + smoke-paths: dokimi/dokimi/repos/ .github/workflows/repo-eval.yml + max-parallel: 4 + # intel-ubuntu-latest: see ci.yml. + runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }} + timeout-minutes: 60 diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml new file mode 100644 index 00000000..a22489d3 --- /dev/null +++ b/.github/workflows/repo-eval.yml @@ -0,0 +1,389 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +name: repo-eval + +# The no-LLM checks of a repository's evals (dokimi/docs/repo-evaluation-contract.md), +# for repositories to call from their own workflow: +# +# validate `repo validate`: the manifest and every task.json (seconds; a gate) +# scan `repo scan`: the L0 claims of the guidance (report only) +# tasks-check `repo tasks check`, one job per selected task: the untouched +# workspace fails the gate, the reference solution passes it +# +# Nothing here calls a model or reads a secret. The caller pins dokimi twice, +# by the same commit SHA (or tag): in `uses: ...@` and in `dokimi-ref`. The +# ref installs dokimi and, through its requirements.lock.txt, every dependency, +# so a rerun checks with the same framework. A branch name is refused. +# +# Task selection (`tasks`): `changed` on a pull request is the tasks whose +# directories it touches, plus `smoke-tasks` when it changes anything else the +# self-check depends on: the eval directory outside its tasks (harness, +# graders, Dockerfile; Markdown excepted), the manifest, the calling workflow +# file, or a path under `smoke-paths`. On a push `changed` diffs against the +# previous tip; on any other event it means `all`. + +on: + workflow_call: + inputs: + dokimi-ref: + description: >- + dokimi commit SHA (40 hex digits) or release tag to install. Use the ref of + the `uses:` line; a branch is refused. + required: true + type: string + dokimi-repository: + description: Repository dokimi-ref is in (owner/name), for testing a fork + required: false + type: string + default: intel/dokimi + manifest: + description: Manifest path relative to the repository root + required: false + type: string + default: .agent-eval/repo-eval.yaml + tasks: + description: "`changed`, `all`, or task ids separated by spaces or commas" + required: false + type: string + default: changed + smoke-tasks: + description: >- + Task ids checked when `changed` finds a change outside the task directories + (empty: every task) + required: false + type: string + default: '' + smoke-paths: + description: >- + More path prefixes (space-separated) whose change selects `smoke-tasks`, + for example a vendored grader library + required: false + type: string + default: '' + exclude-tasks: + description: Task ids never checked here (space- or comma-separated), for example ones too heavy for CI + required: false + type: string + default: '' + max-parallel: + description: At most this many tasks-check jobs at once + required: false + type: number + default: 4 + warm: + description: >- + Pass --warm to `repo tasks check`: null and oracle run with network so a + cold cache fills (an ephemeral runner starts with empty caches) + required: false + type: boolean + default: false + runs-on: + description: Runner label for every job (a tasks-check job needs docker) + required: false + type: string + default: ubuntu-latest + timeout-minutes: + description: Timeout of each tasks-check job (image build included) + required: false + type: number + default: 180 + +permissions: {} + +jobs: + validate: + name: repo validate + runs-on: ${{ inputs.runs-on }} + timeout-minutes: 20 + permissions: + contents: read + outputs: + dokimi-sha: ${{ steps.ref.outputs.sha }} + id: ${{ steps.select.outputs.id }} + tasks: ${{ steps.select.outputs.tasks }} + steps: + # Full history: `validate` resolves every task base, and `changed` diffs + # against the base branch. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + # A SHA is taken as it is; a tag is resolved to the commit it names, so the + # later jobs install the same commit even if the tag moves meanwhile. + - name: Resolve dokimi-ref + id: ref + env: + REF: ${{ inputs.dokimi-ref }} + DOKIMI_REPO: ${{ inputs.dokimi-repository }} + run: | + if ! [[ "$DOKIMI_REPO" =~ ^[A-Za-z0-9-]+/[A-Za-z0-9._-]+$ ]]; then + echo "::error title=dokimi-repository::dokimi-repository is not owner/name" + exit 1 + fi + if [[ "$REF" =~ ^[0-9a-f]{40}$ ]]; then + sha=$REF + elif [[ "$REF" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + # An annotated tag's commit is its peeled (^{}) line. + sha=$(git ls-remote --tags "https://github.com/$DOKIMI_REPO" \ + | awk -v t="refs/tags/$REF" '$2 == t "^{}" { p = $1 } $2 == t { l = $1 } END { print (p != "" ? p : l) }') + fi + if [ -z "${sha:-}" ]; then + echo "::error title=dokimi-ref::dokimi-ref is neither a 40-digit commit SHA nor a tag of $DOKIMI_REPO" + exit 1 + fi + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "dokimi $DOKIMI_REPO@$REF -> $sha" + - name: Install dokimi + env: + SHA: ${{ steps.ref.outputs.sha }} + DOKIMI_REPO: ${{ inputs.dokimi-repository }} + run: | + pip install -c "https://raw.githubusercontent.com/$DOKIMI_REPO/$SHA/dokimi/requirements.lock.txt" \ + "dokimi @ git+https://github.com/$DOKIMI_REPO@$SHA#subdirectory=dokimi" + - name: repo validate + env: + MANIFEST: ${{ inputs.manifest }} + run: | + status=0 + dokimi repo validate . --manifest "$MANIFEST" || status=$? + dokimi repo validate . --manifest "$MANIFEST" --json > "$RUNNER_TEMP/validate.json" || true + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json, os + path = os.path.join(os.environ["RUNNER_TEMP"], "validate.json") + try: + d = json.load(open(path, encoding="utf-8")) + except (OSError, ValueError): + print("### repo validate", "", "The manifest could not be read; see the job log.", sep="\n") + raise SystemExit + tasks = d["tasks"] + print("### repo validate", "", f"{'ok' if d['ok'] else 'FAILED'}: `{d['manifest']['manifest']}`, " + f"{tasks['count']} task(s), {len(d['guidance_files'])} guidance file(s)", sep="\n") + for kind in ("errors", "warnings", "notes"): + for line in d[kind]: + print(f"- {kind[:-1]}: {line}") + PY + exit "$status" + - name: Select tasks + id: select + env: + EVENT: ${{ github.event_name }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + BEFORE: ${{ github.event.before }} + WORKFLOW_REF: ${{ github.workflow_ref }} + REPOSITORY: ${{ github.repository }} + MANIFEST: ${{ inputs.manifest }} + TASKS: ${{ inputs.tasks }} + SMOKE_TASKS: ${{ inputs.smoke-tasks }} + SMOKE_PATHS: ${{ inputs.smoke-paths }} + EXCLUDE_TASKS: ${{ inputs.exclude-tasks }} + run: | + python - <<'PY' + import json, os, re, subprocess, sys + from pathlib import Path + + env = os.environ + d = json.load(open(os.path.join(env["RUNNER_TEMP"], "validate.json"), encoding="utf-8")) + m = d["manifest"] + + def ids(text): + return [t for t in re.split(r"[\s,]+", text) if t] + + def out(**kv): + with open(env["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + for k, v in kv.items(): + f.write(f"{k}={v}\n") + + tasks_dir = m["tasks"] + if not tasks_dir: # a guidance-only manifest + print("guidance-only manifest: no tasks to check") + out(id=m["id"], tasks="[]") + sys.exit() + all_ids = sorted(p.parent.name for p in Path(tasks_dir).glob("*/task.json")) + + def changed_files(): + if env["EVENT"] == "pull_request" and env["BASE_REF"]: + rng = [f"origin/{env['BASE_REF']}...HEAD"] + elif env["EVENT"] == "push" and env["BEFORE"].strip("0"): + rng = [env["BEFORE"], "HEAD"] + else: + return None + r = subprocess.run(["git", "diff", "--name-only", *rng], capture_output=True, text=True) + if r.returncode: + print(f"::warning::git diff {' '.join(rng)} failed, checking every task: {r.stderr.strip()}") + return None + return r.stdout.split() + + mode = env["TASKS"].strip() + if mode == "all": + sel = list(all_ids) + elif mode == "changed": + files = changed_files() + if files is None: + sel = list(all_ids) + else: + prefix = tasks_dir.rstrip("/") + "/" + sel = sorted({f[len(prefix):].split("/", 1)[0] for f in files if f.startswith(prefix)}) + sel = [t for t in sel if t in all_ids] # a deleted task has nothing to check + # the calling workflow: owner/repo/.github/workflows/x.yml@ref + caller = env["WORKFLOW_REF"].split("@", 1)[0].removeprefix(env["REPOSITORY"] + "/") + eval_dir = m["eval_dir"].rstrip("/") + "/" + roots = [p for p in env["SMOKE_PATHS"].split() if p] + + def harness(f): + if f in (env["MANIFEST"], caller) or any(f.startswith(r) for r in roots): + return True + return f.startswith(eval_dir) and not f.startswith(prefix) and not f.endswith(".md") + + hits = [f for f in files if harness(f)] + if hits: + smoke = ids(env["SMOKE_TASKS"]) or all_ids + print(f"outside the task directories: {' '.join(hits[:10])}; adding the smoke set") + unknown = [t for t in smoke if t not in all_ids] + if unknown: + sys.exit(f"::error title=smoke-tasks::no such task(s): {', '.join(unknown)}") + sel = sorted(set(sel) | set(smoke)) + else: + sel = ids(mode) + unknown = [t for t in sel if t not in all_ids] + if unknown: + sys.exit(f"::error title=tasks::no such task(s): {', '.join(unknown)}") + excluded = set(ids(env["EXCLUDE_TASKS"])) + sel = [t for t in dict.fromkeys(sel) if t not in excluded] + print(f"tasks-check: {' '.join(sel) or 'none'}") + out(id=m["id"], tasks=json.dumps(sel)) + with open(env["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as f: + f.write(f"\ntasks-check ({mode}): {', '.join(f'`{t}`' for t in sel) or 'none'}\n") + PY + + scan: + name: repo scan + needs: validate + runs-on: ${{ inputs.runs-on }} + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + - name: Install dokimi + env: + SHA: ${{ needs.validate.outputs.dokimi-sha }} + DOKIMI_REPO: ${{ inputs.dokimi-repository }} + run: | + pip install -c "https://raw.githubusercontent.com/$DOKIMI_REPO/$SHA/dokimi/requirements.lock.txt" \ + "dokimi @ git+https://github.com/$DOKIMI_REPO@$SHA#subdirectory=dokimi" + # Report only: a failing claim does not fail the job (an unreadable + # manifest still does). The table lists the failing claims. + - name: repo scan + env: + MANIFEST: ${{ inputs.manifest }} + run: | + dokimi repo scan . --manifest "$MANIFEST" --out "$RUNNER_TEMP/repo-readiness.json" + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json, os + d = json.load(open(os.path.join(os.environ["RUNNER_TEMP"], "repo-readiness.json"), encoding="utf-8")) + c = d["counts"] + + def cell(text): + return str(text).replace("|", "\\|").replace("\n", " ") + + print("### repo scan (L0, report only)", "", + f"{len(d['files'])} guidance file(s), adapters: {', '.join(d['adapters']) or 'none'}", "", + "| claims | ok | unresolved | fail |", "|---|---|---|---|", + f"| {c['total']} | {c['ok']} | {c['unresolved']} | {c['fail']} |", sep="\n") + for w in d["warnings"]: + print(f"- warning: {cell(w)}") + fails = [x for x in d["claims"] if x["status"].split("_", 1)[0] == "fail"] + if fails: + print("", "| file | line | kind | claim | why |", "|---|---|---|---|---|", sep="\n") + for x in fails[:100]: + print(f"| {cell(x['file'])} | {x['line']} | {cell(x.get('kind'))} | " + f"`{cell(str(x.get('claim'))[:80])}` | {cell(x.get('why'))} |") + if len(fails) > 100: + print(f"\n{len(fails) - 100} more in the repo-readiness.json artifact") + PY + - name: Upload the scan report + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: repo-eval-scan-${{ needs.validate.outputs.id }} + path: ${{ runner.temp }}/repo-readiness.json + if-no-files-found: ignore + + tasks-check: + name: repo tasks check ${{ matrix.task }} + needs: validate + if: needs.validate.outputs.tasks != '[]' + runs-on: ${{ inputs.runs-on }} + timeout-minutes: ${{ inputs.timeout-minutes }} + permissions: + contents: read + strategy: + fail-fast: false + max-parallel: ${{ inputs.max-parallel }} + matrix: + task: ${{ fromJSON(needs.validate.outputs.tasks) }} + steps: + # Full history: task workspaces are checked out from the task's base commit. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + - name: Install dokimi + env: + SHA: ${{ needs.validate.outputs.dokimi-sha }} + DOKIMI_REPO: ${{ inputs.dokimi-repository }} + run: | + pip install -c "https://raw.githubusercontent.com/$DOKIMI_REPO/$SHA/dokimi/requirements.lock.txt" \ + "dokimi @ git+https://github.com/$DOKIMI_REPO@$SHA#subdirectory=dokimi" + # Builds the image from the manifest's Dockerfile (the tree without its + # tasks), then grades the null and the oracle workspace in it. + - name: repo tasks check + env: + MANIFEST: ${{ inputs.manifest }} + TASK: ${{ matrix.task }} + WARM: ${{ inputs.warm }} + run: | + warm=() + if [ "$WARM" = true ]; then warm=(--warm); fi + dokimi repo tasks check . --manifest "$MANIFEST" --task "$TASK" --out "$RUNNER_TEMP/repo-check" "${warm[@]}" + - name: Summary + if: ${{ !cancelled() }} + run: | + python - <<'PY' >> "$GITHUB_STEP_SUMMARY" + import json, os + path = os.path.join(os.environ["RUNNER_TEMP"], "repo-check", "check", "check.json") + if not os.path.isfile(path): + print("### repo tasks check", "", "No check.json: the check stopped before grading; see the job log.", + sep="\n") + raise SystemExit + for r in json.load(open(path, encoding="utf-8"))["tasks"]: + print(f"### repo tasks check `{r['task']}`: {r['verdict'].upper()}", "", sep="\n") + for kind in ("null", "oracle"): + g = r.get(kind) + if g: + print(f"- {kind}: {g.get('status')}, success {g.get('success')}, score {g.get('score')}") + for p in r["problems"]: + print(f"- problem: {p}") + PY + - name: Upload the check results + if: ${{ !cancelled() }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: repo-eval-check-${{ needs.validate.outputs.id }}-${{ matrix.task }} + path: | + ${{ runner.temp }}/repo-check/check/check.json + ${{ runner.temp }}/repo-check/check/*/grade.json + ${{ runner.temp }}/repo-check/check/*/*.log + if-no-files-found: ignore diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 40069f9d..07743953 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -721,3 +721,53 @@ python cli.py repo compare repo-runs/pilot --baseline guidance:none --metric roo `eval_dir` or `image` pins or harness versions. An image `repo run` tagged from the tree (`dokimi/:`) follows the tree, so two revisions are not warned about it; an `--image` of its own is compared. + +CI for a repository: `.github/workflows/repo-eval.yml` is a reusable workflow +that runs the no-LLM checks on the caller's checkout and needs no secret. + +- `validate` runs `repo validate` and gates the other two jobs; `scan` runs `repo scan` + (report only: the claim counts and failing claims go to the job summary, the + `repo_readiness.v1` report to an artifact); `tasks-check` runs `repo tasks check` + for each selected task in a job of its own (docker; the image is built from the + manifest's Dockerfile), and uploads `check.json` and the grade logs. +- `tasks: changed` (the default) checks, on a pull request, the tasks whose + directories it touches, plus `smoke-tasks` (empty: every task) when it changes + the eval directory outside its tasks (Markdown excepted), the manifest, the + calling workflow file or a path under `smoke-paths`. On a push it diffs against + the previous tip, on any other event it checks every task. `all` or a list of + task ids selects directly; `exclude-tasks` always drops tasks. +- Pin dokimi by a commit SHA or a release tag, the same one in `uses:` and in + `dokimi-ref` (a branch is refused). It installs + `dokimi @ git+https://github.com/intel/dokimi@#subdirectory=dokimi` with + that commit's `requirements.lock.txt` as constraints, so a rerun checks with the + same framework and dependencies. Moving the pin is a change to the calling + workflow, so it runs the smoke set. +- Other inputs: `manifest`, `max-parallel` (4), `warm` (`--warm`, for a cache that + starts empty on every runner), `runs-on` (`ubuntu-latest`), `timeout-minutes` + (180, per task) and `dokimi-repository` (`intel/dokimi`). + +```yaml +# .github/workflows/agent-evals.yml in the repository +name: Agent evals +on: + pull_request: + paths: ['dev/agent_evals/**', '**/AGENTS.md', '.github/workflows/agent-evals.yml'] + schedule: + - cron: '0 3 * * 0' + workflow_dispatch: +permissions: {} +jobs: + evals: + permissions: + contents: read + uses: intel/dokimi/.github/workflows/repo-eval.yml@ # vX.Y.Z + with: + dokimi-ref: + manifest: dev/agent_evals/repo-eval.yaml + smoke-tasks: build_kmeans_host fix_finiteness_mut review_seeded_fc + max-parallel: 4 + warm: true +``` + +dokimi runs the workflow on its own `.agent-eval/` (`repo-eval-self.yml`), with +the commit under test as `dokimi-ref`. From 056814fca2572a92b4abcdefd0191887e12b701f Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sun, 18 Oct 2026 22:06:28 -0700 Subject: [PATCH 02/20] ci(repo-eval): unbuffered tasks check log; say what the caller grants Part of #41 (step D8). Signed-off-by: Nikolay Petrov --- .github/workflows/repo-eval.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index a22489d3..2e03148e 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -14,7 +14,8 @@ name: repo-eval # Nothing here calls a model or reads a secret. The caller pins dokimi twice, # by the same commit SHA (or tag): in `uses: ...@` and in `dokimi-ref`. The # ref installs dokimi and, through its requirements.lock.txt, every dependency, -# so a rerun checks with the same framework. A branch name is refused. +# so a rerun checks with the same framework. A branch name is refused. The +# calling job grants `contents: read`, all any job here asks for. # # Task selection (`tasks`): `changed` on a pull request is the tasks whose # directories it touches, plus `smoke-tasks` when it changes anything else the @@ -354,6 +355,8 @@ jobs: MANIFEST: ${{ inputs.manifest }} TASK: ${{ matrix.task }} WARM: ${{ inputs.warm }} + # Each task's verdict in the log as it lands, not when the step ends. + PYTHONUNBUFFERED: '1' run: | warm=() if [ "$WARM" = true ]; then warm=(--warm); fi From ffbb6d171e2f61ad0344e05032826b1b4adafbab Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 19 Oct 2026 00:59:56 -0700 Subject: [PATCH 03/20] fix(repo-eval): handle whitespace and renames in changed-task selector, include .err files in artifacts Signed-off-by: Nikolay Petrov --- .github/workflows/repo-eval.yml | 5 +- dokimi/tests/test_repo_eval_task_selection.py | 93 +++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 dokimi/tests/test_repo_eval_task_selection.py diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index 2e03148e..f316274e 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -212,11 +212,11 @@ jobs: rng = [env["BEFORE"], "HEAD"] else: return None - r = subprocess.run(["git", "diff", "--name-only", *rng], capture_output=True, text=True) + r = subprocess.run(["git", "diff", "--no-renames", "--name-only", "-z", *rng], capture_output=True, text=True) if r.returncode: print(f"::warning::git diff {' '.join(rng)} failed, checking every task: {r.stderr.strip()}") return None - return r.stdout.split() + return r.stdout.split('\0') if r.stdout else [] mode = env["TASKS"].strip() if mode == "all": @@ -389,4 +389,5 @@ jobs: ${{ runner.temp }}/repo-check/check/check.json ${{ runner.temp }}/repo-check/check/*/grade.json ${{ runner.temp }}/repo-check/check/*/*.log + ${{ runner.temp }}/repo-check/check/*/*.err if-no-files-found: ignore diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py new file mode 100644 index 00000000..603f0cc8 --- /dev/null +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -0,0 +1,93 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""Regression tests for repo-eval workflow task selection with NUL-separated filenames.""" + +from __future__ import annotations + +from pathlib import Path + + +def parse_changed_files(output: str) -> list[str]: + """Parse NUL-separated git diff output (regression for PR #113 thread 1).""" + return [f for f in output.split('\0') if f] + + +class TestChangedFilesNulSeparation: + """Verify task selection handles NUL-separated filenames with spaces and renames.""" + + def test_nul_separated_output_with_spaces(self) -> None: + """Files with spaces should be parsed correctly when NUL-separated.""" + # Simulate git diff --no-renames --name-only -z output + output = "tasks/task1/file.txt\0tasks/agent eval/file.py\0tasks/task2/file.md\0" + result = parse_changed_files(output) + assert result == [ + "tasks/task1/file.txt", + "tasks/agent eval/file.py", + "tasks/task2/file.md", + ] + + def test_nul_separated_output_with_quotes(self) -> None: + """Files with quotes (Git's quoted format) should be parsed correctly.""" + # Git quotes filenames with special chars when not using -z + # With -z, they come unquoted + output = 'tasks/file"name".txt\0tasks/file\'name\'.py\0' + result = parse_changed_files(output) + assert result == ['tasks/file"name".txt', "tasks/file'name'.py"] + + def test_nul_separated_empty_output(self) -> None: + """Empty output should produce empty list.""" + output = "" + result = parse_changed_files(output) + assert result == [] + + def test_nul_separated_trailing_nul(self) -> None: + """Trailing NUL should not produce empty entry.""" + output = "file1.txt\0file2.txt\0" + result = parse_changed_files(output) + assert result == ["file1.txt", "file2.txt"] + + def test_no_renames_flag_prevents_omission(self) -> None: + """--no-renames flag ensures both source and dest of moves are reported.""" + # Without --no-renames: file moved from A to B reports only B + # With --no-renames: both A and B are reported + # This test validates the flag is present in the workflow + wf_path = Path(__file__).resolve().parents[2] / ".github/workflows/repo-eval.yml" + with open(wf_path, encoding="utf-8") as f: + workflow = f.read() + # Verify the --no-renames flag is in the git diff command + assert '--no-renames' in workflow + assert '--name-only' in workflow + assert '-z' in workflow + # Check the order and that they're in a git diff command + assert 'git", "diff", "--no-renames", "--name-only", "-z"' in workflow + + +class TestUploadArtifactErrFiles: + """Verify upload artifact includes .err files for grader/oracle stderr.""" + + def test_err_files_in_upload_path(self) -> None: + """Verify *.err pattern is included in upload artifact paths.""" + wf_path = Path(__file__).resolve().parents[2] / ".github/workflows/repo-eval.yml" + with open(wf_path, encoding="utf-8") as f: + workflow = f.read() + # Find the upload artifact section for check results + assert "repo-eval-check-" in workflow + # Verify *.err files are listed + assert '${{ runner.temp }}/repo-check/check/*/*.err' in workflow + # Verify it's in the right section (after the log files) + artifact_section = workflow[ + workflow.find("Upload the check results") : + ] + assert '*.err' in artifact_section or '*/*.err' in artifact_section + + def test_err_files_purpose(self) -> None: + """Document that .err files contain grader and oracle stderr.""" + # This is a documentation test; the actual files are: + # - grade.err: grader stderr + # - oracle.err: oracle stderr + # - isolation.err: isolation probe stderr + # These are all captured by repos/grading.py:124 and the isolation probe + files_expected = ["grade.err", "oracle.err", "isolation.err"] + # This validates the reviewer's comment about these files + assert len(files_expected) == 3 From 48d103e00363c67de62793a2d1131f9a4f926e4e Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 19 Oct 2026 07:02:38 -0700 Subject: [PATCH 04/20] fix(repo-eval): handle shared files under tasks dir, normalize manifest path - Treat files under tasks dir whose first segment is not a task id as harness changes (e.g., tasks/common.py shared by graders, deleted task directories). They now trigger the smoke set as intended. - Normalize manifest path with posixpath.normpath() so changes to manifests like './repo-eval.yaml' are correctly detected (git reports normalized paths). - Add regression tests for shared files and path normalization. Signed-off-by: Nikolay Petrov --- .github/workflows/repo-eval.yml | 10 ++++- dokimi/tests/test_repo_eval_task_selection.py | 37 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index f316274e..c6d4c4ac 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -218,6 +218,7 @@ jobs: return None return r.stdout.split('\0') if r.stdout else [] + import posixpath mode = env["TASKS"].strip() if mode == "all": sel = list(all_ids) @@ -233,10 +234,17 @@ jobs: caller = env["WORKFLOW_REF"].split("@", 1)[0].removeprefix(env["REPOSITORY"] + "/") eval_dir = m["eval_dir"].rstrip("/") + "/" roots = [p for p in env["SMOKE_PATHS"].split() if p] + manifest_normalized = posixpath.normpath(env["MANIFEST"]) def harness(f): - if f in (env["MANIFEST"], caller) or any(f.startswith(r) for r in roots): + if f in (manifest_normalized, caller) or any(f.startswith(r) for r in roots): return True + if f.startswith(prefix): + # a file under prefix (tasks dir) whose first segment is not a task id + # is a shared helper (e.g., tasks/common.py); treat as harness + first_segment = f[len(prefix):].split("/", 1)[0] + if first_segment not in all_ids: + return True return f.startswith(eval_dir) and not f.startswith(prefix) and not f.endswith(".md") hits = [f for f in files if harness(f)] diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index 603f0cc8..08c4f243 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -91,3 +91,40 @@ def test_err_files_purpose(self) -> None: files_expected = ["grade.err", "oracle.err", "isolation.err"] # This validates the reviewer's comment about these files assert len(files_expected) == 3 + + +class TestSharedFilesUnderTasksDir: + """Verify shared helper files under tasks dir are treated as harness changes.""" + + def test_shared_file_under_tasks_dir_selects_smoke(self) -> None: + """A file like tasks/common.py (helper shared by graders) should trigger smoke set.""" + wf_path = Path(__file__).resolve().parents[2] / ".github/workflows/repo-eval.yml" + with open(wf_path, encoding="utf-8") as f: + workflow = f.read() + # Verify the workflow handles non-task-id first segments under prefix as harness + assert "first_segment not in all_ids" in workflow + assert "shared helper" in workflow or "tasks/common.py" in workflow.lower() or "tasks dir" in workflow.lower() + + def test_deleted_task_directory_handled(self) -> None: + """A deleted task directory (first segment not in all_ids) should also be harness.""" + # The workflow should treat deleted task dirs the same way as shared files + wf_path = Path(__file__).resolve().parents[2] / ".github/workflows/repo-eval.yml" + with open(wf_path, encoding="utf-8") as f: + workflow = f.read() + # Verify the logic handles non-task-id segments under prefix + assert "first_segment not in all_ids" in workflow + + +class TestManifestPathNormalization: + """Verify manifest path is normalized for comparison with git output.""" + + def test_manifest_normalized_in_workflow(self) -> None: + """Manifest should be normalized with posixpath.normpath for git comparison.""" + wf_path = Path(__file__).resolve().parents[2] / ".github/workflows/repo-eval.yml" + with open(wf_path, encoding="utf-8") as f: + workflow = f.read() + # Verify posixpath.normpath is used on MANIFEST + assert "posixpath.normpath" in workflow + assert "manifest_normalized" in workflow + # Verify manifest comparison uses the normalized version + assert "manifest_normalized" in workflow From 631ba99246369395144430effda3c6d0233c8cf1 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 19 Oct 2026 07:17:54 -0700 Subject: [PATCH 05/20] test(repo-eval): run the Select tasks step itself against a scratch git repo Signed-off-by: Nikolay Petrov --- dokimi/tests/test_repo_eval_task_selection.py | 268 ++++++++++-------- 1 file changed, 147 insertions(+), 121 deletions(-) diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index 08c4f243..06e05d87 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -1,130 +1,156 @@ # Copyright (C) 2026 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -"""Regression tests for repo-eval workflow task selection with NUL-separated filenames.""" +"""The reusable repo-eval workflow's ``Select tasks`` step, run for real. + +The step's Python is lifted out of ``.github/workflows/repo-eval.yml`` and run in a +scratch git repository with the environment GitHub gives it, so a change to the +selector (path splitting, the smoke set, the manifest comparison) fails here and not +on a caller's pull request. +""" from __future__ import annotations +import json +import os +import re +import shutil +import subprocess +import sys from pathlib import Path +from typing import Any + +import pytest +import yaml + +WORKFLOW = Path(__file__).resolve().parents[2] / ".github" / "workflows" / "repo-eval.yml" +EVAL_DIR = ".agent-eval" +TASKS = f"{EVAL_DIR}/tasks" + +pytestmark = pytest.mark.skipif(shutil.which("git") is None, reason="needs git") + + +def _steps() -> list[dict[str, Any]]: + doc = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + return [s for job in doc["jobs"].values() for s in job.get("steps", [])] + + +def _select_script() -> str: + run = next(s for s in _steps() if s.get("id") == "select")["run"] + match = re.search(r"python - <<'PY'\n(.*)\nPY\s*$", run, re.S) + assert match, "the Select tasks step is no longer a python heredoc" + return match.group(1) + + +def _git(repo: Path, *args: str) -> str: + return subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True, text=True).stdout + + +def _write(repo: Path, rel: str, text: str = "x\n") -> None: + (repo / rel).parent.mkdir(parents=True, exist_ok=True) + (repo / rel).write_text(text, encoding="utf-8") + + +@pytest.fixture +def repo(tmp_path: Path) -> Path: + root = tmp_path / "repo" + root.mkdir() + for cmd in (["init", "-q", "-b", "main"], ["config", "user.email", "t@example.com"], + ["config", "user.name", "t"], ["config", "commit.gpgsign", "false"]): + _git(root, *cmd) + for task in ("t1", "my task"): + _write(root, f"{TASKS}/{task}/task.json", "{}") + _write(root, f"{TASKS}/{task}/grader.py") + _write(root, "README.md") + _write(root, f"{EVAL_DIR}/repo-eval.yaml") + _git(root, "add", "-A") + _git(root, "commit", "-q", "-m", "base") + _git(root, "update-ref", "refs/remotes/origin/main", "HEAD") + return root + + +def _select(repo: Path, tmp_path: Path, **overrides: str) -> list[str]: + """Commit nothing more; run the step on the working tree's diff against origin/main.""" + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "change", "--allow-empty") + temp = tmp_path / "runner" + temp.mkdir(exist_ok=True) + (temp / "validate.json").write_text(json.dumps({"manifest": { + "id": "demo", "tasks": TASKS, "eval_dir": EVAL_DIR}}), encoding="utf-8") + env = { + **os.environ, "RUNNER_TEMP": str(temp), "GITHUB_OUTPUT": str(temp / "out"), + "GITHUB_STEP_SUMMARY": str(temp / "summary"), "EVENT": "pull_request", "BASE_REF": "main", + "BEFORE": "", "WORKFLOW_REF": "o/r/.github/workflows/ci.yml@refs/heads/x", "REPOSITORY": "o/r", + "MANIFEST": f"{EVAL_DIR}/repo-eval.yaml", "TASKS": "changed", "SMOKE_TASKS": "t1", + "SMOKE_PATHS": "", "EXCLUDE_TASKS": "", **overrides, + } + (temp / "out").write_text("", encoding="utf-8") + result = subprocess.run([sys.executable, "-c", _select_script()], cwd=repo, env=env, + capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stdout + result.stderr + line = next(ln for ln in (temp / "out").read_text(encoding="utf-8").splitlines() if ln.startswith("tasks=")) + return json.loads(line.removeprefix("tasks=")) + + +def test_a_task_directory_with_a_space_is_selected_whole(repo, tmp_path): + _write(repo, f"{TASKS}/my task/grader.py", "changed\n") + assert _select(repo, tmp_path) == ["my task"] + + +def test_a_quoted_file_name_does_not_hide_its_task(repo, tmp_path): + _write(repo, f'{TASKS}/t1/we"irdé name.txt') + assert _select(repo, tmp_path) == ["t1"] + + +def test_both_sides_of_a_move_between_tasks_are_selected(repo, tmp_path): + _write(repo, f"{TASKS}/t1/big.py", "".join(f"line {i}\n" for i in range(200))) + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "more") + _git(repo, "update-ref", "refs/remotes/origin/main", "HEAD") + (repo / TASKS / "t1" / "big.py").rename(repo / TASKS / "my task" / "big.py") + assert _select(repo, tmp_path) == ["my task", "t1"] + + +def test_a_shared_helper_directly_in_the_tasks_dir_runs_the_smoke_set(repo, tmp_path): + _write(repo, f"{TASKS}/common.py") + assert _select(repo, tmp_path) == ["t1"] + + +def test_a_shared_directory_in_the_tasks_dir_runs_the_smoke_set(repo, tmp_path): + _write(repo, f"{TASKS}/_shared/lib.py") + assert _select(repo, tmp_path) == ["t1"] + + +def test_a_deleted_task_directory_runs_the_smoke_set(repo, tmp_path): + shutil.rmtree(repo / TASKS / "my task") + assert _select(repo, tmp_path) == ["t1"] + + +def test_a_harness_file_in_the_eval_dir_adds_the_smoke_set_to_the_changed_task(repo, tmp_path): + _write(repo, f"{EVAL_DIR}/Dockerfile") + _write(repo, f"{TASKS}/my task/grader.py", "changed\n") + assert _select(repo, tmp_path) == ["my task", "t1"] -def parse_changed_files(output: str) -> list[str]: - """Parse NUL-separated git diff output (regression for PR #113 thread 1).""" - return [f for f in output.split('\0') if f] - - -class TestChangedFilesNulSeparation: - """Verify task selection handles NUL-separated filenames with spaces and renames.""" - - def test_nul_separated_output_with_spaces(self) -> None: - """Files with spaces should be parsed correctly when NUL-separated.""" - # Simulate git diff --no-renames --name-only -z output - output = "tasks/task1/file.txt\0tasks/agent eval/file.py\0tasks/task2/file.md\0" - result = parse_changed_files(output) - assert result == [ - "tasks/task1/file.txt", - "tasks/agent eval/file.py", - "tasks/task2/file.md", - ] - - def test_nul_separated_output_with_quotes(self) -> None: - """Files with quotes (Git's quoted format) should be parsed correctly.""" - # Git quotes filenames with special chars when not using -z - # With -z, they come unquoted - output = 'tasks/file"name".txt\0tasks/file\'name\'.py\0' - result = parse_changed_files(output) - assert result == ['tasks/file"name".txt', "tasks/file'name'.py"] - - def test_nul_separated_empty_output(self) -> None: - """Empty output should produce empty list.""" - output = "" - result = parse_changed_files(output) - assert result == [] - - def test_nul_separated_trailing_nul(self) -> None: - """Trailing NUL should not produce empty entry.""" - output = "file1.txt\0file2.txt\0" - result = parse_changed_files(output) - assert result == ["file1.txt", "file2.txt"] - - def test_no_renames_flag_prevents_omission(self) -> None: - """--no-renames flag ensures both source and dest of moves are reported.""" - # Without --no-renames: file moved from A to B reports only B - # With --no-renames: both A and B are reported - # This test validates the flag is present in the workflow - wf_path = Path(__file__).resolve().parents[2] / ".github/workflows/repo-eval.yml" - with open(wf_path, encoding="utf-8") as f: - workflow = f.read() - # Verify the --no-renames flag is in the git diff command - assert '--no-renames' in workflow - assert '--name-only' in workflow - assert '-z' in workflow - # Check the order and that they're in a git diff command - assert 'git", "diff", "--no-renames", "--name-only", "-z"' in workflow - - -class TestUploadArtifactErrFiles: - """Verify upload artifact includes .err files for grader/oracle stderr.""" - - def test_err_files_in_upload_path(self) -> None: - """Verify *.err pattern is included in upload artifact paths.""" - wf_path = Path(__file__).resolve().parents[2] / ".github/workflows/repo-eval.yml" - with open(wf_path, encoding="utf-8") as f: - workflow = f.read() - # Find the upload artifact section for check results - assert "repo-eval-check-" in workflow - # Verify *.err files are listed - assert '${{ runner.temp }}/repo-check/check/*/*.err' in workflow - # Verify it's in the right section (after the log files) - artifact_section = workflow[ - workflow.find("Upload the check results") : - ] - assert '*.err' in artifact_section or '*/*.err' in artifact_section - - def test_err_files_purpose(self) -> None: - """Document that .err files contain grader and oracle stderr.""" - # This is a documentation test; the actual files are: - # - grade.err: grader stderr - # - oracle.err: oracle stderr - # - isolation.err: isolation probe stderr - # These are all captured by repos/grading.py:124 and the isolation probe - files_expected = ["grade.err", "oracle.err", "isolation.err"] - # This validates the reviewer's comment about these files - assert len(files_expected) == 3 - - -class TestSharedFilesUnderTasksDir: - """Verify shared helper files under tasks dir are treated as harness changes.""" - - def test_shared_file_under_tasks_dir_selects_smoke(self) -> None: - """A file like tasks/common.py (helper shared by graders) should trigger smoke set.""" - wf_path = Path(__file__).resolve().parents[2] / ".github/workflows/repo-eval.yml" - with open(wf_path, encoding="utf-8") as f: - workflow = f.read() - # Verify the workflow handles non-task-id first segments under prefix as harness - assert "first_segment not in all_ids" in workflow - assert "shared helper" in workflow or "tasks/common.py" in workflow.lower() or "tasks dir" in workflow.lower() - - def test_deleted_task_directory_handled(self) -> None: - """A deleted task directory (first segment not in all_ids) should also be harness.""" - # The workflow should treat deleted task dirs the same way as shared files - wf_path = Path(__file__).resolve().parents[2] / ".github/workflows/repo-eval.yml" - with open(wf_path, encoding="utf-8") as f: - workflow = f.read() - # Verify the logic handles non-task-id segments under prefix - assert "first_segment not in all_ids" in workflow - - -class TestManifestPathNormalization: - """Verify manifest path is normalized for comparison with git output.""" - - def test_manifest_normalized_in_workflow(self) -> None: - """Manifest should be normalized with posixpath.normpath for git comparison.""" - wf_path = Path(__file__).resolve().parents[2] / ".github/workflows/repo-eval.yml" - with open(wf_path, encoding="utf-8") as f: - workflow = f.read() - # Verify posixpath.normpath is used on MANIFEST - assert "posixpath.normpath" in workflow - assert "manifest_normalized" in workflow - # Verify manifest comparison uses the normalized version - assert "manifest_normalized" in workflow +@pytest.mark.parametrize("manifest", [f"{EVAL_DIR}/repo-eval.yaml", f"./{EVAL_DIR}/../repo-eval.yaml"]) +def test_the_manifest_is_compared_normalised(repo, tmp_path, manifest): + if manifest.startswith("./"): + _write(repo, "repo-eval.yaml") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "m") + _git(repo, "update-ref", "refs/remotes/origin/main", "HEAD") + _write(repo, "repo-eval.yaml", "changed\n") + else: + _write(repo, f"{EVAL_DIR}/repo-eval.yaml", "changed\n") + assert _select(repo, tmp_path, MANIFEST=manifest) == ["t1"] + + +def test_an_unrelated_file_selects_nothing(repo, tmp_path): + _write(repo, "README.md", "changed\n") + assert _select(repo, tmp_path) == [] + + +def test_the_check_results_artifact_keeps_the_stderr_files(): + upload = next(s for s in _steps() if "Upload the check results" in str(s.get("name", ""))) + paths = [ln.strip() for ln in str(upload["with"]["path"]).splitlines()] + assert "${{ runner.temp }}/repo-check/check/*/*.err" in paths From 4670ea01494d1672f84aedb724d6c67a5f7f0971 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 19 Oct 2026 13:20:46 -0700 Subject: [PATCH 06/20] fix(ci): repo-eval diffs against the event's base SHA and exempts tasks-dir Markdown The changed-task diff uses github.event.pull_request.base.sha instead of a local origin/ ref, dokimi-ref must resolve to a full 40-digit SHA, and Markdown under the eval dir (the tasks dir included) no longer selects the smoke set. Tests cover explicit lists, all with exclude-tasks, and push with and without a previous tip; the contract says a deleted task's files select the smoke set. Signed-off-by: Nikolay Petrov --- .github/workflows/repo-eval.yml | 17 ++++++----- dokimi/docs/repo-evaluation-contract.md | 5 ++-- dokimi/tests/test_repo_eval_task_selection.py | 30 +++++++++++++++++-- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index c6d4c4ac..08456ca3 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -132,7 +132,7 @@ jobs: sha=$(git ls-remote --tags "https://github.com/$DOKIMI_REPO" \ | awk -v t="refs/tags/$REF" '$2 == t "^{}" { p = $1 } $2 == t { l = $1 } END { print (p != "" ? p : l) }') fi - if [ -z "${sha:-}" ]; then + if ! [[ "${sha:-}" =~ ^[0-9a-f]{40}$ ]]; then echo "::error title=dokimi-ref::dokimi-ref is neither a 40-digit commit SHA nor a tag of $DOKIMI_REPO" exit 1 fi @@ -172,7 +172,7 @@ jobs: id: select env: EVENT: ${{ github.event_name }} - BASE_REF: ${{ github.event.pull_request.base.ref }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} BEFORE: ${{ github.event.before }} WORKFLOW_REF: ${{ github.workflow_ref }} REPOSITORY: ${{ github.repository }} @@ -206,8 +206,9 @@ jobs: all_ids = sorted(p.parent.name for p in Path(tasks_dir).glob("*/task.json")) def changed_files(): - if env["EVENT"] == "pull_request" and env["BASE_REF"]: - rng = [f"origin/{env['BASE_REF']}...HEAD"] + # the base the PR was opened against, from the event: no reliance on a local origin/ ref + if env["EVENT"] == "pull_request" and env["BASE_SHA"]: + rng = [f"{env['BASE_SHA']}...HEAD"] elif env["EVENT"] == "push" and env["BEFORE"].strip("0"): rng = [env["BEFORE"], "HEAD"] else: @@ -237,15 +238,17 @@ jobs: manifest_normalized = posixpath.normpath(env["MANIFEST"]) def harness(f): + if f.startswith(eval_dir) and f.endswith(".md"): + return False # Markdown in the eval dir, its tasks dir included, selects nothing if f in (manifest_normalized, caller) or any(f.startswith(r) for r in roots): return True if f.startswith(prefix): - # a file under prefix (tasks dir) whose first segment is not a task id - # is a shared helper (e.g., tasks/common.py); treat as harness + # a file under prefix (tasks dir) whose first segment is not an existing task id + # is a shared helper (e.g., tasks/common.py) or a deleted task's file: harness first_segment = f[len(prefix):].split("/", 1)[0] if first_segment not in all_ids: return True - return f.startswith(eval_dir) and not f.startswith(prefix) and not f.endswith(".md") + return f.startswith(eval_dir) and not f.startswith(prefix) hits = [f for f in files if harness(f)] if hits: diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 07743953..c6c79b14 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -732,8 +732,9 @@ that runs the no-LLM checks on the caller's checkout and needs no secret. manifest's Dockerfile), and uploads `check.json` and the grade logs. - `tasks: changed` (the default) checks, on a pull request, the tasks whose directories it touches, plus `smoke-tasks` (empty: every task) when it changes - the eval directory outside its tasks (Markdown excepted), the manifest, the - calling workflow file or a path under `smoke-paths`. On a push it diffs against + the eval directory outside its tasks (Markdown anywhere in it excepted), the manifest, the + calling workflow file or a path under `smoke-paths`. A file in the tasks directory that is + not under an existing task (a shared helper, or the files of a deleted task) does too. On a push it diffs against the previous tip, on any other event it checks every task. `all` or a list of task ids selects directly; `exclude-tasks` always drops tasks. - Pin dokimi by a commit SHA or a release tag, the same one in `uses:` and in diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index 06e05d87..60e6b5f2 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -79,8 +79,9 @@ def _select(repo: Path, tmp_path: Path, **overrides: str) -> list[str]: "id": "demo", "tasks": TASKS, "eval_dir": EVAL_DIR}}), encoding="utf-8") env = { **os.environ, "RUNNER_TEMP": str(temp), "GITHUB_OUTPUT": str(temp / "out"), - "GITHUB_STEP_SUMMARY": str(temp / "summary"), "EVENT": "pull_request", "BASE_REF": "main", - "BEFORE": "", "WORKFLOW_REF": "o/r/.github/workflows/ci.yml@refs/heads/x", "REPOSITORY": "o/r", + "GITHUB_STEP_SUMMARY": str(temp / "summary"), "EVENT": "pull_request", + "BEFORE": "", "BASE_SHA": "origin/main", "WORKFLOW_REF": "o/r/.github/workflows/ci.yml@refs/heads/x", + "REPOSITORY": "o/r", "MANIFEST": f"{EVAL_DIR}/repo-eval.yaml", "TASKS": "changed", "SMOKE_TASKS": "t1", "SMOKE_PATHS": "", "EXCLUDE_TASKS": "", **overrides, } @@ -150,6 +151,31 @@ def test_an_unrelated_file_selects_nothing(repo, tmp_path): assert _select(repo, tmp_path) == [] +def test_markdown_in_the_tasks_dir_selects_nothing(repo, tmp_path): + _write(repo, f"{TASKS}/README.md", "changed\n") + assert _select(repo, tmp_path) == [] + + +def test_an_explicit_task_list_is_checked_as_given(repo, tmp_path): + _write(repo, "README.md", "changed\n") + assert _select(repo, tmp_path, TASKS="t1") == ["t1"] + + +def test_all_minus_excluded_tasks(repo, tmp_path): + _write(repo, "README.md", "changed\n") + assert _select(repo, tmp_path, TASKS="all", EXCLUDE_TASKS="t1") == ["my task"] + + +def test_a_push_diffs_against_the_previous_tip(repo, tmp_path): + _write(repo, f"{TASKS}/my task/grader.py", "changed\n") + assert _select(repo, tmp_path, EVENT="push", BEFORE="origin/main") == ["my task"] + + +def test_a_push_without_a_previous_tip_checks_every_task(repo, tmp_path): + _write(repo, f"{TASKS}/my task/grader.py", "changed\n") + assert _select(repo, tmp_path, EVENT="push", BEFORE="0" * 40) == ["my task", "t1"] + + def test_the_check_results_artifact_keeps_the_stderr_files(): upload = next(s for s in _steps() if "Upload the check results" in str(s.get("name", ""))) paths = [ln.strip() for ln in str(upload["with"]["path"]).splitlines()] From a5551bddc98ba2cc651d0959f6765f63098e37cf Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 19 Oct 2026 14:12:56 -0700 Subject: [PATCH 07/20] fix(repo-validate): refuse task ids that contain whitespace The workflow's tasks, smoke-tasks and exclude-tasks inputs are split on whitespace, so a task directory with a space could never be named there. Reject such ids in repo validate and document the rule in the contract. Signed-off-by: Nikolay Petrov --- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/repos/manifest.py | 6 ++++++ dokimi/tests/test_repo_manifest.py | 11 +++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index c6c79b14..7f0d8931 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -217,7 +217,7 @@ belongs to the repository's grader and is passed through unread. | Key | Required | Read by | Meaning | |---|---|---|---| -| `id` | yes | framework | equals the directory name | +| `id` | yes | framework | equals the directory name, and contains no whitespace: the workflow's `tasks`, `smoke-tasks` and `exclude-tasks` inputs are whitespace-separated lists of ids, so `repo validate` refuses an id with whitespace | | `family` | yes | framework (reporting) | free string; common values: `build`, `fix`, `feature`, `test`, `review` | | `base` | yes | framework | commit to snapshot: a hex SHA, optionally followed by `^` (the parent, the usual form for a mined fix). Branch names, tags and `HEAD` are refused, because repetitions of one task must get one snapshot. Use the full 40-character SHA; a shorter one is accepted with a warning. It must resolve in the source. `meta.json` records it as `base_rev` and the resolved full SHA as `base_sha` | | `prompt` | yes | framework | what the agent is told | diff --git a/dokimi/dokimi/repos/manifest.py b/dokimi/dokimi/repos/manifest.py index 7edce9ab..ffc92f06 100644 --- a/dokimi/dokimi/repos/manifest.py +++ b/dokimi/dokimi/repos/manifest.py @@ -416,6 +416,12 @@ def _glob_regex(pattern: str) -> re.Pattern[str]: def _check_task(tree: RepoTree, task_dir: str, report: RepoCheckReport) -> None: name = PurePosixPath(task_dir).name where = f"{task_dir}/task.json" + if re.search(r"\s", name): + # tasks, smoke-tasks and exclude-tasks are whitespace-separated lists of ids, so an id with + # whitespace could never be named there + report.errors.append(f"{where}: task id {name!r} contains whitespace; task ids are " + "whitespace-separated in the workflow's tasks, smoke-tasks and exclude-tasks inputs") + return try: raw = json.loads(tree.read_text(where)) except (json.JSONDecodeError, UnicodeDecodeError) as exc: diff --git a/dokimi/tests/test_repo_manifest.py b/dokimi/tests/test_repo_manifest.py index bdd81c26..1a289cc6 100644 --- a/dokimi/tests/test_repo_manifest.py +++ b/dokimi/tests/test_repo_manifest.py @@ -187,6 +187,17 @@ def test_bad_task_is_reported(tmp_path, task_extra, needle): assert "bad" not in [t.id for t in report.tasks] +@pytest.mark.parametrize("task_id", ["my task", "tab\tid"]) +def test_task_id_with_whitespace_is_refused(tmp_path, task_id): + # the workflow names tasks in whitespace-separated lists, so such an id could never be selected + root = _repo(tmp_path) + _task(root, task_id) + report = check_repo(WorkTree(root), load_manifest(WorkTree(root), "dev/evals/repo-eval.yaml")) + assert not report.ok + assert any(f"task id {task_id!r} contains whitespace" in e for e in report.errors), report.errors + assert task_id not in [t.id for t in report.tasks] + + def test_task_gate_score_and_short_base(tmp_path): root = _repo(tmp_path) _task(root, "rev", family="review", base="cf0c0a0b^", gate=[{"metric": "recall", "min": 0.75}, "format_ok"], From f015ed5b7d9c84c5a089cb421b4986838ca0a208 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Mon, 19 Oct 2026 14:41:00 -0700 Subject: [PATCH 08/20] docs(changelog): add fragment for repo validate whitespace task-id check Signed-off-by: Nikolay Petrov --- .../20261019_144048_napetrov_d8-repo-eval-workflow.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 dokimi/changelog.d/20261019_144048_napetrov_d8-repo-eval-workflow.md diff --git a/dokimi/changelog.d/20261019_144048_napetrov_d8-repo-eval-workflow.md b/dokimi/changelog.d/20261019_144048_napetrov_d8-repo-eval-workflow.md new file mode 100644 index 00000000..061725ff --- /dev/null +++ b/dokimi/changelog.d/20261019_144048_napetrov_d8-repo-eval-workflow.md @@ -0,0 +1,3 @@ +### Fixed + +- **`repo validate` refuses task ids that contain whitespace** — a task directory whose name has a space or tab is now a `task.json` error instead of passing validation. The repo-eval workflow's `tasks`, `smoke-tasks` and `exclude-tasks` inputs are whitespace-separated lists of ids, so such a task could never be named there. Rename the task directory to a name without whitespace. From fe09af06dfa693a1bac27e09663fa7737c82f446 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 20 Oct 2026 00:00:14 -0700 Subject: [PATCH 09/20] ci(repo-eval): diff a pull request's merge commit against its first parent base.sha is the base at PR-open time, so BASE_SHA...HEAD on the merge checkout also counts every commit the base gained since as the PR's change and selects those tasks (and the smoke set). HEAD^1 is the base tip the merge was made on; base.sha stays the fallback for a non-merge HEAD. Signed-off-by: Alexander Rybkin --- .github/workflows/repo-eval.yml | 6 +++-- dokimi/tests/test_repo_eval_task_selection.py | 24 +++++++++++++++---- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index 08456ca3..4c7f91e5 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -206,9 +206,11 @@ jobs: all_ids = sorted(p.parent.name for p in Path(tasks_dir).glob("*/task.json")) def changed_files(): - # the base the PR was opened against, from the event: no reliance on a local origin/ ref + # HEAD is GitHub's merge commit; its first parent is the base tip now, while base.sha is + # the base at PR-open time and would count what the base gained since as the PR's change if env["EVENT"] == "pull_request" and env["BASE_SHA"]: - rng = [f"{env['BASE_SHA']}...HEAD"] + merge = subprocess.run(["git", "rev-parse", "-q", "--verify", "HEAD^2"], capture_output=True).returncode == 0 + rng = ["HEAD^1...HEAD" if merge else f"{env['BASE_SHA']}...HEAD"] elif env["EVENT"] == "push" and env["BEFORE"].strip("0"): rng = [env["BEFORE"], "HEAD"] else: diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index 60e6b5f2..547d1cfa 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -69,10 +69,11 @@ def repo(tmp_path: Path) -> Path: return root -def _select(repo: Path, tmp_path: Path, **overrides: str) -> list[str]: - """Commit nothing more; run the step on the working tree's diff against origin/main.""" - _git(repo, "add", "-A") - _git(repo, "commit", "-q", "-m", "change", "--allow-empty") +def _select(repo: Path, tmp_path: Path, *, commit: bool = True, **overrides: str) -> list[str]: + """Commit the working tree (unless ``commit`` is false) and run the step on HEAD.""" + if commit: + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "change", "--allow-empty") temp = tmp_path / "runner" temp.mkdir(exist_ok=True) (temp / "validate.json").write_text(json.dumps({"manifest": { @@ -176,6 +177,21 @@ def test_a_push_without_a_previous_tip_checks_every_task(repo, tmp_path): assert _select(repo, tmp_path, EVENT="push", BEFORE="0" * 40) == ["my task", "t1"] +def test_a_merge_commit_ignores_what_the_base_gained_after_the_pr_opened(repo, tmp_path): + # base.sha stays at the fork point while the checkout merges into the base's current tip + fork = _git(repo, "rev-parse", "HEAD").strip() + _git(repo, "checkout", "-q", "-b", "pr") + _write(repo, f"{TASKS}/t1/grader.py", "changed\n") + _git(repo, "commit", "-q", "-a", "-m", "pr") + _git(repo, "checkout", "-q", "main") + _write(repo, f"{TASKS}/my task/grader.py", "changed\n") + _write(repo, f"{EVAL_DIR}/Dockerfile") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "main moves on") + _git(repo, "merge", "-q", "--no-ff", "--no-edit", "pr") + assert _select(repo, tmp_path, commit=False, BASE_SHA=fork, SMOKE_TASKS="my task") == ["t1"] + + def test_the_check_results_artifact_keeps_the_stderr_files(): upload = next(s for s in _steps() if "Upload the check results" in str(s.get("name", ""))) paths = [ln.strip() for ln in str(upload["with"]["path"]).splitlines()] From 498b938edae59b3af6e9c4c561fa73b3db5c85ea Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 20 Oct 2026 00:01:14 -0700 Subject: [PATCH 10/20] ci(repo-eval): treat an empty tasks input as changed A caller forwarding an unset dispatch input passes '' on pull_request, which selected an explicit empty list: nothing checked, tasks-check skipped, run green. Empty, blank or separator-only now means the documented default. Signed-off-by: Alexander Rybkin --- .github/workflows/repo-eval.yml | 5 +++-- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/tests/test_repo_eval_task_selection.py | 6 ++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index 4c7f91e5..d4b0b3ea 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -44,7 +44,7 @@ on: type: string default: .agent-eval/repo-eval.yaml tasks: - description: "`changed`, `all`, or task ids separated by spaces or commas" + description: "`changed`, `all`, or task ids separated by spaces or commas (empty: `changed`)" required: false type: string default: changed @@ -222,7 +222,8 @@ jobs: return r.stdout.split('\0') if r.stdout else [] import posixpath - mode = env["TASKS"].strip() + # empty is the default: a caller forwarding an unset dispatch input passes '', not `changed` + mode = env["TASKS"].strip() if ids(env["TASKS"]) else "changed" if mode == "all": sel = list(all_ids) elif mode == "changed": diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 7f0d8931..bd3ba955 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -730,7 +730,7 @@ that runs the no-LLM checks on the caller's checkout and needs no secret. `repo_readiness.v1` report to an artifact); `tasks-check` runs `repo tasks check` for each selected task in a job of its own (docker; the image is built from the manifest's Dockerfile), and uploads `check.json` and the grade logs. -- `tasks: changed` (the default) checks, on a pull request, the tasks whose +- `tasks: changed` (the default, also for an empty value) checks, on a pull request, the tasks whose directories it touches, plus `smoke-tasks` (empty: every task) when it changes the eval directory outside its tasks (Markdown anywhere in it excepted), the manifest, the calling workflow file or a path under `smoke-paths`. A file in the tasks directory that is diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index 547d1cfa..a642effc 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -162,6 +162,12 @@ def test_an_explicit_task_list_is_checked_as_given(repo, tmp_path): assert _select(repo, tmp_path, TASKS="t1") == ["t1"] +@pytest.mark.parametrize("tasks", ["", " ", ","]) +def test_an_empty_task_list_means_changed(repo, tmp_path, tasks): + _write(repo, f"{TASKS}/my task/grader.py", "changed\n") + assert _select(repo, tmp_path, TASKS=tasks) == ["my task"] + + def test_all_minus_excluded_tasks(repo, tmp_path): _write(repo, "README.md", "changed\n") assert _select(repo, tmp_path, TASKS="all", EXCLUDE_TASKS="t1") == ["my task"] From 5e6874f15f4b20bd142c99978d77db8ae8d74fde Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Tue, 20 Oct 2026 00:01:31 -0700 Subject: [PATCH 11/20] test(repo-eval): pin the empty smoke set and the guidance-only manifest Both are what a caller gets without configuring anything, and dropping `or all_ids` or the guidance-only early exit left every test green. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_repo_eval_task_selection.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index a642effc..eeee8f0a 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -69,7 +69,8 @@ def repo(tmp_path: Path) -> Path: return root -def _select(repo: Path, tmp_path: Path, *, commit: bool = True, **overrides: str) -> list[str]: +def _select(repo: Path, tmp_path: Path, *, commit: bool = True, manifest_tasks: str | None = TASKS, + **overrides: str) -> list[str]: """Commit the working tree (unless ``commit`` is false) and run the step on HEAD.""" if commit: _git(repo, "add", "-A") @@ -77,7 +78,7 @@ def _select(repo: Path, tmp_path: Path, *, commit: bool = True, **overrides: str temp = tmp_path / "runner" temp.mkdir(exist_ok=True) (temp / "validate.json").write_text(json.dumps({"manifest": { - "id": "demo", "tasks": TASKS, "eval_dir": EVAL_DIR}}), encoding="utf-8") + "id": "demo", "tasks": manifest_tasks, "eval_dir": EVAL_DIR}}), encoding="utf-8") env = { **os.environ, "RUNNER_TEMP": str(temp), "GITHUB_OUTPUT": str(temp / "out"), "GITHUB_STEP_SUMMARY": str(temp / "summary"), "EVENT": "pull_request", @@ -147,6 +148,16 @@ def test_the_manifest_is_compared_normalised(repo, tmp_path, manifest): assert _select(repo, tmp_path, MANIFEST=manifest) == ["t1"] +def test_an_empty_smoke_set_means_every_task(repo, tmp_path): + _write(repo, f"{EVAL_DIR}/Dockerfile") + assert _select(repo, tmp_path, SMOKE_TASKS="") == ["my task", "t1"] + + +def test_a_guidance_only_manifest_checks_no_task(repo, tmp_path): + _write(repo, f"{EVAL_DIR}/Dockerfile") + assert _select(repo, tmp_path, manifest_tasks=None) == [] + + def test_an_unrelated_file_selects_nothing(repo, tmp_path): _write(repo, "README.md", "changed\n") assert _select(repo, tmp_path) == [] From 279db0f30e989ba5af782295f1542189ed175891 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 20 Oct 2026 04:03:50 -0700 Subject: [PATCH 12/20] fix(repo-eval): smoke paths win over the Markdown exemption; refuse commas in task ids A Markdown file named in smoke-paths selected nothing because the eval-dir Markdown exemption ran first. The explicit smoke path check now runs before it. The workflow splits its task inputs on commas as well as whitespace, so a task id containing a comma passed validation but could not be named, and exclude-tasks silently ignored it. repo validate now refuses commas too. The fragment moves to Changed, since this is a new validation error. Signed-off-by: Nikolay Petrov --- .github/workflows/repo-eval.yml | 4 ++-- .../20261019_144048_napetrov_d8-repo-eval-workflow.md | 4 ++-- dokimi/docs/repo-evaluation-contract.md | 2 +- dokimi/dokimi/repos/manifest.py | 10 +++++----- dokimi/tests/test_repo_eval_task_selection.py | 7 +++++++ dokimi/tests/test_repo_manifest.py | 9 +++++---- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index d4b0b3ea..11b57c12 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -241,10 +241,10 @@ jobs: manifest_normalized = posixpath.normpath(env["MANIFEST"]) def harness(f): + if f in (manifest_normalized, caller) or any(f.startswith(r) for r in roots): + return True # an explicit smoke path wins over the Markdown exemption below if f.startswith(eval_dir) and f.endswith(".md"): return False # Markdown in the eval dir, its tasks dir included, selects nothing - if f in (manifest_normalized, caller) or any(f.startswith(r) for r in roots): - return True if f.startswith(prefix): # a file under prefix (tasks dir) whose first segment is not an existing task id # is a shared helper (e.g., tasks/common.py) or a deleted task's file: harness diff --git a/dokimi/changelog.d/20261019_144048_napetrov_d8-repo-eval-workflow.md b/dokimi/changelog.d/20261019_144048_napetrov_d8-repo-eval-workflow.md index 061725ff..903ab5ba 100644 --- a/dokimi/changelog.d/20261019_144048_napetrov_d8-repo-eval-workflow.md +++ b/dokimi/changelog.d/20261019_144048_napetrov_d8-repo-eval-workflow.md @@ -1,3 +1,3 @@ -### Fixed +### Changed -- **`repo validate` refuses task ids that contain whitespace** — a task directory whose name has a space or tab is now a `task.json` error instead of passing validation. The repo-eval workflow's `tasks`, `smoke-tasks` and `exclude-tasks` inputs are whitespace-separated lists of ids, so such a task could never be named there. Rename the task directory to a name without whitespace. +- **`repo validate` refuses task ids that contain whitespace or a comma** — a task directory whose name has a space, tab or comma is now a `task.json` error instead of passing validation. The repo-eval workflow's `tasks`, `smoke-tasks` and `exclude-tasks` inputs are split on whitespace and commas, so such a task could never be named there. Rename the task directory to a name without whitespace or commas. diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index bd3ba955..904b1f9a 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -217,7 +217,7 @@ belongs to the repository's grader and is passed through unread. | Key | Required | Read by | Meaning | |---|---|---|---| -| `id` | yes | framework | equals the directory name, and contains no whitespace: the workflow's `tasks`, `smoke-tasks` and `exclude-tasks` inputs are whitespace-separated lists of ids, so `repo validate` refuses an id with whitespace | +| `id` | yes | framework | equals the directory name, and contains no whitespace or comma: the workflow's `tasks`, `smoke-tasks` and `exclude-tasks` inputs are lists of ids split on whitespace and commas, so `repo validate` refuses an id containing either | | `family` | yes | framework (reporting) | free string; common values: `build`, `fix`, `feature`, `test`, `review` | | `base` | yes | framework | commit to snapshot: a hex SHA, optionally followed by `^` (the parent, the usual form for a mined fix). Branch names, tags and `HEAD` are refused, because repetitions of one task must get one snapshot. Use the full 40-character SHA; a shorter one is accepted with a warning. It must resolve in the source. `meta.json` records it as `base_rev` and the resolved full SHA as `base_sha` | | `prompt` | yes | framework | what the agent is told | diff --git a/dokimi/dokimi/repos/manifest.py b/dokimi/dokimi/repos/manifest.py index ffc92f06..8ea2ca09 100644 --- a/dokimi/dokimi/repos/manifest.py +++ b/dokimi/dokimi/repos/manifest.py @@ -416,11 +416,11 @@ def _glob_regex(pattern: str) -> re.Pattern[str]: def _check_task(tree: RepoTree, task_dir: str, report: RepoCheckReport) -> None: name = PurePosixPath(task_dir).name where = f"{task_dir}/task.json" - if re.search(r"\s", name): - # tasks, smoke-tasks and exclude-tasks are whitespace-separated lists of ids, so an id with - # whitespace could never be named there - report.errors.append(f"{where}: task id {name!r} contains whitespace; task ids are " - "whitespace-separated in the workflow's tasks, smoke-tasks and exclude-tasks inputs") + if re.search(r"[\s,]", name): + # tasks, smoke-tasks and exclude-tasks are lists of ids split on whitespace and commas, so an id + # containing either could never be named there + report.errors.append(f"{where}: task id {name!r} contains whitespace or a comma; task ids are " + "split on whitespace and commas in the workflow's tasks, smoke-tasks and exclude-tasks inputs") return try: raw = json.loads(tree.read_text(where)) diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index eeee8f0a..e0e9db2e 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -168,6 +168,13 @@ def test_markdown_in_the_tasks_dir_selects_nothing(repo, tmp_path): assert _select(repo, tmp_path) == [] +@pytest.mark.parametrize("smoke_path", [f"{EVAL_DIR}/rubric.md", f"{EVAL_DIR}/prompts/"]) +def test_a_markdown_file_named_in_smoke_paths_runs_the_smoke_set(repo, tmp_path, smoke_path): + _write(repo, f"{EVAL_DIR}/prompts/p.md", "changed\n") + _write(repo, f"{EVAL_DIR}/rubric.md", "changed\n") + assert _select(repo, tmp_path, SMOKE_PATHS=smoke_path) == ["t1"] + + def test_an_explicit_task_list_is_checked_as_given(repo, tmp_path): _write(repo, "README.md", "changed\n") assert _select(repo, tmp_path, TASKS="t1") == ["t1"] diff --git a/dokimi/tests/test_repo_manifest.py b/dokimi/tests/test_repo_manifest.py index 1a289cc6..5f40fcfd 100644 --- a/dokimi/tests/test_repo_manifest.py +++ b/dokimi/tests/test_repo_manifest.py @@ -187,14 +187,15 @@ def test_bad_task_is_reported(tmp_path, task_extra, needle): assert "bad" not in [t.id for t in report.tasks] -@pytest.mark.parametrize("task_id", ["my task", "tab\tid"]) -def test_task_id_with_whitespace_is_refused(tmp_path, task_id): - # the workflow names tasks in whitespace-separated lists, so such an id could never be selected +@pytest.mark.parametrize("task_id", ["my task", "tab\tid", "a,b"]) +def test_task_id_with_whitespace_or_comma_is_refused(tmp_path, task_id): + # the workflow splits its tasks, smoke-tasks and exclude-tasks inputs on whitespace and commas, + # so such an id could never be selected root = _repo(tmp_path) _task(root, task_id) report = check_repo(WorkTree(root), load_manifest(WorkTree(root), "dev/evals/repo-eval.yaml")) assert not report.ok - assert any(f"task id {task_id!r} contains whitespace" in e for e in report.errors), report.errors + assert any(f"task id {task_id!r} contains whitespace or a comma" in e for e in report.errors), report.errors assert task_id not in [t.id for t in report.tasks] From 84fd3e8731cce095db6d27071570431cf483a3e6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 20 Oct 2026 14:36:07 -0700 Subject: [PATCH 13/20] fix(repo-eval): smoke-paths entries are normalised and split like ids() A smoke-paths entry with a ./ prefix or a comma separator matched nothing, so a change under it ran no task and the run stayed green. Entries are now split like the other id lists and normalised like the manifest path, with the trailing slash of a directory kept. The contract says so. Tests cover the calling workflow, the separators, unknown explicit and smoke task ids, and an unreadable previous push tip. Signed-off-by: Nikolay Petrov --- .github/workflows/repo-eval.yml | 3 +- dokimi/docs/repo-evaluation-contract.md | 11 +++-- dokimi/tests/test_repo_eval_task_selection.py | 41 ++++++++++++++++++- 3 files changed, 48 insertions(+), 7 deletions(-) diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index 11b57c12..d5c26e00 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -237,7 +237,8 @@ jobs: # the calling workflow: owner/repo/.github/workflows/x.yml@ref caller = env["WORKFLOW_REF"].split("@", 1)[0].removeprefix(env["REPOSITORY"] + "/") eval_dir = m["eval_dir"].rstrip("/") + "/" - roots = [p for p in env["SMOKE_PATHS"].split() if p] + # the same separators as ids(); normalised like the manifest, a trailing slash kept + roots = [posixpath.normpath(p) + ("/" if p.endswith("/") and p != "/" else "") for p in ids(env["SMOKE_PATHS"])] manifest_normalized = posixpath.normpath(env["MANIFEST"]) def harness(f): diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 904b1f9a..5a364a72 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -732,10 +732,13 @@ that runs the no-LLM checks on the caller's checkout and needs no secret. manifest's Dockerfile), and uploads `check.json` and the grade logs. - `tasks: changed` (the default, also for an empty value) checks, on a pull request, the tasks whose directories it touches, plus `smoke-tasks` (empty: every task) when it changes - the eval directory outside its tasks (Markdown anywhere in it excepted), the manifest, the - calling workflow file or a path under `smoke-paths`. A file in the tasks directory that is - not under an existing task (a shared helper, or the files of a deleted task) does too. On a push it diffs against - the previous tip, on any other event it checks every task. `all` or a list of + the eval directory outside its tasks (Markdown there is excepted, unless it is the manifest, + the calling workflow file or under a `smoke-paths` entry), the manifest, the calling workflow + file or a path under `smoke-paths` (entries are space- or comma-separated prefixes, + normalised like the manifest path; keep the trailing slash for a directory). A file in + the tasks directory that is not under an existing task (a shared helper, or the files of + a deleted task) does too. On a push it diffs against the previous tip, on any other event + it checks every task. `all` or a list of task ids selects directly; `exclude-tasks` always drops tasks. - Pin dokimi by a commit SHA or a release tag, the same one in `uses:` and in `dokimi-ref` (a branch is refused). It installs diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index e0e9db2e..bd0a6b2e 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -69,8 +69,8 @@ def repo(tmp_path: Path) -> Path: return root -def _select(repo: Path, tmp_path: Path, *, commit: bool = True, manifest_tasks: str | None = TASKS, - **overrides: str) -> list[str]: +def _run_step(repo: Path, tmp_path: Path, *, commit: bool = True, manifest_tasks: str | None = TASKS, + **overrides: str) -> tuple[subprocess.CompletedProcess, Path]: """Commit the working tree (unless ``commit`` is false) and run the step on HEAD.""" if commit: _git(repo, "add", "-A") @@ -90,6 +90,11 @@ def _select(repo: Path, tmp_path: Path, *, commit: bool = True, manifest_tasks: (temp / "out").write_text("", encoding="utf-8") result = subprocess.run([sys.executable, "-c", _select_script()], cwd=repo, env=env, capture_output=True, text=True, check=False) + return result, temp + + +def _select(repo: Path, tmp_path: Path, **overrides: str) -> list[str]: + result, temp = _run_step(repo, tmp_path, **overrides) assert result.returncode == 0, result.stdout + result.stderr line = next(ln for ln in (temp / "out").read_text(encoding="utf-8").splitlines() if ln.startswith("tasks=")) return json.loads(line.removeprefix("tasks=")) @@ -216,6 +221,38 @@ def test_a_merge_commit_ignores_what_the_base_gained_after_the_pr_opened(repo, t assert _select(repo, tmp_path, commit=False, BASE_SHA=fork, SMOKE_TASKS="my task") == ["t1"] +def test_a_change_to_the_calling_workflow_runs_the_smoke_set(repo, tmp_path): + _write(repo, ".github/workflows/ci.yml", "changed\n") + assert _select(repo, tmp_path) == ["t1"] + + +@pytest.mark.parametrize("smoke_paths", ["./vendor/", "vendor/,other/", "other/ vendor/"]) +def test_smoke_paths_are_normalised_and_split_like_ids(repo, tmp_path, smoke_paths): + _write(repo, "vendor/grader/lib.py", "changed\n") + assert _select(repo, tmp_path, SMOKE_PATHS=smoke_paths) == ["t1"] + + +def test_a_smoke_directory_without_its_slash_is_a_prefix_as_given(repo, tmp_path): + _write(repo, "vendor-old/x.py", "changed\n") + assert _select(repo, tmp_path, SMOKE_PATHS="vendor/") == [] + + +def test_an_unknown_explicit_task_fails_the_run(repo, tmp_path): + result, _ = _run_step(repo, tmp_path, TASKS="nope") + assert result.returncode != 0 and "no such task(s): nope" in result.stdout + result.stderr + + +def test_an_unknown_smoke_task_fails_the_run(repo, tmp_path): + _write(repo, f"{EVAL_DIR}/Dockerfile") + result, _ = _run_step(repo, tmp_path, SMOKE_TASKS="nope") + assert result.returncode != 0 and "no such task(s): nope" in result.stdout + result.stderr + + +def test_an_unreadable_previous_tip_checks_every_task(repo, tmp_path): + _write(repo, f"{TASKS}/my task/grader.py", "changed\n") + assert _select(repo, tmp_path, EVENT="push", BEFORE="1" * 40) == ["my task", "t1"] + + def test_the_check_results_artifact_keeps_the_stderr_files(): upload = next(s for s in _steps() if "Upload the check results" in str(s.get("name", ""))) paths = [ln.strip() for ln in str(upload["with"]["path"]).splitlines()] From 25686c46d14793f6a0b1480d8298852c92f6ae9b Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 20 Oct 2026 17:31:36 -0700 Subject: [PATCH 14/20] style(repo-eval): ruff format the task-selection tests Signed-off-by: Nikolay Petrov --- dokimi/tests/test_repo_eval_task_selection.py | 67 ++++++++++++++----- 1 file changed, 51 insertions(+), 16 deletions(-) diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index bd0a6b2e..789324f1 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -43,7 +43,9 @@ def _select_script() -> str: def _git(repo: Path, *args: str) -> str: - return subprocess.run(["git", *args], cwd=repo, check=True, capture_output=True, text=True).stdout + return subprocess.run( + ["git", *args], cwd=repo, check=True, capture_output=True, text=True + ).stdout def _write(repo: Path, rel: str, text: str = "x\n") -> None: @@ -55,8 +57,12 @@ def _write(repo: Path, rel: str, text: str = "x\n") -> None: def repo(tmp_path: Path) -> Path: root = tmp_path / "repo" root.mkdir() - for cmd in (["init", "-q", "-b", "main"], ["config", "user.email", "t@example.com"], - ["config", "user.name", "t"], ["config", "commit.gpgsign", "false"]): + for cmd in ( + ["init", "-q", "-b", "main"], + ["config", "user.email", "t@example.com"], + ["config", "user.name", "t"], + ["config", "commit.gpgsign", "false"], + ): _git(root, *cmd) for task in ("t1", "my task"): _write(root, f"{TASKS}/{task}/task.json", "{}") @@ -69,34 +75,61 @@ def repo(tmp_path: Path) -> Path: return root -def _run_step(repo: Path, tmp_path: Path, *, commit: bool = True, manifest_tasks: str | None = TASKS, - **overrides: str) -> tuple[subprocess.CompletedProcess, Path]: +def _run_step( + repo: Path, + tmp_path: Path, + *, + commit: bool = True, + manifest_tasks: str | None = TASKS, + **overrides: str, +) -> tuple[subprocess.CompletedProcess, Path]: """Commit the working tree (unless ``commit`` is false) and run the step on HEAD.""" if commit: _git(repo, "add", "-A") _git(repo, "commit", "-q", "-m", "change", "--allow-empty") temp = tmp_path / "runner" temp.mkdir(exist_ok=True) - (temp / "validate.json").write_text(json.dumps({"manifest": { - "id": "demo", "tasks": manifest_tasks, "eval_dir": EVAL_DIR}}), encoding="utf-8") + (temp / "validate.json").write_text( + json.dumps({"manifest": {"id": "demo", "tasks": manifest_tasks, "eval_dir": EVAL_DIR}}), + encoding="utf-8", + ) env = { - **os.environ, "RUNNER_TEMP": str(temp), "GITHUB_OUTPUT": str(temp / "out"), - "GITHUB_STEP_SUMMARY": str(temp / "summary"), "EVENT": "pull_request", - "BEFORE": "", "BASE_SHA": "origin/main", "WORKFLOW_REF": "o/r/.github/workflows/ci.yml@refs/heads/x", + **os.environ, + "RUNNER_TEMP": str(temp), + "GITHUB_OUTPUT": str(temp / "out"), + "GITHUB_STEP_SUMMARY": str(temp / "summary"), + "EVENT": "pull_request", + "BEFORE": "", + "BASE_SHA": "origin/main", + "WORKFLOW_REF": "o/r/.github/workflows/ci.yml@refs/heads/x", "REPOSITORY": "o/r", - "MANIFEST": f"{EVAL_DIR}/repo-eval.yaml", "TASKS": "changed", "SMOKE_TASKS": "t1", - "SMOKE_PATHS": "", "EXCLUDE_TASKS": "", **overrides, + "MANIFEST": f"{EVAL_DIR}/repo-eval.yaml", + "TASKS": "changed", + "SMOKE_TASKS": "t1", + "SMOKE_PATHS": "", + "EXCLUDE_TASKS": "", + **overrides, } (temp / "out").write_text("", encoding="utf-8") - result = subprocess.run([sys.executable, "-c", _select_script()], cwd=repo, env=env, - capture_output=True, text=True, check=False) + result = subprocess.run( + [sys.executable, "-c", _select_script()], + cwd=repo, + env=env, + capture_output=True, + text=True, + check=False, + ) return result, temp def _select(repo: Path, tmp_path: Path, **overrides: str) -> list[str]: result, temp = _run_step(repo, tmp_path, **overrides) assert result.returncode == 0, result.stdout + result.stderr - line = next(ln for ln in (temp / "out").read_text(encoding="utf-8").splitlines() if ln.startswith("tasks=")) + line = next( + ln + for ln in (temp / "out").read_text(encoding="utf-8").splitlines() + if ln.startswith("tasks=") + ) return json.loads(line.removeprefix("tasks=")) @@ -140,7 +173,9 @@ def test_a_harness_file_in_the_eval_dir_adds_the_smoke_set_to_the_changed_task(r assert _select(repo, tmp_path) == ["my task", "t1"] -@pytest.mark.parametrize("manifest", [f"{EVAL_DIR}/repo-eval.yaml", f"./{EVAL_DIR}/../repo-eval.yaml"]) +@pytest.mark.parametrize( + "manifest", [f"{EVAL_DIR}/repo-eval.yaml", f"./{EVAL_DIR}/../repo-eval.yaml"] +) def test_the_manifest_is_compared_normalised(repo, tmp_path, manifest): if manifest.startswith("./"): _write(repo, "repo-eval.yaml") From 71dda1d2ea7fdd8e62331487955cd1bfd73d2e69 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Tue, 20 Oct 2026 21:41:23 -0700 Subject: [PATCH 15/20] ci(repo-eval): record a timeout verdict when the job is cancelled; widen self-check paths Signed-off-by: Nikolay Petrov --- .github/workflows/repo-eval-self.yml | 4 ++++ .github/workflows/repo-eval.yml | 23 +++++++++++++++---- ...0000_napetrov_repo-eval-timeout-verdict.md | 7 ++++++ 3 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 dokimi/changelog.d/20261021_100000_napetrov_repo-eval-timeout-verdict.md diff --git a/.github/workflows/repo-eval-self.yml b/.github/workflows/repo-eval-self.yml index 21bf4979..3c27f477 100644 --- a/.github/workflows/repo-eval-self.yml +++ b/.github/workflows/repo-eval-self.yml @@ -17,6 +17,10 @@ on: paths: - '.agent-eval/**' - 'dokimi/dokimi/repos/**' + - 'dokimi/dokimi/commands/repo*.py' + - 'dokimi/dokimi/schemas/repo_*.json' + - 'dokimi/requirements.lock.txt' + - 'dokimi/pyproject.toml' - '.github/workflows/repo-eval.yml' - '.github/workflows/repo-eval-self.yml' workflow_dispatch: diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index d5c26e00..384cc469 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -376,16 +376,29 @@ jobs: warm=() if [ "$WARM" = true ]; then warm=(--warm); fi dokimi repo tasks check . --manifest "$MANIFEST" --task "$TASK" --out "$RUNNER_TEMP/repo-check" "${warm[@]}" + # always(): a hung oracle ends in the job timeout, which cancels the job; a + # cancelled job still runs these steps, so the run leaves a verdict. - name: Summary - if: ${{ !cancelled() }} + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + TASK: ${{ matrix.task }} run: | python - <<'PY' >> "$GITHUB_STEP_SUMMARY" import json, os path = os.path.join(os.environ["RUNNER_TEMP"], "repo-check", "check", "check.json") if not os.path.isfile(path): - print("### repo tasks check", "", "No check.json: the check stopped before grading; see the job log.", - sep="\n") - raise SystemExit + if os.environ["JOB_STATUS"] == "cancelled": + # the job timeout (or a cancel) stopped the check before it wrote its verdict + problem = "the job was cancelled (timeout-minutes?) before the check finished" + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump({"tasks": [{"task": os.environ["TASK"], "verdict": "timeout", + "problems": [problem]}]}, f) + else: + print("### repo tasks check", "", "No check.json: the check stopped before grading; see the job log.", + sep="\n") + raise SystemExit for r in json.load(open(path, encoding="utf-8"))["tasks"]: print(f"### repo tasks check `{r['task']}`: {r['verdict'].upper()}", "", sep="\n") for kind in ("null", "oracle"): @@ -396,7 +409,7 @@ jobs: print(f"- problem: {p}") PY - name: Upload the check results - if: ${{ !cancelled() }} + if: ${{ always() }} uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: repo-eval-check-${{ needs.validate.outputs.id }}-${{ matrix.task }} diff --git a/dokimi/changelog.d/20261021_100000_napetrov_repo-eval-timeout-verdict.md b/dokimi/changelog.d/20261021_100000_napetrov_repo-eval-timeout-verdict.md new file mode 100644 index 00000000..2b033cbc --- /dev/null +++ b/dokimi/changelog.d/20261021_100000_napetrov_repo-eval-timeout-verdict.md @@ -0,0 +1,7 @@ +### Fixed + +- **repo-eval: a hung oracle leaves a verdict** — the Summary and Upload steps of + `repo-eval.yml` run with `always()`, and a job cancelled by `timeout-minutes` before + `check.json` exists records a `timeout` verdict for the task. The self-check + (`repo-eval-self.yml`) now also runs on changes to `commands/repo*.py`, + `schemas/repo_*.json`, `requirements.lock.txt` and `pyproject.toml`. From ad1d9b488b435043fabcdaf8ec2bb65e325d839a Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 21 Oct 2026 00:58:11 -0700 Subject: [PATCH 16/20] ci(repo-eval-self): every path that triggers the self-check selects the smoke set The paths added in 71dda1d started the workflow, but `changed` found no task for them, so a change to commands/repo*.py, the repo_ schemas, the lock or pyproject.toml ran no tasks check. smoke-paths now lists them. Signed-off-by: Alexander Rybkin --- .github/workflows/repo-eval-self.yml | 5 +++- dokimi/tests/test_repo_eval_task_selection.py | 24 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/.github/workflows/repo-eval-self.yml b/.github/workflows/repo-eval-self.yml index 3c27f477..ed6d75a0 100644 --- a/.github/workflows/repo-eval-self.yml +++ b/.github/workflows/repo-eval-self.yml @@ -48,7 +48,10 @@ jobs: manifest: .agent-eval/repo-eval.yaml tasks: ${{ inputs.tasks || 'changed' }} smoke-tasks: fix-degrade-pre review-pr-conventions - smoke-paths: dokimi/dokimi/repos/ .github/workflows/repo-eval.yml + # Every `paths:` entry above outside .agent-eval/, as a prefix. + smoke-paths: >- + dokimi/dokimi/repos/ dokimi/dokimi/commands/repo dokimi/dokimi/schemas/repo_ + dokimi/requirements.lock.txt dokimi/pyproject.toml .github/workflows/repo-eval.yml max-parallel: 4 # intel-ubuntu-latest: see ci.yml. runs-on: ${{ github.repository_owner == 'intel' && 'intel-ubuntu-latest' || 'ubuntu-latest' }} diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index 789324f1..eeb84fd6 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -261,6 +261,30 @@ def test_a_change_to_the_calling_workflow_runs_the_smoke_set(repo, tmp_path): assert _select(repo, tmp_path) == ["t1"] +def test_every_path_that_triggers_the_self_check_runs_the_smoke_set(repo, tmp_path): + self_check = WORKFLOW.with_name("repo-eval-self.yml") + doc = yaml.safe_load(self_check.read_text(encoding="utf-8")) + paths = doc[True]["pull_request"]["paths"] # YAML 1.1 reads the `on` key as True + caller = f"o/r/.github/workflows/{self_check.name}@refs/pull/1/merge" + # a file each filter matches: `**` as a nested file, `*` as nothing and as a suffix + files = sorted( + { + p.replace("**", "x/y.py").replace("*", star) + for p in paths + if not p.startswith(f"{EVAL_DIR}/") + for star in ("", "_x") + } + ) + missed = [] + for f in files: + _write(repo, f, "changed\n") + smoke = doc["jobs"]["self"]["with"]["smoke-paths"] + if _select(repo, tmp_path, SMOKE_PATHS=smoke, WORKFLOW_REF=caller) != ["t1"]: + missed.append(f) + _git(repo, "update-ref", "refs/remotes/origin/main", "HEAD") + assert files and not missed + + @pytest.mark.parametrize("smoke_paths", ["./vendor/", "vendor/,other/", "other/ vendor/"]) def test_smoke_paths_are_normalised_and_split_like_ids(repo, tmp_path, smoke_paths): _write(repo, "vendor/grader/lib.py", "changed\n") From c6718f834ef49679dca6e3c527d3e2a9f383f327 Mon Sep 17 00:00:00 2001 From: Alexander Rybkin Date: Wed, 21 Oct 2026 00:58:24 -0700 Subject: [PATCH 17/20] test(repo-eval): run the Summary step for a cancelled and a failed check The timeout verdict from 71dda1d had no test: a cancelled job with no check.json must leave one with verdict timeout, uploaded by an always() step, and a failed one must not invent a verdict. Signed-off-by: Alexander Rybkin --- dokimi/tests/test_repo_eval_task_selection.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index eeb84fd6..6472a759 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -312,6 +312,35 @@ def test_an_unreadable_previous_tip_checks_every_task(repo, tmp_path): assert _select(repo, tmp_path, EVENT="push", BEFORE="1" * 40) == ["my task", "t1"] +def _summary(tmp_path: Path, job_status: str) -> tuple[str, Path]: + """Run the tasks-check Summary step with no check.json, as after a killed check.""" + run = next(s for s in _steps() if s.get("name") == "Summary")["run"] + match = re.search(r"python - <<'PY'[^\n]*\n(.*)\nPY\s*$", run, re.S) + assert match, "the Summary step is no longer a python heredoc" + temp = tmp_path / job_status + temp.mkdir() + env = {**os.environ, "RUNNER_TEMP": str(temp), "JOB_STATUS": job_status, "TASK": "t1"} + result = subprocess.run( + [sys.executable, "-c", match.group(1)], env=env, capture_output=True, text=True, check=False + ) + assert result.returncode == 0, result.stderr + return result.stdout, temp / "repo-check" / "check" / "check.json" + + +def test_a_job_cancelled_before_the_verdict_records_a_timeout(tmp_path): + summary, check = _summary(tmp_path, "cancelled") + assert json.loads(check.read_text(encoding="utf-8"))["tasks"][0]["verdict"] == "timeout" + assert "`t1`: TIMEOUT" in summary + # a timeout cancels the job, and only always() runs a step in a cancelled job + steps = {s.get("name"): s for s in _steps()} + assert steps["Summary"]["if"] == steps["Upload the check results"]["if"] == "${{ always() }}" + + +def test_a_failed_check_without_a_verdict_invents_none(tmp_path): + summary, check = _summary(tmp_path, "failure") + assert not check.exists() and "No check.json" in summary + + def test_the_check_results_artifact_keeps_the_stderr_files(): upload = next(s for s in _steps() if "Upload the check results" in str(s.get("name", ""))) paths = [ln.strip() for ln in str(upload["with"]["path"]).splitlines()] From 21fbdcdbadab75766c369e8b779ac8761f1d0e91 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 21 Oct 2026 14:14:14 +0000 Subject: [PATCH 18/20] ci(repo-eval): smoke-paths is space- or comma-separated Signed-off-by: Anton Volkov --- .github/workflows/repo-eval.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/repo-eval.yml b/.github/workflows/repo-eval.yml index 384cc469..61972036 100644 --- a/.github/workflows/repo-eval.yml +++ b/.github/workflows/repo-eval.yml @@ -57,7 +57,7 @@ on: default: '' smoke-paths: description: >- - More path prefixes (space-separated) whose change selects `smoke-tasks`, + More path prefixes (space- or comma-separated) whose change selects `smoke-tasks`, for example a vendored grader library required: false type: string From 0466d452dfec571a098f8ba11921fc70f60bf4b8 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 21 Oct 2026 14:14:16 +0000 Subject: [PATCH 19/20] docs(repo-eval): document the timeout verdict of a cancelled check Signed-off-by: Anton Volkov --- dokimi/docs/repo-evaluation-contract.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dokimi/docs/repo-evaluation-contract.md b/dokimi/docs/repo-evaluation-contract.md index 34c36e89..e4af28c1 100644 --- a/dokimi/docs/repo-evaluation-contract.md +++ b/dokimi/docs/repo-evaluation-contract.md @@ -878,7 +878,9 @@ that runs the no-LLM checks on the caller's checkout and needs no secret. (report only: the claim counts and failing claims go to the job summary, the `repo_readiness.v1` report to an artifact); `tasks-check` runs `repo tasks check` for each selected task in a job of its own (docker; the image is built from the - manifest's Dockerfile), and uploads `check.json` and the grade logs. + manifest's Dockerfile), and uploads `check.json` and the grade logs. If the job is + cancelled by its `timeout-minutes` before the check finishes, the Summary step + writes a `check.json` with the verdict `timeout` for the task. - `tasks: changed` (the default, also for an empty value) checks, on a pull request, the tasks whose directories it touches, plus `smoke-tasks` (empty: every task) when it changes the eval directory outside its tasks (Markdown there is excepted, unless it is the manifest, From 965ce64b176590c47134c46b7088355102dcc393 Mon Sep 17 00:00:00 2001 From: Anton Volkov Date: Wed, 21 Oct 2026 14:14:43 +0000 Subject: [PATCH 20/20] test(repo-eval): cover an empty base sha and a force-push range Signed-off-by: Anton Volkov --- dokimi/tests/test_repo_eval_task_selection.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dokimi/tests/test_repo_eval_task_selection.py b/dokimi/tests/test_repo_eval_task_selection.py index 6472a759..eff57aa4 100644 --- a/dokimi/tests/test_repo_eval_task_selection.py +++ b/dokimi/tests/test_repo_eval_task_selection.py @@ -241,6 +241,22 @@ def test_a_push_without_a_previous_tip_checks_every_task(repo, tmp_path): assert _select(repo, tmp_path, EVENT="push", BEFORE="0" * 40) == ["my task", "t1"] +def test_a_push_after_a_force_push_diffs_the_two_tips_directly(repo, tmp_path): + # the old tip is not an ancestor of the new HEAD: its own change counts too + _git(repo, "checkout", "-q", "-b", "old") + _write(repo, f"{TASKS}/t1/grader.py", "changed\n") + _git(repo, "commit", "-q", "-a", "-m", "old tip") + old = _git(repo, "rev-parse", "HEAD").strip() + _git(repo, "checkout", "-q", "main") + _write(repo, f"{TASKS}/my task/grader.py", "changed\n") + assert _select(repo, tmp_path, EVENT="push", BEFORE=old) == ["my task", "t1"] + + +def test_a_pull_request_without_a_base_sha_checks_every_task(repo, tmp_path): + _write(repo, f"{TASKS}/my task/grader.py", "changed\n") + assert _select(repo, tmp_path, BASE_SHA="") == ["my task", "t1"] + + def test_a_merge_commit_ignores_what_the_base_gained_after_the_pr_opened(repo, tmp_path): # base.sha stays at the fork point while the checkout merges into the base's current tip fork = _git(repo, "rev-parse", "HEAD").strip()