diff --git a/docs/TASK_DEFINITION_GUIDE.md b/docs/TASK_DEFINITION_GUIDE.md index c2de203..cfcf0ba 100644 --- a/docs/TASK_DEFINITION_GUIDE.md +++ b/docs/TASK_DEFINITION_GUIDE.md @@ -20,6 +20,7 @@ Complete reference for defining evaluation tasks in Coder Eval. - [Template Sources](#template-sources) - [Success Criteria](#success-criteria) - [Continuous Scoring](#continuous-scoring) + - [Glob patterns in path](#glob-patterns-in-path) - [file_exists](#file_exists) - [file_contains](#file_contains) - [file_check](#file_check) @@ -558,6 +559,26 @@ score mattered. **Weighted score:** `weighted_score = sum(score * weight) / sum(weight)` — calculated regardless for quality assessment. +### Glob patterns in `path` + +Every path-based criterion (`file_exists`, `file_contains`, `file_matches_regex`, `file_check`, `json_check`, `import_check`) accepts a glob in `path`. Use one when the prompt does not pin where the file lands — a scaffolding tool that creates a wrapper directory the agent names itself, for example. + +```yaml +- type: "file_contains" + path: "**/*.flow" # matches any depth under the sandbox root + includes: ['"core.logic.decision"'] + description: "flow wires a Decision node" +``` + +Rules: + +- A `path` with no `*`, `?`, or `[` behaves exactly as before. +- Matches are sorted, and directories are skipped. +- `file_exists` passes when the glob matches **at least one** file. +- Content checks require the glob to match **exactly one** file. An ambiguous glob scores 0.0 and reports every match rather than silently grading one of them — narrow the pattern. + +Prefer a glob over a hardcoded path whose leading directory the task prompt never specifies: a correct artifact in an unexpected directory otherwise scores 0.0 on the path alone. + ### `file_exists` Checks if a file exists. **Binary scoring.** diff --git a/src/coder_eval/models/criteria.py b/src/coder_eval/models/criteria.py index 2605e1c..e4ec07b 100644 --- a/src/coder_eval/models/criteria.py +++ b/src/coder_eval/models/criteria.py @@ -194,7 +194,9 @@ class FileExistsCriterion(BaseSuccessCriterion): """ type: Literal["file_exists"] = "file_exists" - path: str = Field(description="Path to the file that must exist") + path: str = Field( + description="Path to the file that must exist; a glob pattern passes when it matches at least one file" + ) class FileContainsCriterion(BaseSuccessCriterion): @@ -204,7 +206,7 @@ class FileContainsCriterion(BaseSuccessCriterion): """ type: Literal["file_contains"] = "file_contains" - path: str = Field(description="Path to the file to check") + path: str = Field(description="Path to the file to check; may be a glob matching exactly one file") includes: list[str] = Field(description="List of strings that must be present in the file") excludes: list[str] | None = Field(default=None, description="List of strings that must NOT be present in the file") @@ -277,7 +279,7 @@ class FileMatchesRegexCriterion(BaseSuccessCriterion): """ type: Literal["file_matches_regex"] = "file_matches_regex" - path: str = Field(description="Path to the file to check") + path: str = Field(description="Path to the file to check; may be a glob matching exactly one file") pattern: str = Field(description="Regex pattern that must match somewhere in the file") must_match: bool = Field(default=True, description="If True, pattern must match; if False, pattern must NOT match") flags: int = Field(default=0, description="Regex flags (e.g., re.IGNORECASE=2, re.MULTILINE=8, re.DOTALL=16)") @@ -408,7 +410,9 @@ class FileCheckCriterion(BaseSuccessCriterion): """ type: Literal["file_check"] = "file_check" - path: str = Field(description="Path to the file to check (relative to sandbox root)") + path: str = Field( + description="Path to the file to check (relative to sandbox root); may be a glob matching exactly one file" + ) includes: list[str] = Field(default_factory=list, description="Strings that must be present in the file") excludes: list[str] = Field(default_factory=list, description="Strings that must NOT be present in the file") patterns: list[RegexPattern] = Field( diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 0721706..4ee5ce9 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -967,38 +967,71 @@ def run_command(self, command: str, timeout: float | int | None = None) -> tuple # needs filesystem access beyond the sandbox root (e.g., reading installed packages, # system headers). Path traversal protection is handled at the agent permission level. + def resolve_files(self, path: str) -> list[Path]: + """Resolve a criterion ``path`` to the sandbox files it addresses. + + A plain path resolves to itself. A path containing a glob + metacharacter (``*``, ``?``, ``[``) is expanded against the sandbox + root, so a criterion can address a file whose exact location the task + prompt does not pin — e.g. ``**/*.flow`` matches a scaffolded wrapper + directory the agent was free to name. Matches are sorted so grading is + deterministic, and directories are dropped so a glob cannot resolve to + something unreadable. + + Args: + path: Relative path or glob pattern + + Returns: + Sorted matching files; empty when nothing matches + """ + if not self.sandbox_dir: + return [] + + if not any(c in path for c in "*?["): + candidate = self.sandbox_dir / path + return [candidate] if candidate.exists() else [] + + return sorted(p for p in self.sandbox_dir.glob(path) if p.is_file()) + def get_file_content(self, path: str) -> str: """Read the content of a file in the sandbox. Args: - path: Relative path to the file + path: Relative path to the file, or a glob pattern matching exactly + one file Returns: File content as string Raises: RuntimeError: If sandbox is not set up - FileNotFoundError: If file doesn't exist + FileNotFoundError: If nothing matches ``path`` + ValueError: If a glob matches more than one file """ if not self.sandbox_dir: raise RuntimeError("Sandbox not set up") - file_path = self.sandbox_dir / path - return file_path.read_text(encoding="utf-8") + matches = self.resolve_files(path) + if not matches: + raise FileNotFoundError(f"No file matches '{path}' in the sandbox") + if len(matches) > 1: + listed = ", ".join(str(p.relative_to(self.sandbox_dir)) for p in matches) + raise ValueError( + f"Pattern '{path}' matches {len(matches)} files — refusing to guess which to grade: {listed}" + ) + + return matches[0].read_text(encoding="utf-8") def file_exists(self, path: str) -> bool: """Check if a file exists in the sandbox. Args: - path: Relative path to the file + path: Relative path to the file, or a glob pattern Returns: - True if file exists, False otherwise + True if at least one file matches, False otherwise """ - if not self.sandbox_dir: - return False - - return (self.sandbox_dir / path).exists() + return bool(self.resolve_files(path)) def list_files(self, path: str = ".") -> list[str]: """List files in a directory within the sandbox. diff --git a/tests/test_glob_paths_in_file_criteria.py b/tests/test_glob_paths_in_file_criteria.py new file mode 100644 index 0000000..c933763 --- /dev/null +++ b/tests/test_glob_paths_in_file_criteria.py @@ -0,0 +1,141 @@ +"""Tests for glob patterns in criterion ``path`` fields. + +A criterion can address a file whose exact location the task prompt does not +pin — e.g. a scaffolded wrapper directory the agent was free to name. Path +resolution lives in ``Sandbox.resolve_files``, so every path-based criterion +type inherits the behavior. +""" + +import pytest + +from coder_eval.evaluation.checker import SuccessChecker +from coder_eval.models import ( + FileCheckCriterion, + FileContainsCriterion, + FileExistsCriterion, + SandboxConfig, +) +from coder_eval.sandbox import Sandbox + + +FLOW_BODY = '{"nodes": [{"type": "uipath.human-in-the-loop.quick-form"}]}' + + +@pytest.fixture +def sandbox(): + sb = Sandbox(SandboxConfig(driver="tempdir", python=None), task_id="test_glob_paths") + sb.setup() + yield sb + sb.cleanup(preserve=False) + + +def _write(sandbox, relpath: str, body: str = FLOW_BODY): + target = sandbox.sandbox_dir / relpath + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(body, encoding="utf-8") + return target + + +class TestResolveFiles: + def test_plain_path_unchanged(self, sandbox): + _write(sandbox, "app.py", "print('hi')") + + assert sandbox.resolve_files("app.py") == [sandbox.sandbox_dir / "app.py"] + assert sandbox.file_exists("app.py") is True + + def test_plain_path_missing(self, sandbox): + assert sandbox.resolve_files("nope.py") == [] + assert sandbox.file_exists("nope.py") is False + + def test_glob_finds_file_under_unpinned_directory(self, sandbox): + _write(sandbox, "InvoiceApprovalSolution/InvoiceApproval/InvoiceApproval.flow") + + assert sandbox.file_exists("**/*.flow") is True + assert sandbox.get_file_content("**/*.flow") == FLOW_BODY + assert sandbox.file_exists("InvoiceApproval/InvoiceApproval/InvoiceApproval.flow") is False + + def test_glob_no_match(self, sandbox): + assert sandbox.resolve_files("**/*.flow") == [] + assert sandbox.file_exists("**/*.flow") is False + with pytest.raises(FileNotFoundError, match="No file matches"): + sandbox.get_file_content("**/*.flow") + + def test_glob_skips_directories(self, sandbox): + (sandbox.sandbox_dir / "build.flow").mkdir() + _write(sandbox, "proj/real.flow") + + assert sandbox.resolve_files("**/*.flow") == [sandbox.sandbox_dir / "proj" / "real.flow"] + + def test_ambiguous_glob_refuses_to_guess(self, sandbox): + _write(sandbox, "a/one.flow") + _write(sandbox, "b/two.flow") + + assert sandbox.file_exists("**/*.flow") is True + with pytest.raises(ValueError, match="refusing to guess"): + sandbox.get_file_content("**/*.flow") + + def test_matches_are_sorted(self, sandbox): + _write(sandbox, "z/last.flow") + _write(sandbox, "a/first.flow") + + assert sandbox.resolve_files("**/*.flow") == sorted(sandbox.resolve_files("**/*.flow")) + + def test_no_sandbox_dir(self): + sb = Sandbox(SandboxConfig(driver="tempdir", python=None), task_id="test_glob_unset") + + assert sb.resolve_files("**/*.flow") == [] + assert sb.file_exists("**/*.flow") is False + with pytest.raises(RuntimeError, match="not set up"): + sb.get_file_content("**/*.flow") + + +class TestGlobThroughCriteria: + def test_file_exists_criterion(self, sandbox): + _write(sandbox, "WrapperSolution/Proj/Proj.flow") + + result = SuccessChecker(sandbox).check(FileExistsCriterion(description="flow exists", path="**/*.flow")) + + assert result.score == 1.0 + + def test_file_contains_criterion(self, sandbox): + _write(sandbox, "WrapperSolution/Proj/Proj.flow") + + result = SuccessChecker(sandbox).check( + FileContainsCriterion( + description="has HITL node", + path="**/*.flow", + includes=['"uipath.human-in-the-loop.quick-form"'], + ) + ) + + assert result.score == 1.0 + + def test_file_check_criterion(self, sandbox): + _write(sandbox, "WrapperSolution/Proj/Proj.flow") + + result = SuccessChecker(sandbox).check( + FileCheckCriterion( + description="has HITL node, no manual trigger", + path="**/*.flow", + includes=['"uipath.human-in-the-loop.quick-form"'], + excludes=['"core.trigger.manual"'], + ) + ) + + assert result.score == 1.0 + + def test_ambiguous_glob_scores_zero_with_message(self, sandbox): + _write(sandbox, "a/one.flow") + _write(sandbox, "b/two.flow") + + result = SuccessChecker(sandbox).check( + FileContainsCriterion(description="ambiguous", path="**/*.flow", includes=["nodes"]) + ) + + assert result.score == 0.0 + assert "refusing to guess" in (result.error or "") + + def test_missing_glob_scores_zero(self, sandbox): + result = SuccessChecker(sandbox).check(FileExistsCriterion(description="no flow", path="**/*.flow")) + + assert result.score == 0.0