refactor(conformance): the judge is a codeupipe pipeline

conformance/README.md

@@ -4,6 +4,14 @@ The rules are in **[API.md](API.md)**, the CUP API contract. This suite
 checks them.
 
 One set of language-neutral cases, a runner, and **one judge** (`run.py`).
+The judge is itself a codeupipe pipeline, in [judging/](judging/): one step
+reads the cases, one runs each target, one compares, and one writes the
+matrix. Each step is tested alone in `tests/conformance/test_judge_pipeline.py`.
+
+The Python runner is built the same way, in
+[runners/python/case_runner/](runners/python/case_runner/): one pipeline plays
+a case and one times a scenario. Its steps are tested alone in
+`tests/conformance/test_case_runner.py`.
 The dynamic run and each compiled form are held to exactly the same
 comparison.
 

conformance/judging/__init__.py

@@ -0,0 +1,5 @@
+"""The judge of the conformance suite, as a codeupipe pipeline (see ``judge_pipeline.py``).
+
+A runner executes the cases and reports what happened. Only the judge
+decides pass or fail, so every target is held to the same comparison.
+"""

conformance/judging/default_targets_filter.py

@@ -0,0 +1,30 @@
+"""DefaultTargetsFilter: each target that this machine can run."""
+
+from codeupipe import Payload
+
+
+class DefaultTargetsFilter:
+    """Filter (sync).
+
+    Input keys: ``targets`` ({name: how to start its runner}), ``needs``
+    ({name: paths that must exist for the target}).
+    Output keys: ``names``. A Python target always runs. A TypeScript target
+    needs a Node that runs ``.ts`` files, and the paths of ``needs``.
+
+    ``node_runs_typescript`` is a function with no arguments that says if
+    the machine has such a Node.
+    """
+
+    def __init__(self, node_runs_typescript):
+        self.node_runs_typescript = node_runs_typescript
+
+    def call(self, payload: Payload) -> Payload:
+        return payload.insert("names", self.choose(payload.get("targets"), payload.get("needs", {})))
+
+    def choose(self, targets, needs) -> list:
+        return [name for name in targets if self.can_run(name, needs)]
+
+    def can_run(self, name, needs) -> bool:
+        if not name.startswith("typescript:"):
+            return True
+        return self.node_runs_typescript() and all(path.exists() for path in needs.get(name, []))

conformance/judging/format_matrix_filter.py

@@ -0,0 +1,32 @@
+"""FormatMatrixFilter: the matrix and the counts as Markdown."""
+
+from codeupipe import Payload
+
+_MARK = {"pass": "✅", "partial": "✅*", "FAIL": "❌", "unsupported": "➖", "—": "—"}
+
+
+class FormatMatrixFilter:
+    """Filter (sync).
+
+    Input keys: ``cases``, ``names``, ``matrix``, ``report``.
+    Output keys: ``markdown``.
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        cases, names = payload.get("cases"), payload.get("names")
+        matrix, report = payload.get("matrix"), payload.get("report")
+        lines = ["| case | " + " | ".join(names) + " |", "|---|" + "---|" * len(names)]
+        for case in cases:
+            lines.append(f"| `{case['id']}` | " + " | ".join(_MARK[matrix[case['id']][n]] for n in names) + " |")
+        lines.append("")
+        for name in names:
+            target = report["targets"][name]
+            if "error" in target:
+                lines.append(f"- **{name}**: not run: {target['error']}")
+            else:
+                lines.append(f"- **{name}**: {target['pass']} pass, {target['partial']} pass*, {target['FAIL']} fail, "
+                             f"{target['unsupported']} unsupported")
+        lines.append("")
+        lines.append("✅* passed everything the target can observe; it does not expose State or Hooks "
+                     "(the fast mode). ➖ the target cannot express the case.")
+        return payload.insert("markdown", "\n".join(lines) + "\n")

conformance/judging/judge_pipeline.py

@@ -0,0 +1,36 @@
+"""JudgePipeline: run the conformance cases on each target and judge them one way."""
+
+from codeupipe import Pipeline
+
+from .default_targets_filter import DefaultTargetsFilter
+from .format_matrix_filter import FormatMatrixFilter
+from .judge_results_filter import JudgeResultsFilter
+from .load_cases_filter import LoadCasesFilter
+from .no_targets_named_predicate import NoTargetsNamedPredicate
+from .run_targets_filter import RunTargetsFilter
+from .summarize_filter import SummarizeFilter
+from .write_results_tap import WriteResultsTap
+
+
+class JudgePipeline(Pipeline):
+    """The judge.
+
+    Input keys: ``cases_dir``, ``results_dir``, ``targets`` ({name: how to
+    start its runner}), ``needs`` ({name: paths that the target needs}), and
+    ``names`` (the targets that the caller named; empty for each target that
+    this machine can run).
+    Output keys: ``report``, ``matrix``, ``markdown``, ``exit_code``.
+
+    ``launcher`` starts the runner of a target. ``node_runs_typescript`` says
+    if the machine can run the TypeScript targets.
+    """
+
+    def __init__(self, launcher, node_runs_typescript):
+        super().__init__()
+        self.add_filter(LoadCasesFilter())
+        self.add_valve(NoTargetsNamedPredicate(), DefaultTargetsFilter(node_runs_typescript))
+        self.add_filter(RunTargetsFilter(launcher))
+        self.add_filter(JudgeResultsFilter())
+        self.add_filter(SummarizeFilter())
+        self.add_filter(FormatMatrixFilter())
+        self.add_tap(WriteResultsTap())

conformance/judging/judge_results_filter.py

@@ -0,0 +1,93 @@
+"""JudgeResultsFilter: compare what each target reported with what each case expects.
+
+This is the one place that decides pass or fail. A verdict is a status and
+its problems:
+
+    pass         every expectation matched
+    partial      every expectation that the target can show matched, but it
+                 does not show some of them at all (the fast compiled forms
+                 show no State and no Hooks)
+    FAIL         something that the target shows is different
+    unsupported  the target cannot express the case. Never counted as a pass
+"""
+
+from codeupipe import Payload
+
+# What a target may not show at all. A case that expects one of these, on a target that does not report it, is "partial".
+_NOT_ALWAYS_SHOWN = ("state", "hooks", "inner_states", "pipeline_state")
+
+
+class JudgeResultsFilter:
+    """Filter (sync).
+
+    Input keys: ``cases``, ``results`` ({name: {case id: result}}).
+    Output keys: ``verdicts`` ({name: {case id: (status, problems)}}).
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        cases = payload.get("cases")
+        verdicts = {name: {case["id"]: self.judge(case, found.get(case["id"])) for case in cases}
+                    for name, found in payload.get("results").items()}
+        return payload.insert("verdicts", verdicts)
+
+    def judge(self, case, result):
+        """The verdict for one case on one target: ``(status, problems)``."""
+        if result is None:
+            return "FAIL", ["no result from runner"]
+        if "unsupported" in result:
+            return "unsupported", [result["unsupported"]]
+        expect = case["expect"]
+        problems = self.output_problems(expect, result)
+
+        if self.norm(result.get("input_after")) != self.norm(case["input"]):
+            problems.append(f"input was mutated: {result.get('input_after')!r}")
+
+        if "taps" in expect and self.norm(result.get("taps")) != self.norm(expect["taps"]):
+            problems.append(f"taps {result.get('taps')!r} != {expect['taps']!r}")
+
+        if "state" in expect and "state" in result:
+            for key, want in expect["state"].items():
+                got = result["state"].get(key)
+                if got != want:
+                    problems.append(f"state.{key} {got!r} != {want!r}")
+
+        for key in ("inner_states", "pipeline_state"):
+            if key in expect and key in result and result[key] != expect[key]:
+                problems.append(f"{key} {result[key]!r} != {expect[key]!r}")
+
+        if "hooks" in expect and "hooks" in result and result["hooks"] != expect["hooks"]:
+            problems.append(f"hooks {result['hooks']!r} != {expect['hooks']!r}")
+
+        if problems:
+            return "FAIL", problems
+        unchecked = [key for key in _NOT_ALWAYS_SHOWN if key in expect and key not in result]
+        if unchecked:
+            return "partial", [f"target cannot observe {', '.join(unchecked)}"]
+        return "pass", []
+
+    def output_problems(self, expect, result) -> list:
+        """What is wrong with the output or the error of a run."""
+        problems = []
+        if "output" in expect:
+            if "error" in result:
+                problems.append(f"expected output, got error {result['error']!r}")
+            elif self.norm(result.get("output")) != self.norm(expect["output"]):
+                problems.append(f"output {result.get('output')!r} != {expect['output']!r}")
+        if "error" in expect:
+            if "error" not in result:
+                problems.append(f"expected an error, got output {result.get('output')!r}")
+            elif expect["error"].get("message_contains", "") not in result["error"]:
+                problems.append(f"error {result['error']!r} lacks {expect['error']['message_contains']!r}")
+        return problems
+
+    def norm(self, value):
+        """JSON equality: 1 == 1.0, but true is not 1."""
+        if isinstance(value, bool) or value is None or isinstance(value, str):
+            return value
+        if isinstance(value, (int, float)):
+            return float(value)
+        if isinstance(value, list):
+            return [self.norm(v) for v in value]
+        if isinstance(value, dict):
+            return {k: self.norm(v) for k, v in value.items()}
+        return value

conformance/judging/load_cases_filter.py

@@ -0,0 +1,28 @@
+"""LoadCasesFilter: read the conformance cases from disk."""
+
+import json
+import pathlib
+
+from codeupipe import Payload
+
+
+class LoadCasesFilter:
+    """Filter (sync).
+
+    Input keys: ``cases_dir`` (the directory of the ``*.json`` case files).
+    Output keys: ``cases`` (each case, in the order of the file names). Two
+    cases with the same id stop the run.
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        return payload.insert("cases", self.load(payload.get("cases_dir")))
+
+    def load(self, cases_dir) -> list:
+        cases = []
+        for path in sorted(pathlib.Path(cases_dir).glob("*.json")):
+            cases.extend(json.loads(path.read_text()))
+        ids = [case["id"] for case in cases]
+        duplicates = {i for i in ids if ids.count(i) > 1}
+        if duplicates:
+            raise SystemExit(f"duplicate case ids: {sorted(duplicates)}")
+        return cases

conformance/judging/no_targets_named_predicate.py

@@ -0,0 +1,10 @@
+"""NoTargetsNamedPredicate: did the caller name no target?"""
+
+from codeupipe import Payload
+
+
+class NoTargetsNamedPredicate:
+    """True when ``names`` is empty: the judge then takes each target that this machine can run."""
+
+    def test(self, payload: Payload) -> bool:
+        return not payload.get("names")

conformance/judging/node_probe.py

@@ -0,0 +1,20 @@
+"""NodeProbe: what this machine has for the TypeScript targets."""
+
+import shutil
+import subprocess
+
+
+# cup: allow-plain the edge to the machine: it asks which Node is installed
+class NodeProbe:
+    """Says if there is a Node that runs a ``.ts`` file as it is (22.18 or later)."""
+
+    def runs_typescript(self) -> bool:
+        node = shutil.which("node")
+        if node is None:
+            return False
+        try:
+            version = subprocess.run([node, "--version"], capture_output=True, text=True, timeout=30).stdout
+            major, minor = (int(part) for part in version.strip().lstrip("v").split(".")[:2])
+        except (OSError, ValueError, subprocess.SubprocessError):
+            return False
+        return (major, minor) >= (22, 18)

conformance/judging/process_launcher.py

@@ -0,0 +1,29 @@
+"""ProcessLauncher: starts the runner of one target and reads what it reports."""
+
+import json
+import os
+import subprocess
+
+
+# cup: allow-plain the edge to the runners: it starts a process, gives it the cases and reads its report
+class ProcessLauncher:
+    """Runs one target. ``spec`` has ``cmd``, ``env`` and ``cwd``.
+
+    Gives ``(results, None)``, with ``results`` as {case id: result}, or
+    ``(None, reason)`` when the runner did not end well or its output
+    cannot be read.
+    """
+
+    def run(self, spec, cases):
+        env = dict(os.environ)
+        env.update(spec["env"])
+        proc = subprocess.run(
+            spec["cmd"], input=json.dumps(cases), capture_output=True, text=True,
+            cwd=spec["cwd"], env=env, timeout=600,
+        )
+        if proc.returncode != 0:
+            return None, f"runner exited {proc.returncode}: {proc.stderr.strip()[-2000:]}"
+        try:
+            return {r["id"]: r for r in json.loads(proc.stdout)}, None
+        except (json.JSONDecodeError, KeyError) as error:
+            return None, f"unreadable runner output ({error}): {proc.stdout[:500]}"

conformance/judging/run_targets_filter.py

@@ -0,0 +1,29 @@
+"""RunTargetsFilter: run the cases on each target."""
+
+from codeupipe import Payload
+
+
+class RunTargetsFilter:
+    """Filter (sync).
+
+    Input keys: ``names`` (the targets to run, in order), ``targets``
+    ({name: how to start its runner}), ``cases``.
+    Output keys: ``results`` ({name: {case id: result}}) for each target that
+    ran, and ``target_errors`` ({name: reason}) for each one that did not.
+
+    ``launcher`` runs one target: ``launcher.run(spec, cases)``.
+    """
+
+    def __init__(self, launcher):
+        self.launcher = launcher
+
+    def call(self, payload: Payload) -> Payload:
+        targets, cases = payload.get("targets"), payload.get("cases")
+        results, errors = {}, {}
+        for name in payload.get("names"):
+            found, error = self.launcher.run(targets[name], cases)
+            if error:
+                errors[name] = error
+            else:
+                results[name] = found
+        return payload.insert("results", results).insert("target_errors", errors)

conformance/judging/summarize_filter.py

@@ -0,0 +1,37 @@
+"""SummarizeFilter: the counts for each target, the matrix, and the exit code."""
+
+from codeupipe import Payload
+
+
+class SummarizeFilter:
+    """Filter (sync).
+
+    Input keys: ``cases``, ``names``, ``verdicts``, ``target_errors``.
+    Output keys: ``report`` (the number of cases, and for each target its
+    counts, its failures and the cases that it cannot express, or the
+    reason that it did not run), ``matrix`` ({case id: {name: status}}) and
+    ``exit_code`` (1 when a case failed or a target did not run).
+    """
+
+    def call(self, payload: Payload) -> Payload:
+        cases, verdicts, errors = payload.get("cases"), payload.get("verdicts"), payload.get("target_errors")
+        report = {"cases": len(cases), "targets": {}}
+        matrix = {case["id"]: {} for case in cases}
+        for name in payload.get("names"):
+            if name in errors:
+                report["targets"][name] = {"error": errors[name]}
+                for case in cases:
+                    matrix[case["id"]][name] = "—"
+                continue
+            summary = {"pass": 0, "partial": 0, "FAIL": 0, "unsupported": 0, "failures": {}, "unsupported_cases": {}}
+            for case in cases:
+                status, problems = verdicts[name][case["id"]]
+                summary[status] += 1
+                matrix[case["id"]][name] = status
+                if status == "FAIL":
+                    summary["failures"][case["id"]] = problems
+                elif status == "unsupported":
+                    summary["unsupported_cases"][case["id"]] = problems[0]
+            report["targets"][name] = summary
+        failed = any(target.get("FAIL") or "error" in target for target in report["targets"].values())
+        return payload.insert("report", report).insert("matrix", matrix).insert("exit_code", 1 if failed else 0)

conformance/judging/write_results_tap.py

@@ -0,0 +1,20 @@
+"""WriteResultsTap: put the report and the matrix on disk."""
+
+import json
+import pathlib
+
+from codeupipe import Payload
+
+
+class WriteResultsTap:
+    """Tap (sync).
+
+    Reads ``results_dir``, ``report`` and ``markdown``, and writes
+    ``latest.json`` and ``latest.md`` in that directory.
+    """
+
+    def observe(self, payload: Payload) -> None:
+        directory = pathlib.Path(payload.get("results_dir"))
+        directory.mkdir(exist_ok=True)
+        (directory / "latest.json").write_text(json.dumps(payload.get("report"), indent=2))
+        (directory / "latest.md").write_text(payload.get("markdown"))