diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb8b865..5c34b7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,11 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: python scripts/check_action_pins.py + - name: Landscape table matches the fetched competitor data + # The table states its own method ("fetched via the GitHub API") + # but was typed by hand. In a sibling repo that drifted into + # publishing that a live competitor's repo was gone. + run: python scripts/generate_landscape.py --check - name: Docs link check run: python scripts/check_docs_links.py - name: Generate static web data diff --git a/README.md b/README.md index d65eed6..486edda 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,20 @@ These are independent projects: no shared library, no coupled releases, and each +## How this compares + +15 projects are tracked in [`docs/competitive-analysis.md`](docs/competitive-analysis.md), +fetched from the GitHub API on 2026-09-09 and committed to +[`data/competitor-meta.json`](data/competitor-meta.json). The table is generated from that +file rather than typed, so it cannot drift from the data it cites. + +They divide into three groups that are easy to confuse: **task suites** (SWE-bench, +tau-bench, OSWorld, WebArena) that define problems, **eval harnesses** (inspect_ai, +promptfoo, DeepEval) that run and grade them, and **tracing platforms** (Langfuse, Phoenix, +AgentOps) that record what happened. ToolTrace Bench spans the first two with a specific +constraint: the score comes from the execution trace and the final workspace, never from a +model's opinion of its own work. + ## Citation See [CITATION.cff](CITATION.cff), or: diff --git a/data/competitor-meta.json b/data/competitor-meta.json new file mode 100644 index 0000000..56f22c3 --- /dev/null +++ b/data/competitor-meta.json @@ -0,0 +1,171 @@ +{ + "fetched_utc": "2026-09-09T10:16:15+00:00", + "tool": "gh api (authenticated)", + "repos": { + "SWE-bench/SWE-bench": { + "repo": "SWE-bench/SWE-bench", + "license_spdx": "MIT", + "stars": 5807, + "pushed_at": "2026-09-02T01:51:01Z", + "archived": false, + "description": "SWE-bench: Can Language Models Resolve Real-world Github Issues?", + "latest_release": null + }, + "microsoft/SWE-bench-Live": { + "repo": "microsoft/SWE-bench-Live", + "license_spdx": "MIT", + "stars": 234, + "pushed_at": "2026-09-07T23:24:48Z", + "archived": false, + "description": "[NeurIPS 2025 D&B] \ud83d\ude80 SWE-bench Goes Live!", + "latest_release": { + "tag": "v1.0-multi-language-multi-os-benchmarking", + "published_at": "2026-03-08T11:23:25Z" + } + }, + "sierra-research/tau-bench": { + "repo": "sierra-research/tau-bench", + "license_spdx": "MIT", + "stars": 1427, + "pushed_at": "2026-03-18T17:36:06Z", + "archived": false, + "description": "Code and Data for Tau-Bench", + "latest_release": null + }, + "sierra-research/tau2-bench": { + "repo": "sierra-research/tau2-bench", + "license_spdx": "MIT", + "stars": 1991, + "pushed_at": "2026-09-07T23:46:37Z", + "archived": false, + "description": "\u03c4-Bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", + "latest_release": { + "tag": "v1.0.1", + "published_at": "2026-07-22T21:22:42Z" + } + }, + "THUDM/AgentBench": { + "repo": "THUDM/AgentBench", + "license_spdx": "Apache-2.0", + "stars": 3717, + "pushed_at": "2026-02-08T17:01:05Z", + "archived": false, + "description": "A Comprehensive Benchmark to Evaluate LLMs as Agents (ICLR'24)", + "latest_release": null + }, + "xlang-ai/OSWorld": { + "repo": "xlang-ai/OSWorld", + "license_spdx": "Apache-2.0", + "stars": 3134, + "pushed_at": "2026-08-30T09:41:51Z", + "archived": false, + "description": "[NeurIPS 2024] OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", + "latest_release": { + "tag": "v0.1.16", + "published_at": "2024-06-26T04:22:29Z" + } + }, + "web-arena-x/webarena": { + "repo": "web-arena-x/webarena", + "license_spdx": "Apache-2.0", + "stars": 1606, + "pushed_at": "2025-11-26T21:16:00Z", + "archived": false, + "description": "Code repo for \"WebArena: A Realistic Web Environment for Building Autonomous Agents\"", + "latest_release": { + "tag": "v0.2.0", + "published_at": "2023-10-21T04:35:19Z" + } + }, + "browser-use/browser-use": { + "repo": "browser-use/browser-use", + "license_spdx": "MIT", + "stars": 113853, + "pushed_at": "2026-09-07T19:55:36Z", + "archived": false, + "description": "Agents that use the browser.", + "latest_release": { + "tag": "0.13.10", + "published_at": "2026-09-04T03:28:53Z" + } + }, + "UKGovernmentBEIS/inspect_ai": { + "repo": "UKGovernmentBEIS/inspect_ai", + "license_spdx": "MIT", + "stars": 2734, + "pushed_at": "2026-09-09T09:02:44Z", + "archived": false, + "description": "Inspect: A framework for large language model evaluations", + "latest_release": null + }, + "promptfoo/promptfoo": { + "repo": "promptfoo/promptfoo", + "license_spdx": "MIT", + "stars": 24964, + "pushed_at": "2026-09-09T09:43:10Z", + "archived": false, + "description": "Test your prompts, agents, and RAGs. Red teaming/pentesting/vulnerability scanning for AI. Compare performance of GPT, Claude, Gemini, DeepSeek, and more. Simple declarative configs with command line and CI/CD integration. Used by OpenAI and Anthropic.", + "latest_release": { + "tag": "code-scan-action-0.2.0", + "published_at": "2026-08-28T07:34:11Z" + } + }, + "confident-ai/deepeval": { + "repo": "confident-ai/deepeval", + "license_spdx": "Apache-2.0", + "stars": 18186, + "pushed_at": "2026-09-08T07:08:59Z", + "archived": false, + "description": "The LLM Evaluation Framework", + "latest_release": { + "tag": "python-v4.2.0", + "published_at": "2026-08-24T21:56:54Z" + } + }, + "AgentOps-AI/agentops": { + "repo": "AgentOps-AI/agentops", + "license_spdx": "MIT", + "stars": 5813, + "pushed_at": "2026-06-25T08:25:03Z", + "archived": false, + "description": "Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI", + "latest_release": { + "tag": "0.4.21", + "published_at": "2025-08-29T06:36:28Z" + } + }, + "Arize-ai/phoenix": { + "repo": "Arize-ai/phoenix", + "license_spdx": "NOASSERTION", + "stars": 11388, + "pushed_at": "2026-09-09T05:22:55Z", + "archived": false, + "description": "AI Observability & Evaluation", + "latest_release": { + "tag": "arize-phoenix-client-v3.5.0", + "published_at": "2026-09-08T22:22:38Z" + } + }, + "langfuse/langfuse": { + "repo": "langfuse/langfuse", + "license_spdx": "NOASSERTION", + "stars": 34392, + "pushed_at": "2026-09-09T10:06:57Z", + "archived": false, + "description": "\ud83e\udea2 Open source AI engineering platform: LLM evals, observability, metrics, prompt management, playground, datasets. Integrates with OpenTelemetry, LangChain, OpenAI SDK, LiteLLM, and more. \ud83c\udf4aYC W23 ", + "latest_release": { + "tag": "v4.32.0", + "published_at": "2026-09-08T12:53:51Z" + } + }, + "openai/evals": { + "repo": "openai/evals", + "license_spdx": "NOASSERTION", + "stars": 19414, + "pushed_at": "2026-04-14T15:29:57Z", + "archived": false, + "description": "Evals is a framework for evaluating LLMs and LLM systems, and an open-source registry of benchmarks.", + "latest_release": null + } + } +} \ No newline at end of file diff --git a/docs/competitive-analysis.md b/docs/competitive-analysis.md index 3ba8e8e..910ce43 100644 --- a/docs/competitive-analysis.md +++ b/docs/competitive-analysis.md @@ -11,6 +11,31 @@ feature claims are taken only from each project's own README/docs/releases. We never claim a competitor lacks a feature without checking its docs; where we could not verify a feature we mark it "unverified" rather than asserting absence. + + +## Landscape snapshot (fetched 2026-09-09) + +| Project | License | Stars | Last push | Latest release | Status | +|---|---|---|---|---|---| +| [browser-use/browser-use](https://github.com/browser-use/browser-use) | MIT | 113,853 | 2026-09-07 | 0.13.10 (2026-09-04) | active | +| [langfuse/langfuse](https://github.com/langfuse/langfuse) | NOASSERTION | 34,392 | 2026-09-09 | v4.32.0 (2026-09-08) | active | +| [promptfoo/promptfoo](https://github.com/promptfoo/promptfoo) | MIT | 24,964 | 2026-09-09 | code-scan-action-0.2.0 (2026-08-28) | active | +| [openai/evals](https://github.com/openai/evals) | NOASSERTION | 19,414 | 2026-04-14 | — | active | +| [confident-ai/deepeval](https://github.com/confident-ai/deepeval) | Apache-2.0 | 18,186 | 2026-09-08 | python-v4.2.0 (2026-08-24) | active | +| [Arize-ai/phoenix](https://github.com/Arize-ai/phoenix) | NOASSERTION | 11,388 | 2026-09-09 | arize-phoenix-client-v3.5.0 (2026-09-08) | active | +| [AgentOps-AI/agentops](https://github.com/AgentOps-AI/agentops) | MIT | 5,813 | 2026-06-25 | 0.4.21 (2025-08-29) | active | +| [SWE-bench/SWE-bench](https://github.com/SWE-bench/SWE-bench) | MIT | 5,807 | 2026-09-02 | — | active | +| [THUDM/AgentBench](https://github.com/THUDM/AgentBench) | Apache-2.0 | 3,717 | 2026-02-08 | — | active | +| [xlang-ai/OSWorld](https://github.com/xlang-ai/OSWorld) | Apache-2.0 | 3,134 | 2026-08-30 | v0.1.16 (2024-06-26) | active | +| [UKGovernmentBEIS/inspect_ai](https://github.com/UKGovernmentBEIS/inspect_ai) | MIT | 2,734 | 2026-09-09 | — | active | +| [sierra-research/tau2-bench](https://github.com/sierra-research/tau2-bench) | MIT | 1,991 | 2026-09-07 | v1.0.1 (2026-07-22) | active | +| [web-arena-x/webarena](https://github.com/web-arena-x/webarena) | Apache-2.0 | 1,606 | 2025-11-26 | v0.2.0 (2023-10-21) | active | +| [sierra-research/tau-bench](https://github.com/sierra-research/tau-bench) | MIT | 1,427 | 2026-03-18 | — | active | +| [microsoft/SWE-bench-Live](https://github.com/microsoft/SWE-bench-Live) | MIT | 234 | 2026-09-07 | v1.0-multi-language-multi-os-benchmarking (2026-03-08) | active | + +Rows are generated from `data/competitor-meta.json` by `scripts/fetch_competitor_meta.py`, which reads the GitHub API. Star counts and dates are facts about the repositories on the fetch date, not judgements. Nothing here claims a project lacks a feature: where a capability was not verified it is absent from this table rather than asserted as missing. + + ## Landscape summary | Project | License | Activity (pushed) | Stars | Category | @@ -189,4 +214,4 @@ project's README/docs/release notes on the refresh date; otherwise - Clones of public websites as eval targets (WebArena style). - Always-on cloud telemetry SDKs (observability-platform style). - Proprietary enterprise-only gating of security features. -- Any proprietary code, assets or branding from the projects above. \ No newline at end of file +- Any proprietary code, assets or branding from the projects above. diff --git a/pyproject.toml b/pyproject.toml index 70adc76..69c4be5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,6 +170,9 @@ exclude_lines = [ ] [tool.ruff.lint.per-file-ignores] +# Same shared-script situation: the gh endpoint is built from the hardcoded +# REPOS list above it, not from user input. +"scripts/fetch_competitor_meta.py" = ["S603", "S607", "RUF100"] # Shared verbatim across the four sibling repositories, whose ruff rule # sets differ: S310 must be suppressed where it is enabled, and that # suppression must not read as unused (RUF100) where it is not. diff --git a/scripts/fetch_competitor_meta.py b/scripts/fetch_competitor_meta.py new file mode 100644 index 0000000..6d8e4f7 --- /dev/null +++ b/scripts/fetch_competitor_meta.py @@ -0,0 +1,123 @@ +"""Fetch live GitHub metadata for competitor repos via the authenticated gh CLI. + +Writes data/competitor-meta.json and prints a compact TSV summary. +Access date is recorded for evidence purposes. Temp-free: writes only the +final artifact. +""" + +from __future__ import annotations + +import json +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +REPOS = [ + "SWE-bench/SWE-bench", + "microsoft/SWE-bench-Live", + "sierra-research/tau-bench", + "sierra-research/tau2-bench", + "THUDM/AgentBench", + "xlang-ai/OSWorld", + "web-arena-x/webarena", + "browser-use/browser-use", + "UKGovernmentBEIS/inspect_ai", + "promptfoo/promptfoo", + "confident-ai/deepeval", + "AgentOps-AI/agentops", + "Arize-ai/phoenix", + "langfuse/langfuse", + "openai/evals", +] + + +class FetchError(RuntimeError): + """A competitor could not be fetched, and the run must not pretend it could.""" + + +def gh_json(endpoint: str, *, allow_404: bool = False) -> dict[str, object] | None: + """Fetch `endpoint`, raising rather than silently returning None. + + This used to swallow every failure into `None`, and the caller recorded + that as `{"error": "repo-not-found-or-error"}` in the artifact. The + competitive analysis then transcribed it as a fact about the world: + "Optic -- repo gone (404)". Optic's repository is not gone. It is + `opticdev/optic`, archived with 1,534 stars; this script was asking for + `useoptic/optic`, which does not exist. A typo in a constant became a + published claim that an archived competitor had disappeared, which is + both wrong and the opposite of useful -- an archived incumbent in this + exact domain is the most interesting fact in the file. + + `allow_404` is for endpoints where absence is a real answer (a repo with + no releases). Everywhere else a 404 stops the run so a human decides + whether the project moved or genuinely went away. + """ + try: + raw = subprocess.run( + ["gh", "api", endpoint], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=60, + check=True, + ).stdout + return json.loads(raw) # type: ignore[no-any-return] + except subprocess.CalledProcessError as exc: + if allow_404 and "404" in (exc.stderr or ""): + return None + detail = (exc.stderr or "").strip()[:200] + raise FetchError( + f"{endpoint}: {detail}. " + "If the project moved, update REPOS. If it really is gone, say so " + "deliberately rather than letting a failed fetch write it." + ) from exc + except (subprocess.TimeoutExpired, json.JSONDecodeError) as exc: + raise FetchError(f"{endpoint}: {type(exc).__name__}") from exc + + +def main() -> None: + results: dict[str, dict[str, object]] = {} + for repo in REPOS: + meta = gh_json(f"repos/{repo}") + rel = gh_json(f"repos/{repo}/releases/latest", allow_404=True) + entry: dict[str, object] = {"repo": repo} + if meta is None: # pragma: no cover - gh_json raises instead now + raise FetchError(f"{repo}: no metadata returned") + else: + lic = meta.get("license") or {} + entry.update( + { + "license_spdx": lic.get("spdx_id"), + "stars": meta.get("stargazers_count"), + "pushed_at": meta.get("pushed_at"), + "archived": meta.get("archived"), + "description": meta.get("description"), + } + ) + if rel is None: + entry["latest_release"] = None + else: + entry["latest_release"] = { + "tag": rel.get("tag_name"), + "published_at": rel.get("published_at"), + } + results[repo] = entry + + payload = { + "fetched_utc": datetime.now(UTC).isoformat(timespec="seconds"), + "tool": "gh api (authenticated)", + "repos": results, + } + out = Path("data/competitor-meta.json") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(payload, indent=2), encoding="utf-8") + for repo, entry in results.items(): + print( + f"{repo} | {entry.get('license_spdx')} | {entry.get('stars')}" + f" | {entry.get('pushed_at')} | {entry.get('latest_release')}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_landscape.py b/scripts/generate_landscape.py new file mode 100644 index 0000000..61f921b --- /dev/null +++ b/scripts/generate_landscape.py @@ -0,0 +1,119 @@ +"""Render the landscape table in docs/competitive-analysis.md from committed data. + +The table states its own method -- "fetched via the GitHub API" -- but was +typed by hand, so it could disagree with the data it cited. In api-verity-lab +it did, in the worst cell available: a competitor was listed as "repo gone +(404)" when the repository is archived and public with 1,534 stars. The 404 +came from the fetcher asking for the wrong org name, and the null it recorded +was transcribed into the published table as a finding. + +So the table is generated, not written. `--check` fails when the committed +markdown differs from what the data renders, which is what stops it drifting +again. + + python scripts/generate_landscape.py # rewrite the table + python scripts/generate_landscape.py --check # verify, exit 1 on drift +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DOC = ROOT / "docs" / "competitive-analysis.md" +DATA = ROOT / "data" / "competitor-meta.json" + +OPEN = "" +CLOSE = "" + +NOTE = ( + "Rows are generated from `data/competitor-meta.json` by " + "`scripts/fetch_competitor_meta.py`, which reads the GitHub API. " + "Star counts and dates are facts about the repositories on the fetch date, " + "not judgements. Nothing here claims a project lacks a feature: where a " + "capability was not verified it is absent from this table rather than " + "asserted as missing." +) + + +def render() -> str: + data = json.loads(DATA.read_text(encoding="utf-8")) + date = data["fetched_utc"][:10] + + rows = [ + "| Project | License | Stars | Last push | Latest release | Status |", + "|---|---|---|---|---|---|", + ] + for entry in sorted(data["repos"].values(), key=lambda r: r.get("stars") or 0, reverse=True): + release = entry.get("latest_release") or {} + tag = release.get("tag") + published = (release.get("published_at") or "")[:10] + release_cell = f"{tag} ({published})" if tag else "—" + stars = f"{entry['stars']:,}" if entry.get("stars") is not None else "—" + name = entry["repo"] + rows.append( + f"| [{name}](https://github.com/{name}) | {entry.get('license_spdx') or '—'} " + f"| {stars} | {(entry.get('pushed_at') or '')[:10]} | {release_cell} " + f"| {'**archived**' if entry.get('archived') else 'active'} |" + ) + + return ( + f"{OPEN}\n## Landscape snapshot (fetched {date})\n\n" + + "\n".join(rows) + + f"\n\n{NOTE}\n{CLOSE}" + ) + + +def splice(text: str) -> str: + if OPEN not in text or CLOSE not in text: + raise SystemExit(f"{DOC.name} is missing the {OPEN} / {CLOSE} markers.") + head, rest = text.split(OPEN, 1) + _, tail = rest.split(CLOSE, 1) + return head + render() + tail + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + + current = DOC.read_text(encoding="utf-8") + updated = splice(current) + + if args.check: + if current == updated: + data = json.loads(DATA.read_text(encoding="utf-8")) + print( + f"ok landscape table matches competitor-meta.json " + f"({len(data['repos'])} projects, fetched {data['fetched_utc'][:10]})" + ) + return 0 + print( + "The landscape table no longer matches competitor-meta.json.\n" + "Run: python scripts/generate_landscape.py", + file=sys.stderr, + ) + import difflib + + for line in list( + difflib.unified_diff( + current.splitlines(), + updated.splitlines(), + fromfile="committed", + tofile="rendered", + lineterm="", + ) + )[:40]: + print(line, file=sys.stderr) + return 1 + + DOC.write_text(updated, encoding="utf-8", newline="\n") + print("wrote docs/competitive-analysis.md (table rendered from committed data)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())