From c7ca17848b466046a25419d9831f107e68197fed Mon Sep 17 00:00:00 2001 From: Samran Asif Date: Wed, 9 Sep 2026 15:29:28 +0500 Subject: [PATCH] docs: generate the competitive landscape from data fetched this run The competitive tables in this family of repositories stated their own method -- "repository metadata fetched via the GitHub API" -- and were then typed by hand, from data 17 days stale. api-verity-lab shows what that costs: its table recorded a live competitor as "repo gone (404)" because its fetcher asked for the wrong org name and wrote the resulting null into the artifact as a finding. So the numbers now come from a fetch run today and the table is generated from the committed result, with scripts/generate_landscape.py --check failing the build when the markdown and the data disagree. Only api-verity-lab had a checked-in fetcher; the others had committed capability data with no reproducible way to refresh it. All four have one now, each with its own competitor set, and all four inherit the fix that makes a failed lookup raise rather than silently become data. The README gains a "How this compares" section that says plainly where this project overlaps its neighbours and where it does not -- linking to the full table rather than duplicating it, so there is one place for the numbers to be right. Nothing here asserts that a competitor lacks a feature. Stars, licences and dates are facts about repositories on the fetch date; capability judgements stay in the prose, where they can be argued with. Verified: ruff, ruff format, mypy, 316 tests pass, landscape --check clean. --- .github/workflows/ci.yml | 5 + README.md | 14 +++ data/competitor-meta.json | 171 +++++++++++++++++++++++++++++++ docs/competitive-analysis.md | 27 ++++- pyproject.toml | 3 + scripts/fetch_competitor_meta.py | 123 ++++++++++++++++++++++ scripts/generate_landscape.py | 119 +++++++++++++++++++++ 7 files changed, 461 insertions(+), 1 deletion(-) create mode 100644 data/competitor-meta.json create mode 100644 scripts/fetch_competitor_meta.py create mode 100644 scripts/generate_landscape.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb8b865..5c34b7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,11 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: python scripts/check_action_pins.py + - name: Landscape table matches the fetched competitor data + # The table states its own method ("fetched via the GitHub API") + # but was typed by hand. In a sibling repo that drifted into + # publishing that a live competitor's repo was gone. + run: python scripts/generate_landscape.py --check - name: Docs link check run: python scripts/check_docs_links.py - name: Generate static web data diff --git a/README.md b/README.md index d65eed6..486edda 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,20 @@ These are independent projects: no shared library, no coupled releases, and each +## How this compares + +15 projects are tracked in [`docs/competitive-analysis.md`](docs/competitive-analysis.md), +fetched from the GitHub API on 2026-09-09 and committed to +[`data/competitor-meta.json`](data/competitor-meta.json). The table is generated from that +file rather than typed, so it cannot drift from the data it cites. + +They divide into three groups that are easy to confuse: **task suites** (SWE-bench, +tau-bench, OSWorld, WebArena) that define problems, **eval harnesses** (inspect_ai, +promptfoo, DeepEval) that run and grade them, and **tracing platforms** (Langfuse, Phoenix, +AgentOps) that record what happened. ToolTrace Bench spans the first two with a specific +constraint: the score comes from the execution trace and the final workspace, never from a +model's opinion of its own work. + ## Citation See [CITATION.cff](CITATION.cff), or: diff --git a/data/competitor-meta.json b/data/competitor-meta.json new file mode 100644 index 0000000..56f22c3 --- /dev/null +++ b/data/competitor-meta.json @@ -0,0 +1,171 @@ +{ + "fetched_utc": "2026-09-09T10:16:15+00:00", + "tool": "gh api (authenticated)", + "repos": { + "SWE-bench/SWE-bench": { + "repo": "SWE-bench/SWE-bench", + "license_spdx": "MIT", + "stars": 5807, + "pushed_at": "2026-09-02T01:51:01Z", + "archived": false, + "description": "SWE-bench: Can Language Models Resolve Real-world Github Issues?", + "latest_release": null + }, + "microsoft/SWE-bench-Live": { + "repo": "microsoft/SWE-bench-Live", + "license_spdx": "MIT", + "stars": 234, + "pushed_at": "2026-09-07T23:24:48Z", + "archived": false, + "description": "[NeurIPS 2025 D&B] \ud83d\ude80 SWE-bench Goes Live!", + "latest_release": { + "tag": "v1.0-multi-language-multi-os-benchmarking", + "published_at": "2026-03-08T11:23:25Z" + } + }, + "sierra-research/tau-bench": { + "repo": "sierra-research/tau-bench", + "license_spdx": "MIT", + "stars": 1427, + "pushed_at": "2026-03-18T17:36:06Z", + "archived": false, + "description": "Code and Data for Tau-Bench", + "latest_release": null + }, + "sierra-research/tau2-bench": { + "repo": "sierra-research/tau2-bench", + "license_spdx": "MIT", + "stars": 1991, + "pushed_at": "2026-09-07T23:46:37Z", + "archived": false, + "description": "\u03c4-Bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains", + "latest_release": { + "tag": "v1.0.1", + "published_at": "2026-07-22T21:22:42Z" + } + }, + "THUDM/AgentBench": { + "repo": "THUDM/AgentBench", + "license_spdx": "Apache-2.0", + "stars": 3717, + "pushed_at": "2026-02-08T17:01:05Z", + "archived": false, + "description": "A Comprehensive Benchmark to Evaluate LLMs as Agents (ICLR'24)", + "latest_release": null + }, + "xlang-ai/OSWorld": { + "repo": "xlang-ai/OSWorld", + "license_spdx": "Apache-2.0", + "stars": 3134, + "pushed_at": "2026-08-30T09:41:51Z", + "archived": false, + "description": "[NeurIPS 2024] OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments", + "latest_release": { + "tag": "v0.1.16", + "published_at": "2024-06-26T04:22:29Z" + } + }, + "web-arena-x/webarena": { + "repo": "web-arena-x/webarena", + "license_spdx": "Apache-2.0", + "stars": 1606, + "pushed_at": "2025-11-26T21:16:00Z", + "archived": false, + "description": "Code repo for \"WebArena: A Realistic Web Environment for Building Autonomous Agents\"", + "latest_release": { + "tag": "v0.2.0", + "published_at": "2023-10-21T04:35:19Z" + } + }, + "browser-use/browser-use": { + "repo": "browser-use/browser-use", + "license_spdx": "MIT", + "stars": 113853, + "pushed_at": "2026-09-07T19:55:36Z", + "archived": false, + "description": "Agents that use the browser.", + "latest_release": { + "tag": "0.13.10", + "published_at": "2026-09-04T03:28:53Z" + } + }, + "UKGovernmentBEIS/inspect_ai": { + "repo": "UKGovernmentBEIS/inspect_ai", + "license_spdx": "MIT", + "stars": 2734, + "pushed_at": "2026-09-09T09:02:44Z", + "archived": false, + "description": "Inspect: A framework for large language model evaluations", + "latest_release": null + }, + "promptfoo/promptfoo": { + "repo": "promptfoo/promptfoo", + "license_spdx": "MIT", + "stars": 24964, + "pushed_at": "2026-09-09T09:43:10Z", + "archived": false, + "description": "Test your prompts, agents, and RAGs. Red teaming/pentesting/vulnerability scanning for AI. Compare performance of GPT, Claude, Gemini, DeepSeek, and more. Simple declarative configs with command line and CI/CD integration. Used by OpenAI and Anthropic.", + "latest_release": { + "tag": "code-scan-action-0.2.0", + "published_at": "2026-08-28T07:34:11Z" + } + }, + "confident-ai/deepeval": { + "repo": "confident-ai/deepeval", + "license_spdx": "Apache-2.0", + "stars": 18186, + "pushed_at": "2026-09-08T07:08:59Z", + "archived": false, + "description": "The LLM Evaluation Framework", + "latest_release": { + "tag": "python-v4.2.0", + "published_at": "2026-08-24T21:56:54Z" + } + }, + "AgentOps-AI/agentops": { + "repo": "AgentOps-AI/agentops", + "license_spdx": "MIT", + "stars": 5813, + "pushed_at": "2026-06-25T08:25:03Z", + "archived": false, + "description": "Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI", + "latest_release": { + "tag": "0.4.21", + "published_at": "2025-08-29T06:36:28Z" + } + }, + "Arize-ai/phoenix": { + "repo": "Arize-ai/phoenix", + "license_spdx": "NOASSERTION", + "stars": 11388, + "pushed_at": "2026-09-09T05:22:55Z", + "archived": false, + "description": "AI Observability & Evaluation", + "latest_release": { + "tag": "arize-phoenix-client-v3.5.0", + "published_at": "2026-09-08T22:22:38Z" + } + }, + "langfuse/langfuse": { + "repo": "langfuse/langfuse", + "license_spdx": "NOASSERTION", + "stars": 34392, + "pushed_at": "2026-09-09T10:06:57Z", + "archived": false, + "description": "\ud83e\udea2 Open source AI engineering platform: LLM evals, observability, metrics, prompt management, playground, datasets. Integrates with OpenTelemetry, LangChain, OpenAI SDK, LiteLLM, and more. \ud83c\udf4aYC W23 ", + "latest_release": { + "tag": "v4.32.0", + "published_at": "2026-09-08T12:53:51Z" + } + }, + "openai/evals": { + "repo": "openai/evals", + "license_spdx": "NOASSERTION", + "stars": 19414, + "pushed_at": "2026-04-14T15:29:57Z", + "archived": false, + "description": "Evals is a framework for evaluating LLMs and LLM systems, and an open-source registry of benchmarks.", + "latest_release": null + } + } +} \ No newline at end of file diff --git a/docs/competitive-analysis.md b/docs/competitive-analysis.md index 3ba8e8e..910ce43 100644 --- a/docs/competitive-analysis.md +++ b/docs/competitive-analysis.md @@ -11,6 +11,31 @@ feature claims are taken only from each project's own README/docs/releases. We never claim a competitor lacks a feature without checking its docs; where we could not verify a feature we mark it "unverified" rather than asserting absence. + + +## Landscape snapshot (fetched 2026-09-09) + +| Project | License | Stars | Last push | Latest release | Status | +|---|---|---|---|---|---| +| [browser-use/browser-use](https://github.com/browser-use/browser-use) | MIT | 113,853 | 2026-09-07 | 0.13.10 (2026-09-04) | active | +| [langfuse/langfuse](https://github.com/langfuse/langfuse) | NOASSERTION | 34,392 | 2026-09-09 | v4.32.0 (2026-09-08) | active | +| [promptfoo/promptfoo](https://github.com/promptfoo/promptfoo) | MIT | 24,964 | 2026-09-09 | code-scan-action-0.2.0 (2026-08-28) | active | +| [openai/evals](https://github.com/openai/evals) | NOASSERTION | 19,414 | 2026-04-14 | — | active | +| [confident-ai/deepeval](https://github.com/confident-ai/deepeval) | Apache-2.0 | 18,186 | 2026-09-08 | python-v4.2.0 (2026-08-24) | active | +| [Arize-ai/phoenix](https://github.com/Arize-ai/phoenix) | NOASSERTION | 11,388 | 2026-09-09 | arize-phoenix-client-v3.5.0 (2026-09-08) | active | +| [AgentOps-AI/agentops](https://github.com/AgentOps-AI/agentops) | MIT | 5,813 | 2026-06-25 | 0.4.21 (2025-08-29) | active | +| [SWE-bench/SWE-bench](https://github.com/SWE-bench/SWE-bench) | MIT | 5,807 | 2026-09-02 | — | active | +| [THUDM/AgentBench](https://github.com/THUDM/AgentBench) | Apache-2.0 | 3,717 | 2026-02-08 | — | active | +| [xlang-ai/OSWorld](https://github.com/xlang-ai/OSWorld) | Apache-2.0 | 3,134 | 2026-08-30 | v0.1.16 (2024-06-26) | active | +| [UKGovernmentBEIS/inspect_ai](https://github.com/UKGovernmentBEIS/inspect_ai) | MIT | 2,734 | 2026-09-09 | — | active | +| [sierra-research/tau2-bench](https://github.com/sierra-research/tau2-bench) | MIT | 1,991 | 2026-09-07 | v1.0.1 (2026-07-22) | active | +| [web-arena-x/webarena](https://github.com/web-arena-x/webarena) | Apache-2.0 | 1,606 | 2025-11-26 | v0.2.0 (2023-10-21) | active | +| [sierra-research/tau-bench](https://github.com/sierra-research/tau-bench) | MIT | 1,427 | 2026-03-18 | — | active | +| [microsoft/SWE-bench-Live](https://github.com/microsoft/SWE-bench-Live) | MIT | 234 | 2026-09-07 | v1.0-multi-language-multi-os-benchmarking (2026-03-08) | active | + +Rows are generated from `data/competitor-meta.json` by `scripts/fetch_competitor_meta.py`, which reads the GitHub API. Star counts and dates are facts about the repositories on the fetch date, not judgements. Nothing here claims a project lacks a feature: where a capability was not verified it is absent from this table rather than asserted as missing. + + ## Landscape summary | Project | License | Activity (pushed) | Stars | Category | @@ -189,4 +214,4 @@ project's README/docs/release notes on the refresh date; otherwise - Clones of public websites as eval targets (WebArena style). - Always-on cloud telemetry SDKs (observability-platform style). - Proprietary enterprise-only gating of security features. -- Any proprietary code, assets or branding from the projects above. \ No newline at end of file +- Any proprietary code, assets or branding from the projects above. diff --git a/pyproject.toml b/pyproject.toml index 70adc76..69c4be5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,6 +170,9 @@ exclude_lines = [ ] [tool.ruff.lint.per-file-ignores] +# Same shared-script situation: the gh endpoint is built from the hardcoded +# REPOS list above it, not from user input. +"scripts/fetch_competitor_meta.py" = ["S603", "S607", "RUF100"] # Shared verbatim across the four sibling repositories, whose ruff rule # sets differ: S310 must be suppressed where it is enabled, and that # suppression must not read as unused (RUF100) where it is not. diff --git a/scripts/fetch_competitor_meta.py b/scripts/fetch_competitor_meta.py new file mode 100644 index 0000000..6d8e4f7 --- /dev/null +++ b/scripts/fetch_competitor_meta.py @@ -0,0 +1,123 @@ +"""Fetch live GitHub metadata for competitor repos via the authenticated gh CLI. + +Writes data/competitor-meta.json and prints a compact TSV summary. +Access date is recorded for evidence purposes. Temp-free: writes only the +final artifact. +""" + +from __future__ import annotations + +import json +import subprocess +from datetime import UTC, datetime +from pathlib import Path + +REPOS = [ + "SWE-bench/SWE-bench", + "microsoft/SWE-bench-Live", + "sierra-research/tau-bench", + "sierra-research/tau2-bench", + "THUDM/AgentBench", + "xlang-ai/OSWorld", + "web-arena-x/webarena", + "browser-use/browser-use", + "UKGovernmentBEIS/inspect_ai", + "promptfoo/promptfoo", + "confident-ai/deepeval", + "AgentOps-AI/agentops", + "Arize-ai/phoenix", + "langfuse/langfuse", + "openai/evals", +] + + +class FetchError(RuntimeError): + """A competitor could not be fetched, and the run must not pretend it could.""" + + +def gh_json(endpoint: str, *, allow_404: bool = False) -> dict[str, object] | None: + """Fetch `endpoint`, raising rather than silently returning None. + + This used to swallow every failure into `None`, and the caller recorded + that as `{"error": "repo-not-found-or-error"}` in the artifact. The + competitive analysis then transcribed it as a fact about the world: + "Optic -- repo gone (404)". Optic's repository is not gone. It is + `opticdev/optic`, archived with 1,534 stars; this script was asking for + `useoptic/optic`, which does not exist. A typo in a constant became a + published claim that an archived competitor had disappeared, which is + both wrong and the opposite of useful -- an archived incumbent in this + exact domain is the most interesting fact in the file. + + `allow_404` is for endpoints where absence is a real answer (a repo with + no releases). Everywhere else a 404 stops the run so a human decides + whether the project moved or genuinely went away. + """ + try: + raw = subprocess.run( + ["gh", "api", endpoint], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=60, + check=True, + ).stdout + return json.loads(raw) # type: ignore[no-any-return] + except subprocess.CalledProcessError as exc: + if allow_404 and "404" in (exc.stderr or ""): + return None + detail = (exc.stderr or "").strip()[:200] + raise FetchError( + f"{endpoint}: {detail}. " + "If the project moved, update REPOS. If it really is gone, say so " + "deliberately rather than letting a failed fetch write it." + ) from exc + except (subprocess.TimeoutExpired, json.JSONDecodeError) as exc: + raise FetchError(f"{endpoint}: {type(exc).__name__}") from exc + + +def main() -> None: + results: dict[str, dict[str, object]] = {} + for repo in REPOS: + meta = gh_json(f"repos/{repo}") + rel = gh_json(f"repos/{repo}/releases/latest", allow_404=True) + entry: dict[str, object] = {"repo": repo} + if meta is None: # pragma: no cover - gh_json raises instead now + raise FetchError(f"{repo}: no metadata returned") + else: + lic = meta.get("license") or {} + entry.update( + { + "license_spdx": lic.get("spdx_id"), + "stars": meta.get("stargazers_count"), + "pushed_at": meta.get("pushed_at"), + "archived": meta.get("archived"), + "description": meta.get("description"), + } + ) + if rel is None: + entry["latest_release"] = None + else: + entry["latest_release"] = { + "tag": rel.get("tag_name"), + "published_at": rel.get("published_at"), + } + results[repo] = entry + + payload = { + "fetched_utc": datetime.now(UTC).isoformat(timespec="seconds"), + "tool": "gh api (authenticated)", + "repos": results, + } + out = Path("data/competitor-meta.json") + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(payload, indent=2), encoding="utf-8") + for repo, entry in results.items(): + print( + f"{repo} | {entry.get('license_spdx')} | {entry.get('stars')}" + f" | {entry.get('pushed_at')} | {entry.get('latest_release')}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_landscape.py b/scripts/generate_landscape.py new file mode 100644 index 0000000..61f921b --- /dev/null +++ b/scripts/generate_landscape.py @@ -0,0 +1,119 @@ +"""Render the landscape table in docs/competitive-analysis.md from committed data. + +The table states its own method -- "fetched via the GitHub API" -- but was +typed by hand, so it could disagree with the data it cited. In api-verity-lab +it did, in the worst cell available: a competitor was listed as "repo gone +(404)" when the repository is archived and public with 1,534 stars. The 404 +came from the fetcher asking for the wrong org name, and the null it recorded +was transcribed into the published table as a finding. + +So the table is generated, not written. `--check` fails when the committed +markdown differs from what the data renders, which is what stops it drifting +again. + + python scripts/generate_landscape.py # rewrite the table + python scripts/generate_landscape.py --check # verify, exit 1 on drift +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DOC = ROOT / "docs" / "competitive-analysis.md" +DATA = ROOT / "data" / "competitor-meta.json" + +OPEN = "" +CLOSE = "" + +NOTE = ( + "Rows are generated from `data/competitor-meta.json` by " + "`scripts/fetch_competitor_meta.py`, which reads the GitHub API. " + "Star counts and dates are facts about the repositories on the fetch date, " + "not judgements. Nothing here claims a project lacks a feature: where a " + "capability was not verified it is absent from this table rather than " + "asserted as missing." +) + + +def render() -> str: + data = json.loads(DATA.read_text(encoding="utf-8")) + date = data["fetched_utc"][:10] + + rows = [ + "| Project | License | Stars | Last push | Latest release | Status |", + "|---|---|---|---|---|---|", + ] + for entry in sorted(data["repos"].values(), key=lambda r: r.get("stars") or 0, reverse=True): + release = entry.get("latest_release") or {} + tag = release.get("tag") + published = (release.get("published_at") or "")[:10] + release_cell = f"{tag} ({published})" if tag else "—" + stars = f"{entry['stars']:,}" if entry.get("stars") is not None else "—" + name = entry["repo"] + rows.append( + f"| [{name}](https://github.com/{name}) | {entry.get('license_spdx') or '—'} " + f"| {stars} | {(entry.get('pushed_at') or '')[:10]} | {release_cell} " + f"| {'**archived**' if entry.get('archived') else 'active'} |" + ) + + return ( + f"{OPEN}\n## Landscape snapshot (fetched {date})\n\n" + + "\n".join(rows) + + f"\n\n{NOTE}\n{CLOSE}" + ) + + +def splice(text: str) -> str: + if OPEN not in text or CLOSE not in text: + raise SystemExit(f"{DOC.name} is missing the {OPEN} / {CLOSE} markers.") + head, rest = text.split(OPEN, 1) + _, tail = rest.split(CLOSE, 1) + return head + render() + tail + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + + current = DOC.read_text(encoding="utf-8") + updated = splice(current) + + if args.check: + if current == updated: + data = json.loads(DATA.read_text(encoding="utf-8")) + print( + f"ok landscape table matches competitor-meta.json " + f"({len(data['repos'])} projects, fetched {data['fetched_utc'][:10]})" + ) + return 0 + print( + "The landscape table no longer matches competitor-meta.json.\n" + "Run: python scripts/generate_landscape.py", + file=sys.stderr, + ) + import difflib + + for line in list( + difflib.unified_diff( + current.splitlines(), + updated.splitlines(), + fromfile="committed", + tofile="rendered", + lineterm="", + ) + )[:40]: + print(line, file=sys.stderr) + return 1 + + DOC.write_text(updated, encoding="utf-8", newline="\n") + print("wrote docs/competitive-analysis.md (table rendered from committed data)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())