webdevsamran · webdevsamran · Sep 9, 2026 · Sep 9, 2026
@@ -53,6 +53,11 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ github.token }}
         run: python scripts/check_action_pins.py
+      - name: Landscape table matches the fetched competitor data
+        # The table states its own method ("fetched via the GitHub API")
+        # but was typed by hand. In a sibling repo that drifted into
+        # publishing that a live competitor's repo was gone.
+        run: python scripts/generate_landscape.py --check
       - name: Docs link check
         run: python scripts/check_docs_links.py
       - name: Generate static web data

@@ -275,6 +275,20 @@ These are independent projects: no shared library, no coupled releases, and each
 
 <!-- /related-projects -->
 
+## How this compares
+
+15 projects are tracked in [`docs/competitive-analysis.md`](docs/competitive-analysis.md),
+fetched from the GitHub API on 2026-09-09 and committed to
+[`data/competitor-meta.json`](data/competitor-meta.json). The table is generated from that
+file rather than typed, so it cannot drift from the data it cites.
+
+They divide into three groups that are easy to confuse: **task suites** (SWE-bench,
+tau-bench, OSWorld, WebArena) that define problems, **eval harnesses** (inspect_ai,
+promptfoo, DeepEval) that run and grade them, and **tracing platforms** (Langfuse, Phoenix,
+AgentOps) that record what happened. ToolTrace Bench spans the first two with a specific
+constraint: the score comes from the execution trace and the final workspace, never from a
+model's opinion of its own work.
+
 ## Citation
 
 See [CITATION.cff](CITATION.cff), or:

@@ -0,0 +1,171 @@
+{
+  "fetched_utc": "2026-09-09T10:16:15+00:00",
+  "tool": "gh api (authenticated)",
+  "repos": {
+    "SWE-bench/SWE-bench": {
+      "repo": "SWE-bench/SWE-bench",
+      "license_spdx": "MIT",
+      "stars": 5807,
+      "pushed_at": "2026-09-02T01:51:01Z",
+      "archived": false,
+      "description": "SWE-bench: Can Language Models Resolve Real-world Github Issues?",
+      "latest_release": null
+    },
+    "microsoft/SWE-bench-Live": {
+      "repo": "microsoft/SWE-bench-Live",
+      "license_spdx": "MIT",
+      "stars": 234,
+      "pushed_at": "2026-09-07T23:24:48Z",
+      "archived": false,
+      "description": "[NeurIPS 2025 D&B] \ud83d\ude80 SWE-bench Goes Live!",
+      "latest_release": {
+        "tag": "v1.0-multi-language-multi-os-benchmarking",
+        "published_at": "2026-03-08T11:23:25Z"
+      }
+    },
+    "sierra-research/tau-bench": {
+      "repo": "sierra-research/tau-bench",
+      "license_spdx": "MIT",
+      "stars": 1427,
+      "pushed_at": "2026-03-18T17:36:06Z",
+      "archived": false,
+      "description": "Code and Data for Tau-Bench",
+      "latest_release": null
+    },
+    "sierra-research/tau2-bench": {
+      "repo": "sierra-research/tau2-bench",
+      "license_spdx": "MIT",
+      "stars": 1991,
+      "pushed_at": "2026-09-07T23:46:37Z",
+      "archived": false,
+      "description": "\u03c4-Bench: A Benchmark for Tool-Agent-User Interaction in Real-World Domains",
+      "latest_release": {
+        "tag": "v1.0.1",
+        "published_at": "2026-07-22T21:22:42Z"
+      }
+    },
+    "THUDM/AgentBench": {
+      "repo": "THUDM/AgentBench",
+      "license_spdx": "Apache-2.0",
+      "stars": 3717,
+      "pushed_at": "2026-02-08T17:01:05Z",
+      "archived": false,
+      "description": "A Comprehensive Benchmark to Evaluate LLMs as Agents (ICLR'24)",
+      "latest_release": null
+    },
+    "xlang-ai/OSWorld": {
+      "repo": "xlang-ai/OSWorld",
+      "license_spdx": "Apache-2.0",
+      "stars": 3134,
+      "pushed_at": "2026-08-30T09:41:51Z",
+      "archived": false,
+      "description": "[NeurIPS 2024] OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments",
+      "latest_release": {
+        "tag": "v0.1.16",
+        "published_at": "2024-06-26T04:22:29Z"
+      }
+    },
+    "web-arena-x/webarena": {
+      "repo": "web-arena-x/webarena",
+      "license_spdx": "Apache-2.0",
+      "stars": 1606,
+      "pushed_at": "2025-11-26T21:16:00Z",
+      "archived": false,
+      "description": "Code repo for \"WebArena: A Realistic Web Environment for Building Autonomous Agents\"",
+      "latest_release": {
+        "tag": "v0.2.0",
+        "published_at": "2023-10-21T04:35:19Z"
+      }
+    },
+    "browser-use/browser-use": {
+      "repo": "browser-use/browser-use",
+      "license_spdx": "MIT",
+      "stars": 113853,
+      "pushed_at": "2026-09-07T19:55:36Z",
+      "archived": false,
+      "description": "Agents that use the browser.",
+      "latest_release": {
+        "tag": "0.13.10",
+        "published_at": "2026-09-04T03:28:53Z"
+      }
+    },
+    "UKGovernmentBEIS/inspect_ai": {
+      "repo": "UKGovernmentBEIS/inspect_ai",
+      "license_spdx": "MIT",
+      "stars": 2734,
+      "pushed_at": "2026-09-09T09:02:44Z",
+      "archived": false,
+      "description": "Inspect: A framework for large language model evaluations",
+      "latest_release": null
+    },
+    "promptfoo/promptfoo": {
+      "repo": "promptfoo/promptfoo",
+      "license_spdx": "MIT",
+      "stars": 24964,
+      "pushed_at": "2026-09-09T09:43:10Z",
+      "archived": false,
+      "description": "Test your prompts, agents, and RAGs. Red teaming/pentesting/vulnerability scanning for AI. Compare performance of GPT, Claude, Gemini, DeepSeek, and more. Simple declarative configs with command line and CI/CD integration.  Used by OpenAI and Anthropic.",
+      "latest_release": {
+        "tag": "code-scan-action-0.2.0",
+        "published_at": "2026-08-28T07:34:11Z"
+      }
+    },
+    "confident-ai/deepeval": {
+      "repo": "confident-ai/deepeval",
+      "license_spdx": "Apache-2.0",
+      "stars": 18186,
+      "pushed_at": "2026-09-08T07:08:59Z",
+      "archived": false,
+      "description": "The LLM Evaluation Framework",
+      "latest_release": {
+        "tag": "python-v4.2.0",
+        "published_at": "2026-08-24T21:56:54Z"
+      }
+    },
+    "AgentOps-AI/agentops": {
+      "repo": "AgentOps-AI/agentops",
+      "license_spdx": "MIT",
+      "stars": 5813,
+      "pushed_at": "2026-06-25T08:25:03Z",
+      "archived": false,
+      "description": "Python SDK for AI agent monitoring, LLM cost tracking, benchmarking, and more. Integrates with most LLMs and agent frameworks including CrewAI, Agno, OpenAI Agents SDK, Langchain, Autogen, AG2, and CamelAI",
+      "latest_release": {
+        "tag": "0.4.21",
+        "published_at": "2025-08-29T06:36:28Z"
+      }
+    },
+    "Arize-ai/phoenix": {
+      "repo": "Arize-ai/phoenix",
+      "license_spdx": "NOASSERTION",
+      "stars": 11388,
+      "pushed_at": "2026-09-09T05:22:55Z",
+      "archived": false,
+      "description": "AI Observability & Evaluation",
+      "latest_release": {
+        "tag": "arize-phoenix-client-v3.5.0",
+        "published_at": "2026-09-08T22:22:38Z"
+      }
+    },
+    "langfuse/langfuse": {
+      "repo": "langfuse/langfuse",
+      "license_spdx": "NOASSERTION",
+      "stars": 34392,
+      "pushed_at": "2026-09-09T10:06:57Z",
+      "archived": false,
+      "description": "\ud83e\udea2 Open source AI engineering platform: LLM evals, observability, metrics, prompt management, playground, datasets. Integrates with OpenTelemetry, LangChain, OpenAI SDK, LiteLLM, and more. \ud83c\udf4aYC W23 ",
+      "latest_release": {
+        "tag": "v4.32.0",
+        "published_at": "2026-09-08T12:53:51Z"
+      }
+    },
+    "openai/evals": {
+      "repo": "openai/evals",
+      "license_spdx": "NOASSERTION",
+      "stars": 19414,
+      "pushed_at": "2026-04-14T15:29:57Z",
+      "archived": false,
+      "description": "Evals is a framework for evaluating LLMs and LLM systems, and an open-source registry of benchmarks.",
+      "latest_release": null
+    }
+  }
+}
@@ -11,6 +11,31 @@ feature claims are taken only from each project's own README/docs/releases.
 We never claim a competitor lacks a feature without checking its docs; where we
 could not verify a feature we mark it "unverified" rather than asserting absence.
 
+
+<!-- landscape:generated -->
+## Landscape snapshot (fetched 2026-09-09)
+
+| Project | License | Stars | Last push | Latest release | Status |
+|---|---|---|---|---|---|
+| [browser-use/browser-use](https://github.com/browser-use/browser-use) | MIT | 113,853 | 2026-09-07 | 0.13.10 (2026-09-04) | active |
+| [langfuse/langfuse](https://github.com/langfuse/langfuse) | NOASSERTION | 34,392 | 2026-09-09 | v4.32.0 (2026-09-08) | active |
+| [promptfoo/promptfoo](https://github.com/promptfoo/promptfoo) | MIT | 24,964 | 2026-09-09 | code-scan-action-0.2.0 (2026-08-28) | active |
+| [openai/evals](https://github.com/openai/evals) | NOASSERTION | 19,414 | 2026-04-14 | — | active |
+| [confident-ai/deepeval](https://github.com/confident-ai/deepeval) | Apache-2.0 | 18,186 | 2026-09-08 | python-v4.2.0 (2026-08-24) | active |
+| [Arize-ai/phoenix](https://github.com/Arize-ai/phoenix) | NOASSERTION | 11,388 | 2026-09-09 | arize-phoenix-client-v3.5.0 (2026-09-08) | active |
+| [AgentOps-AI/agentops](https://github.com/AgentOps-AI/agentops) | MIT | 5,813 | 2026-06-25 | 0.4.21 (2025-08-29) | active |
+| [SWE-bench/SWE-bench](https://github.com/SWE-bench/SWE-bench) | MIT | 5,807 | 2026-09-02 | — | active |
+| [THUDM/AgentBench](https://github.com/THUDM/AgentBench) | Apache-2.0 | 3,717 | 2026-02-08 | — | active |
+| [xlang-ai/OSWorld](https://github.com/xlang-ai/OSWorld) | Apache-2.0 | 3,134 | 2026-08-30 | v0.1.16 (2024-06-26) | active |
+| [UKGovernmentBEIS/inspect_ai](https://github.com/UKGovernmentBEIS/inspect_ai) | MIT | 2,734 | 2026-09-09 | — | active |
+| [sierra-research/tau2-bench](https://github.com/sierra-research/tau2-bench) | MIT | 1,991 | 2026-09-07 | v1.0.1 (2026-07-22) | active |
+| [web-arena-x/webarena](https://github.com/web-arena-x/webarena) | Apache-2.0 | 1,606 | 2025-11-26 | v0.2.0 (2023-10-21) | active |
+| [sierra-research/tau-bench](https://github.com/sierra-research/tau-bench) | MIT | 1,427 | 2026-03-18 | — | active |
+| [microsoft/SWE-bench-Live](https://github.com/microsoft/SWE-bench-Live) | MIT | 234 | 2026-09-07 | v1.0-multi-language-multi-os-benchmarking (2026-03-08) | active |
+
+Rows are generated from `data/competitor-meta.json` by `scripts/fetch_competitor_meta.py`, which reads the GitHub API. Star counts and dates are facts about the repositories on the fetch date, not judgements. Nothing here claims a project lacks a feature: where a capability was not verified it is absent from this table rather than asserted as missing.
+<!-- /landscape:generated -->
+
 ## Landscape summary
 
 | Project | License | Activity (pushed) | Stars | Category |
@@ -189,4 +214,4 @@ project's README/docs/release notes on the refresh date; otherwise
 - Clones of public websites as eval targets (WebArena style).
 - Always-on cloud telemetry SDKs (observability-platform style).
 - Proprietary enterprise-only gating of security features.
-- Any proprietary code, assets or branding from the projects above.
+- Any proprietary code, assets or branding from the projects above.
@@ -170,6 +170,9 @@ exclude_lines = [
 ]
 
 [tool.ruff.lint.per-file-ignores]
+# Same shared-script situation: the gh endpoint is built from the hardcoded
+# REPOS list above it, not from user input.
+"scripts/fetch_competitor_meta.py" = ["S603", "S607", "RUF100"]
 # Shared verbatim across the four sibling repositories, whose ruff rule
 # sets differ: S310 must be suppressed where it is enabled, and that
 # suppression must not read as unused (RUF100) where it is not.

@@ -0,0 +1,123 @@
+"""Fetch live GitHub metadata for competitor repos via the authenticated gh CLI.
+
+Writes data/competitor-meta.json and prints a compact TSV summary.
+Access date is recorded for evidence purposes. Temp-free: writes only the
+final artifact.
+"""
+
+from __future__ import annotations
+
+import json
+import subprocess
+from datetime import UTC, datetime
+from pathlib import Path
+
+REPOS = [
+    "SWE-bench/SWE-bench",
+    "microsoft/SWE-bench-Live",
+    "sierra-research/tau-bench",
+    "sierra-research/tau2-bench",
+    "THUDM/AgentBench",
+    "xlang-ai/OSWorld",
+    "web-arena-x/webarena",
+    "browser-use/browser-use",
+    "UKGovernmentBEIS/inspect_ai",
+    "promptfoo/promptfoo",
+    "confident-ai/deepeval",
+    "AgentOps-AI/agentops",
+    "Arize-ai/phoenix",
+    "langfuse/langfuse",
+    "openai/evals",
+]
+
+
+class FetchError(RuntimeError):
+    """A competitor could not be fetched, and the run must not pretend it could."""
+
+
+def gh_json(endpoint: str, *, allow_404: bool = False) -> dict[str, object] | None:
+    """Fetch `endpoint`, raising rather than silently returning None.
+
+    This used to swallow every failure into `None`, and the caller recorded
+    that as `{"error": "repo-not-found-or-error"}` in the artifact. The
+    competitive analysis then transcribed it as a fact about the world:
+    "Optic -- repo gone (404)". Optic's repository is not gone. It is
+    `opticdev/optic`, archived with 1,534 stars; this script was asking for
+    `useoptic/optic`, which does not exist. A typo in a constant became a
+    published claim that an archived competitor had disappeared, which is
+    both wrong and the opposite of useful -- an archived incumbent in this
+    exact domain is the most interesting fact in the file.
+
+    `allow_404` is for endpoints where absence is a real answer (a repo with
+    no releases). Everywhere else a 404 stops the run so a human decides
+    whether the project moved or genuinely went away.
+    """
+    try:
+        raw = subprocess.run(
+            ["gh", "api", endpoint],
+            capture_output=True,
+            text=True,
+            encoding="utf-8",
+            errors="replace",
+            timeout=60,
+            check=True,
+        ).stdout
+        return json.loads(raw)  # type: ignore[no-any-return]
+    except subprocess.CalledProcessError as exc:
+        if allow_404 and "404" in (exc.stderr or ""):
+            return None
+        detail = (exc.stderr or "").strip()[:200]
+        raise FetchError(
+            f"{endpoint}: {detail}. "
+            "If the project moved, update REPOS. If it really is gone, say so "
+            "deliberately rather than letting a failed fetch write it."
+        ) from exc
+    except (subprocess.TimeoutExpired, json.JSONDecodeError) as exc:
+        raise FetchError(f"{endpoint}: {type(exc).__name__}") from exc
+
+
+def main() -> None:
+    results: dict[str, dict[str, object]] = {}
+    for repo in REPOS:
+        meta = gh_json(f"repos/{repo}")
+        rel = gh_json(f"repos/{repo}/releases/latest", allow_404=True)
+        entry: dict[str, object] = {"repo": repo}
+        if meta is None:  # pragma: no cover - gh_json raises instead now
+            raise FetchError(f"{repo}: no metadata returned")
+        else:
+            lic = meta.get("license") or {}
+            entry.update(
+                {
+                    "license_spdx": lic.get("spdx_id"),
+                    "stars": meta.get("stargazers_count"),
+                    "pushed_at": meta.get("pushed_at"),
+                    "archived": meta.get("archived"),
+                    "description": meta.get("description"),
+                }
+            )
+        if rel is None:
+            entry["latest_release"] = None
+        else:
+            entry["latest_release"] = {
+                "tag": rel.get("tag_name"),
+                "published_at": rel.get("published_at"),
+            }
+        results[repo] = entry
+
+    payload = {
+        "fetched_utc": datetime.now(UTC).isoformat(timespec="seconds"),
+        "tool": "gh api (authenticated)",
+        "repos": results,
+    }
+    out = Path("data/competitor-meta.json")
+    out.parent.mkdir(parents=True, exist_ok=True)
+    out.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+    for repo, entry in results.items():
+        print(
+            f"{repo} | {entry.get('license_spdx')} | {entry.get('stars')}"
+            f" | {entry.get('pushed_at')} | {entry.get('latest_release')}"
+        )
+
+
+if __name__ == "__main__":
+    main()