Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions .github/workflows/scheduled-rescore.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,17 @@ jobs:
- name: Snapshot pre-rescore state
run: sqlite3 data/rank.db "SELECT url, overall_score, badge_embedded FROM repo ORDER BY url;" > /tmp/state-before.txt

- name: Note when the rescore started
id: started
run: echo "at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT"

- name: Rescore curated seeds
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: bun run seed

- name: Commit and push if the repo set, a score or badge adoption changed
id: publish
run: |
sqlite3 data/rank.db "SELECT url, overall_score, badge_embedded FROM repo ORDER BY url;" > /tmp/state-after.txt
if diff -q /tmp/state-before.txt /tmp/state-after.txt > /dev/null; then
Expand All @@ -57,3 +62,13 @@ jobs:
git add data/rank.db
git commit -m "chore: scheduled rescore $(date -u +%Y-%m-%dT%H:%MZ)"
git push
echo "pushed=true" >> "$GITHUB_OUTPUT"

# The push only starts the deploy. A crawler told about a page before the
# new build serves it re-reads the old one and has no reason to come back.
- name: Tell IndexNow which pages changed
if: steps.publish.outputs.pushed == 'true'
continue-on-error: true
run: |
sleep 300
bun run indexnow --since "${{ steps.started.outputs.at }}"
16 changes: 13 additions & 3 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ bun run score <url> # score a single repo
bun run audit-seeds # check every seed is public, original, still there, and still being rescored
bun run parity-check # clone-vs-live-path scores must match (add --pr for the PR subset)
bun run test # unit tests (node --test + tsx) — requires Node ≥22
bun run indexnow # submit sitemap URLs to IndexNow (--since <ISO time> for changed pages only, --dry-run to list them)
```

## Layout
Expand All @@ -52,6 +53,8 @@ app/
methodology/page.tsx # how the static scoring works
language/page.tsx # every language: hub cards, plus the repos of languages too small for a hub
language/[slug]/page.tsx # language hub — that language's ranked repos and its most-missed cross-agent checks
check/page.tsx # every check with its pass rate across the leaderboard
check/[slug]/page.tsx # one check — prose from lib/check-content, pass rates and example repos from lib/check-stats
about/page.tsx # who built this and why (footer-linked, E-E-A-T)
roadmap/page.tsx # upcoming versions (from lib/roadmap.ts)
changelog/page.tsx # what's in this build (from lib/changelog.ts)
Expand All @@ -60,7 +63,7 @@ app/
robots.ts # /robots.txt — wildcard + explicit AI-crawler allows. Filtered leaderboard views are deduped
# with noindex+follow in page.tsx, not a Disallow — a blocked URL hides that directive.
# Next.js `_rsc` prefetch requests are disallowed: they were most of Googlebot's crawl
sitemap.ts # /sitemap.xml — static routes, agent boards, language hubs, every repo (priority scaled by score), top packages
sitemap.ts # /sitemap.xml — static routes, check pages, agent boards, language hubs, every repo (priority scaled by score), top packages
llms.txt/route.ts # /llms.txt — markdown manifest for LLM crawlers (Perplexity, Claude, ChatGPT search)
api/repos/route.ts
api/repo/[id]/route.ts
Expand Down Expand Up @@ -89,6 +92,7 @@ components/ # Tailwind-styled React components
SignalRow.tsx, SuggestionItem.tsx, VersionPill.tsx,
RepoHero.tsx, RepoSummary.tsx, ScoreDeltaPopover.tsx, SignalListCard.tsx, ModelSuggestions.tsx, PerModelScores.tsx,
AlternativesStrip.tsx, BreadcrumbJsonLd.tsx, HomeJsonLd.tsx, ExternalLink.tsx,
CheckRepoList.tsx, CodeText.tsx,
BadgeEmbed.tsx, ActionEmbed.tsx, PeerlistBadge.tsx, ProductHuntBadge.tsx,
CopySnippet.tsx, PackageLookupForm.tsx,
BadgeAdoptedTag.tsx, BackToTop.tsx, GoogleAnalytics.tsx,
Expand All @@ -100,7 +104,8 @@ lib/
hosts.ts # host label map
sort.ts # leaderboard sort keys
utils/
format.ts # compactStars, relativeTime, hostLabel
format.ts # compactStars, relativeTime, hostLabel, percent
check-path.ts # checkSlug / checkPath / signalIdFromSlug — the one place a check URL is built or parsed
score.ts # scoreTier + Tailwind class maps
badge.ts # SVG badge renderer (used by /api/badge)
contact.ts # packageRequestIssueUrl — pre-filled GitHub issue link for unscored packages
Expand All @@ -125,6 +130,8 @@ lib/
score.ts # liveScore(): commit resolve + metadata + materialize + scoreRepo
package-lookup.ts # shared registry → repo lookup (used by /api/package + /package page)
language-hubs.ts # language groups + hub stats (used by /language pages, sitemap)
check-content/ # hand-written prose for each /check/<slug> page, keyed by signal id
check-stats.ts # per-check pass counts, by-language rates, example repos (used by /check pages)
leaderboard-view.ts # search params → the rows, page and filters one board shows (used by / and /model/<id>)
badge-adoption.ts # detectBadgeEmbed — reads the cloned README for an embedded AFC badge (dashboard metadata, NOT a scored signal; never vendored to siblings)
db.ts # better-sqlite3 connection + queries (RANK_DB_PATH overrides the file)
Expand All @@ -141,12 +148,14 @@ scripts/
seed.ts # reconciles the DB with seed-list.ts (rename in place, then prune), then scores every seed
check-db-leaks.ts # findLeaks gate — lefthook, ci.yml, and the rescore workflow before it pushes
parity-check.ts # asserts the live-score path equals `bun run score`. CI gate: .github/workflows/parity.yml
indexnow.ts # submits sitemap URLs to IndexNow; the key is public/<key>.txt
audit-seeds.ts # flags seeds that are forks / mirrors / archived / renamed / gone / no longer rescored.
# Needs a valid GITHUB_TOKEN — unauthenticated it covers ~60 repos/hr.
# Weekly via .github/workflows/seed-audit.yml, which files the findings as an issue.
tests/
_helpers.ts # makeFixture / removeFixture build synthetic trees under os.tmpdir()
format.test.ts # compactStars, relativeTime, hostLabel
check-content.test.ts # every signal has check-page prose; check slugs round-trip
parse-repo-url.test.ts # GH / GL / BB parsing + edge cases
scorer.test.ts # scoreRepo, topImprovements
badge-adoption.test.ts # detectBadgeEmbed — README badge-embed detection
Expand Down Expand Up @@ -230,6 +239,7 @@ If either sibling isn't present locally, flag it; never silently skip the propag
1. New file at `lib/scoring/signals/<kebab-id>.ts` implementing `Signal` (including `improveSuggestion`). Use the `helpers.ts` resolvers for every path lookup (see Conventions).
2. Import and add to the `SIGNALS` array in `lib/scoring/signals/index.ts`.
3. Add a weight entry to **every** model in `lib/scoring/weights.ts` — missing weights default to 0, decide deliberately.
Then write its `/check/<slug>` page in `lib/check-content/`. The `scoring` lines there restate the signal's `check` outcomes by hand, so change them whenever a threshold changes.
4. Re-score: `bun run seed` is idempotent.
5. **Mirror to both siblings** (per "Sibling repos" above): for each of `../agent-friendly-action/` and `../agent-friendly-skill/`, copy the new signal file into `src/scoring/signals/`, add the import + array entry to that repo's `signals/index.ts`, copy the weights changes to its `weights.ts`, and log the addition in its `CHANGELOG.md` under "Unreleased".

Expand Down Expand Up @@ -284,7 +294,7 @@ Hooks docs: <https://docs.claude.com/en/docs/claude-code/hooks.html>.
- `/score/[host]/[owner]/[name]` turns a visitor-supplied slug into host API calls and a `/tmp` directory. Two guards carry that: the `SLUG` regex on the route (host slug alphabet — everything else is a probe, and each miss costs a tree-API call), and `safeAbsolute` in `lib/live-score/materialize.ts`, which is the only thing between an attacker-chosen tree path and the filesystem. Both are load-bearing; `tests/live-score.test.ts` covers the traversal cases. Fetched bytes are written to disk and read back by the scorer — never executed.
- SQL: all queries parameterised. No interpolation.
- `data/rank.db` is committed to a public repo, freed pages included. `bun run check-db-leaks` must pass before any commit of it; the rescore workflow runs it before pushing.
- HTML: React auto-escapes. The only `dangerouslySetInnerHTML` is server-built JSON-LD with `<` escaped to `\u003c` (`app/layout.tsx`, `app/about/page.tsx`, `app/action/page.tsx`, `app/skill/page.tsx`, `app/score/page.tsx`, `app/methodology/page.tsx`, `app/repo/[...slug]/page.tsx`, plus the `HomeJsonLd` component on the leaderboard and the `BreadcrumbJsonLd` component used by About / Agent boards / Changelog / Languages / Methodology / Packages (index and result) / Privacy / Roadmap / Terms); never feed user-controlled strings into it.
- HTML: React auto-escapes. The only `dangerouslySetInnerHTML` is server-built JSON-LD with `<` escaped to `\u003c` (`app/layout.tsx`, `app/about/page.tsx`, `app/action/page.tsx`, `app/skill/page.tsx`, `app/score/page.tsx`, `app/methodology/page.tsx`, `app/repo/[...slug]/page.tsx`, plus the `HomeJsonLd` component on the leaderboard and the `BreadcrumbJsonLd` component used by About / Agent boards / Changelog / Checks / Languages / Methodology / Packages (index and result) / Privacy / Roadmap / Terms); never feed user-controlled strings into it.
- Local-path mode reads files; never writes outside `data/` and the clone workspace passed to `shallowClone`.
- No auth yet. The only write a request can trigger is a package lookup caching its package → repo alias. When auth lands (`tasks/0.9.0/01-opt-out-claim-flow.md`), do it via OAuth and gate DB writes per user.

Expand Down
7 changes: 5 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -155,10 +155,11 @@ app/ Next.js App Router — pages + API + SEO
skill/ agent-skill explainer + install command
package/ registry → repo lookup (form + per-package state pages)
language/ per-language hubs — /language, /language/<slug>
check/ one page per check — /check, /check/<slug>
model/ per-agent leaderboards — /model/<id>
api/ /repos, /repo/[id], /score, /badge/<host>/<owner>/<name>, /package/<registry>/<name>
robots.ts /robots.txt — allows "/", blocks "/api/", "/score/" (unbounded URL space) and Next.js `_rsc` prefetch requests
sitemap.ts /sitemap.xml — static routes, agent boards, language hubs, every repo, top packages
sitemap.ts /sitemap.xml — static routes, check pages, agent boards, language hubs, every repo, top packages
llms.txt/ markdown manifest for LLM crawlers
globals.css Tailwind import + @theme tokens
components/ React components (Tailwind-styled)
Expand All @@ -174,11 +175,13 @@ lib/
db-leaks.ts local-path / token scan of the raw DB bytes before it is committed
package-lookup.ts shared registry → repo lookup (used by /api/package + /package page)
language-hubs.ts language groups + hub stats (used by /language pages, sitemap)
check-content/ prose for the /check pages, keyed by signal id
check-stats.ts per-check pass rates and example repos (used by /check pages)
version.ts app + sibling URLs, install snippets (ACTION_USES, SKILL_INSTALL_CMD), SIBLING_VERSION pin
badge-adoption.ts detectBadgeEmbed — dashboard metadata, not a scored signal
release-notice.ts localStorage "seen" marker for the home-page release announcement
changelog.ts / roadmap.ts / skill-content.ts
scripts/ CLI entries run via `tsx` (Node) — score, seed, init-db, audit-seeds, parity-check, check-db-leaks
scripts/ CLI entries run via `tsx` (Node) — score, seed, init-db, audit-seeds, parity-check, check-db-leaks, indexnow
tests/ `node --test` unit tests — scorer, signals, URL parser, formatters
tasks/ Per-version task breakdown (agent-readable)
public/ Static assets — demo/ screenshots used by the README, IndexNow key (<key>.txt)
Expand Down
190 changes: 190 additions & 0 deletions app/check/[slug]/page.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,190 @@
import type { Metadata } from "next";
import Link from "next/link";
import { notFound } from "next/navigation";

import { BreadcrumbJsonLd } from "@/components/BreadcrumbJsonLd";
import { CheckRepoList } from "@/components/CheckRepoList";
import { CodeText } from "@/components/CodeText";
import { Panel, PanelHeading } from "@/components/Panel";
import { CHECK_CONTENT } from "@/lib/check-content";
import { fullRate, getCheckStats } from "@/lib/check-stats";
import { SIGNAL_BY_ID, SIGNALS } from "@/lib/scoring/signals";
import { checkPath, checkSlug, signalIdFromSlug } from "@/lib/utils/check-path";
import { percent } from "@/lib/utils/format";
import { hubPath } from "@/lib/utils/language";
import { modelPath } from "@/lib/utils/leaderboard-href";
import { DEFAULT_OG_IMAGE, OG_DEFAULTS, TWITTER_DEFAULTS } from "@/lib/version";

type Params = { slug: string };

const PROSE = "m-0 max-w-[76ch] text-[14.5px] leading-relaxed text-ink-dim";
const LINK = "text-ink underline-offset-4 hover:text-ink-soft hover:underline";

export function generateStaticParams(): Params[] {
return SIGNALS.map((s) => ({ slug: checkSlug(s.id) }));
}

export async function generateMetadata({ params }: { params: Promise<Params> }): Promise<Metadata> {
const id = signalIdFromSlug((await params).slug);
if (!id) {
return {};
}

const { title, description } = CHECK_CONTENT[id];
const path = checkPath(id);

return {
title,
description,
alternates: { canonical: path },
twitter: { ...TWITTER_DEFAULTS, images: [DEFAULT_OG_IMAGE], title, description },
openGraph: { ...OG_DEFAULTS, images: [DEFAULT_OG_IMAGE], title, description, url: path, type: "article" },
};
}

export default async function CheckPage({ params }: { params: Promise<Params> }) {
const id = signalIdFromSlug((await params).slug);
if (!id) {
notFound();
}

const signal = SIGNAL_BY_ID[id];
const content = CHECK_CONTENT[id];
const stats = getCheckStats(id);
const languages = [...stats.languages].sort((a, b) => fullRate(b.counts) - fullRate(a.counts));

return (
<>
<BreadcrumbJsonLd
parent={{ name: "Checks", path: "/check" }}
current={{ name: signal.label, path: checkPath(id) }}
/>

<Link
href="/check"
className="my-5 inline-flex items-center gap-1.5 text-sm text-muted no-underline hover:text-ink hover:no-underline"
>
← all checks
</Link>

<section className="mb-7">
<h1 className="mb-2.5 text-[28px] font-bold leading-[1.18] tracking-tight">{content.title}</h1>
<p className="m-0 max-w-[72ch] text-[15.5px] leading-relaxed text-ink-dim">
<CodeText text={content.what} />
</p>
</section>

<Panel>
<PanelHeading>How many repos pass</PanelHeading>
<p className={PROSE}>
Of the {stats.total} repos on the leaderboard,{" "}
<strong className="text-ink">
{stats.full} pass in full ({percent(stats.full, stats.total)})
</strong>
, {stats.partial} pass in part ({percent(stats.partial, stats.total)}) and {stats.none} fail (
{percent(stats.none, stats.total)}).
</p>
</Panel>

<div className="mt-3.5">
<Panel>
<PanelHeading>Why agents need it</PanelHeading>
<p className={PROSE}>
<CodeText text={content.why} />
</p>
</Panel>
</div>

<div className="mt-3.5 grid grid-cols-1 items-stretch gap-3.5 md:grid-cols-2">
<Panel className="h-full">
<PanelHeading>How we score it</PanelHeading>
<ul className="m-0 ml-5 list-disc text-[14.5px] leading-relaxed text-ink-dim">
{content.scoring.map((line) => (
<li key={line}>
<CodeText text={line} />
</li>
))}
</ul>
<p className="mt-3 mb-0 text-[13px] text-muted">
The{" "}
<Link href="/methodology" className="text-ink-dim underline-offset-4 hover:text-ink-soft hover:underline">
methodology
</Link>{" "}
has the formula that turns checks into a score.
</p>
</Panel>

<Panel className="h-full">
<PanelHeading>How to pass</PanelHeading>
<p className={PROSE}>
<CodeText text={content.fix} />
</p>
</Panel>
</div>

<div className="mt-3.5 grid grid-cols-1 items-stretch gap-3.5 md:grid-cols-2">
<Panel className="h-full">
<PanelHeading>Which agents count it most</PanelHeading>
<ul className="m-0 flex list-none flex-col gap-1.5 p-0 text-[14.5px] text-ink-dim">
{stats.agents.map(({ model, share }) => (
<li key={model.id} className="flex items-baseline justify-between gap-3">
<Link href={modelPath(model.id)} className={LINK}>
{model.label}
</Link>
<span className="tabular-nums">{percent(share, 1)} of its score</span>
</li>
))}
</ul>
</Panel>

{languages.length > 0 && (
<Panel className="h-full">
<PanelHeading>Pass rate by language</PanelHeading>
<ul className="m-0 flex list-none flex-col gap-1.5 p-0 text-[14.5px] text-ink-dim">
{languages.map((l) => (
<li key={l.slug} className="flex items-baseline justify-between gap-3">
<Link href={hubPath(l.slug)} className={LINK}>
{l.label}
</Link>
<span className="tabular-nums">
{l.counts.full} of {l.counts.total} ({percent(l.counts.full, l.counts.total)})
</span>
</li>
))}
</ul>
</Panel>
)}
</div>

{stats.examples.length > 0 && (
<div className="mt-3.5">
<Panel>
<PanelHeading>Top-scoring repos that pass</PanelHeading>
<CheckRepoList rows={stats.examples} />
</Panel>
</div>
)}

{stats.missing.length > 0 && (
<div className="mt-3.5">
<Panel>
<PanelHeading>Popular repos that fail it</PanelHeading>
<CheckRepoList rows={stats.missing} />
</Panel>
</div>
)}

<nav
aria-label="Other checks"
className="mt-6 flex flex-wrap items-center gap-x-3 gap-y-1.5 text-[13px] text-muted"
>
<span>Other checks:</span>
{SIGNALS.filter((s) => s.id !== id).map((s) => (
<Link key={s.id} href={checkPath(s.id)} className="text-ink-dim hover:text-ink-soft">
{s.label}
</Link>
))}
</nav>
</>
);
}
Loading
Loading