Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@ The website code lives under `src/`:
- `src/client/` — browser-side TypeScript (search, filter, dark mode), Tailwind entry, and the vendored Outfit fonts under `fonts/` that social cards are rendered with.
- `src/build/` — SSG pipeline (renders pages, compiles assets, emits JSON API, generates a social card per page).
- `src/server/` — Express dev server.
- `src/tracking/` — API usage tracking; the root `middleware.ts` posts one Web Analytics custom event per JSON API request at the edge.
- `src/tracking/` — classifies API paths for the Worker's usage log line in Workers Logs.

Everything the catalog ships beyond the static site:

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ curl https://modelparams.dev/api/v1/models/openai/gpt-5.5.json

Schema at `https://modelparams.dev/api/v1/schema.json`, per the [Model Parameters convention](docs/model-parameters-schema.md).

API requests are counted at the edge and reported to Vercel Web Analytics as `api_request` custom events (endpoint, model, client type), so API usage lands in the same dashboard as page views. On Cloudflare Workers the Worker logs the same event as a JSON line in Workers Logs instead; see [`src/tracking/api-usage.ts`](src/tracking/api-usage.ts).
Requests that reach the Cloudflare Worker (MCP, validate) are logged as an `api_request` JSON line in Workers Logs (endpoint, model, auth mode); see [`src/tracking/api-usage.ts`](src/tracking/api-usage.ts). Static API files are served straight from Cloudflare's assets and are not logged.

### Validate a request

Expand Down
22 changes: 0 additions & 22 deletions middleware.ts

This file was deleted.

2 changes: 1 addition & 1 deletion src/build/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ export async function build(): Promise<{ models: number }> {
const catalog = buildCatalog(models);
const capabilities = buildCapabilityFacets(models);
console.log(`Rendering HTML for ${models.length} model(s)...`);
const html = await renderIndex({ catalog, capabilities, analytics: true });
const html = await renderIndex({ catalog, capabilities });
await fs.writeFile(path.join(DIST_DIR, "index.html"), html, "utf8");

console.log("Writing JSON API...");
Expand Down
5 changes: 0 additions & 5 deletions src/build/render.ts
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,6 @@ export interface ShellMeta {
/** Robots directive; omitted (index, follow) for every indexable page. */
robots?: string;
initialThemeClass?: string;
analytics?: boolean;
}

/** Wrap a rendered body in the shared HTML layout (head, header, footer, modal). */
Expand All @@ -111,7 +110,6 @@ export async function renderShell(meta: ShellMeta, body: string): Promise<string
helpers: viewHelpers,
usageGuide: usageGuideMarkdown(SITE_URL),
initialThemeClass: meta.initialThemeClass ?? "",
analytics: meta.analytics ?? false,
body,
});
}
Expand All @@ -120,8 +118,6 @@ export interface RenderOptions {
catalog: Catalog;
capabilities: CapabilityFacet[];
initialThemeClass?: string;
/** Inject the Vercel Web Analytics snippet. Enabled for production builds; off in dev. */
analytics?: boolean;
}

/**
Expand Down Expand Up @@ -191,7 +187,6 @@ export async function renderIndex(opts: RenderOptions): Promise<string> {
),
providerHubs: hubLinks(opts.catalog.models),
initialThemeClass: opts.initialThemeClass,
analytics: opts.analytics,
},
body,
);
Expand Down
121 changes: 4 additions & 117 deletions src/tracking/api-usage.ts
Original file line number Diff line number Diff line change
@@ -1,32 +1,9 @@
/**
* Server-side usage tracking for the JSON API, reported to Vercel Web
* Analytics so API traffic shows up in the same dashboard as page views.
*
* The API is static files on Vercel's CDN, so the Web Analytics <script> in
* layout.ejs never sees those requests: curl and SDK clients don't execute
* JavaScript. Instead, the root middleware.ts observes every /api/* request
* at the edge and posts an "api_request" custom event to this deployment's
* ingest route (/_vercel/insights/event).
*
* The payload replicates what @vercel/analytics@2.0.1's server track() sends
* (see its dist/server/index.js). We inline it rather than depend on the
* package because its optional framework peer deps (@sveltejs/kit → vite 8)
* conflict with this repo's vite 5 toolchain and would break plain
* `npm install` for contributors.
*
* No configuration: Vercel injects VERCEL_URL, and without it (local dev,
* tests, forks deployed elsewhere) tracking is a no-op. Viewing custom events
* in the Analytics tab requires a Pro plan.
*
* Runs in the Edge runtime: only Web APIs here (fetch, URL), no Node
* built-ins.
* Classifies API paths for the usage log line the Cloudflare Worker writes to
* Workers Logs (see worker/index.ts): which endpoint, auth mode, provider and
* model a request was for.
*/

const SDK_NAME = "@vercel/analytics/server";
const SDK_VERSION = "2.0.1";

export type TrackingEnv = Record<string, string | undefined>;

export interface EndpointInfo {
endpoint:
| "index"
Expand All @@ -49,8 +26,7 @@ export function classifyEndpoint(pathname: string): EndpointInfo {
const path = pathname.length > 1 ? pathname.replace(/\/$/, "") : pathname;

if (path === "/llms.txt" || path === "/llms-full.txt") return { ...none, endpoint: "llms" };
// vercel.json rewrites /mcp to /api/mcp; middleware sees the pre-rewrite
// path, but direct hits on /api/mcp are the same endpoint.
// /mcp is the public path; direct hits on /api/mcp are the same endpoint.
if (path === "/mcp" || path === "/api/mcp") return { ...none, endpoint: "mcp" };
if (path === "/api/v1/index.json") return { ...none, endpoint: "index" };
if (path === "/api/v1/models.json") return { ...none, endpoint: "catalog" };
Expand Down Expand Up @@ -78,92 +54,3 @@ function splitAuthMode(slug: string): { model: string; authMode: "api_key" | "su
? { model: slug.slice(0, -"-subscription".length), authMode: "subscription" }
: { model: slug, authMode: "api_key" };
}

/** Coarse consumer bucket, so the events panel can split SDKs from browsers. */
export function classifyClient(userAgent: string | null): string {
if (!userAgent) return "unknown";
if (/curl\//i.test(userAgent)) return "curl";
if (/wget/i.test(userAgent)) return "wget";
if (/python|httpx|aiohttp/i.test(userAgent)) return "python";
if (/node|undici|axios|\bgot\b/i.test(userAgent)) return "node";
if (/go-http-client|golang/i.test(userAgent)) return "go";
if (/bot|crawler|spider|scrape/i.test(userAgent)) return "bot";
if (/mozilla|chrome|safari|firefox/i.test(userAgent)) return "browser";
return "other";
}

/**
* Custom-event properties. Only present values are sent: Vercel accepts null
* but renders it as noise in the Events panel, and every name/key/value is
* capped at 255 characters.
*/
export function eventProperties(request: Request): Record<string, string> {
const url = new URL(request.url);
const info = classifyEndpoint(url.pathname);
const properties: Record<string, string> = {
path: url.pathname.slice(0, 255),
method: request.method,
endpoint: info.endpoint,
client: classifyClient(request.headers.get("user-agent")),
};
if (info.authMode) properties.auth_mode = info.authMode;
if (info.provider) properties.provider = info.provider;
if (info.model) properties.model = info.model;
return properties;
}

/** Ingest endpoint on this deployment, or null when not running on Vercel. */
export function ingestUrl(env: TrackingEnv): string | null {
const endpoint = env.VERCEL_WEB_ANALYTICS_ENDPOINT || env.VERCEL_URL;
if (!endpoint) return null;
return endpoint.startsWith("http")
? endpoint
: new URL("/_vercel/insights/event", `https://${endpoint}`).toString();
}

/**
* Fire-and-forget capture. Returns the in-flight promise for waitUntil(), or
* null when tracking is off (not on Vercel) or the request is a CORS
* preflight. Never throws and never rejects: an analytics outage must not
* affect the API.
*/
export function trackApiUsage(
request: Request,
env: TrackingEnv = process.env,
): Promise<void> | null {
const url = ingestUrl(env);
if (!url) return null;
if (request.method === "OPTIONS") return null;

// Attribution headers: the ingest route derives visitor, country, and
// browser from these, exactly as it does for @vercel/analytics/server.
const headers: Record<string, string> = {
"content-type": "application/json",
"x-va-server": "1",
};
const userAgent = request.headers.get("user-agent");
if (userAgent) headers["user-agent"] = userAgent;
const forwardedFor = request.headers.get("x-forwarded-for");
if (forwardedFor) headers["x-vercel-ip"] = forwardedFor;
const cookie = request.headers.get("cookie");
if (cookie) headers.cookie = cookie;
// Lets events through on password/SSO-protected preview deployments.
if (env.VERCEL_AUTOMATION_BYPASS_SECRET) {
headers["x-vercel-protection-bypass"] = env.VERCEL_AUTOMATION_BYPASS_SECRET;
}

const body = JSON.stringify({
o: request.url,
ts: Date.now(),
sdkn: SDK_NAME,
sdkv: SDK_VERSION,
r: "",
en: "api_request",
ed: eventProperties(request),
});

return fetch(url, { method: "POST", headers, body }).then(
() => undefined,
() => undefined,
);
}
11 changes: 0 additions & 11 deletions src/views/layout.ejs
Original file line number Diff line number Diff line change
Expand Up @@ -61,16 +61,5 @@
<%- structuredData %>
</script>
<script type="module" src="/assets/main.js"></script>
<% if (analytics) { %>
<!-- Vercel Web Analytics -->
<script>
window.va =
window.va ||
function () {
(window.vaq = window.vaq || []).push(arguments);
};
</script>
<script defer src="/_vercel/insights/script.js"></script>
<% } %>
</body>
</html>
149 changes: 2 additions & 147 deletions tests/api-usage.test.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,5 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import {
classifyClient,
classifyEndpoint,
eventProperties,
ingestUrl,
trackApiUsage,
} from "../src/tracking/api-usage.js";
import { describe, expect, it } from "vitest";
import { classifyEndpoint } from "../src/tracking/api-usage.js";

describe("classifyEndpoint", () => {
it("recognises the fixed endpoints", () => {
Expand Down Expand Up @@ -62,142 +56,3 @@ describe("classifyEndpoint", () => {
expect(classifyEndpoint("/").endpoint).toBe("other");
});
});

describe("classifyClient", () => {
it.each([
["curl/8.6.0", "curl"],
["Wget/1.21", "wget"],
["python-requests/2.32.0", "python"],
["python-httpx/0.27.0", "python"],
["node-fetch/3.3.2", "node"],
["undici", "node"],
["axios/1.7.2", "node"],
["Go-http-client/2.0", "go"],
["Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)", "bot"],
["Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/126.0 Safari/537.36", "browser"],
["SomethingElse/1.0", "other"],
])("classifies %s as %s", (userAgent, expected) => {
expect(classifyClient(userAgent)).toBe(expected);
});

it("treats a missing user agent as unknown", () => {
expect(classifyClient(null)).toBe("unknown");
expect(classifyClient("")).toBe("unknown");
});
});

describe("eventProperties", () => {
it("describes a model params request", () => {
const request = new Request("https://modelparams.dev/api/v1/params/gpt-5.5.json", {
headers: { "user-agent": "python-requests/2.32.0" },
});
expect(eventProperties(request)).toEqual({
path: "/api/v1/params/gpt-5.5.json",
method: "GET",
endpoint: "params_by_model",
client: "python",
auth_mode: "api_key",
model: "gpt-5.5",
});
});

it("omits absent values instead of sending null", () => {
const request = new Request("https://modelparams.dev/api/v1/models.json");
expect(eventProperties(request)).toEqual({
path: "/api/v1/models.json",
method: "GET",
endpoint: "catalog",
client: "unknown",
});
});
});

describe("ingestUrl", () => {
it("builds the endpoint from VERCEL_URL and is off without it", () => {
expect(ingestUrl({ VERCEL_URL: "modelparams-abc123.vercel.app" })).toBe(
"https://modelparams-abc123.vercel.app/_vercel/insights/event",
);
expect(ingestUrl({})).toBe(null);
});

it("uses a full VERCEL_WEB_ANALYTICS_ENDPOINT override verbatim", () => {
expect(
ingestUrl({
VERCEL_WEB_ANALYTICS_ENDPOINT: "https://example.com/ingest",
VERCEL_URL: "ignored.vercel.app",
}),
).toBe("https://example.com/ingest");
});
});

describe("trackApiUsage", () => {
afterEach(() => {
vi.unstubAllGlobals();
});

const env = { VERCEL_URL: "modelparams-abc123.vercel.app" };
const request = (method = "GET") =>
new Request("https://modelparams.dev/api/v1/models.json", {
method,
headers: {
"user-agent": "curl/8.6.0",
"x-forwarded-for": "203.0.113.7",
},
});

it("is a no-op off Vercel and for CORS preflights", () => {
const fetchMock = vi.fn();
vi.stubGlobal("fetch", fetchMock);
expect(trackApiUsage(request(), {})).toBe(null);
expect(trackApiUsage(request("OPTIONS"), env)).toBe(null);
expect(fetchMock).not.toHaveBeenCalled();
});

it("posts the @vercel/analytics wire format to the ingest route", async () => {
const fetchMock = vi.fn().mockResolvedValue(new Response("ok"));
vi.stubGlobal("fetch", fetchMock);

await trackApiUsage(request(), env);

expect(fetchMock).toHaveBeenCalledTimes(1);
const [url, init] = fetchMock.mock.calls[0] as [
string,
{ method: string; headers: Record<string, string>; body: string },
];
expect(url).toBe("https://modelparams-abc123.vercel.app/_vercel/insights/event");
expect(init.method).toBe("POST");
expect(init.headers).toMatchObject({
"content-type": "application/json",
"x-va-server": "1",
"user-agent": "curl/8.6.0",
"x-vercel-ip": "203.0.113.7",
});
expect(init.headers).not.toHaveProperty("cookie");

const body = JSON.parse(init.body) as {
en: string;
o: string;
sdkn: string;
ed: Record<string, string>;
};
expect(body.en).toBe("api_request");
expect(body.o).toBe("https://modelparams.dev/api/v1/models.json");
expect(body.sdkn).toBe("@vercel/analytics/server");
expect(body.ed).toMatchObject({ endpoint: "catalog", client: "curl" });
});

it("sends the protection bypass secret when configured", async () => {
const fetchMock = vi.fn().mockResolvedValue(new Response("ok"));
vi.stubGlobal("fetch", fetchMock);

await trackApiUsage(request(), { ...env, VERCEL_AUTOMATION_BYPASS_SECRET: "shh" });

const [, init] = fetchMock.mock.calls[0] as [string, { headers: Record<string, string> }];
expect(init.headers["x-vercel-protection-bypass"]).toBe("shh");
});

it("swallows network failures", async () => {
vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("offline")));
await expect(trackApiUsage(request(), env)).resolves.toBe(undefined);
});
});
Loading
Loading