From 9729842d86b7901cb164ddd4fdd6e6a9bc1213e8 Mon Sep 17 00:00:00 2001 From: Conductor Adriaan Date: Sun, 25 Oct 2026 20:09:25 +0000 Subject: [PATCH 1/6] Add a traffic-drop check for script deploys `pnpm run traffic-drop query` prints a Kibana Dev Tools request that compares each website's non-robot pageviews since a deploy with the same window 1-4 weeks earlier. `analyze` reads the response, lists websites with a significant drop (Poisson z-score), and prints follow-up requests for their script versions and script errors. Fixes #143 Co-Authored-By: Claude Opus 5.5 (1M context) --- package.json | 1 + traffic-drop.mts | 297 +++++++++++++++++++++++++++++++++++++++++++++++ tsconfig.json | 2 +- 3 files changed, 299 insertions(+), 1 deletion(-) create mode 100644 traffic-drop.mts diff --git a/package.json b/package.json index f2c27f2..2d2b858 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "build": "node compile.js", "build:testing": "node compile.js testing", "deploy": "node --env-file-if-exists=.env deploy.mts", + "traffic-drop": "node traffic-drop.mts", "pretest": "pnpm run build:testing", "test": "node -r dotenv/config ./test/index.js", "test:unit": "pnpm run build && mocha \"test/unit/*.test.js\" --exit", diff --git a/traffic-drop.mts b/traffic-drop.mts new file mode 100644 index 0000000..01e7525 --- /dev/null +++ b/traffic-drop.mts @@ -0,0 +1,297 @@ +// Finds websites whose pageviews dropped after a script deploy. +// +// pnpm run traffic-drop query --from 2026-10-28T08:00:00Z [--to ] +// Prints a request to paste into Kibana Dev Tools. Copy the whole +// response into a file, for example response.json. +// pnpm run traffic-drop analyze response.json --from --to +// Lists the websites with a significant drop, and prints follow-up +// requests for their script versions and script errors. +// +// The current window (--from until --to) is compared with the same window +// 1-4 weeks earlier, so weekdays and hours line up. Websites with fewer than +// --min-daily pageviews per day in the baseline are ignored. +import { readFileSync } from "node:fs"; +import { parseArgs } from "node:util"; + +const INDEX = "live-datapoints-*"; +const FIELD = "hostname_non_robot_pageview"; // only set on non-robot pageviews +const DAY = 24 * 60 * 60 * 1000; + +const { positionals, values } = parseArgs({ + allowPositionals: true, + options: { + from: { type: "string" }, + to: { type: "string" }, + weeks: { type: "string", default: "4" }, + "min-daily": { type: "string", default: "50" }, + "min-drop": { type: "string", default: "30" }, + "min-z": { type: "string", default: "3" }, + limit: { type: "string", default: "100" }, + }, +}); +const [command, file] = positionals; + +type Window = { name: string; gte: string; lt: string }; +type Request = { path: string; body: unknown }; + +const devTools = ({ path, body }: Request) => + `GET ${path}\n${JSON.stringify(body, null, 2)}\n`; + +function windows(from: string, to: string, weeks: number): Window[] { + const start = Date.parse(from); + const end = Date.parse(to); + if (!(start < end)) throw new Error("--from must be before --to"); + return [0, ...Array.from({ length: weeks }, (_, i) => i + 1)].map((w) => ({ + name: w ? `w${w}` : "current", + gte: new Date(start - w * 7 * DAY).toISOString(), + lt: new Date(end - w * 7 * DAY).toISOString(), + })); +} + +const range = ({ gte, lt }: Window) => ({ + range: { "@timestamp": { gte, lt } }, +}); + +function pageviewsRequest(ws: Window[], minWindow: number): Request { + const baselines = ws.filter((w) => w.name !== "current"); + return { + // filter_path keeps the response small enough to copy from Kibana + path: `${INDEX}/_search?request_cache=true&filter_path=took,timed_out,_shards.failed,aggregations.sites.buckets.key,aggregations.sites.buckets.*.doc_count`, + body: { + size: 0, + track_total_hits: false, + query: { + bool: { + filter: [ + { exists: { field: FIELD } }, + { bool: { should: ws.map(range), minimum_should_match: 1 } }, + ], + }, + }, + aggs: { + sites: { + terms: { + field: FIELD, + size: 65000, + // A website needs at least one baseline window above the minimum + min_doc_count: Math.ceil(minWindow), + }, + aggs: { + ...Object.fromEntries( + ws.map((w) => [w.name, { filter: range(w) }]) + ), + keep: { + bucket_selector: { + buckets_path: Object.fromEntries( + baselines.map((w) => [w.name, `${w.name}>_count`]) + ), + script: { + source: baselines + .map((w) => `params.${w.name} >= params.min`) + .join(" || "), + params: { min: Math.ceil(minWindow) }, + }, + }, + }, + }, + }, + }, + }, + }; +} + +function followUpRequests(hostnames: string[], ws: Window[]): Request[] { + const [current, previous] = ws; + return [ + { + // Which script versions sent pageviews, now and a week earlier + path: `${INDEX}/_search?filter_path=aggregations`, + body: { + size: 0, + track_total_hits: false, + query: { + bool: { + filter: [ + { terms: { [FIELD]: hostnames } }, + { + bool: { + should: [range(current), range(previous)], + minimum_should_match: 1, + }, + }, + ], + }, + }, + aggs: { + sites: { + terms: { field: FIELD, size: hostnames.length }, + aggs: Object.fromEntries( + [current, previous].map((w) => [ + w.name, + { + filter: range(w), + aggs: { + versions: { terms: { field: "meta.script_id", size: 5 } }, + }, + }, + ]) + ), + }, + }, + }, + }, + { + // Errors our scripts reported in the current window + path: `${INDEX}/_search?filter_path=aggregations`, + body: { + size: 0, + track_total_hits: false, + query: { + bool: { + filter: [ + { term: { datapoint: "error" } }, + { terms: { hostname: hostnames } }, + range(current), + ], + }, + }, + aggs: { + sites: { + terms: { field: "hostname", size: hostnames.length }, + aggs: { + versions: { terms: { field: "meta.script_id", size: 5 } }, + example: { + top_hits: { + size: 1, + _source: ["metadata.message", "meta.script_id"], + }, + }, + }, + }, + }, + }, + }, + ]; +} + +const median = (list: number[]) => { + const sorted = [...list].sort((a, b) => a - b); + const middle = Math.floor(sorted.length / 2); + return sorted.length % 2 + ? sorted[middle] + : (sorted[middle - 1] + sorted[middle]) / 2; +}; + +function settings() { + const from = values.from; + if (!from) throw new Error("--from is required, for example the deploy time"); + // Leave a few minutes for ingestion when --to is omitted + const to = + values.to ?? + new Date( + Math.floor((Date.now() - 10 * 60 * 1000) / 60000) * 60000 + ).toISOString(); + const weeks = Number(values.weeks); + const ws = windows(from, to, weeks); + const days = (Date.parse(to) - Date.parse(from)) / DAY; + const minWindow = Number(values["min-daily"]) * days; + return { from, to, weeks, ws, days, minWindow }; +} + +if (command === "query") { + const { ws, days, minWindow } = settings(); + if (days < 1 / 24) console.error("Warning: windows under an hour are noisy"); + console.log( + `# Paste into Kibana Dev Tools. Windows: ${ws.map((w) => `${w.name} ${w.gte} → ${w.lt}`).join(", ")}` + ); + console.log( + `# Websites need ${Math.ceil(minWindow)}+ pageviews in a baseline window (${values["min-daily"]}/day)\n` + ); + console.log(devTools(pageviewsRequest(ws, minWindow))); + console.log( + `# Then: pnpm run traffic-drop analyze response.json --from ${values.from} --to ${ws[0].lt}` + ); +} else if (command === "analyze") { + if (!file) throw new Error("Pass the file with the Kibana response"); + if (!values.to) + throw new Error( + "Pass the same --to as the query, printed at the end of `query`" + ); + const { ws, minWindow } = settings(); + const response = JSON.parse(readFileSync(file, "utf8")); + if (response.timed_out || response._shards?.failed) + console.error( + "Warning: the response is incomplete (timeout or failed shards)" + ); + const buckets: Array> = + response.aggregations?.sites?.buckets ?? []; + const baselineNames = ws + .filter((w) => w.name !== "current") + .map((w) => w.name); + const minDrop = Number(values["min-drop"]) / 100; + const minZ = Number(values["min-z"]); + + const sites = buckets + .map((b) => { + const weeks = baselineNames.map((name) => b[name]?.doc_count ?? 0); + const baseline = median(weeks); + const current = b.current?.doc_count ?? 0; + // Pageview counts are roughly Poisson, so the noise is about √baseline + const z = (current - baseline) / Math.sqrt(Math.max(baseline, 1)); + return { + hostname: b.key as string, + baseline, + current, + weeks, + drop: 1 - current / baseline, + z, + }; + }) + .filter((s) => s.baseline >= minWindow); + + const total = sites.reduce((sum, s) => sum + s.current, 0); + const totalBaseline = sites.reduce((sum, s) => sum + s.baseline, 0); + const flagged = sites + .filter((s) => s.drop >= minDrop && s.z <= -minZ) + .sort((a, b) => b.baseline - b.current - (a.baseline - a.current)); + + console.log( + `${sites.length} websites above ${values["min-daily"]} pageviews/day, ${response.took ?? "?"} ms in Elasticsearch` + ); + console.log( + `All of them together: ${total} pageviews vs ${Math.round(totalBaseline)} baseline (${((total / totalBaseline - 1) * 100).toFixed(1)}%)` + ); + console.log( + `${flagged.length} dropped at least ${values["min-drop"]}% with z ≤ -${values["min-z"]}\n` + ); + if (flagged.length) { + console.table( + flagged.slice(0, Number(values.limit)).map((s) => ({ + hostname: s.hostname, + current: s.current, + baseline: s.baseline, + drop: `${Math.round(s.drop * 100)}%`, + z: Number(s.z.toFixed(1)), + weeks: s.weeks.join(" "), + })) + ); + const [versions, errors] = followUpRequests( + flagged.slice(0, Number(values.limit)).map((s) => s.hostname), + ws + ); + console.log( + "\n# Script versions of these websites, now and a week earlier:\n" + ); + console.log(devTools(versions)); + console.log("# Errors our scripts reported on these websites:\n"); + console.log(devTools(errors)); + } +} else { + console.log( + readFileSync(new URL(import.meta.url), "utf8") + .split("\n") + .slice(0, 12) + .join("\n") + .replace(/^\/\/ ?/gm, "") + ); +} diff --git a/tsconfig.json b/tsconfig.json index f1072ad..6e9618f 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,5 +11,5 @@ "types": ["node"], "verbatimModuleSyntax": true }, - "include": ["deploy.mts", "test/deploy.test.mts"] + "include": ["deploy.mts", "traffic-drop.mts", "test/deploy.test.mts"] } From c36b613130b2668362487667b1a2720eb8d7085b Mon Sep 17 00:00:00 2001 From: Conductor Adriaan Date: Sun, 25 Oct 2026 20:13:00 +0000 Subject: [PATCH 2/6] Refuse incomplete responses and account for weekly variance - Stop on Elasticsearch errors, timeouts, failed shards, empty responses, and responses with other --weeks than the analysis. - Print --weeks and --min-daily in the analyze command. - Use the larger of Poisson noise and the weekly variance, so websites that normally vary a lot aren't flagged. - Include metadata_flattened.message in the error follow-up. Co-Authored-By: Claude Opus 5.5 (1M context) --- traffic-drop.mts | 40 +++++++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/traffic-drop.mts b/traffic-drop.mts index 01e7525..f004a89 100644 --- a/traffic-drop.mts +++ b/traffic-drop.mts @@ -163,7 +163,11 @@ function followUpRequests(hostnames: string[], ws: Window[]): Request[] { example: { top_hits: { size: 1, - _source: ["metadata.message", "meta.script_id"], + _source: [ + "metadata_flattened.message", + "metadata.message", + "meta.script_id", + ], }, }, }, @@ -174,6 +178,14 @@ function followUpRequests(hostnames: string[], ws: Window[]): Request[] { ]; } +const variance = (list: number[]) => { + const mean = list.reduce((sum, value) => sum + value, 0) / list.length; + return ( + list.reduce((sum, value) => sum + (value - mean) ** 2, 0) / + Math.max(list.length - 1, 1) + ); +}; + const median = (list: number[]) => { const sorted = [...list].sort((a, b) => a - b); const middle = Math.floor(sorted.length / 2); @@ -209,7 +221,7 @@ if (command === "query") { ); console.log(devTools(pageviewsRequest(ws, minWindow))); console.log( - `# Then: pnpm run traffic-drop analyze response.json --from ${values.from} --to ${ws[0].lt}` + `# Then: pnpm run traffic-drop analyze response.json --from ${values.from} --to ${ws[0].lt} --weeks ${values.weeks} --min-daily ${values["min-daily"]}` ); } else if (command === "analyze") { if (!file) throw new Error("Pass the file with the Kibana response"); @@ -219,15 +231,23 @@ if (command === "query") { ); const { ws, minWindow } = settings(); const response = JSON.parse(readFileSync(file, "utf8")); - if (response.timed_out || response._shards?.failed) - console.error( - "Warning: the response is incomplete (timeout or failed shards)" + // An incomplete response would make broken websites look healthy + if (response.error || response.timed_out || response._shards?.failed) + throw new Error( + `Elasticsearch didn't answer completely: ${JSON.stringify(response.error ?? response._shards ?? "timed out")}` ); - const buckets: Array> = - response.aggregations?.sites?.buckets ?? []; + const buckets: Array> | undefined = + response.aggregations?.sites?.buckets; + if (!buckets?.length) + throw new Error("The response has no websites; paste the whole response"); const baselineNames = ws .filter((w) => w.name !== "current") .map((w) => w.name); + const missing = ws.filter((w) => !(w.name in buckets[0])); + if (missing.length) + throw new Error( + `The response has no ${missing.map((w) => w.name).join(", ")}; use the same --weeks as the query` + ); const minDrop = Number(values["min-drop"]) / 100; const minZ = Number(values["min-z"]); @@ -236,8 +256,10 @@ if (command === "query") { const weeks = baselineNames.map((name) => b[name]?.doc_count ?? 0); const baseline = median(weeks); const current = b.current?.doc_count ?? 0; - // Pageview counts are roughly Poisson, so the noise is about √baseline - const z = (current - baseline) / Math.sqrt(Math.max(baseline, 1)); + // Pageview counts are roughly Poisson, so the noise is at least + // √baseline; websites that vary more from week to week get that instead + const noise = Math.sqrt(Math.max(baseline, variance(weeks), 1)); + const z = (current - baseline) / noise; return { hostname: b.key as string, baseline, From 4466964b1d3aefa4947463bbf30af45cf4fff25d Mon Sep 17 00:00:00 2001 From: Conductor Adriaan Date: Sun, 25 Oct 2026 20:33:53 +0000 Subject: [PATCH 3/6] Only ask for the deploy time and weeks query now takes --deploy (and optionally --weeks) and compares until 10 minutes ago. The request stores its settings in the aggregation's meta, which Elasticsearch returns as is, so analyze only needs the response. Co-Authored-By: Claude Opus 5.5 (1M context) --- traffic-drop.mts | 99 ++++++++++++++++++++++++++---------------------- 1 file changed, 54 insertions(+), 45 deletions(-) diff --git a/traffic-drop.mts b/traffic-drop.mts index f004a89..70c5fa2 100644 --- a/traffic-drop.mts +++ b/traffic-drop.mts @@ -1,15 +1,16 @@ // Finds websites whose pageviews dropped after a script deploy. // -// pnpm run traffic-drop query --from 2026-10-28T08:00:00Z [--to ] -// Prints a request to paste into Kibana Dev Tools. Copy the whole -// response into a file, for example response.json. -// pnpm run traffic-drop analyze response.json --from --to +// pnpm run traffic-drop query --deploy "2026-10-28 08:00" [--weeks 4] +// Prints a request to paste into Kibana Dev Tools. Save the whole +// response to a file, for example response.json. +// pnpm run traffic-drop analyze response.json // Lists the websites with a significant drop, and prints follow-up // requests for their script versions and script errors. // -// The current window (--from until --to) is compared with the same window -// 1-4 weeks earlier, so weekdays and hours line up. Websites with fewer than -// --min-daily pageviews per day in the baseline are ignored. +// The time since the deploy (until 10 minutes ago) is compared with the same +// hours 1 to --weeks weeks earlier, so weekdays line up. Websites with fewer +// than --min-daily pageviews per day are ignored. The request stores these +// settings, so analyze reads them from the response. import { readFileSync } from "node:fs"; import { parseArgs } from "node:util"; @@ -20,7 +21,7 @@ const DAY = 24 * 60 * 60 * 1000; const { positionals, values } = parseArgs({ allowPositionals: true, options: { - from: { type: "string" }, + deploy: { type: "string" }, to: { type: "string" }, weeks: { type: "string", default: "4" }, "min-daily": { type: "string", default: "50" }, @@ -32,15 +33,16 @@ const { positionals, values } = parseArgs({ const [command, file] = positionals; type Window = { name: string; gte: string; lt: string }; +type Settings = { deploy: string; to: string; weeks: number; minDaily: number }; type Request = { path: string; body: unknown }; const devTools = ({ path, body }: Request) => `GET ${path}\n${JSON.stringify(body, null, 2)}\n`; -function windows(from: string, to: string, weeks: number): Window[] { - const start = Date.parse(from); +function windows({ deploy, to, weeks }: Settings): Window[] { + const start = Date.parse(deploy); const end = Date.parse(to); - if (!(start < end)) throw new Error("--from must be before --to"); + if (!(start < end)) throw new Error("The deploy time must be in the past"); return [0, ...Array.from({ length: weeks }, (_, i) => i + 1)].map((w) => ({ name: w ? `w${w}` : "current", gte: new Date(start - w * 7 * DAY).toISOString(), @@ -52,11 +54,13 @@ const range = ({ gte, lt }: Window) => ({ range: { "@timestamp": { gte, lt } }, }); -function pageviewsRequest(ws: Window[], minWindow: number): Request { +function pageviewsRequest(settings: Settings): Request { + const ws = windows(settings); + const minWindow = minPerWindow(settings); const baselines = ws.filter((w) => w.name !== "current"); return { // filter_path keeps the response small enough to copy from Kibana - path: `${INDEX}/_search?request_cache=true&filter_path=took,timed_out,_shards.failed,aggregations.sites.buckets.key,aggregations.sites.buckets.*.doc_count`, + path: `${INDEX}/_search?request_cache=true&filter_path=took,timed_out,_shards.failed,aggregations.sites.meta,aggregations.sites.buckets.key,aggregations.sites.buckets.*.doc_count`, body: { size: 0, track_total_hits: false, @@ -70,6 +74,8 @@ function pageviewsRequest(ws: Window[], minWindow: number): Request { }, aggs: { sites: { + // Elasticsearch returns meta as is, so analyze knows the settings + meta: settings, terms: { field: FIELD, size: 65000, @@ -194,59 +200,59 @@ const median = (list: number[]) => { : (sorted[middle - 1] + sorted[middle]) / 2; }; -function settings() { - const from = values.from; - if (!from) throw new Error("--from is required, for example the deploy time"); - // Leave a few minutes for ingestion when --to is omitted - const to = - values.to ?? - new Date( - Math.floor((Date.now() - 10 * 60 * 1000) / 60000) * 60000 - ).toISOString(); - const weeks = Number(values.weeks); - const ws = windows(from, to, weeks); - const days = (Date.parse(to) - Date.parse(from)) / DAY; - const minWindow = Number(values["min-daily"]) * days; - return { from, to, weeks, ws, days, minWindow }; -} +const days = ({ deploy, to }: Settings) => + (Date.parse(to) - Date.parse(deploy)) / DAY; + +const minPerWindow = (settings: Settings) => + Math.ceil(settings.minDaily * days(settings)); if (command === "query") { - const { ws, days, minWindow } = settings(); - if (days < 1 / 24) console.error("Warning: windows under an hour are noisy"); - console.log( - `# Paste into Kibana Dev Tools. Windows: ${ws.map((w) => `${w.name} ${w.gte} → ${w.lt}`).join(", ")}` - ); + if (!values.deploy) + throw new Error( + 'Pass the deploy time, for example --deploy "2026-10-28 08:00"' + ); + const settings: Settings = { + deploy: new Date(values.deploy).toISOString(), + // Leave a few minutes for ingestion + to: new Date( + values.to ?? Math.floor((Date.now() - 10 * 60 * 1000) / 60000) * 60000 + ).toISOString(), + weeks: Number(values.weeks), + minDaily: Number(values["min-daily"]), + }; + if (days(settings) < 1 / 24) + console.error("Warning: less than an hour since the deploy is noisy"); console.log( - `# Websites need ${Math.ceil(minWindow)}+ pageviews in a baseline window (${values["min-daily"]}/day)\n` + `# Pageviews from ${settings.deploy} to ${settings.to}, compared with the same hours ${settings.weeks} weeks before.` ); - console.log(devTools(pageviewsRequest(ws, minWindow))); console.log( - `# Then: pnpm run traffic-drop analyze response.json --from ${values.from} --to ${ws[0].lt} --weeks ${values.weeks} --min-daily ${values["min-daily"]}` + `# Paste this into Kibana Dev Tools and save the response, then run: pnpm run traffic-drop analyze response.json\n` ); + console.log(devTools(pageviewsRequest(settings))); } else if (command === "analyze") { if (!file) throw new Error("Pass the file with the Kibana response"); - if (!values.to) - throw new Error( - "Pass the same --to as the query, printed at the end of `query`" - ); - const { ws, minWindow } = settings(); const response = JSON.parse(readFileSync(file, "utf8")); // An incomplete response would make broken websites look healthy if (response.error || response.timed_out || response._shards?.failed) throw new Error( `Elasticsearch didn't answer completely: ${JSON.stringify(response.error ?? response._shards ?? "timed out")}` ); + const settings: Settings | undefined = response.aggregations?.sites?.meta; const buckets: Array> | undefined = response.aggregations?.sites?.buckets; - if (!buckets?.length) - throw new Error("The response has no websites; paste the whole response"); + if (!settings || !buckets?.length) + throw new Error( + "The response has no websites; save the whole response of the query" + ); + const ws = windows(settings); + const minWindow = minPerWindow(settings); const baselineNames = ws .filter((w) => w.name !== "current") .map((w) => w.name); const missing = ws.filter((w) => !(w.name in buckets[0])); if (missing.length) throw new Error( - `The response has no ${missing.map((w) => w.name).join(", ")}; use the same --weeks as the query` + `The response has no ${missing.map((w) => w.name).join(", ")}; run the query again` ); const minDrop = Number(values["min-drop"]) / 100; const minZ = Number(values["min-z"]); @@ -278,7 +284,10 @@ if (command === "query") { .sort((a, b) => b.baseline - b.current - (a.baseline - a.current)); console.log( - `${sites.length} websites above ${values["min-daily"]} pageviews/day, ${response.took ?? "?"} ms in Elasticsearch` + `Pageviews from ${settings.deploy} to ${settings.to}, compared with the same hours ${settings.weeks} weeks before` + ); + console.log( + `${sites.length} websites above ${settings.minDaily} pageviews/day, ${response.took ?? "?"} ms in Elasticsearch` ); console.log( `All of them together: ${total} pageviews vs ${Math.round(totalBaseline)} baseline (${((total / totalBaseline - 1) * 100).toFixed(1)}%)` From 81b34e87750138250083f258433243ec67b45ba1 Mon Sep 17 00:00:00 2001 From: Conductor Adriaan Date: Wed, 28 Oct 2026 21:38:07 +0000 Subject: [PATCH 4/6] Take the deploy time from GitHub Deployments Without --deploy, query reads the newest successful deployment of cdn and custom-domains and starts at the first destination that got that release. GH_TOKEN or GITHUB_TOKEN avoids the unauthenticated rate limit. Co-Authored-By: Claude Opus 5.5 (1M context) --- traffic-drop.mts | 71 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 6 deletions(-) diff --git a/traffic-drop.mts b/traffic-drop.mts index 70c5fa2..d8b2756 100644 --- a/traffic-drop.mts +++ b/traffic-drop.mts @@ -1,8 +1,9 @@ // Finds websites whose pageviews dropped after a script deploy. // -// pnpm run traffic-drop query --deploy "2026-10-28 08:00" [--weeks 4] +// pnpm run traffic-drop query [--weeks 4] [--deploy "2026-10-28 08:00"] // Prints a request to paste into Kibana Dev Tools. Save the whole -// response to a file, for example response.json. +// response to a file, for example response.json. Without --deploy, it +// uses the last deploy recorded in GitHub Deployments. // pnpm run traffic-drop analyze response.json // Lists the websites with a significant drop, and prints follow-up // requests for their script versions and script errors. @@ -17,6 +18,10 @@ import { parseArgs } from "node:util"; const INDEX = "live-datapoints-*"; const FIELD = "hostname_non_robot_pageview"; // only set on non-robot pageviews const DAY = 24 * 60 * 60 * 1000; +// GitHub deployment environments that deploy.mts records for latest.js +const GITHUB_DEPLOYMENTS = + "https://api.github.com/repos/simpleanalytics/scripts/deployments"; +const ENVIRONMENTS = ["cdn", "custom-domains"]; const { positionals, values } = parseArgs({ allowPositionals: true, @@ -206,13 +211,67 @@ const days = ({ deploy, to }: Settings) => const minPerWindow = (settings: Settings) => Math.ceil(settings.minDaily * days(settings)); -if (command === "query") { - if (!values.deploy) +type Deployment = { environment: string; ref: string; at: string }; + +// Public data, but a token avoids the 60 requests per hour limit +const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN; + +const readGitHub = async (url: string): Promise => { + const response = await fetch(url, { + headers: { + Accept: "application/vnd.github+json", + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }, + }); + if (!response.ok) throw new Error( - 'Pass the deploy time, for example --deploy "2026-10-28 08:00"' + `GitHub answered HTTP ${response.status} for ${url}. Set GH_TOKEN or pass --deploy.` ); + return (await response.json()) as T; +}; + +// The newest successful deployment per environment +async function lastDeployments(): Promise { + const found: Deployment[] = []; + for (const environment of ENVIRONMENTS) { + const deployments = await readGitHub< + Array<{ id: number; ref: string; created_at: string }> + >(`${GITHUB_DEPLOYMENTS}?environment=${environment}&per_page=10`); + deployments.sort((a, b) => b.created_at.localeCompare(a.created_at)); + for (const { id, ref } of deployments) { + const [status] = await readGitHub< + Array<{ state: string; created_at: string }> + >(`${GITHUB_DEPLOYMENTS}/${id}/statuses?per_page=1`); + if (status?.state !== "success") continue; + found.push({ environment, ref, at: status.created_at }); + break; + } + } + return found; +} + +// The release deployed last, from the first destination that got it +async function deployTime(): Promise { + const deployments = await lastDeployments(); + const newest = deployments.reduce( + (latest, d) => (!latest || d.at > latest.at ? d : latest), + undefined + ); + if (!newest) + throw new Error( + 'No deployment is recorded in GitHub, pass --deploy "2026-10-28 08:00"' + ); + const sameRelease = deployments.filter((d) => d.ref === newest.ref); + const first = sameRelease.reduce((a, b) => (a.at < b.at ? a : b)); + console.log( + `# Last deploy: ${newest.ref}, ${sameRelease.map((d) => `${d.environment} at ${d.at}`).join(", ")}` + ); + return first.at; +} + +if (command === "query") { const settings: Settings = { - deploy: new Date(values.deploy).toISOString(), + deploy: new Date(values.deploy ?? (await deployTime())).toISOString(), // Leave a few minutes for ingestion to: new Date( values.to ?? Math.floor((Date.now() - 10 * 60 * 1000) / 60000) * 60000 From b0b23e454301de5eb41bcc996817d50135f234a3 Mon Sep 17 00:00:00 2001 From: Conductor Adriaan Date: Wed, 28 Oct 2026 21:46:11 +0000 Subject: [PATCH 5/6] Use the newest deployment and read up to 100 per environment Comparing from the newest successful deployment on any destination avoids picking an older rollout of the same release after a rollback. Read up to 100 deployments per environment and warn when one has no successful deployment. Co-Authored-By: Claude Opus 5.5 (1M context) --- traffic-drop.mts | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/traffic-drop.mts b/traffic-drop.mts index d8b2756..4f6c1d7 100644 --- a/traffic-drop.mts +++ b/traffic-drop.mts @@ -3,7 +3,7 @@ // pnpm run traffic-drop query [--weeks 4] [--deploy "2026-10-28 08:00"] // Prints a request to paste into Kibana Dev Tools. Save the whole // response to a file, for example response.json. Without --deploy, it -// uses the last deploy recorded in GitHub Deployments. +// uses the newest deploy recorded in GitHub Deployments. // pnpm run traffic-drop analyze response.json // Lists the websites with a significant drop, and prints follow-up // requests for their script versions and script errors. @@ -236,37 +236,39 @@ async function lastDeployments(): Promise { for (const environment of ENVIRONMENTS) { const deployments = await readGitHub< Array<{ id: number; ref: string; created_at: string }> - >(`${GITHUB_DEPLOYMENTS}?environment=${environment}&per_page=10`); + >(`${GITHUB_DEPLOYMENTS}?environment=${environment}&per_page=100`); deployments.sort((a, b) => b.created_at.localeCompare(a.created_at)); + let success = false; for (const { id, ref } of deployments) { const [status] = await readGitHub< Array<{ state: string; created_at: string }> >(`${GITHUB_DEPLOYMENTS}/${id}/statuses?per_page=1`); if (status?.state !== "success") continue; found.push({ environment, ref, at: status.created_at }); + success = true; break; } + if (deployments.length && !success) + console.error( + `Warning: no successful deployment found for ${environment}` + ); } return found; } -// The release deployed last, from the first destination that got it +// The newest successful deployment on any destination. Comparing from the +// newest one also covers rollbacks and destinations deployed days apart. async function deployTime(): Promise { const deployments = await lastDeployments(); - const newest = deployments.reduce( - (latest, d) => (!latest || d.at > latest.at ? d : latest), - undefined - ); - if (!newest) + if (!deployments.length) throw new Error( 'No deployment is recorded in GitHub, pass --deploy "2026-10-28 08:00"' ); - const sameRelease = deployments.filter((d) => d.ref === newest.ref); - const first = sameRelease.reduce((a, b) => (a.at < b.at ? a : b)); + const newest = deployments.reduce((a, b) => (a.at > b.at ? a : b)); console.log( - `# Last deploy: ${newest.ref}, ${sameRelease.map((d) => `${d.environment} at ${d.at}`).join(", ")}` + `# Last deploys: ${deployments.map((d) => `${d.environment} ${d.ref} at ${d.at}`).join(", ")}. Using ${newest.environment}.` ); - return first.at; + return newest.at; } if (command === "query") { From 91b695a5fe4b209c69ce63cf1f90a8b73ebc1b48 Mon Sep 17 00:00:00 2001 From: Conductor Adriaan Date: Thu, 29 Oct 2026 17:11:17 +0000 Subject: [PATCH 6/6] Address review feedback on traffic-drop - Explain a deploy newer than the window end instead of "must be in the past". - Say "timed out" for a timeout, and check every website's counts. - Print the full help text. - Retry without the token when GitHub rejects GH_TOKEN. Co-Authored-By: Claude Opus 5.5 (1M context) --- traffic-drop.mts | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/traffic-drop.mts b/traffic-drop.mts index 4f6c1d7..814c8f4 100644 --- a/traffic-drop.mts +++ b/traffic-drop.mts @@ -47,7 +47,10 @@ const devTools = ({ path, body }: Request) => function windows({ deploy, to, weeks }: Settings): Window[] { const start = Date.parse(deploy); const end = Date.parse(to); - if (!(start < end)) throw new Error("The deploy time must be in the past"); + if (!(start < end)) + throw new Error( + `The deploy at ${deploy} is newer than ${to}; wait a few minutes after a deploy so there is a window to measure` + ); return [0, ...Array.from({ length: weeks }, (_, i) => i + 1)].map((w) => ({ name: w ? `w${w}` : "current", gte: new Date(start - w * 7 * DAY).toISOString(), @@ -217,12 +220,16 @@ type Deployment = { environment: string; ref: string; at: string }; const token = process.env.GH_TOKEN || process.env.GITHUB_TOKEN; const readGitHub = async (url: string): Promise => { - const response = await fetch(url, { - headers: { - Accept: "application/vnd.github+json", - ...(token ? { Authorization: `Bearer ${token}` } : {}), - }, - }); + const read = (withToken: boolean) => + fetch(url, { + headers: { + Accept: "application/vnd.github+json", + ...(withToken && token ? { Authorization: `Bearer ${token}` } : {}), + }, + }); + let response = await read(true); + // A stale token must not block reading public data + if (token && response.status === 401) response = await read(false); if (!response.ok) throw new Error( `GitHub answered HTTP ${response.status} for ${url}. Set GH_TOKEN or pass --deploy.` @@ -296,7 +303,7 @@ if (command === "query") { // An incomplete response would make broken websites look healthy if (response.error || response.timed_out || response._shards?.failed) throw new Error( - `Elasticsearch didn't answer completely: ${JSON.stringify(response.error ?? response._shards ?? "timed out")}` + `Elasticsearch didn't answer completely: ${JSON.stringify(response.error ?? (response.timed_out ? "timed out" : response._shards))}` ); const settings: Settings | undefined = response.aggregations?.sites?.meta; const buckets: Array> | undefined = @@ -310,10 +317,13 @@ if (command === "query") { const baselineNames = ws .filter((w) => w.name !== "current") .map((w) => w.name); - const missing = ws.filter((w) => !(w.name in buckets[0])); - if (missing.length) + // A missing count would read as zero and look like a drop + const incomplete = buckets.find((b) => + ws.some((w) => typeof b[w.name]?.doc_count !== "number") + ); + if (incomplete) throw new Error( - `The response has no ${missing.map((w) => w.name).join(", ")}; run the query again` + `The response has no counts for every window (${incomplete.key}); run the query again` ); const minDrop = Number(values["min-drop"]) / 100; const minZ = Number(values["min-z"]); @@ -381,9 +391,7 @@ if (command === "query") { } else { console.log( readFileSync(new URL(import.meta.url), "utf8") - .split("\n") - .slice(0, 12) - .join("\n") + .split("\nimport ")[0] .replace(/^\/\/ ?/gm, "") ); }