Skip to content

Only blame a deploy for drops that started after it #152

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adriaandotcom merged 2 commits into from
Oct 29, 2026
Merged

Only blame a deploy for drops that started after it #152

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ed3ba11
Only flag drops that started after the deploy
Oct 29, 2026
89c7995
Check the hour before the deploy and say when a drop started
Oct 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
148 changes: 104 additions & 44 deletions traffic-drop.mts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,14 +10,16 @@
//
// The time since the deploy (until 10 minutes ago) is compared with the same
// hours 1 to --weeks weeks earlier, so weekdays line up. Websites with fewer
// than --min-daily pageviews per day are ignored. The request stores these
// settings, so analyze reads them from the response.
// than --min-daily pageviews per day are ignored. The hour before the deploy is
// checked too, so a drop that started earlier isn't blamed on the deploy. The request stores these settings, so analyze reads them from
// the response.
import { readFileSync } from "node:fs";
import { parseArgs } from "node:util";

const INDEX = "live-datapoints-*";
const FIELD = "hostname_non_robot_pageview"; // only set on non-robot pageviews
const DAY = 24 * 60 * 60 * 1000;
const HOUR = 60 * 60 * 1000;
const DAY = 24 * HOUR;
// GitHub deployment environments that deploy.mts records for latest.js
const GITHUB_DEPLOYMENTS =
"https://api.github.com/repos/simpleanalytics/scripts/deployments";
Expand Down Expand Up @@ -51,11 +53,24 @@ function windows({ deploy, to, weeks }: Settings): Window[] {
throw new Error(
`The deploy at ${deploy} is newer than ${to}; wait a few minutes after a deploy so there is a window to measure`
);
return [0, ...Array.from({ length: weeks }, (_, i) => i + 1)].map((w) => ({
name: w ? `w${w}` : "current",
gte: new Date(start - w * 7 * DAY).toISOString(),
lt: new Date(end - w * 7 * DAY).toISOString(),
}));
const shifts = [0, ...Array.from({ length: weeks }, (_, i) => i + 1)];
const at = (time: number, w: number) =>
new Date(time - w * 7 * DAY).toISOString();
// current/w1..wN cover the time since the deploy, before/bw1..bwN the hour
// before it (shorter windows only see when a drop started more sharply)
const beforeStart = start - Math.min(end - start, HOUR);
return [
...shifts.map((w) => ({
name: w ? `w${w}` : "current",
gte: at(start, w),
lt: at(end, w),
})),
...shifts.map((w) => ({
name: w ? `bw${w}` : "before",
gte: at(beforeStart, w),
lt: at(start, w),
})),
];
}

const range = ({ gte, lt }: Window) => ({
Expand All @@ -65,7 +80,7 @@ const range = ({ gte, lt }: Window) => ({
function pageviewsRequest(settings: Settings): Request {
const ws = windows(settings);
const minWindow = minPerWindow(settings);
const baselines = ws.filter((w) => w.name !== "current");
const baselines = ws.filter((w) => /^w\d+$/.test(w.name));
return {
// filter_path keeps the response small enough to copy from Kibana
path: `${INDEX}/_search?request_cache=true&filter_path=took,timed_out,_shards.failed,aggregations.sites.meta,aggregations.sites.buckets.key,aggregations.sites.buckets.*.doc_count`,
Expand Down Expand Up @@ -291,7 +306,7 @@ if (command === "query") {
if (days(settings) < 1 / 24)
console.error("Warning: less than an hour since the deploy is noisy");
console.log(
`# Pageviews from ${settings.deploy} to ${settings.to}, compared with the same hours ${settings.weeks} weeks before.`
`# Pageviews from ${settings.deploy} to ${settings.to} and the hour before the deploy, compared with the same hours ${settings.weeks} weeks before.`
);
console.log(
`# Paste this into Kibana Dev Tools and save the response, then run: pnpm run traffic-drop analyze response.json\n`
Expand All @@ -314,9 +329,10 @@ if (command === "query") {
);
const ws = windows(settings);
const minWindow = minPerWindow(settings);
const baselineNames = ws
.filter((w) => w.name !== "current")
.map((w) => w.name);
const names = (pattern: RegExp) =>
ws.filter((w) => pattern.test(w.name)).map((w) => w.name);
const afterWeeks = names(/^w\d+$/);
const beforeWeeks = names(/^bw\d+$/);
// A missing count would read as zero and look like a drop
const incomplete = buckets.find((b) =>
ws.some((w) => typeof b[w.name]?.doc_count !== "number")
Expand All @@ -328,34 +344,63 @@ if (command === "query") {
const minDrop = Number(values["min-drop"]) / 100;
const minZ = Number(values["min-z"]);

// Compares a window with the median of the same window in earlier weeks
const compare = (
b: Record<string, any>,
name: string,
weekNames: string[]
) => {
const weeks = weekNames.map((week) => b[week].doc_count as number);
const baseline = median(weeks);
const count = b[name].doc_count as number;
// Pageview counts are roughly Poisson, so the noise is at least
// √baseline; websites that vary more from week to week get that instead
const noise = Math.sqrt(Math.max(baseline, variance(weeks), 1));
const z = (count - baseline) / noise;
const dropped = 1 - count / baseline >= minDrop && z <= -minZ;
return { count, baseline, weeks, drop: 1 - count / baseline, z, dropped };
};

const sites = buckets
.map((b) => {
const weeks = baselineNames.map((name) => b[name]?.doc_count ?? 0);
const baseline = median(weeks);
const current = b.current?.doc_count ?? 0;
// Pageview counts are roughly Poisson, so the noise is at least
// √baseline; websites that vary more from week to week get that instead
const noise = Math.sqrt(Math.max(baseline, variance(weeks), 1));
const z = (current - baseline) / noise;
return {
hostname: b.key as string,
baseline,
current,
weeks,
drop: 1 - current / baseline,
z,
};
})
.filter((s) => s.baseline >= minWindow);
.map((b) => ({
hostname: b.key as string,
after: compare(b, "current", afterWeeks),
before: compare(b, "before", beforeWeeks),
}))
.filter((s) => s.after.baseline >= minWindow);

const total = sites.reduce((sum, s) => sum + s.current, 0);
const totalBaseline = sites.reduce((sum, s) => sum + s.baseline, 0);
const flagged = sites
.filter((s) => s.drop >= minDrop && s.z <= -minZ)
.sort((a, b) => b.baseline - b.current - (a.baseline - a.current));
const total = sites.reduce((sum, s) => sum + s.after.count, 0);
const totalBaseline = sites.reduce((sum, s) => sum + s.after.baseline, 0);
const missing = (s: (typeof sites)[number]) =>
s.after.baseline - s.after.count;
const dropped = sites
.filter((s) => s.after.dropped)
.sort((a, b) => missing(b) - missing(a));
// When did the drop start? Only one that started or got worse after the
// deploy can be caused by it
const when = (s: (typeof sites)[number]) => {
if (s.before.dropped) {
// Already down before: if after kept the same share of the two windows,
// its count is binomial, so a much smaller count means it got worse
const total = s.after.count + s.before.count;
const share = s.after.baseline / (s.after.baseline + s.before.baseline);
const spread = Math.sqrt(total * share * (1 - share));
return spread > 0 && (s.after.count - total * share) / spread <= -minZ
? "worse after the deploy"
: "already down";
}
return s.before.baseline > 0 && s.before.drop < minDrop
? "after the deploy"
: "unclear, little data before";
};
const flagged = dropped
.map((s) => ({ ...s, when: when(s) }))
.filter((s) => s.when !== "already down");
const alreadyDown = dropped.filter((s) => when(s) === "already down");
const percent = (drop: number) => `${Math.round(drop * 100)}%`;

console.log(
`Pageviews from ${settings.deploy} to ${settings.to}, compared with the same hours ${settings.weeks} weeks before`
`Pageviews from ${settings.deploy} to ${settings.to} and the hour before the deploy, compared with the same hours ${settings.weeks} weeks before`
);
console.log(
`${sites.length} websites above ${settings.minDaily} pageviews/day, ${response.took ?? "?"} ms in Elasticsearch`
Expand All @@ -364,19 +409,34 @@ if (command === "query") {
`All of them together: ${total} pageviews vs ${Math.round(totalBaseline)} baseline (${((total / totalBaseline - 1) * 100).toFixed(1)}%)`
);
console.log(
`${flagged.length} dropped at least ${values["min-drop"]}% with z ≤ -${values["min-z"]}\n`
`${flagged.length} dropped after the deploy or can't be ruled out (at least ${values["min-drop"]}% with z ≤ -${values["min-z"]})\n`
);
if (flagged.length) {
if (flagged.length)
console.table(
flagged.slice(0, Number(values.limit)).map((s) => ({
hostname: s.hostname,
current: s.current,
baseline: s.baseline,
drop: `${Math.round(s.drop * 100)}%`,
z: Number(s.z.toFixed(1)),
weeks: s.weeks.join(" "),
current: s.after.count,
baseline: s.after.baseline,
drop: percent(s.after.drop),
z: Number(s.after.z.toFixed(1)),
weeks: s.after.weeks.join(" "),
"hour before": `${s.before.count} vs ${s.before.baseline}`,
when: s.when,
}))
);
if (alreadyDown.length) {
console.log(
`\n${alreadyDown.length} were already down in the hour before the deploy and didn't get clearly worse:\n`
);
console.table(
alreadyDown.slice(0, 10).map((s) => ({
hostname: s.hostname,
before: `${s.before.count} vs ${s.before.baseline} (-${percent(s.before.drop)})`,
after: `${s.after.count} vs ${s.after.baseline} (-${percent(s.after.drop)})`,
}))
);
}
if (flagged.length) {
const [versions, errors] = followUpRequests(
flagged.slice(0, Number(values.limit)).map((s) => s.hostname),
ws
Expand Down
Loading