diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index deadc27..de5558e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: - name: Start DataCat run: | for i in $(seq 1 60); do curl -fsS http://localhost:8123/ping > /dev/null && break; sleep 1; done - PORT=3000 NODE_ENV=production SESSION_SECRET=ci-only-session-secret-0123456789abcdef node artifacts/api-server/dist/index.mjs > server.log 2>&1 & + PORT=3000 NODE_ENV=production PUBLIC_URL=http://localhost:3000 SESSION_SECRET=ci-only-session-secret-0123456789abcdef node artifacts/api-server/dist/index.mjs > server.log 2>&1 & - name: End-to-end known-answer checks run: bash scripts/e2e.sh - name: Server log diff --git a/artifacts/api-server/src/app.ts b/artifacts/api-server/src/app.ts index e0abfce..f07f43b 100644 --- a/artifacts/api-server/src/app.ts +++ b/artifacts/api-server/src/app.ts @@ -35,7 +35,7 @@ app.use( }, serializers: { req(req) { - return { id: req.id, method: req.method, url: req.url?.split("?")[0]?.replace(/(\/(?:invite|invites|agent|input)\/)[^/]+/g, "$1[redacted]") }; + return { id: req.id, method: req.method, url: req.url?.split("?")[0]?.replace(/(\/(?:invite|invites|agent|input)\/)[^/]+/gi, "$1[redacted]") }; }, res(res) { return { statusCode: res.statusCode }; @@ -43,6 +43,12 @@ app.use( }, }), ); +// DataCat is never embedded in another page. +app.use((_req, res, next) => { + res.setHeader("X-Frame-Options", "DENY"); + res.setHeader("Content-Security-Policy", "frame-ancestors 'none'"); + next(); +}); app.use(cors()); // Agent intake routes take raw (possibly compressed / protobuf) bodies, so they mount before JSON parsing. app.use(intakeRouter); diff --git a/artifacts/api-server/src/index.ts b/artifacts/api-server/src/index.ts index 256e4d1..b3fa56d 100644 --- a/artifacts/api-server/src/index.ts +++ b/artifacts/api-server/src/index.ts @@ -6,6 +6,7 @@ import { storage, useClickHouse } from "./lib/storage"; import { ensureChSchema } from "./lib/ch"; import { pool } from "@workspace/db"; import { generatedSetupToken } from "./routes/auth"; +import { flushSpanStats } from "./query/apmStats"; const rawPort = process.env["PORT"] ?? "3000"; const port = Number(rawPort); @@ -17,7 +18,7 @@ logger.info({ storage }, "telemetry storage"); if (generatedSetupToken && !(await pool.query("select 1 from users limit 1")).rowCount) { logger.warn(`First-run setup token (enter it on the setup page): ${generatedSetupToken}`); } -app.listen(port, (err) => { +const server = app.listen(port, (err) => { if (err) { logger.error({ err }, "Error listening on port"); process.exit(1); @@ -31,3 +32,25 @@ app.listen(port, (err) => { void buildBackgroundIndexes((msg, extra) => logger.info(extra ?? {}, msg)); } else logger.info("background jobs disabled (DATACAT_BACKGROUND_JOBS=false)"); }); + +// On SIGTERM/SIGINT: stop accepting connections, let requests in progress finish (up to DATACAT_SHUTDOWN_SECONDS, +// default 25), flush pending APM rollups, close the database pool, then exit. +let stopping = false; +for (const sig of ["SIGTERM", "SIGINT"] as const) + process.once(sig, () => { + if (stopping) return; + stopping = true; + logger.info({ signal: sig }, "shutting down"); + const limitMs = Math.max(1, Number(process.env.DATACAT_SHUTDOWN_SECONDS ?? 25)) * 1000; + const force = setTimeout(() => { + logger.warn("shutdown time limit reached; exiting"); + process.exit(0); + }, limitMs); + force.unref(); + server.close(async () => { + await flushSpanStats().catch((e) => logger.warn({ err: e }, "final span stats flush failed")); + await pool.end().catch(() => undefined); + process.exit(0); + }); + server.closeIdleConnections?.(); + }); diff --git a/artifacts/api-server/src/intake/otlp.ts b/artifacts/api-server/src/intake/otlp.ts index d6dd0e3..c2c21cb 100644 --- a/artifacts/api-server/src/intake/otlp.ts +++ b/artifacts/api-server/src/intake/otlp.ts @@ -380,8 +380,15 @@ function rememberCumulative(k: string, v: any) { } } } -function cumKey(orgId: number, name: string, host: string, tags: string[], start: any) { - return `${orgId}|${name}|${host}|${tags.join(",")}|${start ?? ""}`; +const sortedAttrs = (a: Attrs) => + JSON.stringify( + Object.entries(a) + .filter(([, v]) => v !== null && v !== undefined) + .sort(([x], [y]) => (x < y ? -1 : x > y ? 1 : 0)), + ); +/** Identity of one cumulative stream: every resource, scope and data-point attribute (order-independent) and its start. */ +function cumKey(orgId: number, name: string, resource: string, scope: string, dpAttrs: Attrs, start: any) { + return `${orgId}|${name}|${resource}|${scope}|${sortedAttrs(dpAttrs)}|${start ?? ""}`; } function quantileFromBuckets(bounds: number[], counts: number[], q: number, min?: number, max?: number): number { @@ -402,13 +409,25 @@ function quantileFromBuckets(bounds: number[], counts: number[], q: number, min? return max ?? bounds[bounds.length - 1] ?? NaN; } -function expoToExplicit(dp: any): { bounds: number[]; counts: number[] } { +/** + * Exponential histogram -> explicit buckets in ascending order. Bucket i of a range covers (base^(offset+i), + * base^(offset+i+1)] in absolute value; negative buckets mirror it below zero, the zero bucket sits between. + */ +export function expoToExplicit(dp: any): { bounds: number[]; counts: number[] } { const scale = Number(dp.scale ?? 0); const base = Math.pow(2, Math.pow(2, -scale)); + const bounds: number[] = []; + const counts: number[] = []; + const noff = Number(dp.negative?.offset ?? 0); + const nc = (dp.negative?.bucketCounts ?? []).map(Number); + for (let i = nc.length - 1; i >= 0; i--) { + bounds.push(-Math.pow(base, noff + i)); + counts.push(nc[i]); + } + bounds.push(0); + counts.push(Number(dp.zeroCount ?? 0)); const off = Number(dp.positive?.offset ?? 0); const pc = (dp.positive?.bucketCounts ?? []).map(Number); - const bounds: number[] = [0]; - const counts: number[] = [Number(dp.zeroCount ?? 0)]; pc.forEach((c: number, i: number) => { bounds.push(Math.pow(base, off + i + 1)); counts.push(c); @@ -492,8 +511,18 @@ function hostMappings( } } -export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hosts: Map } { +/** + * Converts an OTLP metrics request. Cumulative streams are turned into deltas against the last stored value; the new + * reference values are applied only by commit(), which the caller runs after the points are stored, so a failed store + * followed by a retry computes the same deltas again instead of losing them. + */ +export function otlpToMetrics( + orgId: number, + req: any, +): { points: PointIn[]; hosts: Map; commit: () => void } { const out: PointIn[] = []; + const refs = new Map(); + const refOf = (k: string) => (refs.has(k) ? refs.get(k) : cumulative.get(k)); const hosts = new Map(); const acc = new Map(); for (const rm of req?.resourceMetrics ?? []) { @@ -506,7 +535,9 @@ export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hos tags: r.tags, }); const baseTags = [...r.tags, ...(r.service ? [`service:${r.service}`] : [])]; + const resKey = sortedAttrs(r.attrs); for (const sm of rm.scopeMetrics ?? []) { + const scopeKey = `${sm.scope?.name ?? ""}@${sm.scope?.version ?? ""}`; for (const m of sm.metrics ?? []) { const name = String(m.name ?? ""); if (!name) continue; @@ -547,9 +578,10 @@ export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hos continue; } if (!delta) { - const k = cumKey(orgId, name, r.host, tags, dp.startTimeUnixNano); - const prev = cumulative.get(k); - rememberCumulative(k, { v, ts }); + const k = cumKey(orgId, name, resKey, scopeKey, a, dp.startTimeUnixNano); + const prev = refOf(k); + if (prev !== undefined && ts < prev.ts) continue; // late or retried older sample: already counted + refs.set(k, { v, ts }); if (prev === undefined || v < prev.v) continue; // first sample or reset const dt = Math.max(1, ts - prev.ts); const d = v - prev.v; @@ -577,10 +609,13 @@ export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hos ? { bounds: (dp.explicitBounds ?? []).map(Number), counts: (dp.bucketCounts ?? []).map(Number) } : expoToExplicit(dp); if (!delta) { - const k = cumKey(orgId, name, r.host, tags, dp.startTimeUnixNano); - const prev = cumulative.get(k); - rememberCumulative(k, { count, sum, counts }); - if (!prev || count < prev.count) continue; + const k = cumKey(orgId, name, resKey, scopeKey, attrs(dp.attributes), dp.startTimeUnixNano); + const prev = refOf(k); + if (prev && ts < prev.ts) continue; + refs.set(k, { count, sum, counts, bounds, ts }); + // buckets are subtracted range by range; a change of bucket layout starts a new reference + const sameLayout = prev && prev.bounds.length === bounds.length && prev.bounds.every((b: number, i: number) => b === bounds[i]); + if (!prev || !sameLayout || count < prev.count) continue; count -= prev.count; sum -= prev.sum; counts = counts.map((c: number, i: number) => c - (prev.counts[i] ?? 0)); @@ -617,19 +652,33 @@ export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hos out.push(e.p); } // derived ratios the Infrastructure list uses - const byHostTs = new Map>(); + const byHostTs = new Map }>(); for (const p of out) if (/^system\.(mem\.(usable|total)|disk\.(used|total))$/.test(p.metric)) { - const k = `${p.host}|${p.points[0][0]}|${p.tags.filter((t) => t.startsWith("device:")).join(",")}`; - (byHostTs.get(k) ?? byHostTs.set(k, {}).get(k)!)[p.metric] = p.points[0][1]; + const device = p.tags.filter((t) => t.startsWith("device:")); + const k = JSON.stringify([p.host, p.points[0][0], device]); + let e = byHostTs.get(k); + if (!e) byHostTs.set(k, (e = { host: p.host, ts: p.points[0][0], device, v: {} })); + e.v[p.metric] = p.points[0][1]; } - for (const [k, v] of byHostTs) { - const [host, ts] = k.split("|"); + for (const { host, ts, device, v } of byHostTs.values()) { const tags = hosts.get(host)?.tags ?? []; if (v["system.mem.total"]) - out.push({ metric: "system.mem.pct_usable", host, tags, type: "gauge", points: [[Number(ts), (v["system.mem.usable"] ?? 0) / v["system.mem.total"]]] }); + out.push({ metric: "system.mem.pct_usable", host, tags, type: "gauge", points: [[ts, (v["system.mem.usable"] ?? 0) / v["system.mem.total"]]] }); if (v["system.disk.total"]) - out.push({ metric: "system.disk.in_use", host, tags, type: "gauge", points: [[Number(ts), (v["system.disk.used"] ?? 0) / v["system.disk.total"]]] }); + out.push({ + metric: "system.disk.in_use", + host, + tags: [...tags, ...device], + type: "gauge", + points: [[ts, (v["system.disk.used"] ?? 0) / v["system.disk.total"]]], + }); } - return { points: out, hosts }; + return { + points: out, + hosts, + commit: () => { + for (const [k, v] of refs) rememberCumulative(k, v); + }, + }; } diff --git a/artifacts/api-server/src/intake/proto.ts b/artifacts/api-server/src/intake/proto.ts index 1a77e9c..6cad609 100644 --- a/artifacts/api-server/src/intake/proto.ts +++ b/artifacts/api-server/src/intake/proto.ts @@ -46,6 +46,8 @@ const descriptor = { max: { type: "double", id: 4 }, avg: { type: "double", id: 5 }, sum: { type: "double", id: 6 }, + k: { rule: "repeated", type: "sint32", id: 7 }, + n: { rule: "repeated", type: "uint32", id: 8 }, }, }, }, diff --git a/artifacts/api-server/src/intake/redact.ts b/artifacts/api-server/src/intake/redact.ts index d957eef..f1eb5de 100644 --- a/artifacts/api-server/src/intake/redact.ts +++ b/artifacts/api-server/src/intake/redact.ts @@ -46,14 +46,13 @@ const custom = String(env.DATACAT_REDACT_PATTERNS ?? "") .map((s) => new RegExp(s, "g")); const CARD = /\b(?:\d[ -]?){12,18}\d\b/g; -const BEARER = /\b(Bearer|Basic)\s+[A-Za-z0-9\-._~+/]{8,}=*/gi; +const BEARER = /\b(Bearer|Basic)\s+([A-Za-z0-9\-._~+/]{4,}=*)/gi; const SECRET_PAIR = /\b(password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key|client[_-]?secret)(["']?\s*[:=]\s*["']?)([^\s"'&,;]{3,})/gi; const AWS_KEY = /\b(AKIA|ASIA)[A-Z0-9]{16}\b/g; -const PRIVATE_KEY = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g; const JWT = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g; const EMAIL = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g; // Cheap pre-check: skip strings that cannot match any rule (most log lines and span attributes). -const MAYBE = /\d{4}|bearer|basic |pass|pwd|secret|token|key|AKIA|ASIA|PRIVATE KEY|eyJ|@/i; +const MAYBE = /\d[ -]?\d[ -]?\d[ -]?\d|bearer|basic|pass|pwd|secret|token|key|AKIA|ASIA|PRIVATE KEY|eyJ|@/i; function luhn(digits: string): boolean { let sum = 0, @@ -72,14 +71,46 @@ function luhn(digits: string): boolean { export const redactionStats = { values: 0, redactions: 0 }; +/** + * Masks PEM private key blocks ("-----BEGIN ... PRIVATE KEY-----" to the next "-----END ... PRIVATE KEY-----") with + * plain string searches, so the cost stays linear however many BEGIN markers a value contains. + */ +function maskPrivateKeys(s: string): string { + const HEAD = /-----BEGIN [A-Z ]{0,40}PRIVATE KEY-----/g; + const TAIL = /-----END [A-Z ]{0,40}PRIVATE KEY-----/g; + let out = ""; + let pos = 0; + for (;;) { + HEAD.lastIndex = pos; + const b = HEAD.exec(s); + if (!b) break; + TAIL.lastIndex = b.index + b[0].length; + const e = TAIL.exec(s); + if (!e) break; // no END after this BEGIN, so none after any later BEGIN either + out += s.slice(pos, b.index) + "[REDACTED:private_key]"; + pos = e.index + e[0].length; + } + return pos === 0 ? s : out + s.slice(pos); +} + +// "Basic ": short values count only when they decode to a user:password pair, so ordinary +// phrases such as "Basic plan" are left alone. +function isCredential(scheme: string, token: string): boolean { + const bare = token.replace(/=+$/, ""); + if (bare.length >= 8) return true; + if (scheme.toLowerCase() !== "basic") return false; + if (!/^[A-Za-z0-9+/]+={0,2}$/.test(token) || token.length % 4 !== 0) return false; + return /^[\x20-\x7e]*:[\x20-\x7e]*$/.test(Buffer.from(token, "base64").toString("latin1")); +} + export function redactString(s: string): string { if (!enabled || typeof s !== "string" || s.length < 6) return s; if (!MAYBE.test(s) && !custom.length) return s; let out = s; const before = out; - out = out.replace(PRIVATE_KEY, "[REDACTED:private_key]"); + out = maskPrivateKeys(out); out = out.replace(JWT, "[REDACTED:jwt]"); - out = out.replace(BEARER, (_m, scheme) => `${scheme} [REDACTED:bearer]`); + out = out.replace(BEARER, (m, scheme, token) => (isCredential(scheme, token) ? `${scheme} [REDACTED:bearer]` : m)); out = out.replace(AWS_KEY, "[REDACTED:aws_key]"); out = out.replace(SECRET_PAIR, (_m, k, sep) => `${k}${sep}[REDACTED:secret]`); out = out.replace(CARD, (m) => { @@ -105,17 +136,16 @@ export function redactObject>(o: T): T { for (const k of Object.keys(o)) { const v = o[k]; if (typeof v === "string") (o as any)[k] = secretKey(k) && v.length ? "[REDACTED:key]" : redactString(v); - else if (v && typeof v === "object") { - if (secretKey(k) && !Array.isArray(v)) (o as any)[k] = "[REDACTED:key]"; - else redactObject(v); - } + else if (secretKey(k) && (typeof v === "number" || typeof v === "bigint" || (v && typeof v === "object"))) (o as any)[k] = "[REDACTED:key]"; + else if (v && typeof v === "object") redactObject(v); } return o; } -export function redactLog; tags?: string[] }>(l: L): L { +export function redactLog; tags?: string[]; service?: string }>(l: L): L { if (!enabled) return l; l.message = redactString(l.message); + if (typeof l.service === "string") l.service = redactString(l.service); redactObject(l.attributes); if (l.tags) l.tags = l.tags.map((t) => { @@ -125,9 +155,11 @@ export function redactLog }>(s: S): S { +export function redactSpan; name?: string; service?: string }>(s: S): S { if (!enabled) return s; s.resource = redactString(s.resource); + if (typeof s.name === "string") s.name = redactString(s.name); + if (typeof s.service === "string") s.service = redactString(s.service); if (s.meta) redactObject(s.meta); return s; } diff --git a/artifacts/api-server/src/intake/routes.ts b/artifacts/api-server/src/intake/routes.ts index 69b6f10..ffb5f1b 100644 --- a/artifacts/api-server/src/intake/routes.ts +++ b/artifacts/api-server/src/intake/routes.ts @@ -7,6 +7,7 @@ import { writeMetrics, touchHost, writeLogs, writeSpans, type PointIn, type LogI import { orgForApiKey } from "../lib/org"; import { decodeOtlp, encodeEmptyResponse, otlpToSpans, otlpToLogs, otlpToMetrics, type OtlpSignal } from "./otlp"; import { logger } from "../lib/logger"; +import { isStorageError } from "../lib/http"; import { pool } from "@workspace/db"; const router: Router = Router(); @@ -38,6 +39,37 @@ export function raw(req: Request, res: Response, next: NextFunction) { const TYPE_V2: Record = { 0: "gauge", 1: "count", 2: "rate", 3: "gauge" }; +// The Datadog Agent's sketch bins (its default configuration: relative accuracy 1/128, minimum value 1e-9). Bin key k +// covers [γ^(k-bias), γ^(k-bias+1)); negative keys mirror it below zero, key 0 is zero. +const SKETCH_GAMMA = 1 + 2 / 128; +const SKETCH_BIAS = -Math.floor(Math.log(1e-9) / Math.log1p(2 / 128)) + 1; +export function sketchBinValue(k: number): number { + if (k === 0) return 0; + const lo = Math.pow(SKETCH_GAMMA, Math.abs(k) - SKETCH_BIAS); + return Math.sign(k) * lo * ((1 + SKETCH_GAMMA) / 2); +} +/** One weighted point per sketch bin (value of the bin, clamped to the sketch's min/max), so counts and percentiles hold. */ +export function sketchPoints(dogsketches: any[]): [number, number, number][] { + const pts: [number, number, number][] = []; + for (const d of dogsketches) { + const ts = Number(d.ts); + const k: number[] = (d.k ?? []).map(Number); + const n: number[] = (d.n ?? []).map(Number); + const min = Number(d.min), + max = Number(d.max); + if (k.length && k.length === n.length) { + for (let i = 0; i < k.length; i++) { + if (!(n[i] > 0)) continue; + let v = sketchBinValue(k[i]); + if (Number.isFinite(min)) v = Math.max(v, min); + if (Number.isFinite(max)) v = Math.min(v, max); + pts.push([ts, v, n[i]]); + } + } else if (Number(d.cnt) > 0) pts.push([ts, Number(d.avg), Number(d.cnt)]); + } + return pts; +} + async function withOrg(req: Request, res: Response, next: NextFunction) { const orgId = await orgForApiKey(apiKeyFrom(req)); if (!orgId) { @@ -54,8 +86,7 @@ function wrap(fn: (req: Request, res: Response) => Promise) { return (req: Request, res: Response) => { fn(req, res).catch((err) => { if (res.headersSent) return; - const fromDb = - (typeof err?.code === "string" && /^[0-9A-Z]{5}$/.test(err.code)) || /^ClickHouse:/.test(String(err?.message ?? "")) || err?.code === "ECONNREFUSED"; + const fromDb = isStorageError(err); if (req.orgId) recordDrop(req.orgId, intakePath(req.path), 1, fromDb ? "storage error" : `rejected: ${String(err?.message ?? err).slice(0, 120)}`); if (fromDb) { logger.error({ err, path: req.path }, "intake storage error"); @@ -144,10 +175,8 @@ router.post( wrap(async (req, res) => { const p = decodeProto(SketchPayload, decompress(req)); const series: PointIn[] = []; - for (const s of p.sketches ?? []) { - const pts: [number, number][] = (s.dogsketches ?? []).map((d: any) => [Number(d.ts), Number(d.avg)]); - series.push({ metric: s.metric, host: s.host ?? "", tags: s.tags ?? [], type: "distribution", points: pts }); - } + for (const s of p.sketches ?? []) + series.push({ metric: s.metric, host: s.host ?? "", tags: s.tags ?? [], type: "distribution", points: sketchPoints(s.dogsketches ?? []) }); await storeMetrics(req.orgId!, "metrics", series); res.status(202).json({}); }), @@ -432,13 +461,14 @@ function otlpRoute(signal: OtlpSignal) { n = logs.length; await writeLogs(orgId, logs); } else { - const { points, hosts } = otlpToMetrics(orgId, body); + const { points, hosts, commit } = otlpToMetrics(orgId, body); for (const [name, h] of hosts) await touchHost(orgId, name, { os: h.os, platform: h.os, agentVersion: h.agentVersion, tags: h.tags, apps: ["system"] }, true); const { valid, invalid } = splitPoints(points); if (invalid) recordDrop(orgId, path, invalid, "points without a numeric value"); n = valid; await writeMetrics(orgId, points); + commit(); } } catch (e) { logger.error({ err: e, signal }, "OTLP write failed"); diff --git a/artifacts/api-server/src/intake/store.ts b/artifacts/api-server/src/intake/store.ts index 5221555..d70bbb9 100644 --- a/artifacts/api-server/src/intake/store.ts +++ b/artifacts/api-server/src/intake/store.ts @@ -34,7 +34,7 @@ export interface PointIn { tags: string[]; type: string; // gauge | count | rate | distribution unit?: string; - points: [number, number][]; // [unix seconds, value] + points: ([number, number] | [number, number, number])[]; // [unix seconds, value, weight (distribution sketches; default 1)] } const seriesCache = new Map(); @@ -102,6 +102,31 @@ async function seriesIds(orgId: number, list: PointIn[], now: number): Promise(); +type DbClient = { query(text: string, values?: unknown[]): Promise }; +/** Runs fn in one transaction, so a batch is stored completely or not at all (a retried request never duplicates). */ +async function inTransaction(fn: (c: DbClient) => Promise): Promise { + const c = await pool.connect(); + try { + await c.query("BEGIN"); + const r = await fn(c); + await c.query("COMMIT"); + return r; + } catch (e) { + await c.query("ROLLBACK").catch((err) => logger.warn({ err }, "rollback failed")); + throw e; + } finally { + c.release(); + } +} +/** Executes the statements on one connection in a transaction when there are several, directly when there is one. */ +async function runAll(stmts: [string, any[]][]) { + if (stmts.length === 1) await pool.query(stmts[0][0], stmts[0][1]); + else if (stmts.length > 1) + await inTransaction(async (c) => { + for (const [q, v] of stmts) await c.query(q, v); + }); +} + /** ClickHouse: deterministic 52-bit series id (fits a JS number exactly), so every API process agrees without a round trip. */ function seriesIdHash(key: string): number { return parseInt(createHash("sha1").update(key).digest("hex").slice(0, 13), 16); @@ -113,6 +138,7 @@ async function writeMetricsCh(orgId: number, series: PointIn[]) { const hosts = new Set(); const valid = series.filter((s) => s.metric && s.points?.length); const meta: object[] = []; + const metaIds = new Set(); const points: object[] = []; trimCaches(); for (const s of valid) { @@ -124,17 +150,28 @@ async function writeMetricsCh(orgId: number, series: PointIn[]) { seriesCache.set(key, id); } // metadata rows are re-written at most every 5 minutes per series (ReplacingMergeTree keeps the latest) - if (now - (lastSeenTouched.get(id) ?? 0) > 300) { - lastSeenTouched.set(id, now); + if (now - (lastSeenTouched.get(id) ?? 0) > 300 && !metaIds.has(id)) { + metaIds.add(id); meta.push({ org_id: orgId, metric: s.metric, series_id: id, host: s.host ?? "", tags, type: s.type || "gauge", unit: s.unit ?? "", last_seen: now }); } if (s.host) hosts.add(s.host); - for (const [ts, v] of s.points) { + for (const [ts, v, w] of s.points) { if (!Number.isFinite(v) || !Number.isFinite(ts)) continue; - points.push({ org_id: orgId, metric: s.metric, series_id: id, host: s.host ?? "", ts: Math.floor(ts), value: v }); + points.push({ + org_id: orgId, + metric: s.metric, + series_id: id, + host: s.host ?? "", + ts: Math.floor(ts), + value: v, + weight: w !== undefined && Number.isFinite(w) ? w : 1, + }); } } - await Promise.all([chInsert("metric_series", meta), chInsert("metric_points", points)]); + // Metadata first: if it fails nothing is written, and a retry writes both. + await chInsert("metric_series", meta); + for (const id of metaIds) lastSeenTouched.set(id, now); + await chInsert("metric_points", points); const apps = newApps(orgId, valid); for (const h of hosts) await touchHost(orgId, h, apps.has(h) ? { apps: apps.get(h) } : {}, apps.has(h)); } @@ -144,6 +181,7 @@ export async function writeMetrics(orgId: number, series: PointIn[]) { const now = Math.floor(Date.now() / 1000); const vals: any[] = []; const rows: string[] = []; + const stmts: [string, any[]][] = []; const hosts = new Set(); const valid = series.filter((s) => s.metric && s.points?.length); const ids = await seriesIds(orgId, valid, now); @@ -158,23 +196,25 @@ export async function writeMetrics(orgId: number, series: PointIn[]) { touch.push(id); } if (s.host) hosts.add(s.host); - for (const [ts, v] of s.points) { + for (const [ts, v, w] of s.points) { if (!Number.isFinite(v) || !Number.isFinite(ts)) continue; - vals.push(id, Math.floor(ts), v); - rows.push(`($${vals.length - 2},$${vals.length - 1},$${vals.length})`); + vals.push(id, Math.floor(ts), v, w !== undefined && Number.isFinite(w) && w !== 1 ? w : null); + rows.push(`($${vals.length - 3},$${vals.length - 2},$${vals.length - 1},$${vals.length})`); if (rows.length >= 2000) { - await pool.query(`insert into metric_points (series_id, ts, value) values ${rows.join(",")}`, vals); + stmts.push([`insert into metric_points (series_id, ts, value, weight) values ${rows.join(",")}`, vals.slice()]); rows.length = 0; vals.length = 0; } } } - if (rows.length) await pool.query(`insert into metric_points (series_id, ts, value) values ${rows.join(",")}`, vals); + if (rows.length) stmts.push([`insert into metric_points (series_id, ts, value, weight) values ${rows.join(",")}`, vals.slice()]); + await runAll(stmts); if (touch.length) { touch.sort((a, b) => a - b); - pool - .query("update metric_series set last_seen=$1 where id = any($2::int[])", [now, touch]) - .catch((e) => logger.warn({ err: e }, "last_seen refresh failed")); + pool.query("update metric_series set last_seen=$1 where id = any($2::int[])", [now, touch]).catch((e) => { + for (const id of touch) lastSeenTouched.delete(id); + logger.warn({ err: e }, "last_seen refresh failed"); + }); } const apps = newApps(orgId, valid); for (const h of hosts) await touchHost(orgId, h, apps.has(h) ? { apps: apps.get(h) } : {}, apps.has(h)); @@ -288,6 +328,8 @@ export async function writeLogs(orgId: number, logs: LogIn[]) { for (const l of logs) { if (typeof l.message === "string") l.message = cut(stripAnsi(l.message), MAX_MESSAGE); limitValues(l.attributes); + if (Array.isArray(l.tags)) l.tags = l.tags.slice(0, MAX_ATTRS).map((t) => (typeof t === "string" ? cut(t, MAX_VALUE) : t)); + if (typeof l.service === "string") l.service = cut(l.service, 1000); redactLog(l); if (l.service) l.service = l.service.toLowerCase(); } @@ -315,6 +357,7 @@ export async function writeLogs(orgId: number, logs: LogIn[]) { for (const h of hosts) await touchHost(orgId, h, {}); return; } + const stmts: [string, any[]][] = []; for (let i = 0; i < logs.length; i += 500) { const chunk = logs.slice(i, i + 500); const vals: any[] = []; @@ -325,8 +368,9 @@ export async function writeLogs(orgId: number, logs: LogIn[]) { rows.push(`(${Array.from({ length: 10 }, (_, j) => `$${b + j + 1}`).join(",")})`); if (l.host) hosts.add(l.host); } - await pool.query(`insert into logs (org_id, ts, status, service, host, source, message, tags, attributes, trace_id) values ${rows.join(",")}`, vals); + stmts.push([`insert into logs (org_id, ts, status, service, host, source, message, tags, attributes, trace_id) values ${rows.join(",")}`, vals]); } + await runAll(stmts); for (const h of hosts) await touchHost(orgId, h, {}); } @@ -366,6 +410,8 @@ export async function writeSpans(orgId: number, spans: SpanIn[]) { for (const s of spans) { limitValues(s.meta); if (s.resource) s.resource = cut(s.resource, 5000); + if (typeof s.name === "string") s.name = cut(s.name, 1000); + if (typeof s.service === "string") s.service = cut(s.service, 1000); redactSpan(s); if (s.service) s.service = s.service.toLowerCase(); if (s.env) s.env = s.env.toLowerCase(); @@ -404,7 +450,8 @@ export async function writeSpans(orgId: number, spans: SpanIn[]) { metrics: toNum(s.metrics), })), ); - } else + } else { + const stmts: [string, any[]][] = []; for (let i = 0; i < spans.length; i += 300) { const chunk = spans.slice(i, i + 300); const vals: any[] = []; @@ -432,12 +479,14 @@ export async function writeSpans(orgId: number, spans: SpanIn[]) { ); rows.push(`(${Array.from({ length: 17 }, (_, j) => `$${b + j + 1}`).join(",")})`); } - await pool.query( + stmts.push([ `insert into spans (org_id, trace_id, span_id, parent_id, service, name, resource, type, env, host, start, start_ms, duration, error, is_top_level, meta, metrics) values ${rows.join(",")}`, vals, - ); + ]); } + await runAll(stmts); + } await recordHostServices(orgId, spans); // Per-minute APM rollups (service/resource counts and latency histograms) for the APM screens. // With ClickHouse a materialized view on spans maintains them (lib/ch.ts). diff --git a/artifacts/api-server/src/intake/tracer.ts b/artifacts/api-server/src/intake/tracer.ts index 31dddbc..4afaf20 100644 --- a/artifacts/api-server/src/intake/tracer.ts +++ b/artifacts/api-server/src/intake/tracer.ts @@ -4,6 +4,7 @@ import { decompress } from "./decode"; import { raw } from "./routes"; import { writeSpans, writeMetrics, markTopLevel, type SpanIn, type PointIn } from "./store"; import { orgForApiKey } from "../lib/org"; +import { isStorageError } from "../lib/http"; import { logger } from "../lib/logger"; /** @@ -53,6 +54,23 @@ function decodeV05(buf: Buffer): any[][] { ); } +/** + * One DogStatsD line, "metric:value|type", split without a regular expression. Returns [line, metric, value, + * type, rest] (rest starts with "|" when present), or null. A trailing "\r" (CRLF bodies) is ignored. + */ +export function parseStatsdLine(line: string): [string, string, string, string, string] | null { + if (line.endsWith("\r")) line = line.slice(0, -1); + const colon = line.indexOf(":"); + const pipe = line.indexOf("|"); + if (colon < 1 || pipe < colon + 2) return null; + let t = pipe + 1; + while (t < line.length && line.charCodeAt(t) >= 97 && line.charCodeAt(t) <= 122) t++; + if (t === pipe + 1) return null; + const rest = line.slice(t); + if (/[\r\n\u2028\u2029]/.test(rest)) return null; + return [line, line.slice(0, colon), line.slice(colon + 1, pipe), line.slice(pipe + 1, t), rest]; +} + function keyFrom(req: Request, spans?: any[][]): string | undefined { if (req.params.key) return String(req.params.key); if (req.headers["dd-api-key"]) return String(req.headers["dd-api-key"]); @@ -102,6 +120,11 @@ async function handleTraces(req: Request, res: Response, version: "v0.4" | "v0.5 await writeSpans(orgId, spans); res.json({ rate_by_service: { "service:,env:": 1 } }); } catch (err: any) { + if (isStorageError(err)) { + logger.error({ err }, "tracer intake storage error"); + res.status(503).json({ error: "Temporarily unable to store data; retry later" }); + return; + } logger.warn({ err: String(err?.message ?? err) }, "tracer intake error"); res.status(err?.status === 413 ? 413 : 400).json({ error: String(err?.message ?? err) }); } @@ -123,11 +146,17 @@ for (const p of PREFIXES) { // DogStatsD over HTTP: newline-separated "metric:value|type|@rate|#tag1,tag2" router.post(`${p}/dogstatsd/v2/proxy`, raw, async (req, res) => { try { + const headerKey = req.params.key ? String(req.params.key) : ((req.headers["dd-api-key"] as string) ?? ""); + // A key given in the path or a header is checked before the body is read. + if (headerKey && !(await orgForApiKey(headerKey))) { + res.status(403).json({ accepted: 0 }); + return; + } const lines = decompress(req).toString("utf8").split("\n").filter(Boolean); const now = Math.floor(Date.now() / 1000); const byKey = new Map(); for (const line of lines) { - const m = /^([^:|]+):([^|]+)\|([a-z]+)(.*)$/.exec(line); + const m = parseStatsdLine(line); if (!m) continue; const parts = m[4].split("|").filter(Boolean); const tags = ( @@ -137,7 +166,7 @@ for (const p of PREFIXES) { .split(",") ?? [] ).filter(Boolean); const rate = Number(parts.find((x) => x.startsWith("@"))?.slice(1) ?? 1) || 1; - let key = req.params.key ? String(req.params.key) : ((req.headers["dd-api-key"] as string) ?? ""); + let key = headerKey; const kt = tags.find((t) => KEY_TAGS.some((k) => t.startsWith(k + ":"))); if (!key && kt) key = kt.split(":").slice(1).join(":"); const clean = tags.filter((t) => !KEY_TAGS.some((k) => t.startsWith(k + ":"))); @@ -158,6 +187,11 @@ for (const p of PREFIXES) { } res.status(accepted || !lines.length ? 202 : 403).json({ accepted }); } catch (err: any) { + if (isStorageError(err)) { + logger.error({ err }, "dogstatsd intake storage error"); + res.status(503).json({ error: "Temporarily unable to store data; retry later" }); + return; + } res.status(err?.status === 413 ? 413 : 400).json({ error: String(err?.message ?? err) }); } }); diff --git a/artifacts/api-server/src/jobs/monitors.ts b/artifacts/api-server/src/jobs/monitors.ts index a03629d..9918127 100644 --- a/artifacts/api-server/src/jobs/monitors.ts +++ b/artifacts/api-server/src/jobs/monitors.ts @@ -74,7 +74,7 @@ export function parseMonitorQuery(type: string, query: string): Parsed { for (const raw of Object.values(queries)) parseMetricQuery(raw); return { kind: "metric", - timeAgg: m[1].startsWith("percentile") ? "p95" : m[1].replace(/\(.*$/, ""), + timeAgg: monitorTimeAgg(m[1]), window: windowSecs(m[2]), expr, op: cmp[1], @@ -82,6 +82,16 @@ export function parseMonitorQuery(type: string, query: string): Parsed { }; } +/** avg|sum|min|max|last as is; percentile(95) or percentile(p95) -> "p95"; change(...) / pct_change(...) by name. */ +function monitorTimeAgg(f: string): string { + if (f.startsWith("percentile")) { + const pm = /^percentile\(\s*p?(\d{1,2}(?:\.\d+)?)\s*\)$/.exec(f); + if (!pm || !(Number(pm[1]) > 0 && Number(pm[1]) < 100)) throw new Error("percentile() takes a number between 0 and 100, e.g. percentile(95)"); + return `p${pm[1]}`; + } + return f.replace(/\(.*$/, ""); +} + export function validateMonitorQuery(type: string, query: string) { parseMonitorQuery(type, query); } @@ -116,14 +126,26 @@ function stateFor(v: number | null, op: string, opts: any, threshold: number, pr // For log and trace monitors, an empty window is a count of 0, or missing data when the monitor has notify_no_data set // (the same on both storage backends). Grouped monitors report each group that matched; groups that stop matching are // handled by the caller as disappeared groups. -async function evaluateValues(orgId: number, p: Parsed, noData = false): Promise> { - const now = Math.floor(Date.now() / 1000); - const out = new Map(); +type GroupValue = { value: number | null; tags: string[]; lastMs?: number }; +// Group-by fields that are real columns of each data set; anything else is an attribute (logs) or a meta key (spans). +const LOG_GROUP_COLS = ["service", "host", "status", "source"]; +const SPAN_GROUP_COLS = ["service", "host", "env", "resource", "name", "type"]; + +async function evaluateValues(orgId: number, p: Parsed, noData = false, delaySec = 0): Promise> { + // evaluation_delay moves the whole window back, so late telemetry is complete before it is judged + const now = Math.floor(Date.now() / 1000) - delaySec; + const out = new Map(); if (p.kind === "metric") { const set = await evalMetricExpression(orgId, p.expr, now - p.window, now); for (const s of set) { - const vals = [...s.points.values()]; - out.set(s.key, { value: vals.length ? reduce(vals, p.timeAgg) : null, tags: s.tags }); + const pts = [...s.points].sort((a, b) => a[0] - b[0]); + const vals = pts.map((x) => x[1]); + const lastTs = pts.at(-1)?.[0]; + out.set(s.key, { + value: vals.length ? reduce(vals, p.timeAgg) : null, + tags: s.tags, + lastMs: lastTs === undefined ? undefined : Math.min(now, lastTs + (s.interval ?? 0)) * 1000, + }); } if (!set.length) out.set("*", { value: null, tags: [] }); return out; @@ -132,24 +154,21 @@ async function evaluateValues(orgId: number, p: Parsed, noData = false): Promise if (useClickHouse) { const { where, b } = isLog ? logWhereCh(orgId, p.logQuery ?? "*", now - p.window, now) : spanWhereCh(orgId, p.logQuery ?? "*", now - p.window, now); const g = p.groupBy ?? []; - const cols = g.map((k) => - ["service", "host", "status", "source", "env"].includes(k) - ? k === "status" && !isLog - ? "if(error = 1, 'error', 'ok')" - : k - : isLog - ? `attrs[${b.p(k.replace(/^@/, ""))}]` - : `meta[${b.p(k.replace(/^@/, ""))}]`, - ); + const cols = g.map((k) => { + if (k === "status" && !isLog) return "if(error = 1, 'error', 'ok')"; + if ((isLog ? LOG_GROUP_COLS : SPAN_GROUP_COLS).includes(k)) return k; + if (isLog && k === "env") return "substring(arrayFirst(t -> startsWith(t, 'env:'), tags), 5)"; + return isLog ? `attrs[${b.p(k.replace(/^@/, ""))}]` : `meta[${b.p(k.replace(/^@/, ""))}]`; + }); const sel = cols.length ? cols.map((c, i) => `${c} AS g${i}`).join(", ") + "," : ""; const rows = await chQuery( - `SELECT ${sel} count() AS c FROM ${isLog ? "logs" : "spans"} WHERE ${where} ${cols.length ? `GROUP BY ${cols.map((_, i) => `g${i}`).join(", ")}` : ""}`, + `SELECT ${sel} count() AS c, max(${isLog ? "ts" : "start_ms"}) AS last FROM ${isLog ? "logs" : "spans"} WHERE ${where} ${cols.length ? `GROUP BY ${cols.map((_, i) => `g${i}`).join(", ")}` : ""}`, b.params, ); for (const r of rows as any[]) { if (!g.length && !Number(r.c)) continue; const tags = g.map((k, i) => `${k.replace(/^@/, "")}:${r[`g${i}`] || "N/A"}`); - out.set(tags.length ? tags.join(",") : "*", { value: Number(r.c), tags }); + out.set(tags.length ? tags.join(",") : "*", { value: Number(r.c), tags, lastMs: Number(r.last) || undefined }); } if (!out.size) out.set("*", { value: noData ? null : 0, tags: [] }); return out; @@ -164,30 +183,37 @@ async function evaluateValues(orgId: number, p: Parsed, noData = false): Promise const g = p.groupBy ?? []; const cols = g.map((k) => { if (k === "status" && !isLog) return "case when error = 1 then 'error' else 'ok' end"; - if (["service", "host", "status", "source", "env"].includes(k)) return k; + if ((isLog ? LOG_GROUP_COLS : SPAN_GROUP_COLS).includes(k)) return k; + if (isLog && k === "env") return "(select substr(t, 5) from unnest(tags) t where t like 'env:%' limit 1)"; const key = k.replace(/^@/, ""); - params.push(isLog ? key.split(".") : key); - return isLog ? `attributes #>> $${params.length}::text[]` : `meta ->> $${params.length}::text`; + if (!isLog) { + params.push(key); + return `meta ->> $${params.length}::text`; + } + // nested path or a flat dotted key, the same lookup as search + params.push(key.split("."), key); + return `COALESCE(attributes #>> $${params.length - 1}::text[], attributes ->> $${params.length}::text)`; }); const sel = cols.length ? cols.map((c, i) => `${c} as g${i}`).join(", ") + "," : ""; const rows = ( await pool.query( - `select ${sel} count(*)::int as c from ${isLog ? "logs" : "spans"} where ${where} ${cols.length ? `group by ${cols.map((_, i) => i + 1).join(",")}` : ""}`, + `select ${sel} count(*)::int as c, max(${isLog ? "ts" : "start_ms"}) as last from ${isLog ? "logs" : "spans"} where ${where} ${cols.length ? `group by ${cols.map((_, i) => i + 1).join(",")}` : ""}`, params, ) ).rows; for (const r of rows) { if (!g.length && !Number(r.c)) continue; const tags = g.map((k, i) => `${k.replace(/^@/, "")}:${r[`g${i}`] ?? "N/A"}`); - out.set(tags.length ? tags.join(",") : "*", { value: Number(r.c), tags }); + out.set(tags.length ? tags.join(",") : "*", { value: Number(r.c), tags, lastMs: Number(r.last) || undefined }); } if (!out.size) out.set("*", { value: noData ? null : 0, tags: [] }); return out; } -function render( +export function render( template: string, ctx: { state: State; prev: State; value: number | null; threshold: number; warn?: number; tags: string[]; name: string; id: number }, + sectionsOnly = false, ) { const isAlert = ctx.state === "Alert", isWarn = ctx.state === "Warn", @@ -202,6 +228,7 @@ function render( is_warning_recovery: isRecovery && ctx.prev === "Warn", }; let s = template.replace(/\{\{([#^])(\w+)\}\}([\s\S]*?)\{\{\/\2\}\}/g, (_m, kind, flag, inner) => ((kind === "#") === !!flags[flag] ? inner : "")); + if (sectionsOnly) return s; const tagVals: Record = {}; for (const t of ctx.tags) { const i = t.indexOf(":"); @@ -218,8 +245,14 @@ function render( return s; } -async function notify(orgId: number, monitorId: number, message: string, title: string, payload: any) { - const handles = [...new Set([...message.matchAll(/@([A-Za-z0-9_\-.@]+)/g)].map((m) => m[1]))]; +/** Notification handles (@webhook-x, @slack-y, @someone) written in a monitor message. */ +export function messageHandles(template: string): string[] { + return [...new Set([...template.matchAll(/@([A-Za-z0-9_\-.@]+)/g)].map((m) => m[1]))]; +} + +// handles come from the monitor's own template (see evaluateMonitorNow), so telemetry values inserted into the +// message cannot add recipients. +async function notify(orgId: number, monitorId: number, message: string, title: string, payload: any, handles: string[]) { const now = Date.now(); const channels = (await pool.query("select * from notification_channels where org_id=$1", [orgId])).rows; const clean = message.replace(/@(webhook|slack)-[A-Za-z0-9_\-]+/g, "").trim(); @@ -290,7 +323,13 @@ export async function retryNotifications(): Promise { [now, now + 60_000], ) ).rows; - for (const n of rows) { + // Retries run in parallel, so one slow destination does not hold up the rest of the batch. + await Promise.all(rows.map((n) => retryOne(n))); + return rows.length; +} + +async function retryOne(n: any) { + { const attempts = Number(n.attempts) + 1; const name = String(n.channel).replace(/^@(webhook|slack)-/, ""); const ch = (await pool.query("select * from notification_channels where org_id=$1 and name=$2", [n.org_id, name])).rows[0]; @@ -299,7 +338,7 @@ export async function retryNotifications(): Promise { n.id, "Channel was deleted before delivery", ]); - continue; + return; } const r = await deliver(ch, n.payload); if (r.ok) @@ -322,7 +361,6 @@ export async function retryNotifications(): Promise { Date.now() + RETRY_BACKOFF_MS[attempts - 1], ]); } - return rows.length; } const inFlight = new Map>(); @@ -349,7 +387,8 @@ async function evaluateMonitorNow(id: number) { logger.warn({ id, err: String(e) }, "invalid monitor"); return; } - const values = await evaluateValues(mon.org_id, parsed, !!mon.options?.notify_no_data); + const delaySec = Math.min(86_400, Math.max(0, Number(mon.options?.evaluation_delay) || 0)); + const values = await evaluateValues(mon.org_id, parsed, !!mon.options?.notify_no_data, delaySec); const prevStates: Record = mon.group_states ?? {}; const next: typeof prevStates = {}; const now = Date.now(); @@ -370,15 +409,21 @@ async function evaluateMonitorNow(id: number) { ? "No Data" : (prev ?? "OK"); next[key] = { state: st, value: v.value, tags: v.tags, since: prev === st ? prevStates[key].since : now }; - (next[key] as any).seen = hasValue ? now : (prevStates[key] as any)?.seen; + // the no-data timer runs from the newest data point, not from this evaluation + (next[key] as any).seen = hasValue ? (v.lastMs ?? now) : (prevStates[key] as any)?.seen; if ((prev ?? "OK") !== st && !(prev === undefined && st === "OK")) transitions.push({ key, from: prev ?? "OK", to: st, value: v.value, tags: v.tags }); } - // groups that disappeared + // Groups that disappeared from the result. With notify_no_data every group is kept until the no-data check resolves + // it (OK groups included, so a host that stops reporting does alert); without it, an alerting group recovers. for (const [key, ps] of Object.entries(prevStates)) { - if (!next[key] && ps.state !== "OK" && now - ps.since < 3600_000) { - const st: State = mon.options?.notify_no_data ? (missingLongEnough(key) ? "No Data" : ps.state) : "OK"; + if (next[key]) continue; + if (mon.options?.notify_no_data) { + const st: State = missingLongEnough(key) ? "No Data" : ps.state; next[key] = { ...ps, state: st, value: null, since: ps.state === st ? ps.since : now }; if (ps.state !== st) transitions.push({ key, from: ps.state, to: st, value: null, tags: ps.tags }); + } else if (ps.state !== "OK" && now - ps.since < 3600_000) { + next[key] = { ...ps, state: "OK", value: null, since: now }; + transitions.push({ key, from: ps.state, to: "OK", value: null, tags: ps.tags }); } } const overall = Object.values(next).reduce((w, s) => (RANK[s.state] > RANK[w] ? s.state : w), "OK") as State; @@ -394,8 +439,15 @@ async function evaluateMonitorNow(id: number) { } else (s as any).notified = lastN; } } + for (const t of transitions) if (t.to === "Alert" || t.to === "Warn") lastTriggered = now; + // Compare-and-set: when two processes evaluate the same monitor at once, only the one whose state write lands sends + // the events and notifications, so a transition is announced once. + const saved = await pool.query( + "update monitors set overall_state=$2, group_states=$3, last_evaluated_at=$4, last_triggered_at=$5 where id=$1 and last_evaluated_at is not distinct from $6::bigint returning id", + [id, overall, next, now, lastTriggered, mon.last_evaluated_at ?? null], + ); + if (!saved.rowCount) return; for (const t of transitions) { - if (t.to === "Alert" || t.to === "Warn") lastTriggered = now; const label = t.to === "OK" ? "Recovered" : t.to === "No Data" ? "No Data" : t.to === "Warn" ? "Warn" : "Triggered"; const ctx = { state: t.to, @@ -409,6 +461,7 @@ async function evaluateMonitorNow(id: number) { }; const title = `[${label}${t.tags.length ? " on " + t.tags.join(",") : ""}] ${render(mon.name, ctx)}`; const text = render(mon.message ?? "", ctx); + const handles = messageHandles(render(mon.message ?? "", ctx, true)); await pool.query( "insert into events (org_id, ts, title, text, alert_type, source, monitor_id, group_key, tags) values ($1,$2,$3,$4,$5,'monitor',$6,$7,$8)", [ @@ -423,41 +476,47 @@ async function evaluateMonitorNow(id: number) { ], ); if (!muted) { - await notify(mon.org_id, mon.id, text, title, { - monitor_id: mon.id, - monitor_name: mon.name, - transition: `${t.from} -> ${t.to}`, - state: t.to, - value: t.value, - threshold: ctx.threshold, - group: t.key, - tags: t.tags, - query: mon.query, - date: new Date(now).toISOString(), - }); + await notify( + mon.org_id, + mon.id, + text, + title, + { + monitor_id: mon.id, + monitor_name: mon.name, + transition: `${t.from} -> ${t.to}`, + state: t.to, + value: t.value, + threshold: ctx.threshold, + group: t.key, + tags: t.tags, + query: mon.query, + date: new Date(now).toISOString(), + }, + handles, + ); } } - await pool.query("update monitors set overall_state=$2, group_states=$3, last_evaluated_at=$4, last_triggered_at=$5 where id=$1", [ - id, - overall, - next, - now, - lastTriggered, - ]); } -let running = false; +// Each organization's monitors run in their own chain, so a slow notification destination in one organization never +// delays monitoring for the others. A chain still running from the previous tick is not started again. +const orgRuns = new Map>(); export function startMonitorLoop() { const tick = async () => { - if (running) return; - running = true; try { - const rows = (await pool.query("select id from monitors")).rows; - for (const r of rows) await evaluateMonitor(r.id).catch((e) => logger.warn({ err: String(e), id: r.id }, "monitor eval failed")); + const rows = (await pool.query("select id, org_id from monitors order by org_id, id")).rows; + const byOrg = new Map(); + for (const r of rows) byOrg.set(r.org_id, [...(byOrg.get(r.org_id) ?? []), r.id]); + for (const [orgId, ids] of byOrg) { + if (orgRuns.has(orgId)) continue; + const run = (async () => { + for (const id of ids) await evaluateMonitor(id).catch((e) => logger.warn({ err: String(e), id }, "monitor eval failed")); + })().finally(() => orgRuns.delete(orgId)); + orgRuns.set(orgId, run); + } } catch (e) { logger.warn({ err: String(e) }, "monitor loop failed"); - } finally { - running = false; } }; setInterval(tick, 30_000); diff --git a/artifacts/api-server/src/jobs/sample.ts b/artifacts/api-server/src/jobs/sample.ts index a8e2b40..27c667f 100644 --- a/artifacts/api-server/src/jobs/sample.ts +++ b/artifacts/api-server/src/jobs/sample.ts @@ -22,8 +22,8 @@ const ROUTES = [ const rnd = (a: number, b: number) => a + Math.random() * (b - a); const id = () => BigInt.asUintN(63, crypto.randomBytes(8).readBigUInt64BE()).toString(); -async function tick(orgId: number, t0: number) { - const now = Math.floor(Date.now() / 1000); +async function tick(orgId: number, t0: number, nowMs = Date.now()) { + const now = Math.floor(nowMs / 1000); const phase = (now - t0) / 600; const series: PointIn[] = []; for (const [i, h] of HOSTS.entries()) { @@ -53,7 +53,7 @@ async function tick(orgId: number, t0: number) { const host = r.svc === "storefront" ? HOSTS[Math.floor(Math.random() * 2)].name : "api-01"; const traceId = id(), rootId = id(); - const startNs = (Date.now() - Math.floor(rnd(0, 14000))) * 1e6; + const startNs = (nowMs - Math.floor(rnd(0, 14000))) * 1e6; const err = Math.random() < (r.err ?? 0.01); const dur = (r.base + rnd(0, r.base)) * (err ? 2 : 1) * 1e6; const dbDur = dur * rnd(0.2, 0.5); @@ -135,7 +135,7 @@ async function tick(orgId: number, t0: number) { } if (Math.random() < 0.3) logs.push({ - ts: Date.now(), + ts: nowMs, status: "warn", service: "postgres", host: "db-01", @@ -148,7 +148,17 @@ async function tick(orgId: number, t0: number) { await writeSpans(orgId, spans); } -export async function startSampleData(orgId: number, minutes = 60) { +const starting = new Map>(); +/** Starts sample data for an organization; a second call while one is starting returns the same result. */ +export function startSampleData(orgId: number, minutes = 60): Promise<{ until: number }> { + const cur = starting.get(orgId); + if (cur) return cur; + const p = startSampleDataNow(orgId, minutes).finally(() => starting.delete(orgId)); + starting.set(orgId, p); + return p; +} + +async function startSampleDataNow(orgId: number, minutes: number) { stopSampleData(orgId); for (const h of HOSTS) await touchHost( @@ -167,15 +177,7 @@ export async function startSampleData(orgId: number, minutes = 60) { const t0 = Math.floor(Date.now() / 1000); // backfill the last 30 minutes so graphs have history const now = Date.now(); - const realNow = Date.now; - try { - for (let back = 30 * 60; back > 0; back -= 60) { - Date.now = () => now - back * 1000; - await tick(orgId, t0 - back); - } - } finally { - Date.now = realNow; - } + for (let back = 30 * 60; back > 0; back -= 60) await tick(orgId, t0 - back, now - back * 1000); const end = Date.now() + minutes * 60_000; const iv = setInterval(() => { if (Date.now() > end) return stopSampleData(orgId); diff --git a/artifacts/api-server/src/lib/auth.ts b/artifacts/api-server/src/lib/auth.ts index 4c6ae75..fe4398b 100644 --- a/artifacts/api-server/src/lib/auth.ts +++ b/artifacts/api-server/src/lib/auth.ts @@ -56,9 +56,8 @@ export async function clearSession(res: Response, sid?: string): Promise { res.clearCookie(SESSION_COOKIE, { path: "/" }); } +// Sessions come only from the (signed) cookie; the raw session id is never accepted as a token. export function getSessionId(req: Request): string | undefined { - const authHeader = req.headers["authorization"]; - if (authHeader?.startsWith("Bearer ")) return authHeader.slice(7); return process.env.SESSION_SECRET ? req.signedCookies?.[SESSION_COOKIE] || undefined : req.cookies?.[SESSION_COOKIE]; } diff --git a/artifacts/api-server/src/lib/ch.ts b/artifacts/api-server/src/lib/ch.ts index abc37a8..e8aa9e8 100644 --- a/artifacts/api-server/src/lib/ch.ts +++ b/artifacts/api-server/src/lib/ch.ts @@ -94,7 +94,7 @@ export async function chInsert(table: string, rows: object[]) { }), { method: "POST", headers: auth(), body }, ); - if (!res.ok) throw new Error(`ClickHouse insert into ${table}: ${(await res.text()).slice(0, 500)}`); + if (!res.ok) throw new Error(`ClickHouse: insert into ${table}: ${(await res.text()).slice(0, 500)}`); await res.arrayBuffer(); } @@ -133,7 +133,8 @@ const DDL = [ unit LowCardinality(String), last_seen UInt32 ) ENGINE = ${engine("ReplacingMergeTree", "last_seen")} ORDER BY (org_id, metric, series_id)`, `CREATE TABLE IF NOT EXISTS ${DB}.metric_points${onCluster} ( - org_id UInt32, metric LowCardinality(String), series_id UInt64, host LowCardinality(String), ts UInt32 CODEC(Delta, ZSTD(1)), value Float64 CODEC(Gorilla, ZSTD(1)) + org_id UInt32, metric LowCardinality(String), series_id UInt64, host LowCardinality(String), ts UInt32 CODEC(Delta, ZSTD(1)), value Float64 CODEC(Gorilla, ZSTD(1)), + weight Float64 DEFAULT 1 CODEC(ZSTD(1)) ) ENGINE = ${engine("MergeTree")} PARTITION BY toYYYYMMDD(toDateTime(ts)) ORDER BY (org_id, metric, series_id, ts) ${ttl("toDateTime(ts)")}`, // Per-minute APM rollups, maintained by ClickHouse itself from top-level spans (the same log-scale histogram @@ -160,6 +161,8 @@ export function ensureChSchema(): Promise { ready = (async () => { // the database itself must be created without ?database= (it does not exist yet) for (const sql of DDL) await chQuery(sql, {}, { noDb: sql.startsWith("CREATE DATABASE") }); + // installs created before weighted distribution points + await chQuery(`ALTER TABLE ${DB}.metric_points${onCluster} ADD COLUMN IF NOT EXISTS weight Float64 DEFAULT 1 CODEC(ZSTD(1))`); // keep TTLs in step with DATACAT_RETENTION_DAYS when it changes between restarts for (const [t, e] of [ ["spans", "toDateTime(intDiv(start_ms, 1000))"], diff --git a/artifacts/api-server/src/lib/config.ts b/artifacts/api-server/src/lib/config.ts index 86ca723..cebc446 100644 --- a/artifacts/api-server/src/lib/config.ts +++ b/artifacts/api-server/src/lib/config.ts @@ -2,6 +2,8 @@ import type { Request } from "express"; export const publicUrl = process.env.PUBLIC_URL?.replace(/\/$/, ""); if (publicUrl && !/^https?:\/\//.test(publicUrl)) throw new Error("PUBLIC_URL must be an http(s) URL"); +// Without it, links and the agent installer would be built from the request's Host header. +if (process.env.NODE_ENV === "production" && !publicUrl) throw new Error("Set PUBLIC_URL to DataCat's external URL, e.g. https://observe.example.com"); export function getOrigin(req: Request): string { return publicUrl ?? `${req.protocol}://${req.get("host")}`; } diff --git a/artifacts/api-server/src/lib/http.ts b/artifacts/api-server/src/lib/http.ts index 7543f7b..3bca94a 100644 --- a/artifacts/api-server/src/lib/http.ts +++ b/artifacts/api-server/src/lib/http.ts @@ -16,6 +16,21 @@ export class HttpError extends Error { * their message. Database errors never reach the caller: invalid input caught by the database is a generic 400, and * anything else is logged and answered with a generic 500. */ +const NET_CODES = new Set(["ECONNREFUSED", "ECONNRESET", "ETIMEDOUT", "EPIPE", "ENOTFOUND", "EAI_AGAIN", "EHOSTUNREACH", "UND_ERR_SOCKET"]); +/** + * True for a temporary storage failure (database or ClickHouse unavailable, connection or server error), which intake + * answers with 503 so agents retry. PostgreSQL data errors (classes 22 and 23) are the payload's fault, not storage's. + */ +export function isStorageError(err: any, depth = 0): boolean { + if (!err || depth > 3) return false; + const code = typeof err.code === "string" ? err.code : ""; + const msg = String(err.message ?? ""); + if (/^[0-9A-Z]{5}$/.test(code)) return !/^2[23]/.test(code); + if (NET_CODES.has(code) || /^ClickHouse[: ]/.test(msg) || msg === "fetch failed") return true; + if (/Connection terminated|timeout exceeded when trying to connect|Client has encountered a connection error/i.test(msg)) return true; + return isStorageError(err.cause, depth + 1); +} + export const h = (fn: (req: Request, res: Response) => Promise) => (req: Request, res: Response) => { fn(req, res).catch((e: any) => { if (res.headersSent) return; diff --git a/artifacts/api-server/src/lib/intakeStats.ts b/artifacts/api-server/src/lib/intakeStats.ts index 7efb5db..a457858 100644 --- a/artifacts/api-server/src/lib/intakeStats.ts +++ b/artifacts/api-server/src/lib/intakeStats.ts @@ -37,7 +37,7 @@ export function intakeStats(orgId: number): Record { } /** Points with a metric name and finite timestamp/value are stored; the rest are dropped. */ -export function splitPoints(series: T[]) { +export function splitPoints(series: T[]) { let valid = 0; let invalid = 0; for (const s of series) { diff --git a/artifacts/api-server/src/lib/migrate.ts b/artifacts/api-server/src/lib/migrate.ts index a0a9b7a..8dde2a9 100644 --- a/artifacts/api-server/src/lib/migrate.ts +++ b/artifacts/api-server/src/lib/migrate.ts @@ -87,6 +87,10 @@ CREATE TABLE IF NOT EXISTS notification_log ( ALTER TABLE notification_log ADD COLUMN IF NOT EXISTS attempts integer NOT NULL DEFAULT 1; ALTER TABLE notification_log ADD COLUMN IF NOT EXISTS next_attempt_at bigint; CREATE INDEX IF NOT EXISTS notification_log_retry ON notification_log(next_attempt_at) WHERE status = 'retrying'; +`, + // 3: weighted distribution points (Datadog Agent sketches store each bin as one value with its count); NULL means 1 + ` +ALTER TABLE metric_points ADD COLUMN IF NOT EXISTS weight double precision; `, ]; diff --git a/artifacts/api-server/src/lib/org.ts b/artifacts/api-server/src/lib/org.ts index cbd9042..33490aa 100644 --- a/artifacts/api-server/src/lib/org.ts +++ b/artifacts/api-server/src/lib/org.ts @@ -1,3 +1,4 @@ +import { logger } from "./logger"; import crypto from "crypto"; import { db, orgsTable, usersTable, apiKeysTable } from "@workspace/db"; import { and, eq } from "drizzle-orm"; @@ -30,13 +31,13 @@ export async function orgForApiKey(key: string | undefined | null): Promise {}); + .catch((e) => logger.debug({ err: String(e) }, "api key last_used_at update failed")); return row.orgId; } diff --git a/artifacts/api-server/src/lib/safeHttp.ts b/artifacts/api-server/src/lib/safeHttp.ts index 0effc82..fa81595 100644 --- a/artifacts/api-server/src/lib/safeHttp.ts +++ b/artifacts/api-server/src/lib/safeHttp.ts @@ -27,18 +27,58 @@ for (const [a, p] of [ for (const [a, p] of [ ["::", 128], ["::1", 128], + ["64:ff9b:1::", 48], // local-use NAT64 + ["100::", 64], // discard-only + ["2001::", 32], // Teredo + ["2001:db8::", 32], // documentation ["fc00::", 7], ["fe80::", 10], + ["fec0::", 10], // site-local (deprecated) ["ff00::", 8], ] as const) blocked.addSubnet(a, p, "ipv6"); +/** The 16 bytes of an IPv6 address (any textual form, including a trailing dotted IPv4), or null. */ +function ipv6Bytes(ip: string): number[] | null { + if (net.isIP(ip) !== 6) return null; + let s = ip.split("%")[0]; + const v4 = /(\d+\.\d+\.\d+\.\d+)$/.exec(s); + if (v4) { + const b = v4[1].split(".").map(Number); + s = s.slice(0, -v4[1].length) + ((b[0] << 8) | b[1]).toString(16) + ":" + ((b[2] << 8) | b[3]).toString(16); + } + const [head, tail] = s.includes("::") ? s.split("::") : [s, null]; + const h = head ? head.split(":") : []; + const t = tail ? tail.split(":") : []; + const groups = tail === null ? h : [...h, ...Array(8 - h.length - t.length).fill("0"), ...t]; + if (groups.length !== 8) return null; + return groups.flatMap((g) => { + const n = parseInt(g || "0", 16); + return [n >> 8, n & 255]; + }); +} + +/** IPv4 address carried inside an IPv6 address (mapped, compatible, translated, NAT64 or 6to4), if any. */ +export function embeddedIpv4(ip: string): string | null { + const b = ipv6Bytes(ip); + if (!b) return null; + const dotted = (o: number) => b.slice(o, o + 4).join("."); + const zero = (from: number, to: number) => b.slice(from, to).every((x) => x === 0); + if (zero(0, 10) && ((b[10] === 0xff && b[11] === 0xff) || (b[10] === 0 && b[11] === 0))) return dotted(12); // ::ffff:a.b.c.d, ::a.b.c.d + if (zero(0, 8) && b[8] === 0xff && b[9] === 0xff && b[10] === 0 && b[11] === 0) return dotted(12); // ::ffff:0:a.b.c.d + if (b[0] === 0 && b[1] === 0x64 && b[2] === 0xff && b[3] === 0x9b && zero(4, 12)) return dotted(12); // 64:ff9b::/96 + if (b[0] === 0x20 && b[1] === 0x02) return dotted(2); // 2002::/16 (6to4) + return null; +} + export function isBlockedAddress(ip: string): boolean { if (allowPrivate) return false; - const mapped = /^::ffff:(\d+\.\d+\.\d+\.\d+)$/i.exec(ip); - if (mapped) return blocked.check(mapped[1], "ipv4"); const family = net.isIP(ip); if (!family) return true; + if (family === 6) { + const v4 = embeddedIpv4(ip); + if (v4) return blocked.check(v4, "ipv4"); + } return blocked.check(ip, family === 4 ? "ipv4" : "ipv6"); } @@ -71,19 +111,41 @@ const safeLookup: net.LookupFunction = (hostname, options, callback) => { }); }; -export function postWebhook(raw: string, headers: Record, body: string, timeoutMs = 10_000): Promise<{ status: number }> { +// Headers that control the connection or message framing; DataCat sets these itself. +const RESERVED_HEADERS = new Set(["host", "transfer-encoding", "connection", "content-length", "upgrade", "te", "keep-alive", "proxy-connection", "trailer"]); + +/** Validates custom webhook headers: a JSON object of string values, without connection or framing headers. */ +export function checkWebhookHeaders(h: unknown): Record | null { + if (h === null || h === undefined) return null; + if (typeof h !== "object" || Array.isArray(h)) throw Object.assign(new Error("headers must be a JSON object"), { status: 400 }); + for (const [k, v] of Object.entries(h as Record)) { + if (!/^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/.test(k)) throw Object.assign(new Error(`Invalid header name: ${k}`), { status: 400 }); + if (RESERVED_HEADERS.has(k.toLowerCase())) throw Object.assign(new Error(`The ${k} header cannot be set on a webhook`), { status: 400 }); + if ((typeof v !== "string" && typeof v !== "number") || /[\r\n]/.test(String(v))) + throw Object.assign(new Error(`Header ${k} must be a single-line string`), { status: 400 }); + } + return h as Record; +} + +/** + * POSTs to a webhook. idleMs limits inactivity; totalMs limits the whole delivery, so a receiver that trickles bytes + * cannot keep it open. + */ +export function postWebhook(raw: string, headers: Record, body: string, idleMs = 10_000, totalMs = 15_000): Promise<{ status: number }> { const u = checkWebhookUrl(raw); const mod = u.protocol === "https:" ? https : http; + const safeHeaders: Record = {}; + for (const [k, v] of Object.entries(headers)) if (!RESERVED_HEADERS.has(k.toLowerCase())) safeHeaders[k] = String(v); + safeHeaders["content-length"] = Buffer.byteLength(body); return new Promise((resolve, reject) => { - const req = mod.request( - u, - { method: "POST", headers: { ...headers, "content-length": Buffer.byteLength(body) }, lookup: safeLookup, timeout: timeoutMs }, - (res) => { - res.resume(); - res.on("end", () => resolve({ status: res.statusCode ?? 0 })); - res.on("error", reject); - }, - ); + const req = mod.request(u, { method: "POST", headers: safeHeaders, lookup: safeLookup, timeout: idleMs }, (res) => { + res.resume(); + res.on("end", () => resolve({ status: res.statusCode ?? 0 })); + res.on("error", reject); + res.on("aborted", () => reject(Object.assign(new Error("response aborted"), { code: "ECONNRESET" }))); + }); + const deadline = setTimeout(() => req.destroy(Object.assign(new Error("timeout"), { code: "ETIMEDOUT" })), totalMs); + req.on("close", () => clearTimeout(deadline)); req.on("timeout", () => req.destroy(Object.assign(new Error("timeout"), { code: "ETIMEDOUT" }))); req.on("error", reject); req.end(body); diff --git a/artifacts/api-server/src/middlewares/authMiddleware.ts b/artifacts/api-server/src/middlewares/authMiddleware.ts index 6553446..e58af85 100644 --- a/artifacts/api-server/src/middlewares/authMiddleware.ts +++ b/artifacts/api-server/src/middlewares/authMiddleware.ts @@ -23,7 +23,7 @@ export async function authMiddleware(req: Request, res: Response, next: NextFunc const sid = getSessionId(req); // Cookie sessions: a state-changing request must come from DataCat's own pages (SameSite=Lax cookies already stop // most cross-site requests; this also covers sibling subdomains). - if (sid && !SAFE_METHODS.has(req.method) && !req.headers["authorization"]) { + if (sid && !SAFE_METHODS.has(req.method)) { const origin = req.get("origin"); if ((origin && origin !== getOrigin(req)) || req.get("sec-fetch-site") === "cross-site") { res.status(403).json({ errors: ["Invalid request origin"] }); diff --git a/artifacts/api-server/src/query/apmStats.ts b/artifacts/api-server/src/query/apmStats.ts index a94bb0a..9aa6657 100644 --- a/artifacts/api-server/src/query/apmStats.ts +++ b/artifacts/api-server/src/query/apmStats.ts @@ -26,7 +26,6 @@ function valueOf(idx: number): number { return MIN_NS * Math.pow(GAMMA, idx) * (2 / (1 + GAMMA)); } -const BOOT_MS = Date.now(); let ready: Promise | null = null; export function ensureSpanStats(): Promise { if (useClickHouse) return ch.ensureSpanStats(); @@ -44,8 +43,12 @@ export function ensureSpanStats(): Promise { pool .query("create index concurrently if not exists spans_errors on spans (org_id, service, start_ms) where error = 1") .catch((e) => logger.warn({ err: e }, "spans_errors index creation failed")); - // First run on an existing install: build rollups for spans stored before this version (in the background). - if (!existed) backfillSpanStats().catch((e) => logger.warn({ err: e }, "span_stats backfill failed")); + // First run on an existing install: build rollups for spans stored before this point (in the background). The + // limit is the highest span id now, so spans inserted later (whatever their start time) are counted once, by intake. + if (!existed) { + const maxId = Number((await pool.query("select coalesce(max(id), 0) as m from spans")).rows[0].m); + backfillSpanStats(maxId).catch((e) => logger.warn({ err: e }, "span_stats backfill failed")); + } })().catch((e) => { ready = null; throw e; @@ -132,10 +135,6 @@ export async function recordSpanStats(orgId: number, spans: SpanLike[]) { process.once("beforeExit", () => { void flushSpanStats(); }); - for (const sig of ["SIGTERM", "SIGINT"] as const) - process.once(sig, () => { - flushSpanStats().finally(() => process.exit(0)); - }); } } @@ -351,7 +350,7 @@ export async function serviceOperations(orgId: number, service: string, from: nu } /** One-time backfill of rollups from raw spans stored before the table existed (spans written since boot are recorded by intake). */ -async function backfillSpanStats() { +async function backfillSpanStats(maxId: number) { const t0 = Date.now(); await pool.query( `insert into span_stats (org_id, bucket, service, env, name, resource, type, lang, hits, errors, dur_sum, dur_max, hist_idx, hist_cnt) @@ -359,7 +358,7 @@ async function backfillSpanStats() { from (select org_id, (start_ms / 60000) * 60 as bucket, service, env, name, resource, max(type) as type, max(meta->>'language') as lang, case when duration <= ${MIN_NS} then 0 else ceil(ln(duration::float8 / ${MIN_NS}) / ln(${GAMMA}))::int end as idx, count(*)::int as c, sum(error)::int as e, sum(duration)::float8 as s, max(duration)::float8 as m - from spans where is_top_level=1 and start_ms < ${BOOT_MS} group by 1,2,3,4,5,6,9) x + from spans where is_top_level=1 and id <= ${Math.floor(maxId)} group by 1,2,3,4,5,6,9) x group by org_id, bucket, service, env, name, resource`, ); logger.info({ ms: Date.now() - t0 }, "span_stats backfilled from spans"); diff --git a/artifacts/api-server/src/query/metrics.ts b/artifacts/api-server/src/query/metrics.ts index 7d3caaa..4c66dce 100644 --- a/artifacts/api-server/src/query/metrics.ts +++ b/artifacts/api-server/src/query/metrics.ts @@ -59,6 +59,8 @@ export function parseFilter(s: string): FilterNode | null { const src = s.trim(); if (!src || src === "*") return null; const toks = tokenizeFilter(src); + for (const t of toks) + if ((t.match(/[*?]/g)?.length ?? 0) > MAX_WILDCARDS) throw new HttpError(400, `A tag filter may use at most ${MAX_WILDCARDS} wildcards`); let p = 0; const peek = () => toks[p]; function parseOr(): FilterNode { @@ -129,25 +131,36 @@ export function parseFilter(s: string): FilterNode | null { return parseOr(); } -function globToRe(g: string): RegExp { - return new RegExp( - "^" + - g - .replace(/[.+^${}()|[\]\\]/g, "\\$&") - .replace(/\*/g, ".*") - .replace(/\?/g, ".") + - "$", - "i", - ); -} -const reCache = new Map(); -function globMatch(pattern: string, v: string) { +const isLineBreak = (c: number) => c === 10 || c === 13 || c === 0x2028 || c === 0x2029; +/** + * Case-insensitive glob match ("*" any run of characters, "?" one character; neither crosses a line break). Greedy + * matching that backs up only to the most recent "*", so the cost stays proportional to pattern × value length. + */ +export function globMatch(pattern: string, v: string) { if (pattern === "*") return true; - if (!pattern.includes("*") && !pattern.includes("?")) return pattern.toLowerCase() === v.toLowerCase(); - let re = reCache.get(pattern); - if (!re) reCache.set(pattern, (re = globToRe(pattern))); - return re.test(v); + const p = pattern.toLowerCase(); + const s = v.toLowerCase(); + if (!p.includes("*") && !p.includes("?")) return p === s; + let i = 0, + j = 0, + star = -1, + mark = 0; + while (j < s.length) { + if (i < p.length && p[i] !== "*" && (p[i] === s[j] || (p[i] === "?" && !isLineBreak(s.charCodeAt(j))))) { + i++; + j++; + } else if (i < p.length && p[i] === "*") { + star = i++; + mark = j; + } else if (star >= 0 && !isLineBreak(s.charCodeAt(mark))) { + i = star + 1; + j = ++mark; + } else return false; + } + while (i < p.length && p[i] === "*") i++; + return i === p.length; } +const MAX_WILDCARDS = 20; const normTag = (v: string) => v.toLowerCase().replace(/[^a-z0-9_:./\-*?]+/g, "_"); @@ -407,7 +420,7 @@ export async function runMetricQuery( p as (select series_id, (ts / $3) * $3 as b, ${col} as v from metric_points where series_id = any($1) and ts >= $4 and ts < $5 group by 1, 2) select s.g, p.b, ${spaceCol} as v from p join s on s.id = p.series_id group by 1, 2 order by 1, 2`, - [matched.map((m) => m.id), gOf, rollInterval, Math.floor(from / rollInterval) * rollInterval, to], + [matched.map((m) => m.id), gOf, rollInterval, from, to], ); // as_rate / as_count scaling is linear, so applying it after sum/avg/min/max equals applying it per series const scale = q.spaceAgg === "count" ? 1 : type === "count" && asRate ? 1 / rollInterval : type === "rate" && asCount ? rollInterval : 1; @@ -440,7 +453,7 @@ function spaceAggregate(agg: string, vs: number[]): number { } function fillSeries(set: SeriesSet, mode: string, from: number, to: number, interval: number) { - if (mode === "null") return; + if (mode === "null" || !(interval > 0)) return; for (const s of set) { let last: number | undefined; for (let b = Math.floor(from / interval) * interval; b < to; b += interval) { @@ -484,17 +497,26 @@ async function runDistributionQuery( to: number, ): Promise { const pct = percentileOf(q.spaceAgg); - const agg = pct !== null ? `p:${pct}` : q.spaceAgg; - const rows = useClickHouse - ? await aggregateDistributionCh(orgId, q.metric, ids, g.gOf, agg, step, from, to) - : ( - await pool.query( - `with s as (select * from unnest($1::int[], $2::int[]) as s(id, g)) - select s.g, (p.ts / $3) * $3 as b, ${pct !== null ? `percentile_cont(${pct}) within group (order by p.value)` : (({ sum: "sum(p.value)", min: "min(p.value)", max: "max(p.value)", count: "count(*)::double precision" } as Record)[q.spaceAgg] ?? "avg(p.value)")} as v - from metric_points p join s on s.id = p.series_id where p.series_id = any($1) and p.ts >= $4 and p.ts < $5 group by 1, 2 order by 1, 2`, - [ids, g.gOf, step, Math.floor(from / step) * step, to], - ) - ).rows; + const agg = pct !== null ? "values" : q.spaceAgg; + let rows: { g: number; b: number; v: number }[]; + if (useClickHouse) rows = await aggregateDistributionCh(orgId, q.metric, ids, g.gOf, agg, step, from, to); + else { + const w = "coalesce(p.weight, 1)"; + const col = + agg === "values" + ? `p.value as v, sum(${w}) as w` + : `${({ sum: `sum(p.value * ${w})`, min: "min(p.value)", max: "max(p.value)", count: `sum(${w})::double precision` } as Record)[agg] ?? `sum(p.value * ${w}) / nullif(sum(${w}), 0)`} as v`; + rows = ( + await pool.query( + `with s as (select * from unnest($1::int[], $2::int[]) as s(id, g)) + select s.g, (p.ts / $3) * $3 as b, ${col} + from metric_points p join s on s.id = p.series_id where p.series_id = any($1) and p.ts >= $4 and p.ts < $5 + group by 1, 2${agg === "values" ? ", p.value" : ""} order by 1, 2`, + [ids, g.gOf, step, from, to], + ) + ).rows; + } + if (pct !== null) rows = weightedPercentileRows(rows as any, pct); const byGroup = new Map>(); for (const r of rows as { g: number; b: number; v: number }[]) { let m = byGroup.get(Number(r.g)); @@ -530,10 +552,15 @@ function binop(op: string, a: Val, b: Val): Val { if (additive && A.length === 0) return op === "+" ? B : B.map((s) => mapPoints(s, (v) => -v)); const out: SeriesSet = []; const bIndex = new Map(B.map((s) => [s.key, s])); - const bAll = B.length === 1 ? B[0] : undefined; + // An ungrouped series ("*") applies to every group of the other side; grouped series combine only when their group + // keys are equal. For + and -, a group present on one side only is kept (the missing side counts as 0). + const bAll = B.length === 1 && B[0].key === "*" ? B[0] : undefined; + const aAll = A.length === 1 && A[0].key === "*" ? A[0] : undefined; + const empty: Series = { key: "", tags: [], points: new Map() }; for (const sa of A) { - const sb = - bIndex.get(sa.key) ?? (bAll && (bAll.key === "*" || A.length === 1) ? bAll : undefined) ?? (A.length === 1 && sa.key === "*" ? undefined : undefined); + let sb = bIndex.get(sa.key) ?? bAll; + if (!sb && aAll && B.length > 1) break; // handled below: A is ungrouped, B is grouped + if (!sb && additive) sb = { ...empty, key: sa.key, tags: sa.tags }; if (!sb) continue; const pts = new Map(); for (const [t, v] of sa.points) { @@ -551,7 +578,19 @@ function binop(op: string, a: Val, b: Val): Val { } out.push({ key: sa.key, tags: sa.tags.length ? sa.tags : sb.tags, points: pts }); } - if (A.length === 1 && A[0].key === "*" && B.length > 1) { + if (additive && !aAll && !bAll) { + const aKeys = new Set(A.map((s) => s.key)); + for (const sb of B) + if (!aKeys.has(sb.key)) { + const pts = new Map(); + for (const [t, w] of sb.points) { + const r = f(0, w); + if (Number.isFinite(r)) pts.set(t, r); + } + out.push({ key: sb.key, tags: sb.tags, points: pts }); + } + } + if (aAll && B.length > 1) { return B.map((sb) => { const pts = new Map(); for (const [t, w] of sb.points) { @@ -692,8 +731,51 @@ export function evalFormula(formula: string, env: Record, int return typeof v === "number" ? [] : v; } +/** + * Percentile per (group, bucket) of weighted values, with the same interpolation as percentile_cont over the values + * repeated by their weights. Rows are (g, b, v, w), one per distinct value. + */ +export function weightedPercentileRows(rows: { g: number; b: number; v: number; w: number }[], q: number): { g: number; b: number; v: number }[] { + const cells = new Map(); + for (const r of rows) { + const k = `${r.g}|${r.b}`; + let c = cells.get(k); + if (!c) cells.set(k, (c = { g: Number(r.g), b: Number(r.b), vs: [] })); + c.vs.push([Number(r.v), Number(r.w)]); + } + const out: { g: number; b: number; v: number }[] = []; + for (const c of cells.values()) { + c.vs.sort((x, y) => x[0] - y[0]); + const total = c.vs.reduce((a, x) => a + x[1], 0); + if (!(total > 0)) continue; + const r = q * (total - 1); + const at = (pos: number) => { + let cum = 0; + for (const [v, w] of c.vs) { + cum += w; + if (cum > pos) return v; + } + return c.vs[c.vs.length - 1][0]; + }; + const lo = at(Math.floor(r)); + out.push({ g: c.g, b: c.b, v: lo + (at(Math.ceil(r)) - lo) * (r - Math.floor(r)) }); + } + return out.sort((x, y) => x.g - y.g || x.b - y.b); +} + +/** Linear interpolation between closest ranks (the method of PostgreSQL's percentile_cont). */ +export function interpolatedPercentile(vs: number[], q: number): number { + const s = [...vs].sort((a, b) => a - b); + const r = q * (s.length - 1); + const lo = Math.floor(r); + return s[lo] + (s[Math.min(s.length - 1, lo + 1)] - s[lo]) * (r - lo); +} + +/** Reduce the values of a window, in time order. agg "pNN" is the NNth percentile. */ export function reduce(vs: number[], agg: string): number { if (!vs.length) return NaN; + const pm = /^p(\d{1,2}(?:\.\d+)?)$/.exec(agg); + if (pm && agg !== "p95") return interpolatedPercentile(vs, Number(pm[1]) / 100); switch (agg) { case "sum": return vs.reduce((a, b) => a + b, 0); @@ -705,11 +787,13 @@ export function reduce(vs: number[], agg: string): number { return vs[vs.length - 1]; case "count": return vs.length; + case "change": + return vs[vs.length - 1] - vs[0]; + case "pct_change": + return vs[0] === 0 ? NaN : ((vs[vs.length - 1] - vs[0]) / Math.abs(vs[0])) * 100; case "percentile": - case "p95": { - const s = [...vs].sort((a, b) => a - b); - return s[Math.min(s.length - 1, Math.floor(s.length * 0.95))]; - } + case "p95": + return interpolatedPercentile(vs, 0.95); default: return vs.reduce((a, b) => a + b, 0) / vs.length; } @@ -725,6 +809,8 @@ export interface QueryRequest { } export interface QueryResultSeries { + /** index of the formula (or query, when there are no formulas) this series belongs to */ + queryIndex: number; name: string; alias?: string; formula: string; @@ -733,27 +819,49 @@ export interface QueryResultSeries { interval: number; // bucket width in seconds } +const MAX_BUCKETS = 10_000; +/** Bucket width in seconds: a requested interval must be a positive number; it is widened to at most MAX_BUCKETS buckets. */ +export function resolveInterval(requested: unknown, from: number, to: number): number { + if (requested === undefined || requested === null) return pickInterval(from, to); + const n = Number(requested); + if (!Number.isFinite(n) || n <= 0) throw new HttpError(400, "interval must be a positive number of seconds"); + return Math.max(1, Math.round(n), Math.ceil((to - from) / MAX_BUCKETS) || 1); +} + /** Run a set of named queries + formulas (v2 timeseries style). Supports inline arithmetic in a query. */ export async function runQueries(orgId: number, req: QueryRequest): Promise<{ series: QueryResultSeries[]; interval: number }> { - const interval = req.interval ?? pickInterval(req.from, req.to); + const interval = resolveInterval(req.interval, req.from, req.to); const env: Record = {}; const exprs: Record = {}; + const raws: Record = {}; for (const q of req.queries) { if (!q.query?.trim()) continue; const { formula, queries } = extractInlineQueries(q.query.trim(), `__q_${q.name.replace(/\W/g, "")}_`); - for (const [n, raw] of Object.entries(queries)) env[n] = await runMetricQuery(orgId, parseMetricQuery(raw), req.from, req.to, interval, req.vars); + for (const [n, raw] of Object.entries(queries)) { + raws[n] = raw; + env[n] = await runMetricQuery(orgId, parseMetricQuery(raw), req.from, req.to, interval, req.vars); + } exprs[q.name] = formula; } + // Operands of a formula must share one bucket size: when a .rollup() widened some query, the others are re-run at + // that width (from raw data, with their own aggregation) instead of being matched point by point. + let widest = interval; + for (const set of Object.values(env)) for (const s of set) if (s.interval && s.interval > widest) widest = s.interval; + if (widest > interval) + for (const [n, raw] of Object.entries(raws)) + if (env[n].some((s) => (s.interval ?? interval) < widest)) + env[n] = await runMetricQuery(orgId, parseMetricQuery(raw), req.from, req.to, widest, req.vars); for (const [name, f] of Object.entries(exprs)) env[name] = /^__q\w+$/.test(f.trim()) && env[f.trim()] ? env[f.trim()] : evalFormula(f, env, interval); const formulas = req.formulas?.length ? req.formulas : req.queries.filter((q) => q.query?.trim()).map((q) => ({ formula: q.name, alias: undefined })); // The reported interval is the widest bucket actually used (a .rollup(…, N) can widen it past the default). let effective = interval; for (const set of Object.values(env)) for (const s of set) if (s.interval && s.interval > effective) effective = s.interval; const series: QueryResultSeries[] = []; - for (const f of formulas) { + for (const [fi, f] of formulas.entries()) { const set = evalFormula(f.formula, env, effective); for (const s of set) { series.push({ + queryIndex: fi, name: f.formula, alias: f.alias, formula: f.formula, diff --git a/artifacts/api-server/src/query/metricsCh.ts b/artifacts/api-server/src/query/metricsCh.ts index 5ebe56d..445193d 100644 --- a/artifacts/api-server/src/query/metricsCh.ts +++ b/artifacts/api-server/src/query/metricsCh.ts @@ -35,7 +35,7 @@ export async function aggregateCh( return chQuery( `SELECT g, b, ${spaceCol} AS v FROM ( SELECT series_id, intDiv(ts, ${step}) * ${step} AS b, ${timeCol} AS v FROM metric_points - WHERE org_id = ${b.p(orgId, "UInt32")} AND metric = ${b.p(metric)} AND ts >= ${b.p(Math.floor(from / step) * step, "UInt32")} AND ts < ${b.p(to, "UInt32")} + WHERE org_id = ${b.p(orgId, "UInt32")} AND metric = ${b.p(metric)} AND ts >= ${b.p(from, "UInt32")} AND ts < ${b.p(to, "UInt32")} AND series_id IN (SELECT id FROM sel) GROUP BY series_id, b ) AS p INNER JOIN sel ON sel.id = p.series_id GROUP BY g, b ORDER BY g, b`, b.params, @@ -54,18 +54,20 @@ export async function aggregateDistributionCh( from: number, to: number, ) { - const col = agg.startsWith("p:") - ? `quantileExact(${Number(agg.slice(2))})(value)` - : (({ sum: "sum(value)", min: "min(value)", max: "max(value)", count: "toFloat64(count())" } as Record)[agg] ?? "avg(value)"); + // agg "values": one row per distinct value with its total weight (percentiles are computed by the caller) + const col = + agg === "values" + ? "p.value AS v, sum(p.weight) AS w" + : `${({ sum: "sum(p.value * p.weight)", min: "min(p.value)", max: "max(p.value)", count: "sum(p.weight)" } as Record)[agg] ?? "sum(p.value * p.weight) / nullIf(sum(p.weight), 0)"} AS v`; const b = new ChParams(); const tsv = ids.map((id, i) => `${id}\t${groups[i]}`).join("\n"); return chQuery( - `SELECT sel.g AS g, intDiv(p.ts, ${step}) * ${step} AS b, ${col} AS v FROM metric_points AS p INNER JOIN sel ON sel.id = p.series_id - WHERE p.org_id = ${b.p(orgId, "UInt32")} AND p.metric = ${b.p(metric)} AND p.ts >= ${b.p(Math.floor(from / step) * step, "UInt32")} AND p.ts < ${b.p(to, "UInt32")} - GROUP BY g, b ORDER BY g, b`, + `SELECT sel.g AS g, intDiv(p.ts, ${step}) * ${step} AS b, ${col} FROM metric_points AS p INNER JOIN sel ON sel.id = p.series_id + WHERE p.org_id = ${b.p(orgId, "UInt32")} AND p.metric = ${b.p(metric)} AND p.ts >= ${b.p(from, "UInt32")} AND p.ts < ${b.p(to, "UInt32")} + GROUP BY g, b${agg === "values" ? ", p.value" : ""} ORDER BY g, b`, b.params, { external: [{ name: "sel", structure: "id UInt64, g UInt32", tsv }] }, - ) as Promise<{ g: number; b: number; v: number }[]>; + ) as Promise<{ g: number; b: number; v: number; w?: number }[]>; } export async function listMetricsCh(orgId: number, search: string, limit: number) { diff --git a/artifacts/api-server/src/query/search.ts b/artifacts/api-server/src/query/search.ts index 6bfc3fa..a891a2f 100644 --- a/artifacts/api-server/src/query/search.ts +++ b/artifacts/api-server/src/query/search.ts @@ -139,7 +139,7 @@ export function compileSearch(query: string, resolve: FieldResolver, b: SqlBuild const cmp = /^(>=|<=|>|<)(-?[\d.]+)([a-z]*)$/.exec(val); const range = /^\[(-?[\d.]+)([a-z]*)\s+TO\s+(-?[\d.]+)([a-z]*)\]$/i.exec(val); if (cmp || range) { - const num = f.numSql ?? `NULLIF(${f.sql}, '')::double precision`; + const num = f.numSql ?? safeNumber(f.sql); const conv = (n: string, unit: string) => { let x = Number(n); if (f.transformNum) x = f.transformNum(x); @@ -183,6 +183,11 @@ export function compileSearch(query: string, resolve: FieldResolver, b: SqlBuild } /** Nested JSON lookup with the key bound as a parameter (never spliced into SQL). */ +/** A text expression as a number; values that are not numbers become NULL instead of failing the query. */ +export function safeNumber(x: string) { + return `(CASE WHEN (${x}) ~ '^\\s*-?([0-9]+\\.?[0-9]*|\\.[0-9]+)([eE][-+]?[0-9]+)?\\s*$' THEN (${x})::double precision END)`; +} + export function jsonPath(col: string, key: string, b: SqlBuilder) { return `${col} #>> ${b.p(key.split("."))}::text[]`; } @@ -219,5 +224,5 @@ export const spanResolver: FieldResolver = (key, isAttr, b) => { } if (key === "duration") return { kind: "num", sql: "duration", numSql: "duration::double precision" }; const k = `${b.p(key)}::text`; - return { kind: "json", sql: `COALESCE(meta ->> ${k}, metrics ->> ${k})`, numSql: `NULLIF(COALESCE(metrics ->> ${k}, meta ->> ${k}), '')::double precision` }; + return { kind: "json", sql: `COALESCE(meta ->> ${k}, metrics ->> ${k})`, numSql: safeNumber(`COALESCE(metrics ->> ${k}, meta ->> ${k})`) }; }; diff --git a/artifacts/api-server/src/query/v2.ts b/artifacts/api-server/src/query/v2.ts index 67c22b5..ca64f6a 100644 --- a/artifacts/api-server/src/query/v2.ts +++ b/artifacts/api-server/src/query/v2.ts @@ -105,6 +105,39 @@ export function parseAggregate(body: any): AggregateRequest { return { query: String(filter.query ?? "*") || "*", from, to, computes, groupBy }; } +/** + * Splits computes into query passes: one for all totals (no time grouping) and one per distinct timeseries interval, + * so a total is never replaced by its last time bucket and each timeseries keeps its own interval. + */ +export function aggregatePasses( + req: AggregateRequest, + auto: (from: number, to: number) => number, +): { interval: number | null; computes: AggregateRequest["computes"] }[] { + const passes: { interval: number | null; computes: AggregateRequest["computes"] }[] = []; + const totals = req.computes.filter((c) => !c.timeseries); + if (totals.length) passes.push({ interval: null, computes: totals }); + const byInterval = new Map(); + for (const c of req.computes.filter((x) => x.timeseries)) { + const iv = c.interval ?? auto(req.from, req.to); + byInterval.set(iv, [...(byInterval.get(iv) ?? []), c]); + } + for (const [interval, computes] of byInterval) passes.push({ interval, computes }); + return passes; +} + +/** Merges the buckets of several passes (same group-by) into one bucket per group. */ +export function mergeBuckets(parts: { by: Record; computes: Record }[][]) { + const m = new Map; computes: Record }>(); + for (const part of parts) + for (const b of part) { + const k = JSON.stringify(b.by); + const cur = m.get(k); + if (cur) Object.assign(cur.computes, b.computes); + else m.set(k, { by: b.by, computes: { ...b.computes } }); + } + return [...m.values()]; +} + /** Shape aggregate rows into response buckets: one bucket per group, with total or timeseries computes. */ export function toBuckets(req: AggregateRequest, rows: any[]): { by: Record; computes: Record }[] { const ts = req.computes.some((c) => c.timeseries); diff --git a/artifacts/api-server/src/routes/auth.ts b/artifacts/api-server/src/routes/auth.ts index 201703b..2247bf1 100644 --- a/artifacts/api-server/src/routes/auth.ts +++ b/artifacts/api-server/src/routes/auth.ts @@ -19,9 +19,24 @@ function setupAllowed(req: Request) { const router: IRouter = Router(); -function getSafeReturnTo(value: unknown): string { - if (typeof value !== "string" || !value.startsWith("/") || value.startsWith("//")) return "/"; - return value; +// Only same-origin paths: parsed against a fixed origin so "//host", "/\\host" and similar forms are refused. +export function getSafeReturnTo(value: unknown): string { + if (typeof value !== "string" || !value.startsWith("/")) return "/"; + const base = "http://datacat.invalid"; + try { + const u = new URL(value, base); + return u.origin === base ? u.pathname + u.search + u.hash : "/"; + } catch { + return "/"; + } +} + +// Same accept/reject result as /^[^\s@]+@[^\s@]+\.[^\s@]+$/, in linear time and with the length checked first. +export function validEmail(email: string): boolean { + if (email.length > 254 || /\s/.test(email)) return false; + const at = email.indexOf("@"); + if (at < 1 || at !== email.lastIndexOf("@")) return false; + return email.slice(at + 2, -1).includes("."); } router.get("/auth/user", async (req: Request, res: Response) => { @@ -85,7 +100,7 @@ router.use(["/auth/login", "/auth/setup", "/auth/invites"], (req, res, next) => function credentials(req: Request) { const email = typeof req.body?.email === "string" ? req.body.email.trim().toLowerCase() : ""; const password = req.body?.password; - if (!/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(email) || email.length > 254 || typeof password !== "string" || password.length < 12 || password.length > 256) { + if (!validEmail(email) || typeof password !== "string" || password.length < 12 || password.length > 256) { throw new Error("Enter a valid email and a password of 12–256 characters"); } return { email, password }; diff --git a/artifacts/api-server/src/routes/resources.ts b/artifacts/api-server/src/routes/resources.ts index 66916a8..a0ae353 100644 --- a/artifacts/api-server/src/routes/resources.ts +++ b/artifacts/api-server/src/routes/resources.ts @@ -5,11 +5,12 @@ import { pool, db, apiKeysTable, invitesTable, usersTable, orgsTable } from "@wo import { getOrigin } from "../lib/config"; import { and, eq } from "drizzle-orm"; import { requireOrg, requireAdmin, requireAdminOrKeys } from "../middlewares/authMiddleware"; -import { checkWebhookUrl } from "../lib/safeHttp"; +import { checkWebhookUrl, checkWebhookHeaders } from "../lib/safeHttp"; import { newKey, invalidateKey } from "../lib/org"; import { evaluateMonitor, validateMonitorQuery } from "../jobs/monitors"; import { startSampleData, stopSampleData, sampleRunning } from "../jobs/sample"; import { intakeStats } from "../lib/intakeStats"; +import { logger } from "../lib/logger"; const router: Router = Router(); router.use( @@ -31,6 +32,16 @@ function intId(v: unknown, what: string): number { const monitorId = (v: unknown) => intId(v, "monitor"); const userName = (req: Request) => [req.authUser?.firstName, req.authUser?.lastName].filter(Boolean).join(" ") || req.authUser?.email || "API"; +/** + * LIMIT/OFFSET for list routes that accept paging (dashboards: count/start; monitors: page_size/page, page counted from + * 0). Without the parameters the full list is returned, as before. + */ +function listPage(q: any, sizeKey: string, posKey: string): string { + if (q?.[sizeKey] === undefined && q?.[posKey] === undefined) return ""; + const size = pageLimit(q[sizeKey], 100, 1000); + const pos = Math.max(0, Math.floor(Number(q[posKey] ?? 0)) || 0); + return `limit ${size} offset ${posKey === "page" ? pos * size : pos}`; +} function dashOut(r: any) { return { id: r.id, @@ -54,7 +65,7 @@ router.get( h(async (req, res) => { const rows = ( await pool.query( - "select id, title, description, layout_type, author_name, created_at, modified_at, jsonb_array_length(coalesce(definition->'widgets','[]'::jsonb)) as n from dashboards where org_id=$1 order by modified_at desc", + `select id, title, description, layout_type, author_name, created_at, modified_at, jsonb_array_length(coalesce(definition->'widgets','[]'::jsonb)) as n from dashboards where org_id=$1 order by modified_at desc, id ${listPage(req.query, "count", "start")}`, [req.orgId], ) ).rows; @@ -85,6 +96,18 @@ router.get( res.json(dashOut(r)); }), ); +/** Validates the parts of a dashboard the UI relies on: widgets and template_variables must be lists of objects. */ +function checkDashboard(b: any) { + if (b.title !== undefined && (typeof b.title !== "string" || !b.title.trim())) throw new HttpError(400, "title must be a non-empty string"); + for (const k of ["widgets", "template_variables"]) { + const v = b[k]; + if (v === undefined || v === null) continue; + if (!Array.isArray(v)) throw new HttpError(400, `${k} must be a list`); + if (v.some((x: unknown) => !x || typeof x !== "object" || Array.isArray(x))) throw new HttpError(400, `each entry of ${k} must be an object`); + } + for (const t of b.template_variables ?? []) if (typeof t.name !== "string" || !t.name) throw new HttpError(400, "each template variable needs a name"); + for (const w of b.widgets ?? []) if (!w.definition || typeof w.definition !== "object") throw new HttpError(400, "each widget needs a definition object"); +} function assignWidgetIds(ws: any[]) { for (const w of ws ?? []) { if (!w.id) w.id = crypto.randomInt(1, 2 ** 47); @@ -97,6 +120,7 @@ router.post( h(async (req, res) => { const b = req.body ?? {}; if (!b.title) throw new Error("title is required"); + checkDashboard(b); const id = dashId(); const def = { widgets: assignWidgetIds(b.widgets ?? []), template_variables: b.template_variables ?? [], reflow_type: b.reflow_type }; const r = ( @@ -112,19 +136,32 @@ router.put( "/v1/dashboard/:id", h(async (req, res) => { const b = req.body ?? {}; - const cur = (await pool.query("select definition, description from dashboards where org_id=$1 and id=$2", [req.orgId, req.params.id])).rows[0]; - const prev = cur?.definition ?? {}; - const def = { - widgets: b.widgets !== undefined ? assignWidgetIds(b.widgets ?? []) : (prev.widgets ?? []), - template_variables: b.template_variables ?? prev.template_variables ?? [], - reflow_type: b.reflow_type ?? prev.reflow_type, - }; - const r = ( - await pool.query( - "update dashboards set title=coalesce($3,title), description=$4, layout_type=coalesce($5,layout_type), definition=$6, modified_at=now() where org_id=$1 and id=$2 returning *", - [req.orgId, req.params.id, b.title ?? null, b.description !== undefined ? b.description : (cur?.description ?? null), b.layout_type ?? null, def], - ) - ).rows[0]; + checkDashboard(b); + // read-merge-write under a row lock, so two concurrent partial updates both keep their changes + const c = await pool.connect(); + let r: any; + try { + await c.query("BEGIN"); + const cur = (await c.query("select definition, description from dashboards where org_id=$1 and id=$2 for update", [req.orgId, req.params.id])).rows[0]; + const prev = cur?.definition ?? {}; + const def = { + widgets: b.widgets !== undefined ? assignWidgetIds(b.widgets ?? []) : (prev.widgets ?? []), + template_variables: b.template_variables ?? prev.template_variables ?? [], + reflow_type: b.reflow_type ?? prev.reflow_type, + }; + r = ( + await c.query( + "update dashboards set title=coalesce($3,title), description=$4, layout_type=coalesce($5,layout_type), definition=$6, modified_at=now() where org_id=$1 and id=$2 returning *", + [req.orgId, req.params.id, b.title ?? null, b.description !== undefined ? b.description : (cur?.description ?? null), b.layout_type ?? null, def], + ) + ).rows[0]; + await c.query("COMMIT"); + } catch (e) { + await c.query("ROLLBACK").catch(() => undefined); + throw e; + } finally { + c.release(); + } if (!r) { res.status(404).json({ errors: ["Dashboard not found"] }); return; @@ -165,7 +202,7 @@ function monOut(r: any) { router.get( "/v1/monitor", h(async (req, res) => { - const rows = (await pool.query("select * from monitors where org_id=$1 order by name", [req.orgId])).rows; + const rows = (await pool.query(`select * from monitors where org_id=$1 order by name, id ${listPage(req.query, "page_size", "page")}`, [req.orgId])).rows; res.json(rows.map(monOut)); }), ); @@ -200,7 +237,7 @@ router.post( [req.orgId, b.name, type, b.query, b.message ?? "", b.tags ?? [], b.priority ?? null, b.options ?? {}, req.authUser?.id ?? null, userName(req)], ) ).rows[0]; - evaluateMonitor(r.id).catch(() => {}); + evaluateMonitor(r.id).catch((e) => logger.warn({ err: String(e), id: r.id }, "monitor evaluation after create failed")); res.json(monOut(r)); }), ); @@ -208,7 +245,15 @@ router.put( "/v1/monitor/:id", h(async (req, res) => { const b = req.body ?? {}; - if (b.query) validateMonitorQuery(b.type ?? "metric alert", b.query); + const cur = (await pool.query("select type, query, priority from monitors where org_id=$1 and id=$2", [req.orgId, monitorId(req.params.id)])).rows[0]; + if (!cur) { + res.status(404).json({ errors: ["Monitor not found"] }); + return; + } + // the monitor as it will be saved must be valid, whichever fields the request changes + if (b.query !== undefined && !String(b.query ?? "").trim()) throw new HttpError(400, "query cannot be empty"); + if (b.name !== undefined && !String(b.name ?? "").trim()) throw new HttpError(400, "name cannot be empty"); + validateMonitorQuery(b.type ?? cur.type, b.query ?? cur.query); const r = ( await pool.query( `update monitors set name=coalesce($3,name), type=coalesce($4,type), query=coalesce($5,query), message=coalesce($6,message), tags=coalesce($7,tags), @@ -221,7 +266,7 @@ router.put( b.query ?? null, b.message ?? null, b.tags ?? null, - b.priority ?? null, + b.priority !== undefined ? b.priority : cur.priority, b.options ?? null, ], ) @@ -230,7 +275,7 @@ router.put( res.status(404).json({ errors: ["Monitor not found"] }); return; } - evaluateMonitor(r.id).catch(() => {}); + evaluateMonitor(r.id).catch((e) => logger.warn({ err: String(e), id: r.id }, "monitor evaluation after update failed")); res.json(monOut(r)); }), ); @@ -488,7 +533,7 @@ router.post( req.orgId, kind === "slack" ? "slack" : "webhook", name, - { url, headers: headers ?? null, payload: payload ?? null }, + { url, headers: checkWebhookHeaders(headers), payload: payload ?? null }, ]) ).rows[0]; res.json(r); @@ -537,6 +582,7 @@ function webhookConfig(b: any, prev: any = {}) { throw new HttpError(400, "custom_headers must be a JSON object"); } if (!headers || typeof headers !== "object" || Array.isArray(headers)) throw new HttpError(400, "custom_headers must be a JSON object"); + checkWebhookHeaders(headers); } } const encode_as = b.encode_as ?? prev.encode_as ?? "json"; diff --git a/artifacts/api-server/src/routes/telemetry.ts b/artifacts/api-server/src/routes/telemetry.ts index e18042d..c8f68f6 100644 --- a/artifacts/api-server/src/routes/telemetry.ts +++ b/artifacts/api-server/src/routes/telemetry.ts @@ -15,8 +15,8 @@ import { evalFormula, type SeriesSet, } from "../query/metrics"; -import { compileSearch, logResolver, spanResolver, SqlBuilder, jsonPath } from "../query/search"; -import { v2Search, v2Page, parseAggregate, toBuckets, aggregateInterval, limitBuckets, PERCENTILE, type AggregateRequest } from "../query/v2"; +import { compileSearch, logResolver, spanResolver, SqlBuilder, jsonPath, safeNumber } from "../query/search"; +import { v2Search, v2Page, parseAggregate, toBuckets, aggregatePasses, mergeBuckets, limitBuckets, PERCENTILE, type AggregateRequest } from "../query/v2"; import { filterHosts } from "../query/hosts"; import { normalizeTraceId } from "../intake/decode"; @@ -91,7 +91,7 @@ router.post( type: "timeseries_response", attributes: { times, - series: r.series.map((s, i) => ({ query_index: i, group_tags: s.group, unit: null })), + series: r.series.map((s) => ({ query_index: s.queryIndex, group_tags: s.group, unit: null })), values: r.series.map((s) => { const m = new Map(s.pointlist); return times.map((t) => m.get(t) ?? null); @@ -251,13 +251,16 @@ const likeEsc = (s: string) => s.replace(/[\\%_]/g, (m) => "\\" + m); /** SQL for a log facet (column, @attribute or tag key); keys are bound as parameters. */ function logFacetSql(f: string, b: SqlBuilder): string { if (LOG_COLUMNS.includes(f)) return f; - if (f.startsWith("@")) return jsonPath("attributes", f.slice(1), b); + if (f.startsWith("@")) return logAttrSql(f.slice(1), b); return `(select substr(t, ${b.p(f.length + 2)}::int) from unnest(tags) t where t like ${b.p(likeEsc(f) + ":%")} limit 1)`; } +/** A log attribute as text: a nested path (a.b.c) or a flat dotted key ("http.status_code"), as search resolves it. */ +function logAttrSql(key: string, b: SqlBuilder): string { + return `COALESCE(${jsonPath("attributes", key, b)}, attributes ->> ${b.p(key)}::text)`; +} /** A numeric log attribute (non-numeric values become NULL instead of failing the query). */ function logNumberSql(metric: string, b: SqlBuilder): string { - const x = jsonPath("attributes", metric.replace(/^@/, ""), b); - return `(CASE WHEN ${x} ~ '^-?[0-9]+(\\.[0-9]+)?([eE][-+]?[0-9]+)?$' THEN (${x})::double precision END)`; + return safeNumber(logAttrSql(metric.replace(/^@/, ""), b)); } router.post( "/ui/logs/search", @@ -403,20 +406,23 @@ router.post( h(async (req, res) => { const started = Date.now(); const a = parseAggregate(req.body); - const { where, b } = logWhere(req.orgId!, a.query, a.from, a.to); - const groups = a.groupBy.map((g, i) => `${logFacetSql(g.facet, b)} as g${i}`); - const computes = a.computes.map((c) => `${aggregateComputeSql(c, b)} as ${c.name}`); - const interval = aggregateInterval(a, (f, t) => pickInterval(f, t, 60)); - const ts = a.computes.some((c) => c.timeseries); - const sel = [...groups, ...(ts ? [`(ts / 1000 / ${interval}) * ${interval} as t`] : []), ...computes]; - const by = [...a.groupBy.map((_, i) => i + 1), ...(ts ? [groups.length + 1] : [])]; - const rows = ( - await pool.query( - `select ${sel.join(", ")} from logs where ${where} ${by.length ? `group by ${by.join(",")}` : ""} ${ts ? "order by t" : ""} limit 100000`, - b.params, - ) - ).rows; - res.json({ data: { buckets: limitBuckets(a, toBuckets(a, rows)) }, meta: { status: "done", elapsed: Date.now() - started } }); + const parts = []; + for (const pass of aggregatePasses(a, (f, t) => pickInterval(f, t, 60))) { + const { where, b } = logWhere(req.orgId!, a.query, a.from, a.to); + const groups = a.groupBy.map((g, i) => `${logFacetSql(g.facet, b)} as g${i}`); + const computes = pass.computes.map((c) => `${aggregateComputeSql(c, b)} as ${c.name}`); + const ts = pass.interval !== null; + const sel = [...groups, ...(ts ? [`(ts / 1000 / ${pass.interval}) * ${pass.interval} as t`] : []), ...computes]; + const by = [...a.groupBy.map((_, i) => i + 1), ...(ts ? [groups.length + 1] : [])]; + const rows = ( + await pool.query( + `select ${sel.join(", ")} from logs where ${where} ${by.length ? `group by ${by.join(",")}` : ""} ${ts ? "order by t" : ""} limit 100000`, + b.params, + ) + ).rows; + parts.push(toBuckets({ ...a, computes: pass.computes }, rows)); + } + res.json({ data: { buckets: limitBuckets(a, mergeBuckets(parts)) }, meta: { status: "done", elapsed: Date.now() - started } }); }), ); diff --git a/artifacts/api-server/src/routes/telemetryCh.ts b/artifacts/api-server/src/routes/telemetryCh.ts index a12d601..eba0cf8 100644 --- a/artifacts/api-server/src/routes/telemetryCh.ts +++ b/artifacts/api-server/src/routes/telemetryCh.ts @@ -3,7 +3,7 @@ import { serviceSummary, serviceTimeseries, serviceResources, serviceOperations, import { chQuery, ChParams } from "../lib/ch"; import { hostMetricsCh } from "../query/metricsCh"; import { logWhereCh, spanWhereCh } from "../query/searchCh"; -import { v2Search, v2Page, parseAggregate, toBuckets, aggregateInterval, limitBuckets, PERCENTILE, type AggregateRequest } from "../query/v2"; +import { v2Search, v2Page, parseAggregate, toBuckets, limitBuckets, PERCENTILE, type AggregateRequest, aggregatePasses, mergeBuckets } from "../query/v2"; import { filterHosts } from "../query/hosts"; import { spanV2 } from "./telemetry"; import { correlate } from "../query/correlate"; @@ -95,7 +95,7 @@ router.post( type: "timeseries_response", attributes: { times, - series: r.series.map((s, i) => ({ query_index: i, group_tags: s.group, unit: null })), + series: r.series.map((s) => ({ query_index: s.queryIndex, group_tags: s.group, unit: null })), values: r.series.map((s) => { const m = new Map(s.pointlist); return times.map((t) => m.get(t) ?? null); @@ -392,7 +392,7 @@ function aggregateComputeSql(c: AggregateRequest["computes"][number], b: ChParam const metric = c.metric!; if (c.aggregation === "cardinality") return `toFloat64(uniqExact(${facetExpr(metric, b)}))`; const x = `toFloat64OrNull(attrs[${b.p(metric.replace(/^@/, ""))}])`; - if (c.aggregation in PERCENTILE) return `quantileExact(${PERCENTILE[c.aggregation]})(${x})`; + if (c.aggregation in PERCENTILE) return `quantileExactInclusive(${PERCENTILE[c.aggregation]})(${x})`; return `${c.aggregation}(${x})`; } router.post( @@ -400,18 +400,21 @@ router.post( h(async (req, res) => { const started = Date.now(); const a = parseAggregate(req.body); - const { where, b } = logWhere(req.orgId!, a.query, a.from, a.to); - const groups = a.groupBy.map((g, i) => `${facetExpr(g.facet, b)} AS g${i}`); - const computes = a.computes.map((c) => `${aggregateComputeSql(c, b)} AS ${c.name}`); - const interval = aggregateInterval(a, (f, t) => pickInterval(f, t, 60)); - const ts = a.computes.some((c) => c.timeseries); - const sel = [...groups, ...(ts ? [`intDiv(ts, ${interval * 1000}) * ${interval} AS t`] : []), ...computes]; - const by = [...a.groupBy.map((_, i) => `g${i}`), ...(ts ? ["t"] : [])]; - const rows = await chQuery( - `SELECT ${sel.join(", ")} FROM logs WHERE ${where} ${by.length ? `GROUP BY ${by.join(", ")}` : ""} ${ts ? "ORDER BY t" : ""} LIMIT 100000`, - b.params, - ); - res.json({ data: { buckets: limitBuckets(a, toBuckets(a, rows)) }, meta: { status: "done", elapsed: Date.now() - started } }); + const parts = []; + for (const pass of aggregatePasses(a, (f, t) => pickInterval(f, t, 60))) { + const { where, b } = logWhere(req.orgId!, a.query, a.from, a.to); + const groups = a.groupBy.map((g, i) => `${facetExpr(g.facet, b)} AS g${i}`); + const computes = pass.computes.map((c) => `${aggregateComputeSql(c, b)} AS ${c.name}`); + const ts = pass.interval !== null; + const sel = [...groups, ...(ts ? [`intDiv(ts, ${pass.interval! * 1000}) * ${pass.interval} AS t`] : []), ...computes]; + const by = [...a.groupBy.map((_, i) => `g${i}`), ...(ts ? ["t"] : [])]; + const rows = await chQuery( + `SELECT ${sel.join(", ")} FROM logs WHERE ${where} ${by.length ? `GROUP BY ${by.join(", ")}` : ""} ${ts ? "ORDER BY t" : ""} LIMIT 100000`, + b.params, + ); + parts.push(toBuckets({ ...a, computes: pass.computes }, rows)); + } + res.json({ data: { buckets: limitBuckets(a, mergeBuckets(parts)) }, meta: { status: "done", elapsed: Date.now() - started } }); }), ); diff --git a/artifacts/api-server/test/correctness.test.ts b/artifacts/api-server/test/correctness.test.ts new file mode 100644 index 0000000..6f4ad57 --- /dev/null +++ b/artifacts/api-server/test/correctness.test.ts @@ -0,0 +1,193 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { otlpToMetrics, expoToExplicit } from "../src/intake/otlp"; +import { evalFormula, reduce, interpolatedPercentile, weightedPercentileRows, type SeriesSet } from "../src/query/metrics"; +import { isStorageError } from "../src/lib/http"; +import { sketchPoints } from "../src/intake/routes"; +import { aggregatePasses, mergeBuckets, toBuckets, parseAggregate } from "../src/query/v2"; +import { parseMonitorQuery } from "../src/jobs/monitors"; + +const kv = (key: string, v: string) => ({ key, value: { stringValue: v } }); +const sumReq = (v: number, t: number, inst = "i1", attrs: any[] = [kv("a", "1"), kv("b", "2")]) => ({ + resourceMetrics: [ + { + resource: { attributes: [kv("service.name", "svc"), kv("service.instance.id", inst)] }, + scopeMetrics: [ + { + scope: { name: "s" }, + metrics: [ + { + name: "reqs", + sum: { + isMonotonic: true, + aggregationTemporality: 2, + dataPoints: [{ startTimeUnixNano: "1", timeUnixNano: String(t * 1e9), asInt: v, attributes: attrs }], + }, + }, + ], + }, + ], + }, + ], +}); +const counts = (r: ReturnType) => r.points.filter((p) => p.metric === "reqs").map((p) => p.points[0][1]); +let org = 1000; + +test("cumulative counters: a failed store and retry keep the increment (reference applied on commit)", () => { + const o = org++; + otlpToMetrics(o, sumReq(100, 10)).commit(); + otlpToMetrics(o, sumReq(110, 20)).commit(); + otlpToMetrics(o, sumReq(140, 30)); // store failed: no commit + const retry = otlpToMetrics(o, sumReq(140, 30)); + assert.deepEqual(counts(retry), [30]); +}); + +test("cumulative counters: stream key includes the instance and ignores attribute order; older samples are ignored", () => { + const o = org++; + otlpToMetrics(o, sumReq(100, 10, "a")).commit(); + otlpToMetrics(o, sumReq(70, 10, "b")).commit(); + assert.deepEqual(counts(otlpToMetrics(o, sumReq(130, 20, "a"))), [30]); + const o2 = org++; + otlpToMetrics(o2, sumReq(100, 10)).commit(); + assert.deepEqual(counts(otlpToMetrics(o2, sumReq(130, 20, "i1", [kv("b", "2"), kv("a", "1")]))), [30]); + const o3 = org++; + for (const [v, t] of [ + [100, 0], + [130, 30], + [120, 20], + [140, 40], + ]) { + const r = otlpToMetrics(o3, sumReq(v, t)); + r.commit(); + if (t === 20) assert.deepEqual(counts(r), []); + } +}); + +test("exponential histograms keep negative buckets in ascending order", () => { + const { bounds, counts } = expoToExplicit({ + scale: 0, + zeroCount: 1, + positive: { offset: 0, bucketCounts: [2] }, + negative: { offset: 0, bucketCounts: [3, 4] }, + }); + assert.deepEqual(bounds, [-2, -1, 0, 2]); + assert.deepEqual(counts, [4, 3, 1, 2, 0]); +}); + +test("disk usage keeps the device tag", () => { + const dp = (st: string, dev: string, v: number) => ({ timeUnixNano: "60000000000", asInt: v, attributes: [kv("state", st), kv("device", dev)] }); + const r = otlpToMetrics(org++, { + resourceMetrics: [ + { + resource: { attributes: [kv("host.name", "h")] }, + scopeMetrics: [ + { + scope: { name: "hostmetricsreceiver/filesystem" }, + metrics: [ + { + name: "system.filesystem.usage", + sum: { isMonotonic: false, dataPoints: [dp("used", "a", 2), dp("free", "a", 8), dp("used", "b", 8), dp("free", "b", 2)] }, + }, + ], + }, + ], + }, + ], + }); + const inUse = r.points.filter((p) => p.metric === "system.disk.in_use").map((p) => [p.tags.find((t) => t.startsWith("device:")), p.points[0][1]]); + assert.deepEqual(inUse.sort(), [ + ["device:a", 0.2], + ["device:b", 0.8], + ]); +}); + +test("ClickHouse insert and connection failures are storage errors (503); data errors are not", () => { + assert.ok(isStorageError(new Error("ClickHouse: insert into logs: Code: 241"))); + assert.ok(isStorageError(Object.assign(new Error("x"), { code: "ECONNREFUSED" }))); + assert.ok(isStorageError(Object.assign(new Error("fetch failed"), { cause: { code: "ECONNRESET" } }))); + assert.ok(isStorageError(Object.assign(new Error("db"), { code: "57P01" }))); + assert.ok(!isStorageError(Object.assign(new Error("bad value"), { code: "22P02" }))); + assert.ok(!isStorageError(new Error("Expected a JSON body"))); +}); + +test("sketch bins become weighted points; counts and percentiles hold", () => { + const pts = sketchPoints([{ ts: 60, cnt: 100, min: 0, max: 100, avg: 99, sum: 9900, k: [0, 1635], n: [1, 99] }]); + assert.equal( + pts.reduce((a, p) => a + p[2], 0), + 100, + ); + const rows = pts.map(([, v, w]) => ({ g: 0, b: 60, v, w })); + const p95 = weightedPercentileRows(rows, 0.95)[0].v; + assert.ok(Math.abs(p95 - 100) / 100 < 0.02, String(p95)); + assert.equal( + weightedPercentileRows( + [10, 20, 30, 40].map((v) => ({ g: 0, b: 0, v, w: 1 })), + 0.5, + )[0].v, + 25, + ); +}); + +test("formulas combine only matching groups; one-sided groups are kept for + and -", () => { + const s = (key: string, pts: [number, number][]): SeriesSet[number] => ({ key, tags: key === "*" ? [] : [key], points: new Map(pts) }); + const env = { + b: [ + s("host:a", [ + [0, 2], + [60, 4], + ]), + ], + c: [ + s("host:c", [ + [0, 3], + [60, 6], + ]), + ], + t: [s("*", [[0, 10]])], + }; + const r = evalFormula("b + c", env, 60); + assert.deepEqual(r.map((x) => [x.key, [...x.points.values()]]).sort(), [ + ["host:a", [2, 4]], + ["host:c", [3, 6]], + ]); + assert.deepEqual( + evalFormula("b * c", env, 60).map((x) => x.key), + [], + ); + assert.deepEqual( + evalFormula("b + t", env, 60).map((x) => [x.key, x.points.get(0)]), + [["host:a", 12]], + ); +}); + +test("monitor aggregations: percentile(N), change and pct_change", () => { + const vals = Array.from({ length: 20 }, (_, i) => i); + assert.equal(reduce(vals, parseMonitorQuery("metric alert", "percentile(50)(last_5m):avg:x{*} > 1").timeAgg), 9.5); + assert.equal(reduce(vals, "p95"), interpolatedPercentile(vals, 0.95)); + assert.equal(reduce([100, 0], parseMonitorQuery("metric alert", "change(last_5m,last_5m)(last_5m):avg:x{*} > 1").timeAgg), -100); + assert.equal(reduce([50, 75], "pct_change"), 50); + assert.throws(() => parseMonitorQuery("metric alert", "percentile(150)(last_5m):avg:x{*} > 1")); +}); + +test("log aggregate: totals and timeseries are separate passes, merged per group", () => { + const a = parseAggregate({ + filter: { from: "1000", to: "1180" }, + compute: [{ aggregation: "count" }, { aggregation: "count", type: "timeseries", interval: "60s" }], + }); + const passes = aggregatePasses(a, () => 60); + assert.deepEqual( + passes.map((p) => [p.interval, p.computes.map((c) => c.name)]), + [ + [null, ["c0"]], + [60, ["c1"]], + ], + ); + const total = toBuckets({ ...a, computes: passes[0].computes }, [{ c0: 3 }]); + const ts = toBuckets({ ...a, computes: passes[1].computes }, [ + { t: 960, c1: 2 }, + { t: 1020, c1: 1 }, + ]); + const merged = mergeBuckets([total, ts]); + assert.equal(merged[0].computes.c0, 3); + assert.equal((merged[0].computes.c1 as any[]).length, 2); +}); diff --git a/artifacts/api-server/test/monitor-states.test.ts b/artifacts/api-server/test/monitor-states.test.ts new file mode 100644 index 0000000..5f11bb5 --- /dev/null +++ b/artifacts/api-server/test/monitor-states.test.ts @@ -0,0 +1,9 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { validateMonitorQuery } from "../src/jobs/monitors"; + +test("monitor queries must not be empty and must end with a comparison", () => { + assert.throws(() => validateMonitorQuery("metric alert", "")); + assert.throws(() => validateMonitorQuery("log alert", 'logs("x").last("5m")')); + validateMonitorQuery("log alert", 'logs("service:x").by("env").last("5m") > 5'); +}); diff --git a/artifacts/api-server/test/security.test.ts b/artifacts/api-server/test/security.test.ts new file mode 100644 index 0000000..fab564a --- /dev/null +++ b/artifacts/api-server/test/security.test.ts @@ -0,0 +1,151 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { validEmail, getSafeReturnTo } from "../src/routes/auth"; +import { parseStatsdLine } from "../src/intake/tracer"; +import { globMatch, matchTags, parseFilter, resolveInterval } from "../src/query/metrics"; +import { redactString, redactLog, redactSpan } from "../src/intake/redact"; +import { isBlockedAddress, checkWebhookHeaders, checkWebhookUrl } from "../src/lib/safeHttp"; +import { getSessionId } from "../src/lib/auth"; +import { render, messageHandles } from "../src/jobs/monitors"; + +// Sample secrets are assembled at runtime so secret scanners do not flag this file. +const PEM = (edge: "BEGIN" | "END") => `-----${edge} RSA PRIV` + "ATE KEY-----"; +const TOKEN_PAIR = "tok" + "en=abcdefSECRET1"; + +const timed = (fn: () => unknown) => { + const t = performance.now(); + fn(); + return performance.now() - t; +}; + +test("email check: same answers as the old pattern, and linear on long input", () => { + const old = (e: string) => /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(e) && e.length <= 254; + for (const e of [ + "a@b.co", + "first.last@example.com", + "a@b.c", + "a@.b.c", + "a@b..c", + "a@b.", + "@b.co", + "a@@b.co", + "a b@c.de", + "a@b", + "a@b@c.de", + "x@y.z.w", + "a@bc.", + ]) + assert.equal(validEmail(e), old(e), e); + assert.ok(timed(() => validEmail("a@" + ".".repeat(64000) + "@")) < 50); +}); + +test("DogStatsD lines parse like before, and a long line with a carriage return is fast", () => { + const old = (l: string) => /^([^:|]+):([^|]+)\|([a-z]+)(.*)$/.exec(l); + for (const l of ["page.views:1|c", "lat:12.5|d|@0.5|#env:prod,host:a", "g:3|g|#a:b", "bad", ":1|c", "x:|c", "x:1|", "x:1|C", "a:b:c|g", "x:1|c|#t:1\r"]) { + const o = old(l.replace(/\r$/, "")); + const n = parseStatsdLine(l); + assert.deepEqual(n && n.slice(1), o && o.slice(1), JSON.stringify(l)); + } + assert.ok(timed(() => parseStatsdLine("x:1|" + "a".repeat(65536) + "\rX")) < 50); +}); + +test("tag globs match like the old regular expressions, and many wildcards are fast or refused", () => { + const toRe = (g: string) => + new RegExp( + "^" + + g + .replace(/[.+^${}()|[\]\\]/g, "\\$&") + .replace(/\*/g, ".*") + .replace(/\?/g, ".") + + "$", + "i", + ); + const pats = ["*", "web-*", "*-01", "w?b-0*", "*a*b*", "API-*", "x", "?", "*.*", "a*a*a"]; + const vals = ["web-01", "WEB-02", "api-01", "db-01", "ab", "aab", "a.b", "", "x", "aaa", "line\nbreak"]; + // the old matcher: "*" matched everything, a pattern without wildcards compared case-insensitively, else the regex + const old = (p: string, v: string) => (p === "*" ? true : !/[*?]/.test(p) ? p.toLowerCase() === v.toLowerCase() : toRe(p).test(v)); + for (const p of pats) for (const v of vals) assert.equal(globMatch(p, v), old(p, v), `${p} ~ ${JSON.stringify(v)}`); + assert.ok(timed(() => globMatch("****************Z", "a".repeat(5000))) < 50); + assert.ok(timed(() => matchTags(parseFilter("host:****************Z"), ["host:" + "a".repeat(5000)])) < 50); + assert.throws(() => parseFilter("host:" + "*".repeat(21)), /at most 20 wildcards/); +}); + +test("metric intervals must be positive and are widened to a bounded number of buckets", () => { + assert.throws(() => resolveInterval(-1, 0, 3600), /positive/); + assert.throws(() => resolveInterval(0, 0, 3600), /positive/); + assert.throws(() => resolveInterval("abc", 0, 3600), /positive/); + assert.equal(resolveInterval(60, 0, 3600), 60); + assert.equal(resolveInterval(1, 0, 86400 * 30), Math.ceil((86400 * 30) / 10_000)); + assert.ok(resolveInterval(undefined, 0, 3600) > 0); +}); + +test("raw session ids are not accepted as Bearer tokens", () => { + const req: any = { headers: { authorization: "Bearer abc123" }, signedCookies: {}, cookies: {} }; + assert.equal(getSessionId(req), undefined); +}); + +test("alert recipients come from the monitor template, not from telemetry values", () => { + const ctx = { state: "Alert" as const, prev: "OK" as const, value: 5, threshold: 1, tags: ["service:checkout @webhook-two"], name: "m", id: 1 }; + const tpl = "Errors on {{service.name}} @webhook-one"; + assert.deepEqual(messageHandles(render(tpl, ctx, true)), ["webhook-one"]); + assert.match(render(tpl, ctx), /@webhook-two/); + const cond = "{{#is_alert}}@webhook-page{{/is_alert}}{{#is_recovery}}@webhook-ok{{/is_recovery}}"; + assert.deepEqual(messageHandles(render(cond, ctx, true)), ["webhook-page"]); +}); + +test("private-key redaction is linear and still masks real blocks", () => { + assert.ok(timed(() => redactString(PEM("BEGIN").repeat(70000))) < 200); + assert.equal(redactString(`k=${PEM("BEGIN")}\nMIIabc\n${PEM("END")} done`), "k=[REDACTED:private_key] done"); +}); + +test("secret-named keys are masked for every value type; names and short Basic credentials too", () => { + const l = redactLog({ + message: "ok", + service: `svc ${TOKEN_PAIR}`, + attributes: { headers: { "set-cookie": ["sid=R4nd0mValue"] }, password: 123456, plan: "Basic plan", note: "order 1234" }, + }); + assert.equal(l.attributes.headers["set-cookie"], "[REDACTED:key]"); + assert.equal(l.attributes.password, "[REDACTED:key]"); + assert.equal(l.attributes.plan, "Basic plan"); + assert.equal(l.attributes.note, "order 1234"); + assert.equal(l.service, "svc token=[REDACTED:secret]"); + assert.equal(redactString("auth Basic YTpi end"), "auth Basic [REDACTED:bearer] end"); + assert.equal(redactString("card 4-1-1-1-1-1-1-1-1-1-1-1-1-1-1-1"), "card [REDACTED:card]"); + const s = redactSpan({ resource: "GET /", name: "token=abcdef", service: "token-service" }); + assert.equal(s.name, "token=[REDACTED:secret]"); + assert.equal(s.service, "token-service"); +}); + +test("webhook filter refuses private IPv4 addresses inside IPv6 forms", () => { + for (const ip of [ + "64:ff9b::a9fe:a9fe", + "::127.0.0.1", + "::ffff:10.0.0.1", + "::ffff:0:192.168.1.1", + "2002:a9fe:a9fe::1", + "2001::1", + "fec0::1", + "::1", + "fd00::1", + ]) + assert.equal(isBlockedAddress(ip), true, ip); + for (const ip of ["8.8.8.8", "2606:4700:4700::1111", "64:ff9b::808:808", "2002:808:808::1"]) assert.equal(isBlockedAddress(ip), false, ip); + assert.throws(() => checkWebhookUrl("http://[64:ff9b::a9fe:a9fe]/"), /private or local/); +}); + +test("webhook headers cannot set connection or framing headers", () => { + for (const k of ["Host", "transfer-encoding", "Connection", "Content-Length", "Upgrade", "TE"]) + assert.throws(() => checkWebhookHeaders({ [k]: "x" }), /cannot be set/); + assert.throws(() => checkWebhookHeaders({ "X-A": "a\r\nInjected: 1" }), /single-line/); + assert.deepEqual(checkWebhookHeaders({ Authorization: "Bearer x", "X-Retry": 3 }), { Authorization: "Bearer x", "X-Retry": 3 }); + assert.equal(checkWebhookHeaders(null), null); +}); + +test("logout returnTo accepts only same-origin paths", () => { + for (const v of ["/\\evil.example/login", "//evil.example", "/\\/evil.example", "https://evil.example", "/%5Cevil", "javascript:alert(1)"]) { + const r = getSafeReturnTo(v === "/%5Cevil" ? decodeURIComponent(v) : v); + assert.ok(r.startsWith("/") && !r.startsWith("//") && !r.startsWith("/\\"), `${v} -> ${r}`); + } + assert.equal(getSafeReturnTo("/dashboards?x=1#y"), "/dashboards?x=1#y"); + assert.equal(getSafeReturnTo(undefined), "/"); +}); diff --git a/artifacts/api-server/test/webhook-timeout.test.ts b/artifacts/api-server/test/webhook-timeout.test.ts new file mode 100644 index 0000000..f0ef034 --- /dev/null +++ b/artifacts/api-server/test/webhook-timeout.test.ts @@ -0,0 +1,19 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; + +test("a receiver that trickles bytes is cut off at the total time limit", async () => { + process.env.DATACAT_ALLOW_PRIVATE_WEBHOOKS = "true"; + const { postWebhook } = await import("../src/lib/safeHttp"); + const server = http.createServer((_req, res) => { + res.writeHead(200); + const iv = setInterval(() => res.write("x"), 100); + res.on("close", () => clearInterval(iv)); + }); + await new Promise((r) => server.listen(0, "127.0.0.1", r)); + const port = (server.address() as any).port; + const t = Date.now(); + await assert.rejects(postWebhook(`http://127.0.0.1:${port}/`, {}, "{}", 1000, 800), /timeout/); + assert.ok(Date.now() - t < 2000); + server.close(); +}); diff --git a/artifacts/web/src/App.tsx b/artifacts/web/src/App.tsx index e44fb08..2f2204e 100644 --- a/artifacts/web/src/App.tsx +++ b/artifacts/web/src/App.tsx @@ -24,7 +24,7 @@ export default function App() { - {() => } + {(p: { id: string }) => } diff --git a/artifacts/web/src/lib/format.ts b/artifacts/web/src/lib/format.ts index e8516fe..460bb49 100644 --- a/artifacts/web/src/lib/format.ts +++ b/artifacts/web/src/lib/format.ts @@ -6,7 +6,10 @@ export function fmtNum(v: number | null | undefined, digits = 2): string { if (a >= 1e6) return (v / 1e6).toFixed(1) + "M"; if (a >= 1e4) return (v / 1e3).toFixed(1) + "K"; if (a >= 100) return v.toFixed(0); - if (a >= 1) return v.toFixed(digits).replace(/\.?0+$/, ""); + if (a >= 1) { + const s = v.toFixed(digits); + return s.includes(".") ? s.replace(/\.?0+$/, "") : s; + } if (a === 0) return "0"; return v.toPrecision(2); } diff --git a/artifacts/web/src/pages/Logs.tsx b/artifacts/web/src/pages/Logs.tsx index f7f5221..e87de6a 100644 --- a/artifacts/web/src/pages/Logs.tsx +++ b/artifacts/web/src/pages/Logs.tsx @@ -65,7 +65,6 @@ export function LogsExplorer() { const [groupBy, setGroupBy] = useState("service"); const [sel, setSel] = useState(null); const [liveRows, setLiveRows] = useState([]); - const lastLive = useRef(Date.now() - 60_000); const effFrom = live ? Math.floor(Date.now() / 1000) - 900 : from; const effTo = live ? Math.floor(Date.now() / 1000) + 5 : to; const list = useQuery({ @@ -95,23 +94,43 @@ export function LogsExplorer() { useEffect(() => { if (!live) return; setLiveRows([]); - lastLive.current = Date.now() - 30_000; + // The position is the API cursor (time and id of the last log seen), so a burst of logs with one timestamp is read + // page by page instead of repeating the first page. Requests from an older query are aborted and their results + // dropped. + const ctl = new AbortController(); + const since = Math.floor((Date.now() - 30_000) / 1000); + let cursor: string | null = null; + let busy = false; const poll = async () => { - const now = Math.floor(Date.now() / 1000) + 2; - const r = await api<{ logs: LogRow[] }>("/ui/logs/search", { - body: { query, from: Math.floor(lastLive.current / 1000), to: now, limit: 200, sort: "asc" }, - }).catch(() => null); - if (r?.logs.length) { - setLiveRows((rows) => { - const ids = new Set(rows.map((x) => x.id)); - return [...r.logs.filter((l) => !ids.has(l.id)).reverse(), ...rows].slice(0, 500); - }); - lastLive.current = r.logs[r.logs.length - 1].ts; + if (busy || ctl.signal.aborted) return; + busy = true; + try { + for (let page = 0; page < 10 && !ctl.signal.aborted; page++) { + const r: { logs: LogRow[]; cursor: string | null } | null = await api<{ logs: LogRow[]; cursor: string | null }>("/ui/logs/search", { + body: { query, from: since, to: Math.floor(Date.now() / 1000) + 2, limit: 200, sort: "asc", ...(cursor ? { cursor } : {}) }, + signal: ctl.signal, + }).catch(() => null); + if (!r || ctl.signal.aborted) return; + if (r.logs.length) { + const last = r.logs[r.logs.length - 1]; + cursor = `${last.ts}_${last.id}`; + setLiveRows((rows) => { + const ids = new Set(rows.map((x) => x.id)); + return [...r.logs.filter((l) => !ids.has(l.id)).reverse(), ...rows].slice(0, 500); + }); + } + if (!r.cursor) break; + } + } finally { + busy = false; } }; poll(); const iv = setInterval(poll, 2000); - return () => clearInterval(iv); + return () => { + clearInterval(iv); + ctl.abort(); + }; }, [live, query]); const histData = useMemo(() => { diff --git a/docs/SELF_HOSTING.md b/docs/SELF_HOSTING.md index c4d1930..87c83c7 100644 --- a/docs/SELF_HOSTING.md +++ b/docs/SELF_HOSTING.md @@ -162,7 +162,7 @@ use systemd `EnvironmentFile`. | --- | --- | | `DATABASE_URL` | **Required.** PostgreSQL connection URL. | | `PORT` | `3000`, HTTP listener. In Compose this controls the published host port; the container listens on 3000. | -| `PUBLIC_URL` | Canonical external origin, e.g. `https://observe.example.com`. Used for invite links, installer and in-app documentation. Falls back to the request origin; set explicitly in production. No path prefix. | +| `PUBLIC_URL` | Canonical external origin, e.g. `https://observe.example.com`. Used for invite links, installer and in-app documentation. **Required when `NODE_ENV=production`**; in development it falls back to the request origin. No path prefix. | | `SESSION_SECRET` | **Required in production**, at least 32 random characters. Signs session cookies. Changing it signs browser sessions out. | | `DATACAT_RETENTION_DAYS` | `15`, positive telemetry retention window. Legacy `RETENTION_DAYS` is accepted with lower priority. With PostgreSQL, cleanup runs hourly; with ClickHouse it is a table TTL. | | `DATACAT_STORAGE` | `postgres`. Telemetry storage: `postgres` or `clickhouse` (see "Choosing telemetry storage"). App data always stays in PostgreSQL. | diff --git a/docs/images/dashboard.jpg b/docs/images/dashboard.jpg index d4ec8e1..306fd27 100644 Binary files a/docs/images/dashboard.jpg and b/docs/images/dashboard.jpg differ diff --git a/sdks/node/lib/span-exporter.js b/sdks/node/lib/span-exporter.js index fe11c72..8c40962 100644 --- a/sdks/node/lib/span-exporter.js +++ b/sdks/node/lib/span-exporter.js @@ -251,20 +251,25 @@ class DataCatSpanExporter { // One POST; resolves { ok, retryable, retryAfterMs, error }. post(body) { return new Promise((resolve) => { + let settled = false; + const done = (r) => { if (!settled) { settled = true; resolve(r); } }; const req = this.transport.request(this.url, { method: 'POST', agent: this.agent, timeout: this.timeout, headers: { ...this.headers, 'content-length': body.length }, }, (res) => { + // A reset after the headers must count as a failed (retryable) attempt, not crash the application. + res.on('error', (error) => done({ ok: false, retryable: true, error })); + res.on('aborted', () => done({ ok: false, retryable: true, error: Object.assign(new Error('OTLP export response aborted'), { code: 'ECONNRESET' }) })); res.resume(); res.on('end', () => { const code = res.statusCode; - if (code >= 200 && code < 300) return resolve({ ok: true }); + if (code >= 200 && code < 300) return done({ ok: true }); const ra = Number(res.headers['retry-after']); - resolve({ ok: false, retryable: RETRYABLE_STATUS.has(code), retryAfterMs: Number.isFinite(ra) ? ra * 1000 : 0, error: new Error(`OTLP export failed: HTTP ${code}`) }); + done({ ok: false, retryable: RETRYABLE_STATUS.has(code), retryAfterMs: Number.isFinite(ra) ? ra * 1000 : 0, error: new Error(`OTLP export failed: HTTP ${code}`) }); }); }); req.on('timeout', () => req.destroy(Object.assign(new Error('OTLP export timed out'), { code: 'ETIMEDOUT' }))); - req.on('error', (error) => resolve({ ok: false, retryable: true, error })); + req.on('error', (error) => done({ ok: false, retryable: true, error })); req.end(body); }); } diff --git a/sdks/node/test/exporter-reset.test.js b/sdks/node/test/exporter-reset.test.js new file mode 100644 index 0000000..986b3df --- /dev/null +++ b/sdks/node/test/exporter-reset.test.js @@ -0,0 +1,27 @@ +'use strict'; +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const http = require('node:http'); +const { DataCatSpanExporter } = require('../lib/span-exporter'); + +test('a connection reset after the response headers is a failed attempt, not a crash', async () => { + let n = 0; + const server = http.createServer((req, res) => { + req.resume(); + n++; + if (n === 1) { + res.writeHead(200, { 'content-length': '100' }); + res.write('x'); + setTimeout(() => res.socket.destroy(), 20); + } else res.end(); + }); + await new Promise((r) => server.listen(0, '127.0.0.1', r)); + const exp = new DataCatSpanExporter({ url: `http://127.0.0.1:${server.address().port}/v1/traces` }); + const r = await exp.post(Buffer.from('a')); + assert.equal(r.ok, false); + assert.equal(r.retryable, true); + const ok = await exp.post(Buffer.from('a')); + assert.equal(ok.ok, true); + await exp.shutdown(); + server.close(); +}); diff --git a/sdks/python/src/datacat/_fast_flask.py b/sdks/python/src/datacat/_fast_flask.py index 5f864d1..6032209 100644 --- a/sdks/python/src/datacat/_fast_flask.py +++ b/sdks/python/src/datacat/_fast_flask.py @@ -16,6 +16,38 @@ _patched = False +class _ClosingBody: + """Iterates a WSGI response body inside the request span and ends the span once, on close or exhaustion.""" + + def __init__(self, body, span, finish): + self._body = body + self._span = span + self._finish = finish + self._done = False + + def __iter__(self): + token = otel_context.attach(trace.set_span_in_context(self._span)) + try: + for chunk in self._body: + yield chunk + except BaseException as exc: + self._span.record_exception(exc) + self._span.set_status(Status(StatusCode.ERROR)) + raise + finally: + otel_context.detach(token) + + def close(self): + try: + close = getattr(self._body, "close", None) + if close is not None: + close() + finally: + if not self._done: + self._done = True + self._finish() + + def _parent_from(environ): tp = environ.get("HTTP_TRACEPARENT") if not tp or len(tp) < 55: @@ -60,18 +92,24 @@ def _start_response(s, headers, exc_info=None): pass return start_response(s, headers, exc_info) - try: - return original_wsgi_app(self, environ, _start_response) - except BaseException as exc: - span.record_exception(exc) - raise - finally: + def finish(): code = status[0] span.set_attribute("http.response.status_code", code) if code >= 500: span.set_status(Status(StatusCode.ERROR)) span.end() + + try: + body = original_wsgi_app(self, environ, _start_response) + except BaseException as exc: + span.record_exception(exc) + finish() otel_context.detach(token) + raise + otel_context.detach(token) + # The span covers the whole response: it ends when the server has consumed or closed the body, so streamed + # responses (downloads, server-sent events) report their real duration and errors. + return _ClosingBody(body, span, finish) def dispatch_request(self, *args, **kwargs): try: diff --git a/sdks/python/tests/test_flask_stream.py b/sdks/python/tests/test_flask_stream.py new file mode 100644 index 0000000..371544d --- /dev/null +++ b/sdks/python/tests/test_flask_stream.py @@ -0,0 +1,45 @@ +import time +import unittest + +try: + import flask +except ImportError: # pragma: no cover + flask = None + +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + +@unittest.skipIf(flask is None, "flask not installed") +class FlaskStreamTests(unittest.TestCase): + def test_span_covers_streamed_body(self): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + trace.set_tracer_provider(provider) + from datacat._fast_flask import patch_flask + self.assertTrue(patch_flask()) + app = flask.Flask("t") + + @app.route("/stream") + def stream(): + def gen(): + for _ in range(3): + time.sleep(0.05) + yield b"x" + return flask.Response(gen()) + + resp = app.test_client().get("/stream") + body = resp.data + resp.close() # a WSGI server closes the body after sending it + self.assertEqual(body, b"xxx") + spans = [s for s in exporter.get_finished_spans() if s.name.endswith("/stream")] + self.assertEqual(len(spans), 1) + self.assertGreaterEqual((spans[0].end_time - spans[0].start_time) / 1e9, 0.14) + self.assertEqual(spans[0].attributes.get("http.response.status_code"), 200) + + +if __name__ == "__main__": + unittest.main()