Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 24 additions & 1 deletion artifacts/api-server/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { storage, useClickHouse } from "./lib/storage";
import { ensureChSchema } from "./lib/ch";
import { pool } from "@workspace/db";
import { generatedSetupToken } from "./routes/auth";
import { flushSpanStats } from "./query/apmStats";

const rawPort = process.env["PORT"] ?? "3000";
const port = Number(rawPort);
Expand All @@ -17,7 +18,7 @@ logger.info({ storage }, "telemetry storage");
if (generatedSetupToken && !(await pool.query("select 1 from users limit 1")).rowCount) {
logger.warn(`First-run setup token (enter it on the setup page): ${generatedSetupToken}`);
}
app.listen(port, (err) => {
const server = app.listen(port, (err) => {
if (err) {
logger.error({ err }, "Error listening on port");
process.exit(1);
Expand All @@ -31,3 +32,25 @@ app.listen(port, (err) => {
void buildBackgroundIndexes((msg, extra) => logger.info(extra ?? {}, msg));
} else logger.info("background jobs disabled (DATACAT_BACKGROUND_JOBS=false)");
});

// On SIGTERM/SIGINT: stop accepting connections, let requests in progress finish (up to DATACAT_SHUTDOWN_SECONDS,
// default 25), flush pending APM rollups, close the database pool, then exit.
let stopping = false;
for (const sig of ["SIGTERM", "SIGINT"] as const)
process.once(sig, () => {
if (stopping) return;
stopping = true;
logger.info({ signal: sig }, "shutting down");
const limitMs = Math.max(1, Number(process.env.DATACAT_SHUTDOWN_SECONDS ?? 25)) * 1000;
const force = setTimeout(() => {
logger.warn("shutdown time limit reached; exiting");
process.exit(0);
}, limitMs);
force.unref();
server.close(async () => {
await flushSpanStats().catch((e) => logger.warn({ err: e }, "final span stats flush failed"));
await pool.end().catch(() => undefined);
process.exit(0);
});
server.closeIdleConnections?.();
});
91 changes: 70 additions & 21 deletions artifacts/api-server/src/intake/otlp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -380,8 +380,15 @@ function rememberCumulative(k: string, v: any) {
}
}
}
function cumKey(orgId: number, name: string, host: string, tags: string[], start: any) {
return `${orgId}|${name}|${host}|${tags.join(",")}|${start ?? ""}`;
const sortedAttrs = (a: Attrs) =>
JSON.stringify(
Object.entries(a)
.filter(([, v]) => v !== null && v !== undefined)
.sort(([x], [y]) => (x < y ? -1 : x > y ? 1 : 0)),
);
/** Identity of one cumulative stream: every resource, scope and data-point attribute (order-independent) and its start. */
function cumKey(orgId: number, name: string, resource: string, scope: string, dpAttrs: Attrs, start: any) {
return `${orgId}|${name}|${resource}|${scope}|${sortedAttrs(dpAttrs)}|${start ?? ""}`;
}

function quantileFromBuckets(bounds: number[], counts: number[], q: number, min?: number, max?: number): number {
Expand All @@ -402,13 +409,25 @@ function quantileFromBuckets(bounds: number[], counts: number[], q: number, min?
return max ?? bounds[bounds.length - 1] ?? NaN;
}

function expoToExplicit(dp: any): { bounds: number[]; counts: number[] } {
/**
* Exponential histogram -> explicit buckets in ascending order. Bucket i of a range covers (base^(offset+i),
* base^(offset+i+1)] in absolute value; negative buckets mirror it below zero, the zero bucket sits between.
*/
export function expoToExplicit(dp: any): { bounds: number[]; counts: number[] } {
const scale = Number(dp.scale ?? 0);
const base = Math.pow(2, Math.pow(2, -scale));
const bounds: number[] = [];
const counts: number[] = [];
const noff = Number(dp.negative?.offset ?? 0);
const nc = (dp.negative?.bucketCounts ?? []).map(Number);
for (let i = nc.length - 1; i >= 0; i--) {
bounds.push(-Math.pow(base, noff + i));
counts.push(nc[i]);
}
bounds.push(0);
counts.push(Number(dp.zeroCount ?? 0));
const off = Number(dp.positive?.offset ?? 0);
const pc = (dp.positive?.bucketCounts ?? []).map(Number);
const bounds: number[] = [0];
const counts: number[] = [Number(dp.zeroCount ?? 0)];
pc.forEach((c: number, i: number) => {
bounds.push(Math.pow(base, off + i + 1));
counts.push(c);
Expand Down Expand Up @@ -492,8 +511,18 @@ function hostMappings(
}
}

export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hosts: Map<string, { os?: string; agentVersion?: string; tags: string[] }> } {
/**
* Converts an OTLP metrics request. Cumulative streams are turned into deltas against the last stored value; the new
* reference values are applied only by commit(), which the caller runs after the points are stored, so a failed store
* followed by a retry computes the same deltas again instead of losing them.
*/
export function otlpToMetrics(
orgId: number,
req: any,
): { points: PointIn[]; hosts: Map<string, { os?: string; agentVersion?: string; tags: string[] }>; commit: () => void } {
const out: PointIn[] = [];
const refs = new Map<string, any>();
const refOf = (k: string) => (refs.has(k) ? refs.get(k) : cumulative.get(k));
const hosts = new Map<string, { os?: string; agentVersion?: string; tags: string[] }>();
const acc = new Map<string, { sum: number; n: number; p: PointIn }>();
for (const rm of req?.resourceMetrics ?? []) {
Expand All @@ -506,7 +535,9 @@ export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hos
tags: r.tags,
});
const baseTags = [...r.tags, ...(r.service ? [`service:${r.service}`] : [])];
const resKey = sortedAttrs(r.attrs);
for (const sm of rm.scopeMetrics ?? []) {
const scopeKey = `${sm.scope?.name ?? ""}@${sm.scope?.version ?? ""}`;
for (const m of sm.metrics ?? []) {
const name = String(m.name ?? "");
if (!name) continue;
Expand Down Expand Up @@ -547,9 +578,10 @@ export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hos
continue;
}
if (!delta) {
const k = cumKey(orgId, name, r.host, tags, dp.startTimeUnixNano);
const prev = cumulative.get(k);
rememberCumulative(k, { v, ts });
const k = cumKey(orgId, name, resKey, scopeKey, a, dp.startTimeUnixNano);
const prev = refOf(k);
if (prev !== undefined && ts < prev.ts) continue; // late or retried older sample: already counted
refs.set(k, { v, ts });
if (prev === undefined || v < prev.v) continue; // first sample or reset
const dt = Math.max(1, ts - prev.ts);
const d = v - prev.v;
Expand Down Expand Up @@ -577,10 +609,13 @@ export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hos
? { bounds: (dp.explicitBounds ?? []).map(Number), counts: (dp.bucketCounts ?? []).map(Number) }
: expoToExplicit(dp);
if (!delta) {
const k = cumKey(orgId, name, r.host, tags, dp.startTimeUnixNano);
const prev = cumulative.get(k);
rememberCumulative(k, { count, sum, counts });
if (!prev || count < prev.count) continue;
const k = cumKey(orgId, name, resKey, scopeKey, attrs(dp.attributes), dp.startTimeUnixNano);
const prev = refOf(k);
if (prev && ts < prev.ts) continue;
refs.set(k, { count, sum, counts, bounds, ts });
// buckets are subtracted range by range; a change of bucket layout starts a new reference
const sameLayout = prev && prev.bounds.length === bounds.length && prev.bounds.every((b: number, i: number) => b === bounds[i]);
if (!prev || !sameLayout || count < prev.count) continue;
count -= prev.count;
sum -= prev.sum;
counts = counts.map((c: number, i: number) => c - (prev.counts[i] ?? 0));
Expand Down Expand Up @@ -617,19 +652,33 @@ export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hos
out.push(e.p);
}
// derived ratios the Infrastructure list uses
const byHostTs = new Map<string, Record<string, number>>();
const byHostTs = new Map<string, { host: string; ts: number; device: string[]; v: Record<string, number> }>();
for (const p of out)
if (/^system\.(mem\.(usable|total)|disk\.(used|total))$/.test(p.metric)) {
const k = `${p.host}|${p.points[0][0]}|${p.tags.filter((t) => t.startsWith("device:")).join(",")}`;
(byHostTs.get(k) ?? byHostTs.set(k, {}).get(k)!)[p.metric] = p.points[0][1];
const device = p.tags.filter((t) => t.startsWith("device:"));
const k = JSON.stringify([p.host, p.points[0][0], device]);
let e = byHostTs.get(k);
if (!e) byHostTs.set(k, (e = { host: p.host, ts: p.points[0][0], device, v: {} }));
e.v[p.metric] = p.points[0][1];
}
for (const [k, v] of byHostTs) {
const [host, ts] = k.split("|");
for (const { host, ts, device, v } of byHostTs.values()) {
const tags = hosts.get(host)?.tags ?? [];
if (v["system.mem.total"])
out.push({ metric: "system.mem.pct_usable", host, tags, type: "gauge", points: [[Number(ts), (v["system.mem.usable"] ?? 0) / v["system.mem.total"]]] });
out.push({ metric: "system.mem.pct_usable", host, tags, type: "gauge", points: [[ts, (v["system.mem.usable"] ?? 0) / v["system.mem.total"]]] });
if (v["system.disk.total"])
out.push({ metric: "system.disk.in_use", host, tags, type: "gauge", points: [[Number(ts), (v["system.disk.used"] ?? 0) / v["system.disk.total"]]] });
out.push({
metric: "system.disk.in_use",
host,
tags: [...tags, ...device],
type: "gauge",
points: [[ts, (v["system.disk.used"] ?? 0) / v["system.disk.total"]]],
});
}
return { points: out, hosts };
return {
points: out,
hosts,
commit: () => {
for (const [k, v] of refs) rememberCumulative(k, v);
},
};
}
2 changes: 2 additions & 0 deletions artifacts/api-server/src/intake/proto.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ const descriptor = {
max: { type: "double", id: 4 },
avg: { type: "double", id: 5 },
sum: { type: "double", id: 6 },
k: { rule: "repeated", type: "sint32", id: 7 },
n: { rule: "repeated", type: "uint32", id: 8 },
},
},
},
Expand Down
44 changes: 37 additions & 7 deletions artifacts/api-server/src/intake/routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { writeMetrics, touchHost, writeLogs, writeSpans, type PointIn, type LogI
import { orgForApiKey } from "../lib/org";
import { decodeOtlp, encodeEmptyResponse, otlpToSpans, otlpToLogs, otlpToMetrics, type OtlpSignal } from "./otlp";
import { logger } from "../lib/logger";
import { isStorageError } from "../lib/http";
import { pool } from "@workspace/db";

const router: Router = Router();
Expand Down Expand Up @@ -38,6 +39,37 @@ export function raw(req: Request, res: Response, next: NextFunction) {

const TYPE_V2: Record<number, string> = { 0: "gauge", 1: "count", 2: "rate", 3: "gauge" };

// The Datadog Agent's sketch bins (its default configuration: relative accuracy 1/128, minimum value 1e-9). Bin key k
// covers [γ^(k-bias), γ^(k-bias+1)); negative keys mirror it below zero, key 0 is zero.
const SKETCH_GAMMA = 1 + 2 / 128;
const SKETCH_BIAS = -Math.floor(Math.log(1e-9) / Math.log1p(2 / 128)) + 1;
export function sketchBinValue(k: number): number {
if (k === 0) return 0;
const lo = Math.pow(SKETCH_GAMMA, Math.abs(k) - SKETCH_BIAS);
return Math.sign(k) * lo * ((1 + SKETCH_GAMMA) / 2);
}
/** One weighted point per sketch bin (value of the bin, clamped to the sketch's min/max), so counts and percentiles hold. */
export function sketchPoints(dogsketches: any[]): [number, number, number][] {
const pts: [number, number, number][] = [];
for (const d of dogsketches) {
const ts = Number(d.ts);
const k: number[] = (d.k ?? []).map(Number);
const n: number[] = (d.n ?? []).map(Number);
const min = Number(d.min),
max = Number(d.max);
if (k.length && k.length === n.length) {
for (let i = 0; i < k.length; i++) {
if (!(n[i] > 0)) continue;
let v = sketchBinValue(k[i]);
if (Number.isFinite(min)) v = Math.max(v, min);
if (Number.isFinite(max)) v = Math.min(v, max);
pts.push([ts, v, n[i]]);
}
} else if (Number(d.cnt) > 0) pts.push([ts, Number(d.avg), Number(d.cnt)]);
}
return pts;
}

async function withOrg(req: Request, res: Response, next: NextFunction) {
const orgId = await orgForApiKey(apiKeyFrom(req));
if (!orgId) {
Expand All @@ -54,8 +86,7 @@ function wrap(fn: (req: Request, res: Response) => Promise<void>) {
return (req: Request, res: Response) => {
fn(req, res).catch((err) => {
if (res.headersSent) return;
const fromDb =
(typeof err?.code === "string" && /^[0-9A-Z]{5}$/.test(err.code)) || /^ClickHouse:/.test(String(err?.message ?? "")) || err?.code === "ECONNREFUSED";
const fromDb = isStorageError(err);
if (req.orgId) recordDrop(req.orgId, intakePath(req.path), 1, fromDb ? "storage error" : `rejected: ${String(err?.message ?? err).slice(0, 120)}`);
if (fromDb) {
logger.error({ err, path: req.path }, "intake storage error");
Expand Down Expand Up @@ -144,10 +175,8 @@ router.post(
wrap(async (req, res) => {
const p = decodeProto(SketchPayload, decompress(req));
const series: PointIn[] = [];
for (const s of p.sketches ?? []) {
const pts: [number, number][] = (s.dogsketches ?? []).map((d: any) => [Number(d.ts), Number(d.avg)]);
series.push({ metric: s.metric, host: s.host ?? "", tags: s.tags ?? [], type: "distribution", points: pts });
}
for (const s of p.sketches ?? [])
series.push({ metric: s.metric, host: s.host ?? "", tags: s.tags ?? [], type: "distribution", points: sketchPoints(s.dogsketches ?? []) });
await storeMetrics(req.orgId!, "metrics", series);
res.status(202).json({});
}),
Expand Down Expand Up @@ -432,13 +461,14 @@ function otlpRoute(signal: OtlpSignal) {
n = logs.length;
await writeLogs(orgId, logs);
} else {
const { points, hosts } = otlpToMetrics(orgId, body);
const { points, hosts, commit } = otlpToMetrics(orgId, body);
for (const [name, h] of hosts)
await touchHost(orgId, name, { os: h.os, platform: h.os, agentVersion: h.agentVersion, tags: h.tags, apps: ["system"] }, true);
const { valid, invalid } = splitPoints(points);
if (invalid) recordDrop(orgId, path, invalid, "points without a numeric value");
n = valid;
await writeMetrics(orgId, points);
commit();
}
} catch (e) {
logger.error({ err: e, signal }, "OTLP write failed");
Expand Down
Loading
Loading