Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
- name: Start DataCat
run: |
for i in $(seq 1 60); do curl -fsS http://localhost:8123/ping > /dev/null && break; sleep 1; done
PORT=3000 NODE_ENV=production SESSION_SECRET=ci-only-session-secret-0123456789abcdef node artifacts/api-server/dist/index.mjs > server.log 2>&1 &
PORT=3000 NODE_ENV=production PUBLIC_URL=http://localhost:3000 SESSION_SECRET=ci-only-session-secret-0123456789abcdef node artifacts/api-server/dist/index.mjs > server.log 2>&1 &
- name: End-to-end known-answer checks
run: bash scripts/e2e.sh
- name: Server log
Expand Down
8 changes: 7 additions & 1 deletion artifacts/api-server/src/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,20 @@ app.use(
},
serializers: {
req(req) {
return { id: req.id, method: req.method, url: req.url?.split("?")[0]?.replace(/(\/(?:invite|invites|agent|input)\/)[^/]+/g, "$1[redacted]") };
return { id: req.id, method: req.method, url: req.url?.split("?")[0]?.replace(/(\/(?:invite|invites|agent|input)\/)[^/]+/gi, "$1[redacted]") };
},
res(res) {
return { statusCode: res.statusCode };
},
},
}),
);
// DataCat is never embedded in another page.
app.use((_req, res, next) => {
res.setHeader("X-Frame-Options", "DENY");
res.setHeader("Content-Security-Policy", "frame-ancestors 'none'");
next();
});
app.use(cors());
// Agent intake routes take raw (possibly compressed / protobuf) bodies, so they mount before JSON parsing.
app.use(intakeRouter);
Expand Down
25 changes: 24 additions & 1 deletion artifacts/api-server/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { storage, useClickHouse } from "./lib/storage";
import { ensureChSchema } from "./lib/ch";
import { pool } from "@workspace/db";
import { generatedSetupToken } from "./routes/auth";
import { flushSpanStats } from "./query/apmStats";

const rawPort = process.env["PORT"] ?? "3000";
const port = Number(rawPort);
Expand All @@ -17,7 +18,7 @@ logger.info({ storage }, "telemetry storage");
if (generatedSetupToken && !(await pool.query("select 1 from users limit 1")).rowCount) {
logger.warn(`First-run setup token (enter it on the setup page): ${generatedSetupToken}`);
}
app.listen(port, (err) => {
const server = app.listen(port, (err) => {
if (err) {
logger.error({ err }, "Error listening on port");
process.exit(1);
Expand All @@ -31,3 +32,25 @@ app.listen(port, (err) => {
void buildBackgroundIndexes((msg, extra) => logger.info(extra ?? {}, msg));
} else logger.info("background jobs disabled (DATACAT_BACKGROUND_JOBS=false)");
});

// On SIGTERM/SIGINT: stop accepting connections, let requests in progress finish (up to DATACAT_SHUTDOWN_SECONDS,
// default 25), flush pending APM rollups, close the database pool, then exit.
let stopping = false;
for (const sig of ["SIGTERM", "SIGINT"] as const)
process.once(sig, () => {
if (stopping) return;
stopping = true;
logger.info({ signal: sig }, "shutting down");
const limitMs = Math.max(1, Number(process.env.DATACAT_SHUTDOWN_SECONDS ?? 25)) * 1000;
const force = setTimeout(() => {
logger.warn("shutdown time limit reached; exiting");
process.exit(0);
}, limitMs);
force.unref();
server.close(async () => {
await flushSpanStats().catch((e) => logger.warn({ err: e }, "final span stats flush failed"));
await pool.end().catch(() => undefined);
process.exit(0);
});
server.closeIdleConnections?.();
});
91 changes: 70 additions & 21 deletions artifacts/api-server/src/intake/otlp.ts
Original file line number Diff line number Diff line change
Expand Up @@ -380,8 +380,15 @@ function rememberCumulative(k: string, v: any) {
}
}
}
function cumKey(orgId: number, name: string, host: string, tags: string[], start: any) {
return `${orgId}|${name}|${host}|${tags.join(",")}|${start ?? ""}`;
const sortedAttrs = (a: Attrs) =>
JSON.stringify(
Object.entries(a)
.filter(([, v]) => v !== null && v !== undefined)
.sort(([x], [y]) => (x < y ? -1 : x > y ? 1 : 0)),
);
/** Identity of one cumulative stream: every resource, scope and data-point attribute (order-independent) and its start. */
function cumKey(orgId: number, name: string, resource: string, scope: string, dpAttrs: Attrs, start: any) {
return `${orgId}|${name}|${resource}|${scope}|${sortedAttrs(dpAttrs)}|${start ?? ""}`;
}

function quantileFromBuckets(bounds: number[], counts: number[], q: number, min?: number, max?: number): number {
Expand All @@ -402,13 +409,25 @@ function quantileFromBuckets(bounds: number[], counts: number[], q: number, min?
return max ?? bounds[bounds.length - 1] ?? NaN;
}

function expoToExplicit(dp: any): { bounds: number[]; counts: number[] } {
/**
* Exponential histogram -> explicit buckets in ascending order. Bucket i of a range covers (base^(offset+i),
* base^(offset+i+1)] in absolute value; negative buckets mirror it below zero, the zero bucket sits between.
*/
export function expoToExplicit(dp: any): { bounds: number[]; counts: number[] } {
const scale = Number(dp.scale ?? 0);
const base = Math.pow(2, Math.pow(2, -scale));
const bounds: number[] = [];
const counts: number[] = [];
const noff = Number(dp.negative?.offset ?? 0);
const nc = (dp.negative?.bucketCounts ?? []).map(Number);
for (let i = nc.length - 1; i >= 0; i--) {
bounds.push(-Math.pow(base, noff + i));
counts.push(nc[i]);
}
bounds.push(0);
counts.push(Number(dp.zeroCount ?? 0));
const off = Number(dp.positive?.offset ?? 0);
const pc = (dp.positive?.bucketCounts ?? []).map(Number);
const bounds: number[] = [0];
const counts: number[] = [Number(dp.zeroCount ?? 0)];
pc.forEach((c: number, i: number) => {
bounds.push(Math.pow(base, off + i + 1));
counts.push(c);
Expand Down Expand Up @@ -492,8 +511,18 @@ function hostMappings(
}
}

export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hosts: Map<string, { os?: string; agentVersion?: string; tags: string[] }> } {
/**
* Converts an OTLP metrics request. Cumulative streams are turned into deltas against the last stored value; the new
* reference values are applied only by commit(), which the caller runs after the points are stored, so a failed store
* followed by a retry computes the same deltas again instead of losing them.
*/
export function otlpToMetrics(
orgId: number,
req: any,
): { points: PointIn[]; hosts: Map<string, { os?: string; agentVersion?: string; tags: string[] }>; commit: () => void } {
const out: PointIn[] = [];
const refs = new Map<string, any>();
const refOf = (k: string) => (refs.has(k) ? refs.get(k) : cumulative.get(k));
const hosts = new Map<string, { os?: string; agentVersion?: string; tags: string[] }>();
const acc = new Map<string, { sum: number; n: number; p: PointIn }>();
for (const rm of req?.resourceMetrics ?? []) {
Expand All @@ -506,7 +535,9 @@ export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hos
tags: r.tags,
});
const baseTags = [...r.tags, ...(r.service ? [`service:${r.service}`] : [])];
const resKey = sortedAttrs(r.attrs);
for (const sm of rm.scopeMetrics ?? []) {
const scopeKey = `${sm.scope?.name ?? ""}@${sm.scope?.version ?? ""}`;
for (const m of sm.metrics ?? []) {
const name = String(m.name ?? "");
if (!name) continue;
Expand Down Expand Up @@ -547,9 +578,10 @@ export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hos
continue;
}
if (!delta) {
const k = cumKey(orgId, name, r.host, tags, dp.startTimeUnixNano);
const prev = cumulative.get(k);
rememberCumulative(k, { v, ts });
const k = cumKey(orgId, name, resKey, scopeKey, a, dp.startTimeUnixNano);
const prev = refOf(k);
if (prev !== undefined && ts < prev.ts) continue; // late or retried older sample: already counted
refs.set(k, { v, ts });
if (prev === undefined || v < prev.v) continue; // first sample or reset
const dt = Math.max(1, ts - prev.ts);
const d = v - prev.v;
Expand Down Expand Up @@ -577,10 +609,13 @@ export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hos
? { bounds: (dp.explicitBounds ?? []).map(Number), counts: (dp.bucketCounts ?? []).map(Number) }
: expoToExplicit(dp);
if (!delta) {
const k = cumKey(orgId, name, r.host, tags, dp.startTimeUnixNano);
const prev = cumulative.get(k);
rememberCumulative(k, { count, sum, counts });
if (!prev || count < prev.count) continue;
const k = cumKey(orgId, name, resKey, scopeKey, attrs(dp.attributes), dp.startTimeUnixNano);
const prev = refOf(k);
if (prev && ts < prev.ts) continue;
refs.set(k, { count, sum, counts, bounds, ts });
// buckets are subtracted range by range; a change of bucket layout starts a new reference
const sameLayout = prev && prev.bounds.length === bounds.length && prev.bounds.every((b: number, i: number) => b === bounds[i]);
if (!prev || !sameLayout || count < prev.count) continue;
count -= prev.count;
sum -= prev.sum;
counts = counts.map((c: number, i: number) => c - (prev.counts[i] ?? 0));
Expand Down Expand Up @@ -617,19 +652,33 @@ export function otlpToMetrics(orgId: number, req: any): { points: PointIn[]; hos
out.push(e.p);
}
// derived ratios the Infrastructure list uses
const byHostTs = new Map<string, Record<string, number>>();
const byHostTs = new Map<string, { host: string; ts: number; device: string[]; v: Record<string, number> }>();
for (const p of out)
if (/^system\.(mem\.(usable|total)|disk\.(used|total))$/.test(p.metric)) {
const k = `${p.host}|${p.points[0][0]}|${p.tags.filter((t) => t.startsWith("device:")).join(",")}`;
(byHostTs.get(k) ?? byHostTs.set(k, {}).get(k)!)[p.metric] = p.points[0][1];
const device = p.tags.filter((t) => t.startsWith("device:"));
const k = JSON.stringify([p.host, p.points[0][0], device]);
let e = byHostTs.get(k);
if (!e) byHostTs.set(k, (e = { host: p.host, ts: p.points[0][0], device, v: {} }));
e.v[p.metric] = p.points[0][1];
}
for (const [k, v] of byHostTs) {
const [host, ts] = k.split("|");
for (const { host, ts, device, v } of byHostTs.values()) {
const tags = hosts.get(host)?.tags ?? [];
if (v["system.mem.total"])
out.push({ metric: "system.mem.pct_usable", host, tags, type: "gauge", points: [[Number(ts), (v["system.mem.usable"] ?? 0) / v["system.mem.total"]]] });
out.push({ metric: "system.mem.pct_usable", host, tags, type: "gauge", points: [[ts, (v["system.mem.usable"] ?? 0) / v["system.mem.total"]]] });
if (v["system.disk.total"])
out.push({ metric: "system.disk.in_use", host, tags, type: "gauge", points: [[Number(ts), (v["system.disk.used"] ?? 0) / v["system.disk.total"]]] });
out.push({
metric: "system.disk.in_use",
host,
tags: [...tags, ...device],
type: "gauge",
points: [[ts, (v["system.disk.used"] ?? 0) / v["system.disk.total"]]],
});
}
return { points: out, hosts };
return {
points: out,
hosts,
commit: () => {
for (const [k, v] of refs) rememberCumulative(k, v);
},
};
}
2 changes: 2 additions & 0 deletions artifacts/api-server/src/intake/proto.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ const descriptor = {
max: { type: "double", id: 4 },
avg: { type: "double", id: 5 },
sum: { type: "double", id: 6 },
k: { rule: "repeated", type: "sint32", id: 7 },
n: { rule: "repeated", type: "uint32", id: 8 },
},
},
},
Expand Down
54 changes: 43 additions & 11 deletions artifacts/api-server/src/intake/redact.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,13 @@ const custom = String(env.DATACAT_REDACT_PATTERNS ?? "")
.map((s) => new RegExp(s, "g"));

const CARD = /\b(?:\d[ -]?){12,18}\d\b/g;
const BEARER = /\b(Bearer|Basic)\s+[A-Za-z0-9\-._~+/]{8,}=*/gi;
const BEARER = /\b(Bearer|Basic)\s+([A-Za-z0-9\-._~+/]{4,}=*)/gi;
const SECRET_PAIR = /\b(password|passwd|pwd|secret|token|api[_-]?key|access[_-]?key|client[_-]?secret)(["']?\s*[:=]\s*["']?)([^\s"'&,;]{3,})/gi;
const AWS_KEY = /\b(AKIA|ASIA)[A-Z0-9]{16}\b/g;
const PRIVATE_KEY = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g;
const JWT = /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g;
const EMAIL = /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b/g;
// Cheap pre-check: skip strings that cannot match any rule (most log lines and span attributes).
const MAYBE = /\d{4}|bearer|basic |pass|pwd|secret|token|key|AKIA|ASIA|PRIVATE KEY|eyJ|@/i;
const MAYBE = /\d[ -]?\d[ -]?\d[ -]?\d|bearer|basic|pass|pwd|secret|token|key|AKIA|ASIA|PRIVATE KEY|eyJ|@/i;

function luhn(digits: string): boolean {
let sum = 0,
Expand All @@ -72,14 +71,46 @@ function luhn(digits: string): boolean {

export const redactionStats = { values: 0, redactions: 0 };

/**
* Masks PEM private key blocks ("-----BEGIN ... PRIVATE KEY-----" to the next "-----END ... PRIVATE KEY-----") with
* plain string searches, so the cost stays linear however many BEGIN markers a value contains.
*/
function maskPrivateKeys(s: string): string {
const HEAD = /-----BEGIN [A-Z ]{0,40}PRIVATE KEY-----/g;
const TAIL = /-----END [A-Z ]{0,40}PRIVATE KEY-----/g;
let out = "";
let pos = 0;
for (;;) {
HEAD.lastIndex = pos;
const b = HEAD.exec(s);
if (!b) break;
TAIL.lastIndex = b.index + b[0].length;
const e = TAIL.exec(s);
if (!e) break; // no END after this BEGIN, so none after any later BEGIN either
out += s.slice(pos, b.index) + "[REDACTED:private_key]";
pos = e.index + e[0].length;
}
return pos === 0 ? s : out + s.slice(pos);
}

// "Basic <base64 of user:password>": short values count only when they decode to a user:password pair, so ordinary
// phrases such as "Basic plan" are left alone.
function isCredential(scheme: string, token: string): boolean {
const bare = token.replace(/=+$/, "");
if (bare.length >= 8) return true;
if (scheme.toLowerCase() !== "basic") return false;
if (!/^[A-Za-z0-9+/]+={0,2}$/.test(token) || token.length % 4 !== 0) return false;
return /^[\x20-\x7e]*:[\x20-\x7e]*$/.test(Buffer.from(token, "base64").toString("latin1"));
}

export function redactString(s: string): string {
if (!enabled || typeof s !== "string" || s.length < 6) return s;
if (!MAYBE.test(s) && !custom.length) return s;
let out = s;
const before = out;
out = out.replace(PRIVATE_KEY, "[REDACTED:private_key]");
out = maskPrivateKeys(out);
out = out.replace(JWT, "[REDACTED:jwt]");
out = out.replace(BEARER, (_m, scheme) => `${scheme} [REDACTED:bearer]`);
out = out.replace(BEARER, (m, scheme, token) => (isCredential(scheme, token) ? `${scheme} [REDACTED:bearer]` : m));
out = out.replace(AWS_KEY, "[REDACTED:aws_key]");
out = out.replace(SECRET_PAIR, (_m, k, sep) => `${k}${sep}[REDACTED:secret]`);
out = out.replace(CARD, (m) => {
Expand All @@ -105,17 +136,16 @@ export function redactObject<T extends Record<string, any>>(o: T): T {
for (const k of Object.keys(o)) {
const v = o[k];
if (typeof v === "string") (o as any)[k] = secretKey(k) && v.length ? "[REDACTED:key]" : redactString(v);
else if (v && typeof v === "object") {
if (secretKey(k) && !Array.isArray(v)) (o as any)[k] = "[REDACTED:key]";
else redactObject(v);
}
else if (secretKey(k) && (typeof v === "number" || typeof v === "bigint" || (v && typeof v === "object"))) (o as any)[k] = "[REDACTED:key]";
else if (v && typeof v === "object") redactObject(v);
}
return o;
}

export function redactLog<L extends { message: string; attributes: Record<string, any>; tags?: string[] }>(l: L): L {
export function redactLog<L extends { message: string; attributes: Record<string, any>; tags?: string[]; service?: string }>(l: L): L {
if (!enabled) return l;
l.message = redactString(l.message);
if (typeof l.service === "string") l.service = redactString(l.service);
redactObject(l.attributes);
if (l.tags)
l.tags = l.tags.map((t) => {
Expand All @@ -125,9 +155,11 @@ export function redactLog<L extends { message: string; attributes: Record<string
return l;
}

export function redactSpan<S extends { resource: string; meta?: Record<string, string> }>(s: S): S {
export function redactSpan<S extends { resource: string; meta?: Record<string, string>; name?: string; service?: string }>(s: S): S {
if (!enabled) return s;
s.resource = redactString(s.resource);
if (typeof s.name === "string") s.name = redactString(s.name);
if (typeof s.service === "string") s.service = redactString(s.service);
if (s.meta) redactObject(s.meta);
return s;
}
Expand Down
Loading
Loading