diff --git a/apps/api/package.json b/apps/api/package.json index 34ddcdc8c..8bc588f11 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -36,12 +36,14 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/clickhouse-builder": "workspace:*", "@maple-dev/effect-sdk": "workspace:*", + "@maple/alerting-core": "workspace:*", "@maple/auth": "workspace:*", "@maple/cache": "workspace:*", "@maple/db": "workspace:*", "@maple/domain": "workspace:*", "@maple/effect-cloudflare": "workspace:*", "@maple/email": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/infra": "workspace:*", "@maple/llm": "workspace:*", "@maple/query-engine": "workspace:*", diff --git a/apps/api/src/planetscale-webhook-runtime.test.ts b/apps/api/src/planetscale-webhook-runtime.test.ts index 5b0c77887..c9773726f 100644 --- a/apps/api/src/planetscale-webhook-runtime.test.ts +++ b/apps/api/src/planetscale-webhook-runtime.test.ts @@ -1,27 +1,44 @@ import type { MessageBatch } from "@cloudflare/workers-types" import { afterEach, assert, describe, it } from "@effect/vitest" -import { Effect, Layer } from "effect" +import { OrgId } from "@maple/domain/http" +import { Effect, Layer, Schema } from "effect" import { Database, DatabaseError } from "@/platform/DatabaseLive" import { cleanupTestDbs, createTestDb, queryFirstRow, type TestDb } from "@/platform/test-pglite" import { processPlanetScaleWebhookBatch } from "./planetscale-webhook-runtime" +import { + projectPlanetScaleWebhookEvent, + type PlanetScaleWebhookPayload, +} from "./services/integrations/planetscale/webhook-events" import type { PlanetScaleWebhookJob } from "./services/integrations/planetscale/PlanetScaleWebhookQueue" const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) -const job: PlanetScaleWebhookJob = { +const orgId = Schema.decodeUnknownSync(OrgId)("org_1") + +const basePayload: PlanetScaleWebhookPayload = { + timestamp: 1, + event: "branch.out_of_memory", + organization: "acme", + database: "shop", + resource: { name: "main" }, +} + +const makeJob = (payload: PlanetScaleWebhookPayload = basePayload): PlanetScaleWebhookJob => ({ kind: "planetscale-webhook", - orgId: "org_1", + orgId, connectionId: "connection_1", - payload: { - event: "branch.out_of_memory", - organization: "acme", - database: "shop", - resource: { name: "main" }, - }, receivedAt: 1_000, -} + event: projectPlanetScaleWebhookEvent({ + orgId, + connectionId: "connection_1", + payload, + receivedAt: 1_000, + }), +}) + +const job = makeJob() const makeBatch = (body: unknown) => { let acknowledged = false @@ -70,6 +87,120 @@ describe("PlanetScale webhook queue consumer", () => { }).pipe(Effect.provide(testDb.layer)) }) + it.effect("applies an issue event exactly once across duplicate queue deliveries", () => { + const testDb = createTestDb(trackedDbs) + const first = makeBatch(job) + const duplicate = makeBatch(job) + return Effect.gen(function* () { + yield* processPlanetScaleWebhookBatch(first.batch) + yield* Effect.promise(() => + testDb.pglite.exec( + "UPDATE error_issues SET workflow_state = 'done', resolved_at = '2026-08-20T00:00:00Z'", + ), + ) + yield* processPlanetScaleWebhookBatch(duplicate.batch) + assert.isTrue(first.acknowledged()) + assert.isTrue(duplicate.acknowledged()) + const issue = yield* Effect.promise(() => + queryFirstRow<{ occurrence_count: number; workflow_state: string }>( + testDb, + "SELECT occurrence_count, workflow_state FROM error_issues WHERE org_id = $1", + ["org_1"], + ), + ) + assert.strictEqual(issue?.occurrence_count, 1) + assert.strictEqual(issue?.workflow_state, "done") + const history = yield* Effect.promise(() => + queryFirstRow<{ count: number }>( + testDb, + "SELECT count(*)::int AS count FROM error_issue_events WHERE org_id = $1", + ["org_1"], + ), + ) + assert.strictEqual(history?.count, 1) + }).pipe(Effect.provide(testDb.layer)) + }) + + it.effect("recovers exactly once after the timeline commits but the issue transaction fails", () => { + const testDb = createTestDb(trackedDbs) + const failed = makeBatch(job) + const retry = makeBatch(job) + return Effect.gen(function* () { + yield* Effect.promise(() => + testDb.pglite.exec(`CREATE FUNCTION reject_planetscale_issue_event() RETURNS trigger AS $$ + BEGIN RAISE EXCEPTION 'forced issue event failure'; END; + $$ LANGUAGE plpgsql; + CREATE TRIGGER reject_planetscale_issue_event + BEFORE INSERT ON error_issue_events + FOR EACH ROW EXECUTE FUNCTION reject_planetscale_issue_event();`), + ) + yield* processPlanetScaleWebhookBatch(failed.batch) + assert.isTrue(failed.retried()) + yield* Effect.promise(() => + testDb.pglite.exec(`DROP TRIGGER reject_planetscale_issue_event ON error_issue_events; + DROP FUNCTION reject_planetscale_issue_event();`), + ) + yield* processPlanetScaleWebhookBatch(retry.batch) + assert.isTrue(retry.acknowledged()) + const counts = yield* Effect.promise(() => + queryFirstRow<{ timeline: number; issues: number; receipts: number }>( + testDb, + `SELECT + (SELECT count(*)::int FROM planetscale_events) AS timeline, + (SELECT count(*)::int FROM error_issues) AS issues, + (SELECT count(*)::int FROM planetscale_issue_receipts) AS receipts`, + ), + ) + assert.deepStrictEqual(counts, { timeline: 1, issues: 1, receipts: 1 }) + }).pipe(Effect.provide(testDb.layer)) + }) + + it.effect("processes the exact pre-event-envelope queue body during rolling upgrades", () => { + const testDb = createTestDb(trackedDbs) + const legacyJob = { + kind: "planetscale-webhook", + orgId, + connectionId: "connection_1", + payload: basePayload, + receivedAt: 1_000, + } + const delivery = makeBatch(legacyJob) + return Effect.gen(function* () { + yield* processPlanetScaleWebhookBatch(delivery.batch) + assert.isTrue(delivery.acknowledged()) + assert.isFalse(delivery.retried()) + const row = yield* Effect.promise(() => + queryFirstRow<{ workflow_state: string; occurrence_count: number }>( + testDb, + "SELECT workflow_state, occurrence_count FROM error_issues WHERE org_id = $1", + ["org_1"], + ), + ) + assert.strictEqual(row?.workflow_state, "triage") + assert.strictEqual(row?.occurrence_count, 1) + }).pipe(Effect.provide(testDb.layer)) + }) + + it.effect("terminally acknowledges timestamp-less legacy queue bodies", () => { + const testDb = createTestDb(trackedDbs) + const delivery = makeBatch({ + kind: "planetscale-webhook", + orgId, + connectionId: "connection_1", + payload: { ...basePayload, timestamp: null }, + receivedAt: 1_000, + }) + return processPlanetScaleWebhookBatch(delivery.batch).pipe( + Effect.tap(() => + Effect.sync(() => { + assert.isTrue(delivery.acknowledged()) + assert.isFalse(delivery.retried()) + }), + ), + Effect.provide(testDb.layer), + ) + }) + it.effect("acknowledges terminal malformed jobs", () => { const testDb = createTestDb(trackedDbs) const delivery = makeBatch({ kind: "not-a-planetscale-job" }) @@ -84,12 +215,26 @@ describe("PlanetScale webhook queue consumer", () => { ) }) - it.effect("writes a lifecycle event to the timeline but not to the issue hub", () => { + it.effect("terminally acknowledges schema-valid jobs with contradictory event identity", () => { const testDb = createTestDb(trackedDbs) const delivery = makeBatch({ ...job, - payload: { ...job.payload, event: "branch.ready" }, + event: { ...job.event, tenantid: Schema.decodeUnknownSync(OrgId)("org_2") }, }) + return processPlanetScaleWebhookBatch(delivery.batch).pipe( + Effect.tap(() => + Effect.sync(() => { + assert.isTrue(delivery.acknowledged()) + assert.isFalse(delivery.retried()) + }), + ), + Effect.provide(testDb.layer), + ) + }) + + it.effect("writes a lifecycle event to the timeline but not to the issue hub", () => { + const testDb = createTestDb(trackedDbs) + const delivery = makeBatch(makeJob({ ...basePayload, event: "branch.ready" })) return Effect.gen(function* () { yield* processPlanetScaleWebhookBatch(delivery.batch) assert.isTrue(delivery.acknowledged()) @@ -138,14 +283,13 @@ describe("PlanetScale webhook queue consumer", () => { it.effect("carries the deploy-request number so redelivery dedupes", () => { const testDb = createTestDb(trackedDbs) - const delivery = makeBatch({ - ...job, - payload: { - ...job.payload, + const delivery = makeBatch( + makeJob({ + ...basePayload, event: "deploy_request.schema_applied", resource: { number: 42 }, - }, - }) + }), + ) return Effect.gen(function* () { yield* processPlanetScaleWebhookBatch(delivery.batch) const event = yield* Effect.promise(() => diff --git a/apps/api/src/planetscale-webhook-runtime.ts b/apps/api/src/planetscale-webhook-runtime.ts index 71bc6c596..cafa1569c 100644 --- a/apps/api/src/planetscale-webhook-runtime.ts +++ b/apps/api/src/planetscale-webhook-runtime.ts @@ -10,9 +10,12 @@ import { deployRequestNumber, insertPlanetScaleEvent, planetScaleBranchName, + planetScaleWebhookPayloadFromEvent, + planetScaleWebhookTimestampMillis, + projectPlanetScaleWebhookEvent, upsertPlanetScaleIssue, } from "./services/integrations/planetscale/webhook-events" -import { PlanetScaleWebhookJob } from "./services/integrations/planetscale/PlanetScaleWebhookQueue" +import { PlanetScaleWebhookQueueMessage } from "./services/integrations/planetscale/PlanetScaleWebhookQueue" const telemetry = MapleCloudflareSDK.make({ serviceName: "maple-api", @@ -32,7 +35,7 @@ export const buildPlanetScaleWebhookLayer = (_env: Record) => { export const flushPlanetScaleWebhookTelemetry = (env: Record) => telemetry.flush(env) -const decodeJob = Schema.decodeUnknownEffect(PlanetScaleWebhookJob) +const decodeJob = Schema.decodeUnknownEffect(PlanetScaleWebhookQueueMessage) export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => Effect.forEach( @@ -54,11 +57,52 @@ export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => ), ), onSuccess: (job) => { - const classified = classifyPlanetScaleEvent(job.payload.event) + if (!("event" in job) && planetScaleWebhookTimestampMillis(job.payload) === null) + return Effect.logWarning( + "Discarding timestamp-less legacy PlanetScale webhook queue message", + ).pipe( + Effect.annotateLogs({ + orgId: job.orgId, + connectionId: job.connectionId, + event: job.payload.event, + }), + Effect.flatMap(() => Effect.sync(() => message.ack())), + Effect.tap(() => + Effect.annotateCurrentSpan({ + "maple.planetscale.webhook.queue.outcome": "timestamp_missing_ack", + }), + ), + ) + // Old jobs can remain in Cloudflare Queue across a deploy. Rebuild the + // event from the durable legacy fields instead of malformed-acking them. + const event = + "event" in job + ? job.event + : projectPlanetScaleWebhookEvent({ + orgId: job.orgId, + connectionId: job.connectionId, + payload: job.payload, + receivedAt: job.receivedAt, + }) + const payload = + "event" in job + ? planetScaleWebhookPayloadFromEvent(event, job.orgId, job.connectionId) + : job.payload + const eventData = + typeof event.data === "object" && + event.data !== null && + !Array.isArray(event.data) + ? (event.data as { readonly [key: string]: unknown }) + : null + const eventName = + typeof eventData?.event === "string" ? eventData.event : payload.event + const classified = classifyPlanetScaleEvent(eventName) const annotateJob = Effect.annotateCurrentSpan({ orgId: job.orgId, + "maple.event.id": event.id, + "maple.event.type": event.type, "maple.planetscale.connection_id": job.connectionId, - "maple.planetscale.webhook.event": job.payload.event, + "maple.planetscale.webhook.event": payload.event, }) if (classified.action !== "issue" && classified.action !== "timeline") { return annotateJob.pipe( @@ -70,38 +114,38 @@ export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => Effect.annotateLogs({ orgId: job.orgId, connectionId: job.connectionId, - event: job.payload.event, + event: payload.event, }), Effect.flatMap(() => Effect.sync(() => message.ack())), ) } const timestamp = - job.payload.timestamp != null && job.payload.timestamp > 0 - ? job.payload.timestamp * 1000 + payload.timestamp != null && payload.timestamp > 0 + ? payload.timestamp * 1000 : job.receivedAt const spec = classified.timeline const timeline = insertPlanetScaleEvent({ orgId: job.orgId, - databaseName: job.payload.database ?? "unknown", + databaseName: payload.database ?? "unknown", branchName: - spec.category === "deploy_request" ? "" : planetScaleBranchName(job.payload), + spec.category === "deploy_request" ? "" : planetScaleBranchName(payload), category: spec.category, - eventType: job.payload.event, + eventType: payload.event, state: spec.state, externalId: - spec.category === "deploy_request" ? deployRequestNumber(job.payload) : "", - title: spec.title(job.payload), + spec.category === "deploy_request" ? deployRequestNumber(payload) : "", + title: spec.title(payload), source: "webhook", - payload: job.payload.resource ?? null, + payload: payload.resource ?? null, occurredAtMs: timestamp, createdAtMs: job.receivedAt, }).pipe( Effect.withSpan("PlanetScaleWebhookQueue.persistTimelineEvent", { attributes: { orgId: job.orgId, - "maple.planetscale.webhook.event": job.payload.event, + "maple.planetscale.webhook.event": payload.event, }, }), ) @@ -119,10 +163,11 @@ export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => Effect.flatMap(() => upsertPlanetScaleIssue({ orgId: job.orgId, - payload: job.payload, + eventId: event.id, + payload, severity: classified.severity, title: classified.title, - description: classified.describe(job.payload), + description: classified.describe(payload), timestamp, }), ), @@ -130,7 +175,7 @@ export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => attributes: { orgId: job.orgId, "maple.planetscale.connection_id": job.connectionId, - "maple.planetscale.webhook.event": job.payload.event, + "maple.planetscale.webhook.event": payload.event, }, }), ) @@ -142,7 +187,7 @@ export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => Effect.annotateLogs({ orgId: job.orgId, connectionId: job.connectionId, - event: job.payload.event, + event: payload.event, attempt: message.attempts, error: error.message, }), @@ -158,7 +203,7 @@ export const processPlanetScaleWebhookBatch = (batch: MessageBatch) => Effect.annotateLogs({ orgId: job.orgId, connectionId: job.connectionId, - event: job.payload.event, + event: payload.event, issueId: result.issueId, issueAction: result.action, }), diff --git a/apps/api/src/routes/v1/planetscale-webhook.http.test.ts b/apps/api/src/routes/v1/planetscale-webhook.http.test.ts index 3b22e91f4..bc085ce17 100644 --- a/apps/api/src/routes/v1/planetscale-webhook.http.test.ts +++ b/apps/api/src/routes/v1/planetscale-webhook.http.test.ts @@ -8,6 +8,7 @@ import { Database } from "@/platform/DatabaseLive" import { Env } from "@/platform/Env" import { cleanupTestDbs, createTestDb, type TestDb } from "@/platform/test-pglite" import { + MAX_PLANETSCALE_WEBHOOK_QUEUE_BYTES, PlanetScaleWebhookQueue, PlanetScaleWebhookQueueError, type PlanetScaleWebhookJob, @@ -146,6 +147,7 @@ describe("PlanetScaleWebhookRouter", () => { }), ) const issueBody = JSON.stringify({ + timestamp: 1_698_252_879, event: "branch.out_of_memory", organization: "acme", database: "shop", @@ -171,6 +173,52 @@ describe("PlanetScaleWebhookRouter", () => { assert.strictEqual(rejected.status, 401) assert.strictEqual(jobs.length, 0) + const timestampLessBody = JSON.stringify({ + event: "branch.out_of_memory", + organization: "acme", + database: "shop", + }) + const timestampLess = yield* Effect.promise(() => + handler( + new Request(`http://api.localhost${WEBHOOK_PATH}`, { + method: "POST", + headers: { + "x-planetscale-signature": createHmac("sha256", SECRET) + .update(timestampLessBody, "utf8") + .digest("hex"), + }, + body: timestampLessBody, + }), + Context.make(Database, database), + ), + ) + assert.strictEqual(timestampLess.status, 400) + assert.strictEqual(jobs.length, 0) + + const oversizedBody = JSON.stringify({ + timestamp: 1_698_252_879, + event: "branch.out_of_memory", + organization: "acme", + database: "shop", + resource: { payload: "x".repeat(MAX_PLANETSCALE_WEBHOOK_QUEUE_BYTES) }, + }) + const oversized = yield* Effect.promise(() => + handler( + new Request(`http://api.localhost${WEBHOOK_PATH}`, { + method: "POST", + headers: { + "x-planetscale-signature": createHmac("sha256", SECRET) + .update(oversizedBody, "utf8") + .digest("hex"), + }, + body: oversizedBody, + }), + Context.make(Database, database), + ), + ) + assert.strictEqual(oversized.status, 413) + assert.strictEqual(jobs.length, 0) + const accepted = yield* Effect.promise(() => handler( new Request(`http://api.localhost${WEBHOOK_PATH}`, { @@ -186,12 +234,17 @@ describe("PlanetScaleWebhookRouter", () => { assert.strictEqual(jobs[0]?.kind, "planetscale-webhook") assert.strictEqual(jobs[0]?.orgId, "org_1") assert.strictEqual(jobs[0]?.connectionId, CONNECTION_ID) - assert.strictEqual(jobs[0]?.payload.event, "branch.out_of_memory") + assert.strictEqual( + (jobs[0]?.event.data as { readonly event: string }).event, + "branch.out_of_memory", + ) + assert.strictEqual(jobs[0]?.event.type, "dev.maple.planetscale.webhook.received.v1") + assert.strictEqual(jobs[0]?.event.tenantid, "org_1") }).pipe(Effect.ensuring(Effect.promise(dispose))) }).pipe(Effect.provide(testDb.layer)) }) - it.effect("enqueues lifecycle events too, and still drops genuinely unknown ones", () => { + it.effect("enqueues every verified factual event before downstream classification", () => { const testDb = createTestDb(trackedDbs) const jobs: PlanetScaleWebhookJob[] = [] return Effect.gen(function* () { @@ -221,7 +274,7 @@ describe("PlanetScaleWebhookRouter", () => { ) const post = (payload: Record) => { - const body = JSON.stringify(payload) + const body = JSON.stringify({ timestamp: 1_698_252_879, ...payload }) return Effect.promise(() => handler( new Request(`http://api.localhost${WEBHOOK_PATH}`, { @@ -249,7 +302,10 @@ describe("PlanetScaleWebhookRouter", () => { }) assert.strictEqual(deploy.status, 202) assert.strictEqual(jobs.length, 1) - assert.strictEqual(jobs[0]?.payload.event, "deploy_request.schema_applied") + assert.strictEqual( + (jobs[0]?.event.data as { readonly event: string }).event, + "deploy_request.schema_applied", + ) const branchReady = yield* post({ event: "branch.ready", @@ -260,15 +316,19 @@ describe("PlanetScaleWebhookRouter", () => { assert.strictEqual(branchReady.status, 202) assert.strictEqual(jobs.length, 2) - // Forward-compatibility must not become "enqueue everything": an - // event neither side knows is acknowledged and dropped. + // Unknown provider facts also enter the typed event layer. The current + // issue/timeline consumer may ignore them, but other consumers can opt in. const unknown = yield* post({ event: "branch.some_future_event", organization: "acme", database: "shop", }) assert.strictEqual(unknown.status, 202) - assert.strictEqual(jobs.length, 2) + assert.strictEqual(jobs.length, 3) + assert.strictEqual( + (jobs[2]?.event.data as { readonly event: string }).event, + "branch.some_future_event", + ) }).pipe(Effect.ensuring(Effect.promise(dispose))) }).pipe(Effect.provide(testDb.layer)) }) @@ -297,6 +357,7 @@ describe("PlanetScaleWebhookRouter", () => { }), ) const issueBody = JSON.stringify({ + timestamp: 1_698_252_879, event: "branch.anomaly", organization: "acme", database: "shop", diff --git a/apps/api/src/routes/v1/planetscale-webhook.http.ts b/apps/api/src/routes/v1/planetscale-webhook.http.ts index f09355f6b..732f87810 100644 --- a/apps/api/src/routes/v1/planetscale-webhook.http.ts +++ b/apps/api/src/routes/v1/planetscale-webhook.http.ts @@ -9,9 +9,15 @@ import { Env } from "@/platform/Env" import { classifyPlanetScaleEvent, decodePlanetScaleWebhookPayload, + projectPlanetScaleWebhookEvent, + planetScaleWebhookTimestampMillis, verifyPlanetScaleSignature, } from "@/services/integrations/planetscale/webhook-events" -import { PlanetScaleWebhookQueue } from "@/services/integrations/planetscale/PlanetScaleWebhookQueue" +import { + MAX_PLANETSCALE_WEBHOOK_QUEUE_BYTES, + PlanetScaleWebhookQueue, + planetScaleWebhookQueueJobBytes, +} from "@/services/integrations/planetscale/PlanetScaleWebhookQueue" // Public PlanetScale webhook receiver. NOT behind auth — authenticity comes // from the per-connection HMAC secret (`X-PlanetScale-Signature`, SHA-256 hex @@ -162,32 +168,53 @@ export const PlanetScaleWebhookRouter = HttpRouter.use((router) => }) return textResponse("ok", 200) } + if (planetScaleWebhookTimestampMillis(payload) === null) + return yield* reject(400, "timestamp_rejected", "Webhook timestamp is required") - // Both issue-worthy and timeline-only events go through the queue: the - // durable retry is what makes a missed deploy marker recoverable. - if (classified.action === "issue" || classified.action === "timeline") { + // Every verified factual event is normalized and projected before the + // durable queue boundary. The queued CloudEvent is the stable contract. + { const now = yield* Clock.currentTimeMillis - const enqueued = yield* webhookQueue - .send({ - kind: "planetscale-webhook", - orgId: decodeOrgIdSync(connection.orgId), - connectionId, - payload, - receivedAt: now, - }) - .pipe( - Effect.tapError((error) => - Effect.logError("PlanetScale webhook enqueue failed").pipe( - Effect.annotateLogs({ - orgId: connection.orgId, - connectionId, - event: payload.event, - error: error.message, - }), - ), - ), - Effect.option, + const orgId = decodeOrgIdSync(connection.orgId) + const event = yield* Effect.try({ + try: () => + projectPlanetScaleWebhookEvent({ + orgId, + connectionId, + payload, + receivedAt: now, + }), + catch: () => + new PlanetScaleWebhookUnavailable({ + message: "Webhook event projection unavailable", + }), + }) + const job = { + kind: "planetscale-webhook" as const, + orgId, + connectionId, + receivedAt: now, + event, + } + if (planetScaleWebhookQueueJobBytes(job) > MAX_PLANETSCALE_WEBHOOK_QUEUE_BYTES) + return yield* reject( + 413, + "queue_message_too_large", + "Webhook payload exceeds the durable queue limit", ) + const enqueued = yield* webhookQueue.send(job).pipe( + Effect.tapError((error) => + Effect.logError("PlanetScale webhook enqueue failed").pipe( + Effect.annotateLogs({ + orgId: connection.orgId, + connectionId, + event: payload.event, + error: error.message, + }), + ), + ), + Effect.option, + ) if (Option.isNone(enqueued)) { return yield* unavailable("queue_unavailable", "Webhook queue unavailable") } @@ -198,10 +225,6 @@ export const PlanetScaleWebhookRouter = HttpRouter.use((router) => event: payload.event, }), ) - } else { - yield* Effect.logInfo("PlanetScale webhook lifecycle event acknowledged").pipe( - Effect.annotateLogs({ orgId: connection.orgId, event: payload.event }), - ) } yield* Effect.annotateCurrentSpan({ diff --git a/apps/api/src/services/alerts/AlertDestinationDelivery.ts b/apps/api/src/services/alerts/AlertDestinationDelivery.ts index 42c9ba691..9de266653 100644 --- a/apps/api/src/services/alerts/AlertDestinationDelivery.ts +++ b/apps/api/src/services/alerts/AlertDestinationDelivery.ts @@ -8,6 +8,7 @@ import { type AlertIncidentId, type AlertRuleId, } from "@maple/domain/http" +import { projectAlertLifecycleEvent } from "@maple/alerting-core" import type { AlertDestinationRow } from "@maple/db" import { Effect } from "effect" import { parseBase64Aes256GcmKey } from "@/platform/Crypto" @@ -129,8 +130,26 @@ export const makeAlertDestinationDelivery = (options: { { sendEmail, resolveSlackBotToken: options.resolveSlackBotToken }, ) - const buildPayload = (context: AlertDeliveryPayloadContext) => + const buildPayload = (context: AlertDeliveryPayloadContext, tenantId: string) => ({ + event: projectAlertLifecycleEvent({ + tenantId, + ruleId: context.ruleId, + ruleName: context.ruleName, + incidentId: context.incidentId, + eventType: context.eventType, + incidentStatus: context.incidentStatus, + groupKey: context.groupKey, + signalType: context.signalType, + severity: context.severity, + comparator: context.comparator, + threshold: context.threshold, + thresholdUpper: context.thresholdUpper, + windowMinutes: context.windowMinutes, + value: context.value, + sampleCount: context.sampleCount, + occurredAtMs: context.sentAtMs, + }), eventType: context.eventType, incidentId: context.incidentId, incidentStatus: context.incidentStatus, @@ -164,6 +183,7 @@ export const makeAlertDestinationDelivery = (options: { chatUrl: buildAlertChatUrl(options.appBaseUrl, context), sentAt: new Date(context.sentAtMs).toISOString(), }) satisfies { + readonly event: ReturnType readonly eventType: AlertDeliveryPayloadContext["eventType"] readonly incidentId: AlertIncidentId | null readonly incidentStatus: AlertDeliveryPayloadContext["incidentStatus"] @@ -192,7 +212,7 @@ export const makeAlertDestinationDelivery = (options: { secretConfig: enrichedSecret, ...context, } - const payload = buildPayload(fullContext) + const payload = buildPayload(fullContext, destinationRow.orgId) return yield* dispatchDelivery(fullContext, JSON.stringify(payload)) }) diff --git a/apps/api/src/services/alerts/AlertsService.test.ts b/apps/api/src/services/alerts/AlertsService.test.ts index c29f59101..10f89434b 100644 --- a/apps/api/src/services/alerts/AlertsService.test.ts +++ b/apps/api/src/services/alerts/AlertsService.test.ts @@ -3,6 +3,7 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { Cause, Clock, ConfigProvider, Duration, Effect, Exit, Layer, Option, Schema } from "effect" import { TestClock } from "effect/testing" +import { projectAlertLifecycleEvent } from "@maple/alerting-core" import { AlertDestinationInUseError, AlertForbiddenError, @@ -1909,6 +1910,24 @@ describe("AlertsService", () => { const userId = asUserId("user_timeout") const destination = yield* createWebhookDestination(alerts, orgId, userId) const rule = yield* createErrorRateRule(alerts, orgId, userId, destination.id) + const lifecycleEvent = projectAlertLifecycleEvent({ + tenantId: orgId, + ruleId: rule.id, + ruleName: rule.name, + incidentId: null, + eventType: "test", + incidentStatus: "resolved", + groupKey: null, + signalType: rule.signalType, + severity: rule.severity, + comparator: rule.comparator, + threshold: rule.threshold, + thresholdUpper: rule.thresholdUpper, + windowMinutes: rule.windowMinutes, + value: 0, + sampleCount: 0, + occurredAtMs: fixedTime, + }) yield* Effect.promise(() => insertDeliveryEventRow(testDb, { @@ -1923,6 +1942,7 @@ describe("AlertsService", () => { status: "queued", scheduledAt: fixedTime - 1, payloadJson: JSON.stringify({ + event: lifecycleEvent, eventType: "test", incidentId: null, incidentStatus: "resolved", @@ -1943,6 +1963,7 @@ describe("AlertsService", () => { }, linkUrl: "http://127.0.0.1:3471/alerts", sentAt: new Date(fixedTime).toISOString(), + futureAdditiveField: { preserve: true }, }), }), ) @@ -1951,6 +1972,18 @@ describe("AlertsService", () => { // live runtime clock, so the timeout fires on its own in real time. const tick = yield* alerts.runSchedulerTick() const events = yield* alerts.listDeliveryEvents(orgId) + const retryPayload = yield* Effect.promise(() => + queryFirstRow<{ + payload_json: { + event?: unknown + futureAdditiveField?: unknown + } + }>( + testDb, + "select payload_json from alert_delivery_events where delivery_key = $1 and attempt_number = 2", + ["timeout-delivery-key"], + ), + ) assert.strictEqual(tick.processedCount, 1) assert.strictEqual(tick.deliveryFailureCount, 1) @@ -1963,6 +1996,8 @@ describe("AlertsService", () => { assert.strictEqual(timeoutEvent?.status, "failed") assert.include(timeoutEvent?.errorMessage ?? "", "timed out") assert.strictEqual(retryEvent?.status, "queued") + assert.deepStrictEqual(retryPayload?.payload_json.event, lifecycleEvent) + assert.deepStrictEqual(retryPayload?.payload_json.futureAdditiveField, { preserve: true }) }).pipe( Effect.provide( makeLayer(testDb, makeWarehouseStub({ tracesAggregateRows: emptyWarehouseRows }), { diff --git a/apps/api/src/services/alerts/AlertsService.ts b/apps/api/src/services/alerts/AlertsService.ts index 0992a9261..04afe683a 100644 --- a/apps/api/src/services/alerts/AlertsService.ts +++ b/apps/api/src/services/alerts/AlertsService.ts @@ -1,6 +1,17 @@ +import { + alertDeliveryRetryDelayMs, + canRetryAlertDelivery, + evaluateAlertObservation, + interleaveAlertRulesByTenant, + makeAlertDeliveryKey, + planAlertLifecycle, + type AlertLifecycleInput, +} from "@maple/alerting-core" import { formatWarehouseDateTime, snapAlertWindowEndMs } from "@maple/query-engine" +import { MapleCloudEventSchema } from "@maple/eventing-core" import { AlertComparator as AlertComparatorSchema, + type AlertComparator, AlertDeliveryError, type AlertDeliveryFailure, AlertDestinationDecryptionError, @@ -27,7 +38,6 @@ import { AlertSignalType as AlertSignalTypeSchema, AlertValidationError, AlertNotificationTemplate, - type AlertComparator, type AlertDestinationType, type AlertEventType as AlertEventTypeValue, type AlertRuleUpsertRequest, @@ -185,6 +195,7 @@ type DatabaseExecutor = DatabaseClient | DatabaseTransaction /* -------------------------------------------------------------------------- */ const StoredDeliveryPayloadSchema = Schema.Struct({ + event: Schema.optionalKey(MapleCloudEventSchema), eventType: Schema.optionalKey(Schema.String), incidentId: Schema.optionalKey(Schema.NullOr(Schema.String)), incidentStatus: Schema.optionalKey(Schema.String), @@ -250,25 +261,7 @@ const MAX_PREVIEW_BUCKETS = 200 /** Preserve each org's oldest-first order while preventing one org from monopolizing a tick. */ export const interleaveAlertRulesByOrg = ( rows: ReadonlyArray, -): ReadonlyArray => { - const queues = new Map() - for (const row of rows) { - const queue = queues.get(row.orgId) - if (queue) queue.push(row) - else queues.set(row.orgId, [row]) - } - - const fair: T[] = [] - let index = 0 - while (fair.length < rows.length) { - for (const queue of queues.values()) { - const row = queue[index] - if (row !== undefined) fair.push(row) - } - index += 1 - } - return fair -} +): ReadonlyArray => interleaveAlertRulesByTenant(rows, (row) => row.orgId) // Tinybird DateTime64(3) wire format for alert_checks ingest: // "YYYY-MM-DD HH:MM:SS.SSS" (UTC, no timezone). @@ -278,27 +271,6 @@ const toIngestDateTime64 = (epochMs: number) => { return `${d.getUTCFullYear()}-${pad(d.getUTCMonth() + 1)}-${pad(d.getUTCDate())} ${pad(d.getUTCHours())}:${pad(d.getUTCMinutes())}:${pad(d.getUTCSeconds())}.${pad(d.getUTCMilliseconds(), 3)}` } -const compareThreshold = ( - value: number, - comparator: AlertComparator, - threshold: number, - thresholdUpper: number | null = null, -): boolean => - Match.value(comparator).pipe( - Match.when("gt", () => value > threshold), - Match.when("gte", () => value >= threshold), - Match.when("lt", () => value < threshold), - Match.when("lte", () => value <= threshold), - Match.when("eq", () => value === threshold), - Match.when("neq", () => value !== threshold), - Match.when("between", () => thresholdUpper != null && value >= threshold && value <= thresholdUpper), - Match.when( - "not_between", - () => thresholdUpper != null && (value < threshold || value > thresholdUpper), - ), - Match.exhaustive, - ) - const makeDeliveryError = (message: string, destinationType?: AlertDestinationType, cause?: unknown) => new AlertDeliveryError({ message, @@ -530,79 +502,26 @@ export class AlertsService extends Context.Service, reasonOverride?: string, - ): EvaluatedRule => { - const noDataBehavior = rule.compiledPlan.noDataBehavior - // Sample-weighted counts arrive fractional from the warehouse - // (`sum(SampleRate)`), and this flows into `last_sample_count`, an - // `integer` column — an unrounded value fails the insert outright. - const sampleCount = Math.round(obs.sampleCount) - const value = obs.hasData ? obs.value : noDataBehavior === "zero" ? 0 : null - - if (!obs.hasData && noDataBehavior === "skip") { - return { - status: "skipped", - value: null, - sampleCount, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, - comparator: rule.comparator, - reason: "No data in the selected window", - // Inert: `skipped` never resolves an incident, so this branch - // short-circuits before any status is derived from a synthesized value. - derivedFromNoData: false, - } - } - - if (sampleCount < rule.minimumSampleCount) { - return { - status: "skipped", - value, - sampleCount, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, + ): EvaluatedRule => + evaluateAlertObservation( + { comparator: rule.comparator, - reason: `Sample count ${sampleCount} is below minimum ${rule.minimumSampleCount}`, - derivedFromNoData: false, - } - } - - if (value == null) { - return { - status: "skipped", - value: null, - sampleCount, threshold: rule.threshold, thresholdUpper: rule.thresholdUpper, - comparator: rule.comparator, - reason: "Alert evaluation did not return a scalar value", - derivedFromNoData: false, - } - } - - return { - status: compareThreshold(value, rule.comparator, rule.threshold, rule.thresholdUpper) - ? "breached" - : "healthy", - value, - sampleCount, - threshold: rule.threshold, - thresholdUpper: rule.thresholdUpper, - comparator: rule.comparator, - reason: - reasonOverride ?? + minimumSampleCount: rule.minimumSampleCount, + noDataBehavior: rule.compiledPlan.noDataBehavior, + }, + obs, + reasonOverride ?? `${rule.signalType} ${formatComparator(rule.comparator, rule.threshold, rule.thresholdUpper)}`, - // Only reachable with `noDataBehavior: "zero"` — the "skip" branch - // returned above. The comparison ran against a fabricated 0. - derivedFromNoData: !obs.hasData, - } - } + ) const buildDeliveryKey = ( incidentId: string, destinationId: string, eventType: AlertEventTypeValue, scheduledAt: number, - ) => [incidentId, destinationId, eventType, scheduledAt].join(":") + ) => makeAlertDeliveryKey(incidentId, destinationId, eventType, scheduledAt) const insertDeliveryEventRecord = ( db: DatabaseExecutor, @@ -851,28 +770,31 @@ export class AlertsService extends Context.Service row.payloadJson as Record), Effect.orElseSucceed(() => ({})), - )) as Record + ) yield* insertDeliveryEvent( row.orgId, row.incidentId, @@ -1764,7 +1689,7 @@ export class AlertsService extends Context.Service= against *Required, so saturating keeps open/resolve behavior - // identical while letting steady-state ticks skip the state upsert above. - const consecutiveBreaches = - evaluation.status === "breached" - ? Math.min( - (state?.consecutiveBreaches ?? 0) + 1, - normalized.consecutiveBreachesRequired, - ) - : 0 - const consecutiveHealthy = - evaluation.status === "healthy" - ? Math.min( - (state?.consecutiveHealthy ?? 0) + 1, - normalized.consecutiveHealthyRequired, - ) - : 0 - + let lifecycle = planAlertLifecycle(lifecycleInput) + // Persist the counter/state decision before follow-up adapter work, as + // before extraction. A failed flap-history or liveness query must not + // discard an evaluation that already completed successfully. yield* upsertState({ - consecutiveBreaches, - consecutiveHealthy, + consecutiveBreaches: lifecycle.state.consecutiveBreaches, + consecutiveHealthy: lifecycle.state.consecutiveHealthy, lastStatus: evaluation.status, lastValue: evaluation.value, lastSampleCount: evaluation.sampleCount, }) - if ( - evaluation.status === "breached" && - openIncident == null && - consecutiveBreaches >= normalized.consecutiveBreachesRequired - ) { - // Flap suppression: a metric oscillating around the threshold opens - // a fresh incident per flap, which would email an identical trigger - // notification every few minutes. If the previous incident for this - // (rule, group) was notified within the renotify interval, open the - // incident but skip the trigger notification and carry the prior - // lastNotifiedAt forward — the renotify gate then enforces one - // email per interval while the flapping persists. + // Ask the persistence adapter for flap history only when the pure core + // has decided that a new incident is otherwise ready to open. + if (lifecycle.transition === "opened") { const priorNotified = (yield* dbExecute((db) => db @@ -1944,9 +1857,46 @@ export class AlertsService extends Context.Service db.insert(alertIncidents).values(incident)) - if (flapSuppressedAt != null) { + if (lifecycle.notificationSuppression === "flapping") { yield* Effect.logInfo("Skipping trigger notification for flapping incident").pipe( Effect.annotateLogs({ ruleId: row.id, incidentId, groupKey, - priorNotifiedAt: flapSuppressedAt.toISOString(), + priorNotifiedAt: inheritedNotificationAt?.toISOString(), }), ) - } else { + } else if (lifecycle.eventType === "trigger") { yield* queueIncidentNotifications( row.orgId, normalized, incident, evaluation, - "trigger", + lifecycle.eventType, timestamp, pushBudget, ) } return { - transition: "opened" as const, + transition: lifecycle.transition, incidentId, openedIncidentId: incidentId, - consecutiveBreaches, - consecutiveHealthy, + consecutiveBreaches: lifecycle.state.consecutiveBreaches, + consecutiveHealthy: lifecycle.state.consecutiveHealthy, } } - if (evaluation.status === "breached" && openIncident != null) { + if (lifecycle.transition === "continued" && openIncident != null) { const refreshedIncident = { ...openIncident, lastTriggeredAt: new Date(timestamp), @@ -2016,19 +1963,6 @@ export class AlertsService extends Context.Service db .update(alertIncidents) @@ -2038,73 +1972,33 @@ export class AlertsService extends Context.Service= normalized.consecutiveHealthyRequired - ) { - // A "healthy" synthesized from an empty window is a statement - // about missing data, not about a recovered system: with - // `noDataBehavior: "zero"` a total ingest outage compares as 0 < - // threshold and would resolve every incident it touches, paging - // out a wave of false all-clears. Believe it only once telemetry - // is provably still arriving. - if (evaluation.derivedFromNoData) { - const liveness = yield* telemetryStillFlowing( - row.orgId, - normalized, - openIncident.firstTriggeredAt.getTime(), - timestamp, - ) - if (!liveness.dataFlowing) { - yield* Effect.logWarning( - "Holding incident open: healthy evaluation came from missing telemetry", - ).pipe( - Effect.annotateLogs({ - orgId: row.orgId, - ruleId: row.id, - incidentId: openIncident.id, - groupKey, - livenessReason: liveness.reason, - observedCount: liveness.observedCount, - baselineCount: liveness.baselineCount, - }), - ) - return { - transition: "none" as const, - incidentId: carriedIncidentId, - openedIncidentId: null, - consecutiveBreaches, - consecutiveHealthy, - } - } - } - + if (lifecycle.transition === "resolved" && openIncident != null) { const resolvedIncident = { ...openIncident, status: "resolved" as const, @@ -2114,7 +2008,6 @@ export class AlertsService extends Context.Service db .update(alertIncidents) @@ -2128,14 +2021,7 @@ export class AlertsService extends Context.Service) => @@ -22,8 +38,35 @@ const provideQueue = (environment: Record) => ) describe("PlanetScaleWebhookQueue", () => { + it("decodes tenant, source, connection, type, schema, and time as one relational boundary", () => { + const contradictions: readonly PlanetScaleWebhookJob[] = [ + { ...job, event: { ...job.event, tenantid: Schema.decodeUnknownSync(OrgId)("org_2") } }, + { ...job, event: { ...job.event, source: "urn:maple:planetscale:connection_2" } }, + { + ...job, + event: { + ...job.event, + data: { + connectionId: "connection_2", + event: payload.event, + organization: payload.organization, + database: payload.database, + resource: payload.resource, + }, + }, + }, + { ...job, event: { ...job.event, type: "dev.maple.unsupported.v1" } }, + { ...job, event: { ...job.event, dataschema: "urn:maple:event-schema:unsupported:v1" } }, + { ...job, event: { ...job.event, time: "2026-99-99T00:00:00Z" } }, + ] + for (const contradiction of contradictions) + assert.throws(() => Schema.decodeUnknownSync(PlanetScaleWebhookQueueMessage)(contradiction)) + assert.deepStrictEqual(Schema.decodeUnknownSync(PlanetScaleWebhookQueueMessage)(job), job) + }) + it.effect("schema-encodes the internal job onto the dedicated binding", () => { const sent: unknown[] = [] + assert.isBelow(planetScaleWebhookQueueJobBytes(job), MAX_PLANETSCALE_WEBHOOK_QUEUE_BYTES) return Effect.gen(function* () { const queue = yield* PlanetScaleWebhookQueue yield* queue.send(job) @@ -66,4 +109,39 @@ describe("PlanetScaleWebhookQueue", () => { }), ) }) + + it.effect("accepts the serialized cap and rejects one byte above it", () => { + let attempts = 0 + const withPayload = (payload: string): PlanetScaleWebhookJob => ({ + ...job, + event: { + ...job.event, + data: { payload }, + }, + }) + const empty = withPayload("") + const envelopeBytes = planetScaleWebhookQueueJobBytes(empty) + const atCap = withPayload("x".repeat(MAX_PLANETSCALE_WEBHOOK_QUEUE_BYTES - envelopeBytes)) + const oversized = withPayload("x".repeat(MAX_PLANETSCALE_WEBHOOK_QUEUE_BYTES - envelopeBytes + 1)) + assert.strictEqual(planetScaleWebhookQueueJobBytes(atCap), MAX_PLANETSCALE_WEBHOOK_QUEUE_BYTES) + assert.strictEqual( + planetScaleWebhookQueueJobBytes(oversized), + MAX_PLANETSCALE_WEBHOOK_QUEUE_BYTES + 1, + ) + return Effect.gen(function* () { + const queue = yield* PlanetScaleWebhookQueue + yield* queue.send(atCap) + const error = yield* queue.send(oversized).pipe(Effect.flip) + assert.match(error.message, /queue job exceeds/) + assert.strictEqual(attempts, 1) + }).pipe( + provideQueue({ + PLANETSCALE_WEBHOOK_QUEUE: { + send: async () => { + attempts += 1 + }, + }, + }), + ) + }) }) diff --git a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts index 89093f9e5..f9d6d33ce 100644 --- a/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts +++ b/apps/api/src/services/integrations/planetscale/PlanetScaleWebhookQueue.ts @@ -1,20 +1,63 @@ import type { Queue } from "@cloudflare/workers-types" import { OrgId } from "@maple/domain/http" import { WorkerEnvironment } from "@maple/effect-cloudflare/worker-environment" +import { MapleCloudEventSchema } from "@maple/eventing-core" import { Context, Effect, Layer, Schema } from "effect" -import { PlanetScaleWebhookPayload } from "./webhook-events" +import { PlanetScaleWebhookPayload, planetScaleWebhookPayloadFromEvent } from "./webhook-events" const QUEUE_BINDING = "PLANETSCALE_WEBHOOK_QUEUE" -export const PlanetScaleWebhookJob = Schema.Struct({ +const PlanetScaleWebhookJobBase = { kind: Schema.Literal("planetscale-webhook"), orgId: OrgId, connectionId: Schema.String, - payload: PlanetScaleWebhookPayload, receivedAt: Schema.Number, +} as const + +/** Exact queue body emitted before the typed CloudEvent migration. */ +export const LegacyPlanetScaleWebhookJob = Schema.Struct({ + ...PlanetScaleWebhookJobBase, + payload: PlanetScaleWebhookPayload, +}) + +/** Current producer contract. New writers queue only the canonical event. */ +export const PlanetScaleWebhookJob = Schema.Struct({ + ...PlanetScaleWebhookJobBase, + event: MapleCloudEventSchema, }) export type PlanetScaleWebhookJob = Schema.Schema.Type +/** Consumer contract kept backward-compatible during rolling deployments. */ +const PlanetScaleWebhookQueueMessageBase = Schema.Union([ + PlanetScaleWebhookJob, + Schema.Struct({ + ...PlanetScaleWebhookJobBase, + payload: PlanetScaleWebhookPayload, + event: MapleCloudEventSchema, + }), + LegacyPlanetScaleWebhookJob, +]) +export const PlanetScaleWebhookQueueMessage = PlanetScaleWebhookQueueMessageBase.pipe( + Schema.check( + Schema.makeFilter( + (job) => { + if (!("event" in job)) return true + try { + planetScaleWebhookPayloadFromEvent(job.event, job.orgId, job.connectionId) + return true + } catch { + return false + } + }, + { expected: "a supported, tenant-bound PlanetScale webhook event" }, + ), + ), +) +export type PlanetScaleWebhookQueueMessage = Schema.Schema.Type + +/** Cloudflare's 128 KB body limit includes the complete serialized queue job. */ +export const MAX_PLANETSCALE_WEBHOOK_QUEUE_BYTES = 120 * 1024 + export class PlanetScaleWebhookQueueError extends Schema.TaggedError()( "@maple/api/services/planetscale/PlanetScaleWebhookQueueError", { @@ -29,6 +72,9 @@ export interface PlanetScaleWebhookQueueApi { const encodeJob = Schema.encodeSync(PlanetScaleWebhookJob) +export const planetScaleWebhookQueueJobBytes = (job: PlanetScaleWebhookJob): number => + new TextEncoder().encode(JSON.stringify(encodeJob(job))).byteLength + export class PlanetScaleWebhookQueue extends Context.Service< PlanetScaleWebhookQueue, PlanetScaleWebhookQueueApi @@ -47,8 +93,14 @@ export class PlanetScaleWebhookQueue extends Context.Service< message: `Missing queue binding: ${QUEUE_BINDING}`, }) } + const encoded = encodeJob(job) + const encodedBytes = planetScaleWebhookQueueJobBytes(job) + if (encodedBytes > MAX_PLANETSCALE_WEBHOOK_QUEUE_BYTES) + return yield* new PlanetScaleWebhookQueueError({ + message: `PlanetScale queue job exceeds ${MAX_PLANETSCALE_WEBHOOK_QUEUE_BYTES} bytes`, + }) yield* Effect.tryPromise({ - try: () => queue.send(encodeJob(job)), + try: () => queue.send(encoded), catch: (cause) => new PlanetScaleWebhookQueueError({ message: cause instanceof Error ? cause.message : "PlanetScale queue send failed", diff --git a/apps/api/src/services/integrations/planetscale/webhook-events.test.ts b/apps/api/src/services/integrations/planetscale/webhook-events.test.ts index eb4a0f1d7..a4f8b6af5 100644 --- a/apps/api/src/services/integrations/planetscale/webhook-events.test.ts +++ b/apps/api/src/services/integrations/planetscale/webhook-events.test.ts @@ -10,6 +10,7 @@ import { deployRequestNumber, insertPlanetScaleEvent, planetScaleIssueFingerprint, + projectPlanetScaleWebhookEvent, truncateToSecond, upsertPlanetScaleIssue, verifyPlanetScaleSignature, @@ -47,6 +48,58 @@ describe("verifyPlanetScaleSignature", () => { }) describe("classifyPlanetScaleEvent", () => { + it("normalizes queued webhooks into deterministic common CloudEvents", () => { + const payload = Schema.decodeUnknownSync(PlanetScaleWebhookPayload)(JSON.parse(OOM_PAYLOAD)) + const input = { + orgId: "org_events", + connectionId: "connection-1", + payload, + receivedAt: 1_698_252_880_000, + } + const event = projectPlanetScaleWebhookEvent(input) + assert.deepStrictEqual(event, projectPlanetScaleWebhookEvent(input)) + assert.strictEqual(event.type, "dev.maple.planetscale.webhook.received.v1") + assert.strictEqual(event.tenantid, "org_events") + assert.strictEqual(event.subject, "planetscale-databases/main-db") + assert.strictEqual((event.data as { readonly event: string }).event, "branch.out_of_memory") + assert.throws( + () => projectPlanetScaleWebhookEvent({ ...input, receivedAt: Number.MAX_SAFE_INTEGER }), + /outside the supported date range/, + ) + }) + + it("keeps source-timestamp retries byte-identical and rejects missing timestamps", () => { + const timestamped = Schema.decodeUnknownSync(PlanetScaleWebhookPayload)(JSON.parse(OOM_PAYLOAD)) + const first = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: timestamped, + receivedAt: 1_698_252_880_000, + }) + const redelivery = projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: timestamped, + receivedAt: 1_698_252_990_000, + }) + assert.deepStrictEqual(first, redelivery) + + const withoutTimestamp = Schema.decodeUnknownSync(PlanetScaleWebhookPayload)({ + event: "branch.ready", + database: "main-db", + }) + assert.throws( + () => + projectPlanetScaleWebhookEvent({ + orgId: "org_events", + connectionId: "connection-1", + payload: withoutTimestamp, + receivedAt: 1_698_252_880_000, + }), + /requires a positive source timestamp/, + ) + }) + it("maps health events to issues and lifecycle events to timeline rows", () => { assert.strictEqual(classifyPlanetScaleEvent("branch.out_of_memory").action, "issue") assert.strictEqual(classifyPlanetScaleEvent("branch.anomaly").action, "issue") @@ -210,7 +263,7 @@ describe("upsertPlanetScaleIssue", () => { description: "Branch main of main-db was restarted after running out of memory.", } - const first = yield* upsertPlanetScaleIssue({ ...base, timestamp: 1_000 }) + const first = yield* upsertPlanetScaleIssue({ ...base, eventId: "event-1", timestamp: 1_000 }) assert.strictEqual(first.action, "created") assert.isNotNull(first.issueId) @@ -228,7 +281,7 @@ describe("upsertPlanetScaleIssue", () => { ) // Repeat firing dedupes into the same issue and bumps the count. - const second = yield* upsertPlanetScaleIssue({ ...base, timestamp: 2_000 }) + const second = yield* upsertPlanetScaleIssue({ ...base, eventId: "event-2", timestamp: 2_000 }) assert.strictEqual(second.action, "refreshed") assert.strictEqual(second.issueId, first.issueId) @@ -238,7 +291,7 @@ describe("upsertPlanetScaleIssue", () => { first.issueId, ]), ) - const third = yield* upsertPlanetScaleIssue({ ...base, timestamp: 3_000 }) + const third = yield* upsertPlanetScaleIssue({ ...base, eventId: "event-3", timestamp: 3_000 }) assert.strictEqual(third.action, "reopened") const reopened = yield* Effect.promise(() => @@ -253,6 +306,95 @@ describe("upsertPlanetScaleIssue", () => { }).pipe(Effect.provide(testDb.layer)) }) + it.effect("counts concurrent distinct events against an initially absent issue", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const payload = yield* decodePlanetScaleWebhookPayload(OOM_PAYLOAD) + const base = { + orgId: asOrgId("org_1"), + payload, + severity: "high" as const, + title: "PlanetScale branch out of memory", + description: "Branch main of main-db was restarted after running out of memory.", + } + const results = yield* Effect.all( + [ + upsertPlanetScaleIssue({ ...base, eventId: "event-a", timestamp: 1_000 }), + upsertPlanetScaleIssue({ ...base, eventId: "event-b", timestamp: 2_000 }), + ], + { concurrency: "unbounded" }, + ) + assert.deepStrictEqual(results.map(({ action }) => action).sort(), ["created", "refreshed"]) + assert.strictEqual(results[0].issueId, results[1].issueId) + + const aggregate = yield* Effect.promise(() => + queryFirstRow<{ occurrence_count: number; receipts: number; created_events: number }>( + testDb, + `SELECT i.occurrence_count, + (SELECT count(*)::int FROM planetscale_issue_receipts) AS receipts, + (SELECT count(*)::int FROM error_issue_events WHERE issue_id = i.id AND type = 'created') AS created_events + FROM error_issues i`, + ), + ) + assert.strictEqual(aggregate?.occurrence_count, 2) + assert.strictEqual(aggregate?.receipts, 2) + assert.strictEqual(aggregate?.created_events, 1) + }).pipe(Effect.provide(testDb.layer)) + }) + + it.effect("serializes concurrent distinct events when reopening a resolved issue", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const payload = yield* decodePlanetScaleWebhookPayload(OOM_PAYLOAD) + const base = { + orgId: asOrgId("org_1"), + payload, + severity: "high" as const, + title: "PlanetScale branch out of memory", + description: "Branch main of main-db was restarted after running out of memory.", + } + const initial = yield* upsertPlanetScaleIssue({ + ...base, + eventId: "event-initial", + timestamp: 1_000, + }) + yield* Effect.promise(() => + executeSql(testDb, "UPDATE error_issues SET workflow_state = 'done' WHERE id = $1", [ + initial.issueId, + ]), + ) + + const results = yield* Effect.all( + [ + upsertPlanetScaleIssue({ ...base, eventId: "event-a", timestamp: 2_000 }), + upsertPlanetScaleIssue({ ...base, eventId: "event-b", timestamp: 3_000 }), + ], + { concurrency: "unbounded" }, + ) + assert.deepStrictEqual(results.map(({ action }) => action).sort(), ["refreshed", "reopened"]) + + const aggregate = yield* Effect.promise(() => + queryFirstRow<{ + workflow_state: string + occurrence_count: number + state_changes: number + regressions: number + }>( + testDb, + `SELECT i.workflow_state, i.occurrence_count, + (SELECT count(*)::int FROM error_issue_events WHERE issue_id = i.id AND type = 'state_change') AS state_changes, + (SELECT count(*)::int FROM error_issue_events WHERE issue_id = i.id AND type = 'regression') AS regressions + FROM error_issues i WHERE i.id = $1`, + [initial.issueId], + ), + ) + assert.strictEqual(aggregate?.workflow_state, "triage") + assert.strictEqual(aggregate?.occurrence_count, 3) + assert.strictEqual(aggregate?.state_changes, 1) + assert.strictEqual(aggregate?.regressions, 1) + }).pipe(Effect.provide(testDb.layer)) + }) + it.effect("leaves a wontfix issue with an active snooze entirely alone", () => { const testDb = createTestDb(trackedDbs) return Effect.gen(function* () { @@ -266,7 +408,7 @@ describe("upsertPlanetScaleIssue", () => { description: "Branch main of main-db was restarted after running out of memory.", } - const first = yield* upsertPlanetScaleIssue({ ...base, timestamp: 1_000 }) + const first = yield* upsertPlanetScaleIssue({ ...base, eventId: "event-1", timestamp: 1_000 }) assert.strictEqual(first.action, "created") // Operator marks it wontfix with a snooze that has not yet expired. @@ -278,7 +420,7 @@ describe("upsertPlanetScaleIssue", () => { ), ) - const second = yield* upsertPlanetScaleIssue({ ...base, timestamp: 5_000 }) + const second = yield* upsertPlanetScaleIssue({ ...base, eventId: "event-2", timestamp: 5_000 }) assert.strictEqual(second.action, "skipped") assert.strictEqual(second.issueId, first.issueId) @@ -326,7 +468,7 @@ describe("upsertPlanetScaleIssue", () => { description: "Branch main of main-db was restarted after running out of memory.", } - const first = yield* upsertPlanetScaleIssue({ ...base, timestamp: 1_000 }) + const first = yield* upsertPlanetScaleIssue({ ...base, eventId: "event-1", timestamp: 1_000 }) assert.strictEqual(first.action, "created") // "Won't fix" with snooze_until NULL means "stop resurfacing this" — @@ -340,7 +482,11 @@ describe("upsertPlanetScaleIssue", () => { ) const farFuture = Date.UTC(2099, 0, 1) - const second = yield* upsertPlanetScaleIssue({ ...base, timestamp: farFuture }) + const second = yield* upsertPlanetScaleIssue({ + ...base, + eventId: "event-2", + timestamp: farFuture, + }) assert.strictEqual(second.action, "skipped") assert.strictEqual(second.issueId, first.issueId) @@ -369,7 +515,7 @@ describe("upsertPlanetScaleIssue", () => { description: "Branch main of main-db was restarted after running out of memory.", } - const first = yield* upsertPlanetScaleIssue({ ...base, timestamp: 1_000 }) + const first = yield* upsertPlanetScaleIssue({ ...base, eventId: "event-1", timestamp: 1_000 }) assert.strictEqual(first.action, "created") // Snooze deadline is before the next firing's timestamp → expired. @@ -381,7 +527,7 @@ describe("upsertPlanetScaleIssue", () => { ), ) - const second = yield* upsertPlanetScaleIssue({ ...base, timestamp: 10_000 }) + const second = yield* upsertPlanetScaleIssue({ ...base, eventId: "event-2", timestamp: 10_000 }) assert.strictEqual(second.action, "reopened") assert.strictEqual(second.issueId, first.issueId) @@ -428,6 +574,7 @@ describe("upsertPlanetScaleIssue", () => { const payload = yield* decodePlanetScaleWebhookPayload(OOM_PAYLOAD) const input = { orgId: asOrgId("org_1"), + eventId: "event-1", payload, severity: "high" as const, title: "PlanetScale branch out of memory", diff --git a/apps/api/src/services/integrations/planetscale/webhook-events.ts b/apps/api/src/services/integrations/planetscale/webhook-events.ts index 201f9dbdb..470999887 100644 --- a/apps/api/src/services/integrations/planetscale/webhook-events.ts +++ b/apps/api/src/services/integrations/planetscale/webhook-events.ts @@ -1,10 +1,23 @@ -import { createHmac, randomUUID, timingSafeEqual } from "node:crypto" +import { createHash, createHmac, randomUUID, timingSafeEqual } from "node:crypto" +import { + canonicalJson, + CompiledProjectionRegistry, + defineSignalFields, + isJsonValue, + ProjectorRegistry, + SignalSourceRegistry, + type JsonValue, + type MapleCloudEvent, + type SignalProjector, + type SignalSourceAdapter, +} from "@maple/eventing-core" import type { IssueSeverity, OrgId, WorkflowState } from "@maple/domain/http" import { ActorId, ErrorIssueEventId, ErrorIssueId } from "@maple/domain/primitives" import { actors, errorIssues, errorIssueEvents, + planetscaleIssueReceipts, planetscaleDatabases, planetscaleEvents, type ErrorIssueRow, @@ -49,6 +62,205 @@ export const decodePlanetScaleWebhookPayload = Schema.decodeUnknownEffect( Schema.fromJsonString(PlanetScaleWebhookPayload), ) +export interface PlanetScaleWebhookEventInput { + readonly orgId: string + readonly connectionId: string + readonly payload: PlanetScaleWebhookPayload + readonly receivedAt: number +} + +interface PlanetScaleWebhookAdapterInput { + readonly connectionId: string + readonly payload: PlanetScaleWebhookPayload +} + +interface PlanetScaleWebhookAdapterContext { + readonly tenantId: string + readonly acceptedAt: string +} + +const validDate = (epochMs: number, label: string): Date => { + if (!Number.isSafeInteger(epochMs) || epochMs < 0) + throw new Error(`${label} must be a non-negative epoch millisecond`) + const date = new Date(epochMs) + if (Number.isNaN(date.getTime())) throw new Error(`${label} is outside the supported date range`) + return date +} + +export const planetScaleWebhookTimestampMillis = (payload: PlanetScaleWebhookPayload): number | null => { + if (payload.timestamp == null || !Number.isFinite(payload.timestamp) || payload.timestamp <= 0) + return null + const epochMs = Math.trunc(payload.timestamp * 1_000) + return Number.isSafeInteger(epochMs) ? epochMs : null +} + +export const PLANETSCALE_WEBHOOK_ADAPTER: SignalSourceAdapter< + PlanetScaleWebhookAdapterInput, + PlanetScaleWebhookAdapterContext +> = { + definition: { + sourceKind: "planetscale.webhook", + fields: [ + { + field: { namespace: "signal", key: "event.name", type: "string" }, + operators: ["exists", "eq", "neq", "contains", "in"], + sensitivity: "public", + replay: "unavailable", + }, + ], + }, + normalize: ({ connectionId, payload }, context) => { + const observedAtDate = new Date(context.acceptedAt) + if (Number.isNaN(observedAtDate.getTime())) + throw new Error("PlanetScale receipt time is outside the supported date range") + if (!isJsonValue(payload)) throw new Error("PlanetScale webhook payload must be finite JSON") + const payloadJson = payload + const occurredAtMs = planetScaleWebhookTimestampMillis(payload) + if (occurredAtMs === null) throw new Error("PlanetScale webhook requires a positive source timestamp") + const occurredAt = validDate(occurredAtMs, "PlanetScale event timestamp").toISOString() + const occurrenceId = `derived:sha256:${createHash("sha256") + .update(connectionId) + .update("\0") + .update(canonicalJson(payloadJson)) + .update("\0") + .update(occurredAt) + .digest("hex")}` + return [ + { + sourceKind: "planetscale.webhook", + source: `urn:maple:planetscale:${connectionId}`, + tenantId: context.tenantId, + occurrenceId, + identityQuality: "derived", + occurredAt, + observedAt: observedAtDate.toISOString(), + subject: + payload.database == null + ? `planetscale-connections/${connectionId}` + : `planetscale-databases/${payload.database}`, + fields: defineSignalFields([ + { + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: payload.event }, + }, + ]), + data: { + connectionId, + event: payload.event, + organization: payload.organization ?? null, + database: payload.database ?? null, + resource: (payload.resource ?? null) as JsonValue, + }, + }, + ] + }, +} + +const PlanetScaleWebhookEventDataSchema = Schema.Struct({ + connectionId: Schema.String, + event: Schema.String, + organization: Schema.NullOr(Schema.String), + database: Schema.NullOr(Schema.String), + resource: Schema.NullOr(Schema.Record(Schema.String, Schema.Unknown)), +}) + +const decodePlanetScaleWebhookEventData = Schema.decodeUnknownSync(PlanetScaleWebhookEventDataSchema) + +const decodePlanetScaleWebhookProjectorOutput = (value: unknown): JsonValue => { + const decoded = decodePlanetScaleWebhookEventData(value) + if (!isJsonValue(decoded)) throw new Error("PlanetScale projector output must be finite JSON") + return decoded +} + +const PLANETSCALE_WEBHOOK_PROJECTOR: SignalProjector> = { + id: "planetscale.webhook", + version: 1, + sourceKinds: ["planetscale.webhook"], + outputType: "dev.maple.planetscale.webhook.received.v1", + dataSchema: "urn:maple:event-schema:planetscale-webhook:v1", + decodeOutput: decodePlanetScaleWebhookProjectorOutput, + decodeConfig: (value) => { + if ( + typeof value !== "object" || + value === null || + Array.isArray(value) || + Object.keys(value).length > 0 + ) + throw new Error("PlanetScale webhook projector config must be empty") + return {} + }, + project: (signal) => ({ data: signal.data as JsonValue }), +} + +const PLANETSCALE_SOURCES = new SignalSourceRegistry().register(PLANETSCALE_WEBHOOK_ADAPTER.definition) +const PLANETSCALE_PROJECTORS = new ProjectorRegistry().register(PLANETSCALE_WEBHOOK_PROJECTOR) + +/** Normalize and project one verified, durably queued PlanetScale webhook through the common layer. */ +export const projectPlanetScaleWebhookEvent = (input: PlanetScaleWebhookEventInput): MapleCloudEvent => { + const observedAt = validDate(input.receivedAt, "PlanetScale receipt time").toISOString() + const [signal] = PLANETSCALE_WEBHOOK_ADAPTER.normalize( + { connectionId: input.connectionId, payload: input.payload }, + { tenantId: input.orgId, acceptedAt: observedAt }, + ) + if (!signal) throw new Error("PlanetScale webhook adapter produced no signal") + const registry = CompiledProjectionRegistry.compile( + [ + { + id: "planetscale-webhook", + revision: 1, + enabled: true, + tenantId: input.orgId, + sourceKind: "planetscale.webhook", + selector: { + op: "exists", + field: { namespace: "signal", key: "event.name", type: "string" }, + }, + projector: { id: "planetscale.webhook", version: 1, config: {} }, + activeFrom: observedAt, + }, + ], + PLANETSCALE_SOURCES, + PLANETSCALE_PROJECTORS, + ) + const result = registry.evaluate(signal, observedAt) + if (result.failures.length > 0) throw new Error(result.failures[0]!.message) + if (result.events.length !== 1) throw new Error("PlanetScale webhook projection produced no event") + return result.events[0]! +} + +export const planetScaleWebhookPayloadFromEvent = ( + event: Pick & { + readonly data: unknown + }, + orgId: string, + connectionId: string, +): PlanetScaleWebhookPayload => { + if ( + event.type !== "dev.maple.planetscale.webhook.received.v1" || + event.dataschema !== "urn:maple:event-schema:planetscale-webhook:v1" + ) + throw new Error("queued PlanetScale event contract is invalid") + if (event.tenantid !== orgId) throw new Error("queued PlanetScale event tenant identity is contradictory") + if (event.source !== `urn:maple:planetscale:${connectionId}`) + throw new Error("queued PlanetScale event source identity is contradictory") + const data = decodePlanetScaleWebhookEventData(event.data) + if (data.connectionId !== connectionId) + throw new Error("queued PlanetScale event connection identity is contradictory") + const timestamp = Date.parse(event.time) + if (!Number.isSafeInteger(timestamp) || timestamp <= 0) + throw new Error("queued PlanetScale event timestamp is invalid") + return { + timestamp: timestamp / 1_000, + event: data.event, + organization: data.organization, + database: data.database, + resource: data.resource, + } +} + +// --------------------------------------------------------------------------- +// Classification +// --------------------------------------------------------------------------- /** Where an event belongs on the timeline. Mirrored by the web vocabulary table. */ export type PlanetScaleEventCategory = "deploy_request" | "branch" | "database" | "cluster" | "keyspace" @@ -315,6 +527,7 @@ export const planetScaleIssueFingerprint = (database: string, event: string) => export interface UpsertPlanetScaleIssueInput { readonly orgId: OrgId + readonly eventId: string readonly payload: PlanetScaleWebhookPayload readonly severity: IssueSeverity readonly title: string @@ -330,7 +543,8 @@ export interface UpsertPlanetScaleIssueResult { /** * Create-or-refresh the triage issue backing a PlanetScale health event. * Database failures stay typed so the durable queue consumer can retry the - * delivery. The fingerprint makes successful redelivery idempotent. + * delivery. The event receipt makes redelivery idempotent; the fingerprint + * groups distinct source occurrences into the same issue. */ export const upsertPlanetScaleIssue: ( input: UpsertPlanetScaleIssueInput, @@ -352,6 +566,39 @@ export const upsertPlanetScaleIssue: ( return yield* database.execute((db) => db.transaction(async (tx) => { + // Distinct source events can share one issue fingerprint and queue batches + // process concurrently. Serialize that aggregate before claiming a receipt + // so every committed receipt corresponds to exactly one applied occurrence. + await tx.execute( + sql`select pg_advisory_xact_lock(hashtext(${input.orgId}), hashtext(${fingerprintHash}))`, + ) + const receipt = await tx + .insert(planetscaleIssueReceipts) + .values({ + orgId: input.orgId, + eventId: input.eventId, + processedAt: new Date(actorTimestamp), + }) + .onConflictDoNothing() + .returning({ eventId: planetscaleIssueReceipts.eventId }) + if (receipt.length === 0) { + const existing = ( + await tx + .select({ id: errorIssues.id }) + .from(errorIssues) + .where( + and( + eq(errorIssues.orgId, input.orgId), + eq(errorIssues.fingerprintHash, fingerprintHash), + ), + ) + .limit(1) + )[0] + if (existing === undefined) + throw new Error("PlanetScale issue receipt exists without its atomic issue mutation") + return { issueId: existing.id, action: "skipped" as const } + } + const ensureActor = async (): Promise => { const selectActor = () => tx @@ -420,13 +667,13 @@ export const upsertPlanetScaleIssue: ( ), ) .limit(1) + .for("update") )[0] if (prior === undefined) { const candidateId = decodeIssueId(randomUUID()) - // READ COMMITTED does not hold the gap between the select above and - // this insert, so a concurrent webhook for the same event can slip in - // and raise `error_issues_org_fp_idx`. + // The transaction-scoped fingerprint lock protects the absent-row gap. + // Keep the conflict handling defensive for writers that predate the lock. const claimed = await tx .insert(errorIssues) .values({ @@ -473,11 +720,11 @@ export const upsertPlanetScaleIssue: ( }) return { issueId: insertedId, action: "created" as const } } - // The concurrent writer won and already emitted `created`; report the - // sighting against their issue rather than duplicating the history. + // A writer outside this lock won. Re-read it under a row lock and apply + // this distinct occurrence instead of committing a receipt-only skip. const winner = ( await tx - .select({ id: errorIssues.id }) + .select() .from(errorIssues) .where( and( @@ -486,54 +733,61 @@ export const upsertPlanetScaleIssue: ( ), ) .limit(1) + .for("update") )[0] - return { issueId: winner?.id ?? candidateId, action: "skipped" as const } + if (winner === undefined) + throw new Error("PlanetScale issue conflict winner was not visible in the transaction") + return await applyExistingIssue(winner) } - const issueId = prior.id - // A wontfix issue with an active or indefinite snooze stays untouched. - const snoozeActive = - prior.workflowState === "wontfix" && - (prior.snoozeUntil == null || prior.snoozeUntil.getTime() > input.timestamp) - if (snoozeActive) return { issueId, action: "skipped" as const } - - await tx - .update(errorIssues) - .set({ - lastSeenAt: new Date(input.timestamp), - occurrenceCount: sql`${errorIssues.occurrenceCount} + 1`, - exceptionMessage: input.description, - sourceRefJson, - updatedAt: new Date(input.timestamp), + return await applyExistingIssue(prior) + + async function applyExistingIssue(prior: ErrorIssueRow): Promise { + const issueId = prior.id + // A wontfix issue with an active or indefinite snooze stays untouched. + const snoozeActive = + prior.workflowState === "wontfix" && + (prior.snoozeUntil == null || prior.snoozeUntil.getTime() > input.timestamp) + if (snoozeActive) return { issueId, action: "skipped" as const } + + await tx + .update(errorIssues) + .set({ + lastSeenAt: new Date(input.timestamp), + occurrenceCount: sql`${errorIssues.occurrenceCount} + 1`, + exceptionMessage: input.description, + sourceRefJson, + updatedAt: new Date(input.timestamp), + }) + .where(and(eq(errorIssues.orgId, input.orgId), eq(errorIssues.id, prior.id))) + + const reopenFrom: WorkflowState | null = + prior.workflowState === "done" || prior.workflowState === "wontfix" + ? prior.workflowState + : null + if (reopenFrom === null) return { issueId, action: "refreshed" as const } + + await tx + .update(errorIssues) + .set({ + workflowState: "triage", + resolvedAt: null, + resolvedByActorId: null, + snoozeUntil: null, + updatedAt: new Date(input.timestamp), + }) + .where(and(eq(errorIssues.orgId, input.orgId), eq(errorIssues.id, prior.id))) + const actorId = await ensureActor() + await recordEvent(issueId, actorId, "state_change", { + fromState: reopenFrom, + toState: "triage", + payload: { viaRegression: true, event: input.payload.event }, }) - .where(and(eq(errorIssues.orgId, input.orgId), eq(errorIssues.id, prior.id))) - - const reopenFrom: WorkflowState | null = - prior.workflowState === "done" || prior.workflowState === "wontfix" - ? prior.workflowState - : null - if (reopenFrom === null) return { issueId, action: "refreshed" as const } - - await tx - .update(errorIssues) - .set({ - workflowState: "triage", - resolvedAt: null, - resolvedByActorId: null, - snoozeUntil: null, - updatedAt: new Date(input.timestamp), + await recordEvent(issueId, actorId, "regression", { + payload: { event: input.payload.event, database: databaseName }, }) - .where(and(eq(errorIssues.orgId, input.orgId), eq(errorIssues.id, prior.id))) - const actorId = await ensureActor() - await recordEvent(issueId, actorId, "state_change", { - fromState: reopenFrom, - toState: "triage", - payload: { viaRegression: true, event: input.payload.event }, - }) - await recordEvent(issueId, actorId, "regression", { - payload: { event: input.payload.event, database: databaseName }, - }) - return { issueId, action: "reopened" as const } + return { issueId, action: "reopened" as const } + } }), ) }) diff --git a/apps/cli/package.json b/apps/cli/package.json index 59409a75f..650cdf5ff 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -14,6 +14,7 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/effect-sdk": "workspace:*", "@maple/domain": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect", "protobufjs": "^8.6.1" diff --git a/apps/cli/src/server/checkpoints.ts b/apps/cli/src/server/checkpoints.ts index a9ce98594..204c388f9 100644 --- a/apps/cli/src/server/checkpoints.ts +++ b/apps/cli/src/server/checkpoints.ts @@ -1,5 +1,5 @@ // BOUNDARY: This module owns unparsed external values and narrows them before domain use. -import { randomUUID } from "node:crypto" +import { createHash, randomUUID } from "node:crypto" import { spawnSync } from "node:child_process" import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from "node:fs" import { cp, lstat, mkdir, readFile, readdir, rm, stat } from "node:fs/promises" @@ -19,6 +19,11 @@ import { syncDirectory, syncTree, } from "./durable-files" +import { + eventingControlSnapshotPath, + LocalEventingControlStore, + type EventingControlSnapshotValidation, +} from "./eventing/control-store" import { CURRENT_LOCAL_SCHEMA, SCHEMA_FINGERPRINT } from "./schema-identity" import schemaSql from "./schema/local-schema.sql" with { type: "text" } import { @@ -29,12 +34,12 @@ import { } from "./store-version" const STATE_FORMAT_VERSION = 1 -const MANIFEST_FORMAT_VERSION = 1 +const MANIFEST_FORMAT_VERSION = 2 const OPERATION_FORMAT_VERSION = 1 const RESTORE_TRANSACTION_FORMAT_VERSION = 1 const RESET_TRANSACTION_FORMAT_VERSION = 1 const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i -const RESETTABLE_CHDB_ENTRIES = new Set(["data", "metadata", "status", "store", "tmp"]) +const RESETTABLE_LIVE_ENTRIES = new Set(["control", "data", "metadata", "status", "store", "tmp"]) export const CHECKPOINT_REOPEN_PROBE_ENV = "MAPLE_INTERNAL_CHECKPOINT_REOPEN_DATA_DIR" const CheckpointUuid = Schema.String.check(Schema.isPattern(CHECKPOINT_ID)) @@ -84,8 +89,7 @@ const CheckpointValidationSchema = Schema.Struct({ export type CheckpointValidation = Schema.Schema.Type -const CheckpointManifestSchema = Schema.Struct({ - formatVersion: Schema.Literal(MANIFEST_FORMAT_VERSION), +const CheckpointManifestFields = { checkpointId: CheckpointId, operationId: CheckpointOperationId, mapleVersion: Schema.String, @@ -96,8 +100,28 @@ const CheckpointManifestSchema = Schema.Struct({ backupRelativePath: Schema.String, backupBytes: NonNegativeInt, validation: CheckpointValidationSchema, +} as const + +const EventingControlSnapshotValidationSchema = Schema.Struct({ + schemaVersion: NonNegativeInt, + projectionRevisions: NonNegativeInt, + projectionFailures: NonNegativeInt, + stagedEvents: NonNegativeInt, + readyEvents: NonNegativeInt, }) +const CheckpointManifestSchema = Schema.Union([ + Schema.Struct({ formatVersion: Schema.Literal(1), ...CheckpointManifestFields }), + Schema.Struct({ + formatVersion: Schema.Literal(MANIFEST_FORMAT_VERSION), + ...CheckpointManifestFields, + controlRelativePath: Schema.String, + controlBytes: NonNegativeInt, + controlSha256: Schema.String.check(Schema.isPattern(/^[0-9a-f]{64}$/)), + controlValidation: EventingControlSnapshotValidationSchema, + }), +]) + export type CheckpointManifest = Schema.Schema.Type const CheckpointStateSchema = Schema.Struct({ @@ -125,7 +149,7 @@ const RestoreTransactionPhase = Schema.Literals([ "markers-committed", ]) const ResetTransactionPhase = Schema.Literals(["intent", "live-cleared", "markers-cleared"]) -const ResetTarget = Schema.Literals(["data", "metadata", "status", "store", "tmp"]) +const ResetTarget = Schema.Literals(["control", "data", "metadata", "status", "store", "tmp"]) const CheckpointOperationSchema = Schema.Struct({ formatVersion: Schema.Literal(OPERATION_FORMAT_VERSION), @@ -342,9 +366,23 @@ const snapshotManifestPath = (dataDir: string, checkpointId: CheckpointId): stri const snapshotBackupDir = (dataDir: string, checkpointId: CheckpointId): string => join(checkpointSnapshotDir(dataDir, checkpointId), "backup") const snapshotBackupRelativePath = (checkpointId: CheckpointId): string => `snapshots/${checkpointId}/backup` +const snapshotControlRelativePath = (checkpointId: CheckpointId): string => + `snapshots/${checkpointId}/control.sqlite` const snapshotBackupSqlPath = (checkpointId: CheckpointId): string => `backups/${snapshotBackupRelativePath(checkpointId)}` +const sha256File = (path: string): string => createHash("sha256").update(readFileSync(path)).digest("hex") + +const controlValidationMatches = ( + left: EventingControlSnapshotValidation, + right: EventingControlSnapshotValidation, +): boolean => + left.schemaVersion === right.schemaVersion && + left.projectionRevisions === right.projectionRevisions && + left.projectionFailures === right.projectionFailures && + left.stagedEvents === right.stagedEvents && + left.readyEvents === right.readyEvents + const assertContained = (root: string, candidate: string, label: string): string => { const absoluteRoot = resolve(root) const absoluteCandidate = resolve(candidate) @@ -682,6 +720,12 @@ export const parseCheckpointManifest = ( if (manifest.backupRelativePath !== snapshotBackupRelativePath(manifest.checkpointId)) { throw new Error("checkpoint backup path does not match its immutable ID") } + if ( + manifest.formatVersion === MANIFEST_FORMAT_VERSION && + manifest.controlRelativePath !== snapshotControlRelativePath(manifest.checkpointId) + ) { + throw new Error("checkpoint control-store path does not match its immutable ID") + } if (manifest.chdbVersion !== CHDB_VERSION) { throw new Error( `checkpoint chDB version mismatch (checkpoint: ${manifest.chdbVersion}; build: ${CHDB_VERSION})`, @@ -828,6 +872,24 @@ const resolveCheckpointById = async ( `checkpoint backup size mismatch (manifest: ${manifest.backupBytes}; actual: ${actualBackupBytes})`, ) } + const controlPath = eventingControlSnapshotPath(dataDir, checkpointId) + if (manifest.formatVersion === MANIFEST_FORMAT_VERSION) { + await assertNoSymlink(snapshotsRoot, controlPath) + await assertRealFile(controlPath, "checkpoint eventing control snapshot") + const controlBytes = (await stat(controlPath)).size + if (controlBytes !== manifest.controlBytes) + throw new Error( + `checkpoint control-store size mismatch (manifest: ${manifest.controlBytes}; actual: ${controlBytes})`, + ) + const controlSha256 = sha256File(controlPath) + if (controlSha256 !== manifest.controlSha256) + throw new Error("checkpoint control-store digest mismatch") + const controlValidation = LocalEventingControlStore.validateSnapshot(controlPath) + if (!controlValidationMatches(manifest.controlValidation, controlValidation)) + throw new Error("checkpoint control-store validation does not match its manifest") + } else if (existsSync(controlPath)) { + throw new Error("legacy checkpoint contains an unsigned eventing control snapshot") + } return { checkpointId, snapshotDir, @@ -869,6 +931,15 @@ const restoreResolvedInto = async ( `RESTORE DATABASE default FROM Disk('src', '${resolvedCheckpoint.backupSqlPath}') ` + "SETTINGS allow_different_database_def=1", ) + if (resolvedCheckpoint.manifest.formatVersion === MANIFEST_FORMAT_VERSION) { + await LocalEventingControlStore.restoreSnapshot( + join(resolvedCheckpoint.snapshotDir, "control.sqlite"), + targetDataDir, + ) + } else { + const controlStore = await LocalEventingControlStore.open(targetDataDir) + controlStore.close() + } return { db, validation: validateRestoredDatabase(db) } } catch (error) { db?.close() @@ -1620,10 +1691,14 @@ const createCheckpointTraced = Effect.fn("CheckpointService.create")(function* ( const { oldState, snapshot, startedAt } = prepared let { operation } = prepared await syncTree(snapshotBackupDir(options.dataDir, checkpointId)) + const controlPath = eventingControlSnapshotPath(options.dataDir, checkpointId) + await assertNoSymlink(checkpointSnapshotsRoot(options.dataDir), controlPath) + await assertRealFile(controlPath, "checkpoint eventing control snapshot") + const controlValidation = LocalEventingControlStore.validateSnapshot(controlPath) operation = { ...operation, phase: "backup-complete" } await writeOperation(options.dataDir, operation, options.faults) const provisionalManifest: CheckpointManifest = { - formatVersion: 1, + formatVersion: MANIFEST_FORMAT_VERSION, checkpointId, operationId, mapleVersion: MAPLE_VERSION, @@ -1633,6 +1708,10 @@ const createCheckpointTraced = Effect.fn("CheckpointService.create")(function* ( sourceDataDir: resolve(options.dataDir), backupRelativePath: snapshotBackupRelativePath(checkpointId), backupBytes: await dirSize(snapshotBackupDir(options.dataDir, checkpointId)), + controlRelativePath: snapshotControlRelativePath(checkpointId), + controlBytes: (await stat(controlPath)).size, + controlSha256: sha256File(controlPath), + controlValidation, validation: { validatedAt: startedAt, traces: 0, @@ -1821,7 +1900,7 @@ const beginResetTransactionUnlocked = async ( const entries = await readdir(live, { withFileTypes: true }) for (const entry of entries) { if (entry.name === "backups") continue - if (!RESETTABLE_CHDB_ENTRIES.has(entry.name)) { + if (!RESETTABLE_LIVE_ENTRIES.has(entry.name)) { unknown.push(join(live, entry.name)) continue } @@ -2069,9 +2148,10 @@ export const reconcileCheckpointRecovery = Effect.fn("CheckpointService.reconcil }) /** - * Explicitly remove the live chDB store while preserving the checkpoint - * registry below `/backups`. The maintenance lock serializes this - * destructive operation with checkpoint, restore, and archive work. + * Explicitly remove the live chDB and eventing control stores while preserving + * the checkpoint registry below `/backups`. The maintenance lock + * serializes this destructive operation with checkpoint, restore, and archive + * work. */ export const resetLiveStorePreservingCheckpoints = Effect.fn("CheckpointService.reset")(function* ( dataDir: string, diff --git a/apps/cli/src/server/eventing/consumer-auth.ts b/apps/cli/src/server/eventing/consumer-auth.ts new file mode 100644 index 000000000..3751a4b5c --- /dev/null +++ b/apps/cli/src/server/eventing/consumer-auth.ts @@ -0,0 +1,35 @@ +import { randomBytes, timingSafeEqual } from "node:crypto" +import { lstatSync, readFileSync } from "node:fs" +import { resolve } from "node:path" +import { durableWrite } from "../durable-files" + +const TOKEN_BYTES = 32 + +export const eventConsumerTokenPath = (dataDir: string): string => `${resolve(dataDir)}.event-consumer-token` + +const readRealFile = (path: string): string => { + const stat = lstatSync(path) + if (stat.isSymbolicLink() || !stat.isFile()) + throw new Error(`event consumer token is not a real file: ${path}`) + return readFileSync(path, "utf8") +} + +export const ensureEventConsumerToken = async (dataDir: string): Promise => { + const path = eventConsumerTokenPath(dataDir) + try { + readRealFile(path) + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error + await durableWrite(path, `${randomBytes(TOKEN_BYTES).toString("hex")}\n`) + } + const token = readRealFile(path).trim() + if (!/^[0-9a-f]{64}$/.test(token)) throw new Error("event consumer token is malformed") + return token +} + +export const eventConsumerTokenMatches = (expected: string, supplied: string | null): boolean => { + if (supplied === null) return false + const left = Buffer.from(expected) + const right = Buffer.from(supplied) + return left.length === right.length && timingSafeEqual(left, right) +} diff --git a/apps/cli/src/server/eventing/control-store.ts b/apps/cli/src/server/eventing/control-store.ts new file mode 100644 index 000000000..e2ed30267 --- /dev/null +++ b/apps/cli/src/server/eventing/control-store.ts @@ -0,0 +1,1292 @@ +import { constants as sqliteConstants, Database } from "bun:sqlite" +import { createHash, randomBytes, timingSafeEqual } from "node:crypto" +import { chmodSync, existsSync, lstatSync, mkdtempSync, readFileSync, rmSync } from "node:fs" +import { dirname, join, resolve } from "node:path" +import { pathToFileURL } from "node:url" +import { + canonicalJson, + isJsonValue, + SignalProjectionSpecSchema, + validateMapleCloudEvent, + type MapleCloudEvent, + type JsonValue, + type ProjectionFailure, + type SignalProjectionSpec, +} from "@maple/eventing-core" +import { Schema } from "effect" +import { durableWrite, ensurePrivateDirectory } from "../durable-files" +import { NOOP_EVENTING_TELEMETRY, type EventingTelemetry } from "./telemetry" + +const CONTROL_SCHEMA_VERSION = 4 +const CONTROL_DIRECTORY = "control" +const CONTROL_DATABASE = "eventing.sqlite" +const MAX_FAILURES_PER_TENANT = 10_000 +export const DEFAULT_MAX_OUTBOX_EVENTS = 10_000 +export const DEFAULT_MAX_OUTBOX_BYTES = 256 * 1024 * 1024 +export const DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS = 1_000 + +export const eventingControlDirectory = (dataDir: string): string => join(resolve(dataDir), CONTROL_DIRECTORY) +export const eventingControlPath = (dataDir: string): string => + join(eventingControlDirectory(dataDir), CONTROL_DATABASE) +export const eventingControlSnapshotPath = (dataDir: string, checkpointId: string): string => + join(resolve(dataDir), "backups", "snapshots", checkpointId, "control.sqlite") + +const CREATE_SCHEMA = ` +CREATE TABLE projection_revisions ( + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + revision INTEGER NOT NULL CHECK (revision > 0), + enabled INTEGER NOT NULL CHECK (enabled IN (0, 1)), + spec_json TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY (tenant_id, projection_id, revision) +) STRICT; + +CREATE TABLE active_projections ( + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + revision INTEGER NOT NULL, + PRIMARY KEY (tenant_id, projection_id), + FOREIGN KEY (tenant_id, projection_id, revision) + REFERENCES projection_revisions (tenant_id, projection_id, revision) + ON DELETE RESTRICT +) STRICT; + +CREATE TABLE outbox_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL UNIQUE, + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + projection_revision INTEGER NOT NULL CHECK (projection_revision > 0), + source_kind TEXT, + source TEXT, + source_occurrence_id TEXT, + source_fingerprint TEXT, + state TEXT NOT NULL CHECK (state IN ('staged', 'ready')), + event_json TEXT NOT NULL, + staged_at TEXT NOT NULL, + ready_at TEXT +) STRICT; + +CREATE INDEX outbox_events_staged_sequence + ON outbox_events (state, sequence); + +CREATE INDEX outbox_events_staged_occurrence + ON outbox_events (tenant_id, source_kind, source, source_occurrence_id, state); + +CREATE TABLE outbox_ready_events ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + event_id TEXT NOT NULL UNIQUE, + ready_at TEXT NOT NULL, + FOREIGN KEY (event_id) + REFERENCES outbox_events (event_id) + ON DELETE RESTRICT +) STRICT; + +CREATE TABLE projection_failures ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + tenant_id TEXT NOT NULL, + projection_id TEXT NOT NULL, + projection_revision INTEGER NOT NULL CHECK (projection_revision > 0), + occurrence_id TEXT, + message TEXT NOT NULL, + created_at TEXT NOT NULL +) STRICT; + +CREATE UNIQUE INDEX projection_failures_occurrence + ON projection_failures (tenant_id, projection_id, projection_revision, occurrence_id) + WHERE occurrence_id IS NOT NULL; + +CREATE TABLE event_consumers ( + consumer_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + active INTEGER NOT NULL CHECK (active IN (0, 1)), + last_acked_sequence INTEGER NOT NULL CHECK (last_acked_sequence >= 0), + lease_token_hash TEXT, + lease_expires_at TEXT, + claimed_through_sequence INTEGER CHECK (claimed_through_sequence > 0), + registered_at TEXT NOT NULL, + disabled_at TEXT, + CHECK ( + (active = 1 AND disabled_at IS NULL) OR + (active = 0 AND disabled_at IS NOT NULL) + ), + CHECK ( + (lease_token_hash IS NULL AND lease_expires_at IS NULL AND claimed_through_sequence IS NULL) OR + (lease_token_hash IS NOT NULL AND lease_expires_at IS NOT NULL AND claimed_through_sequence IS NOT NULL) + ), + CHECK (claimed_through_sequence IS NULL OR claimed_through_sequence > last_acked_sequence) +) STRICT; + +CREATE INDEX event_consumers_tenant_active_ack + ON event_consumers (tenant_id, active, last_acked_sequence); + +PRAGMA user_version = 4; +` + +const MIGRATE_SCHEMA_1_TO_2 = ` +CREATE TABLE event_consumers ( + consumer_id TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + active INTEGER NOT NULL CHECK (active IN (0, 1)), + last_acked_sequence INTEGER NOT NULL CHECK (last_acked_sequence >= 0), + lease_token_hash TEXT, + lease_expires_at TEXT, + claimed_through_sequence INTEGER CHECK (claimed_through_sequence > 0), + registered_at TEXT NOT NULL, + disabled_at TEXT, + CHECK ( + (active = 1 AND disabled_at IS NULL) OR + (active = 0 AND disabled_at IS NOT NULL) + ), + CHECK ( + (lease_token_hash IS NULL AND lease_expires_at IS NULL AND claimed_through_sequence IS NULL) OR + (lease_token_hash IS NOT NULL AND lease_expires_at IS NOT NULL AND claimed_through_sequence IS NOT NULL) + ), + CHECK (claimed_through_sequence IS NULL OR claimed_through_sequence > last_acked_sequence) +) STRICT; + +CREATE INDEX event_consumers_tenant_active_ack + ON event_consumers (tenant_id, active, last_acked_sequence); + +PRAGMA user_version = 2; +` + +const MIGRATE_SCHEMA_2_TO_3 = ` +ALTER TABLE outbox_events ADD COLUMN source_kind TEXT; +ALTER TABLE outbox_events ADD COLUMN source TEXT; +ALTER TABLE outbox_events ADD COLUMN source_occurrence_id TEXT; + +UPDATE outbox_events +SET source_kind = ( + SELECT json_extract(spec_json, '$.sourceKind') + FROM projection_revisions + WHERE projection_revisions.tenant_id = outbox_events.tenant_id + AND projection_revisions.projection_id = outbox_events.projection_id + AND projection_revisions.revision = outbox_events.projection_revision + ), + source = json_extract(event_json, '$.source'), + source_occurrence_id = json_extract(event_json, '$.sourceoccurrenceid'); + +CREATE INDEX outbox_events_staged_occurrence + ON outbox_events (tenant_id, source_kind, source, source_occurrence_id, state); + +PRAGMA user_version = 3; +` + +const MIGRATE_SCHEMA_3_TO_4 = ` +ALTER TABLE outbox_events ADD COLUMN source_fingerprint TEXT; + +PRAGMA user_version = 4; +` + +interface UserVersionRow { + readonly user_version: number | bigint +} + +interface RevisionRow { + readonly revision: number | bigint | null +} + +interface ProjectionJsonRow { + readonly spec_json: string +} + +interface EventRow { + readonly event_id: string + readonly event_json: string + readonly state: "staged" | "ready" + readonly source_fingerprint: string | null +} + +interface EventJsonRow { + readonly sequence: number | bigint + readonly event_json: string + readonly staged_at: string + readonly ready_at: string | null +} + +interface CountRow { + readonly count: number | bigint +} + +interface QuickCheckRow { + readonly quick_check: string +} + +interface WalCheckpointRow { + readonly busy: number | bigint + readonly log: number | bigint + readonly checkpointed: number | bigint +} + +interface OutboxUsageRow { + readonly count: number | bigint + readonly bytes: number | bigint +} + +interface SequenceRow { + readonly sequence: number | bigint | null +} + +interface ConsumerRow { + readonly consumer_id: string + readonly tenant_id: string + readonly active: number | bigint + readonly last_acked_sequence: number | bigint + readonly lease_token_hash: string | null + readonly lease_expires_at: string | null + readonly claimed_through_sequence: number | bigint | null + readonly registered_at: string + readonly disabled_at: string | null +} + +interface EventIdRow { + readonly event_id: string +} + +interface StagedOccurrenceRow extends EventIdRow { + readonly source_fingerprint: string | null +} + +interface ActiveRevisionRow { + readonly revision: number | bigint +} + +export interface StageEventsResult { + readonly inserted: number + readonly deduplicated: number + readonly eventIds: readonly string[] +} + +export interface EventingControlSnapshotValidation { + readonly schemaVersion: number + readonly projectionRevisions: number + readonly projectionFailures: number + readonly stagedEvents: number + readonly readyEvents: number +} + +export interface LocalEventingControlLimits { + readonly maxOutboxEvents: number + readonly maxOutboxBytes: number + readonly retainAcknowledgedReadyEvents?: number +} + +interface ResolvedLocalEventingControlLimits { + readonly maxOutboxEvents: number + readonly maxOutboxBytes: number + readonly retainAcknowledgedReadyEvents: number +} + +export interface EventingOutboxRecord { + readonly sequence: number + readonly event: MapleCloudEvent + readonly stagedAt: string + readonly readyAt: string | null +} + +export interface EventingOutboxPage { + readonly events: readonly EventingOutboxRecord[] + readonly nextCursor: number | null +} + +export type EventConsumerStart = "beginning" | "latest" + +export interface EventConsumer { + readonly consumerId: string + readonly tenantId: string + readonly active: boolean + readonly lastAcknowledgedSequence: number + readonly leaseExpiresAt: string | null + readonly claimedThroughSequence: number | null + readonly registeredAt: string + readonly disabledAt: string | null +} + +export interface EventConsumerClaim { + readonly consumerId: string + readonly leaseToken: string | null + readonly leaseExpiresAt: string | null + readonly throughSequence: number | null + readonly events: readonly EventingOutboxRecord[] +} + +export interface EventConsumerAcknowledgement { + readonly consumerId: string + readonly acknowledgedThrough: number + readonly prunedEvents: number +} + +export class EventConsumerInputError extends Error {} +export class EventConsumerNotFoundError extends Error {} +export class EventConsumerConflictError extends Error {} + +const asNumber = (value: number | bigint): number => { + const number = Number(value) + if (!Number.isSafeInteger(number) || number < 0) throw new Error(`invalid SQLite integer: ${value}`) + return number +} + +const decodeProjection = (json: string): SignalProjectionSpec => + Schema.decodeUnknownSync(SignalProjectionSpecSchema)(JSON.parse(json) as unknown) + +const decodeEvent = (json: string): MapleCloudEvent => { + return validateMapleCloudEvent(JSON.parse(json) as unknown).event +} + +const assertRealDatabaseFile = (path: string): void => { + let info + try { + info = lstatSync(path) + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return + throw error + } + if (info.isSymbolicLink() || !info.isFile()) + throw new Error(`eventing control database is not a real file: ${path}`) +} + +const configure = (db: Database): void => { + db.exec("PRAGMA foreign_keys = ON") + db.exec("PRAGMA trusted_schema = OFF") + db.exec("PRAGMA busy_timeout = 5000") +} + +const assertSchema3MigrationSafe = (db: Database): void => { + const row = db + .query( + "SELECT count(*) AS count FROM outbox_events WHERE state = 'staged' AND source_occurrence_id IS NOT NULL", + ) + .get() + if (row === null) throw new Error("schema-3 staged-source preflight returned no row") + if (asNumber(row.count) > 0) + throw new Error( + "cannot migrate eventing control schema 3 with staged source occurrences: the legacy rows have no source fingerprint; complete or explicitly abandon them with the schema-3 build before upgrading", + ) +} + +const checkpointWal = (db: Database): void => { + const result = db.query("PRAGMA wal_checkpoint(TRUNCATE)").get() + if (!result) throw new Error("eventing control WAL checkpoint returned no result") + const busy = asNumber(result.busy) + const log = asNumber(result.log) + const checkpointed = asNumber(result.checkpointed) + if (busy !== 0 || log !== 0) + throw new Error( + `eventing control WAL checkpoint incomplete (busy=${busy}, log=${log}, checkpointed=${checkpointed})`, + ) +} + +const validateLimits = (limits: LocalEventingControlLimits): ResolvedLocalEventingControlLimits => { + if (!Number.isSafeInteger(limits.maxOutboxEvents) || limits.maxOutboxEvents < 1) + throw new Error("maxOutboxEvents must be a positive safe integer") + if (!Number.isSafeInteger(limits.maxOutboxBytes) || limits.maxOutboxBytes < 1) + throw new Error("maxOutboxBytes must be a positive safe integer") + const retainAcknowledgedReadyEvents = + limits.retainAcknowledgedReadyEvents ?? DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS + if (!Number.isSafeInteger(retainAcknowledgedReadyEvents) || retainAcknowledgedReadyEvents < 0) + throw new Error("retainAcknowledgedReadyEvents must be a non-negative safe integer") + return { ...limits, retainAcknowledgedReadyEvents } +} + +const validateOpenDatabase = ( + db: Database, + acceptedSchemaVersions: readonly number[] = [CONTROL_SCHEMA_VERSION], +): EventingControlSnapshotValidation => { + const quick = db.query("PRAGMA quick_check").get() + if (quick?.quick_check !== "ok") throw new Error(`eventing control database quick_check failed`) + const version = db.query("PRAGMA user_version").get() + if (!version) throw new Error("eventing control database has no schema version") + const schemaVersion = asNumber(version.user_version) + if (!acceptedSchemaVersions.includes(schemaVersion)) + throw new Error( + `unsupported eventing control schema ${schemaVersion}; expected ${acceptedSchemaVersions.join(" or ")}`, + ) + const count = (where: string): number => { + const row = db.query(`SELECT count(*) AS count FROM outbox_events ${where}`).get() + if (!row) throw new Error("eventing control count query returned no row") + return asNumber(row.count) + } + const revisions = db.query("SELECT count(*) AS count FROM projection_revisions").get() + if (!revisions) throw new Error("eventing projection count query returned no row") + const failures = db.query("SELECT count(*) AS count FROM projection_failures").get() + if (!failures) throw new Error("eventing projection-failure count query returned no row") + const invalidReadiness = db + .query( + `SELECT count(*) AS count + FROM outbox_events AS event + LEFT JOIN outbox_ready_events AS readiness ON readiness.event_id = event.event_id + WHERE (event.state = 'ready' AND ( + readiness.event_id IS NULL OR event.ready_at IS NULL OR event.ready_at <> readiness.ready_at + )) OR (event.state = 'staged' AND ( + readiness.event_id IS NOT NULL OR event.ready_at IS NOT NULL + ))`, + ) + .get() + if (!invalidReadiness) throw new Error("eventing readiness validation query returned no row") + if (asNumber(invalidReadiness.count) !== 0) + throw new Error("eventing control database has inconsistent outbox readiness state") + if (schemaVersion >= 2) { + const consumers = db + .query, []>( + "SELECT lease_expires_at, registered_at, disabled_at FROM event_consumers", + ) + .all() + for (const consumer of consumers) { + canonicalInstant(consumer.registered_at, "event consumer registeredAt") + if (consumer.lease_expires_at !== null) + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") + if (consumer.disabled_at !== null) + canonicalInstant(consumer.disabled_at, "event consumer disabledAt") + } + } + if (schemaVersion >= 4) { + const statement = db.prepare( + `SELECT count(*) AS count + FROM outbox_events + WHERE state = 'staged' + AND source_occurrence_id IS NOT NULL + AND ( + source_fingerprint IS NULL + OR length(source_fingerprint) <> 71 + OR substr(source_fingerprint, 1, 7) <> 'sha256:' + OR substr(source_fingerprint, 8) GLOB '*[^0-9a-f]*' + )`, + ) + let invalidFingerprints: CountRow | null + try { + invalidFingerprints = statement.get() + } finally { + statement.finalize() + } + if (invalidFingerprints === null) + throw new Error("eventing staged source-fingerprint validation returned no row") + if (asNumber(invalidFingerprints.count) > 0) + throw new Error("eventing control database has an invalid staged source fingerprint") + } + return { + schemaVersion, + projectionRevisions: asNumber(revisions.count), + projectionFailures: asNumber(failures.count), + stagedEvents: count("WHERE state = 'staged'"), + readyEvents: count("WHERE state = 'ready'"), + } +} + +const CONSUMER_ID = /^[a-z][a-z0-9._-]{0,63}$/ +const LEASE_TOKEN = /^[0-9a-f]{64}$/ + +const validateConsumerId = (consumerId: string): string => { + if (!CONSUMER_ID.test(consumerId)) + throw new EventConsumerInputError( + "consumerId must start with a lowercase letter and contain at most 64 lowercase letters, digits, dots, underscores, or hyphens", + ) + return consumerId +} + +const canonicalInstant = (value: string, label: string): number => { + const milliseconds = Date.parse(value) + if (Number.isNaN(milliseconds) || new Date(milliseconds).toISOString() !== value) + throw new EventConsumerInputError(`${label} must be canonical ISO-8601`) + return milliseconds +} + +const tokenHash = (token: string): string => createHash("sha256").update(token).digest("hex") + +const tokenHashMatches = (expected: string, token: string): boolean => { + if (!LEASE_TOKEN.test(token)) return false + const left = Buffer.from(expected, "hex") + const right = Buffer.from(tokenHash(token), "hex") + return left.length === right.length && timingSafeEqual(left, right) +} + +const decodeConsumer = (row: ConsumerRow): EventConsumer => ({ + consumerId: row.consumer_id, + tenantId: row.tenant_id, + active: asNumber(row.active) === 1, + lastAcknowledgedSequence: asNumber(row.last_acked_sequence), + leaseExpiresAt: row.lease_expires_at, + claimedThroughSequence: + row.claimed_through_sequence === null ? null : asNumber(row.claimed_through_sequence), + registeredAt: row.registered_at, + disabledAt: row.disabled_at, +}) + +export class LocalEventingControlStore { + readonly #db: Database + readonly #limits: ResolvedLocalEventingControlLimits + readonly #telemetry: EventingTelemetry + readonly path: string + + private constructor( + path: string, + db: Database, + limits: ResolvedLocalEventingControlLimits, + telemetry: EventingTelemetry, + ) { + this.path = path + this.#db = db + this.#limits = limits + this.#telemetry = telemetry + } + + static async open( + dataDir: string, + limits: LocalEventingControlLimits = { + maxOutboxEvents: DEFAULT_MAX_OUTBOX_EVENTS, + maxOutboxBytes: DEFAULT_MAX_OUTBOX_BYTES, + retainAcknowledgedReadyEvents: DEFAULT_RETAIN_ACKNOWLEDGED_READY_EVENTS, + }, + telemetry: EventingTelemetry = NOOP_EVENTING_TELEMETRY, + ): Promise { + const validatedLimits = validateLimits(limits) + const directory = eventingControlDirectory(dataDir) + await ensurePrivateDirectory(directory) + const path = eventingControlPath(dataDir) + assertRealDatabaseFile(path) + const db = new Database(path, { create: true, readwrite: true, strict: true, safeIntegers: true }) + try { + configure(db) + db.exec("PRAGMA journal_mode = WAL") + db.exec("PRAGMA synchronous = FULL") + const version = db.query("PRAGMA user_version").get() + if (!version) throw new Error("eventing control database has no schema version") + let schemaVersion = asNumber(version.user_version) + if (schemaVersion === 0) { + db.transaction(() => db.exec(CREATE_SCHEMA)).exclusive() + schemaVersion = CONTROL_SCHEMA_VERSION + } + if (schemaVersion === 1) { + db.transaction(() => db.exec(MIGRATE_SCHEMA_1_TO_2)).exclusive() + schemaVersion = 2 + } + if (schemaVersion === 2) { + db.transaction(() => db.exec(MIGRATE_SCHEMA_2_TO_3)).exclusive() + schemaVersion = 3 + } + if (schemaVersion === 3) { + db.transaction(() => { + assertSchema3MigrationSafe(db) + db.exec(MIGRATE_SCHEMA_3_TO_4) + }).exclusive() + schemaVersion = 4 + } + if (schemaVersion !== CONTROL_SCHEMA_VERSION) + throw new Error( + `unsupported eventing control schema ${schemaVersion}; expected ${CONTROL_SCHEMA_VERSION}`, + ) + chmodSync(path, 0o600) + validateOpenDatabase(db) + return new LocalEventingControlStore(path, db, validatedLimits, telemetry) + } catch (error) { + db.close() + throw error + } + } + + close(): void { + checkpointWal(this.#db) + this.#db.close(true) + } + + saveProjection(spec: SignalProjectionSpec, createdAt = new Date().toISOString()): void { + const decoded = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(spec) + if (!isJsonValue(decoded)) throw new Error("projection spec must be finite JSON") + const specJson = canonicalJson(decoded) + this.#db + .transaction(() => { + const latest = this.#db + .query( + "SELECT max(revision) AS revision FROM projection_revisions WHERE tenant_id = ? AND projection_id = ?", + ) + .get(decoded.tenantId, decoded.id) + const latestRevision = latest?.revision == null ? null : asNumber(latest.revision) + const existing = this.#db + .query( + "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", + ) + .get(decoded.tenantId, decoded.id, decoded.revision) + if (existing) { + if (existing.spec_json !== specJson) + throw new Error( + `projection revision is immutable: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + if (latestRevision !== decoded.revision) + throw new Error( + `stale projection revision: ${decoded.tenantId}:${decoded.id}@${decoded.revision}; latest is ${latestRevision}`, + ) + const active = this.#db + .query( + "SELECT revision FROM active_projections WHERE tenant_id = ? AND projection_id = ?", + ) + .get(decoded.tenantId, decoded.id) + const activeRevision = active === null ? null : asNumber(active.revision) + const expectedActiveRevision = decoded.enabled ? decoded.revision : null + if (activeRevision !== expectedActiveRevision) + throw new Error( + `projection active state conflicts with exact revision replay: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + return + } else { + const expected = latestRevision === null ? 1 : latestRevision + 1 + if (decoded.revision !== expected) + throw new Error( + `projection revision must be ${expected}: ${decoded.tenantId}:${decoded.id}@${decoded.revision}`, + ) + this.#db.run( + "INSERT INTO projection_revisions (tenant_id, projection_id, revision, enabled, spec_json, created_at) VALUES (?, ?, ?, ?, ?, ?)", + [ + decoded.tenantId, + decoded.id, + decoded.revision, + decoded.enabled ? 1 : 0, + specJson, + createdAt, + ], + ) + } + + if (decoded.enabled) + this.#db.run( + "INSERT INTO active_projections (tenant_id, projection_id, revision) VALUES (?, ?, ?) ON CONFLICT (tenant_id, projection_id) DO UPDATE SET revision = excluded.revision", + [decoded.tenantId, decoded.id, decoded.revision], + ) + else + this.#db.run("DELETE FROM active_projections WHERE tenant_id = ? AND projection_id = ?", [ + decoded.tenantId, + decoded.id, + ]) + }) + .immediate() + } + + loadEnabledProjections(tenantId: string): readonly SignalProjectionSpec[] { + return this.#db + .query( + `SELECT r.spec_json + FROM active_projections a + JOIN projection_revisions r + ON r.tenant_id = a.tenant_id + AND r.projection_id = a.projection_id + AND r.revision = a.revision + WHERE a.tenant_id = ? + ORDER BY a.projection_id`, + ) + .all(tenantId) + .map(({ spec_json }) => decodeProjection(spec_json)) + } + + stageEvents( + events: readonly MapleCloudEvent[], + sourceFingerprints: ReadonlyMap = new Map(), + stagedAt = new Date().toISOString(), + ): StageEventsResult { + let inserted = 0 + let deduplicated = 0 + const eventIds: string[] = [] + try { + this.#db + .transaction(() => { + const usage = this.#db + .query( + "SELECT count(*) AS count, coalesce(sum(length(CAST(event_json AS BLOB))), 0) AS bytes FROM outbox_events", + ) + .get() + if (!usage) throw new Error("event outbox usage query returned no row") + let outboxEvents = asNumber(usage.count) + let outboxBytes = asNumber(usage.bytes) + for (const candidate of events) { + const validated = validateMapleCloudEvent(candidate) + const { event, canonicalJson: eventJson, byteLength: eventBytes } = validated + const sourceFingerprint = sourceFingerprints.get(event.id) ?? null + if (sourceFingerprint !== null && !/^sha256:[0-9a-f]{64}$/.test(sourceFingerprint)) + throw new Error(`event has invalid source fingerprint: ${event.id}`) + if (event.sourceoccurrenceid !== undefined && sourceFingerprint === null) + throw new Error( + `event with source occurrence ID requires a source fingerprint: ${event.id}`, + ) + let sourceKind: string | null = null + if (event.sourceoccurrenceid !== undefined) { + const projection = this.#db + .query( + "SELECT spec_json FROM projection_revisions WHERE tenant_id = ? AND projection_id = ? AND revision = ?", + ) + .get(event.tenantid, event.projectionid, event.projectionrevision) + if (projection === null) + throw new Error( + `event references unknown projection revision: ${event.tenantid}:${event.projectionid}@${event.projectionrevision}`, + ) + sourceKind = decodeProjection(projection.spec_json).sourceKind + } + const existing = this.#db + .query( + "SELECT event_id, event_json, state, source_fingerprint FROM outbox_events WHERE event_id = ?", + ) + .get(event.id) + if (existing) { + if (existing.event_json !== eventJson) + throw new Error(`event ID collision with different payload: ${event.id}`) + if ( + sourceFingerprint !== null && + existing.source_fingerprint !== null && + existing.source_fingerprint !== sourceFingerprint + ) + throw new Error( + `event ID collision with different source occurrence: ${event.id}`, + ) + if ( + existing.state === "staged" && + sourceFingerprint !== null && + existing.source_fingerprint === null + ) + throw new Error(`staged event has no recovery fingerprint: ${event.id}`) + deduplicated += 1 + } else { + if ( + outboxEvents + 1 > this.#limits.maxOutboxEvents || + outboxBytes + eventBytes > this.#limits.maxOutboxBytes + ) + throw new Error( + `event outbox capacity exceeded (${outboxEvents}/${this.#limits.maxOutboxEvents} events, ${outboxBytes}/${this.#limits.maxOutboxBytes} bytes)`, + ) + this.#db.run( + "INSERT INTO outbox_events (event_id, tenant_id, projection_id, projection_revision, source_kind, source, source_occurrence_id, source_fingerprint, state, event_json, staged_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, 'staged', ?, ?)", + [ + event.id, + event.tenantid, + event.projectionid, + event.projectionrevision, + sourceKind, + event.sourceoccurrenceid === undefined ? null : event.source, + event.sourceoccurrenceid ?? null, + sourceFingerprint, + eventJson, + stagedAt, + ], + ) + inserted += 1 + outboxEvents += 1 + outboxBytes += eventBytes + } + eventIds.push(event.id) + } + }) + .immediate() + } catch (error) { + this.#telemetry.record({ operation: "outbox_stage", outcome: "failure" }) + throw error + } + this.#telemetry.record({ operation: "outbox_stage", outcome: "success", count: inserted }) + this.#telemetry.record({ operation: "outbox_dedup", outcome: "success", count: deduplicated }) + return { inserted, deduplicated, eventIds } + } + + hasStagedSourceKind(tenantId: string, sourceKind: string): boolean { + const row = this.#db + .query( + "SELECT count(*) AS count FROM outbox_events WHERE tenant_id = ? AND source_kind = ? AND state = 'staged'", + ) + .get(tenantId, sourceKind) + if (row === null) throw new Error("staged source-kind query returned no row") + return asNumber(row.count) > 0 + } + + hasStagedSourceOccurrence( + tenantId: string, + sourceKind: string, + source: string, + sourceOccurrenceId: string, + ): boolean { + const row = this.#db + .query( + "SELECT count(*) AS count FROM outbox_events WHERE tenant_id = ? AND source_kind = ? AND source = ? AND source_occurrence_id = ? AND state = 'staged'", + ) + .get(tenantId, sourceKind, source, sourceOccurrenceId) + if (row === null) throw new Error("staged source-occurrence query returned no row") + return asNumber(row.count) > 0 + } + + stagedEventIdsForOccurrence( + tenantId: string, + sourceKind: string, + source: string, + sourceOccurrenceId: string, + sourceFingerprint: string, + ): readonly string[] { + const rows = this.#db + .query( + "SELECT event_id, source_fingerprint FROM outbox_events WHERE tenant_id = ? AND source_kind = ? AND source = ? AND source_occurrence_id = ? AND state = 'staged' ORDER BY sequence", + ) + .all(tenantId, sourceKind, source, sourceOccurrenceId) + for (const row of rows) { + if (row.source_fingerprint === null) + throw new Error(`staged source occurrence has no recovery fingerprint: ${row.event_id}`) + if (row.source_fingerprint !== sourceFingerprint) + throw new Error(`staged source occurrence collision: ${sourceOccurrenceId}`) + } + return rows.map(({ event_id }) => event_id) + } + + markReady(eventIds: readonly string[], readyAt = new Date().toISOString()): void { + let markedReady = 0 + try { + this.#db + .transaction(() => { + for (const eventId of eventIds) { + const row = this.#db + .query, [string]>( + "SELECT state FROM outbox_events WHERE event_id = ?", + ) + .get(eventId) + if (!row) throw new Error(`cannot mark unknown event ready: ${eventId}`) + if (row.state === "ready") continue + this.#db.run("INSERT INTO outbox_ready_events (event_id, ready_at) VALUES (?, ?)", [ + eventId, + readyAt, + ]) + this.#db.run( + "UPDATE outbox_events SET state = 'ready', ready_at = ? WHERE event_id = ? AND state = 'staged'", + [readyAt, eventId], + ) + markedReady += 1 + } + }) + .immediate() + } catch (error) { + this.#telemetry.record({ operation: "outbox_ready", outcome: "failure" }) + throw error + } + this.#telemetry.record({ operation: "outbox_ready", outcome: "success", count: markedReady }) + } + + #listOutbox(state: "ready" | "staged", limit = 100, after = 0): EventingOutboxPage { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw new Error("outbox-event limit must be between 1 and 1000") + if (!Number.isSafeInteger(after) || after < 0) + throw new Error("outbox cursor must be a non-negative safe integer") + const rows = + state === "ready" + ? this.#db + .query( + `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.state = 'ready' AND readiness.sequence > ? + ORDER BY readiness.sequence + LIMIT ?`, + ) + .all(after, limit + 1) + : this.#db + .query( + `SELECT sequence, event_json, staged_at, ready_at + FROM outbox_events + WHERE state = 'staged' AND sequence > ? + ORDER BY sequence + LIMIT ?`, + ) + .all(after, limit + 1) + const hasMore = rows.length > limit + const pageRows = hasMore ? rows.slice(0, limit) : rows + const page = pageRows.map(({ sequence, event_json, staged_at, ready_at }) => ({ + sequence: asNumber(sequence), + event: decodeEvent(event_json), + stagedAt: staged_at, + readyAt: ready_at, + })) + return { + events: page, + nextCursor: hasMore ? (page.at(-1)?.sequence ?? null) : null, + } + } + + listReady(limit = 100, after = 0): EventingOutboxPage { + return this.#listOutbox("ready", limit, after) + } + + listStaged(limit = 100, after = 0): EventingOutboxPage { + return this.#listOutbox("staged", limit, after) + } + + listConsumers(tenantId: string): readonly EventConsumer[] { + return this.#db + .query( + `SELECT consumer_id, tenant_id, active, last_acked_sequence, lease_token_hash, + lease_expires_at, claimed_through_sequence, registered_at, disabled_at + FROM event_consumers + WHERE tenant_id = ? + ORDER BY consumer_id`, + ) + .all(tenantId) + .map(decodeConsumer) + } + + registerConsumer( + tenantId: string, + consumerId: string, + startAt: EventConsumerStart, + registeredAt = new Date().toISOString(), + ): EventConsumer { + validateConsumerId(consumerId) + if (startAt !== "beginning" && startAt !== "latest") + throw new EventConsumerInputError("startAt must be beginning or latest") + canonicalInstant(registeredAt, "event consumer registeredAt") + return this.#db + .transaction(() => { + const existing = this.#consumer(tenantId, consumerId) + if (existing) + throw new EventConsumerConflictError(`event consumer already exists: ${consumerId}`) + const boundary = this.#db + .query( + startAt === "latest" + ? `SELECT max(readiness.sequence) AS sequence + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ?` + : `SELECT min(readiness.sequence) AS sequence + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ?`, + ) + .get(tenantId) + const sequence = boundary?.sequence == null ? 0 : asNumber(boundary.sequence) + const lastAcknowledged = startAt === "beginning" ? Math.max(0, sequence - 1) : sequence + this.#db.run( + "INSERT INTO event_consumers (consumer_id, tenant_id, active, last_acked_sequence, registered_at) VALUES (?, ?, 1, ?, ?)", + [consumerId, tenantId, lastAcknowledged, registeredAt], + ) + return decodeConsumer(this.#consumer(tenantId, consumerId)!) + }) + .immediate() + } + + disableConsumer( + tenantId: string, + consumerId: string, + disabledAt = new Date().toISOString(), + ): EventConsumer { + validateConsumerId(consumerId) + canonicalInstant(disabledAt, "event consumer disabledAt") + return this.#db + .transaction(() => { + const existing = this.#consumer(tenantId, consumerId) + if (!existing) throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) + if (asNumber(existing.active) === 0) return decodeConsumer(existing) + this.#db.run( + `UPDATE event_consumers + SET active = 0, lease_token_hash = NULL, lease_expires_at = NULL, + claimed_through_sequence = NULL, disabled_at = ? + WHERE tenant_id = ? AND consumer_id = ?`, + [disabledAt, tenantId, consumerId], + ) + this.#pruneAcknowledgedReady(tenantId) + return decodeConsumer(this.#consumer(tenantId, consumerId)!) + }) + .immediate() + } + + claimReady( + tenantId: string, + consumerId: string, + limit: number, + leaseSeconds: number, + now = new Date().toISOString(), + ): EventConsumerClaim { + let reclaimedExpiredLease = false + let lag = 0 + try { + validateConsumerId(consumerId) + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 1_000) + throw new EventConsumerInputError("claim limit must be between 1 and 1000") + if (!Number.isSafeInteger(leaseSeconds) || leaseSeconds < 5 || leaseSeconds > 300) + throw new EventConsumerInputError("leaseSeconds must be between 5 and 300") + const nowMilliseconds = canonicalInstant(now, "claim time") + const claim = this.#db + .transaction(() => { + const consumer = this.#consumer(tenantId, consumerId) + if (!consumer) + throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) + if (asNumber(consumer.active) === 0) + throw new EventConsumerConflictError(`event consumer is disabled: ${consumerId}`) + if ( + consumer.lease_expires_at !== null && + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") > + nowMilliseconds + ) + throw new EventConsumerConflictError( + `event consumer already has an active lease: ${consumerId}`, + ) + if (consumer.lease_expires_at !== null) reclaimedExpiredLease = true + lag = this.#consumerLag(tenantId, asNumber(consumer.last_acked_sequence)) + + const rows = this.#db + .query( + `SELECT readiness.sequence, event.event_json, event.staged_at, readiness.ready_at + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND event.state = 'ready' AND readiness.sequence > ? + ORDER BY readiness.sequence + LIMIT ?`, + ) + .all(tenantId, asNumber(consumer.last_acked_sequence), limit) + if (rows.length === 0) { + this.#db.run( + "UPDATE event_consumers SET lease_token_hash = NULL, lease_expires_at = NULL, claimed_through_sequence = NULL WHERE tenant_id = ? AND consumer_id = ?", + [tenantId, consumerId], + ) + return { + consumerId, + leaseToken: null, + leaseExpiresAt: null, + throughSequence: null, + events: [], + } + } + + const leaseToken = randomBytes(32).toString("hex") + const leaseExpiresAt = new Date(nowMilliseconds + leaseSeconds * 1_000).toISOString() + const throughSequence = asNumber(rows.at(-1)!.sequence) + this.#db.run( + `UPDATE event_consumers + SET lease_token_hash = ?, lease_expires_at = ?, claimed_through_sequence = ? + WHERE tenant_id = ? AND consumer_id = ?`, + [tokenHash(leaseToken), leaseExpiresAt, throughSequence, tenantId, consumerId], + ) + return { + consumerId, + leaseToken, + leaseExpiresAt, + throughSequence, + events: rows.map(({ sequence, event_json, staged_at, ready_at }) => ({ + sequence: asNumber(sequence), + event: decodeEvent(event_json), + stagedAt: staged_at, + readyAt: ready_at, + })), + } + }) + .immediate() + this.#telemetry.record({ + operation: "consumer_claim", + outcome: claim.events.length === 0 ? "empty" : "success", + count: Math.max(1, claim.events.length), + }) + this.#telemetry.record({ operation: "consumer_lag", outcome: "observed", lag }) + if (reclaimedExpiredLease) + this.#telemetry.record({ operation: "consumer_lease", outcome: "reclaimed" }) + return claim + } catch (error) { + this.#telemetry.record({ operation: "consumer_claim", outcome: "failure" }) + if (error instanceof EventConsumerConflictError && /lease/.test(error.message)) + this.#telemetry.record({ operation: "consumer_lease", outcome: "failure" }) + throw error + } + } + + acknowledgeClaim( + tenantId: string, + consumerId: string, + leaseToken: string, + throughSequence: number, + now = new Date().toISOString(), + ): EventConsumerAcknowledgement { + try { + validateConsumerId(consumerId) + if (!Number.isSafeInteger(throughSequence) || throughSequence < 1) + throw new EventConsumerInputError("throughSequence must be a positive safe integer") + const nowMilliseconds = canonicalInstant(now, "acknowledgement time") + const acknowledgement = this.#db + .transaction(() => { + const consumer = this.#consumer(tenantId, consumerId) + if (!consumer) + throw new EventConsumerNotFoundError(`unknown event consumer: ${consumerId}`) + if (asNumber(consumer.active) === 0) + throw new EventConsumerConflictError(`event consumer is disabled: ${consumerId}`) + if ( + consumer.lease_token_hash === null || + consumer.lease_expires_at === null || + consumer.claimed_through_sequence === null + ) + throw new EventConsumerConflictError( + `event consumer has no active lease: ${consumerId}`, + ) + if ( + canonicalInstant(consumer.lease_expires_at, "event consumer leaseExpiresAt") <= + nowMilliseconds + ) + throw new EventConsumerConflictError( + `event consumer lease has expired: ${consumerId}`, + ) + if (!tokenHashMatches(consumer.lease_token_hash, leaseToken)) + throw new EventConsumerConflictError("event consumer lease token does not match") + const claimedThrough = asNumber(consumer.claimed_through_sequence) + if (throughSequence !== claimedThrough) + throw new EventConsumerConflictError( + `acknowledgement must cover the complete claimed batch through sequence ${claimedThrough}`, + ) + this.#db.run( + `UPDATE event_consumers + SET last_acked_sequence = ?, lease_token_hash = NULL, lease_expires_at = NULL, + claimed_through_sequence = NULL + WHERE tenant_id = ? AND consumer_id = ?`, + [throughSequence, tenantId, consumerId], + ) + return { + consumerId, + acknowledgedThrough: throughSequence, + prunedEvents: this.#pruneAcknowledgedReady(tenantId), + } + }) + .immediate() + this.#telemetry.record({ operation: "consumer_ack", outcome: "success" }) + this.#telemetry.record({ + operation: "consumer_lag", + outcome: "observed", + lag: this.#consumerLag(tenantId, acknowledgement.acknowledgedThrough), + }) + return acknowledgement + } catch (error) { + this.#telemetry.record({ operation: "consumer_ack", outcome: "failure" }) + if (error instanceof EventConsumerConflictError && /lease/.test(error.message)) + this.#telemetry.record({ operation: "consumer_lease", outcome: "failure" }) + throw error + } + } + + #consumerLag(tenantId: string, lastAcknowledgedSequence: number): number { + const latest = this.#db + .query( + `SELECT max(readiness.sequence) AS sequence + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND event.state = 'ready'`, + ) + .get(tenantId) + return Math.max( + 0, + (latest?.sequence == null ? 0 : asNumber(latest.sequence)) - lastAcknowledgedSequence, + ) + } + + #consumer(tenantId: string, consumerId: string): ConsumerRow | null { + return this.#db + .query( + `SELECT consumer_id, tenant_id, active, last_acked_sequence, lease_token_hash, + lease_expires_at, claimed_through_sequence, registered_at, disabled_at + FROM event_consumers + WHERE tenant_id = ? AND consumer_id = ?`, + ) + .get(tenantId, consumerId) + } + + #pruneAcknowledgedReady(tenantId: string): number { + const boundary = this.#db + .query( + "SELECT min(last_acked_sequence) AS sequence FROM event_consumers WHERE tenant_id = ? AND active = 1", + ) + .get(tenantId) + if (boundary?.sequence == null) return 0 + const rows = this.#db + .query( + `SELECT readiness.event_id + FROM outbox_ready_events AS readiness + INNER JOIN outbox_events AS event ON event.event_id = readiness.event_id + WHERE event.tenant_id = ? AND readiness.sequence <= ? + ORDER BY readiness.sequence`, + ) + .all(tenantId, asNumber(boundary.sequence)) + const pruneCount = Math.max(0, rows.length - this.#limits.retainAcknowledgedReadyEvents) + for (const { event_id } of rows.slice(0, pruneCount)) { + this.#db.run("DELETE FROM outbox_ready_events WHERE event_id = ?", [event_id]) + this.#db.run("DELETE FROM outbox_events WHERE event_id = ? AND state = 'ready'", [event_id]) + } + return pruneCount + } + + outboxCapacity(): LocalEventingControlLimits & { + readonly currentEvents: number + readonly currentBytes: number + } { + const usage = this.#db + .query( + "SELECT count(*) AS count, coalesce(sum(length(CAST(event_json AS BLOB))), 0) AS bytes FROM outbox_events", + ) + .get() + if (!usage) throw new Error("event outbox usage query returned no row") + return { + ...this.#limits, + currentEvents: asNumber(usage.count), + currentBytes: asNumber(usage.bytes), + } + } + + recordProjectionFailures( + tenantId: string, + failures: readonly ProjectionFailure[], + createdAt = new Date().toISOString(), + ): void { + this.#db + .transaction(() => { + for (const failure of failures) + this.#db.run( + "INSERT OR IGNORE INTO projection_failures (tenant_id, projection_id, projection_revision, occurrence_id, message, created_at) VALUES (?, ?, ?, ?, ?, ?)", + [ + tenantId, + failure.projectionId, + failure.projectionRevision, + failure.occurrenceId, + failure.message.slice(0, 4_096), + createdAt, + ], + ) + this.#db.run( + "DELETE FROM projection_failures WHERE tenant_id = ? AND sequence NOT IN (SELECT sequence FROM projection_failures WHERE tenant_id = ? ORDER BY sequence DESC LIMIT ?)", + [tenantId, tenantId, MAX_FAILURES_PER_TENANT], + ) + }) + .immediate() + } + + validate(): EventingControlSnapshotValidation { + return validateOpenDatabase(this.#db) + } + + async backupTo(path: string): Promise { + // sqlite3_serialize() snapshots the main database file. In WAL mode a + // committed transaction may still live only in the sidecar, so force and + // verify a complete checkpoint before copying the file image. + checkpointWal(this.#db) + const bytes = this.#db.serialize() + await durableWrite(path, bytes) + return LocalEventingControlStore.validateSnapshot(path) + } + + static validateSnapshot(path: string): EventingControlSnapshotValidation { + assertRealDatabaseFile(path) + if (!existsSync(path)) throw new Error(`eventing control snapshot is missing: ${path}`) + const uri = `${pathToFileURL(path).href}?immutable=1` + const db = new Database(uri, sqliteConstants.SQLITE_OPEN_READONLY | sqliteConstants.SQLITE_OPEN_URI) + try { + configure(db) + return validateOpenDatabase(db, [1, 2, 3, CONTROL_SCHEMA_VERSION]) + } finally { + db.close(true) + } + } + + static async restoreSnapshot(snapshotPath: string, dataDir: string): Promise { + LocalEventingControlStore.validateSnapshot(snapshotPath) + const stagingDataDir = mkdtempSync( + join(dirname(resolve(dataDir)), ".maple-eventing-control-restore-"), + ) + let restored: LocalEventingControlStore | undefined + try { + await durableWrite(eventingControlPath(stagingDataDir), readFileSync(snapshotPath)) + restored = await LocalEventingControlStore.open(stagingDataDir) + await restored.backupTo(eventingControlPath(dataDir)) + } finally { + restored?.close() + rmSync(stagingDataDir, { recursive: true, force: true }) + } + } +} diff --git a/apps/cli/src/server/eventing/otlp.ts b/apps/cli/src/server/eventing/otlp.ts new file mode 100644 index 000000000..95bf2232b --- /dev/null +++ b/apps/cli/src/server/eventing/otlp.ts @@ -0,0 +1,538 @@ +import { createHash } from "node:crypto" +import { + canonicalJson, + defineSignalFields, + type JsonValue, + type NormalizedSignal, + type SignalFieldCatalogEntry, + type SignalScalar, + type SignalSourceAdapter, + type SignalSourceDefinition, +} from "@maple/eventing-core" +import { OtlpFieldError, spanIdHex, traceIdHex } from "../otlp/encode" + +interface AnyValue { + readonly stringValue?: string + readonly boolValue?: boolean + readonly intValue?: string | number + readonly doubleValue?: number + readonly bytesValue?: string + readonly arrayValue?: { readonly values?: readonly AnyValue[] } + readonly kvlistValue?: { readonly values?: readonly KeyValue[] } +} + +interface KeyValue { + readonly key?: string + readonly value?: AnyValue +} + +interface OtlpLogRecord { + readonly timeUnixNano?: string | number + readonly observedTimeUnixNano?: string | number + readonly severityNumber?: number + readonly severityText?: string + readonly eventName?: string + readonly body?: AnyValue + readonly attributes?: readonly KeyValue[] + readonly traceId?: string + readonly spanId?: string +} + +interface OtlpLogsRequest { + readonly resourceLogs?: readonly { + readonly resource?: { readonly attributes?: readonly KeyValue[] } + readonly scopeLogs?: readonly { + readonly scope?: { + readonly name?: string + readonly version?: string + readonly attributes?: readonly KeyValue[] + } + readonly logRecords?: readonly OtlpLogRecord[] + }[] + }[] +} + +const MAX_ATTRIBUTES = 256 +const MAX_STRING_BYTES = 16 * 1024 +const MAX_DATA_BYTES = 256 * 1024 +const MAX_VALUE_DEPTH = 8 +const MAX_VALUE_NODES = 1_024 +const SENSITIVE_KEY = + /(?:^|[._-])(authorization|cookie|password|passwd|secret|token|api[._-]?key)(?:$|[._-])/i + +const allOperators = ["exists", "eq", "neq", "gt", "gte", "lt", "lte", "contains", "in"] as const +const equalityOperators = ["exists", "eq", "neq", "contains", "in"] as const + +const catalog = ( + key: string, + type: SignalScalar["type"], + operators: SignalFieldCatalogEntry["operators"] = allOperators, +): SignalFieldCatalogEntry => ({ + field: { namespace: "signal", key, type }, + operators, + sensitivity: "public", + replay: "exact", +}) + +export const OTLP_LOG_SOURCE: SignalSourceDefinition = { + sourceKind: "otel.log", + fields: [ + catalog("event.name", "string", equalityOperators), + catalog("severity.number", "int64"), + catalog("severity.text", "string", equalityOperators), + catalog("trace.id", "string", equalityOperators), + catalog("span.id", "string", equalityOperators), + catalog("time", "timestamp"), + catalog("observed_time", "timestamp"), + { + field: { namespace: "body", key: "value" }, + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + ], + openFields: [ + { + namespace: "resource", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + { + namespace: "scope", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + { + namespace: "attribute", + types: ["string", "boolean", "int64", "float64"], + operators: allOperators, + sensitivity: "public", + replay: "coerced", + }, + ], +} + +interface ValueBudget { + nodes: number +} + +const assertStringBound = (value: string, label: string): string => { + if (Buffer.byteLength(value, "utf8") > MAX_STRING_BYTES) + throw new OtlpFieldError(`${label} exceeds ${MAX_STRING_BYTES} UTF-8 bytes`) + return value +} + +const int64 = (value: string | number, label: string): string => { + if (typeof value === "number" && !Number.isSafeInteger(value)) + throw new OtlpFieldError( + `${label} must encode int64 as a decimal string when outside safe integer range`, + ) + const decimal = String(value) + if (!/^-?(?:0|[1-9][0-9]*)$/.test(decimal)) throw new OtlpFieldError(`${label} is not an int64`) + const parsed = BigInt(decimal) + if (parsed < -(1n << 63n) || parsed > (1n << 63n) - 1n) + throw new OtlpFieldError(`${label} is outside the int64 range`) + return decimal +} + +const anyValueScalar = (value: AnyValue | undefined, label: string): SignalScalar | null => { + if (!value) return null + if (value.stringValue !== undefined) + return { type: "string", value: assertStringBound(value.stringValue, label) } + if (value.boolValue !== undefined) return { type: "boolean", value: value.boolValue } + if (value.intValue !== undefined) return { type: "int64", value: int64(value.intValue, label) } + if (value.doubleValue !== undefined) { + if (!Number.isFinite(value.doubleValue)) throw new OtlpFieldError(`${label} must be finite`) + return { type: "float64", value: value.doubleValue } + } + return null +} + +const anyValueJson = ( + value: AnyValue | undefined, + label: string, + depth = 0, + budget: ValueBudget = { nodes: 0 }, +): JsonValue | null => { + budget.nodes += 1 + if (budget.nodes > MAX_VALUE_NODES) throw new OtlpFieldError(`${label} exceeds value node limit`) + if (depth > MAX_VALUE_DEPTH) throw new OtlpFieldError(`${label} exceeds value depth limit`) + const scalar = anyValueScalar(value, label) + if (scalar) return scalar.value + if (!value) return null + if (value.bytesValue !== undefined) return assertStringBound(value.bytesValue, `${label}.bytesValue`) + if (value.arrayValue !== undefined) + return (value.arrayValue.values ?? []).map((item, index) => + anyValueJson(item, `${label}[${index}]`, depth + 1, budget), + ) + if (value.kvlistValue !== undefined) { + const output: Record = Object.create(null) + for (const [index, entry] of (value.kvlistValue.values ?? []).entries()) { + const key = assertStringBound(entry.key ?? "", `${label}.key[${index}]`) + if (key.length === 0 || SENSITIVE_KEY.test(key)) continue + output[key] = anyValueJson(entry.value, `${label}.${key}`, depth + 1, budget) + } + return output + } + return null +} + +interface NormalizedAttributes { + readonly scalars: ReadonlyArray<{ readonly key: string; readonly value: SignalScalar }> + readonly data: Readonly> +} + +const attributes = (values: readonly KeyValue[] | undefined, label: string): NormalizedAttributes => { + if ((values?.length ?? 0) > MAX_ATTRIBUTES) + throw new OtlpFieldError(`${label} exceeds ${MAX_ATTRIBUTES} attributes`) + const scalars = new Map() + const data: Record = Object.create(null) + for (const [index, entry] of (values ?? []).entries()) { + const key = assertStringBound(entry.key ?? "", `${label}[${index}].key`) + if (key.length === 0 || SENSITIVE_KEY.test(key)) continue + const scalar = anyValueScalar(entry.value, `${label}.${key}`) + if (scalar) scalars.set(key, scalar) + data[key] = anyValueJson(entry.value, `${label}.${key}`) + } + return { scalars: [...scalars].map(([key, value]) => ({ key, value })), data } +} + +const epochNanos = (value: string | number | undefined): bigint | null => { + if (value === undefined || value === "" || value === 0 || value === "0") return null + try { + const parsed = BigInt(value) + return parsed >= 0 ? parsed : null + } catch { + return null + } +} + +const nanosToTimestamp = (nanos: bigint): string => { + const seconds = nanos / 1_000_000_000n + const fraction = nanos % 1_000_000_000n + const milliseconds = Number(seconds) * 1_000 + const date = new Date(milliseconds) + if (!Number.isFinite(milliseconds) || Number.isNaN(date.getTime())) + throw new OtlpFieldError("OTLP timestamp is outside the supported date range") + return `${date.toISOString().slice(0, 19)}.${fraction.toString().padStart(9, "0")}Z` +} + +const stringAttribute = (attrs: NormalizedAttributes, key: string): string | null => { + const scalar = attrs.scalars.find((entry) => entry.key === key)?.value + return scalar?.type === "string" ? scalar.value : null +} + +const boundedIdentity = (value: string, prefix: string): string => + value.length <= 256 + ? value + : `${prefix}:sha256:${createHash("sha256").update(value, "utf8").digest("hex")}` + +const sourceUri = (resource: NormalizedAttributes, record: NormalizedAttributes): string => { + const explicit = ( + stringAttribute(record, "event.source") ?? stringAttribute(record, "cloudevents.source") + )?.trim() + if (explicit) return boundedIdentity(assertStringBound(explicit, "event source"), "urn:maple:source") + const service = stringAttribute(resource, "service.name")?.trim() + const source = service + ? `urn:maple:source:otel:${encodeURIComponent(service)}` + : "urn:maple:source:otel:local" + return boundedIdentity(source, "urn:maple:source") +} + +const sourceOccurrenceId = (record: NormalizedAttributes): string | null => { + for (const key of ["event.id", "cloudevents.id"]) { + const value = stringAttribute(record, key)?.trim() + if (value) return boundedIdentity(value, "source") + } + return null +} + +export interface OtlpRecoveryIdentity { + readonly sourceKind: "otel.log" + readonly source: string + readonly tenantId: string + readonly occurrenceId: string + readonly occurredAt: string | null +} + +const recoveryStringAttribute = (values: readonly KeyValue[] | undefined, key: string): string | null => { + let value: string | null = null + for (const entry of values ?? []) { + if (entry.key !== key) continue + if (typeof entry.value?.stringValue === "string") value = entry.value.stringValue + } + return value +} + +const recoveryBoundedIdentity = (value: string, prefix: string): string | null => + Buffer.byteLength(value, "utf8") > MAX_STRING_BYTES ? null : boundedIdentity(value, prefix) + +const recoveryIdentity = ( + resourceAttributes: readonly KeyValue[] | undefined, + log: OtlpLogRecord, + tenantId: string, +): OtlpRecoveryIdentity | null => { + let occurrenceId: string | null = null + for (const key of ["event.id", "cloudevents.id"]) { + const value = recoveryStringAttribute(log.attributes, key)?.trim() + if (!value) continue + occurrenceId = recoveryBoundedIdentity(value, "source") + break + } + if (occurrenceId === null) return null + + const explicit = ( + recoveryStringAttribute(log.attributes, "event.source") ?? + recoveryStringAttribute(log.attributes, "cloudevents.source") + )?.trim() + let source: string | null + if (explicit) source = recoveryBoundedIdentity(explicit, "urn:maple:source") + else { + const service = recoveryStringAttribute(resourceAttributes, "service.name")?.trim() + source = recoveryBoundedIdentity( + service ? `urn:maple:source:otel:${encodeURIComponent(service)}` : "urn:maple:source:otel:local", + "urn:maple:source", + ) + } + if (source === null) return null + + const occurredNanos = epochNanos(log.timeUnixNano) ?? epochNanos(log.observedTimeUnixNano) + let occurredAt: string | null = null + if (occurredNanos !== null) + try { + occurredAt = nanosToTimestamp(occurredNanos) + } catch (error) { + if (!(error instanceof OtlpFieldError)) throw error + } + return { sourceKind: "otel.log", source, tenantId, occurrenceId, occurredAt } +} + +const derivedOccurrenceId = (input: JsonValue): string => + `derived:sha256:${createHash("sha256").update(canonicalJson(input)).digest("hex")}` + +const normalizeOtlpLogsStrict = ( + request: unknown, + _acceptedAt = new Date().toISOString(), + tenantId = "local", +): readonly NormalizedSignal[] => { + const input = (request ?? {}) as OtlpLogsRequest + const signals: NormalizedSignal[] = [] + for (const resourceLogs of input.resourceLogs ?? []) { + const resource = attributes(resourceLogs.resource?.attributes, "resource.attributes") + for (const scopeLogs of resourceLogs.scopeLogs ?? []) { + const scope = attributes(scopeLogs.scope?.attributes, "scope.attributes") + for (const log of scopeLogs.logRecords ?? []) { + const record = attributes(log.attributes, "log.attributes") + const occurredNanos = epochNanos(log.timeUnixNano) ?? epochNanos(log.observedTimeUnixNano) + // OTLP permits both timestamps to be absent or zero. Such records still + // belong in the warehouse, but cannot acquire a durable event identity. + if (occurredNanos === null) continue + const observedNanos = epochNanos(log.observedTimeUnixNano) + const occurredAt = nanosToTimestamp(occurredNanos) + const sourceObservedAt = observedNanos ? nanosToTimestamp(observedNanos) : occurredAt + const bodyScalar = anyValueScalar(log.body, "log.body") + const traceId = traceIdHex(log.traceId, "logRecord.traceId") + const spanId = spanIdHex(log.spanId, "logRecord.spanId") + const data: JsonValue = { + resource: resource.data, + scope: { + name: assertStringBound(scopeLogs.scope?.name ?? "", "scope.name"), + version: assertStringBound(scopeLogs.scope?.version ?? "", "scope.version"), + attributes: scope.data, + }, + record: { + eventName: assertStringBound(log.eventName ?? "", "log.eventName"), + severityNumber: log.severityNumber ?? 0, + severityText: assertStringBound(log.severityText ?? "", "log.severityText"), + traceId, + spanId, + body: anyValueJson(log.body, "log.body"), + attributes: record.data, + }, + } + if (Buffer.byteLength(canonicalJson(data), "utf8") > MAX_DATA_BYTES) + throw new OtlpFieldError(`normalized log event exceeds ${MAX_DATA_BYTES} UTF-8 bytes`) + const source = sourceUri(resource, record) + const occurrenceId = sourceOccurrenceId(record) + const subject = + stringAttribute(record, "event.subject") ?? stringAttribute(record, "cloudevents.subject") + signals.push({ + sourceKind: "otel.log", + source, + tenantId, + occurrenceId: + occurrenceId ?? + derivedOccurrenceId({ source, occurredAt, signalKind: "otel.log", data }), + identityQuality: occurrenceId === null ? "derived" : "source", + occurredAt, + observedAt: sourceObservedAt, + subject, + fields: defineSignalFields([ + ...(log.eventName + ? [ + { + field: { + namespace: "signal" as const, + key: "event.name", + type: "string" as const, + }, + value: { type: "string" as const, value: log.eventName }, + }, + ] + : []), + { + field: { namespace: "signal", key: "severity.number", type: "int64" }, + value: { + type: "int64", + value: int64(log.severityNumber ?? 0, "severity.number"), + }, + }, + ...(log.severityText + ? [ + { + field: { + namespace: "signal" as const, + key: "severity.text", + type: "string" as const, + }, + value: { type: "string" as const, value: log.severityText }, + }, + ] + : []), + ...(traceId + ? [ + { + field: { + namespace: "signal" as const, + key: "trace.id", + type: "string" as const, + }, + value: { type: "string" as const, value: traceId }, + }, + ] + : []), + ...(spanId + ? [ + { + field: { + namespace: "signal" as const, + key: "span.id", + type: "string" as const, + }, + value: { type: "string" as const, value: spanId }, + }, + ] + : []), + { + field: { namespace: "signal", key: "time", type: "timestamp" }, + value: { type: "timestamp", value: occurredAt }, + }, + { + field: { namespace: "signal", key: "observed_time", type: "timestamp" }, + value: { type: "timestamp", value: sourceObservedAt }, + }, + ...resource.scalars.map(({ key, value }) => ({ + field: { namespace: "resource" as const, key, type: value.type }, + value, + })), + ...scope.scalars.map(({ key, value }) => ({ + field: { namespace: "scope" as const, key, type: value.type }, + value, + })), + ...record.scalars.map(({ key, value }) => ({ + field: { namespace: "attribute" as const, key, type: value.type }, + value, + })), + ...(bodyScalar + ? [ + { + field: { + namespace: "body" as const, + key: "value", + type: bodyScalar.type, + }, + value: bodyScalar, + }, + ] + : []), + ]), + data, + }) + } + } + } + return signals +} + +export interface OtlpLogNormalizationResult { + readonly signals: readonly NormalizedSignal[] + readonly unprojectedIdentities: readonly OtlpRecoveryIdentity[] + readonly ineligible: number + readonly failures: number +} + +/** + * Event projection is an optional branch beside the established warehouse + * encoder. Projection-only bounds make one occurrence ineligible; they must + * not narrow the OTLP request contract or suppress unrelated warehouse rows. + * The warehouse encoder independently rejects fields that are invalid for + * both paths. + */ +export const normalizeOtlpLogsWithDiagnostics = ( + request: unknown, + acceptedAt = new Date().toISOString(), + tenantId = "local", +): OtlpLogNormalizationResult => { + const input = (request ?? {}) as OtlpLogsRequest + const signals: NormalizedSignal[] = [] + const unprojectedIdentities: OtlpRecoveryIdentity[] = [] + let ineligible = 0 + let failures = 0 + for (const resourceLogs of input.resourceLogs ?? []) + for (const scopeLogs of resourceLogs.scopeLogs ?? []) + for (const log of scopeLogs.logRecords ?? []) { + const identity = recoveryIdentity(resourceLogs.resource?.attributes, log, tenantId) + try { + const normalized = normalizeOtlpLogsStrict( + { + resourceLogs: [ + { + ...resourceLogs, + scopeLogs: [{ ...scopeLogs, logRecords: [log] }], + }, + ], + }, + acceptedAt, + tenantId, + ) + if (normalized.length === 0) { + ineligible += 1 + if (identity !== null) unprojectedIdentities.push(identity) + } else signals.push(...normalized) + } catch (error) { + if (!(error instanceof OtlpFieldError)) throw error + failures += 1 + if (identity !== null) unprojectedIdentities.push(identity) + } + } + return { signals, unprojectedIdentities, ineligible, failures } +} + +export const normalizeOtlpLogs = ( + request: unknown, + acceptedAt = new Date().toISOString(), + tenantId = "local", +): readonly NormalizedSignal[] => normalizeOtlpLogsWithDiagnostics(request, acceptedAt, tenantId).signals + +export const OTLP_LOG_ADAPTER: SignalSourceAdapter< + unknown, + { readonly acceptedAt: string; readonly tenantId: string } +> = { + definition: OTLP_LOG_SOURCE, + normalize: (raw, context) => normalizeOtlpLogs(raw, context.acceptedAt, context.tenantId), +} diff --git a/apps/cli/src/server/eventing/runtime.ts b/apps/cli/src/server/eventing/runtime.ts new file mode 100644 index 000000000..5cff06ecf --- /dev/null +++ b/apps/cli/src/server/eventing/runtime.ts @@ -0,0 +1,317 @@ +import { createHash } from "node:crypto" +import { + canonicalJson, + CompiledProjectionRegistry, + isJsonValue, + ProjectorRegistry, + SignalSourceRegistry, + assertSignalProjectionInputBudget, + SignalProjectionSpecSchema, + type MapleCloudEvent, + type JsonValue, + type NormalizedSignal, + type ProjectionFailure, + type SignalProjectionSpec, +} from "@maple/eventing-core" +import { Schema } from "effect" +import { LocalEventingControlStore } from "./control-store" +import type { EventConsumerStart } from "./control-store" +import { normalizeOtlpLogsWithDiagnostics, OTLP_LOG_ADAPTER, type OtlpRecoveryIdentity } from "./otlp" +import { NOOP_EVENTING_TELEMETRY, type EventingTelemetry } from "./telemetry" + +const TENANT_ID = "local" + +export interface LocalProjectionEvaluation { + readonly events: readonly MapleCloudEvent[] + readonly eventSourceFingerprints: ReadonlyMap + readonly recoveredEventIds: readonly string[] + readonly failures: readonly ProjectionFailure[] + readonly typeMismatchFields: readonly string[] +} + +export interface LocalProjectionActivation { + readonly spec: SignalProjectionSpec + readonly next: readonly SignalProjectionSpec[] + readonly compiled: CompiledProjectionRegistry + readonly generation: number +} + +const emptyEvaluation = (): LocalProjectionEvaluation => ({ + events: [], + eventSourceFingerprints: new Map(), + recoveredEventIds: [], + failures: [], + typeMismatchFields: [], +}) + +export const sourceOccurrenceFingerprint = (signal: NormalizedSignal): string => { + if (!isJsonValue(signal.data)) throw new Error("normalized source occurrence must contain finite JSON") + const content: JsonValue = { + sourceKind: signal.sourceKind, + source: signal.source, + tenantId: signal.tenantId, + occurrenceId: signal.occurrenceId, + identityQuality: signal.identityQuality, + occurredAt: signal.occurredAt, + observedAt: signal.observedAt, + subject: signal.subject, + fields: [...signal.fields.entries()] + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([key, value]) => ({ key, value })), + data: signal.data, + } + return `sha256:${createHash("sha256").update(canonicalJson(content)).digest("hex")}` +} + +const sourceOccurrenceKey = ( + occurrence: Pick, +): string | null => + occurrence.occurrenceId === null + ? null + : canonicalJson([ + occurrence.tenantId, + occurrence.sourceKind, + occurrence.source, + occurrence.occurrenceId, + ]) + +const recoveryIdentityKey = (identity: OtlpRecoveryIdentity): string => + canonicalJson([identity.tenantId, identity.sourceKind, identity.source, identity.occurrenceId]) + +export class LocalEventingRuntime { + readonly #store: LocalEventingControlStore + readonly #sources: SignalSourceRegistry + readonly #projectors: ProjectorRegistry + readonly #telemetry: EventingTelemetry + #compiled: CompiledProjectionRegistry + #activeSourceKinds = new Set() + #generation = 0 + + constructor( + store: LocalEventingControlStore, + telemetry: EventingTelemetry = NOOP_EVENTING_TELEMETRY, + projectors: ProjectorRegistry = new ProjectorRegistry(), + ) { + this.#store = store + this.#telemetry = telemetry + this.#sources = new SignalSourceRegistry().register(OTLP_LOG_ADAPTER.definition) + this.#projectors = projectors + const specs = store.loadEnabledProjections(TENANT_ID) + this.#compiled = CompiledProjectionRegistry.compile(specs, this.#sources, this.#projectors) + this.#activeSourceKinds = new Set(specs.map(({ sourceKind }) => sourceKind)) + } + + hasActiveSource(sourceKind: string): boolean { + return this.#activeSourceKinds.has(sourceKind) + } + + prepareActivation(candidate: unknown): LocalProjectionActivation { + assertSignalProjectionInputBudget(candidate) + const spec = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) + if (spec.tenantId !== TENANT_ID) + throw new Error(`Maple Local only accepts projections for tenant ${TENANT_ID}`) + const active = this.#store + .loadEnabledProjections(TENANT_ID) + .filter((candidate) => candidate.id !== spec.id) + const next = spec.enabled ? [...active, spec] : active + const compiled = CompiledProjectionRegistry.compile(next, this.#sources, this.#projectors) + return { spec, next, compiled, generation: this.#generation } + } + + commitActivation(activation: LocalProjectionActivation): void { + if (activation.generation !== this.#generation) + throw new Error("projection registry changed during activation; retry the request") + this.#store.saveProjection(activation.spec) + this.#compiled = activation.compiled + this.#activeSourceKinds = new Set(activation.next.map(({ sourceKind }) => sourceKind)) + this.#generation += 1 + } + + activate(candidate: unknown): void { + this.commitActivation(this.prepareActivation(candidate)) + } + + listActive(): readonly SignalProjectionSpec[] { + return this.#store.loadEnabledProjections(TENANT_ID) + } + + evaluateOtlp( + signal: "traces" | "logs" | "metrics", + decoded: unknown, + isRetiredUtcDay: (rangeDate: string) => boolean = () => false, + ): LocalProjectionEvaluation { + const sourceKind = signal === "logs" ? "otel.log" : signal === "traces" ? "otel.span" : "otel.metric" + if (!this.hasActiveSource(sourceKind) && !this.#store.hasStagedSourceKind(TENANT_ID, sourceKind)) + return emptyEvaluation() + const startedAt = performance.now() + const acceptedAt = new Date().toISOString() + let normalized + let unprojectedIdentities: readonly OtlpRecoveryIdentity[] + try { + const result = + signal === "logs" + ? normalizeOtlpLogsWithDiagnostics(decoded, acceptedAt, TENANT_ID) + : { signals: [], unprojectedIdentities: [], ineligible: 0, failures: 0 } + normalized = result.signals + unprojectedIdentities = result.unprojectedIdentities + this.#telemetry.record({ + operation: "normalization", + outcome: "success", + count: normalized.length, + durationMs: performance.now() - startedAt, + sourceKind, + }) + if (result.failures > 0) + this.#telemetry.record({ + operation: "normalization", + outcome: "failure", + count: result.failures, + sourceKind, + }) + } catch (error) { + this.#telemetry.record({ + operation: "normalization", + outcome: "failure", + durationMs: performance.now() - startedAt, + sourceKind, + }) + throw error + } + const sourceFingerprints = new Map() + for (const occurrence of normalized) { + const key = sourceOccurrenceKey(occurrence) + if (key === null) continue + const fingerprint = sourceOccurrenceFingerprint(occurrence) + const prior = sourceFingerprints.get(key) + if (prior !== undefined && prior !== fingerprint) + throw new Error( + `source occurrence collision within one ingest batch: ${occurrence.occurrenceId}`, + ) + sourceFingerprints.set(key, fingerprint) + } + for (const identity of unprojectedIdentities) { + if (sourceFingerprints.has(recoveryIdentityKey(identity))) + throw new Error( + `source occurrence collision with an unprojectable record within one ingest batch: ${identity.occurrenceId}`, + ) + if ( + this.#store.hasStagedSourceOccurrence( + identity.tenantId, + identity.sourceKind, + identity.source, + identity.occurrenceId, + ) + ) + throw new Error( + `cannot safely recover staged source occurrence after projection normalization failed: ${identity.occurrenceId}`, + ) + } + const snapshot = this.#compiled + const events: MapleCloudEvent[] = [] + const eventSourceFingerprints = new Map() + const recoveredEventIds: string[] = [] + const failures: ProjectionFailure[] = [] + const typeMismatchFields = new Set() + for (const occurrence of normalized) { + const sourceFingerprint = sourceOccurrenceFingerprint(occurrence) + if (occurrence.occurrenceId !== null) { + const staged = this.#store.stagedEventIdsForOccurrence( + occurrence.tenantId, + occurrence.sourceKind, + occurrence.source, + occurrence.occurrenceId, + sourceFingerprint, + ) + if (staged.length > 0) { + recoveredEventIds.push(...staged) + continue + } + } + if (isRetiredUtcDay(occurrence.occurredAt.slice(0, 10))) continue + const result = snapshot.evaluate(occurrence, acceptedAt) + this.#telemetry.record({ + operation: "projection", + outcome: "success", + count: result.events.length, + sourceKind, + }) + this.#telemetry.record({ + operation: "projection", + outcome: "failure", + count: result.failures.length, + sourceKind, + }) + events.push(...result.events) + for (const event of result.events) { + const priorFingerprint = eventSourceFingerprints.get(event.id) + if (priorFingerprint !== undefined && priorFingerprint !== sourceFingerprint) + throw new Error(`source occurrence collision within one ingest batch: ${event.id}`) + eventSourceFingerprints.set(event.id, sourceFingerprint) + } + failures.push(...result.failures) + for (const mismatch of result.typeMismatchFields) typeMismatchFields.add(mismatch) + } + if (typeMismatchFields.size > 0) + this.#telemetry.record({ + operation: "selector_type_mismatch", + outcome: "observed", + count: typeMismatchFields.size, + sourceKind, + }) + return { + events, + eventSourceFingerprints, + recoveredEventIds, + failures, + typeMismatchFields: [...typeMismatchFields], + } + } + + persistFailures(failures: readonly ProjectionFailure[]): void { + if (failures.length > 0) this.#store.recordProjectionFailures(TENANT_ID, failures) + } + + stage(events: readonly MapleCloudEvent[], sourceFingerprints: ReadonlyMap = new Map()) { + return this.#store.stageEvents(events, sourceFingerprints) + } + + markReady(eventIds: readonly string[]): void { + this.#store.markReady(eventIds) + } + + listReady(limit?: number, after?: number) { + return this.#store.listReady(limit, after) + } + + listStaged(limit?: number, after?: number) { + return this.#store.listStaged(limit, after) + } + + listConsumers() { + return this.#store.listConsumers(TENANT_ID) + } + + registerConsumer(consumerId: string, startAt: EventConsumerStart) { + return this.#store.registerConsumer(TENANT_ID, consumerId, startAt) + } + + disableConsumer(consumerId: string) { + return this.#store.disableConsumer(TENANT_ID, consumerId) + } + + claimReady(consumerId: string, limit: number, leaseSeconds: number) { + return this.#store.claimReady(TENANT_ID, consumerId, limit, leaseSeconds) + } + + acknowledgeClaim(consumerId: string, leaseToken: string, throughSequence: number) { + return this.#store.acknowledgeClaim(TENANT_ID, consumerId, leaseToken, throughSequence) + } + + health() { + return { + activeProjections: this.listActive().length, + outboxCapacity: this.#store.outboxCapacity(), + ...this.#store.validate(), + } + } +} diff --git a/apps/cli/src/server/eventing/telemetry.ts b/apps/cli/src/server/eventing/telemetry.ts new file mode 100644 index 000000000..2885e27ad --- /dev/null +++ b/apps/cli/src/server/eventing/telemetry.ts @@ -0,0 +1,81 @@ +import { Effect, Metric } from "effect" + +export type EventingTelemetryOperation = + | "normalization" + | "projection" + | "selector_type_mismatch" + | "outbox_stage" + | "outbox_ready" + | "outbox_dedup" + | "consumer_claim" + | "consumer_ack" + | "consumer_lease" + | "consumer_lag" + +export type EventingTelemetryOutcome = + | "success" + | "failure" + | "empty" + | "active" + | "expired" + | "reclaimed" + | "observed" + +export type EventingTelemetrySourceKind = "otel.log" | "otel.span" | "otel.metric" | "unknown" + +/** Deliberately excludes tenant, consumer, event, projection, payload, and credential values. */ +export interface EventingTelemetryObservation { + readonly operation: EventingTelemetryOperation + readonly outcome: EventingTelemetryOutcome + readonly count?: number + readonly durationMs?: number + readonly lag?: number + readonly sourceKind?: EventingTelemetrySourceKind +} + +export interface EventingTelemetry { + record(observation: EventingTelemetryObservation): void +} + +export const NOOP_EVENTING_TELEMETRY: EventingTelemetry = { record: () => {} } + +const operations = Metric.counter("maple.eventing.operations_total", { + description: "Eventing operations by bounded operation and outcome", + incremental: true, +}) +const durations = Metric.histogram("maple.eventing.operation_duration_ms", { + description: "Eventing operation duration in milliseconds", + boundaries: [0.1, 0.5, 1, 5, 10, 50, 100, 500, 1_000, 5_000], +}) +const consumerLag = Metric.histogram("maple.eventing.consumer_lag_events", { + description: "Ready-event sequence lag observed by event consumers", + boundaries: [0, 1, 5, 10, 50, 100, 500, 1_000, 10_000], +}) + +export const makeEffectEventingTelemetry = ( + run: (effect: Effect.Effect) => void, +): EventingTelemetry => ({ + record(observation) { + const attributes = { + operation: observation.operation, + outcome: observation.outcome, + source_kind: observation.sourceKind ?? "unknown", + } + const effects: Effect.Effect[] = [] + const count = observation.count ?? 1 + if (Number.isFinite(count) && count > 0) + effects.push(Metric.update(Metric.withAttributes(operations, attributes), count)) + if (observation.durationMs !== undefined && Number.isFinite(observation.durationMs)) + effects.push( + Metric.update( + Metric.withAttributes(durations, attributes), + Math.max(0, observation.durationMs), + ), + ) + if (observation.lag !== undefined && Number.isSafeInteger(observation.lag)) + effects.push( + Metric.update(Metric.withAttributes(consumerLag, attributes), Math.max(0, observation.lag)), + ) + if (effects.length > 0) run(Effect.all(effects, { discard: true })) + }, +}) diff --git a/apps/cli/src/server/serve.ts b/apps/cli/src/server/serve.ts index 8f142cc7d..727d07727 100644 --- a/apps/cli/src/server/serve.ts +++ b/apps/cli/src/server/serve.ts @@ -18,6 +18,16 @@ import { rawTelemetryTtlStatements, } from "./chdb" import { buildInsertStatements } from "./inserts" +import { + eventingControlSnapshotPath, + EventConsumerConflictError, + EventConsumerInputError, + EventConsumerNotFoundError, + LocalEventingControlStore, +} from "./eventing/control-store" +import { ensureEventConsumerToken, eventConsumerTokenMatches } from "./eventing/consumer-auth" +import { LocalEventingRuntime } from "./eventing/runtime" +import { makeEffectEventingTelemetry } from "./eventing/telemetry" import { encodeLogs, encodeMetrics, encodeTraces, type EncodedBatch, OtlpFieldError } from "./otlp/encode" import { decodeLogsRequest, @@ -110,7 +120,8 @@ export const corsHeadersForAllowedOrigin = ( // `x-maple-sdk` is the SDK identity hint every browser SDK sends on // every request; a listener that does not allow it fails preflight // for the whole SDK. - "access-control-allow-headers": "content-type, content-encoding, authorization, x-maple-sdk", + "access-control-allow-headers": + "content-type, content-encoding, authorization, x-maple-sdk, x-maple-maintenance-token", "access-control-allow-private-network": "true", vary: "Origin", } @@ -217,6 +228,7 @@ interface IngestResult { async function ingest( db: Chdb, authority: RetiredDayAuthority, + eventing: LocalEventingRuntime, signal: Signal, req: Request, ): Promise { @@ -246,6 +258,17 @@ async function ingest( requestBytes, } } + let evaluation: ReturnType + try { + evaluation = eventing.evaluateOtlp(signal, decoded, (rangeDate) => authority.isRetired(rangeDate)) + } catch (error) { + const status = error instanceof OtlpFieldError ? 400 : 503 + return { + response: text(`event projection ${signal}: ${(error as Error).message}`, status), + accepted: 0, + requestBytes, + } + } let batches: EncodedBatch[] try { batches = encodeFor(signal, decoded) @@ -260,6 +283,19 @@ async function ingest( requestBytes, } } + let stagedEventIds: readonly string[] = [] + try { + eventing.persistFailures(evaluation.failures) + if (evaluation.events.length > 0) + stagedEventIds = eventing.stage(evaluation.events, evaluation.eventSourceFingerprints).eventIds + } catch (error) { + const status = error instanceof OtlpFieldError ? 400 : 503 + return { + response: text(`event projection ${signal}: ${(error as Error).message}`, status), + accepted: 0, + requestBytes, + } + } let rejected = 0 batches = batches.map((batch) => { const filtered = authority.filterBatch(batch.datasource, batch.ndjson) @@ -282,6 +318,16 @@ async function ingest( accepted += statement.rowCount } } + try { + const readyEventIds = [...evaluation.recoveredEventIds, ...stagedEventIds] + if (readyEventIds.length > 0) eventing.markReady(readyEventIds) + } catch (error) { + return { + response: text(`event outbox readiness ${signal}: ${(error as Error).message}`, 503), + accepted, + requestBytes, + } + } const errorMessage = rejected > 0 ? "telemetry from permanently retired UTC days was rejected" : "" if (contentType.includes("json")) { const rejectedField = @@ -447,6 +493,7 @@ const ingestSpan = ( runSpan: SpanRunner, db: Chdb, authority: RetiredDayAuthority, + eventing: LocalEventingRuntime, signal: Signal, req: Request, ): Promise => @@ -459,7 +506,7 @@ const ingestSpan = ( // catch — so it escaped as an untyped, unlabelled span error instead of // the 500 the caller should have received. const { response, accepted, requestBytes } = yield* Effect.tryPromise({ - try: () => ingest(db, authority, signal, req), + try: () => ingest(db, authority, eventing, signal, req), catch: (error): IngestFailed => new IngestFailed({ message: describeThrown(error) }), }).pipe( Effect.catchTag("@maple/cli/IngestFailed", (error) => @@ -543,7 +590,7 @@ export class RequestQuiescenceGate { } async exclusive(work: () => Promise): Promise { - if (this.#closed) throw new Error("another server maintenance operation is active") + if (this.#closed) throw new MaintenanceInProgressError() this.#closed = true try { if (this.#active > 0) await new Promise((resolve) => this.#drained.push(resolve)) @@ -554,6 +601,57 @@ export class RequestQuiescenceGate { } } +class MaintenanceInProgressError extends Error { + constructor() { + super("another server maintenance operation is active") + this.name = "MaintenanceInProgressError" + } +} + +class RequestBodyTooLargeError extends Error { + constructor(readonly maximumBytes: number) { + super(`request body exceeds ${maximumBytes} bytes`) + this.name = "RequestBodyTooLargeError" + } +} + +const readBoundedJson = async (req: Request, maximumBytes: number): Promise => { + const contentLength = req.headers.get("content-length") + if (contentLength !== null && /^[0-9]+$/.test(contentLength)) { + const declared = Number(contentLength) + if (!Number.isSafeInteger(declared) || declared > maximumBytes) + throw new RequestBodyTooLargeError(maximumBytes) + } + if (req.body === null) return JSON.parse("") as unknown + const reader = req.body.getReader() + const chunks: Uint8Array[] = [] + let total = 0 + try { + while (true) { + const { done, value } = await reader.read() + if (done) break + total += value.byteLength + if (total > maximumBytes) { + await reader.cancel() + throw new RequestBodyTooLargeError(maximumBytes) + } + chunks.push(value) + } + } finally { + reader.releaseLock() + } + const bytes = new Uint8Array(total) + let offset = 0 + for (const chunk of chunks) { + bytes.set(chunk, offset) + offset += chunk.byteLength + } + return JSON.parse(new TextDecoder().decode(bytes)) as unknown +} + +const invalidJsonResponse = (error: unknown): Response => + error instanceof RequestBodyTooLargeError ? text(error.message, 413) : text("invalid JSON body", 400) + const admitted = async (gate: RequestQuiescenceGate, work: () => Promise): Promise => { const leave = gate.enter() if (!leave) return text("server maintenance in progress", 503) @@ -606,16 +704,26 @@ const handleRetirement = async ( } const CHECKPOINT_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i +const MAX_CHECKPOINT_BODY_BYTES = 4 * 1024 +const MAX_PROJECTION_BODY_BYTES = 512 * 1024 +const MAX_CONSUMER_BODY_BYTES = 16 * 1024 /** Typed, authenticated replacement for sending BACKUP through /local/query. */ -const handleCheckpointBackup = async (db: Chdb, token: string, req: Request): Promise => { +const handleCheckpointBackup = async ( + db: Chdb, + controlStore: LocalEventingControlStore, + dataDir: string, + gate: RequestQuiescenceGate, + token: string, + req: Request, +): Promise => { if (!maintenanceTokenMatches(token, req.headers.get("x-maple-maintenance-token"))) return text("maintenance authorization required", 403) let body: unknown try { - body = await req.json() - } catch { - return text("invalid JSON body", 400) + body = await readBoundedJson(req, MAX_CHECKPOINT_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) } if (!Predicate.isObject(body)) return text("invalid body", 400) const record = body @@ -623,11 +731,14 @@ const handleCheckpointBackup = async (db: Chdb, token: string, req: Request): Pr return text("invalid checkpoint fields", 400) if (!CHECKPOINT_ID.test(record.checkpointId)) return text("invalid checkpoint ID", 400) try { - db.exec( - `BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${record.checkpointId.toLowerCase()}/backup')`, - ) - return json({ checkpointId: record.checkpointId.toLowerCase() }) + const checkpointId = record.checkpointId.toLowerCase() + return await gate.exclusive(async () => { + const control = await controlStore.backupTo(eventingControlSnapshotPath(dataDir, checkpointId)) + db.exec(`BACKUP DATABASE default TO Disk('default', 'backups/snapshots/${checkpointId}/backup')`) + return json({ checkpointId, control }) + }) } catch (error) { + if (error instanceof MaintenanceInProgressError) return text(error.message, 409) return text( `checkpoint backup failed: ${error instanceof Error ? error.message : String(error)}`, 400, @@ -635,6 +746,218 @@ const handleCheckpointBackup = async (db: Chdb, token: string, req: Request): Pr } } +const eventingAuthorized = (token: string, req: Request): Response | null => + maintenanceTokenMatches(token, req.headers.get("x-maple-maintenance-token")) + ? null + : text("maintenance authorization required", 403) + +const handleProjectionActivation = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + token: string, + req: Request, +): Promise => { + const unauthorized = eventingAuthorized(token, req) + if (unauthorized) return unauthorized + let body: unknown + try { + body = await readBoundedJson(req, MAX_PROJECTION_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + let activation + try { + // Recursive schema validation and full registry compilation happen while + // normal ingest/query admission remains open. + activation = eventing.prepareActivation(body) + } catch (error) { + return text( + `invalid event projection: ${error instanceof Error ? error.message : String(error)}`, + 400, + ) + } + try { + await gate.exclusive(async () => eventing.commitActivation(activation)) + return json({ active: eventing.listActive() }) + } catch (error) { + if (error instanceof MaintenanceInProgressError) return text(error.message, 409) + return text( + `invalid event projection: ${error instanceof Error ? error.message : String(error)}`, + 400, + ) + } +} + +const eventConsumerErrorResponse = (error: unknown): Response => { + const message = error instanceof Error ? error.message : String(error) + if (error instanceof EventConsumerNotFoundError) return text(message, 404) + if (error instanceof EventConsumerConflictError) return text(message, 409) + if (error instanceof EventConsumerInputError) return text(message, 400) + return text(`event consumer operation failed: ${message}`, 500) +} + +const isRequestRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const handleConsumerRegistration = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + maintenanceToken: string, + req: Request, +): Promise => { + const unauthorized = eventingAuthorized(maintenanceToken, req) + if (unauthorized) return unauthorized + let body: unknown + try { + body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + if (!isRequestRecord(body)) return text("invalid body", 400) + const record = body + const consumerId = record.consumerId + const startAt = record.startAt + if ( + Object.keys(record).sort().join(",") !== "consumerId,startAt" || + !Schema.is(Schema.String)(consumerId) || + (startAt !== "beginning" && startAt !== "latest") + ) + return text("invalid event consumer registration fields", 400) + return admitted(gate, async () => { + try { + return json(eventing.registerConsumer(consumerId, startAt), 201) + } catch (error) { + return eventConsumerErrorResponse(error) + } + }) +} + +const handleConsumerDisable = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + maintenanceToken: string, + req: Request, +): Promise => { + const unauthorized = eventingAuthorized(maintenanceToken, req) + if (unauthorized) return unauthorized + let body: unknown + try { + body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + if (!isRequestRecord(body)) return text("invalid body", 400) + const record = body + const consumerId = record.consumerId + if (Object.keys(record).join(",") !== "consumerId" || !Schema.is(Schema.String)(consumerId)) + return text("invalid event consumer disable fields", 400) + return admitted(gate, async () => { + try { + return json(eventing.disableConsumer(consumerId)) + } catch (error) { + return eventConsumerErrorResponse(error) + } + }) +} + +const handleConsumerClaim = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + consumerToken: string, + req: Request, +): Promise => { + if (!eventConsumerTokenMatches(consumerToken, req.headers.get("x-maple-event-consumer-token"))) + return text("event consumer authorization required", 403) + let body: unknown + try { + body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + if (!isRequestRecord(body)) return text("invalid body", 400) + const record = body + const consumerId = record.consumerId + const limit = record.limit + const leaseSeconds = record.leaseSeconds + if ( + Object.keys(record).sort().join(",") !== "consumerId,leaseSeconds,limit" || + !Schema.is(Schema.String)(consumerId) || + !Schema.is(Schema.Number)(limit) || + !Schema.is(Schema.Number)(leaseSeconds) + ) + return text("invalid event consumer claim fields", 400) + return admitted(gate, async () => { + try { + return json(eventing.claimReady(consumerId, limit, leaseSeconds)) + } catch (error) { + return eventConsumerErrorResponse(error) + } + }) +} + +const handleConsumerAcknowledgement = async ( + eventing: LocalEventingRuntime, + gate: RequestQuiescenceGate, + consumerToken: string, + req: Request, +): Promise => { + if (!eventConsumerTokenMatches(consumerToken, req.headers.get("x-maple-event-consumer-token"))) + return text("event consumer authorization required", 403) + let body: unknown + try { + body = await readBoundedJson(req, MAX_CONSUMER_BODY_BYTES) + } catch (error) { + return invalidJsonResponse(error) + } + if (!isRequestRecord(body)) return text("invalid body", 400) + const record = body + const consumerId = record.consumerId + const leaseToken = record.leaseToken + const throughSequence = record.throughSequence + if ( + Object.keys(record).sort().join(",") !== "consumerId,leaseToken,throughSequence" || + !Schema.is(Schema.String)(consumerId) || + !Schema.is(Schema.String)(leaseToken) || + !Schema.is(Schema.Number)(throughSequence) + ) + return text("invalid event consumer acknowledgement fields", 400) + return admitted(gate, async () => { + try { + return json(eventing.acknowledgeClaim(consumerId, leaseToken, throughSequence)) + } catch (error) { + return eventConsumerErrorResponse(error) + } + }) +} + +const handleEventingRead = ( + eventing: LocalEventingRuntime, + token: string, + req: Request, + url: URL, +): Response => { + const unauthorized = eventingAuthorized(token, req) + if (unauthorized) return unauthorized + if (url.pathname === "/local/eventing/health") return json(eventing.health()) + if (url.pathname === "/local/eventing/projections") return json(eventing.listActive()) + if (url.pathname === "/local/eventing/consumers") return json(eventing.listConsumers()) + if (url.pathname === "/local/eventing/outbox") { + const rawLimit = url.searchParams.get("limit") + const limit = rawLimit === null ? 100 : Number(rawLimit) + const rawAfter = url.searchParams.get("after") + const after = rawAfter === null ? 0 : Number(rawAfter) + const state = url.searchParams.get("state") ?? "ready" + try { + if (state === "ready") return json(eventing.listReady(limit, after)) + if (state === "staged") return json(eventing.listStaged(limit, after)) + return text("outbox state must be ready or staged", 400) + } catch (error) { + return text(error instanceof Error ? error.message : String(error), 400) + } + } + return text("not found", 404) +} + /** The `Bun.serve` fetch handler, closed over the chDB connection. Each ingest * and query request is run through `runSpan` so it leaves a trace; `/health` * and `OPTIONS` are skipped (loop-prevention convention — no health-check noise). */ @@ -646,6 +969,9 @@ const makeFetch = authority: RetiredDayAuthority, gate: RequestQuiescenceGate, maintenanceToken: string, + consumerToken: string, + controlStore: LocalEventingControlStore, + eventing: LocalEventingRuntime, ) => async (req: Request): Promise => { const url = new URL(req.url) @@ -659,18 +985,45 @@ const makeFetch = if (url.pathname === "/health") return respond(text("OK")) if (req.method === "POST") { if (url.pathname === "/v1/traces") - return respond(await admitted(gate, () => ingestSpan(runSpan, db, authority, "traces", req))) + return respond( + await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "traces", req)), + ) if (url.pathname === "/v1/logs") - return respond(await admitted(gate, () => ingestSpan(runSpan, db, authority, "logs", req))) + return respond( + await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "logs", req)), + ) if (url.pathname === "/v1/metrics") - return respond(await admitted(gate, () => ingestSpan(runSpan, db, authority, "metrics", req))) + return respond( + await admitted(gate, () => ingestSpan(runSpan, db, authority, eventing, "metrics", req)), + ) if (url.pathname === "/local/query") return respond(await admitted(gate, () => querySpan(runSpan, db, authority, req))) if (url.pathname === "/local/checkpoint/backup") - return respond(await admitted(gate, () => handleCheckpointBackup(db, maintenanceToken, req))) + return respond( + await handleCheckpointBackup( + db, + controlStore, + options.dataDir, + gate, + maintenanceToken, + req, + ), + ) + if (url.pathname === "/local/eventing/projections") + return respond(await handleProjectionActivation(eventing, gate, maintenanceToken, req)) + if (url.pathname === "/local/eventing/consumers") + return respond(await handleConsumerRegistration(eventing, gate, maintenanceToken, req)) + if (url.pathname === "/local/eventing/consumers/disable") + return respond(await handleConsumerDisable(eventing, gate, maintenanceToken, req)) + if (url.pathname === "/local/eventing/claims") + return respond(await handleConsumerClaim(eventing, gate, consumerToken, req)) + if (url.pathname === "/local/eventing/acks") + return respond(await handleConsumerAcknowledgement(eventing, gate, consumerToken, req)) if (url.pathname === "/local/retention/retire") return respond(await handleRetirement(db, authority, gate, maintenanceToken, req)) } + if (req.method === "GET" && url.pathname.startsWith("/local/eventing/")) + return respond(handleEventingRead(eventing, maintenanceToken, req, url)) if (req.method === "GET" && options.assets) return respond(serveAsset(options.assets, url.pathname)) return respond(text("not found", 404)) } @@ -706,6 +1059,32 @@ export const startServer = ( configFile: options.configFile, rawTelemetryRetentionDays: retention.effective, }) + // The request handler and synchronous eventing store share one telemetry + // runtime; eventing observations contain only bounded operation labels. + const telemetry = yield* Effect.acquireRelease( + Effect.sync(() => ManagedRuntime.make(TelemetryLayer)), + (rt) => Effect.promise(() => rt.dispose()), + ) + const eventingTelemetry = makeEffectEventingTelemetry((effect) => { + telemetry.runFork(effect) + }) + const controlStore = yield* Effect.acquireRelease( + Effect.tryPromise({ + try: () => LocalEventingControlStore.open(options.dataDir, undefined, eventingTelemetry), + catch: (error) => + new ChdbError({ + message: `failed to open local eventing control store: ${error instanceof Error ? error.message : String(error)}`, + }), + }), + (store) => Effect.sync(() => store.close()), + ) + const eventing = yield* Effect.try({ + try: () => new LocalEventingRuntime(controlStore, eventingTelemetry), + catch: (error) => + new ChdbError({ + message: `failed to compile local event projections: ${error instanceof Error ? error.message : String(error)}`, + }), + }) // `CREATE ... IF NOT EXISTS` does not repair a table whose physical // definition was altered out of band. Inspect the opened store before the // listener is bound; a mismatch fails startup rather than allowing new @@ -764,15 +1143,14 @@ export const startServer = ( message: `failed to load maintenance token: ${error instanceof Error ? error.message : String(error)}`, }), }) + const consumerToken = yield* Effect.tryPromise({ + try: () => ensureEventConsumerToken(options.dataDir), + catch: (error) => + new ChdbError({ + message: `failed to load event consumer token: ${error instanceof Error ? error.message : String(error)}`, + }), + }) const gate = new RequestQuiescenceGate() - // A dedicated runtime carrying the OTel tracer for per-request spans: the - // Bun.serve handler runs outside Effect, so each request's span effect is - // run through this runtime. Disposed on scope close, which flushes any - // pending spans (bounded by the layer's shutdownTimeout). - const telemetry = yield* Effect.acquireRelease( - Effect.sync(() => ManagedRuntime.make(TelemetryLayer)), - (rt) => Effect.promise(() => rt.dispose()), - ) const runSpan: SpanRunner = (effect) => telemetry.runPromise(effect) const server = yield* Effect.acquireRelease( Effect.try({ @@ -780,7 +1158,17 @@ export const startServer = ( Bun.serve({ port: options.port, hostname: options.hostname, - fetch: makeFetch(db, options, runSpan, authority, gate, maintenanceToken), + fetch: makeFetch( + db, + options, + runSpan, + authority, + gate, + maintenanceToken, + consumerToken, + controlStore, + eventing, + ), }), catch: (error) => new ServerBindError({ @@ -794,4 +1182,16 @@ export const startServer = ( return { port: server.port ?? options.port } }) -export const __testables = { recordServerResponse } +export const __testables = { + handleConsumerAcknowledgement, + handleConsumerClaim, + handleConsumerDisable, + handleConsumerRegistration, + handleCheckpointBackup, + handleEventingRead, + handleProjectionActivation, + ingest, + readBoundedJson, + recordServerResponse, + RequestQuiescenceGate, +} diff --git a/apps/cli/test/checkpoints.test.ts b/apps/cli/test/checkpoints.test.ts index 42d74dff1..d045e20af 100644 --- a/apps/cli/test/checkpoints.test.ts +++ b/apps/cli/test/checkpoints.test.ts @@ -1,5 +1,6 @@ // BOUNDARY: Test doubles preserve opaque values so the consuming boundary can be exercised. import { describe, it } from "@effect/vitest" +import { createHash } from "node:crypto" import { Effect, Exit, Option } from "effect" import { deepStrictEqual, match, ok, rejects, strictEqual, throws } from "node:assert" import { @@ -56,6 +57,7 @@ import { import { SCHEMA_FINGERPRINT } from "../src/server/schema-identity" import { storeMarkerPath, storeOpenMarkerPath } from "../src/server/store-version" import { CHDB_VERSION, MAPLE_VERSION } from "../src/version" +import { eventingControlSnapshotPath, LocalEventingControlStore } from "../src/server/eventing/control-store" const withDataDir = async (run: (dataDir: string) => Promise | void): Promise => { const parent = mkdtempSync(join(tmpdir(), "maple-checkpoint-test-")) @@ -308,6 +310,41 @@ describe("checkpoint IDs and strict parsers", () => { }) describe("checkpoint state resolution", () => { + it("binds a version-2 checkpoint to its eventing control snapshot", async () => { + await withDataDir(async (dataDir) => { + const checkpointId = newCheckpointId() + const operationId = newCheckpointOperationId() + const snapshot = checkpointSnapshotDir(dataDir, checkpointId) + mkdirSync(join(snapshot, "backup"), { recursive: true }) + writeFileSync(join(snapshot, "backup", "data.bin"), "backup") + + const store = await LocalEventingControlStore.open(dataDir) + const controlPath = eventingControlSnapshotPath(dataDir, checkpointId) + const controlValidation = await store.backupTo(controlPath) + store.close() + const controlBytes = readFileSync(controlPath) + writeFileSync( + join(snapshot, "manifest.json"), + `${JSON.stringify({ + ...manifest(checkpointId, operationId, dataDir), + formatVersion: 2, + backupBytes: 6, + controlRelativePath: `snapshots/${checkpointId}/control.sqlite`, + controlBytes: controlBytes.byteLength, + controlSha256: createHash("sha256").update(controlBytes).digest("hex"), + controlValidation, + })}\n`, + ) + writeState(dataDir, checkpointId) + strictEqual((await resolveCheckpoint(dataDir)).manifest.formatVersion, 2) + + const corrupted = Buffer.from(controlBytes) + corrupted[corrupted.length - 1] ^= 1 + writeFileSync(controlPath, corrupted) + await rejects(resolveCheckpoint(dataDir), /digest mismatch|quick_check failed/) + }) + }) + it("resolves immutable current, previous, and explicit IDs", async () => { await withDataDir(async (dataDir) => { const current = newCheckpointId() @@ -691,9 +728,11 @@ describe("live-store reset safety", () => { writeSnapshot(dataDir, checkpointId) writeState(dataDir, checkpointId) mkdirSync(join(dataDir, "store"), { recursive: true }) + mkdirSync(join(dataDir, "control"), { recursive: true }) mkdirSync(join(dataDir, "metadata"), { recursive: true }) mkdirSync(join(dataDir, "tmp"), { recursive: true }) writeFileSync(join(dataDir, "store", "part.bin"), "live") + writeFileSync(join(dataDir, "control", "eventing.sqlite"), "live") writeFileSync(join(dataDir, "metadata", "table.sql"), "live") writeFileSync(join(dataDir, "status"), "live") writeFileSync(join(dataDir, "tmp", "scratch.bin"), "live") @@ -705,6 +744,7 @@ describe("live-store reset safety", () => { strictEqual((await readCheckpointState(dataDir)).current, checkpointId) ok(existsSync(checkpointSnapshotDir(dataDir, checkpointId))) ok(!existsSync(join(dataDir, "store"))) + ok(!existsSync(join(dataDir, "control"))) ok(!existsSync(join(dataDir, "metadata"))) ok(!existsSync(join(dataDir, "status"))) ok(!existsSync(join(dataDir, "tmp"))) @@ -760,7 +800,7 @@ describe("live-store reset safety", () => { const checkpointId = newCheckpointId() writeSnapshot(dataDir, checkpointId) writeState(dataDir, checkpointId) - for (const entry of ["data", "metadata", "store", "tmp"]) { + for (const entry of ["control", "data", "metadata", "store", "tmp"]) { mkdirSync(join(dataDir, entry), { recursive: true }) writeFileSync(join(dataDir, entry, "live.bin"), "live") } @@ -782,7 +822,7 @@ describe("live-store reset safety", () => { ) await Effect.runPromise(reconcileCheckpointRecovery(dataDir)) - for (const entry of ["data", "metadata", "status", "store", "tmp"]) { + for (const entry of ["control", "data", "metadata", "status", "store", "tmp"]) { ok(!existsSync(join(dataDir, entry)), `${boundary}: ${entry}`) } strictEqual((await readCheckpointState(dataDir)).current, checkpointId) diff --git a/apps/cli/test/local-eventing-consumer-auth.test.ts b/apps/cli/test/local-eventing-consumer-auth.test.ts new file mode 100644 index 000000000..6e264bdcd --- /dev/null +++ b/apps/cli/test/local-eventing-consumer-auth.test.ts @@ -0,0 +1,48 @@ +import { strictEqual } from "node:assert" +import { mkdirSync, mkdtempSync, rmSync, statSync, symlinkSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "vitest" +import { + ensureEventConsumerToken, + eventConsumerTokenMatches, + eventConsumerTokenPath, +} from "../src/server/eventing/consumer-auth" + +describe("local event consumer authorization", () => { + it("creates a stable private token separate from the data directory", async () => { + const parent = mkdtempSync(join(tmpdir(), "maple-event-consumer-auth-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir) + try { + const first = await ensureEventConsumerToken(dataDir) + const second = await ensureEventConsumerToken(dataDir) + strictEqual(first.length, 64) + strictEqual(second, first) + strictEqual(statSync(eventConsumerTokenPath(dataDir)).mode & 0o777, 0o600) + strictEqual(eventConsumerTokenMatches(first, first), true) + strictEqual(eventConsumerTokenMatches(first, `${first}0`), false) + strictEqual(eventConsumerTokenMatches(first, null), false) + } finally { + rmSync(parent, { recursive: true, force: true }) + } + }) + + it("refuses a symlink in place of the token", async () => { + const parent = mkdtempSync(join(tmpdir(), "maple-event-consumer-auth-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir) + try { + symlinkSync(join(parent, "target"), eventConsumerTokenPath(dataDir)) + let message = "" + try { + await ensureEventConsumerToken(dataDir) + } catch (error) { + message = error instanceof Error ? error.message : String(error) + } + strictEqual(message.includes("not a real file"), true) + } finally { + rmSync(parent, { recursive: true, force: true }) + } + }) +}) diff --git a/apps/cli/test/local-eventing-control-store.test.ts b/apps/cli/test/local-eventing-control-store.test.ts new file mode 100644 index 000000000..88b76c07e --- /dev/null +++ b/apps/cli/test/local-eventing-control-store.test.ts @@ -0,0 +1,745 @@ +import { deepStrictEqual, ok, rejects, strictEqual, throws } from "node:assert" +import { Database } from "bun:sqlite" +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + statSync, + symlinkSync, +} from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "vitest" +import type { MapleCloudEvent, SignalProjectionSpec } from "@maple/eventing-core" +import { eventingControlPath, LocalEventingControlStore } from "../src/server/eventing/control-store" +import type { EventingTelemetryObservation } from "../src/server/eventing/telemetry" + +const withDataDir = async (run: (dataDir: string) => Promise): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-eventing-control-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir, { recursive: true }) + try { + await run(dataDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ + id: "example-record-observed", + revision: 1, + enabled: true, + tenantId: "tenant-a", + sourceKind: "otel.log", + selector: { + op: "eq", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "example.record.observed" }, + }, + projector: { id: "example.record", version: 1, config: { includeLabel: true } }, + activeFrom: "2026-08-07T00:00:00Z", + ...overrides, +}) + +const event = (overrides: Partial = {}): MapleCloudEvent => ({ + specversion: "1.0", + id: "sha256:b01688f3c4a04b29206ff9d9949339b8fadc0de8fbf99c8282eae7e863c265e6", + source: "urn:maple:source:otel:local", + type: "dev.maple.example.record.observed.v1", + subject: "records/42", + time: "2026-08-07T19:42:00.123456789Z", + datacontenttype: "application/json", + dataschema: "urn:maple:event-schema:example-record:v1", + tenantid: "tenant-a", + projectionid: "example-record-observed", + projectionrevision: 1, + projectorid: "example.record", + projectorversion: 1, + data: { recordId: 42, label: "Example" }, + ...overrides, +}) + +const SOURCE_FINGERPRINT = `sha256:${"a".repeat(64)}` + +describe("LocalEventingControlStore", () => { + it("records bounded outbox and consumer telemetry without identifiers or payloads", async () => + withDataDir(async (dataDir) => { + const observations: EventingTelemetryObservation[] = [] + const store = await LocalEventingControlStore.open(dataDir, undefined, { + record: (observation) => observations.push(observation), + }) + try { + const sensitiveEvent = event({ + data: { recordId: 42, label: "PAYLOAD-MUST-NOT-BE-METRIC-DATA" }, + }) + store.stageEvents([sensitiveEvent, sensitiveEvent]) + throws(() => store.stageEvents([event({ data: { recordId: 43 } })]), /collision/) + throws(() => store.markReady(["unknown-event-identifier"]), /unknown event/) + store.markReady([sensitiveEvent.id]) + store.registerConsumer("tenant-a", "private-consumer-identifier", "beginning") + const claim = store.claimReady("tenant-a", "private-consumer-identifier", 10, 30) + throws( + () => store.claimReady("tenant-a", "private-consumer-identifier", 10, 30), + /active lease/, + ) + throws( + () => + store.acknowledgeClaim( + "tenant-a", + "private-consumer-identifier", + "incorrect-private-token", + claim.throughSequence!, + ), + /token does not match/, + ) + store.acknowledgeClaim( + "tenant-a", + "private-consumer-identifier", + claim.leaseToken!, + claim.throughSequence!, + ) + + const operationOutcomes = observations.map( + ({ operation, outcome }) => `${operation}:${outcome}`, + ) + for (const expected of [ + "outbox_stage:success", + "outbox_stage:failure", + "outbox_ready:success", + "outbox_ready:failure", + "outbox_dedup:success", + "consumer_claim:success", + "consumer_claim:failure", + "consumer_ack:success", + "consumer_ack:failure", + "consumer_lease:failure", + "consumer_lag:observed", + ]) + ok(operationOutcomes.includes(expected), `missing telemetry observation ${expected}`) + + const serialized = JSON.stringify(observations) + for (const forbidden of [ + "PAYLOAD-MUST-NOT-BE-METRIC-DATA", + "private-consumer-identifier", + "incorrect-private-token", + sensitiveEvent.id, + claim.leaseToken!, + ]) + strictEqual(serialized.includes(forbidden), false) + } finally { + store.close() + } + })) + + it("stores immutable sequential revisions and only loads the active revision", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + store.saveProjection(projection()) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) + throws( + () => + store.saveProjection( + projection({ projector: { id: "changed", version: 1, config: {} } }), + ), + /immutable/, + ) + throws(() => store.saveProjection(projection({ revision: 3 })), /must be 2/) + + store.saveProjection(projection({ revision: 2, enabled: false })) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), []) + store.saveProjection(projection({ revision: 3 })) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection({ revision: 3 })]) + throws( + () => store.saveProjection(projection({ revision: 2, enabled: false })), + /stale projection revision/, + ) + throws(() => store.saveProjection(projection()), /stale projection revision/) + store.saveProjection(projection({ revision: 3 })) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection({ revision: 3 })]) + deepStrictEqual(store.validate(), { + schemaVersion: 4, + projectionRevisions: 3, + projectionFailures: 0, + stagedEvents: 0, + readyEvents: 0, + }) + } finally { + store.close() + } + })) + + it("deduplicates staged events, rejects collisions, and preserves ready order", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + deepStrictEqual(store.stageEvents([event(), event()]), { + inserted: 1, + deduplicated: 1, + eventIds: [event().id, event().id], + }) + throws(() => store.stageEvents([event({ data: { recordId: 43 } })]), /collision/) + throws(() => store.markReady(["unknown"]), /unknown event/) + store.markReady([event().id]) + store.markReady([event().id]) + deepStrictEqual(store.listStaged().events, []) + deepStrictEqual( + store.listReady().events.map(({ event }) => event), + [event()], + ) + } finally { + store.close() + } + })) + + it("binds staged source recovery to the normalized occurrence fingerprint", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + store.saveProjection(projection()) + const sourced = event({ sourceoccurrenceid: "record-42" }) + throws(() => store.stageEvents([sourced]), /requires a source fingerprint/) + store.stageEvents([sourced], new Map([[sourced.id, SOURCE_FINGERPRINT]])) + deepStrictEqual( + store.stagedEventIdsForOccurrence( + sourced.tenantid, + "otel.log", + sourced.source, + sourced.sourceoccurrenceid!, + SOURCE_FINGERPRINT, + ), + [sourced.id], + ) + throws( + () => + store.stagedEventIdsForOccurrence( + sourced.tenantid, + "otel.log", + sourced.source, + sourced.sourceoccurrenceid!, + `sha256:${"b".repeat(64)}`, + ), + /staged source occurrence collision/, + ) + strictEqual(store.listStaged().events.length, 1) + } finally { + store.close() + } + })) + + it("survives restart and round-trips through a validated standalone snapshot", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + store.saveProjection(projection()) + store.stageEvents([event()]) + store.markReady([event().id]) + store.recordProjectionFailures("tenant-a", [ + { + projectionId: "example-record-observed", + projectionRevision: 1, + occurrenceId: "record-42", + message: "test failure", + }, + ]) + store.close() + + store = await LocalEventingControlStore.open(dataDir) + deepStrictEqual(store.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual( + store.listReady().events.map(({ event }) => event), + [event()], + ) + const snapshot = join(dataDir, "backups", "snapshot", "control.sqlite") + const validation = await store.backupTo(snapshot) + deepStrictEqual(validation, { + schemaVersion: 4, + projectionRevisions: 1, + projectionFailures: 1, + stagedEvents: 0, + readyEvents: 1, + }) + store.close() + + const restored = join(dataDir, "restored") + await LocalEventingControlStore.restoreSnapshot(snapshot, restored) + deepStrictEqual( + LocalEventingControlStore.validateSnapshot(eventingControlPath(restored)), + validation, + ) + const restoredStore = await LocalEventingControlStore.open(restored) + try { + deepStrictEqual(restoredStore.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual( + restoredStore.listReady().events.map(({ event }) => event), + [event()], + ) + } finally { + restoredStore.close() + } + })) + + it("checkpoints committed live WAL state before serializing", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + store.saveProjection(projection()) + store.stageEvents([event()]) + store.markReady([event().id]) + const walPath = `${eventingControlPath(dataDir)}-wal` + ok(existsSync(walPath)) + ok(statSync(walPath).size > 0, "test requires uncheckpointed WAL frames") + + const snapshot = join(dataDir, "backups", "live-wal", "control.sqlite") + await store.backupTo(snapshot) + strictEqual(statSync(walPath).size, 0) + + const restored = join(dataDir, "restored-live-wal") + await LocalEventingControlStore.restoreSnapshot(snapshot, restored) + const restoredStore = await LocalEventingControlStore.open(restored) + try { + deepStrictEqual(restoredStore.loadEnabledProjections("tenant-a"), [projection()]) + deepStrictEqual( + restoredStore.listReady().events.map(({ event }) => event), + [event()], + ) + } finally { + restoredStore.close() + } + } finally { + store.close() + } + })) + + it("paginates every ready event and applies fail-closed outbox capacity", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir, { + maxOutboxEvents: 2, + maxOutboxBytes: 1024 * 1024, + }) + try { + const second = event({ id: "event-2", data: { recordId: 43, label: "Second" } }) + const third = event({ id: "event-3", data: { recordId: 44, label: "Third" } }) + const staged = store.stageEvents([event(), second]) + store.markReady(staged.eventIds) + + const firstPage = store.listReady(1) + strictEqual(firstPage.events.length, 1) + strictEqual(firstPage.nextCursor, firstPage.events[0]?.sequence) + const secondPage = store.listReady(1, firstPage.nextCursor!) + deepStrictEqual( + [...firstPage.events, ...secondPage.events].map(({ event }) => event.id), + [event().id, second.id], + ) + strictEqual(secondPage.nextCursor, null) + deepStrictEqual(store.stageEvents([event()]).deduplicated, 1) + throws(() => store.stageEvents([third]), /outbox capacity exceeded/) + } finally { + store.close() + } + })) + + it("pages recovered events by first readiness transition instead of staging order", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const first = event({ id: "event-a" }) + const second = event({ id: "event-b" }) + store.stageEvents([first]) + store.stageEvents([second]) + store.markReady([second.id]) + + const initialPage = store.listReady(1) + deepStrictEqual( + initialPage.events.map(({ event }) => event.id), + [second.id], + ) + const cursor = initialPage.events[0]!.sequence + + store.markReady([first.id]) + const recoveredPage = store.listReady(1, cursor) + deepStrictEqual( + recoveredPage.events.map(({ event }) => event.id), + [first.id], + ) + strictEqual(recoveredPage.events[0]!.sequence > cursor, true) + } finally { + store.close() + } + })) + + it("migrates schema 1 in place and keeps schema-1 snapshots restorable", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + store.saveProjection(projection()) + const migratedEvent = event({ sourceoccurrenceid: "record-42" }) + store.stageEvents([migratedEvent], new Map([[migratedEvent.id, SOURCE_FINGERPRINT]])) + store.markReady([migratedEvent.id]) + store.close() + + const database = new Database(eventingControlPath(dataDir), { + readwrite: true, + strict: true, + safeIntegers: true, + }) + database.exec("DROP INDEX outbox_events_staged_occurrence") + database.exec("ALTER TABLE outbox_events DROP COLUMN source_fingerprint") + database.exec("ALTER TABLE outbox_events DROP COLUMN source_kind") + database.exec("ALTER TABLE outbox_events DROP COLUMN source") + database.exec("ALTER TABLE outbox_events DROP COLUMN source_occurrence_id") + database.exec("DROP TABLE event_consumers") + database.exec("PRAGMA user_version = 1") + database.close(true) + + strictEqual( + LocalEventingControlStore.validateSnapshot(eventingControlPath(dataDir)).schemaVersion, + 1, + ) + store = await LocalEventingControlStore.open(dataDir) + try { + strictEqual(store.validate().schemaVersion, 4) + deepStrictEqual( + store.listReady().events.map(({ event }) => event.id), + [migratedEvent.id], + ) + strictEqual( + store.stageEvents([migratedEvent], new Map([[migratedEvent.id, SOURCE_FINGERPRINT]])) + .deduplicated, + 1, + ) + deepStrictEqual(store.listConsumers("tenant-a"), []) + } finally { + store.close() + } + })) + + it("refuses schema-3 migration when staged source rows lack recovery fingerprints", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + store.saveProjection(projection()) + const staged = event({ sourceoccurrenceid: "record-42" }) + store.stageEvents([staged], new Map([[staged.id, SOURCE_FINGERPRINT]])) + store.close() + + const database = new Database(eventingControlPath(dataDir), { + readwrite: true, + strict: true, + safeIntegers: true, + }) + database.exec("ALTER TABLE outbox_events DROP COLUMN source_fingerprint") + database.exec("PRAGMA user_version = 3") + database.close(true) + + strictEqual( + LocalEventingControlStore.validateSnapshot(eventingControlPath(dataDir)).schemaVersion, + 3, + ) + await rejects( + () => LocalEventingControlStore.open(dataDir), + /schema 3 with staged source occurrences/, + ) + const unchanged = new Database(eventingControlPath(dataDir), { + readwrite: true, + strict: true, + safeIntegers: true, + }) + try { + const version = unchanged + .query<{ readonly user_version: number | bigint }, []>("PRAGMA user_version") + .get() + strictEqual(Number(version!.user_version), 3) + } finally { + unchanged.close(true) + } + })) + + it("rejects invalid schema-4 staged fingerprints during snapshot validation", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + store.saveProjection(projection()) + const missing = event({ id: "event-missing-fingerprint", sourceoccurrenceid: "record-1" }) + const malformed = event({ id: "event-malformed-fingerprint", sourceoccurrenceid: "record-2" }) + store.stageEvents( + [missing, malformed], + new Map([ + [missing.id, SOURCE_FINGERPRINT], + [malformed.id, SOURCE_FINGERPRINT], + ]), + ) + store.close() + + const database = new Database(eventingControlPath(dataDir), { + readwrite: true, + strict: true, + safeIntegers: true, + }) + database.run("UPDATE outbox_events SET source_fingerprint = NULL WHERE event_id = ?", [ + missing.id, + ]) + database.run("UPDATE outbox_events SET source_fingerprint = ? WHERE event_id = ?", [ + "sha256:not-a-digest", + malformed.id, + ]) + database.close(true) + + throws( + () => LocalEventingControlStore.validateSnapshot(eventingControlPath(dataDir)), + /invalid staged source fingerprint/, + ) + await rejects(() => LocalEventingControlStore.open(dataDir), /invalid staged source fingerprint/) + })) + + it("rejects an unsafe schema-3 restore before replacing an openable target", async () => + withDataDir(async (dataDir) => { + const unsafeDataDir = join(dataDir, "unsafe") + let store = await LocalEventingControlStore.open(unsafeDataDir) + store.saveProjection(projection()) + const unsafeEvent = event({ sourceoccurrenceid: "unsafe-record" }) + store.stageEvents([unsafeEvent], new Map([[unsafeEvent.id, SOURCE_FINGERPRINT]])) + store.close() + const unsafeDatabase = new Database(eventingControlPath(unsafeDataDir), { + readwrite: true, + strict: true, + safeIntegers: true, + }) + unsafeDatabase.exec("ALTER TABLE outbox_events DROP COLUMN source_fingerprint") + unsafeDatabase.exec("PRAGMA user_version = 3") + unsafeDatabase.close(true) + + const liveDataDir = join(dataDir, "live") + store = await LocalEventingControlStore.open(liveDataDir) + store.saveProjection(projection()) + const liveEvent = event({ id: "live-event" }) + store.stageEvents([liveEvent]) + store.markReady([liveEvent.id]) + store.close() + const liveBefore = readFileSync(eventingControlPath(liveDataDir)) + + await rejects( + () => + LocalEventingControlStore.restoreSnapshot( + eventingControlPath(unsafeDataDir), + liveDataDir, + ), + /schema 3 with staged source occurrences/, + ) + deepStrictEqual(readFileSync(eventingControlPath(liveDataDir)), liveBefore) + deepStrictEqual( + readdirSync(dataDir).filter((name) => name.startsWith(".maple-eventing-control-restore-")), + [], + ) + store = await LocalEventingControlStore.open(liveDataDir) + try { + deepStrictEqual( + store.listReady().events.map(({ event }) => event.id), + [liveEvent.id], + ) + } finally { + store.close() + } + })) + + it("leases whole batches, redelivers after expiry, and rejects stale acknowledgements", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir, { + maxOutboxEvents: 10, + maxOutboxBytes: 1024 * 1024, + retainAcknowledgedReadyEvents: 0, + }) + try { + const second = event({ id: "event-2" }) + const third = event({ id: "event-3" }) + const staged = store.stageEvents([event(), second, third]) + store.markReady(staged.eventIds) + store.registerConsumer("tenant-a", "automation", "beginning", "2026-08-13T12:00:00.000Z") + + const firstClaim = store.claimReady( + "tenant-a", + "automation", + 2, + 10, + "2026-08-13T12:00:01.000Z", + ) + strictEqual(firstClaim.leaseToken?.length, 64) + deepStrictEqual( + firstClaim.events.map(({ event }) => event.id), + [event().id, second.id], + ) + throws( + () => store.claimReady("tenant-a", "automation", 2, 10, "2026-08-13T12:00:02.000Z"), + /active lease/, + ) + throws( + () => + store.acknowledgeClaim( + "tenant-a", + "automation", + "0".repeat(64), + firstClaim.throughSequence!, + "2026-08-13T12:00:03.000Z", + ), + /token does not match/, + ) + throws( + () => + store.acknowledgeClaim( + "tenant-a", + "automation", + firstClaim.leaseToken!, + firstClaim.events[0]!.sequence, + "2026-08-13T12:00:03.000Z", + ), + /complete claimed batch/, + ) + + const retry = store.claimReady("tenant-a", "automation", 2, 10, "2026-08-13T12:00:12.000Z") + deepStrictEqual( + retry.events.map(({ event }) => event.id), + [event().id, second.id], + ) + strictEqual(retry.leaseToken === firstClaim.leaseToken, false) + deepStrictEqual( + store.acknowledgeClaim( + "tenant-a", + "automation", + retry.leaseToken!, + retry.throughSequence!, + "2026-08-13T12:00:13.000Z", + ), + { + consumerId: "automation", + acknowledgedThrough: retry.throughSequence, + prunedEvents: 2, + }, + ) + deepStrictEqual( + store.listReady().events.map(({ event }) => event.id), + [third.id], + ) + throws( + () => + store.acknowledgeClaim( + "tenant-a", + "automation", + firstClaim.leaseToken!, + firstClaim.throughSequence!, + "2026-08-13T12:00:14.000Z", + ), + /no active lease/, + ) + } finally { + store.close() + } + })) + + it("prunes only after every active consumer advances and never prunes staged events", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir, { + maxOutboxEvents: 10, + maxOutboxBytes: 1024 * 1024, + retainAcknowledgedReadyEvents: 0, + }) + try { + const second = event({ id: "event-2" }) + const third = event({ id: "event-3" }) + const stranded = event({ id: "event-staged" }) + const ready = store.stageEvents([event(), second, third]) + store.markReady(ready.eventIds) + store.stageEvents([stranded]) + store.registerConsumer("tenant-a", "automation-a", "beginning") + store.registerConsumer("tenant-a", "automation-b", "beginning") + + const fast = store.claimReady("tenant-a", "automation-a", 3, 30) + strictEqual( + store.acknowledgeClaim( + "tenant-a", + "automation-a", + fast.leaseToken!, + fast.throughSequence!, + ).prunedEvents, + 0, + ) + const slow = store.claimReady("tenant-a", "automation-b", 2, 30) + strictEqual( + store.acknowledgeClaim( + "tenant-a", + "automation-b", + slow.leaseToken!, + slow.throughSequence!, + ).prunedEvents, + 2, + ) + deepStrictEqual( + store.listReady().events.map(({ event }) => event.id), + [third.id], + ) + store.disableConsumer("tenant-a", "automation-b") + deepStrictEqual(store.listReady().events, []) + deepStrictEqual( + store.listStaged().events.map(({ event }) => event.id), + [stranded.id], + ) + } finally { + store.close() + } + })) + + it("starts latest consumers after backlog and checkpoints active leases", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + store.stageEvents([event()]) + store.markReady([event().id]) + const registered = store.registerConsumer( + "tenant-a", + "automation", + "latest", + "2099-01-01T00:00:00.000Z", + ) + strictEqual(registered.lastAcknowledgedSequence, store.listReady().events[0]!.sequence) + deepStrictEqual( + store.claimReady("tenant-a", "automation", 10, 300, "2099-01-01T00:00:01.000Z").events, + [], + ) + + const second = event({ id: "event-2" }) + store.stageEvents([second]) + store.markReady([second.id]) + const claim = store.claimReady("tenant-a", "automation", 10, 300, "2099-01-01T00:00:02.000Z") + const snapshot = join(dataDir, "backups", "consumer", "control.sqlite") + await store.backupTo(snapshot) + store.close() + + const restored = join(dataDir, "restored-consumer") + await LocalEventingControlStore.restoreSnapshot(snapshot, restored) + store = await LocalEventingControlStore.open(restored) + try { + deepStrictEqual( + store.listConsumers("tenant-a")[0]?.claimedThroughSequence, + claim.throughSequence, + ) + strictEqual( + store.acknowledgeClaim( + "tenant-a", + "automation", + claim.leaseToken!, + claim.throughSequence!, + "2099-01-01T00:00:03.000Z", + ).acknowledgedThrough, + claim.throughSequence, + ) + } finally { + store.close() + } + })) + + it("refuses a symlink in place of the database", async () => + withDataDir(async (dataDir) => { + const controlPath = eventingControlPath(dataDir) + mkdirSync(join(dataDir, "control"), { recursive: true }) + symlinkSync(join(dataDir, "target.sqlite"), controlPath) + await rejects(() => LocalEventingControlStore.open(dataDir), /not a real file/) + strictEqual(controlPath.endsWith("control/eventing.sqlite"), true) + })) +}) diff --git a/apps/cli/test/local-eventing-ingest.test.ts b/apps/cli/test/local-eventing-ingest.test.ts new file mode 100644 index 000000000..b1250821f --- /dev/null +++ b/apps/cli/test/local-eventing-ingest.test.ts @@ -0,0 +1,545 @@ +import { deepStrictEqual, ok, rejects, strictEqual } from "node:assert" +import { describe, it } from "vitest" +import { normalizeOtlpLogs } from "../src/server/eventing/otlp" +import { __testables } from "../src/server/serve" + +describe("Local eventing ingest seam", () => { + it("requires maintenance authorization and exposes staged records only when requested", async () => { + const eventing = { + health: () => ({ activeProjections: 1 }), + listActive: () => [], + listReady: () => ({ events: [{ sequence: 1, event: { id: "ready" } }], nextCursor: null }), + listStaged: (_limit: number, after: number) => ({ + events: [{ sequence: after + 1, event: { id: "staged" } }], + nextCursor: null, + }), + } + const unauthorized = __testables.handleEventingRead( + eventing as never, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/outbox?state=staged"), + new URL("http://127.0.0.1/local/eventing/outbox?state=staged"), + ) + strictEqual(unauthorized.status, 403) + + const request = new Request("http://127.0.0.1/local/eventing/outbox?state=staged&after=41", { + headers: { "x-maple-maintenance-token": "maintenance-secret" }, + }) + const authorized = __testables.handleEventingRead( + eventing as never, + "maintenance-secret", + request, + new URL(request.url), + ) + strictEqual(authorized.status, 200) + deepStrictEqual(await authorized.json(), { + events: [{ sequence: 42, event: { id: "staged" } }], + nextCursor: null, + }) + }) + + it("authenticates and reads activation bodies before closing admission", async () => { + const gate = new __testables.RequestQuiescenceGate() + const neverClosed = new ReadableStream() + const unauthorized = await __testables.handleProjectionActivation( + {} as never, + gate, + "maintenance-secret", + { + headers: new Headers(), + body: neverClosed, + } as Request, + ) + strictEqual(unauthorized.status, 403) + const afterUnauthorized = gate.enter() + ok(afterUnauthorized, "invalid authorization must not close admission") + afterUnauthorized() + + const checkpointUnauthorized = await __testables.handleCheckpointBackup( + {} as never, + {} as never, + "/unused", + gate, + "maintenance-secret", + { headers: new Headers(), body: neverClosed } as Request, + ) + strictEqual(checkpointUnauthorized.status, 403) + const afterCheckpointUnauthorized = gate.enter() + ok(afterCheckpointUnauthorized, "checkpoint authorization must precede exclusivity") + afterCheckpointUnauthorized() + + let controller!: ReadableStreamDefaultController + const slowBody = new ReadableStream({ + start(value) { + controller = value + }, + }) + let committed = false + const pending = __testables.handleProjectionActivation( + { + prepareActivation: (body: unknown) => ({ body }), + commitActivation: () => { + committed = true + }, + listActive: () => [], + } as never, + gate, + "maintenance-secret", + { + headers: new Headers({ "x-maple-maintenance-token": "maintenance-secret" }), + body: slowBody, + } as Request, + ) + await Promise.resolve() + const whileReading = gate.enter() + ok(whileReading, "an incomplete request body must not close admission") + whileReading() + controller.enqueue(new TextEncoder().encode("{}")) + controller.close() + strictEqual((await pending).status, 200) + strictEqual(committed, true) + }) + + it("bounds activation bodies and reports concurrent maintenance intentionally", async () => { + const oversized = new Request("http://127.0.0.1/local/eventing/projections", { + method: "POST", + body: "123456789", + }) + await rejects(() => __testables.readBoundedJson(oversized, 8), /exceeds 8 bytes/) + + const gate = new __testables.RequestQuiescenceGate() + let releaseMaintenance!: () => void + const maintenance = gate.exclusive( + () => + new Promise((resolve) => { + releaseMaintenance = resolve + }), + ) + await Promise.resolve() + const response = await __testables.handleProjectionActivation( + { + prepareActivation: () => ({}), + commitActivation: () => undefined, + listActive: () => [], + } as never, + gate, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/projections", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: "{}", + }), + ) + strictEqual(response.status, 409) + releaseMaintenance() + await maintenance + }) + + it("separates consumer administration from claim and acknowledgement authorization", async () => { + const gate = new __testables.RequestQuiescenceGate() + const calls: string[] = [] + const eventing = { + registerConsumer: (consumerId: string, startAt: string) => { + calls.push(`register:${consumerId}:${startAt}`) + return { consumerId, active: true } + }, + disableConsumer: (consumerId: string) => { + calls.push(`disable:${consumerId}`) + return { consumerId, active: false } + }, + claimReady: (consumerId: string, limit: number, leaseSeconds: number) => { + calls.push(`claim:${consumerId}:${limit}:${leaseSeconds}`) + return { + consumerId, + leaseToken: "a".repeat(64), + throughSequence: 7, + events: [{ sequence: 7, event: { id: "event-7" } }], + } + }, + acknowledgeClaim: (consumerId: string, _leaseToken: string, throughSequence: number) => { + calls.push(`ack:${consumerId}:${throughSequence}`) + return { consumerId, acknowledgedThrough: throughSequence, prunedEvents: 0 } + }, + } + + const registration = await __testables.handleConsumerRegistration( + eventing as never, + gate, + "maintenance-secret", + new Request("http://127.0.0.1/local/eventing/consumers", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: JSON.stringify({ consumerId: "automation", startAt: "beginning" }), + }), + ) + strictEqual(registration.status, 201) + + const wrongClaimCredential = await __testables.handleConsumerClaim( + eventing as never, + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-maintenance-token": "maintenance-secret", + }, + body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), + }), + ) + strictEqual(wrongClaimCredential.status, 403) + + const claim = await __testables.handleConsumerClaim( + eventing as never, + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), + }), + ) + strictEqual(claim.status, 200) + const claimed = (await claim.json()) as { leaseToken: string; throughSequence: number } + + const acknowledgement = await __testables.handleConsumerAcknowledgement( + eventing as never, + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/acks", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ + consumerId: "automation", + leaseToken: claimed.leaseToken, + throughSequence: claimed.throughSequence, + }), + }), + ) + strictEqual(acknowledgement.status, 200) + deepStrictEqual(calls, [ + "register:automation:beginning", + "claim:automation:10:30", + "ack:automation:7", + ]) + + let releaseMaintenance!: () => void + const maintenance = gate.exclusive( + () => + new Promise((resolve) => { + releaseMaintenance = resolve + }), + ) + await Promise.resolve() + const blockedClaim = await __testables.handleConsumerClaim( + eventing as never, + gate, + "consumer-secret", + new Request("http://127.0.0.1/local/eventing/claims", { + method: "POST", + headers: { + "content-type": "application/json", + "x-maple-event-consumer-token": "consumer-secret", + }, + body: JSON.stringify({ consumerId: "automation", limit: 10, leaseSeconds: 30 }), + }), + ) + strictEqual(blockedClaim.status, 503) + releaseMaintenance() + await maintenance + }) + + it("isolates projection failures, stores telemetry, and makes sibling events ready", async () => { + const order: string[] = [] + const event = { id: "event-1" } + const db = { + exec: () => { + order.push("chdb-insert") + }, + } + const authority = { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => { + order.push("retention-filter") + return { ndjson, accepted: 1, rejected: 0 } + }, + } + const eventing = { + evaluateOtlp: () => { + order.push("evaluate") + return { + events: [event], + recoveredEventIds: [], + failures: [ + { + projectionId: "oversized-projector", + projectionRevision: 1, + occurrenceId: "occurrence-1", + message: "CloudEvent exceeds 262144 UTF-8 bytes", + }, + ], + typeMismatchFields: [], + } + }, + persistFailures: () => order.push("persist-failures"), + stage: () => { + order.push("stage") + return { inserted: 1, deduplicated: 0, eventIds: [event.id] } + }, + markReady: () => order.push("ready"), + } + const request = new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + resourceLogs: [ + { + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: "1786131720123456789", + body: { stringValue: "one" }, + }, + ], + }, + ], + }, + ], + }), + }) + + const result = await __testables.ingest( + db as never, + authority as never, + eventing as never, + "logs", + request, + ) + strictEqual(result.response.status, 200) + strictEqual(result.accepted, 1) + deepStrictEqual(order, [ + "evaluate", + "persist-failures", + "stage", + "retention-filter", + "chdb-insert", + "ready", + ]) + }) + + it("leaves a staged event non-ready when the warehouse write fails", async () => { + let markedReady = false + const request = new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + resourceLogs: [{ scopeLogs: [{ logRecords: [{ body: { stringValue: "one" } }] }] }], + }), + }) + const result = await __testables.ingest( + { + exec: () => { + throw new Error("write failed") + }, + } as never, + { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => ({ + ndjson, + accepted: 1, + rejected: 0, + }), + } as never, + { + evaluateOtlp: () => ({ + events: [{ id: "event-1" }], + recoveredEventIds: [], + failures: [], + typeMismatchFields: [], + }), + persistFailures: () => undefined, + stage: () => ({ inserted: 1, deduplicated: 0, eventIds: ["event-1"] }), + markReady: () => { + markedReady = true + }, + } as never, + "logs", + request, + ) + strictEqual(result.response.status, 500) + strictEqual(markedReady, false) + }) + + it("promotes recovered staged IDs only after the retry reaches the warehouse commit point", async () => { + let readyIds: readonly string[] = [] + const request = new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + resourceLogs: [ + { + scopeLogs: [ + { + logRecords: [ + { timeUnixNano: "1786131720123456789", body: { stringValue: "retry" } }, + ], + }, + ], + }, + ], + }), + }) + const result = await __testables.ingest( + { exec: () => undefined } as never, + { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => ({ + ndjson, + accepted: 1, + rejected: 0, + }), + } as never, + { + evaluateOtlp: () => ({ + events: [], + recoveredEventIds: ["revision-1-event"], + failures: [], + typeMismatchFields: [], + }), + persistFailures: () => undefined, + stage: () => ({ inserted: 0, deduplicated: 0, eventIds: [] }), + markReady: (eventIds: readonly string[]) => { + readyIds = eventIds + }, + } as never, + "logs", + request, + ) + strictEqual(result.response.status, 200) + deepStrictEqual(readyIds, ["revision-1-event"]) + }) + + it("accepts mixed OTLP batches while projecting only records with durable source time", async () => { + let inserted = false + let stagedIds: readonly string[] = [] + const body = { + resourceLogs: [ + { + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: "1786131720123456789", + eventName: "project.me", + body: { stringValue: "projectable" }, + }, + { + eventName: "ignore.me", + body: { stringValue: "timestamp-less" }, + attributes: Array.from({ length: 257 }, (_, index) => ({ + key: `projection-only-${index}`, + value: { stringValue: "warehouse-valid" }, + })), + }, + { + timeUnixNano: "1786131721123456789", + eventName: "ignore.me", + body: { stringValue: "ordinary" }, + }, + ], + }, + ], + }, + { + resource: { + attributes: Array.from({ length: 257 }, (_, index) => ({ + key: `resource-projection-only-${index}`, + value: { stringValue: "warehouse-valid" }, + })), + }, + scopeLogs: [ + { + logRecords: [ + { + timeUnixNano: "1786131722123456789", + eventName: "ignore.me", + }, + ], + }, + ], + }, + { + scopeLogs: [ + { + scope: { + attributes: Array.from({ length: 257 }, (_, index) => ({ + key: `scope-projection-only-${index}`, + value: { stringValue: "warehouse-valid" }, + })), + }, + logRecords: [ + { + timeUnixNano: "1786131723123456789", + eventName: "ignore.me", + }, + ], + }, + ], + }, + ], + } + const result = await __testables.ingest( + { exec: () => (inserted = true) } as never, + { + isRetired: () => false, + filterBatch: (_datasource: string, ndjson: string) => ({ + ndjson, + accepted: ndjson.trim().split("\n").length, + rejected: 0, + }), + } as never, + { + evaluateOtlp: (_signal: string, decoded: unknown) => { + const projected = normalizeOtlpLogs(decoded).filter( + (signal) => signal.fields.get("signal:event.name")?.value === "project.me", + ) + return { + events: projected.map((_signal, index) => ({ id: `event-${index + 1}` })), + recoveredEventIds: [], + failures: [], + typeMismatchFields: [], + } + }, + persistFailures: () => undefined, + stage: (events: readonly { readonly id: string }[]) => { + stagedIds = events.map(({ id }) => id) + return { inserted: events.length, deduplicated: 0, eventIds: stagedIds } + }, + markReady: () => undefined, + } as never, + "logs", + new Request("http://127.0.0.1/v1/logs", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ) + strictEqual(result.response.status, 200) + strictEqual(result.accepted, 5) + strictEqual(inserted, true) + deepStrictEqual(stagedIds, ["event-1"]) + }) +}) diff --git a/apps/cli/test/local-eventing-runtime.test.ts b/apps/cli/test/local-eventing-runtime.test.ts new file mode 100644 index 000000000..7a3a534bd --- /dev/null +++ b/apps/cli/test/local-eventing-runtime.test.ts @@ -0,0 +1,597 @@ +import { deepStrictEqual, ok, strictEqual, throws } from "node:assert" +import { mkdirSync, mkdtempSync, rmSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "vitest" +import { + fieldKey, + isJsonValue, + ProjectorRegistry, + type JsonValue, + type NormalizedSignal, + type SignalProjectionSpec, + type SignalScalar, +} from "@maple/eventing-core" +import { LocalEventingControlStore } from "../src/server/eventing/control-store" +import { normalizeOtlpLogs, normalizeOtlpLogsWithDiagnostics } from "../src/server/eventing/otlp" +import { LocalEventingRuntime, sourceOccurrenceFingerprint } from "../src/server/eventing/runtime" +import type { EventingTelemetryObservation } from "../src/server/eventing/telemetry" +import { encodeLogs } from "../src/server/otlp/encode" + +const withDataDir = async (run: (dataDir: string) => Promise): Promise => { + const parent = mkdtempSync(join(tmpdir(), "maple-eventing-runtime-")) + const dataDir = join(parent, "data") + mkdirSync(dataDir, { recursive: true }) + try { + await run(dataDir) + } finally { + rmSync(parent, { recursive: true, force: true }) + } +} + +const attr = (key: string, value: Record) => ({ key, value }) + +const exampleRecordObserved = { + resourceLogs: [ + { + resource: { + attributes: [ + attr("service.name", { stringValue: "example-service" }), + attr("service.version", { stringValue: "19.1.0" }), + ], + }, + scopeLogs: [ + { + scope: { name: "example.event_store", version: "1.0.0" }, + logRecords: [ + { + timeUnixNano: "1786131720123456789", + observedTimeUnixNano: "1786131721123456789", + eventName: "example.record.observed", + severityNumber: 9, + severityText: "INFO", + body: { stringValue: "Record 42 observed" }, + attributes: [ + attr("event.id", { stringValue: "01K20EXAMPLERECORD42" }), + attr("event.source", { stringValue: "https://events.example.test" }), + attr("example.collection.id", { intValue: "7" }), + attr("example.collection.name", { stringValue: "example/widgets" }), + attr("example.record.id", { intValue: "4200" }), + attr("example.record.sequence", { intValue: "42" }), + attr("example.record.title", { stringValue: "Observe example events" }), + attr("example.record.url", { + stringValue: "https://events.example.test/collections/widgets/records/42", + }), + attr("example.actor.id", { intValue: "9" }), + attr("example.actor.name", { stringValue: "observer" }), + ], + }, + ], + }, + ], + }, + ], +} + +const firstLogRecord = (request: typeof exampleRecordObserved) => + request.resourceLogs[0]!.scopeLogs[0]!.logRecords[0]! + +const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ + id: "example-record-observed", + revision: 1, + enabled: true, + tenantId: "local", + sourceKind: "otel.log", + selector: { + op: "all", + clauses: [ + { + op: "eq", + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: "example.record.observed" }, + }, + { + op: "gte", + field: { namespace: "attribute", key: "example.record.sequence", type: "int64" }, + value: { type: "int64", value: "1" }, + }, + ], + }, + projector: { id: "example.record.observed", version: 1, config: {} }, + activeFrom: "2000-01-01T00:00:00Z", + ...overrides, +}) + +const eventNameProjection = (id: string, eventName: string): SignalProjectionSpec => + projection({ + id, + selector: { + op: "eq", + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: eventName }, + }, + }) + +const signalField = ( + signal: NormalizedSignal, + namespace: "resource" | "attribute", + key: string, +): SignalScalar | undefined => signal.fields.get(fieldKey({ namespace, key })) + +const stringField = ( + signal: NormalizedSignal, + namespace: "resource" | "attribute", + key: string, + required = false, +): string | undefined => { + const value = signalField(signal, namespace, key) + if (value === undefined) { + if (required) throw new Error(`example event is missing ${key}`) + return undefined + } + if (value.type !== "string") throw new Error(`example event ${key} must be a string`) + return value.value +} + +const int64Field = (signal: NormalizedSignal, key: string, required = false): string | undefined => { + const value = signalField(signal, "attribute", key) + if (value === undefined) { + if (required) throw new Error(`example event is missing ${key}`) + return undefined + } + if (value.type !== "int64") throw new Error(`example event ${key} must be an int64`) + return value.value +} + +const exampleProjectors = (): ProjectorRegistry => + new ProjectorRegistry().register({ + id: "example.record.observed", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.example.record.observed.v1", + dataSchema: "urn:maple:event-schema:example-record-observed:v1", + decodeOutput: (value): JsonValue => { + if (!isJsonValue(value)) throw new Error("example projector output must be finite JSON") + return value + }, + decodeConfig: (value) => { + if (typeof value !== "object" || value === null || Array.isArray(value)) + throw new Error("example projector config must be an object") + return {} + }, + project: (signal) => { + const collectionName = stringField(signal, "attribute", "example.collection.name", true)! + const sequence = int64Field(signal, "example.record.sequence", true)! + return { + subject: `${collectionName}/records/${sequence}`, + data: { + collection: { + id: int64Field(signal, "example.collection.id"), + name: collectionName, + }, + record: { + id: int64Field(signal, "example.record.id"), + sequence, + title: stringField(signal, "attribute", "example.record.title"), + url: stringField(signal, "attribute", "example.record.url"), + }, + actor: { + id: int64Field(signal, "example.actor.id"), + name: stringField(signal, "attribute", "example.actor.name"), + }, + serviceName: stringField(signal, "resource", "service.name"), + }, + } + }, + }) + +describe("LocalEventingRuntime", () => { + it("records bounded normalization and projection outcomes without signal data", async () => + withDataDir(async (dataDir) => { + const observations: EventingTelemetryObservation[] = [] + const telemetry = { + record: (observation: EventingTelemetryObservation) => observations.push(observation), + } + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store, telemetry, exampleProjectors()) + runtime.activate(projection()) + strictEqual(runtime.evaluateOtlp("logs", exampleRecordObserved).events.length, 1) + + const malformed = structuredClone(exampleRecordObserved) + firstLogRecord(malformed).attributes = firstLogRecord(malformed).attributes.filter( + ({ key }) => key !== "example.collection.name", + ) + strictEqual(runtime.evaluateOtlp("logs", malformed).failures.length, 1) + + const mismatched = structuredClone(exampleRecordObserved) + firstLogRecord(mismatched).attributes = firstLogRecord(mismatched).attributes.map((entry) => + entry.key === "example.record.sequence" ? attr(entry.key, { stringValue: "42" }) : entry, + ) + deepStrictEqual(runtime.evaluateOtlp("logs", mismatched).typeMismatchFields, [ + "attribute:example.record.sequence", + ]) + + const projectionBoundFailure = structuredClone(exampleRecordObserved) + firstLogRecord(projectionBoundFailure).attributes.push( + ...Array.from({ length: 257 }, (_, index) => + attr(`projection-only-${index}`, { stringValue: "warehouse-valid" }), + ), + ) + strictEqual(runtime.evaluateOtlp("logs", projectionBoundFailure).events.length, 0) + + const operationOutcomes = observations.map( + ({ operation, outcome }) => `${operation}:${outcome}`, + ) + ok(operationOutcomes.includes("normalization:success")) + ok(operationOutcomes.includes("normalization:failure")) + ok(operationOutcomes.includes("projection:success")) + ok(operationOutcomes.includes("projection:failure")) + ok(operationOutcomes.includes("selector_type_mismatch:observed")) + const serialized = JSON.stringify(observations) + strictEqual(serialized.includes("Observe example events"), false) + strictEqual(serialized.includes("01K20EXAMPLERECORD42"), false) + strictEqual(serialized.includes("example-record-observed"), false) + strictEqual(serialized.includes("example.record.sequence"), false) + } finally { + store.close() + } + })) + + it("normalizes typed generic OTLP fields while preserving the existing warehouse encoding", () => { + const [signal] = normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z") + strictEqual(signal?.occurrenceId, "01K20EXAMPLERECORD42") + strictEqual(signal?.identityQuality, "source") + strictEqual(signal?.source, "https://events.example.test") + deepStrictEqual(signal?.fields.get("attribute:example.record.sequence"), { + type: "int64", + value: "42", + }) + const batches = encodeLogs(exampleRecordObserved) + strictEqual(batches.length, 1) + strictEqual(batches[0]?.rowCount, 1) + strictEqual(JSON.parse(batches[0]!.ndjson).log_attributes["example.record.sequence"], "42") + }) + + it("uses the first nonblank occurrence alias and derives identity when every alias is blank", () => { + const aliased = structuredClone(exampleRecordObserved) + const aliasedRecord = firstLogRecord(aliased) + aliasedRecord.attributes = [ + attr("event.id", { stringValue: " " }), + attr("cloudevents.id", { stringValue: " cloud-event-42 " }), + ...aliasedRecord.attributes.filter(({ key }) => !["event.id", "cloudevents.id"].includes(key)), + ] + const [aliasedSignal] = normalizeOtlpLogs(aliased, "2026-08-07T20:00:00Z") + strictEqual(aliasedSignal?.occurrenceId, "cloud-event-42") + strictEqual(aliasedSignal?.identityQuality, "source") + + const derivedA = structuredClone(aliased) + const derivedARecord = firstLogRecord(derivedA) + derivedARecord.attributes = derivedARecord.attributes.map((entry) => + ["event.id", "cloudevents.id"].includes(entry.key) + ? attr(entry.key, { stringValue: entry.key === "event.id" ? "" : " \t " }) + : entry, + ) + const derivedB = structuredClone(derivedA) + firstLogRecord(derivedB).body = { stringValue: "A different record occurrence" } + const [signalA] = normalizeOtlpLogs(derivedA, "2026-08-07T20:00:00Z") + const [signalB] = normalizeOtlpLogs(derivedB, "2026-08-07T20:00:00Z") + strictEqual(signalA?.identityQuality, "derived") + strictEqual(signalB?.identityQuality, "derived") + strictEqual(signalA?.occurrenceId?.startsWith("derived:sha256:"), true) + strictEqual(signalA?.occurrenceId === signalB?.occurrenceId, false) + }) + + it("keeps projectable retries byte-identical and skips timestamp-less durable logs", () => { + const first = normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z") + const retry = normalizeOtlpLogs(exampleRecordObserved, "2026-08-08T20:00:00Z") + deepStrictEqual(first, retry) + + const timestampLess = structuredClone(exampleRecordObserved) + const timestampLessRecord = firstLogRecord(timestampLess) as { + timeUnixNano?: string + observedTimeUnixNano?: string + } + delete timestampLessRecord.timeUnixNano + delete timestampLessRecord.observedTimeUnixNano + deepStrictEqual(normalizeOtlpLogs(timestampLess, "2026-08-07T20:00:00Z"), []) + deepStrictEqual( + normalizeOtlpLogsWithDiagnostics(timestampLess, "2026-08-07T20:00:00Z").unprojectedIdentities, + [ + { + sourceKind: "otel.log", + source: "https://events.example.test", + tenantId: "local", + occurrenceId: "01K20EXAMPLERECORD42", + occurredAt: null, + }, + ], + ) + }) + + it("uses a locale-independent source-fingerprint field order", () => { + const [signal] = normalizeOtlpLogs(exampleRecordObserved, "2026-08-07T20:00:00Z") + const fields = new Map(signal!.fields) + fields.set("attribute:ä", { type: "string", value: "umlaut" }) + fields.set("attribute:z", { type: "string", value: "ascii" }) + const forward = { ...signal!, fields } + const reverse = { ...signal!, fields: new Map([...fields].reverse()) } + strictEqual(sourceOccurrenceFingerprint(forward), sourceOccurrenceFingerprint(reverse)) + strictEqual( + sourceOccurrenceFingerprint(forward), + "sha256:4ed4d210645f2df1959e5c56acb5b22140a01aa267fdf1fab8b62e56ea63e31e", + ) + }) + + it("preserves __proto__ as ordinary OTLP data without prototype mutation", () => { + const request = structuredClone(exampleRecordObserved) + firstLogRecord(request).attributes.push( + attr("__proto__", { + kvlistValue: { values: [attr("nested", { stringValue: "top-level" })] }, + }), + attr("safe", { + kvlistValue: { values: [attr("__proto__", { stringValue: "nested" })] }, + }), + ) + const [signal] = normalizeOtlpLogs(request, "2026-08-07T20:00:00Z") + const record = (signal!.data as { record: { attributes: Record } }).record + ok(Object.prototype.hasOwnProperty.call(record.attributes, "__proto__")) + deepStrictEqual(record.attributes["__proto__"], { nested: "top-level" }) + const safe = record.attributes.safe as Record + ok(Object.prototype.hasOwnProperty.call(safe, "__proto__")) + strictEqual(safe["__proto__"], "nested") + strictEqual(Object.prototype.hasOwnProperty.call({}, "nested"), false) + }) + + it("catalogs only the scalar body field that the OTLP adapter can populate", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) + throws( + () => + runtime.prepareActivation( + projection({ + selector: { + op: "exists", + field: { namespace: "body", key: "text", type: "string" }, + }, + }), + ), + /unknown field body:text/, + ) + const activation = runtime.prepareActivation( + projection({ + selector: { + op: "exists", + field: { namespace: "body", key: "value", type: "boolean" }, + }, + }), + ) + strictEqual(activation.spec.selector.op, "exists") + } finally { + store.close() + } + })) + + it("projects before storage, deduplicates retry delivery, and makes the event ready after commit", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) + strictEqual(runtime.hasActiveSource("otel.log"), false) + runtime.activate(projection()) + const first = runtime.evaluateOtlp("logs", exampleRecordObserved) + strictEqual(first.failures.length, 0) + strictEqual(first.events.length, 1) + deepStrictEqual(first.events[0], { + specversion: "1.0", + id: first.events[0]!.id, + source: "https://events.example.test", + type: "dev.maple.example.record.observed.v1", + subject: "example/widgets/records/42", + time: "2026-08-07T19:42:00.123456789Z", + datacontenttype: "application/json", + dataschema: "urn:maple:event-schema:example-record-observed:v1", + tenantid: "local", + projectionid: "example-record-observed", + projectionrevision: 1, + projectorid: "example.record.observed", + projectorversion: 1, + sourceoccurrenceid: "01K20EXAMPLERECORD42", + sourceidentityquality: "source", + data: { + collection: { id: "7", name: "example/widgets" }, + record: { + id: "4200", + sequence: "42", + title: "Observe example events", + url: "https://events.example.test/collections/widgets/records/42", + }, + actor: { id: "9", name: "observer" }, + serviceName: "example-service", + }, + }) + const staged = runtime.stage(first.events, first.eventSourceFingerprints) + strictEqual(staged.inserted, 1) + strictEqual(runtime.listReady().events.length, 0) + deepStrictEqual( + runtime.listStaged().events.map(({ event }) => event), + first.events, + ) + runtime.activate(projection({ revision: 2, enabled: false })) + const projectionIneligibleRetry = structuredClone(exampleRecordObserved) + firstLogRecord(projectionIneligibleRetry).attributes.push( + ...Array.from({ length: 257 }, (_, index) => + attr(`retry-projection-only-${index}`, { stringValue: "warehouse-valid" }), + ), + ) + throws( + () => runtime.evaluateOtlp("logs", projectionIneligibleRetry, () => true), + /cannot safely recover staged source occurrence/, + ) + strictEqual(runtime.listStaged().events.length, 1) + strictEqual(runtime.listReady().events.length, 0) + const changedRetry = structuredClone(exampleRecordObserved) + firstLogRecord(changedRetry).body = { stringValue: "changed retry content" } + throws( + () => runtime.evaluateOtlp("logs", changedRetry, () => true), + /staged source occurrence collision/, + ) + strictEqual(runtime.listStaged().events.length, 1) + strictEqual(runtime.listReady().events.length, 0) + const retry = runtime.evaluateOtlp("logs", exampleRecordObserved, () => true) + deepStrictEqual(retry.events, []) + deepStrictEqual(retry.recoveredEventIds, staged.eventIds) + runtime.markReady(retry.recoveredEventIds) + deepStrictEqual( + runtime.listReady().events.map(({ event }) => event), + first.events, + ) + deepStrictEqual(runtime.listStaged().events, []) + } finally { + store.close() + } + })) + + it("rejects same event bytes with conflicting source content within one batch", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) + runtime.activate(projection()) + const request = structuredClone(exampleRecordObserved) + const first = firstLogRecord(request) + first.attributes.push(attr("example.projector.ignored", { stringValue: "first" })) + const second = structuredClone(first) + second.attributes = second.attributes.map((entry) => + entry.key === "example.projector.ignored" + ? attr(entry.key, { stringValue: "second" }) + : entry, + ) + request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(second) + throws( + () => runtime.evaluateOtlp("logs", request), + /source occurrence collision within one ingest batch/, + ) + strictEqual(runtime.listStaged().events.length, 0) + strictEqual(runtime.listReady().events.length, 0) + } finally { + store.close() + } + })) + + it("rejects matching and nonmatching records that reuse one source occurrence", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) + runtime.activate(eventNameProjection("observed-only", "example.record.observed")) + const request = structuredClone(exampleRecordObserved) + const sibling = structuredClone(firstLogRecord(request)) + sibling.eventName = "example.record.ignored" + request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(sibling) + throws( + () => runtime.evaluateOtlp("logs", request), + /source occurrence collision within one ingest batch/, + ) + strictEqual(runtime.listStaged().events.length, 0) + strictEqual(runtime.listReady().events.length, 0) + } finally { + store.close() + } + })) + + it("rejects projectable and projection-ineligible records with one source occurrence", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) + runtime.activate(projection()) + const request = structuredClone(exampleRecordObserved) + const sibling = structuredClone(firstLogRecord(request)) + sibling.attributes.push( + ...Array.from({ length: 257 }, (_, index) => + attr(`projection-only-sibling-${index}`, { stringValue: "warehouse-valid" }), + ), + ) + request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(sibling) + throws( + () => runtime.evaluateOtlp("logs", request), + /source occurrence collision with an unprojectable record within one ingest batch/, + ) + strictEqual(runtime.listStaged().events.length, 0) + strictEqual(runtime.listReady().events.length, 0) + } finally { + store.close() + } + })) + + it("rejects disjoint projections over conflicting records with one source occurrence", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) + runtime.activate(eventNameProjection("observed-events", "example.record.observed")) + runtime.activate(eventNameProjection("alternate-events", "example.record.alternate")) + const request = structuredClone(exampleRecordObserved) + const sibling = structuredClone(firstLogRecord(request)) + sibling.eventName = "example.record.alternate" + request.resourceLogs[0]!.scopeLogs[0]!.logRecords.push(sibling) + throws( + () => runtime.evaluateOtlp("logs", request), + /source occurrence collision within one ingest batch/, + ) + strictEqual(runtime.listStaged().events.length, 0) + strictEqual(runtime.listReady().events.length, 0) + } finally { + store.close() + } + })) + + it("activates a validated revision without restart and reloads it after restart", async () => + withDataDir(async (dataDir) => { + let store = await LocalEventingControlStore.open(dataDir) + let runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) + runtime.activate(projection()) + strictEqual(runtime.evaluateOtlp("logs", exampleRecordObserved).events.length, 1) + runtime.activate( + projection({ + revision: 2, + selector: { + op: "eq", + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: "example.record.closed" }, + }, + }), + ) + strictEqual(runtime.evaluateOtlp("logs", exampleRecordObserved).events.length, 0) + store.close() + + store = await LocalEventingControlStore.open(dataDir) + try { + runtime = new LocalEventingRuntime(store, undefined, exampleProjectors()) + strictEqual(runtime.listActive()[0]?.revision, 2) + strictEqual(runtime.evaluateOtlp("logs", exampleRecordObserved).events.length, 0) + } finally { + store.close() + } + })) + + it("does no normalization or event work for a source with no active projection", async () => + withDataDir(async (dataDir) => { + const store = await LocalEventingControlStore.open(dataDir) + try { + const runtime = new LocalEventingRuntime(store) + deepStrictEqual(runtime.evaluateOtlp("logs", { malformed: Symbol("not decoded") }), { + events: [], + eventSourceFingerprints: new Map(), + recoveredEventIds: [], + failures: [], + typeMismatchFields: [], + }) + } finally { + store.close() + } + })) +}) diff --git a/apps/cli/test/server-args.test.ts b/apps/cli/test/server-args.test.ts index b86f1d937..0021679c8 100644 --- a/apps/cli/test/server-args.test.ts +++ b/apps/cli/test/server-args.test.ts @@ -30,7 +30,10 @@ describe("local server bind host", () => { it("separates the bind address from the client-facing address", () => { strictEqual(resolveAdvertiseHost(undefined, undefined, "0.0.0.0"), "127.0.0.1") - strictEqual(resolveAdvertiseHost(undefined, " srvmini2.lan ", "0.0.0.0"), "srvmini2.lan") + strictEqual( + resolveAdvertiseHost(undefined, " node-a.example.test ", "0.0.0.0"), + "node-a.example.test", + ) strictEqual(resolveAdvertiseHost(" 192.0.2.10 ", "ignored", "0.0.0.0"), "192.0.2.10") strictEqual(resolveAdvertiseHost(" ", " [::1] ", "0.0.0.0"), "::1") }) @@ -70,7 +73,7 @@ describe("buildDetachedChildArgs", () => { const args = buildDetachedChildArgs({ entry: "/repo/apps/cli/src/bin.ts", host: "0.0.0.0", - advertiseHost: "srvmini2.lan", + advertiseHost: "node-a.example.test", port: 4318, dataDir: "/tmp/maple data", offline: true, @@ -84,7 +87,7 @@ describe("buildDetachedChildArgs", () => { "--host", "0.0.0.0", "--advertise-host", - "srvmini2.lan", + "node-a.example.test", "--port", "4318", "--data-dir", diff --git a/apps/cli/test/server-network.test.ts b/apps/cli/test/server-network.test.ts index 79918db84..d9203937c 100644 --- a/apps/cli/test/server-network.test.ts +++ b/apps/cli/test/server-network.test.ts @@ -100,18 +100,23 @@ describe("local listener addresses", () => { }) describe("browser origin policy", () => { - const requestUrl = new URL("http://srvmini2.lan:4418/local/query") + const requestUrl = new URL("http://node-a.example.test:4418/local/query") const hostedOrigin = "https://local.maple.dev" - const browserHosts = ["srvmini2.lan", "127.0.0.1"] + const browserHosts = ["node-a.example.test", "127.0.0.1"] it("allows non-browser clients, the advertised same-origin UI, and the hosted UI", () => { strictEqual(isBrowserOriginAllowed(requestUrl, null, hostedOrigin, browserHosts), true) strictEqual( - isBrowserOriginAllowed(requestUrl, "http://srvmini2.lan:4418", hostedOrigin, browserHosts), + isBrowserOriginAllowed(requestUrl, "http://node-a.example.test:4418", hostedOrigin, browserHosts), true, ) strictEqual( - isBrowserOriginAllowed(requestUrl, "https://srvmini2.lan:4418", hostedOrigin, browserHosts), + isBrowserOriginAllowed( + requestUrl, + "https://node-a.example.test:4418", + hostedOrigin, + browserHosts, + ), true, ) strictEqual(isBrowserOriginAllowed(requestUrl, hostedOrigin, hostedOrigin, browserHosts), true) @@ -171,7 +176,8 @@ describe("browser origin policy", () => { deepStrictEqual(corsHeadersForAllowedOrigin(hostedOrigin), { "access-control-allow-origin": hostedOrigin, "access-control-allow-methods": "GET, POST, OPTIONS", - "access-control-allow-headers": "content-type, content-encoding, authorization, x-maple-sdk", + "access-control-allow-headers": + "content-type, content-encoding, authorization, x-maple-sdk, x-maple-maintenance-token", "access-control-allow-private-network": "true", vary: "Origin", }) diff --git a/apps/local-ui/src/lib/constants.test.ts b/apps/local-ui/src/lib/constants.test.ts index cf150b2a5..54ad10873 100644 --- a/apps/local-ui/src/lib/constants.test.ts +++ b/apps/local-ui/src/lib/constants.test.ts @@ -19,9 +19,9 @@ describe("local UI endpoint selection", () => { }) it("keeps an embedded LAN or TLS-proxied UI same-origin", () => { - const page = location("https://srvmini2.lan:4418/?api_key=not-propagated") + const page = location("https://node-a.example.test:4418/?api_key=not-propagated") expect(localApiBaseForLocation(page)).toBe("") - expect(localOtlpEndpointForLocation(page)).toBe("https://srvmini2.lan:4418") + expect(localOtlpEndpointForLocation(page)).toBe("https://node-a.example.test:4418") }) it("keeps the Vite development UI same-origin for its proxied query and OTLP routes", () => { diff --git a/bun.lock b/bun.lock index cefbd7f37..75d923a7c 100644 --- a/bun.lock +++ b/bun.lock @@ -53,12 +53,14 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/clickhouse-builder": "workspace:*", "@maple-dev/effect-sdk": "workspace:*", + "@maple/alerting-core": "workspace:*", "@maple/auth": "workspace:*", "@maple/cache": "workspace:*", "@maple/db": "workspace:*", "@maple/domain": "workspace:*", "@maple/effect-cloudflare": "workspace:*", "@maple/email": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/infra": "workspace:*", "@maple/llm": "workspace:*", "@maple/query-engine": "workspace:*", @@ -93,6 +95,7 @@ "@effect/platform-bun": "catalog:effect", "@maple-dev/effect-sdk": "workspace:*", "@maple/domain": "workspace:*", + "@maple/eventing-core": "workspace:*", "@maple/query-engine": "workspace:*", "effect": "catalog:effect", "protobufjs": "^8.6.1", @@ -470,6 +473,18 @@ "effect": ">=4.0.0-beta.100 || >=4.0.0", }, }, + "packages/alerting-core": { + "name": "@maple/alerting-core", + "version": "0.0.0", + "dependencies": { + "@maple/eventing-core": "workspace:*", + }, + "devDependencies": { + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:", + }, + }, "packages/auth": { "name": "@maple/auth", "dependencies": { @@ -600,6 +615,19 @@ "typescript": "catalog:tooling", }, }, + "packages/eventing-core": { + "name": "@maple/eventing-core", + "version": "0.0.0", + "dependencies": { + "effect": "catalog:effect", + }, + "devDependencies": { + "@effect/language-service": "catalog:effect", + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:", + }, + }, "packages/infra": { "name": "@maple/infra", "devDependencies": { @@ -1292,6 +1320,8 @@ "@maple/alerting": ["@maple/alerting@workspace:apps/alerting"], + "@maple/alerting-core": ["@maple/alerting-core@workspace:packages/alerting-core"], + "@maple/api": ["@maple/api@workspace:apps/api"], "@maple/auth": ["@maple/auth@workspace:packages/auth"], @@ -1316,6 +1346,8 @@ "@maple/email": ["@maple/email@workspace:packages/email"], + "@maple/eventing-core": ["@maple/eventing-core@workspace:packages/eventing-core"], + "@maple/infra": ["@maple/infra@workspace:packages/infra"], "@maple/ingest": ["@maple/ingest@workspace:apps/ingest"], diff --git a/docs/eventing-extension-guide.md b/docs/eventing-extension-guide.md new file mode 100644 index 000000000..6c5995f76 --- /dev/null +++ b/docs/eventing-extension-guide.md @@ -0,0 +1,509 @@ +# Extending Maple's signal-to-event system + +This guide walks through adding either of the two main eventing extensions: + +- a **source adapter**, which turns an authenticated source payload into typed, + normalized signals; or +- a **semantic projector**, which turns matching signals into versioned factual + events. + +You can add one or both, depending on what the source already provides. + +First, one important naming point: an eventing extension is a compile-time +registered module. It is not a runtime-loaded plugin, and projection +configuration cannot introduce executable code. + +The host decides which adapters and projectors are installed. An operator can +then activate installed projectors through bounded, durable projection +revisions. + +The contracts in `@maple/eventing-core` are host-neutral. Hosted Maple and Maple +Local can install the same source and projector definitions while using +different authentication, persistence, transaction, and consumer +implementations. + +## Where the extension fits + +A factual occurrence moves through the system like this: + +```text +authenticated input + -> source adapter + -> typed normalized signal + -> registered field catalog and selector + -> pure registered projector + -> schema-validated CloudEvent + -> host-owned durable outbox + -> named consumer or hosted delivery path +``` + +The extension is responsible for: + +- normalizing an authenticated source payload into bounded signals; +- defining stable source and occurrence identity; +- declaring the selectable field catalog and sensitivity policy; +- decoding projector configuration and output; +- translating a matching signal into factual event data; and +- owning the versioned event type and data-schema names. + +The host is responsible for: + +- authenticating the source or verifying its signature before normalization; +- decoding requests and enforcing input-size limits; +- storing projection revisions and activating compiled registries atomically; +- defining the warehouse or source-of-record commit boundary; +- staging events durably and detecting recovery collisions; +- checkpointing or providing equivalent hosted transactional persistence; and +- authorizing consumers, delivering events, retrying work, and performing side + effects. + +That last boundary matters: projectors never perform I/O. Sending a message, +calling a provider, mutating source state, or deciding what action to take +belongs to a consumer after the event has crossed the durable boundary. + +## Do you need a new source adapter? + +A useful rule of thumb is to reuse an installed source kind whenever it already +preserves the fact you need. + +For example, when a semantic fact already arrives in an OTLP log, you will +usually need only: + +1. a new projector; and +2. projection configuration that selects the relevant logs. + +You do not need another OTLP decoder. + +Add a source adapter when the source has its own authenticated payload, identity +contract, or field vocabulary. A provider webhook is the usual example. + +A new adapter should not decode the same request a second time just for +eventing. The host should decode once, authenticate once, and pass the +already-decoded value to the adapter. + +Before writing the implementation, settle the following contracts: + +| Decision | What to decide | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `sourceKind` | A stable name for the normalized input contract. | +| `source` | A stable URI identifying the logical producer or integration. Never include credentials. | +| `occurrenceId` | Prefer an ID issued by the source that remains stable across retries and rebatching. Document the collision limits of any derived identity. | +| Event time | Use source time. Do not put a changing server receipt time into durable event bytes. | +| Fields | Expose only the bounded scalar values needed for selection. | +| `data` | Preserve only bounded, schema-validated projector input. Do not retain an unchecked raw request. | +| Sensitivity | Mark fields as sensitive when generic projection must not expose them by default. | +| Replay | State whether each field can be reconstructed exactly, only through an explicit coercion, or not at all. | + +When the source provides no stable occurrence identity, say so through the +weaker identity quality. When it provides no stable source timestamp, do not +substitute a changing host receipt time. A host may decline durable projection +rather than pretend the source offers retry-safe identity or time. + +## Complete example + +The following example takes a build-system message that the host has already +authenticated and runtime-decoded, normalizes it, and projects successful builds +into a versioned factual event. + +The example is deliberately provider-neutral. + +### 1. Define the source and normalize its messages + +```ts +import { defineSignalFields, type SignalSourceAdapter } from "@maple/eventing-core" + +interface BuildMessage { + readonly id: string + readonly projectId: string + readonly status: "running" | "success" | "failed" + readonly occurredAt: string +} + +interface BuildContext { + readonly tenantId: string + readonly integrationId: string +} + +export const BUILD_SOURCE: SignalSourceAdapter = { + definition: { + sourceKind: "example.build", + fields: [ + { + field: { namespace: "signal", key: "event.name", type: "string" }, + operators: ["exists", "eq", "neq", "in"], + sensitivity: "public", + replay: "exact", + }, + { + field: { namespace: "attribute", key: "build.status", type: "string" }, + operators: ["exists", "eq", "neq", "in"], + sensitivity: "public", + replay: "exact", + }, + ], + }, + normalize: (message, context) => [ + { + sourceKind: "example.build", + source: `urn:example:builds:${context.integrationId}`, + tenantId: context.tenantId, + occurrenceId: message.id, + identityQuality: "source", + occurredAt: message.occurredAt, + // This provider has no separate, stable observation timestamp. + // Use source time rather than a changing host receipt time. + observedAt: message.occurredAt, + subject: `projects/${message.projectId}/builds/${message.id}`, + fields: defineSignalFields([ + { + field: { namespace: "signal", key: "event.name", type: "string" }, + value: { type: "string", value: "build.status.changed" }, + }, + { + field: { namespace: "attribute", key: "build.status", type: "string" }, + value: { type: "string", value: message.status }, + }, + ]), + data: { + buildId: message.id, + projectId: message.projectId, + status: message.status, + }, + }, + ], +} +``` + +Authentication is intentionally absent from `normalize`. The host must verify +the message before calling the adapter. + +Normalization must also be deterministic. Given the same source occurrence, it +should produce the same normalized signal. It must not call `Date.now()`, +generate a UUID, query a database, make a network request, or depend on mutable +host state. + +### 2. Define the projector's runtime codecs + +Projector configuration and projector output both cross trust boundaries, so +each needs a runtime decoder. + +The output decoder is especially important: it makes `dataschema` an enforced +contract rather than a hopeful annotation. + +```ts +import { type JsonValue, type SignalProjector } from "@maple/eventing-core" +import { Schema } from "effect" + +const BuildData = Schema.Struct({ + buildId: Schema.String, + projectId: Schema.String, + status: Schema.Literals(["running", "success", "failed"]), +}) + +const BuildCompletedConfig = Schema.Struct({ + includeProject: Schema.Boolean, +}) + +const BuildCompletedData = Schema.Struct({ + build_id: Schema.String, + project_id: Schema.optionalKey(Schema.String), + status: Schema.Literal("success"), +}) + +const decodeBuildData = Schema.decodeUnknownSync(BuildData) +const decodeConfig = Schema.decodeUnknownSync(BuildCompletedConfig) +const decodeOutput = (value: unknown): JsonValue => Schema.decodeUnknownSync(BuildCompletedData)(value) + +export const BUILD_COMPLETED_PROJECTOR: SignalProjector> = { + id: "example.build-completed", + version: 1, + sourceKinds: ["example.build"], + outputType: "dev.maple.example.build.completed.v1", + dataSchema: "urn:maple:event-schema:example-build-completed:v1", + decodeConfig, + decodeOutput, + project: (signal, config) => { + const build = decodeBuildData(signal.data) + if (build.status !== "success") { + throw new Error("build-completed projector requires a successful build") + } + return { + subject: signal.subject, + time: signal.occurredAt, + data: { + build_id: build.buildId, + ...(config.includeProject ? { project_id: build.projectId } : {}), + status: "success", + }, + } + }, +} +``` + +The selector should normally stop incompatible statuses from reaching this +projector. The explicit check is still useful: if the projection configuration +and implementation ever drift apart, the projector fails closed instead of +emitting a misleading event. + +### 3. Register the code and compile a projection + +Registration installs trusted code. A projection revision selects that +installed code and supplies bounded data configuration. + +That distinction is the core safety model: configuration chooses among +registered behavior, but it cannot introduce new executable behavior. + +```ts +import { + CompiledProjectionRegistry, + ProjectorRegistry, + SignalSourceRegistry, + type SignalProjectionSpec, +} from "@maple/eventing-core" + +const sources = new SignalSourceRegistry().register(BUILD_SOURCE.definition) +const projectors = new ProjectorRegistry().register(BUILD_COMPLETED_PROJECTOR) + +const projection: SignalProjectionSpec = { + id: "successful-builds", + revision: 1, + enabled: true, + tenantId: "tenant-a", + sourceKind: "example.build", + selector: { + op: "eq", + field: { namespace: "attribute", key: "build.status", type: "string" }, + value: { type: "string", value: "success" }, + }, + projector: { + id: "example.build-completed", + version: 1, + config: { includeProject: true }, + }, + activeFrom: "2026-08-21T00:00:00Z", +} + +const compiled = CompiledProjectionRegistry.compile([projection], sources, projectors) +``` + +Compilation rejects: + +- unknown source kinds; +- unknown projector IDs or versions; +- unsupported fields or operators; +- malformed projector configuration; and +- projectors that do not accept the selected source kind. + +A host should replace its complete compiled registry snapshot atomically, and +only after compilation succeeds. + +### 4. Evaluate at the host's commit boundary + +```ts +const acceptedAt = "2026-08-21T12:00:01Z" +const [signal] = BUILD_SOURCE.normalize( + { + id: "build-42", + projectId: "project-7", + status: "success", + occurredAt: "2026-08-21T12:00:00Z", + }, + { + tenantId: "tenant-a", + integrationId: "integration-3", + }, +) + +if (!signal) throw new Error("build adapter produced no signal") +const result = compiled.evaluate(signal, acceptedAt) +``` + +`acceptedAt` is host control metadata used for `activeFrom` gating. It is not +part of the source fact. + +Do not put a changing acceptance timestamp into source identity, normalized +event content, or projector output. Otherwise, a retry could produce different +durable bytes for the same source occurrence. + +`evaluate` is pure: it returns results but does not persist them. + +The host must then: + +1. stage every successfully projected event durably; +2. commit the original source occurrence to the warehouse or other source of + record; +3. mark the staged events ready only after that commit succeeds; and +4. on retry, recover the original staged events rather than reevaluating the + occurrence under a newer projection revision. + +That last step is important. A projection may be edited or disabled between the +first attempt and a retry. Recovery must complete the original durable +obligation, not quietly replace it with whatever the current registry would +produce. + +Maple Local implements this with its SQLite eventing control store and the chDB +commit seam. A hosted implementation may use a database transaction or another +durable outbox, as long as it provides the same ordering and recovery guarantees. + +## Registering an extension in a host + +### Maple Local + +Maple Local already normalizes OTLP logs in `apps/cli/src/server/eventing`. + +When the new fact is already carried by those logs, the usual path is: + +1. register the projector in the `ProjectorRegistry` supplied to + `LocalEventingRuntime`; and +2. activate a durable projection revision through the authenticated + configuration boundary. + +A genuinely new Local source requires a little more wiring: + +1. authenticate and decode its ingest request; +2. pass the decoded value to a `SignalSourceAdapter`; +3. register the adapter definition in the Local composition root; and +4. preserve the existing stage → warehouse commit → ready ordering. + +Do not bypass `LocalEventingRuntime` by writing directly to the outbox. That +would skip the shared identity, collision, activation, and recovery rules. + +### Hosted Maple + +A hosted source should: + +1. verify and decode the request at the route boundary; +2. invoke its adapter once; +3. register the source and projector definitions in the service composition; + and +4. persist the resulting events through the hosted durable boundary. + +The PlanetScale webhook composition in +[`apps/api/src/services/integrations/planetscale/webhook-events.ts`](../apps/api/src/services/integrations/planetscale/webhook-events.ts) +is the current reference implementation. Provider verification stays outside +the projector, while the normalized fact uses the shared registry and +CloudEvent contracts. + +A hosted implementation does not need to use Maple Local's SQLite store. It +does, however, need equivalent guarantees for: + +- tenant isolation; +- idempotent staging; +- event and source-identity collision detection; +- durable recovery; and +- retries. + +## Versioning without surprises + +There are four separate kinds of versioning here. They solve different +problems, so do not collapse them into one number. + +### Source kind + +`sourceKind` identifies the normalized fields, identity rules, and source +contract. + +Keep changes backward compatible. When you need an incompatible normalized +contract, introduce a new source kind. + +### Projector version + +Increment the projector version when its semantics or configuration +compatibility changes. + +Never replace an old implementation under the same projector ID and version. +Existing durable projection revisions must continue to refer to the behavior +they originally selected. + +### Event type and data-schema version + +Change the event type or data-schema version when a consumer would observe an +incompatible payload contract. + +Do not reuse an event type or schema URI for a differently shaped or differently +interpreted event. + +### Projection revision + +Create a new projection revision whenever you change: + +- the selector; +- `activeFrom`; +- enabled or disabled state; +- the projector ID or version; or +- projector configuration. + +Projection revisions are immutable and monotonic. A rollback is not an edit to +an older revision; it is a new revision that restores the earlier behavior. + +## Schemas and fixtures + +Every public or cross-runtime event contract should include the following: + +1. A closed runtime decoder for projector output. +2. A matching, versioned JSON Schema in the package that owns the event. +3. Valid and invalid output fixtures. +4. A deterministic complete-event fixture with its expected canonical event ID. +5. Schema-generation and drift checks in the package test suite. + +The shared schemas and fixtures in `packages/eventing-core` define the common +selector, envelope, and event-identity behavior. + +Source-specific payload schemas should stay with the module that owns their +meaning. + +## Required tests + +An extension is not complete until its tests demonstrate all of the following: + +- Authentication or signature verification happens before normalization. +- Normalization is bounded. +- Sensitive raw values are rejected or redacted according to the source policy. +- The same source occurrence normalizes deterministically. +- A retry under the same projection revision produces a byte-identical + CloudEvent for the same source occurrence. +- Reusing a source ID with changed content is detected as a collision by the + host. +- The field catalog accepts valid selector fields and operators. +- The field catalog rejects unknown or incompatible fields and operators. +- The projector configuration decoder rejects malformed configuration. +- The projector output decoder rejects malformed event data. +- One failing projector does not suppress successful sibling projections. +- Tenant mismatches and source-kind mismatches do not project. +- Event-size and string-size limits are enforced. +- A warehouse failure leaves the event staged. +- A retry promotes the original staged event exactly once. +- Checkpoint restore, or the hosted equivalent, preserves event and consumer + state. + +For pure contract examples, start with +[`packages/eventing-core/src/registry.test.ts`](../packages/eventing-core/src/registry.test.ts). + +For durability examples, see the Local runtime and control-store tests under +[`apps/cli/test`](../apps/cli/test). + +## Review checklist + +Before registering an extension, reviewers should be able to answer yes to each +of these: + +- Is the source authenticated before adapter code runs? +- Is occurrence identity stable across retries and rebatching? +- Are source time and host acceptance time kept separate? +- Are selectable fields typed, bounded, and classified for sensitivity? +- Is projector input bounded and schema validated? +- Are projector configuration and output decoded at runtime? +- Is the projector deterministic, pure, and free of I/O? +- Is ownership of the event type, data schema, and their versions explicit? +- Does the host preserve stage → source commit → ready ordering? +- Does retry recover the original staged event instead of reevaluating it under + new configuration? +- Does the consumer use the immutable event ID as an idempotency key where the + destination supports one? +- Are external side effects kept behind the durable event and consumer boundary? + +For the underlying contracts and processing guarantees, see +[`signal-to-event-projection.md`](./signal-to-event-projection.md). + +For Maple Local's consumer administration and lease semantics, see +[`local-event-consumers.md`](./local-event-consumers.md). diff --git a/docs/local-event-consumers.md b/docs/local-event-consumers.md new file mode 100644 index 000000000..6492fcf88 --- /dev/null +++ b/docs/local-event-consumers.md @@ -0,0 +1,135 @@ +# Maple Local event consumer protocol + +Status: version 1 durable downstream-consumer boundary for the Maple Local event outbox. + +This protocol lets a local consumer deliver ready Maple CloudEvents without destructive reads or a +second delivery database. It is intentionally transport-neutral: Maple does not select a downstream +transport, store downstream credentials, or choose delivery destinations. + +## Credentials + +Maple creates two independent 32-byte hexadecimal credentials beside the configured data directory: + +- `.maintenance-token` administers projection and consumer configuration. +- `.event-consumer-token` permits only claim and acknowledgement requests. + +Both files must be real regular files. The consumer token is sent in +`x-maple-event-consumer-token`; it does not grant access to projection configuration, outbox +inspection, checkpoints, or retention controls. The existing maintenance token is sent in +`x-maple-maintenance-token` and cannot be substituted for the consumer token. + +## Consumer administration + +Consumer IDs match `^[a-z][a-z0-9._-]{0,63}$` and are unique. Disabled IDs remain reserved so an +operator cannot accidentally replace one consumer's durable position with an unrelated process. + +Register a consumer with the maintenance credential: + +```http +POST /local/eventing/consumers +Content-Type: application/json +X-Maple-Maintenance-Token: + +{"consumerId":"automation","startAt":"beginning"} +``` + +`startAt` is exact: + +- `beginning` starts immediately before the earliest ready event still retained for the tenant. +- `latest` atomically skips every ready event visible at registration and receives later events. + +Successful registration returns `201` and the consumer record. Reusing any existing or disabled ID +returns `409`. `GET /local/eventing/consumers` lists records under maintenance authorization. + +Disable a consumer explicitly: + +```http +POST /local/eventing/consumers/disable +Content-Type: application/json +X-Maple-Maintenance-Token: + +{"consumerId":"automation"} +``` + +Disabling clears any active lease and removes that cursor from the retention quorum. It does not +delete the audit record or permit the ID to be reused. + +## Claim and acknowledgement + +Claim between 1 and 1,000 ready events for a lease of 5 through 300 seconds: + +```http +POST /local/eventing/claims +Content-Type: application/json +X-Maple-Event-Consumer-Token: + +{"consumerId":"automation","limit":100,"leaseSeconds":60} +``` + +A non-empty response has this shape: + +```json +{ + "consumerId": "automation", + "leaseToken": "<64 lowercase hexadecimal characters>", + "leaseExpiresAt": "2026-08-13T16:01:00.000Z", + "throughSequence": 42, + "events": [ + { + "sequence": 42, + "event": { + "specversion": "1.0", + "id": "sha256:...", + "type": "dev.maple.example.record.observed.v1" + }, + "stagedAt": "2026-08-13T16:00:00.000Z", + "readyAt": "2026-08-13T16:00:00.010Z" + } + ] +} +``` + +The real `event` member is the complete validated CloudEvent. An empty claim returns null lease +fields and an empty event array. Only a SHA-256 hash of the lease token is stored. A second claim +while the lease is live returns `409`; at or after expiry it returns the same unacknowledged prefix, +possibly with a new token. + +After every event in the claimed batch has been accepted by the downstream system, acknowledge the +exact `throughSequence` returned by the claim: + +```http +POST /local/eventing/acks +Content-Type: application/json +X-Maple-Event-Consumer-Token: + +{"consumerId":"automation","leaseToken":"","throughSequence":42} +``` + +Partial, extended, expired, missing, and wrong-token acknowledgements return `409`. Success returns: + +```json +{ "consumerId": "automation", "acknowledgedThrough": 42, "prunedEvents": 0 } +``` + +Claims are at-least-once. A consumer crash after a downstream send and before acknowledgement causes +re-delivery after lease expiry. A consumer must therefore use the immutable Maple CloudEvent `id` as +its downstream idempotency key whenever the destination supports one. + +## Retention, capacity, and checkpoints + +Ready events are eligible for pruning only through the lowest acknowledged sequence among all active +consumers for the tenant. Maple retains the newest 1,000 otherwise-prunable ready events by default. +Disabled consumers do not block pruning; staged events are never pruned by consumer acknowledgement. +If no consumer is active, acknowledgement retention performs no deletion. + +The eventing control database uses schema 4. Schemas 1 through 3 are accepted and each migration step +is applied transactionally on open. A schema-3 database containing a staged source-backed event is +rejected before the schema-4 migration: schema 3 did not persist the normalized-source fingerprint +needed to distinguish an exact retry from source-ID reuse. Complete or explicitly abandon those +staged events with the schema-3 build before upgrading. Ready schema-3 events and older snapshots +without that unresolved state remain migratable. Restore opens and migrates a private scratch copy +before publishing restored state, so an unrecoverable legacy snapshot cannot replace the live store. +Schema-4 validation also rejects staged source-backed rows with missing or malformed fingerprints. +Consumer cursors and leases are part of the same SQLite backup as projection and outbox state. +Consumer mutations enter the server admission gate, so checkpoint exclusivity cannot capture a +half-applied claim or acknowledgement. diff --git a/docs/signal-to-event-projection.md b/docs/signal-to-event-projection.md new file mode 100644 index 000000000..240126aec --- /dev/null +++ b/docs/signal-to-event-projection.md @@ -0,0 +1,991 @@ +# Signal-to-event projection architecture + +Status: implemented on `codex/issue-222-alerting-core`; downstream delivery remains out of scope + +Related work: [issue #222](https://github.com/MapleTechLabs/maple/issues/222), +`@maple/alerting-core` + +Audience: Maple maintainers and implementers of hosted or Maple Local runtimes + +Implementers adding a source adapter or semantic projector should also read +[`eventing-extension-guide.md`](./eventing-extension-guide.md), which provides a +complete compile-time extension example, host wiring patterns, and review and +test checklists. + +## Decision summary + +Maple will treat immediate, per-occurrence event generation as an ingest concern, +not as a scheduled warehouse-query concern. + +- Each accepted OTLP record or provider webhook is decoded and normalized into a + typed signal once. +- An immutable snapshot of enabled signal projections is evaluated against that + signal before its scalar types are flattened for warehouse storage. +- Every matching projection invokes a registered, pure projector that produces a + factual [CloudEvents 1.0](https://github.com/cloudevents/spec/blob/main/cloudevents/spec.md) + event. +- Produced events enter a durable, idempotent outbox. Consumers and delivery + transports are downstream of that boundary. +- The original telemetry continues through the existing warehouse write path. +- chDB is not polled to discover newly arrived records. It remains the analytics + store and an optional, explicitly invoked replay source. +- Scheduled aggregate alerts remain query-driven. Alert lifecycle transitions + become another producer of typed events and use the same outbox as ingest-time + projections. + +The configurable matching model is a small, structured, typed predicate tree. It +is not arbitrary SQL and it is not a new textual expression language. The live +runtime evaluates the tree in memory. A warehouse adapter may lower the supported +subset to parameterized ClickHouse expressions for explicit historical replay, +but SQL behavior does not define the predicate semantics. + +## Problem + +Maple currently contains several mechanisms that are related but not expressed +through one event boundary: + +- hosted alert rules periodically query telemetry, update incident lifecycle + state, and request deliveries; +- PlanetScale receives signed webhooks and performs provider-specific work; +- Maple Local accepts OTLP records and writes them directly to chDB; +- future automation needs individual facts, such as a source record being + observed, to become events that agents or other consumers can act on. + +Using the alert scheduler for the last case would give it the wrong semantics. +A windowed query answers a question about a set of stored records and normally +produces one aggregate observation. It cannot faithfully represent every +individual occurrence without cursors, overlap windows, late-arrival handling, +and deduplication. + +The current Local `logs` table has no ingestion sequence or native event ID. Its +sort key is designed for observability queries, and arbitrary OTLP attributes are +stored as strings. Repeatedly querying that table once per rule would therefore: + +- compete with ingest, UI queries, checkpoints, retention, and archive work; +- miss late records or repeatedly rediscover records unless a second deduplication + system is added; +- require casts that cannot always recover the source value's original type; +- turn an embedded analytical database into an inefficient message queue. + +The event layer is still useful. It belongs in front of chDB for live signals, +with chDB retained behind it for analytics and aggregate alert evaluation. + +## Goals + +1. Allow operators and integrations to configure which incoming signals become + typed events without writing SQL or changing core runtime code. +2. Evaluate each delivered signal in one ingest pass against all applicable + projections; do not issue one warehouse query per projection. +3. Preserve scalar types for string, boolean, integer, floating-point, + timestamp, and duration comparisons. +4. Make source adapters, projectors, event persistence, and consumers replaceable + behind explicit interfaces. +5. Give emitted events stable identities so retries do not create duplicate + logical events when the source provides stable occurrence identity. +6. Reuse the same event envelope and outbox for query-alert lifecycle events. +7. Keep Maple Local headless: matching and event persistence must work while no + browser is open. +8. Keep the core deterministic, bounded, tenant-scoped, and independent of a + database, network, scheduler, wall clock, or particular deployment host. + +## Non-goals + +- Adding NATS, JetStream, Kafka, or another general-purpose broker as a required + Maple component. +- Loading arbitrary third-party code into a running Maple process. A "plugin" in + this document is a compile-time registered module behind a stable interface. +- Defining sink delivery, consumer-specific behavior, agent authorization, or + action policy. +- Replacing the Collector's routing, filtering, queueing, or authentication. +- Replacing scheduled queries for rates, percentiles, absence, threshold state, + or other aggregate alerts. +- Guaranteeing exactly-once external side effects across an uncooperative source, + Maple, and an arbitrary consumer. +- Automatically replaying old telemetry whenever a projection is created or + changed. +- Providing a general scripting language, joins, aggregation, arithmetic, + regular expressions, or user-provided SQL in the first version. + +## Terminology + +**Signal** +: One factual input occurrence after authentication, decoding, and normalization. +It may originate as an OTLP log/span/metric point or a provider webhook. + +**Source adapter** +: A module that verifies or accepts a source payload, normalizes occurrences into +typed signals, declares known fields, and supplies source identity when +available. + +**Signal projection** +: Durable configuration pairing a source kind, typed selector, and registered +projector. It says which source occurrences should be promoted into which +event representation. It is distinct from a downstream event subscription. + +**Selector** +: A bounded structured predicate over typed signal fields. + +**Projector** +: A pure, versioned function that maps one matching signal to a declared event +type and data schema. Provider-specific meaning belongs here rather than in the +eventing core. + +**Event** +: An immutable CloudEvents 1.0 envelope containing a typed factual payload. + +**Event outbox** +: Durable host storage that makes event creation idempotent and separates event +production from downstream delivery. + +**Event consumer** +: A downstream component interested in one or more event types. Webhooks, +automation workers, agents, and provider responses are consumer concerns, not +selector or projector concerns. + +## Architecture + +There are two intentionally different event-production paths. They converge only +after a factual event has been produced. + +```mermaid +flowchart LR + Source["OTLP or provider source"] --> Gate["Authenticate / verify"] + Gate --> Decode["Decode once"] + Decode --> Signal["Typed normalized signal"] + + Signal --> Match["Ingest-time selector evaluation"] + Match --> Project["Registered signal projector"] + Project --> Outbox["Durable event outbox"] + + Signal --> Encode["Warehouse encoder"] + Encode --> Warehouse["chDB / hosted warehouse"] + + Warehouse --> Scheduled["Scheduled aggregate query"] + Scheduled --> Lifecycle["Alert evaluation and lifecycle"] + Lifecycle --> AlertProjector["Alert lifecycle projector"] + AlertProjector --> Outbox +``` + +The upper path handles occurrences such as "this source record was observed". The +lower path handles conclusions such as "the error rate has remained above five +percent for ten minutes". Both can ultimately notify the same consumers without +pretending they have the same input or timing semantics. + +### Required module boundaries + +The architecture has four replaceable boundaries: + +1. **Source adapters** turn authenticated source payloads into typed signals. +2. **Selectors** determine whether a normalized signal qualifies. +3. **Projectors** map a qualifying signal to a typed factual event. +4. **Consumers** subscribe to event types downstream of the durable outbox. + +The eventing core owns the contracts and deterministic behavior. It does not know +about particular providers, consumers, databases, queues, or network transports. + +PlanetScale is therefore one installed composition, not the model itself. Its +module can register a webhook source adapter and PlanetScale-specific projectors. +Those projectors can be replaced or supplemented without changing the selector +evaluator or downstream event contract. Existing PlanetScale behavior can later +be moved behind consumers of those typed events without putting provider actions +inside the projector. + +## Core data contracts + +The TypeScript below is illustrative. Canonical persisted encodings must be +defined with runtime schemas and shared conformance fixtures. + +### Typed values + +```ts +type SignalScalar = + | { readonly type: "string"; readonly value: string } + | { readonly type: "boolean"; readonly value: boolean } + | { readonly type: "int64"; readonly value: string } + | { readonly type: "float64"; readonly value: number } + | { readonly type: "timestamp"; readonly value: string } + | { readonly type: "duration"; readonly value: string } +``` + +`int64` and `duration` use decimal strings in serialized form so JavaScript does +not lose precision. Runtime evaluators may compile them to native `bigint` or the +equivalent host type. Timestamp values use canonical RFC 3339 with an explicit +offset in serialized form and compare as UTC instants. Duration values represent +integer nanoseconds. `float64` values must be finite; `NaN` and infinities are +rejected during normalization. + +Arrays and objects may be preserved for projector payloads, but selectors operate +only on declared scalar fields in version 1. + +### Normalized signal + +```ts +interface NormalizedSignal { + readonly sourceKind: string + readonly source: string + readonly tenantId: string + readonly occurrenceId: string | null + readonly identityQuality: "source" | "derived" | "none" + readonly occurredAt: string + readonly observedAt: string + readonly subject: string | null + readonly fields: ReadonlyMap + readonly data: unknown +} +``` + +- `sourceKind` chooses the compatible field catalog and projector registry. +- `source` is a stable URI identifying the logical producer or integration. +- `occurrenceId` is a source-issued stable identifier when one exists. +- `identityQuality: "source"` means the adapter expects the ID to survive source + retries and rebatching. `"derived"` identifies a canonical content fingerprint + with documented collision/collapse limitations. `"none"` cannot support a + durable once-only automation guarantee. +- `occurredAt` is source event time; `observedAt` is the stable source-observation + time when the source provides one. Maple acceptance time is host control + metadata passed separately to activation gating, so retries cannot leak a new + receipt timestamp into projector output. +- `fields` contains canonical built-ins and namespaced source attributes. It must + not contain secrets merely because they were present in the incoming payload. +- `data` is a bounded, schema-validated, source-specific representation available + to compatible projectors. It may contain arrays and objects that are not + selector-addressable, but it follows the adapter's redaction policy and is not + an unvalidated raw request body. + +The source adapter must not expose an unbounded raw payload as the selector field +space or projector input. + +### Field references and catalogs + +A selector uses logical field references, never physical column names: + +```ts +interface FieldRef { + readonly namespace: "signal" | "resource" | "scope" | "attribute" | "body" + readonly key: string + readonly type: SignalScalar["type"] +} +``` + +Each source adapter exposes a field catalog for known fields. A catalog entry +declares: + +- logical name and one or more scalar types; +- allowed selector operators; +- sensitivity and whether a projector may expose it by default; +- whether historical replay is `exact`, `coerced`, or `unavailable`; +- an optional backend-owned replay binding. This binding is not user SQL. + +OTLP resource, scope, and record attributes are open-ended. A projection may +reference an uncatalogued attribute by explicitly declaring its expected scalar +type. At runtime a differently typed value does not get coerced; it does not +match, and a bounded type-mismatch metric is recorded. Source-specific modules +should publish catalogs for common attributes so users do not need to repeat +those declarations. OTLP log bodies are deliberately closed in version 1: only +the polymorphic `body:value` field is selectable, and only when the entire body +is a scalar. Structured body objects and arrays remain available to projectors +through normalized signal data but do not advertise child selector fields that +the adapter cannot populate. + +### Selector AST + +```ts +type SignalPredicate = + | { readonly op: "all"; readonly clauses: readonly SignalPredicate[] } + | { readonly op: "any"; readonly clauses: readonly SignalPredicate[] } + | { readonly op: "not"; readonly clause: SignalPredicate } + | { readonly op: "exists"; readonly field: FieldRef } + | { + readonly op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" + readonly field: FieldRef + readonly value: SignalLiteral + } + | { + readonly op: "in" + readonly field: FieldRef + readonly values: readonly SignalLiteral[] + } +``` + +Version 1 has the following semantics: + +| Operation | Supported types | Semantics | +| ------------------------ | ------------------------------------------- | ------------------------------------------------------------------------- | +| `exists` | all | True only when the field is present with a valid typed scalar. | +| `eq`, `neq` | all | Exact same-type comparison. A missing or mistyped field makes both false. | +| `gt`, `gte`, `lt`, `lte` | `int64`, `float64`, `timestamp`, `duration` | Ordered same-type comparison. | +| `contains` | `string` | Case-sensitive Unicode substring comparison. | +| `in` | all | Exact same-type membership; all literals must share the field type. | +| `all`, `any`, `not` | predicates | Total boolean composition with short-circuit evaluation. | + +There are no implicit casts. The string `"3"` is not the integer `3`; an integer +is not silently promoted to a float; and a string that resembles a date is not a +timestamp. Adapters may deliberately normalize a provider value into a declared +type, but that conversion is part of the source contract and is tested there. + +Missing values are not equivalent to null. Null source values are treated as +missing in version 1. Consequently `neq` requires a present field, whereas +`not(eq(...))` also matches a missing field. Configuration tooling should prefer +the explicit form that expresses the intended behavior. + +Validation happens before a projection can become active. Version 1 limits a +selector to: + +- nesting depth of 8; +- 64 total predicate nodes; +- 100 members in one `in` predicate; +- 4 KiB per string literal; +- no regular expressions, functions, arithmetic, joins, or user code. + +These bounds keep evaluation predictable and leave room for indexing active +projections by source kind and simple discriminating fields. `SignalScalar` +describes normalized source data and does not inherit the literal-only 4 KiB +limit; the OTLP adapter accepts source strings up to its separate 16 KiB bound. +The literal limit is normative in UTF-8 bytes. Because JSON Schema `maxLength` +counts characters rather than encoded bytes, the generated schema documents the +constraint and the shared multibyte conformance vectors enforce its exact edge. + +### Signal projection + +```ts +interface SignalProjectionSpec { + readonly id: string + readonly revision: number + readonly enabled: boolean + readonly tenantId: string + readonly sourceKind: string + readonly selector: SignalPredicate + readonly projector: { + readonly id: string + readonly version: number + readonly config: unknown + } + readonly activeFrom: string +} +``` + +Every semantic edit creates a new immutable revision. Activation is not +retroactive: the new revision sees signals accepted after the runtime atomically +installs its compiled registry snapshot. Historical processing requires an +explicit replay operation. + +Replaying the exact latest revision is a no-op only while its enabled/disabled +state still matches the active pointer. Replaying an older revision is a stale +revision conflict; an intentional rollback is a new monotonic revision that +copies the earlier configuration. + +The configuration record is data. Source adapters and projector implementations +are registered code. This is how matching remains configurable without making +authentication, provider semantics, or executable code user-supplied. + +For example, an installed source adapter and projector can use this neutral +contract: + +```json +{ + "id": "example-record-observed", + "revision": 1, + "enabled": true, + "tenantId": "local", + "sourceKind": "otel.log", + "selector": { + "op": "all", + "clauses": [ + { + "op": "eq", + "field": { "namespace": "signal", "key": "event.name", "type": "string" }, + "value": { "type": "string", "value": "example.record.observed" } + }, + { + "op": "gte", + "field": { "namespace": "attribute", "key": "record.sequence", "type": "int64" }, + "value": { "type": "int64", "value": "1" } + } + ] + }, + "projector": { "id": "example.record", "version": 1, "config": {} }, + "activeFrom": "2026-08-07T00:00:00Z" +} +``` + +The `gte` comparison above is an integer comparison, not lexicographic string +ordering. A timestamp predicate would similarly carry a `timestamp` literal and +compare normalized instants rather than formatted text. No query is generated +for either comparison on the live path. + +### Projector contract + +```ts +interface SignalProjector { + readonly id: string + readonly version: number + readonly sourceKinds: readonly string[] + readonly outputType: string + readonly dataSchema: string + readonly decodeConfig: (value: unknown) => ProjectorConfig + readonly decodeOutput: (value: unknown) => JsonValue + readonly project: (signal: NormalizedSignal, config: ProjectorConfig) => ProjectedEventData +} +``` + +A projector must be pure, deterministic, bounded, versioned, and free of I/O. It +does not invoke downstream systems, send notifications, or mutate source state. +It produces a factual event payload conforming to its declared schema. The +registry invokes the output decoder before constructing the CloudEvent, so +`dataschema` is a checked contract rather than documentation. + +The registry may include a bounded generic field-mapping projector for +operator-defined factual events. Provider modules register semantic projectors +when field copying is insufficient. No runtime module loading is required. + +### Event envelope + +Produced events use CloudEvents 1.0 structured representation: + +```json +{ + "specversion": "1.0", + "id": "sha256:...", + "source": "urn:maple:source:otel:local", + "type": "dev.maple.example.record.observed.v1", + "subject": "records/42", + "time": "2026-08-07T19:42:00.000000000Z", + "datacontenttype": "application/json", + "dataschema": "urn:maple:event-schema:example-record:v1", + "tenantid": "...", + "projectionid": "...", + "projectionrevision": 3, + "data": {} +} +``` + +Names above are illustrative until the repository reserves its canonical event +type and schema namespace. + +The event ID is deterministic when stable occurrence identity exists: + +```text +SHA-256(tenant ID, source kind, source URI, occurrence ID, projection ID, projection revision) +``` + +The hash input uses a canonical length-delimited encoding, not string +concatenation. Projector version and output schema version are already fixed by +the immutable projection revision and must be recorded with the event. + +Sensitive source details belong in `data`, under the projector's explicit schema +and redaction policy. They must not be copied into CloudEvents context attributes, +logs, metrics labels, or idempotency keys. + +## Runtime behavior + +### Projection compilation and activation + +The host loads enabled projections for a tenant, validates them against the +source and projector registries, and compiles them into immutable predicate +functions. The active registry is swapped atomically. Every decoded ingest batch +uses exactly one registry snapshot, even if configuration changes while the batch +is being processed. + +The initial implementation may evaluate all projections in the applicable +`sourceKind` bucket. The registry may later index projections by exact-match +discriminators such as event name or service name. This is an optimization and +must not alter selector semantics or ordering. + +Projection evaluation is deterministic and side-effect free. All matching +projections run; this is not first-match routing. A signal may therefore produce +zero, one, or several different factual events. + +### Maple Local OTLP ingest + +Maple Local already decodes an OTLP request and then passes the decoded payload +to the warehouse encoder. The event seam belongs between those operations. + +The implementation should refactor decoding/normalization so that: + +1. the OTLP request is parsed once; +2. typed record values remain available to the matcher; +3. the existing warehouse rows are produced without changing their stored shape; +4. matched events are staged idempotently before ingest acknowledges success; +5. the telemetry insert completes; +6. staged events are marked ready for downstream consumption; +7. only then is the OTLP request acknowledged. + +When no projection matches, the path adds only bounded predicate work before the +existing chDB insert. + +If the event store cannot stage a required event, ingest returns a retryable +failure rather than silently losing automation. A source retry reuses the same +event ID and canonical event bytes, so staging is idempotent. Durable OTLP log +projection requires `timeUnixNano` or `observedTimeUnixNano`; server receipt +time is never incorporated into durable identity or event content. +OTLP permits both timestamp fields to be absent or zero; those records remain +accepted by the warehouse path but are skipped by durable event projection. +The same isolation applies to eventing-specific normalization bounds: an +oversized attribute map or nested value makes only that source occurrence +ineligible for projection and records a bounded normalization failure. The +existing warehouse encoder still decides independently whether the OTLP record +is valid for storage, so enabling a projection does not narrow ingest. Before +full event normalization, Maple performs a tolerant, bounded extraction of the +stable source URI and source-issued occurrence ID. An ineligible occurrence +that matches an existing staged obligation fails ingest rather than silently +acknowledging a changed retry and stranding the earlier event. + +Staging and chDB insertion are not one transaction. A process crash after the +chDB insert but before the OTLP acknowledgement can still cause a duplicate raw +telemetry row on retry; that is already possible with at-least-once OTLP +delivery. The staged/ready outbox protocol prevents an event from becoming +dispatchable before the ingest attempt reaches its warehouse commit point. +Staged rows retain the source occurrence identity and original projection +revision plus a bounded hash of the normalized source content. On redelivery, +Maple recovers those exact event IDs and does not reevaluate that occurrence +against a newer or disabled projection snapshot. Recovery requires the source +hash to match; reuse of the same source identity with changed content fails as a +collision and leaves the staged event non-ready. Within one ingest batch, two +records that reuse the same tenant, source kind, source URI, and source-issued +occurrence ID must also have the same normalized source hash. Maple checks that +source tuple for every normalized occurrence before selectors divide it into +zero, one, or several projected event IDs. A projection-ineligible record that +reuses a normalized tuple makes the batch ambiguous and is rejected before any +event is staged. The fingerprint contract orders field keys by explicit +JavaScript code-unit order, not locale collation, so checkpoint recovery is +independent of host locale. + +Schema 4 introduced this source fingerprint. Opening a schema-3 control store +therefore fails before migration if it contains staged source-backed rows whose +fingerprints cannot be reconstructed. Ready rows and stores without unresolved +source-backed staging remain eligible for migration. Restore copies a signed +control snapshot into a private scratch store, opens and migrates that copy, and +serializes the validated current-schema database into the restored data +directory. An unsafe legacy snapshot therefore fails before restore readiness +or the live-directory swap; the signed checkpoint artifact itself is unchanged. + +If atomic exactly-once storage across both systems later becomes a requirement, +the correct addition is a durable ingress journal before both writes. chDB +polling does not solve that problem. + +### Provider webhooks + +Provider authentication and replay protection run before normalization. The +host must establish a durable event boundary before acknowledging the provider. +The hosted PlanetScale route therefore requires the provider timestamp, +projects a verified payload first, and enqueues only the resulting canonical +CloudEvent plus bounded routing metadata; the queue is its durable event +boundary. The complete serialized job is measured against a 120 KiB cap before +send; oversized factual payloads receive a deterministic `413` rather than a +retryable queue failure. Consumers continue to read legacy payload-only and +transitional jobs. + +Current queue jobs are decoded as one relational contract: the event tenant, +source, embedded connection, type, schema, and timestamp must agree with the +bounded routing fields. Unsupported or contradictory jobs are terminally +acknowledged as poison messages. Health-event issue mutations use a durable +`(org_id, event_id)` receipt inserted in the same PostgreSQL transaction as the +issue mutation; timeline insertion remains independently idempotent. A retry +after a failure between those phases therefore completes the issue once without +duplicating its occurrence count or history. Transactions also take a scoped +lock for `(org_id, issue fingerprint)` before claiming the receipt. Concurrent, +distinct events for the same issue are therefore all counted, while only one +transition reopens a resolved issue. + +The provider source adapter supplies the strongest available delivery or event +identity. It then uses the same selector, projector, event ID, and outbox +contracts as OTLP. Provider-specific response behavior does not live in the core; +it can be migrated behind consumers of the emitted event types. + +### Query-driven alerts + +Scheduled alert rules retain their existing execution model: + +1. the host schedules and claims a rule; +2. a warehouse query produces an aggregate `AlertObservation`; +3. `@maple/alerting-core` evaluates threshold and lifecycle state; +4. an alert lifecycle projector converts `trigger`, `resolve`, `renotify`, or + `test` intent into a CloudEvent; +5. the host persists it through the common event outbox. + +This path queries chDB or the hosted warehouse because its input is an aggregate +over time. It does not reuse the ingest-time signal selector, and the ingest-time +path does not impersonate an alert incident. + +### Historical replay + +Replay is an operator-invoked batch operation, never the live event mechanism. +It evaluates one projection revision over a bounded time range and must support a +dry-run count/sample mode before it can persist events. + +Every field catalog entry declares replay capability: + +- `exact`: stored data retains enough type and identity information to reproduce + live semantics; +- `coerced`: the adapter can apply an explicit cast, but the source type was lost + or identity is derived; +- `unavailable`: the backend cannot implement the live predicate faithfully. + +A replay request using a `coerced` field requires explicit operator +acknowledgement. A request using an unavailable field is rejected. The warehouse +compiler emits parameterized expressions through existing query-building +facilities; it never interpolates field names or literals supplied directly by a +user. + +Current Local OTLP attribute maps store strings, so arbitrary typed attributes +will generally be `coerced`, not `exact`. Replay event IDs are guaranteed to +deduplicate against live events only when the warehouse retained the same stable +source occurrence ID. + +## Processing and delivery guarantees + +The architecture uses precise, layered guarantees rather than the blanket phrase +"exactly once". + +| Boundary | Guarantee | +| -------------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| Source to Maple | At least once when the source/Collector retries; source-specific otherwise. | +| One accepted batch | One evaluation against one immutable projection-registry snapshot. | +| Projection with source-stable identity | Effectively-once event creation through deterministic ID plus unique outbox insertion. | +| Projection with derived identity | Best-effort deduplication; identical real occurrences may collapse and re-encoded retries may diverge. | +| Projection with no identity | At-least-once event creation only; durable automation should reject this configuration by default. | +| Outbox to consumer | At least once with an event ID/idempotency key; consumer-side external effects are outside this specification. | +| chDB telemetry row | Existing OTLP semantics; duplicate storage remains possible after ambiguous failures. | + +A projection intended to trigger external automation must require +`identityQuality: "source"` unless an operator explicitly accepts weaker +semantics. An installed source adapter should therefore furnish a stable event +or delivery identifier as part of its source contract. + +## chDB responsibilities + +chDB is responsible for: + +- storing telemetry for interactive and analytical queries; +- serving scheduled aggregate-alert queries; +- serving bounded explicit replay where field capabilities allow it; +- participating in existing checkpoint, retention, and archive workflows. + +chDB is not responsible for: + +- acting as a live queue; +- maintaining one cursor per signal projection; +- deduplicating event delivery; +- storing mutable projection configuration or delivery attempts merely because + it stores the source telemetry; +- defining selector type semantics through ClickHouse casts. + +Version 1 requires no new column or sort-key change to the existing telemetry +tables. A future narrow event journal or ingress-identity column may improve +replay, but it must be justified separately and must not turn wide raw-telemetry +tables into queue state. + +## Alternatives considered + +| Alternative | Decision | +| ------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| One periodic chDB query per projection | Rejected. It repeats wide scans, introduces cursor/late-arrival problems, and competes with the analytical workload. | +| One shared query that tails all recent chDB rows | Rejected as the live path. It reduces query count but still lacks a reliable ingestion cursor and evaluates after scalar type loss. It may inform an explicit replay implementation. | +| ClickHouse materialized views per projection | Rejected. Mutable user configuration would become DDL, current attribute storage has already flattened types, and lifecycle/deduplication state still needs another store. | +| Collector OTTL as Maple's rule language | Kept as an optional deployment optimization. It is valuable for OTel-only routing but does not define provider-webhook behavior or Maple-managed dynamic configuration. | +| CEL as the first expression language | Deferred. CEL is safe and capable, but embedding compatible runtimes and defining warehouse lowering is more surface than the initial predicates require. Reconsider it if the bounded AST is demonstrably insufficient. | +| CloudEvents SQL as the signal selector | Rejected for raw signals. [CESQL 1.0](https://github.com/cloudevents/spec/blob/main/cesql/spec.md) filters CloudEvent context attributes but does not address arbitrary event `data`; it may be useful for downstream CloudEvent subscriptions. | +| NATS or another broker as the event abstraction | Rejected as a requirement. A broker can later implement an event transport port, but it does not replace source normalization, selector semantics, projectors, identity, or host persistence. | +| A custom textual DSL | Rejected. The structured predicate tree is the persisted intermediate representation; configuration UIs and APIs do not need a parser. | + +## Durable host ports + +The core needs interfaces rather than a prescribed database: + +```ts +interface SignalProjectionStore { + loadEnabled(tenantId: string): Promise +} + +interface EventOutboxStore { + stage(events: readonly CloudEvent[]): Promise + markReady(eventIds: readonly string[]): Promise +} +``` + +The real contracts also need revision/change notification, unique event IDs, +bounded batch operations, health inspection, and recovery of staged records. + +Hosted Maple may implement these ports with its relational state and queue +infrastructure. Maple Local needs a small transactional control-state store whose +rules, outbox, and migration identity survive restart. That state is not covered +by chDB checkpoints automatically; backup, restore, and schema migration are part +of the Local host adapter's acceptance criteria. + +The physical Local store is an implementation decision, but it must provide: + +- uniqueness on event ID; +- atomic projection revision writes; +- atomic event staging and readiness transitions; +- bounded recovery of stranded staged events; +- crash-safe migrations and explicit backup/restore behavior; +- no dependency on a browser process. + +## Package and host ownership + +The intended ownership is: + +- `packages/eventing-core` (new): language-neutral schemas, selector validation, + the reference TypeScript evaluator, projector registry contracts, canonical + event identity, and conformance fixtures. No database, network, scheduler, or + global clock dependencies. +- `packages/alerting-core` (existing): aggregate alert evaluation and incident + lifecycle. It remains distinct and later emits through an eventing-core port. +- `packages/domain`: public/API schemas when projection CRUD becomes public. +- `apps/cli`: Maple Local OTLP source adapter, compiled-registry lifecycle, + durable Local ports, ingest staging, and optional replay adapter. +- `apps/api`: provider webhook adapters and hosted persistence wiring. +- `apps/ingest`: a future Rust OTLP adapter only when hosted per-signal projection + is required. + +The canonical JSON schemas and fixture corpus, rather than TypeScript source +types, define cross-language behavior. A Rust implementation must pass the same +valid/invalid selector cases, typed comparison cases, canonical event-ID vectors, +and projection fixtures before it can claim compatibility. + +[OpenTelemetry Transformation Language](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/pkg/ottl) +can remain a Collector-side optimization or adapter. It is not the universal +Maple contract because it is coupled to OTel Collector contexts and does not +cover provider webhooks. [CEL](https://cel.dev/overview/cel-overview) is the +preferred language to reconsider if real requirements outgrow the bounded AST; +version 1 does not embed CEL runtimes or define a CEL-to-ClickHouse compiler. + +## Security and tenancy + +- Authentication or provider signature verification occurs before a source + adapter may produce a signal. +- Every signal, projection, event, and outbox operation carries an explicit + tenant ID. Cross-tenant registry lookup or event fanout is forbidden. +- User configuration cannot name SQL columns, inject SQL fragments, load code, + call functions, or select secrets outside the source field catalog. +- Source adapters mark sensitive fields. Generic projectors exclude them by + default; provider projectors must opt in deliberately and document why. +- Projected event size and source-field size are bounded before outbox insertion. +- Runtime errors and telemetry must not record full sensitive payloads. +- Sink URL validation, private-network policy, signing, and agent authorization + remain downstream policies. General eventing must not weaken hosted SSRF + protections. + +## Failure handling and observability + +Malformed projection configuration is rejected before activation. The reference +evaluator is total: missing fields and runtime type mismatches produce defined +non-matches rather than exceptions. + +A projector must return either schema-valid event data or a bounded typed +projection failure. A bad occurrence must not create an infinite source retry +loop. The host records the failure against projection ID/revision and occurrence +identity, exposes degraded health, and quarantines or dead-letters according to a +bounded policy. Exact quarantine policy belongs to the host adapter, but silently +dismissing a durable projection failure is not allowed. + +Required low-cardinality telemetry includes: + +- received signals by source kind; +- selector evaluations and matches by projection ID; +- bounded selector type-mismatch counts by source kind, without arbitrary open + field names as metric labels; +- projection failures; +- outbox staged, deduplicated, ready, and stranded counts; +- evaluation and staging latency; +- active projection count and registry revision; +- replay scanned, matched, emitted, and deduplicated counts. + +Raw field values, subjects, event IDs, and arbitrary event types must not become +unbounded metric labels. + +Maple Local implements the ingest-time subset as +`maple.eventing.operations_total`, `maple.eventing.operation_duration_ms`, and +`maple.eventing.consumer_lag_events`. Their only attributes are bounded +`operation`, `outcome`, and `source_kind` values. The operations cover +normalization, projection success/failure, outbox stage/ready/deduplication, and +consumer claim/ack/lease/lag. Tenant IDs, projection and consumer IDs, event +types and IDs, URLs, payload fields, lease tokens, and credentials are never +metric attributes. Replay and stranded-outbox telemetry remain applicable only +when those optional host operations run. + +## Compatibility and migration + +This design extends rather than replaces the host-neutral alert-core extraction +already on the issue-222 branch. + +1. Existing hosted aggregate alerts continue using their scheduler, query, + lifecycle, and delivery behavior while the event contract is introduced. +2. The new eventing core lands without runtime activation and with conformance + fixtures. +3. Maple Local adds ingest-time projection behind an explicit feature/config + gate. With no active projections, observable ingest and chDB behavior remain + unchanged. +4. A neutral OTLP record fixture proves the end-to-end source identity, typed + selector, projector, retry deduplication, and durable outbox path. +5. PlanetScale is adapted behind the same source/projector interfaces while its + existing externally visible behavior remains intact. A compare/dual-observe + period should precede removal of direct hard-coded handling. +6. Alert lifecycle intents are projected into the same CloudEvents/outbox model + after parity tests show no change to trigger, resolve, renotify, test, + suppression, or retry semantics. +7. Warehouse replay is added only after the live path is proven and replay + capability metadata is implemented. + +No migration step requires NATS, a per-rule chDB cursor, or a new raw-telemetry +sort key. + +## Implementation slices for the next goal + +### Slice 1 — Contract and evaluator + +- Add `packages/eventing-core`. +- Define runtime schemas for typed values, fields, predicates, projection specs, + projector registrations, and CloudEvent output. +- Implement validation, compilation, and the pure reference evaluator. +- Add canonical JSON and event-ID test vectors. +- Add complexity-limit and hostile-input tests. + +### Slice 2 — Local durable control state + +- Select and document the Local transactional store. +- Implement projection revision and outbox ports, migrations, recovery, and + backup/restore hooks. +- Expose headless health inspection before UI work. + +### Slice 3 — Local ingest seam + +- Refactor OTLP normalization to preserve typed values without decoding twice. +- Load and atomically swap compiled projection snapshots. +- Stage matching events, insert telemetry, mark events ready, and acknowledge. +- Prove that the live path executes no chDB `SELECT` and adds no scheduler. + +### Slice 4 — Example extension: source record to durable Maple event + +- Define a source adapter's OTLP field contract and stable occurrence identity. +- Register its field catalog and a pure semantic projector outside the core. +- Configure a record-observed projection without hard-coded selector values in + the evaluator. +- Verify duplicate source deliveries create one logical outbox event. + +This slice stops at the outbox. Transport and agent-action behavior are +downstream concerns using the produced typed event. + +### Slice 5 — Existing producer convergence + +- Adapt PlanetScale webhook inputs to the source/projector contracts. +- Project alert lifecycle intents into CloudEvents. +- Preserve existing provider and alert behavior with parity fixtures before + switching consumers. + +### Slice 6 — Optional replay + +- Add per-field replay capability declarations. +- Implement bounded dry-run and explicit emission modes. +- Add evaluator-versus-ClickHouse conformance tests for every `exact` binding. + +## Acceptance criteria + +The first usable implementation is complete when all of the following are true: + +1. A configured OTLP record signal is matched before chDB encoding and produces + a schema-valid CloudEvent while the telemetry record is still stored normally. +2. Re-delivery of a source-stable occurrence produces the same event ID and one + logical outbox record. +3. A nonmatching signal performs no warehouse read and creates no event. +4. Several active projections are evaluated from one registry snapshot, and all + matches run. +5. Integer, float, timestamp, duration, boolean, and string truth-table fixtures + pass with no implicit coercion. +6. Projection changes are validated, revisioned, persisted, and activated + atomically without restarting Maple Local. +7. Rules and ready/staged outbox records survive process restart and participate + in documented backup and recovery. +8. chDB query alerts retain their existing aggregate and lifecycle behavior. +9. No implementation requires a browser, a new broker, arbitrary runtime code, + raw SQL configuration, or a per-projection chDB poller. +10. The event envelope and selector fixture corpus are sufficient for a second + language implementation to demonstrate semantic parity. + +## Settled implementation choices + +The TypeScript reference implementation settles the remaining host choices as +follows: + +- Maple Local stores projection revisions, failures, and the staged/ready outbox + in SQLite at `/control/eventing.sqlite`, using WAL and `synchronous = +FULL`. While ingest is quiesced, backup first completes and verifies a blocking + `wal_checkpoint(TRUNCATE)` so the serialized database contains every committed + control-store transaction rather than only the main SQLite file. A version-2 + Maple checkpoint contains `control.sqlite` beside the chDB + backup and binds its byte count, SHA-256 digest, schema version, and row counts + in the checkpoint manifest. Version-1 checkpoints remain readable and restore + an empty control store. +- The reference OTLP extension example uses a LogRecord event name such as + `example.record.observed`. An installed adapter defines its own accepted stable + occurrence identifiers, field catalog, validation rules, and semantic + projector. The eventing core neither synthesizes provider fields nor assigns + provider meaning to arbitrary attributes. +- Maple-owned event types use `dev.maple.*.v1`; schemas use + `urn:maple:event-schema:*:v1`. Installed projectors reserve their concrete + event type and schema names; neutral fixtures use + `dev.maple.example.record.observed.v1` with + `urn:maple:event-schema:example-record:v1`. +- Attribute strings are limited to 16 KiB, source/event identities to 256 + characters (long stable inputs are represented by a SHA-256 URN), each + attribute namespace to 256 entries, + nested values to depth 8 and 1,024 nodes, normalized source data to 256 KiB, + and a canonical outbox CloudEvent to 256 KiB. Secret-like attribute names are + excluded from the projection field and data views. +- The Local TypeScript path is the reference live implementation. Hosted Rust + ingest remains a later adapter and must pass the shared schemas and fixture + corpus before claiming parity. +- Verified non-test PlanetScale webhooks run through a registered + `planetscale.webhook` source adapter, selector, and projector before the route + acknowledges them. The dedicated Cloudflare Queue durably carries + `dev.maple.planetscale.webhook.received.v1` without duplicating the provider + payload. Queue consumers accept the event-only message plus transitional and + exact pre-migration shapes, reconstructing the deterministic event from older + timestamped messages during rolling upgrades. Timestamp-less legacy jobs are + terminally acknowledged without durable projection. +- Hosted query-alert delivery rows remain that producer's durable outbox. Their + payload now includes an additive deterministic + `dev.maple.alert.lifecycle.{trigger,resolve,renotify,test}.v1` CloudEvent while + retaining every legacy top-level delivery field. Retry creation preserves the + originally stored JSON, including the CloudEvent ID and future additive fields, + instead of round-tripping it through a lossy legacy schema. +- Historical replay execution remains deliberately unimplemented in this + change. Field catalogs already declare `exact`, `coerced`, or `unavailable`, + but Local's current arbitrary attribute maps have lost source scalar type and + its warehouse rows do not furnish a native occurrence ID. A later bounded, + operator-invoked replay adapter must require explicit coercion acknowledgement + and pass live-evaluator conformance tests; the live path never falls back to a + chDB poller in the meantime. +- Projector failures with a source occurrence ID are idempotent per projection + revision. Local retains a bounded newest 10,000 failure rows per tenant and + exposes the count through the authenticated headless health endpoint. A + projector failure does not retry a valid telemetry occurrence forever; + infrastructure failure to persist required state remains retryable. + +Maple Local activates immutable revisions with authenticated +`POST /local/eventing/projections`. The same maintenance credential protects +`GET /local/eventing/projections`, `/local/eventing/health`, +`/local/eventing/outbox`, and consumer administration. Ready records receive a separate, append-only +readiness `sequence` on their first staged-to-ready transition; +`?after=&limit=` therefore cannot skip an older staged event that is +recovered after newer events were already read. `?state=staged` uses the original +staging sequence for bounded inspection of records stranded before the chDB +commit point. The Local store defaults to at most 10,000 events and 256 MiB of +canonical event JSON. Staging fails closed with a retryable ingest error before +either cap can be exceeded. Inspection remains non-destructive. Named downstream +consumers use the separate [Maple Local event consumer protocol](./local-event-consumers.md) for +leased, at-least-once claims and exact whole-batch acknowledgement. Ready-event pruning advances only +through the slowest active consumer and retains a bounded acknowledged tail; staged events are never +pruned by delivery acknowledgement. + +Re-delivery is the safe recovery operation: it locates staged rows by stable +source occurrence, preserves their original projection snapshot, and promotes +those exact event IDs only after the warehouse write succeeds. Maple never blindly +promotes an old staged record because, after a crash, the control store alone +cannot prove whether the corresponding chDB write committed. Activation requires +authentication, a bounded request body, structural budget validation, and full +registry compilation before acquiring global quiescence. Only the projection +revision commit and immutable runtime-registry swap occur while ingest is +quiesced, so invalid credentials, incomplete bodies, and expensive validation do +not close admission and every ingest request still observes exactly one registry +version. Concurrent maintenance requests receive an intentional conflict response. diff --git a/packages/alerting-core/README.md b/packages/alerting-core/README.md new file mode 100644 index 000000000..0571b49e0 --- /dev/null +++ b/packages/alerting-core/README.md @@ -0,0 +1,40 @@ +# `@maple/alerting-core` + +Host-neutral alert evaluation and incident-lifecycle semantics shared by Maple +deployment targets. + +The core is deliberately free of database, telemetry warehouse, scheduler, +network, and wall-clock dependencies. A host supplies observations and durable +state, calls the pure decision functions, then applies the returned transition +and delivery intent through its own adapters. + +Current hosted adapters live in `apps/api` and are scheduled by +`apps/alerting`. A Maple Local adapter can use the same core with chDB-backed +queries, Local durable state, an in-process scheduler, and its own outbound URL +policy without importing either hosted application. + +This package covers scheduled aggregate alerts. Immediate per-occurrence events +use the separate ingest-time architecture described in +[`docs/signal-to-event-projection.md`](../../docs/signal-to-event-projection.md). +Both paths may ultimately publish through the same typed event outbox, but raw +signal matching does not poll chDB or impersonate an alert lifecycle. + +The boundary is: + +- query adapter -> `AlertObservation`; +- evaluation policy + observation -> `AlertEvaluation`; +- persistence snapshot + evaluation -> `AlertLifecyclePlan`; +- host persists the plan, projects its optional `eventType` into the common + CloudEvents envelope, and sends that event through a delivery adapter; +- delivery adapters share idempotency-key and bounded retry policy helpers; +- host clock supplies `nowMs`; the core never reads global time. + +Rule CRUD, storage schemas, scheduler claims, destination configuration, and +delivery transports remain host concerns. This keeps Local UI work optional: +the alert runtime can evaluate and deliver while no browser is open. + +Hosted alert delivery rows are the existing durable outbox for this producer. +Their additive `event` payload contains the deterministic +`dev.maple.alert.lifecycle.{trigger,resolve,renotify,test}.v1` envelope; current +destinations continue to receive the legacy top-level payload fields during the +migration. diff --git a/packages/alerting-core/package.json b/packages/alerting-core/package.json new file mode 100644 index 000000000..9c7034969 --- /dev/null +++ b/packages/alerting-core/package.json @@ -0,0 +1,21 @@ +{ + "name": "@maple/alerting-core", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "test": "vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@maple/eventing-core": "workspace:*" + }, + "devDependencies": { + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:" + } +} diff --git a/packages/alerting-core/src/index.test.ts b/packages/alerting-core/src/index.test.ts new file mode 100644 index 000000000..f0786e57a --- /dev/null +++ b/packages/alerting-core/src/index.test.ts @@ -0,0 +1,227 @@ +import { describe, expect, it } from "vitest" +import { + alertDeliveryRetryDelayMs, + canRetryAlertDelivery, + evaluateAlertObservation, + interleaveAlertRulesByTenant, + makeAlertDeliveryKey, + planAlertLifecycle, + projectAlertLifecycleEvent, + type AlertEvaluation, +} from "./index" + +const breached: AlertEvaluation = { + status: "breached", + value: 11, + sampleCount: 5, + threshold: 10, + thresholdUpper: null, + comparator: "gt", + reason: "above threshold", + derivedFromNoData: false, +} + +const healthy: AlertEvaluation = { ...breached, status: "healthy", value: 9 } + +const policy = { + consecutiveBreachesRequired: 2, + consecutiveHealthyRequired: 2, + renotifyIntervalMinutes: 10, +} + +describe("evaluateAlertObservation", () => { + it("applies thresholds and rounds weighted sample counts", () => { + expect( + evaluateAlertObservation( + { + comparator: "between", + threshold: 10, + thresholdUpper: 20, + minimumSampleCount: 2, + noDataBehavior: "skip", + }, + { value: 15, sampleCount: 2.4, hasData: true }, + "inside range", + ), + ).toMatchObject({ status: "breached", sampleCount: 2, reason: "inside range" }) + }) + + it("marks a zero synthesized from no data so lifecycle resolution can fail closed", () => { + expect( + evaluateAlertObservation( + { + comparator: "gt", + threshold: 10, + thresholdUpper: null, + minimumSampleCount: 0, + noDataBehavior: "zero", + }, + { value: null, sampleCount: 0, hasData: false }, + "above threshold", + ), + ).toMatchObject({ status: "healthy", value: 0, derivedFromNoData: true }) + }) +}) + +describe("planAlertLifecycle", () => { + it("opens only after the configured breach count", () => { + const first = planAlertLifecycle({ + policy, + evaluation: breached, + state: null, + openIncident: null, + nowMs: 1_000, + }) + expect(first).toMatchObject({ transition: "none", state: { consecutiveBreaches: 1 } }) + + const second = planAlertLifecycle({ + policy, + evaluation: breached, + state: first.state, + openIncident: null, + nowMs: 2_000, + }) + expect(second).toMatchObject({ transition: "opened", eventType: "trigger" }) + }) + + it("suppresses a flapping trigger and its matching resolve", () => { + const opened = planAlertLifecycle({ + policy, + evaluation: breached, + state: { consecutiveBreaches: 1, consecutiveHealthy: 0 }, + openIncident: null, + nowMs: 600_000, + previousNotificationAtMs: 300_000, + }) + expect(opened).toMatchObject({ + transition: "opened", + eventType: null, + notificationSuppression: "flapping", + inheritedNotificationAtMs: 300_000, + }) + + const resolved = planAlertLifecycle({ + policy, + evaluation: healthy, + state: { consecutiveBreaches: 0, consecutiveHealthy: 1 }, + openIncident: { + firstTriggeredAtMs: 600_000, + lastNotifiedAtMs: opened.inheritedNotificationAtMs, + lastDeliveredEventType: null, + }, + nowMs: 700_000, + }) + expect(resolved).toMatchObject({ + transition: "resolved", + eventType: null, + notificationSuppression: "flap_resolution", + }) + }) + + it("advances the notification anchor when renotify becomes due", () => { + const plan = planAlertLifecycle({ + policy, + evaluation: breached, + state: { consecutiveBreaches: 2, consecutiveHealthy: 0 }, + openIncident: { + firstTriggeredAtMs: 0, + lastNotifiedAtMs: 1_000, + lastDeliveredEventType: "trigger", + }, + nowMs: 601_000, + }) + expect(plan).toMatchObject({ + transition: "continued", + eventType: "renotify", + advanceNotificationAnchor: true, + }) + }) + + it("holds a no-data recovery until the host proves telemetry liveness", () => { + const noDataHealthy = { ...healthy, derivedFromNoData: true } + const input = { + policy, + evaluation: noDataHealthy, + state: { consecutiveBreaches: 0, consecutiveHealthy: 1 }, + openIncident: { + firstTriggeredAtMs: 0, + lastNotifiedAtMs: 0, + lastDeliveredEventType: "trigger" as const, + }, + nowMs: 1_000, + } + expect(planAlertLifecycle(input)).toMatchObject({ transition: "none", hold: "missing_telemetry" }) + expect(planAlertLifecycle({ ...input, allowNoDataResolution: true })).toMatchObject({ + transition: "resolved", + eventType: "resolve", + }) + }) +}) + +describe("interleaveAlertRulesByTenant", () => { + it("preserves each tenant's order while round-robining tenants", () => { + const rows = [ + { tenantId: "a", id: "a1" }, + { tenantId: "a", id: "a2" }, + { tenantId: "b", id: "b1" }, + { tenantId: "a", id: "a3" }, + { tenantId: "b", id: "b2" }, + ] + expect(interleaveAlertRulesByTenant(rows, (row) => row.tenantId).map(({ id }) => id)).toEqual([ + "a1", + "b1", + "a2", + "b2", + "a3", + ]) + }) +}) + +describe("delivery policy", () => { + it("projects lifecycle intents into deterministic common CloudEvents", () => { + const input = { + tenantId: "org-1", + ruleId: "rule-1", + ruleName: "High errors", + incidentId: "incident-1", + eventType: "trigger" as const, + incidentStatus: "open", + groupKey: "checkout", + signalType: "error_rate", + severity: "critical", + comparator: "gt" as const, + threshold: 5, + thresholdUpper: null, + windowMinutes: 5, + value: 7.2, + sampleCount: 12, + occurredAtMs: 1_786_131_720_123, + } + const event = projectAlertLifecycleEvent(input) + expect(event).toEqual(projectAlertLifecycleEvent(input)) + expect(event).toMatchObject({ + type: "dev.maple.alert.lifecycle.trigger.v1", + subject: "alert-incidents/incident-1", + tenantid: "org-1", + projectionid: "alert-lifecycle", + data: { eventType: "trigger", incidentId: "incident-1" }, + }) + expect(() => projectAlertLifecycleEvent({ ...input, occurredAtMs: Number.MAX_SAFE_INTEGER })).toThrow( + "outside the supported date range", + ) + }) + + it("builds stable idempotency keys", () => { + expect(makeAlertDeliveryKey("incident", "destination", "trigger", 42)).toBe( + "incident:destination:trigger:42", + ) + }) + + it("caps exponential retry delay and attempts", () => { + expect(alertDeliveryRetryDelayMs(1, 123)).toBe(60_123) + expect(alertDeliveryRetryDelayMs(5, 999)).toBe(900_999) + expect(canRetryAlertDelivery(4, true)).toBe(true) + expect(canRetryAlertDelivery(5, true)).toBe(false) + expect(canRetryAlertDelivery(1, false)).toBe(false) + }) +}) diff --git a/packages/alerting-core/src/index.ts b/packages/alerting-core/src/index.ts new file mode 100644 index 000000000..6d5b523d9 --- /dev/null +++ b/packages/alerting-core/src/index.ts @@ -0,0 +1,404 @@ +import { makeCloudEvent, type MapleCloudEvent } from "@maple/eventing-core" + +export type AlertComparator = "gt" | "gte" | "lt" | "lte" | "eq" | "neq" | "between" | "not_between" + +export type AlertEvaluationStatus = "breached" | "healthy" | "skipped" + +export interface AlertObservation { + readonly value: number | null + readonly sampleCount: number + readonly hasData: boolean +} + +export interface AlertEvaluationPolicy { + readonly comparator: AlertComparator + readonly threshold: number + readonly thresholdUpper: number | null + readonly minimumSampleCount: number + readonly noDataBehavior: "skip" | "zero" +} + +export interface AlertEvaluation { + readonly status: AlertEvaluationStatus + readonly value: number | null + readonly sampleCount: number + readonly threshold: number + readonly thresholdUpper: number | null + readonly comparator: AlertComparator + readonly reason: string + /** A healthy result derived from an empty window synthesized as zero. */ + readonly derivedFromNoData: boolean +} + +export const compareAlertThreshold = ( + value: number, + comparator: AlertComparator, + threshold: number, + thresholdUpper: number | null = null, +): boolean => { + switch (comparator) { + case "gt": + return value > threshold + case "gte": + return value >= threshold + case "lt": + return value < threshold + case "lte": + return value <= threshold + case "eq": + return value === threshold + case "neq": + return value !== threshold + case "between": + return thresholdUpper != null && value >= threshold && value <= thresholdUpper + case "not_between": + return thresholdUpper != null && (value < threshold || value > thresholdUpper) + } +} + +export const evaluateAlertObservation = ( + policy: AlertEvaluationPolicy, + observation: AlertObservation, + reason: string, +): AlertEvaluation => { + // Sample-weighted counts can be fractional while durable alert state commonly + // stores an integer. Normalize at the host-neutral boundary. + const sampleCount = Math.round(observation.sampleCount) + const value = observation.hasData ? observation.value : policy.noDataBehavior === "zero" ? 0 : null + + if (!observation.hasData && policy.noDataBehavior === "skip") { + return { + status: "skipped", + value: null, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: "No data in the selected window", + derivedFromNoData: false, + } + } + + if (sampleCount < policy.minimumSampleCount) { + return { + status: "skipped", + value, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: `Sample count ${sampleCount} is below minimum ${policy.minimumSampleCount}`, + derivedFromNoData: false, + } + } + + if (value == null) { + return { + status: "skipped", + value: null, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason: "Alert evaluation did not return a scalar value", + derivedFromNoData: false, + } + } + + return { + status: compareAlertThreshold(value, policy.comparator, policy.threshold, policy.thresholdUpper) + ? "breached" + : "healthy", + value, + sampleCount, + threshold: policy.threshold, + thresholdUpper: policy.thresholdUpper, + comparator: policy.comparator, + reason, + derivedFromNoData: !observation.hasData, + } +} + +export interface AlertLifecyclePolicy { + readonly consecutiveBreachesRequired: number + readonly consecutiveHealthyRequired: number + readonly renotifyIntervalMinutes: number +} + +export interface AlertLifecycleState { + readonly consecutiveBreaches: number + readonly consecutiveHealthy: number +} + +export interface AlertLifecycleIncident { + readonly firstTriggeredAtMs: number + readonly lastNotifiedAtMs: number | null + readonly lastDeliveredEventType: AlertEventType | null +} + +export type AlertEventType = "trigger" | "resolve" | "renotify" | "test" +export type AlertIncidentTransition = "none" | "opened" | "continued" | "resolved" +export type AlertNotificationSuppression = "flapping" | "flap_resolution" | null +export type AlertLifecycleHold = "missing_telemetry" | null + +export interface AlertLifecycleEventInput { + readonly tenantId: string + readonly ruleId: string + readonly ruleName: string + readonly incidentId: string | null + readonly eventType: AlertEventType + readonly incidentStatus: string + readonly groupKey: string | null + readonly signalType: string + readonly severity: string + readonly comparator: AlertComparator + readonly threshold: number + readonly thresholdUpper: number | null + readonly windowMinutes: number + readonly value: number | null + readonly sampleCount: number | null + readonly occurredAtMs: number +} + +/** Project a query-alert lifecycle intent into the common factual event envelope. */ +export const projectAlertLifecycleEvent = (input: AlertLifecycleEventInput): MapleCloudEvent => { + if (!Number.isSafeInteger(input.occurredAtMs) || input.occurredAtMs < 0) + throw new Error("alert lifecycle event time must be a non-negative epoch millisecond") + const occurredAtDate = new Date(input.occurredAtMs) + if (Number.isNaN(occurredAtDate.getTime())) + throw new Error("alert lifecycle event time is outside the supported date range") + const occurredAt = occurredAtDate.toISOString() + const occurrenceId = `${input.incidentId ?? input.ruleId}:${input.eventType}:${input.occurredAtMs}` + return makeCloudEvent({ + signal: { + sourceKind: "alert.lifecycle", + source: `urn:maple:alert-rule:${input.ruleId}`, + tenantId: input.tenantId, + occurrenceId, + identityQuality: "source", + occurredAt, + observedAt: occurredAt, + subject: + input.incidentId === null + ? `alert-rules/${input.ruleId}` + : `alert-incidents/${input.incidentId}`, + fields: new Map(), + data: {}, + }, + projection: { + id: "alert-lifecycle", + revision: 1, + enabled: true, + tenantId: input.tenantId, + sourceKind: "alert.lifecycle", + selector: { + op: "exists", + field: { namespace: "signal", key: "event_type", type: "string" }, + }, + projector: { id: "alert.lifecycle", version: 1, config: {} }, + activeFrom: occurredAt, + }, + projectorId: "alert.lifecycle", + projectorVersion: 1, + outputType: `dev.maple.alert.lifecycle.${input.eventType}.v1`, + dataSchema: "urn:maple:event-schema:alert-lifecycle:v1", + data: { + eventType: input.eventType, + incidentId: input.incidentId, + incidentStatus: input.incidentStatus, + rule: { + id: input.ruleId, + name: input.ruleName, + signalType: input.signalType, + severity: input.severity, + groupKey: input.groupKey, + comparator: input.comparator, + threshold: input.threshold, + thresholdUpper: input.thresholdUpper, + windowMinutes: input.windowMinutes, + }, + observed: { value: input.value, sampleCount: input.sampleCount }, + }, + }) +} + +export interface AlertLifecycleInput { + readonly policy: AlertLifecyclePolicy + readonly evaluation: AlertEvaluation + readonly state: AlertLifecycleState | null + readonly openIncident: AlertLifecycleIncident | null + readonly nowMs: number + /** Most recent notification for a resolved incident with the same rule and group. */ + readonly previousNotificationAtMs?: number | null + /** Set only after the host's telemetry-query adapter proves data is still arriving. */ + readonly allowNoDataResolution?: boolean +} + +export interface AlertLifecyclePlan { + readonly state: AlertLifecycleState + readonly transition: AlertIncidentTransition + readonly eventType: AlertEventType | null + readonly notificationSuppression: AlertNotificationSuppression + readonly hold: AlertLifecycleHold + /** Notification anchor to copy to a newly opened, flap-suppressed incident. */ + readonly inheritedNotificationAtMs: number | null + /** Whether the host must advance lastNotifiedAt before queueing the event. */ + readonly advanceNotificationAnchor: boolean +} + +export interface AlertDeliveryRetryPolicy { + readonly maxAttempts: number + readonly baseDelayMs: number + readonly maxDelayMs: number +} + +export const DEFAULT_ALERT_DELIVERY_RETRY_POLICY: AlertDeliveryRetryPolicy = { + maxAttempts: 5, + baseDelayMs: 60_000, + maxDelayMs: 15 * 60_000, +} + +/** Stable idempotency key shared by every alert delivery adapter. */ +export const makeAlertDeliveryKey = ( + incidentId: string, + destinationId: string, + eventType: AlertEventType, + scheduledAtMs: number, +): string => [incidentId, destinationId, eventType, scheduledAtMs].join(":") + +export const canRetryAlertDelivery = ( + attemptNumber: number, + retryable: boolean, + policy: AlertDeliveryRetryPolicy = DEFAULT_ALERT_DELIVERY_RETRY_POLICY, +): boolean => retryable && attemptNumber < policy.maxAttempts + +/** Exponential retry delay; the host supplies jitter from its own random source. */ +export const alertDeliveryRetryDelayMs = ( + attemptNumber: number, + jitterMs: number, + policy: AlertDeliveryRetryPolicy = DEFAULT_ALERT_DELIVERY_RETRY_POLICY, +): number => { + const exponent = Math.max(0, attemptNumber - 1) + const base = Math.min(policy.baseDelayMs * Math.pow(2, exponent), policy.maxDelayMs) + return base + Math.max(0, Math.floor(jitterMs)) +} + +const noTransition = (state: AlertLifecycleState, hold: AlertLifecycleHold = null): AlertLifecyclePlan => ({ + state, + transition: "none", + eventType: null, + notificationSuppression: null, + hold, + inheritedNotificationAtMs: null, + advanceNotificationAnchor: false, +}) + +/** + * Decide the next alert state and lifecycle intent without performing I/O. + * + * The caller owns persistence, incident identifiers, delivery, telemetry + * liveness checks, and time. This makes the same lifecycle semantics usable by + * the hosted PostgreSQL/Tinybird adapter and a future Maple Local adapter. + */ +export const planAlertLifecycle = (input: AlertLifecycleInput): AlertLifecyclePlan => { + const { evaluation, policy, openIncident, nowMs } = input + const previous = input.state ?? { consecutiveBreaches: 0, consecutiveHealthy: 0 } + + if (evaluation.status === "skipped") return noTransition(previous) + + const state: AlertLifecycleState = { + consecutiveBreaches: + evaluation.status === "breached" + ? Math.min(previous.consecutiveBreaches + 1, policy.consecutiveBreachesRequired) + : 0, + consecutiveHealthy: + evaluation.status === "healthy" + ? Math.min(previous.consecutiveHealthy + 1, policy.consecutiveHealthyRequired) + : 0, + } + + if ( + evaluation.status === "breached" && + openIncident == null && + state.consecutiveBreaches >= policy.consecutiveBreachesRequired + ) { + const previousNotificationAtMs = input.previousNotificationAtMs ?? null + const flapSuppressed = + previousNotificationAtMs != null && + previousNotificationAtMs >= nowMs - policy.renotifyIntervalMinutes * 60_000 + return { + state, + transition: "opened", + eventType: flapSuppressed ? null : "trigger", + notificationSuppression: flapSuppressed ? "flapping" : null, + hold: null, + inheritedNotificationAtMs: flapSuppressed ? previousNotificationAtMs : null, + advanceNotificationAnchor: false, + } + } + + if (evaluation.status === "breached" && openIncident != null) { + const renotifyDueAt = + (openIncident.lastNotifiedAtMs ?? openIncident.firstTriggeredAtMs) + + policy.renotifyIntervalMinutes * 60_000 + const renotifyDue = renotifyDueAt <= nowMs + return { + state, + transition: "continued", + eventType: renotifyDue ? "renotify" : null, + notificationSuppression: null, + hold: null, + inheritedNotificationAtMs: null, + advanceNotificationAnchor: renotifyDue, + } + } + + if ( + evaluation.status === "healthy" && + openIncident != null && + state.consecutiveHealthy >= policy.consecutiveHealthyRequired + ) { + if (evaluation.derivedFromNoData && input.allowNoDataResolution !== true) { + return noTransition(state, "missing_telemetry") + } + + const flapResolutionSuppressed = + openIncident.lastDeliveredEventType == null && openIncident.lastNotifiedAtMs != null + return { + state, + transition: "resolved", + eventType: flapResolutionSuppressed ? null : "resolve", + notificationSuppression: flapResolutionSuppressed ? "flap_resolution" : null, + hold: null, + inheritedNotificationAtMs: null, + advanceNotificationAnchor: false, + } + } + + return noTransition(state) +} + +/** Preserve per-tenant order while preventing one tenant from monopolizing a tick. */ +export const interleaveAlertRulesByTenant = ( + rows: ReadonlyArray, + tenantIdOf: (row: T) => string, +): ReadonlyArray => { + const queues = new Map() + for (const row of rows) { + const tenantId = tenantIdOf(row) + const queue = queues.get(tenantId) + if (queue) queue.push(row) + else queues.set(tenantId, [row]) + } + + const fair: T[] = [] + let index = 0 + while (fair.length < rows.length) { + for (const queue of queues.values()) { + const row = queue[index] + if (row !== undefined) fair.push(row) + } + index += 1 + } + return fair +} diff --git a/packages/alerting-core/tsconfig.json b/packages/alerting-core/tsconfig.json new file mode 100644 index 000000000..3d83a7d0c --- /dev/null +++ b/packages/alerting-core/tsconfig.json @@ -0,0 +1,17 @@ +{ + "include": ["**/*.ts"], + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022"], + "types": ["node"], + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "skipLibCheck": true, + "strict": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true + } +} diff --git a/packages/db/drizzle/0047_planetscale_issue_receipts.sql b/packages/db/drizzle/0047_planetscale_issue_receipts.sql new file mode 100644 index 000000000..51c175dd3 --- /dev/null +++ b/packages/db/drizzle/0047_planetscale_issue_receipts.sql @@ -0,0 +1,6 @@ +CREATE TABLE "planetscale_issue_receipts" ( + "org_id" text NOT NULL, + "event_id" text NOT NULL, + "processed_at" timestamp with time zone NOT NULL, + CONSTRAINT "planetscale_issue_receipts_org_id_event_id_pk" PRIMARY KEY("org_id","event_id") +); diff --git a/packages/db/drizzle/meta/0047_snapshot.json b/packages/db/drizzle/meta/0047_snapshot.json new file mode 100644 index 000000000..09cce5b64 --- /dev/null +++ b/packages/db/drizzle/meta/0047_snapshot.json @@ -0,0 +1,8339 @@ +{ + "id": "3acdc496-7811-4415-ae63-b354ef8b0e15", + "prevId": "22510189-9670-45c9-ae72-860908016f38", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.ai_triage_settings": { + "name": "ai_triage_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "max_runs_per_day": { + "name": "max_runs_per_day", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 20 + }, + "max_passes_per_day": { + "name": "max_passes_per_day", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 90 + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_delivery_events": { + "name": "alert_delivery_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "destination_id": { + "name": "destination_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delivery_key": { + "name": "delivery_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt_number": { + "name": "attempt_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scheduled_at": { + "name": "scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claim_expires_at": { + "name": "claim_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempted_at": { + "name": "attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "provider_message": { + "name": "provider_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "provider_reference": { + "name": "provider_reference", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_delivery_events_org_idx": { + "name": "alert_delivery_events_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_org_incident_idx": { + "name": "alert_delivery_events_org_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_due_idx": { + "name": "alert_delivery_events_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scheduled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_claim_idx": { + "name": "alert_delivery_events_claim_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "claim_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scheduled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_delivery_events_delivery_attempt_idx": { + "name": "alert_delivery_events_delivery_attempt_idx", + "columns": [ + { + "expression": "delivery_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "attempt_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_destinations": { + "name": "alert_destinations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "config_json": { + "name": "config_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "secret_ciphertext": { + "name": "secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_iv": { + "name": "secret_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_tag": { + "name": "secret_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_tested_at": { + "name": "last_tested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_test_error": { + "name": "last_test_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consecutive_failures": { + "name": "consecutive_failures", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_failure_at": { + "name": "last_failure_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "disabled_at": { + "name": "disabled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "disabled_reason": { + "name": "disabled_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_destinations_org_idx": { + "name": "alert_destinations_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_destinations_org_enabled_idx": { + "name": "alert_destinations_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_destinations_org_name_idx": { + "name": "alert_destinations_org_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_incidents": { + "name": "alert_incidents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_key": { + "name": "incident_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rule_name": { + "name": "rule_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_key": { + "name": "group_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "comparator": { + "name": "comparator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "threshold": { + "name": "threshold", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "threshold_upper": { + "name": "threshold_upper", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_observed_value": { + "name": "last_observed_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_delivered_event_type": { + "name": "last_delivered_event_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_notified_at": { + "name": "last_notified_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error_issue_id": { + "name": "error_issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_incidents_org_idx": { + "name": "alert_incidents_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_org_status_idx": { + "name": "alert_incidents_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_org_rule_idx": { + "name": "alert_incidents_org_rule_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "rule_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_org_issue_idx": { + "name": "alert_incidents_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "error_issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_incidents_incident_key_idx": { + "name": "alert_incidents_incident_key_idx", + "columns": [ + { + "expression": "incident_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_rule_claims": { + "name": "alert_rule_claims", + "schema": "", + "columns": { + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_scheduled_at": { + "name": "last_scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_rule_claims_org_idx": { + "name": "alert_rule_claims_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_rule_states": { + "name": "alert_rule_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "rule_id": { + "name": "rule_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "group_key": { + "name": "group_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'__total__'" + }, + "consecutive_breaches": { + "name": "consecutive_breaches", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "consecutive_healthy": { + "name": "consecutive_healthy", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_value": { + "name": "last_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_rule_states_org_idx": { + "name": "alert_rule_states_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "alert_rule_states_org_id_rule_id_group_key_pk": { + "name": "alert_rule_states_org_id_rule_id_group_key_pk", + "columns": [ + "org_id", + "rule_id", + "group_key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.alert_rules": { + "name": "alert_rules", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notification_template_json": { + "name": "notification_template_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_names_json": { + "name": "service_names_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "exclude_service_names_json": { + "name": "exclude_service_names_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "environments_json": { + "name": "environments_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "tags_json": { + "name": "tags_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "comparator": { + "name": "comparator", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "threshold": { + "name": "threshold", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "threshold_upper": { + "name": "threshold_upper", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "window_minutes": { + "name": "window_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "minimum_sample_count": { + "name": "minimum_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "consecutive_breaches_required": { + "name": "consecutive_breaches_required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "consecutive_healthy_required": { + "name": "consecutive_healthy_required", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 2 + }, + "renotify_interval_minutes": { + "name": "renotify_interval_minutes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 30 + }, + "apdex_threshold_ms": { + "name": "apdex_threshold_ms", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "query_builder_draft_json": { + "name": "query_builder_draft_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "raw_query_sql": { + "name": "raw_query_sql", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "group_by": { + "name": "group_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "destination_ids_json": { + "name": "destination_ids_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "query_spec_json": { + "name": "query_spec_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "reducer": { + "name": "reducer", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sample_count_strategy": { + "name": "sample_count_strategy", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "no_data_behavior": { + "name": "no_data_behavior", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_scheduled_at": { + "name": "last_scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "alert_rules_org_idx": { + "name": "alert_rules_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_rules_org_enabled_idx": { + "name": "alert_rules_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "alert_rules_org_name_idx": { + "name": "alert_rules_org_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.anomaly_detector_settings": { + "name": "anomaly_detector_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sensitivity": { + "name": "sensitivity", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'normal'" + }, + "muted_signals_json": { + "name": "muted_signals_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "last_tick_at": { + "name": "last_tick_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.anomaly_detector_states": { + "name": "anomaly_detector_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detector_key": { + "name": "detector_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_env": { + "name": "deployment_env", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "consecutive_breaches": { + "name": "consecutive_breaches", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "consecutive_healthy": { + "name": "consecutive_healthy", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_status": { + "name": "last_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_value": { + "name": "last_value", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "baseline_median": { + "name": "baseline_median", + "type": "double precision", + "primaryKey": false, + "notNull": false + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "open_incident_id": { + "name": "open_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_resolved_at": { + "name": "last_resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_incident_id": { + "name": "last_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "anomaly_detector_states_open_incident_idx": { + "name": "anomaly_detector_states_open_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "open_incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_detector_states_evaluated_idx": { + "name": "anomaly_detector_states_evaluated_idx", + "columns": [ + { + "expression": "last_evaluated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "anomaly_detector_states_org_id_detector_key_pk": { + "name": "anomaly_detector_states_org_id_detector_key_pk", + "columns": [ + "org_id", + "detector_key" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.anomaly_incidents": { + "name": "anomaly_incidents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "detector_key": { + "name": "detector_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "signal_type": { + "name": "signal_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "deployment_env": { + "name": "deployment_env", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "error_issue_id": { + "name": "error_issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "opened_value": { + "name": "opened_value", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "baseline_median": { + "name": "baseline_median", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "baseline_sigma": { + "name": "baseline_sigma", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "threshold_value": { + "name": "threshold_value", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "last_observed_value": { + "name": "last_observed_value", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "last_sample_count": { + "name": "last_sample_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "resolve_reason": { + "name": "resolve_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "triage_status": { + "name": "triage_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprints_json": { + "name": "fingerprints_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "reopen_count": { + "name": "reopen_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "last_reopened_at": { + "name": "last_reopened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "anomaly_incidents_org_status_idx": { + "name": "anomaly_incidents_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_org_triggered_idx": { + "name": "anomaly_incidents_org_triggered_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_triggered_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_org_detector_idx": { + "name": "anomaly_incidents_org_detector_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "detector_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "anomaly_incidents_org_issue_idx": { + "name": "anomaly_incidents_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "error_issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.api_keys": { + "name": "api_keys", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "key_hash": { + "name": "key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key_prefix": { + "name": "key_prefix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revoked": { + "name": "revoked", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "metadata_json": { + "name": "metadata_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scopes": { + "name": "scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'standard'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by_email": { + "name": "created_by_email", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "api_keys_key_hash_unique": { + "name": "api_keys_key_hash_unique", + "columns": [ + { + "expression": "key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "api_keys_org_id_idx": { + "name": "api_keys_org_id_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloudflare_analytics_state": { + "name": "cloudflare_analytics_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dataset": { + "name": "dataset", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "zone_id": { + "name": "zone_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "watermark_at": { + "name": "watermark_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "backfill_at": { + "name": "backfill_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "settings_json": { + "name": "settings_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "settings_fetched_at": { + "name": "settings_fetched_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "quantiles_available": { + "name": "quantiles_available", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "discovered_at": { + "name": "discovered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "live_scripts_json": { + "name": "live_scripts_json", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error_at": { + "name": "last_error_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cf_analytics_state_org_dataset_zone_idx": { + "name": "cf_analytics_state_org_dataset_zone_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dataset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "zone_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cf_analytics_state_org_idx": { + "name": "cf_analytics_state_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloudflare_hyperdrive_configs": { + "name": "cloudflare_hyperdrive_configs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_host": { + "name": "origin_host", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "origin_port": { + "name": "origin_port", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "origin_scheme": { + "name": "origin_scheme", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_database": { + "name": "origin_database", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "origin_user": { + "name": "origin_user", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cloudflare_hyperdrive_configs_org_config_idx": { + "name": "cloudflare_hyperdrive_configs_org_config_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "config_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cloudflare_hyperdrive_configs_org_idx": { + "name": "cloudflare_hyperdrive_configs_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cloudflare_logpush_connectors": { + "name": "cloudflare_logpush_connectors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "zone_name": { + "name": "zone_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dataset": { + "name": "dataset", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'http_requests'" + }, + "secret_ciphertext": { + "name": "secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_iv": { + "name": "secret_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_tag": { + "name": "secret_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_hash": { + "name": "secret_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_received_at": { + "name": "last_received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "secret_rotated_at": { + "name": "secret_rotated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cloudflare_logpush_connectors_org_idx": { + "name": "cloudflare_logpush_connectors_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cloudflare_logpush_connectors_org_enabled_idx": { + "name": "cloudflare_logpush_connectors_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cloudflare_logpush_connectors_secret_hash_unique": { + "name": "cloudflare_logpush_connectors_secret_hash_unique", + "columns": [ + { + "expression": "secret_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.cli_device_authorizations": { + "name": "cli_device_authorizations", + "schema": "", + "columns": { + "device_code_hash": { + "name": "device_code_hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_code_hash": { + "name": "user_code_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "device_name": { + "name": "device_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "approved_org_id": { + "name": "approved_org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_user_id": { + "name": "approved_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_roles": { + "name": "approved_roles", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "approved_user_email": { + "name": "approved_user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_id": { + "name": "api_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_at": { + "name": "approved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "denied_at": { + "name": "denied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "cli_device_authorizations_user_code_unique": { + "name": "cli_device_authorizations_user_code_unique", + "columns": [ + { + "expression": "user_code_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "cli_device_authorizations_expires_idx": { + "name": "cli_device_authorizations_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_oauth_authorizations": { + "name": "mcp_oauth_authorizations", + "schema": "", + "columns": { + "request_id_hash": { + "name": "request_id_hash", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_name": { + "name": "client_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uri": { + "name": "redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "code_challenge": { + "name": "code_challenge", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "authorization_code_hash": { + "name": "authorization_code_hash", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_org_id": { + "name": "approved_org_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_user_id": { + "name": "approved_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_roles": { + "name": "approved_roles", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "approved_user_email": { + "name": "approved_user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "approved_at": { + "name": "approved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "denied_at": { + "name": "denied_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "used_at": { + "name": "used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mcp_oauth_authorizations_code_unique": { + "name": "mcp_oauth_authorizations_code_unique", + "columns": [ + { + "expression": "authorization_code_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_oauth_authorizations_expires_idx": { + "name": "mcp_oauth_authorizations_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_oauth_clients": { + "name": "mcp_oauth_clients", + "schema": "", + "columns": { + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "client_name": { + "name": "client_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uris": { + "name": "redirect_uris", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "client_uri": { + "name": "client_uri", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mcp_oauth_refresh_tokens": { + "name": "mcp_oauth_refresh_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "family_id": { + "name": "family_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "client_id": { + "name": "client_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "resource": { + "name": "resource", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scopes": { + "name": "scopes", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "roles": { + "name": "roles", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "user_email": { + "name": "user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_key_id": { + "name": "access_key_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "replaced_by_id": { + "name": "replaced_by_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mcp_oauth_refresh_tokens_hash_unique": { + "name": "mcp_oauth_refresh_tokens_hash_unique", + "columns": [ + { + "expression": "token_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_oauth_refresh_tokens_family_idx": { + "name": "mcp_oauth_refresh_tokens_family_idx", + "columns": [ + { + "expression": "family_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mcp_oauth_refresh_tokens_expires_idx": { + "name": "mcp_oauth_refresh_tokens_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.mobile_devices": { + "name": "mobile_devices", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "platform": { + "name": "platform", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "environment": { + "name": "environment", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "bundle_id": { + "name": "bundle_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "app_version": { + "name": "app_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "live_activity_start_token": { + "name": "live_activity_start_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "device_name": { + "name": "device_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "preferences": { + "name": "preferences", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "disabled_at": { + "name": "disabled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "disabled_reason": { + "name": "disabled_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_pushed_at": { + "name": "last_pushed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "mobile_devices_org_platform_token_unique": { + "name": "mobile_devices_org_platform_token_unique", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "platform", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "token", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mobile_devices_org_idx": { + "name": "mobile_devices_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "mobile_devices_user_idx": { + "name": "mobile_devices_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dashboard_shares": { + "name": "dashboard_shares", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dashboard_id": { + "name": "dashboard_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "widget_id": { + "name": "widget_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_hash": { + "name": "token_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_ciphertext": { + "name": "token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_iv": { + "name": "token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_tag": { + "name": "token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token_suffix": { + "name": "token_suffix", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "dashboard_shares_token_hash_unq": { + "name": "dashboard_shares_token_hash_unq", + "columns": [ + { + "expression": "token_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_shares_live_unq": { + "name": "dashboard_shares_live_unq", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "coalesce(widget_id, '')", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "revoked_at is null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_shares_org_dashboard_idx": { + "name": "dashboard_shares_org_dashboard_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_shares_id_idx": { + "name": "dashboard_shares_id_idx", + "columns": [ + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "dashboard_shares_dashboard_fk": { + "name": "dashboard_shares_dashboard_fk", + "tableFrom": "dashboard_shares", + "tableTo": "dashboards", + "columnsFrom": [ + "org_id", + "dashboard_id" + ], + "columnsTo": [ + "org_id", + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "dashboard_shares_org_id_id_pk": { + "name": "dashboard_shares_org_id_id_pk", + "columns": [ + "org_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dashboard_versions": { + "name": "dashboard_versions", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dashboard_id": { + "name": "dashboard_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version_number": { + "name": "version_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "change_kind": { + "name": "change_kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "change_summary": { + "name": "change_summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_version_id": { + "name": "source_version_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "dashboard_versions_org_dashboard_idx": { + "name": "dashboard_versions_org_dashboard_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboard_versions_org_dashboard_version_unq": { + "name": "dashboard_versions_org_dashboard_version_unq", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dashboard_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "version_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "dashboard_versions_org_id_id_pk": { + "name": "dashboard_versions_org_id_id_pk", + "columns": [ + "org_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.dashboards": { + "name": "dashboards", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "id": { + "name": "id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "version": { + "name": "version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + } + }, + "indexes": { + "dashboards_org_updated_idx": { + "name": "dashboards_org_updated_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "dashboards_org_name_idx": { + "name": "dashboards_org_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "dashboards_org_id_id_pk": { + "name": "dashboards_org_id_id_pk", + "columns": [ + "org_id", + "id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.digest_subscriptions": { + "name": "digest_subscriptions", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "day_of_week": { + "name": "day_of_week", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "timezone": { + "name": "timezone", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'UTC'" + }, + "last_sent_at": { + "name": "last_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_attempted_at": { + "name": "last_attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "digest_subscriptions_org_user_idx": { + "name": "digest_subscriptions_org_user_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "digest_subscriptions_org_enabled_idx": { + "name": "digest_subscriptions_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.actors": { + "name": "actors", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "agent_name": { + "name": "agent_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "capabilities_json": { + "name": "capabilities_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_active_at": { + "name": "last_active_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "actors_org_user_idx": { + "name": "actors_org_user_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "actors_org_agent_name_idx": { + "name": "actors_org_agent_name_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "agent_name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "actors_org_type_idx": { + "name": "actors_org_type_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_fingerprint_candidates": { + "name": "error_fingerprint_candidates", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_type": { + "name": "exception_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_message": { + "name": "exception_message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error_label": { + "name": "error_label", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "top_frame": { + "name": "top_frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_versions_json": { + "name": "service_versions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_fingerprint_candidates_last_seen_idx": { + "name": "error_fingerprint_candidates_last_seen_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_seen_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "error_fingerprint_candidates_org_id_fingerprint_hash_pk": { + "name": "error_fingerprint_candidates_org_id_fingerprint_hash_pk", + "columns": [ + "org_id", + "fingerprint_hash" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_incidents": { + "name": "error_incidents", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "first_triggered_at": { + "name": "first_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_triggered_at": { + "name": "last_triggered_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_incidents_org_issue_idx": { + "name": "error_incidents_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_incidents_org_status_idx": { + "name": "error_incidents_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issue_events": { + "name": "error_issue_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_state": { + "name": "from_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "to_state": { + "name": "to_state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issue_events_issue_idx": { + "name": "error_issue_events_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_events_actor_idx": { + "name": "error_issue_events_actor_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issue_events_type_idx": { + "name": "error_issue_events_type_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issue_states": { + "name": "error_issue_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_observed_occurrence_at": { + "name": "last_observed_occurrence_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_evaluated_at": { + "name": "last_evaluated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "open_incident_id": { + "name": "open_incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "error_issue_states_org_id_issue_id_pk": { + "name": "error_issue_states_org_id_issue_id_pk", + "columns": [ + "org_id", + "issue_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_issues": { + "name": "error_issues", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'error'" + }, + "source_ref_json": { + "name": "source_ref_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "fingerprint_hash": { + "name": "fingerprint_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "fingerprint_version": { + "name": "fingerprint_version", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_type": { + "name": "exception_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "exception_message": { + "name": "exception_message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error_label": { + "name": "error_label", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "top_frame": { + "name": "top_frame", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_state": { + "name": "workflow_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'triage'" + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 3 + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "severity_source": { + "name": "severity_source", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "assigned_actor_id": { + "name": "assigned_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_holder_actor_id": { + "name": "lease_holder_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lease_expires_at": { + "name": "lease_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "first_seen_at": { + "name": "first_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "last_seen_at": { + "name": "last_seen_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "occurrence_count": { + "name": "occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "resolved_by_actor_id": { + "name": "resolved_by_actor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_resolved_at": { + "name": "last_resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_regressed_at": { + "name": "last_regressed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "regression_count": { + "name": "regression_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "seen_versions_json": { + "name": "seen_versions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "resolved_versions_json": { + "name": "resolved_versions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "snooze_until": { + "name": "snooze_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "archived_at": { + "name": "archived_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_issues_org_fp_idx": { + "name": "error_issues_org_fp_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_workflow_idx": { + "name": "error_issues_org_workflow_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "workflow_state", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_severity_idx": { + "name": "error_issues_org_severity_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "severity", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_last_seen_idx": { + "name": "error_issues_org_last_seen_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "last_seen_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_fp_version_idx": { + "name": "error_issues_org_fp_version_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "fingerprint_version", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_assignee_idx": { + "name": "error_issues_org_assignee_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "assigned_actor_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_lease_expiry_idx": { + "name": "error_issues_lease_expiry_idx", + "columns": [ + { + "expression": "lease_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_issues_org_archived_idx": { + "name": "error_issues_org_archived_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "archived_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"error_issues\".\"archived_at\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_notification_deliveries": { + "name": "error_notification_deliveries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "destination_id": { + "name": "destination_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "delivery_key": { + "name": "delivery_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt_count": { + "name": "attempt_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "scheduled_at": { + "name": "scheduled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "claimed_at": { + "name": "claimed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claim_expires_at": { + "name": "claim_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "claimed_by": { + "name": "claimed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "attempted_at": { + "name": "attempted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_notification_deliveries_due_idx": { + "name": "error_notification_deliveries_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "scheduled_at", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "claim_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_notification_deliveries_org_idx": { + "name": "error_notification_deliveries_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "error_notification_deliveries_key_destination_idx": { + "name": "error_notification_deliveries_key_destination_idx", + "columns": [ + { + "expression": "delivery_key", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "destination_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_notification_policies": { + "name": "error_notification_policies", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "destination_ids_json": { + "name": "destination_ids_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "notify_on_first_seen": { + "name": "notify_on_first_seen", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "notify_on_regression": { + "name": "notify_on_regression", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "notify_on_resolve": { + "name": "notify_on_resolve", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "notify_on_transition_in_review": { + "name": "notify_on_transition_in_review", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "notify_on_transition_done": { + "name": "notify_on_transition_done", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "notify_on_claim": { + "name": "notify_on_claim", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "min_occurrence_count": { + "name": "min_occurrence_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'warning'" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.error_tick_states": { + "name": "error_tick_states", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "processed_through": { + "name": "processed_through", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "bootstrap_completed": { + "name": "bootstrap_completed", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "claim_token": { + "name": "claim_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "claim_expires_at": { + "name": "claim_expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "error_tick_states_claim_idx": { + "name": "error_tick_states_claim_idx", + "columns": [ + { + "expression": "claim_expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.issue_escalation_policies": { + "name": "issue_escalation_policies", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "rules_json": { + "name": "rules_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.issue_escalations": { + "name": "issue_escalations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "investigation_id": { + "name": "investigation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "delivery_results_json": { + "name": "delivery_results_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "dedupe_key": { + "name": "dedupe_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "issue_escalations_dedupe_idx": { + "name": "issue_escalations_dedupe_idx", + "columns": [ + { + "expression": "dedupe_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "issue_escalations_due_idx": { + "name": "issue_escalations_due_idx", + "columns": [ + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "issue_escalations_org_issue_idx": { + "name": "issue_escalations_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.investigation_lens_runs": { + "name": "investigation_lens_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "investigation_id": { + "name": "investigation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "lens_id": { + "name": "lens_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "ordinal": { + "name": "ordinal", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "verdict": { + "name": "verdict", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "claim": { + "name": "claim", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "progress_note": { + "name": "progress_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "tool_count": { + "name": "tool_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "elapsed_ms": { + "name": "elapsed_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "lens_name": { + "name": "lens_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "lens_question": { + "name": "lens_question", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "priority": { + "name": "priority", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "deadline_hit": { + "name": "deadline_hit", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "hypothesis_json": { + "name": "hypothesis_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "evidence_json": { + "name": "evidence_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "mechanism": { + "name": "mechanism", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "self_doubt": { + "name": "self_doubt", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "suggested_actions_json": { + "name": "suggested_actions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "reported_at": { + "name": "reported_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "ranked_at": { + "name": "ranked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "investigation_lens_runs_lens_idx": { + "name": "investigation_lens_runs_lens_idx", + "columns": [ + { + "expression": "investigation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "attempt", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "lens_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigation_lens_runs_org_inv_idx": { + "name": "investigation_lens_runs_org_inv_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "investigation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "investigation_lens_runs_investigation_id_investigations_id_fk": { + "name": "investigation_lens_runs_investigation_id_investigations_id_fk", + "tableFrom": "investigation_lens_runs", + "tableTo": "investigations", + "columnsFrom": [ + "investigation_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.investigations": { + "name": "investigations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'investigating'" + }, + "seeded_by": { + "name": "seeded_by", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'user'" + }, + "subject_json": { + "name": "subject_json", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "snapshot_json": { + "name": "snapshot_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "incident_kind": { + "name": "incident_kind", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "issue_id": { + "name": "issue_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "report_json": { + "name": "report_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "severity": { + "name": "severity", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "confidence": { + "name": "confidence", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "model": { + "name": "model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fanout_state": { + "name": "fanout_state", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "fanout_size": { + "name": "fanout_size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "plan_json": { + "name": "plan_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "planner_model": { + "name": "planner_model", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "planner_elapsed_ms": { + "name": "planner_elapsed_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "validator_note": { + "name": "validator_note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validator_elapsed_ms": { + "name": "validator_elapsed_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "fanout_deadline_at": { + "name": "fanout_deadline_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "workflow_instance_id": { + "name": "workflow_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "fanout_attempt": { + "name": "fanout_attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "autonomous_turns": { + "name": "autonomous_turns", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "diagnosed_at": { + "name": "diagnosed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "investigations_incident_idx": { + "name": "investigations_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"investigations\".\"incident_id\" is not null", + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_org_created_idx": { + "name": "investigations_org_created_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_org_issue_idx": { + "name": "investigations_org_issue_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "issue_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "investigations_org_status_idx": { + "name": "investigations_org_status_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.live_activities": { + "name": "live_activities", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "device_id": { + "name": "device_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "incident_id": { + "name": "incident_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "activity_id": { + "name": "activity_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "push_token": { + "name": "push_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ended_at": { + "name": "ended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "ended_reason": { + "name": "ended_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "live_activities_device_incident_unique": { + "name": "live_activities_device_incident_unique", + "columns": [ + { + "expression": "device_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "live_activities_incident_idx": { + "name": "live_activities_incident_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "incident_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_auth_states": { + "name": "oauth_auth_states", + "schema": "", + "columns": { + "state": { + "name": "state", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "initiated_by_user_id": { + "name": "initiated_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "redirect_uri": { + "name": "redirect_uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "return_to": { + "name": "return_to", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "code_verifier": { + "name": "code_verifier", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_auth_states_expires_idx": { + "name": "oauth_auth_states_expires_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.oauth_connections": { + "name": "oauth_connections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_user_id": { + "name": "external_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_user_email": { + "name": "external_user_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_account_name": { + "name": "external_account_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "connected_by_user_id": { + "name": "connected_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "access_token_ciphertext": { + "name": "access_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_iv": { + "name": "access_token_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token_tag": { + "name": "access_token_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refresh_token_ciphertext": { + "name": "refresh_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token_iv": { + "name": "refresh_token_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token_tag": { + "name": "refresh_token_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "oauth_connections_org_provider_idx": { + "name": "oauth_connections_org_provider_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "oauth_connections_org_idx": { + "name": "oauth_connections_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_onboarding_state": { + "name": "org_onboarding_state", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "demo_data_requested": { + "name": "demo_data_requested", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "checklist_dismissed_at": { + "name": "checklist_dismissed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "first_data_received_at": { + "name": "first_data_received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "welcome_email_sent_at": { + "name": "welcome_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "connect_nudge_email_sent_at": { + "name": "connect_nudge_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "stalled_email_sent_at": { + "name": "stalled_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "activation_email_sent_at": { + "name": "activation_email_sent_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_ingest_attribute_mappings": { + "name": "org_ingest_attribute_mappings", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_context": { + "name": "source_context", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_key": { + "name": "target_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "operation": { + "name": "operation", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "org_ingest_attribute_mappings_org_idx": { + "name": "org_ingest_attribute_mappings_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_recommendation_issues": { + "name": "org_recommendation_issues", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "number": { + "name": "number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "recommendation_key": { + "name": "recommendation_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source_key": { + "name": "source_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "canonical_key": { + "name": "canonical_key", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'open'" + }, + "usage_count": { + "name": "usage_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "opened_at": { + "name": "opened_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "org_recommendation_issues_org_idx": { + "name": "org_recommendation_issues_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_recommendation_issues_org_key_idx": { + "name": "org_recommendation_issues_org_key_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "recommendation_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_ingest_keys": { + "name": "org_ingest_keys", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_key": { + "name": "public_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_key_hash": { + "name": "public_key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_ciphertext": { + "name": "private_key_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_iv": { + "name": "private_key_iv", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_tag": { + "name": "private_key_tag", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "private_key_hash": { + "name": "private_key_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "public_rotated_at": { + "name": "public_rotated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "private_rotated_at": { + "name": "private_rotated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "org_ingest_keys_public_key_unique": { + "name": "org_ingest_keys_public_key_unique", + "columns": [ + { + "expression": "public_key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_ingest_keys_public_key_hash_unique": { + "name": "org_ingest_keys_public_key_hash_unique", + "columns": [ + { + "expression": "public_key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "org_ingest_keys_private_key_hash_unique": { + "name": "org_ingest_keys_private_key_hash_unique", + "columns": [ + { + "expression": "private_key_hash", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "org_ingest_keys_org_id_pk": { + "name": "org_ingest_keys_org_id_pk", + "columns": [ + "org_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_ingest_sampling_policies": { + "name": "org_ingest_sampling_policies", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "trace_sample_ratio": { + "name": "trace_sample_ratio", + "type": "double precision", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "always_keep_error_spans": { + "name": "always_keep_error_spans", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "always_keep_slow_spans_ms": { + "name": "always_keep_slow_spans_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_clickhouse_settings": { + "name": "org_clickhouse_settings", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ch_url": { + "name": "ch_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ch_user": { + "name": "ch_user", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ch_password_ciphertext": { + "name": "ch_password_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ch_password_iv": { + "name": "ch_password_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ch_password_tag": { + "name": "ch_password_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "ch_database": { + "name": "ch_database", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "last_sync_at": { + "name": "last_sync_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "schema_version": { + "name": "schema_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "updated_by": { + "name": "updated_by", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "org_clickhouse_settings_org_id_pk": { + "name": "org_clickhouse_settings_org_id_pk", + "columns": [ + "org_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.org_clickhouse_schema_apply_runs": { + "name": "org_clickhouse_schema_apply_runs", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "workflow_instance_id": { + "name": "workflow_instance_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "phase": { + "name": "phase", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "current_migration": { + "name": "current_migration", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "steps_total": { + "name": "steps_total", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "steps_done": { + "name": "steps_done", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "applied_versions": { + "name": "applied_versions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "skipped": { + "name": "skipped", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "error_message": { + "name": "error_message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "finished_at": { + "name": "finished_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "org_clickhouse_schema_apply_runs_org_id_pk": { + "name": "org_clickhouse_schema_apply_runs_org_id_pk", + "columns": [ + "org_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_connections": { + "name": "planetscale_connections", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "ps_organization": { + "name": "ps_organization", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "connected_by_user_id": { + "name": "connected_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scrape_target_id": { + "name": "scrape_target_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "webhook_secret_ciphertext": { + "name": "webhook_secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "webhook_secret_iv": { + "name": "webhook_secret_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "webhook_secret_tag": { + "name": "webhook_secret_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "detected_permissions_json": { + "name": "detected_permissions_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "last_inventory_at": { + "name": "last_inventory_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_inventory_error": { + "name": "last_inventory_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "planetscale_connections_org_idx": { + "name": "planetscale_connections_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_databases": { + "name": "planetscale_databases", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "database_id": { + "name": "database_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'mysql'" + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "plan": { + "name": "plan", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "branches_json": { + "name": "branches_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "planetscale_databases_org_db_idx": { + "name": "planetscale_databases_org_db_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "database_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "planetscale_databases_org_idx": { + "name": "planetscale_databases_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_events": { + "name": "planetscale_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "database_id": { + "name": "database_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "database_name": { + "name": "database_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch_name": { + "name": "branch_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_login": { + "name": "actor_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload_json": { + "name": "payload_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "planetscale_events_dedupe_idx": { + "name": "planetscale_events_dedupe_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "database_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "planetscale_events_org_db_time_idx": { + "name": "planetscale_events_org_db_time_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "database_name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "planetscale_events_org_time_idx": { + "name": "planetscale_events_org_time_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_issue_receipts": { + "name": "planetscale_issue_receipts", + "schema": "", + "columns": { + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_id": { + "name": "event_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "planetscale_issue_receipts_org_id_event_id_pk": { + "name": "planetscale_issue_receipts_org_id_event_id_pk", + "columns": [ + "org_id", + "event_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.planetscale_poll_state": { + "name": "planetscale_poll_state", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "dataset": { + "name": "dataset", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "database_id": { + "name": "database_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "watermark_at": { + "name": "watermark_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_success_at": { + "name": "last_success_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_error_at": { + "name": "last_error_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "planetscale_poll_state_org_dataset_db_idx": { + "name": "planetscale_poll_state_org_dataset_db_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "dataset", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "database_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "planetscale_poll_state_org_idx": { + "name": "planetscale_poll_state_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scrape_target_checks": { + "name": "scrape_target_checks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "byDefault", + "name": "scrape_target_checks_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "2147483647", + "cache": "1", + "cycle": false + } + }, + "target_id": { + "name": "target_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sub_target_key": { + "name": "sub_target_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "''" + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "samples_scraped": { + "name": "samples_scraped", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "samples_post_relabel": { + "name": "samples_post_relabel", + "type": "integer", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "scrape_target_checks_target_checked_idx": { + "name": "scrape_target_checks_target_checked_idx", + "columns": [ + { + "expression": "target_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "checked_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "scrape_target_checks_target_id_scrape_targets_id_fk": { + "name": "scrape_target_checks_target_id_scrape_targets_id_fk", + "tableFrom": "scrape_target_checks", + "tableTo": "scrape_targets", + "columnsFrom": [ + "target_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.scrape_targets": { + "name": "scrape_targets", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "service_name": { + "name": "service_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "target_type": { + "name": "target_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'prometheus'" + }, + "discovery_config_json": { + "name": "discovery_config_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "scrape_interval_seconds": { + "name": "scrape_interval_seconds", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 15 + }, + "labels_json": { + "name": "labels_json", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "auth_type": { + "name": "auth_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'none'" + }, + "managed_by": { + "name": "managed_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_credentials_ciphertext": { + "name": "auth_credentials_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_credentials_iv": { + "name": "auth_credentials_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "auth_credentials_tag": { + "name": "auth_credentials_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled": { + "name": "enabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "last_scrape_at": { + "name": "last_scrape_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_scrape_error": { + "name": "last_scrape_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "scrape_targets_org_idx": { + "name": "scrape_targets_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "scrape_targets_org_enabled_idx": { + "name": "scrape_targets_org_enabled_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "enabled", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.slack_workspaces": { + "name": "slack_workspaces", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "team_id": { + "name": "team_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "team_name": { + "name": "team_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_user_id": { + "name": "bot_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_token_ciphertext": { + "name": "bot_token_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_token_iv": { + "name": "bot_token_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "bot_token_tag": { + "name": "bot_token_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_id": { + "name": "api_key_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_secret_ciphertext": { + "name": "api_key_secret_ciphertext", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_secret_iv": { + "name": "api_key_secret_iv", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "api_key_secret_tag": { + "name": "api_key_secret_tag", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "installed_by_user_id": { + "name": "installed_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_reason": { + "name": "revoked_reason", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "slack_workspaces_team_id_idx": { + "name": "slack_workspaces_team_id_idx", + "columns": [ + { + "expression": "team_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "slack_workspaces_org_idx": { + "name": "slack_workspaces_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "slack_workspaces_active_org_idx": { + "name": "slack_workspaces_active_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"slack_workspaces\".\"revoked_at\" IS NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_commits": { + "name": "vcs_commits", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha": { + "name": "sha", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "author_name": { + "name": "author_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_email": { + "name": "author_email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_login": { + "name": "author_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "author_avatar_url": { + "name": "author_avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "authored_at": { + "name": "authored_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "committed_at": { + "name": "committed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_commits_repo_sha_idx": { + "name": "vcs_commits_repo_sha_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sha", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_commits_org_sha_idx": { + "name": "vcs_commits_org_sha_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "sha", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_installations": { + "name": "vcs_installations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_installation_id": { + "name": "external_installation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_login": { + "name": "account_login", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_type": { + "name": "account_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_account_id": { + "name": "external_account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "account_avatar_url": { + "name": "account_avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "repository_selection": { + "name": "repository_selection", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'all'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "suspended_at": { + "name": "suspended_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "installed_by_user_id": { + "name": "installed_by_user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_installations_provider_external_idx": { + "name": "vcs_installations_provider_external_idx", + "columns": [ + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_installations_org_idx": { + "name": "vcs_installations_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_repositories": { + "name": "vcs_repositories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installation_id": { + "name": "installation_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_repo_id": { + "name": "external_repo_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'main'" + }, + "tracked_branch": { + "name": "tracked_branch", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "html_url": { + "name": "html_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_private": { + "name": "is_private", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "is_archived": { + "name": "is_archived", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'active'" + }, + "sync_status": { + "name": "sync_status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "last_sync_error": { + "name": "last_sync_error", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_repositories_org_repo_idx": { + "name": "vcs_repositories_org_repo_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "provider", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_repo_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_repositories_org_idx": { + "name": "vcs_repositories_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_repositories_installation_idx": { + "name": "vcs_repositories_installation_idx", + "columns": [ + { + "expression": "installation_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vcs_repository_branches": { + "name": "vcs_repository_branches", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "org_id": { + "name": "org_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "repository_id": { + "name": "repository_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "is_default": { + "name": "is_default", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "head_sha": { + "name": "head_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "vcs_repository_branches_repo_name_idx": { + "name": "vcs_repository_branches_repo_name_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "vcs_repository_branches_org_idx": { + "name": "vcs_repository_branches_org_idx", + "columns": [ + { + "expression": "org_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index ab877701f..6eb193e8f 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -323,6 +323,13 @@ "when": 1787136812507, "tag": "0046_live_activities", "breakpoints": true + }, + { + "idx": 46, + "version": "7", + "when": 1787274609583, + "tag": "0047_planetscale_issue_receipts", + "breakpoints": true } ] -} \ No newline at end of file +} diff --git a/packages/db/src/schema/planetscale-inventory.ts b/packages/db/src/schema/planetscale-inventory.ts index 1773b3401..e3a5e2f87 100644 --- a/packages/db/src/schema/planetscale-inventory.ts +++ b/packages/db/src/schema/planetscale-inventory.ts @@ -1,5 +1,5 @@ import type { OrgId } from "@maple/domain" -import { boolean, index, jsonb, pgTable, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core" +import { boolean, index, jsonb, pgTable, primaryKey, text, timestamp, uniqueIndex } from "drizzle-orm/pg-core" /** * Poll-state for the PlanetScale management-API poller, mirroring @@ -161,3 +161,20 @@ export const planetscaleEvents = pgTable( export type PlanetScaleEventRow = typeof planetscaleEvents.$inferSelect export type PlanetScaleEventInsert = typeof planetscaleEvents.$inferInsert + +/** + * Exactly-once guard for issue mutations driven by an at-least-once queue. + * The receipt is inserted in the same transaction as the issue update, so a + * crash before commit leaves both absent and a retry can safely finish them. + */ +export const planetscaleIssueReceipts = pgTable( + "planetscale_issue_receipts", + { + orgId: text("org_id").$type().notNull(), + eventId: text("event_id").notNull(), + processedAt: timestamp("processed_at", { withTimezone: true, mode: "date" }).notNull(), + }, + (table) => [primaryKey({ columns: [table.orgId, table.eventId] })], +) + +export type PlanetScaleIssueReceiptRow = typeof planetscaleIssueReceipts.$inferSelect diff --git a/packages/eventing-core/README.md b/packages/eventing-core/README.md new file mode 100644 index 000000000..f6adec6be --- /dev/null +++ b/packages/eventing-core/README.md @@ -0,0 +1,27 @@ +# `@maple/eventing-core` + +Host-neutral signal-to-event contracts and deterministic runtime semantics. + +The package owns typed signal values, bounded selectors, pure projector +registration, canonical event identity, and an immutable compiled projection +registry. It has no database, network, scheduler, or wall-clock dependency. A +host authenticates and normalizes source input, supplies durable projection and +outbox adapters, and decides when compiled registries become active. + +See [`docs/signal-to-event-projection.md`](../../docs/signal-to-event-projection.md) +for the architecture and acceptance contract. + +See [`docs/eventing-extension-guide.md`](../../docs/eventing-extension-guide.md) +for a complete source adapter and projector example, registration and host +wiring patterns, versioning rules, and the required test checklist. Eventing +extensions are compile-time registered modules, not dynamically loaded plugins. + +The versioned interoperability artifacts are generated under `schemas/`, with +valid comparison and identity vectors in `fixtures/v1.json`. Run `bun test` to +verify generated-schema drift, hostile selector bounds, typed comparison +semantics, deterministic event IDs, and projector isolation. + +The first host adapter is Maple Local in `apps/cli/src/server/eventing`. It uses +an authenticated configuration endpoint, a SQLite projection/outbox store, and +the pre-chDB OTLP seam. The package itself deliberately contains none of those +host decisions. diff --git a/packages/eventing-core/fixtures/v1.json b/packages/eventing-core/fixtures/v1.json new file mode 100644 index 000000000..5e0d95985 --- /dev/null +++ b/packages/eventing-core/fixtures/v1.json @@ -0,0 +1,191 @@ +{ + "version": 1, + "eventIdVectors": [ + { + "name": "tenant-scoped projected occurrence", + "input": { + "tenantId": "tenant-a", + "sourceKind": "otel.log", + "source": "urn:maple:source:otel:local", + "occurrenceId": "event-123", + "projectionId": "example-record-observed", + "projectionRevision": 3 + }, + "output": "sha256:f278b407b3384ae705126120fb7e0919ac3ea63530979f8d43eca10879e4da6a" + } + ], + "stringLiteralByteVectors": [ + { + "name": "multibyte literal exactly at the UTF-8 byte limit", + "unit": "é", + "repeat": 2048, + "valid": true + }, + { + "name": "multibyte literal one code point beyond the UTF-8 byte limit", + "unit": "é", + "repeat": 2049, + "valid": false + } + ], + "predicateVectors": [ + { + "name": "int64 remains exact above JavaScript safe integer range", + "predicate": { + "op": "gt", + "field": { "namespace": "attribute", "key": "counter", "type": "int64" }, + "value": { "type": "int64", "value": "9007199254740992" } + }, + "fields": [ + { + "namespace": "attribute", + "key": "counter", + "value": { "type": "int64", "value": "9007199254740993" } + } + ], + "matches": true + }, + { + "name": "timestamps compare as UTC instants", + "predicate": { + "op": "eq", + "field": { "namespace": "signal", "key": "occurred_at", "type": "timestamp" }, + "value": { "type": "timestamp", "value": "2026-08-07T19:42:00.123456789Z" } + }, + "fields": [ + { + "namespace": "signal", + "key": "occurred_at", + "value": { "type": "timestamp", "value": "2026-08-07T15:42:00.123456789-04:00" } + } + ], + "matches": true + }, + { + "name": "numeric strings do not coerce", + "predicate": { + "op": "gte", + "field": { "namespace": "attribute", "key": "attempt", "type": "int64" }, + "value": { "type": "int64", "value": "3" } + }, + "fields": [ + { + "namespace": "attribute", + "key": "attempt", + "value": { "type": "string", "value": "12" } + } + ], + "matches": false, + "typeMismatches": ["attribute:attempt"] + }, + { + "name": "neq does not match a missing field", + "predicate": { + "op": "neq", + "field": { "namespace": "attribute", "key": "state", "type": "string" }, + "value": { "type": "string", "value": "closed" } + }, + "fields": [], + "matches": false + }, + { + "name": "boolean composition and string containment", + "predicate": { + "op": "all", + "clauses": [ + { + "op": "eq", + "field": { "namespace": "attribute", "key": "active", "type": "boolean" }, + "value": { "type": "boolean", "value": true } + }, + { + "op": "contains", + "field": { "namespace": "body", "key": "text", "type": "string" }, + "value": { "type": "string", "value": "record observed" } + } + ] + }, + "fields": [ + { + "namespace": "attribute", + "key": "active", + "value": { "type": "boolean", "value": true } + }, + { + "namespace": "body", + "key": "text", + "value": { "type": "string", "value": "example record observed successfully" } + } + ], + "matches": true + }, + { + "name": "float64 ordering is numeric", + "predicate": { + "op": "lt", + "field": { "namespace": "attribute", "key": "ratio", "type": "float64" }, + "value": { "type": "float64", "value": 10.25 } + }, + "fields": [ + { + "namespace": "attribute", + "key": "ratio", + "value": { "type": "float64", "value": 9.5 } + } + ], + "matches": true + }, + { + "name": "durations compare as exact nanoseconds", + "predicate": { + "op": "gte", + "field": { "namespace": "signal", "key": "duration", "type": "duration" }, + "value": { "type": "duration", "value": "1000000000" } + }, + "fields": [ + { + "namespace": "signal", + "key": "duration", + "value": { "type": "duration", "value": "1000000001" } + } + ], + "matches": true + }, + { + "name": "boolean equality has no string coercion", + "predicate": { + "op": "eq", + "field": { "namespace": "attribute", "key": "enabled", "type": "boolean" }, + "value": { "type": "boolean", "value": true } + }, + "fields": [ + { + "namespace": "attribute", + "key": "enabled", + "value": { "type": "string", "value": "true" } + } + ], + "matches": false, + "typeMismatches": ["attribute:enabled"] + }, + { + "name": "string membership is exact and case-sensitive", + "predicate": { + "op": "in", + "field": { "namespace": "attribute", "key": "state", "type": "string" }, + "values": [ + { "type": "string", "value": "opened" }, + { "type": "string", "value": "closed" } + ] + }, + "fields": [ + { + "namespace": "attribute", + "key": "state", + "value": { "type": "string", "value": "Closed" } + } + ], + "matches": false + } + ] +} diff --git a/packages/eventing-core/package.json b/packages/eventing-core/package.json new file mode 100644 index 000000000..3dd387ac9 --- /dev/null +++ b/packages/eventing-core/package.json @@ -0,0 +1,24 @@ +{ + "name": "@maple/eventing-core", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "schemas": "bun run scripts/generate-schemas.ts", + "schemas:check": "bun run scripts/generate-schemas.ts --check", + "test": "bun run schemas:check && vitest run", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "effect": "catalog:effect" + }, + "devDependencies": { + "@effect/language-service": "catalog:effect", + "@types/node": "catalog:tooling", + "typescript": "catalog:tooling", + "vitest": "catalog:" + } +} diff --git a/packages/eventing-core/schemas/cloud-event.v1.schema.json b/packages/eventing-core/schemas/cloud-event.v1.schema.json new file mode 100644 index 000000000..323056917 --- /dev/null +++ b/packages/eventing-core/schemas/cloud-event.v1.schema.json @@ -0,0 +1,180 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:eventing:schema:cloud-event:v1", + "$ref": "#/$defs/MapleCloudEvent", + "$defs": { + "MapleCloudEvent": { + "type": "object", + "properties": { + "specversion": { + "type": "string", + "enum": ["1.0"] + }, + "id": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "source": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "type": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "subject": { + "type": "string" + }, + "time": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + }, + "datacontenttype": { + "type": "string", + "enum": ["application/json"] + }, + "dataschema": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "tenantid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "projectionid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "projectionrevision": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "projectorid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "projectorversion": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "sourceoccurrenceid": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "sourceidentityquality": { + "type": "string", + "enum": ["source", "derived", "none"] + }, + "data": {} + }, + "required": [ + "specversion", + "id", + "source", + "type", + "time", + "datacontenttype", + "dataschema", + "tenantid", + "projectionid", + "projectionrevision", + "projectorid", + "projectorversion", + "data" + ], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/signal-projection.v1.schema.json b/packages/eventing-core/schemas/signal-projection.v1.schema.json new file mode 100644 index 000000000..0523586da --- /dev/null +++ b/packages/eventing-core/schemas/signal-projection.v1.schema.json @@ -0,0 +1,380 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:eventing:schema:signal-projection:v1", + "$ref": "#/$defs/SignalProjectionSpec", + "$defs": { + "SignalFieldRef": { + "type": "object", + "properties": { + "namespace": { + "type": "string", + "enum": ["signal", "resource", "scope", "attribute", "body"] + }, + "key": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 512 + } + ] + }, + "type": { + "type": "string", + "enum": ["string", "boolean", "int64", "float64", "timestamp", "duration"] + } + }, + "required": ["namespace", "key", "type"], + "additionalProperties": false + }, + "SignalLiteral": { + "anyOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["string"] + }, + "value": { + "type": "string", + "description": "Predicate string literal limited to 4096 UTF-8 bytes; JSON Schema cannot express this byte-count constraint" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["boolean"] + }, + "value": { + "type": "boolean" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["int64"] + }, + "value": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["float64"] + }, + "value": { + "type": "number" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["timestamp"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["duration"] + }, + "value": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + ] + }, + "SignalPredicate": { + "anyOf": [ + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["all"] + }, + "clauses": { + "type": "array", + "items": { + "$ref": "#/$defs/SignalPredicate" + }, + "allOf": [ + { + "minItems": 1 + }, + { + "maxItems": 64 + } + ] + } + }, + "required": ["op", "clauses"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["any"] + }, + "clauses": { + "type": "array", + "items": { + "$ref": "#/$defs/SignalPredicate" + }, + "allOf": [ + { + "minItems": 1 + }, + { + "maxItems": 64 + } + ] + } + }, + "required": ["op", "clauses"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["not"] + }, + "clause": { + "$ref": "#/$defs/SignalPredicate" + } + }, + "required": ["op", "clause"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["exists"] + }, + "field": { + "$ref": "#/$defs/SignalFieldRef" + } + }, + "required": ["op", "field"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["eq", "neq", "gt", "gte", "lt", "lte", "contains"] + }, + "field": { + "$ref": "#/$defs/SignalFieldRef" + }, + "value": { + "$ref": "#/$defs/SignalLiteral" + } + }, + "required": ["op", "field", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "op": { + "type": "string", + "enum": ["in"] + }, + "field": { + "$ref": "#/$defs/SignalFieldRef" + }, + "values": { + "type": "array", + "items": { + "$ref": "#/$defs/SignalLiteral" + }, + "allOf": [ + { + "minItems": 1 + }, + { + "maxItems": 100 + } + ] + } + }, + "required": ["op", "field", "values"], + "additionalProperties": false + } + ] + }, + "SignalProjectionSpec": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "revision": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "enabled": { + "type": "boolean" + }, + "tenantId": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "sourceKind": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "selector": { + "$ref": "#/$defs/SignalPredicate" + }, + "projector": { + "type": "object", + "properties": { + "id": { + "type": "string", + "allOf": [ + { + "minLength": 1 + }, + { + "maxLength": 256 + }, + { + "pattern": "^\\S[\\s\\S]*\\S$|^\\S$|^$" + } + ] + }, + "version": { + "type": "integer", + "allOf": [ + { + "exclusiveMinimum": 0 + } + ] + }, + "config": {} + }, + "required": ["id", "version", "config"], + "additionalProperties": false + }, + "activeFrom": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + } + }, + "required": [ + "id", + "revision", + "enabled", + "tenantId", + "sourceKind", + "selector", + "projector", + "activeFrom" + ], + "additionalProperties": false + } + } +} diff --git a/packages/eventing-core/schemas/signal-scalar.v1.schema.json b/packages/eventing-core/schemas/signal-scalar.v1.schema.json new file mode 100644 index 000000000..d83d0d9a2 --- /dev/null +++ b/packages/eventing-core/schemas/signal-scalar.v1.schema.json @@ -0,0 +1,116 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:maple:eventing:schema:signal-scalar:v1", + "$ref": "#/$defs/SignalScalar", + "$defs": { + "SignalScalar": { + "anyOf": [ + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["string"] + }, + "value": { + "type": "string" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["boolean"] + }, + "value": { + "type": "boolean" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["int64"] + }, + "value": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["float64"] + }, + "value": { + "type": "number" + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["timestamp"] + }, + "value": { + "type": "string", + "allOf": [ + { + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d{1,9})?(?:Z|[+-]\\d{2}:\\d{2})$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "type": { + "type": "string", + "enum": ["duration"] + }, + "value": { + "type": "string", + "allOf": [ + { + "maxLength": 20 + }, + { + "pattern": "^-?(?:0|[1-9][0-9]*)$" + } + ] + } + }, + "required": ["type", "value"], + "additionalProperties": false + } + ] + } + } +} diff --git a/packages/eventing-core/scripts/generate-schemas.ts b/packages/eventing-core/scripts/generate-schemas.ts new file mode 100644 index 000000000..d2c1db3ee --- /dev/null +++ b/packages/eventing-core/scripts/generate-schemas.ts @@ -0,0 +1,61 @@ +import { execFileSync } from "node:child_process" +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs" +import { dirname, resolve } from "node:path" +import { Schema } from "effect" +import { MapleCloudEventSchema, SignalProjectionSpecSchema, SignalScalarSchema } from "../src/model" + +const root = resolve(import.meta.dirname, "..") +const check = process.argv.includes("--check") + +const documents = [ + { + path: "schemas/signal-scalar.v1.schema.json", + id: "urn:maple:eventing:schema:signal-scalar:v1", + schema: SignalScalarSchema, + }, + { + path: "schemas/signal-projection.v1.schema.json", + id: "urn:maple:eventing:schema:signal-projection:v1", + schema: SignalProjectionSpecSchema, + }, + { + path: "schemas/cloud-event.v1.schema.json", + id: "urn:maple:eventing:schema:cloud-event:v1", + schema: MapleCloudEventSchema, + }, +] as const + +let stale = false +for (const entry of documents) { + const document = Schema.toJsonSchemaDocument(entry.schema) + const schemaDocument = + Object.keys(document.definitions).length === 0 + ? { $schema: "https://json-schema.org/draft/2020-12/schema", $id: entry.id, ...document.schema } + : { + $schema: "https://json-schema.org/draft/2020-12/schema", + $id: entry.id, + ...document.schema, + $defs: document.definitions, + } + const unformatted = `${JSON.stringify(schemaDocument, null, "\t")}\n` + const serialized = execFileSync( + resolve(root, "../../node_modules/.bin/oxfmt"), + ["--stdin-filepath", entry.path], + { + input: unformatted, + encoding: "utf8", + }, + ) + const path = resolve(root, entry.path) + if (check) { + if (!existsSync(path) || readFileSync(path, "utf8") !== serialized) { + console.error(`${entry.path} is stale; run bun run schemas`) + stale = true + } + } else { + mkdirSync(dirname(path), { recursive: true }) + writeFileSync(path, serialized) + } +} + +if (stale) process.exitCode = 1 diff --git a/packages/eventing-core/src/event.ts b/packages/eventing-core/src/event.ts new file mode 100644 index 000000000..af16e2338 --- /dev/null +++ b/packages/eventing-core/src/event.ts @@ -0,0 +1,151 @@ +import { createHash } from "node:crypto" +import { Schema } from "effect" +import { + MapleCloudEventSchema, + type JsonValue, + type MapleCloudEvent, + type NormalizedSignal, + type SignalProjectionSpec, +} from "./model" +import { timestampToEpochNanos } from "./predicate" + +export const MAX_CLOUD_EVENT_BYTES = 256 * 1024 + +export interface EventIdentityInput { + readonly tenantId: string + readonly sourceKind: string + readonly source: string + readonly occurrenceId: string + readonly projectionId: string + readonly projectionRevision: number +} + +const updateLengthDelimited = (hash: ReturnType, value: string): void => { + const encoded = Buffer.from(value, "utf8") + const length = Buffer.allocUnsafe(4) + length.writeUInt32BE(encoded.byteLength) + hash.update(length) + hash.update(encoded) +} + +/** Canonical v1 identity shared by every host implementation. */ +export const makeEventId = (input: EventIdentityInput): string => { + const hash = createHash("sha256") + for (const field of [ + "maple-event-v1", + input.tenantId, + input.sourceKind, + input.source, + input.occurrenceId, + input.projectionId, + String(input.projectionRevision), + ]) + updateLengthDelimited(hash, field) + return `sha256:${hash.digest("hex")}` +} + +export const isJsonValue = (value: unknown, seen: Set = new Set()): value is JsonValue => { + if (value === null || typeof value === "string" || typeof value === "boolean") return true + if (typeof value === "number") return Number.isFinite(value) + if (typeof value !== "object") return false + if (seen.has(value)) return false + seen.add(value) + try { + if (Array.isArray(value)) return value.every((item) => isJsonValue(item, seen)) + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return false + return Object.values(value).every((item) => isJsonValue(item, seen)) + } finally { + // Track the active recursion path. Repeated references serialize as a + // JSON tree and are not themselves cycles. + seen.delete(value) + } +} + +const canonicalizeJson = (value: JsonValue): JsonValue => { + if (value === null || typeof value !== "object") return value + if (Array.isArray(value)) return value.map(canonicalizeJson) + const record = value as { readonly [key: string]: JsonValue } + return Object.fromEntries( + Object.keys(record) + .sort() + .map((key) => [key, canonicalizeJson(record[key]!)]), + ) +} + +/** Stable JSON encoding for outbox collision checks and cross-host fixtures. */ +export const canonicalJson = (value: JsonValue): string => { + if (!isJsonValue(value)) throw new Error("value must be finite acyclic JSON") + return JSON.stringify(canonicalizeJson(value)) +} + +export interface ValidatedMapleCloudEvent { + readonly event: MapleCloudEvent + readonly canonicalJson: string + readonly byteLength: number +} + +/** Validate the complete persisted envelope, including its canonical byte budget. */ +export const validateMapleCloudEvent = (candidate: unknown): ValidatedMapleCloudEvent => { + const event = Schema.decodeUnknownSync(MapleCloudEventSchema)(candidate) + if (!isJsonValue(event)) throw new Error("CloudEvent must be finite JSON") + // SAFETY: the envelope schema and finite-JSON guard establish MapleCloudEvent's complete contract. + const validatedEvent = event as MapleCloudEvent + const eventJson = canonicalJson(event) + const byteLength = Buffer.byteLength(eventJson, "utf8") + if (byteLength > MAX_CLOUD_EVENT_BYTES) + throw new Error(`CloudEvent exceeds ${MAX_CLOUD_EVENT_BYTES} UTF-8 bytes`) + return { event: validatedEvent, canonicalJson: eventJson, byteLength } +} + +export const makeCloudEvent = (input: { + readonly signal: NormalizedSignal + readonly projection: SignalProjectionSpec + readonly projectorId: string + readonly projectorVersion: number + readonly outputType: string + readonly dataSchema: string + readonly subject?: string | null + readonly time?: string + readonly data: JsonValue +}): MapleCloudEvent => { + if ( + input.signal.occurrenceId === null || + input.signal.occurrenceId.trim().length === 0 || + input.signal.identityQuality === "none" + ) + throw new Error("durable event projection requires stable or derived occurrence identity") + if (!isJsonValue(input.data)) throw new Error("projected event data must be finite JSON") + if (input.outputType.trim().length === 0) throw new Error("projected event type must not be empty") + if (input.dataSchema.trim().length === 0) throw new Error("projected event data schema must not be empty") + if (input.signal.source.trim().length === 0) throw new Error("signal source must not be empty") + + const subject = input.subject ?? input.signal.subject + const time = input.time ?? input.signal.occurredAt + if (timestampToEpochNanos(time) === null) throw new Error("projected event time must be a valid instant") + const envelope = { + specversion: "1.0", + id: makeEventId({ + tenantId: input.signal.tenantId, + sourceKind: input.signal.sourceKind, + source: input.signal.source, + occurrenceId: input.signal.occurrenceId, + projectionId: input.projection.id, + projectionRevision: input.projection.revision, + }), + source: input.signal.source, + type: input.outputType, + time, + datacontenttype: "application/json", + dataschema: input.dataSchema, + tenantid: input.signal.tenantId, + projectionid: input.projection.id, + projectionrevision: input.projection.revision, + projectorid: input.projectorId, + projectorversion: input.projectorVersion, + sourceoccurrenceid: input.signal.occurrenceId, + sourceidentityquality: input.signal.identityQuality, + data: input.data, + } + return validateMapleCloudEvent(subject == null ? envelope : { ...envelope, subject }).event +} diff --git a/packages/eventing-core/src/index.ts b/packages/eventing-core/src/index.ts new file mode 100644 index 000000000..87253218d --- /dev/null +++ b/packages/eventing-core/src/index.ts @@ -0,0 +1,5 @@ +export * from "./event" +export * from "./model" +export * from "./predicate" +export * from "./registry" +export * from "./source" diff --git a/packages/eventing-core/src/model.ts b/packages/eventing-core/src/model.ts new file mode 100644 index 000000000..bce303e33 --- /dev/null +++ b/packages/eventing-core/src/model.ts @@ -0,0 +1,267 @@ +import { Schema } from "effect" + +export const MAX_PREDICATE_DEPTH = 8 +export const MAX_PREDICATE_NODES = 64 +export const MAX_IN_VALUES = 100 +export const MAX_STRING_LITERAL_BYTES = 4 * 1024 +export const MAX_DECIMAL_INT64_LENGTH = 20 + +const utf8Bytes = (value: string): number => new TextEncoder().encode(value).byteLength + +const NonEmptyIdentifier = Schema.String.check( + Schema.isMinLength(1), + Schema.isMaxLength(256), + Schema.isTrimmed(), +) + +const DecimalInt64 = Schema.String.check( + Schema.isMaxLength(MAX_DECIMAL_INT64_LENGTH), + Schema.isPattern(/^-?(?:0|[1-9][0-9]*)$/), +) + +const Rfc3339Timestamp = Schema.String.check( + Schema.isPattern(/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,9})?(?:Z|[+-]\d{2}:\d{2})$/), +) + +export const StringSignalScalar = Schema.Struct({ + type: Schema.Literal("string"), + value: Schema.String, +}) + +const StringLiteralValue = Schema.String.annotate({ + description: `Predicate string literal limited to ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes; JSON Schema cannot express this byte-count constraint`, +}).check( + Schema.makeFilter((value) => utf8Bytes(value) <= MAX_STRING_LITERAL_BYTES, { + expected: `a string no larger than ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes`, + description: `Predicate string literal limited to ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes`, + }), +) + +export const StringSignalLiteral = Schema.Struct({ + type: Schema.Literal("string"), + value: StringLiteralValue, +}) + +export const BooleanSignalScalar = Schema.Struct({ + type: Schema.Literal("boolean"), + value: Schema.Boolean, +}) + +export const Int64SignalScalar = Schema.Struct({ + type: Schema.Literal("int64"), + value: DecimalInt64, +}) + +export const Float64SignalScalar = Schema.Struct({ + type: Schema.Literal("float64"), + value: Schema.Finite, +}) + +export const TimestampSignalScalar = Schema.Struct({ + type: Schema.Literal("timestamp"), + value: Rfc3339Timestamp, +}) + +export const DurationSignalScalar = Schema.Struct({ + type: Schema.Literal("duration"), + value: DecimalInt64, +}) + +export const SignalScalarSchema = Schema.Union([ + StringSignalScalar, + BooleanSignalScalar, + Int64SignalScalar, + Float64SignalScalar, + TimestampSignalScalar, + DurationSignalScalar, +]).annotate({ identifier: "SignalScalar" }) +export type SignalScalar = Schema.Schema.Type +export type SignalScalarType = SignalScalar["type"] + +export const SignalLiteralSchema = Schema.Union([ + StringSignalLiteral, + BooleanSignalScalar, + Int64SignalScalar, + Float64SignalScalar, + TimestampSignalScalar, + DurationSignalScalar, +]).annotate({ identifier: "SignalLiteral" }) +export type SignalLiteral = Schema.Schema.Type + +export const FieldNamespaceSchema = Schema.Literals(["signal", "resource", "scope", "attribute", "body"]) +export type FieldNamespace = Schema.Schema.Type + +export const FieldRefSchema = Schema.Struct({ + namespace: FieldNamespaceSchema, + key: Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(512)), + type: Schema.Literals(["string", "boolean", "int64", "float64", "timestamp", "duration"]), +}).annotate({ identifier: "SignalFieldRef" }) +export type FieldRef = Schema.Schema.Type + +export interface AllPredicate { + readonly op: "all" + readonly clauses: readonly SignalPredicate[] +} + +export interface AnyPredicate { + readonly op: "any" + readonly clauses: readonly SignalPredicate[] +} + +export interface NotPredicate { + readonly op: "not" + readonly clause: SignalPredicate +} + +export interface ExistsPredicate { + readonly op: "exists" + readonly field: FieldRef +} + +export interface ComparisonPredicate { + readonly op: "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" + readonly field: FieldRef + readonly value: SignalLiteral +} + +export interface InPredicate { + readonly op: "in" + readonly field: FieldRef + readonly values: readonly SignalLiteral[] +} + +export type SignalPredicate = + | AllPredicate + | AnyPredicate + | NotPredicate + | ExistsPredicate + | ComparisonPredicate + | InPredicate + +export const SignalPredicateSchema: Schema.Codec = Schema.suspend( + (): Schema.Codec => + Schema.Union([ + Schema.Struct({ + op: Schema.Literal("all"), + clauses: Schema.Array(SignalPredicateSchema).check( + Schema.isMinLength(1), + Schema.isMaxLength(MAX_PREDICATE_NODES), + ), + }), + Schema.Struct({ + op: Schema.Literal("any"), + clauses: Schema.Array(SignalPredicateSchema).check( + Schema.isMinLength(1), + Schema.isMaxLength(MAX_PREDICATE_NODES), + ), + }), + Schema.Struct({ + op: Schema.Literal("not"), + clause: SignalPredicateSchema, + }), + Schema.Struct({ + op: Schema.Literal("exists"), + field: FieldRefSchema, + }), + Schema.Struct({ + op: Schema.Literals(["eq", "neq", "gt", "gte", "lt", "lte", "contains"]), + field: FieldRefSchema, + value: SignalLiteralSchema, + }), + Schema.Struct({ + op: Schema.Literal("in"), + field: FieldRefSchema, + values: Schema.Array(SignalLiteralSchema).check( + Schema.isMinLength(1), + Schema.isMaxLength(MAX_IN_VALUES), + ), + }), + ]) as Schema.Codec, +).annotate({ identifier: "SignalPredicate" }) + +export const ProjectorRefSchema = Schema.Struct({ + id: NonEmptyIdentifier, + version: Schema.Int.check(Schema.isGreaterThan(0)), + config: Schema.Unknown, +}) +export type ProjectorRef = Schema.Schema.Type + +export const SignalProjectionSpecSchema = Schema.Struct({ + id: NonEmptyIdentifier, + revision: Schema.Int.check(Schema.isGreaterThan(0)), + enabled: Schema.Boolean, + tenantId: NonEmptyIdentifier, + sourceKind: NonEmptyIdentifier, + selector: SignalPredicateSchema, + projector: ProjectorRefSchema, + activeFrom: Rfc3339Timestamp, +}).annotate({ identifier: "SignalProjectionSpec" }) +export type SignalProjectionSpec = Schema.Schema.Type + +export interface NormalizedSignal { + readonly sourceKind: string + readonly source: string + readonly tenantId: string + readonly occurrenceId: string | null + readonly identityQuality: "source" | "derived" | "none" + readonly occurredAt: string + readonly observedAt: string + readonly subject: string | null + readonly fields: ReadonlyMap + readonly data: TData +} + +export type JsonPrimitive = string | number | boolean | null +export type JsonValue = JsonPrimitive | { readonly [key: string]: JsonValue } | readonly JsonValue[] + +export interface ProjectedEventData { + readonly subject?: string | null + readonly time?: string + readonly data: TData +} + +export interface MapleCloudEvent { + readonly specversion: "1.0" + readonly id: string + readonly source: string + readonly type: string + readonly subject?: string + readonly time: string + readonly datacontenttype: "application/json" + readonly dataschema: string + readonly tenantid: string + readonly projectionid: string + readonly projectionrevision: number + readonly projectorid: string + readonly projectorversion: number + readonly sourceoccurrenceid?: string + readonly sourceidentityquality?: "source" | "derived" | "none" + readonly data: JsonValue +} + +export const MapleCloudEventSchema = Schema.Struct({ + specversion: Schema.Literal("1.0"), + id: NonEmptyIdentifier, + source: NonEmptyIdentifier, + type: NonEmptyIdentifier, + subject: Schema.optionalKey(Schema.String), + time: Rfc3339Timestamp, + datacontenttype: Schema.Literal("application/json"), + dataschema: NonEmptyIdentifier, + tenantid: NonEmptyIdentifier, + projectionid: NonEmptyIdentifier, + projectionrevision: Schema.Int.check(Schema.isGreaterThan(0)), + projectorid: NonEmptyIdentifier, + projectorversion: Schema.Int.check(Schema.isGreaterThan(0)), + sourceoccurrenceid: Schema.optionalKey(NonEmptyIdentifier), + sourceidentityquality: Schema.optionalKey(Schema.Literals(["source", "derived", "none"])), + data: Schema.Unknown, +}).annotate({ identifier: "MapleCloudEvent" }) + +export const fieldKey = (field: Pick): string => + `${field.namespace}:${field.key}` + +export const defineSignalFields = ( + fields: ReadonlyArray<{ readonly field: FieldRef; readonly value: SignalScalar }>, +): ReadonlyMap => + new Map(fields.map(({ field, value }) => [fieldKey(field), value] as const)) diff --git a/packages/eventing-core/src/predicate.test.ts b/packages/eventing-core/src/predicate.test.ts new file mode 100644 index 000000000..a2887fb50 --- /dev/null +++ b/packages/eventing-core/src/predicate.test.ts @@ -0,0 +1,226 @@ +import { readFileSync } from "node:fs" +import { Schema } from "effect" +import { describe, expect, it } from "vitest" +import { + assertSignalProjectionInputBudget, + compileSignalPredicate, + defineSignalFields, + fieldKey, + makeEventId, + MAX_PREDICATE_DEPTH, + SignalLiteralSchema, + SignalPredicateSchema, + SignalScalarSchema, + timestampToEpochNanos, + validateSignalPredicate, + type EventIdentityInput, + type FieldNamespace, + type FieldRef, + type NormalizedSignal, + type SignalPredicate, +} from "./index" + +interface ConformanceFixture { + readonly eventIdVectors: ReadonlyArray<{ + readonly name: string + readonly input: EventIdentityInput + readonly output: string + }> + readonly stringLiteralByteVectors: ReadonlyArray<{ + readonly name: string + readonly unit: string + readonly repeat: number + readonly valid: boolean + }> + readonly predicateVectors: ReadonlyArray<{ + readonly name: string + readonly predicate: unknown + readonly fields: ReadonlyArray<{ + readonly namespace: FieldNamespace + readonly key: string + readonly value: unknown + }> + readonly matches: boolean + readonly typeMismatches?: readonly string[] + }> +} + +// SAFETY: the conformance suite exercises every decoded fixture field below against its owning schema. +const fixture = JSON.parse( + readFileSync(new URL("../fixtures/v1.json", import.meta.url), "utf8"), +) as ConformanceFixture + +const signalFor = (fields: ConformanceFixture["predicateVectors"][number]["fields"]): NormalizedSignal => ({ + sourceKind: "otel.log", + source: "urn:maple:source:otel:local", + tenantId: "tenant-a", + occurrenceId: "occurrence-1", + identityQuality: "source", + occurredAt: "2026-08-07T19:42:00Z", + observedAt: "2026-08-07T19:42:01Z", + subject: null, + fields: defineSignalFields( + fields.map(({ namespace, key, value }) => ({ + field: { + namespace, + key, + type: Schema.decodeUnknownSync(SignalScalarSchema)(value).type, + }, + value: Schema.decodeUnknownSync(SignalScalarSchema)(value), + })), + ), + data: {}, +}) + +describe("cross-language conformance vectors", () => { + for (const vector of fixture.eventIdVectors) { + it(`event ID: ${vector.name}`, () => { + expect(makeEventId(vector.input)).toBe(vector.output) + }) + } + + for (const vector of fixture.stringLiteralByteVectors) { + it(`string literal bytes: ${vector.name}`, () => { + const candidate = { type: "string", value: vector.unit.repeat(vector.repeat) } + if (vector.valid) + expect(() => Schema.decodeUnknownSync(SignalLiteralSchema)(candidate)).not.toThrow() + else expect(() => Schema.decodeUnknownSync(SignalLiteralSchema)(candidate)).toThrow() + }) + } + + for (const vector of fixture.predicateVectors) { + it(`predicate: ${vector.name}`, () => { + const predicate = Schema.decodeUnknownSync(SignalPredicateSchema)(vector.predicate) + const result = compileSignalPredicate(predicate)(signalFor(vector.fields)) + expect(result.matches).toBe(vector.matches) + expect(result.typeMismatches.map(fieldKey)).toEqual(vector.typeMismatches ?? []) + }) + } +}) + +describe("selector validation", () => { + it("rejects wrong literal types and unsupported ordering", () => { + const field: FieldRef = { namespace: "attribute", key: "enabled", type: "boolean" } + expect( + validateSignalPredicate({ op: "gt", field, value: { type: "string", value: "true" } }), + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ message: "gt is not supported for boolean" }), + expect.objectContaining({ message: "field and literal types must match" }), + ]), + ) + }) + + it("rejects empty combinators and excessive nesting", () => { + expect(validateSignalPredicate({ op: "all", clauses: [] })).toContainEqual({ + path: "selector.clauses", + message: "all requires at least one clause", + }) + + let nested = { + op: "exists" as const, + field: { namespace: "attribute" as const, key: "x", type: "string" as const }, + } + for (let i = 0; i < MAX_PREDICATE_DEPTH; i++) nested = { op: "not", clause: nested } as never + expect(validateSignalPredicate(nested)).toEqual( + expect.arrayContaining([expect.objectContaining({ message: `predicate depth exceeds 8` })]), + ) + }) + + it("rejects invalid calendar dates and int64 overflow", () => { + expect(timestampToEpochNanos("2026-02-31T00:00:00Z")).toBeNull() + expect( + validateSignalPredicate({ + op: "eq", + field: { namespace: "attribute", key: "n", type: "int64" }, + value: { type: "int64", value: "9223372036854775808" }, + }), + ).toContainEqual( + expect.objectContaining({ message: "int64 must be a signed 64-bit decimal integer" }), + ) + }) + + it("rejects hostile raw predicate topology before recursive schema decoding", () => { + let deeplyNested: SignalPredicate = { + op: "exists", + field: { namespace: "attribute", key: "x", type: "string" }, + } + for (let index = 0; index < MAX_PREDICATE_DEPTH; index++) + deeplyNested = { op: "not", clause: deeplyNested } + expect(() => assertSignalProjectionInputBudget({ selector: deeplyNested })).toThrow( + "predicate depth exceeds", + ) + + expect(() => + assertSignalProjectionInputBudget({ + selector: { + op: "all", + clauses: Array.from({ length: 65 }, () => ({ + op: "exists", + field: { namespace: "attribute", key: "x", type: "string" }, + })), + }, + }), + ).toThrow("clause list exceeds") + + expect(() => + assertSignalProjectionInputBudget({ + selector: { + op: "eq", + field: { namespace: "attribute", key: "n", type: "int64" }, + value: { type: "int64", value: "1".repeat(21) }, + }, + }), + ).toThrow("int64 literal exceeds") + }) +}) + +describe("total runtime behavior", () => { + it("accepts valid large source strings for exists and small contains literals", () => { + const largeValue = `${"a".repeat(5 * 1024)}needle` + const signal = signalFor([ + { + namespace: "attribute", + key: "large.description", + value: { type: "string", value: largeValue }, + }, + ]) + const field = { + namespace: "attribute" as const, + key: "large.description", + type: "string" as const, + } + + expect(compileSignalPredicate({ op: "exists", field })(signal).matches).toBe(true) + expect( + compileSignalPredicate({ + op: "contains", + field, + value: { type: "string", value: "needle" }, + })(signal).matches, + ).toBe(true) + }) + + it("treats malformed source scalars as mismatches rather than throwing", () => { + const field: FieldRef = { namespace: "attribute", key: "n", type: "int64" } + const evaluate = compileSignalPredicate({ + op: "gte", + field, + value: { type: "int64", value: "1" }, + }) + const signal = signalFor([]) + const fields = new Map(signal.fields) + fields.set(fieldKey(field), { type: "int64", value: "not-an-integer" }) + expect(evaluate({ ...signal, fields })).toMatchObject({ + matches: false, + typeMismatches: [field], + }) + }) + + it("distinguishes neq from not(eq) for a missing field", () => { + const field: FieldRef = { namespace: "attribute", key: "state", type: "string" } + const eq = { op: "eq" as const, field, value: { type: "string" as const, value: "closed" } } + expect(compileSignalPredicate({ ...eq, op: "neq" })(signalFor([])).matches).toBe(false) + expect(compileSignalPredicate({ op: "not", clause: eq })(signalFor([])).matches).toBe(true) + }) +}) diff --git a/packages/eventing-core/src/predicate.ts b/packages/eventing-core/src/predicate.ts new file mode 100644 index 000000000..d63a7b201 --- /dev/null +++ b/packages/eventing-core/src/predicate.ts @@ -0,0 +1,414 @@ +import type { + FieldRef, + NormalizedSignal, + SignalLiteral, + SignalPredicate, + SignalProjectionSpec, + SignalScalar, + SignalScalarType, +} from "./model" +import { + fieldKey, + MAX_DECIMAL_INT64_LENGTH, + MAX_IN_VALUES, + MAX_PREDICATE_DEPTH, + MAX_PREDICATE_NODES, + MAX_STRING_LITERAL_BYTES, +} from "./model" + +const INT64_MIN = -(1n << 63n) +const INT64_MAX = (1n << 63n) - 1n +const ORDERED_TYPES = new Set(["int64", "float64", "timestamp", "duration"]) + +export interface ValidationIssue { + readonly path: string + readonly message: string +} + +export class SignalPredicateValidationError extends Error { + readonly issues: readonly ValidationIssue[] + + constructor(issues: readonly ValidationIssue[]) { + super(issues.map(({ path, message }) => `${path}: ${message}`).join("; ")) + this.name = "SignalPredicateValidationError" + this.issues = issues + } +} + +const stringBytes = (value: string): number => new TextEncoder().encode(value).byteLength + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const assertScalarInputBudget = (value: unknown): void => { + if (!isRecord(value) || typeof value.type !== "string") return + if (value.type === "string" && typeof value.value === "string") { + if (stringBytes(value.value) > MAX_STRING_LITERAL_BYTES) + throw new Error(`selector string exceeds ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes`) + return + } + if ( + (value.type === "int64" || value.type === "duration") && + typeof value.value === "string" && + value.value.length > MAX_DECIMAL_INT64_LENGTH + ) + throw new Error(`${value.type} literal exceeds ${MAX_DECIMAL_INT64_LENGTH} characters`) +} + +/** + * Reject hostile selector topology before the recursive runtime schema sees it. + * HTTP hosts should additionally bound the serialized request body. + */ +export const assertSignalProjectionInputBudget = (candidate: unknown): void => { + if (!isRecord(candidate) || candidate.selector === undefined) return + const stack: Array<{ readonly value: unknown; readonly depth: number }> = [ + { value: candidate.selector, depth: 1 }, + ] + const seen = new Set() + let nodes = 0 + while (stack.length > 0) { + const current = stack.pop()! + if (current.depth > MAX_PREDICATE_DEPTH) + throw new Error(`predicate depth exceeds ${MAX_PREDICATE_DEPTH}`) + nodes += 1 + if (nodes > MAX_PREDICATE_NODES) throw new Error(`predicate exceeds ${MAX_PREDICATE_NODES} nodes`) + if (!isRecord(current.value)) continue + if (seen.has(current.value)) throw new Error("predicate must be acyclic JSON") + seen.add(current.value) + + switch (current.value.op) { + case "all": + case "any": { + const clauses = current.value.clauses + if (!Array.isArray(clauses)) break + if (clauses.length > MAX_PREDICATE_NODES) + throw new Error(`predicate clause list exceeds ${MAX_PREDICATE_NODES} entries`) + for (let index = clauses.length - 1; index >= 0; index--) + stack.push({ value: clauses[index], depth: current.depth + 1 }) + break + } + case "not": + stack.push({ value: current.value.clause, depth: current.depth + 1 }) + break + case "in": { + const values = current.value.values + if (!Array.isArray(values)) break + if (values.length > MAX_IN_VALUES) throw new Error(`in exceeds ${MAX_IN_VALUES} values`) + for (const value of values) assertScalarInputBudget(value) + break + } + default: + assertScalarInputBudget(current.value.value) + } + } +} + +const parseInt64 = (value: string): bigint | null => { + try { + const parsed = BigInt(value) + return parsed >= INT64_MIN && parsed <= INT64_MAX ? parsed : null + } catch { + return null + } +} + +const isLeapYear = (year: number): boolean => year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0) + +const daysInMonth = (year: number, month: number): number => { + switch (month) { + case 2: + return isLeapYear(year) ? 29 : 28 + case 4: + case 6: + case 9: + case 11: + return 30 + default: + return 31 + } +} + +const TIMESTAMP = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|([+-])(\d{2}):(\d{2}))$/ + +/** Parse the v1 RFC 3339 subset into exact UTC nanoseconds. */ +export const timestampToEpochNanos = (value: string): bigint | null => { + const match = TIMESTAMP.exec(value) + if (!match) return null + const year = Number(match[1]) + const month = Number(match[2]) + const day = Number(match[3]) + const hour = Number(match[4]) + const minute = Number(match[5]) + const second = Number(match[6]) + const fraction = match[7] ?? "" + if ( + month < 1 || + month > 12 || + day < 1 || + day > daysInMonth(year, month) || + hour > 23 || + minute > 59 || + second > 59 + ) + return null + + let offsetMinutes = 0 + if (match[8] !== "Z") { + const offsetHours = Number(match[10]) + const offsetMinutePart = Number(match[11]) + if (offsetHours > 23 || offsetMinutePart > 59) return null + offsetMinutes = offsetHours * 60 + offsetMinutePart + if (match[9] === "-") offsetMinutes = -offsetMinutes + } + + const date = new Date(0) + date.setUTCFullYear(year, month - 1, day) + date.setUTCHours(hour, minute, second, 0) + const milliseconds = date.getTime() - offsetMinutes * 60_000 + if (!Number.isFinite(milliseconds)) return null + const nanos = BigInt(fraction.padEnd(9, "0")) + return BigInt(milliseconds) * 1_000_000n + nanos +} + +export const validateSignalScalar = (scalar: SignalScalar, path = "value"): readonly ValidationIssue[] => { + const issues: ValidationIssue[] = [] + switch (scalar.type) { + case "string": + case "boolean": + break + case "int64": + case "duration": + if (parseInt64(scalar.value) === null) + issues.push({ path, message: `${scalar.type} must be a signed 64-bit decimal integer` }) + break + case "float64": + if (!Number.isFinite(scalar.value)) issues.push({ path, message: "float64 must be finite" }) + break + case "timestamp": + if (timestampToEpochNanos(scalar.value) === null) + issues.push({ + path, + message: "timestamp must be a valid RFC 3339 instant with an explicit offset", + }) + break + } + return issues +} + +export const validateSignalLiteral = (literal: SignalLiteral, path = "value"): readonly ValidationIssue[] => [ + ...validateSignalScalar(literal, path), + ...(literal.type === "string" && stringBytes(literal.value) > MAX_STRING_LITERAL_BYTES + ? [{ path, message: `string exceeds ${MAX_STRING_LITERAL_BYTES} UTF-8 bytes` }] + : []), +] + +export const validateSignalPredicate = (predicate: SignalPredicate): readonly ValidationIssue[] => { + const issues: ValidationIssue[] = [] + let nodes = 0 + + const visit = (node: SignalPredicate, path: string, depth: number): void => { + nodes += 1 + if (nodes > MAX_PREDICATE_NODES) return + if (depth > MAX_PREDICATE_DEPTH) { + issues.push({ path, message: `predicate depth exceeds ${MAX_PREDICATE_DEPTH}` }) + return + } + + switch (node.op) { + case "all": + case "any": + if (node.clauses.length === 0) + issues.push({ + path: `${path}.clauses`, + message: `${node.op} requires at least one clause`, + }) + for (let i = 0; i < node.clauses.length; i++) + visit(node.clauses[i]!, `${path}.clauses[${i}]`, depth + 1) + break + case "not": + visit(node.clause, `${path}.clause`, depth + 1) + break + case "exists": + break + case "contains": + if (node.field.type !== "string" || node.value.type !== "string") + issues.push({ path, message: "contains requires a string field and string literal" }) + issues.push(...validateSignalLiteral(node.value, `${path}.value`)) + break + case "gt": + case "gte": + case "lt": + case "lte": + if (!ORDERED_TYPES.has(node.field.type)) + issues.push({ path, message: `${node.op} is not supported for ${node.field.type}` }) + if (node.field.type !== node.value.type) + issues.push({ path, message: "field and literal types must match" }) + issues.push(...validateSignalLiteral(node.value, `${path}.value`)) + break + case "eq": + case "neq": + if (node.field.type !== node.value.type) + issues.push({ path, message: "field and literal types must match" }) + issues.push(...validateSignalLiteral(node.value, `${path}.value`)) + break + case "in": + if (node.values.length === 0) + issues.push({ path: `${path}.values`, message: "in requires at least one value" }) + if (node.values.length > MAX_IN_VALUES) + issues.push({ path: `${path}.values`, message: `in exceeds ${MAX_IN_VALUES} values` }) + for (let i = 0; i < node.values.length; i++) { + const value = node.values[i]! + if (value.type !== node.field.type) + issues.push({ + path: `${path}.values[${i}]`, + message: "field and literal types must match", + }) + issues.push(...validateSignalLiteral(value, `${path}.values[${i}]`)) + } + break + } + } + + visit(predicate, "selector", 1) + if (nodes > MAX_PREDICATE_NODES) + issues.push({ path: "selector", message: `predicate exceeds ${MAX_PREDICATE_NODES} nodes` }) + return issues +} + +export const assertValidSignalPredicate = (predicate: SignalPredicate): void => { + const issues = validateSignalPredicate(predicate) + if (issues.length > 0) throw new SignalPredicateValidationError(issues) +} + +export const validateSignalProjectionSpec = ( + projection: SignalProjectionSpec, +): readonly ValidationIssue[] => [ + ...(timestampToEpochNanos(projection.activeFrom) === null + ? [{ path: "activeFrom", message: "must be a valid RFC 3339 instant with an explicit offset" }] + : []), + ...validateSignalPredicate(projection.selector), +] + +export interface PredicateEvaluation { + readonly matches: boolean + readonly typeMismatches: readonly FieldRef[] +} + +const scalarEquals = (left: SignalScalar, right: SignalScalar): boolean => { + if (left.type !== right.type) return false + switch (left.type) { + case "string": + return right.type === "string" && left.value === right.value + case "boolean": + return right.type === "boolean" && left.value === right.value + case "float64": + return right.type === "float64" && left.value === right.value + case "int64": + return right.type === "int64" && BigInt(left.value) === BigInt(right.value) + case "duration": + return right.type === "duration" && BigInt(left.value) === BigInt(right.value) + case "timestamp": + return ( + right.type === "timestamp" && + timestampToEpochNanos(left.value) === timestampToEpochNanos(right.value) + ) + } +} + +const scalarOrder = (left: SignalScalar, right: SignalScalar): number | null => { + if (left.type !== right.type || !ORDERED_TYPES.has(left.type)) return null + switch (left.type) { + case "int64": { + if (right.type !== "int64") return null + const a = BigInt(left.value) + const b = BigInt(right.value) + return a < b ? -1 : a > b ? 1 : 0 + } + case "duration": { + if (right.type !== "duration") return null + const a = BigInt(left.value) + const b = BigInt(right.value) + return a < b ? -1 : a > b ? 1 : 0 + } + case "float64": + return right.type !== "float64" + ? null + : left.value < right.value + ? -1 + : left.value > right.value + ? 1 + : 0 + case "timestamp": { + if (right.type !== "timestamp") return null + const a = timestampToEpochNanos(left.value)! + const b = timestampToEpochNanos(right.value)! + return a < b ? -1 : a > b ? 1 : 0 + } + default: + return null + } +} + +export type CompiledSignalPredicate = (signal: NormalizedSignal) => PredicateEvaluation + +export const compileSignalPredicate = (predicate: SignalPredicate): CompiledSignalPredicate => { + assertValidSignalPredicate(predicate) + + return (signal) => { + const typeMismatches: FieldRef[] = [] + const readField = (field: FieldRef): SignalScalar | undefined => { + const value = signal.fields.get(fieldKey(field)) + if (value === undefined) return undefined + if (value.type !== field.type || validateSignalScalar(value).length > 0) { + typeMismatches.push(field) + return undefined + } + return value + } + + const evaluate = (node: SignalPredicate): boolean => { + switch (node.op) { + case "all": + return node.clauses.every(evaluate) + case "any": + return node.clauses.some(evaluate) + case "not": + return !evaluate(node.clause) + case "exists": { + return readField(node.field) !== undefined + } + case "eq": + case "neq": + case "gt": + case "gte": + case "lt": + case "lte": + case "contains": { + const value = readField(node.field) + if (value === undefined) return false + if (node.op === "eq") return scalarEquals(value, node.value) + if (node.op === "neq") return !scalarEquals(value, node.value) + if (node.op === "contains") + return ( + value.type === "string" && + node.value.type === "string" && + value.value.includes(node.value.value) + ) + const order = scalarOrder(value, node.value) + if (order === null) return false + if (node.op === "gt") return order > 0 + if (node.op === "gte") return order >= 0 + if (node.op === "lt") return order < 0 + return order <= 0 + } + case "in": { + const value = readField(node.field) + if (value === undefined) return false + return node.values.some((candidate) => scalarEquals(value, candidate)) + } + } + } + + return { matches: evaluate(predicate), typeMismatches } + } +} diff --git a/packages/eventing-core/src/registry.test.ts b/packages/eventing-core/src/registry.test.ts new file mode 100644 index 000000000..7fb8421c2 --- /dev/null +++ b/packages/eventing-core/src/registry.test.ts @@ -0,0 +1,370 @@ +import { describe, expect, it } from "vitest" +import { + CompiledProjectionRegistry, + canonicalJson, + defineSignalFields, + isJsonValue, + makeEventId, + MAX_CLOUD_EVENT_BYTES, + ProjectorRegistry, + SignalSourceRegistry, + validateMapleCloudEvent, + type NormalizedSignal, + type JsonValue, + type SignalProjectionSpec, +} from "./index" + +interface CyclicJsonFixture { + self?: CyclicJsonFixture +} + +const decodeJsonOutput = (value: unknown): JsonValue => { + if (!isJsonValue(value)) throw new Error("projector output must be finite JSON") + return value +} + +const signal = (overrides: Partial = {}): NormalizedSignal => ({ + sourceKind: "otel.log", + source: "urn:maple:source:otel:local", + tenantId: "tenant-a", + occurrenceId: "event-123", + identityQuality: "source", + occurredAt: "2026-08-07T19:42:00.123456789Z", + observedAt: "2026-08-07T19:42:01Z", + subject: "records/42", + fields: defineSignalFields([ + { + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "example.record.observed" }, + }, + ]), + data: { record: { id: 42, label: "Example" } }, + ...overrides, +}) + +const projection = (overrides: Partial = {}): SignalProjectionSpec => ({ + id: "example-record-observed", + revision: 3, + enabled: true, + tenantId: "tenant-a", + sourceKind: "otel.log", + selector: { + op: "eq", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "example.record.observed" }, + }, + projector: { id: "example.record", version: 1, config: { includeLabel: true } }, + activeFrom: "2026-08-07T00:00:00Z", + ...overrides, +}) + +const projectors = (): ProjectorRegistry => + new ProjectorRegistry().register({ + id: "example.record", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.example.record.observed.v1", + dataSchema: "urn:maple:event-schema:example-record:v1", + decodeOutput: decodeJsonOutput, + decodeConfig: (value) => { + if (typeof value !== "object" || value === null) throw new Error("invalid projector config") + return value + }, + project: (input) => ({ data: input.data as { record: { id: number; label: string } } }), + }) + +const sources = (): SignalSourceRegistry => + new SignalSourceRegistry().register({ + sourceKind: "otel.log", + fields: [ + { + field: { namespace: "attribute", key: "event.name", type: "string" }, + operators: ["exists", "eq", "neq", "contains", "in"], + sensitivity: "public", + replay: "coerced", + }, + ], + openFields: [ + { + namespace: "attribute", + types: ["string", "boolean", "int64", "float64", "timestamp", "duration"], + operators: ["exists", "eq", "neq", "gt", "gte", "lt", "lte", "contains", "in"], + sensitivity: "public", + replay: "coerced", + }, + ], + }) + +describe("CompiledProjectionRegistry", () => { + const acceptedAt = "2026-08-07T20:00:00Z" + it("canonicalizes JSON independently of object insertion order", () => { + expect(canonicalJson({ z: 1, nested: { b: true, a: [2, 1] }, a: "first" })).toBe( + '{"a":"first","nested":{"a":[2,1],"b":true},"z":1}', + ) + const shared = { value: 1 } + expect(canonicalJson({ left: shared, right: shared })).toBe( + '{"left":{"value":1},"right":{"value":1}}', + ) + const cyclic: CyclicJsonFixture = {} + cyclic.self = cyclic + // SAFETY: this fixture deliberately violates JsonValue to exercise cycle rejection. + expect(() => canonicalJson(cyclic as JsonValue)).toThrow("finite acyclic JSON") + expect(() => canonicalJson({ invalid: Number.NaN })).toThrow("finite acyclic JSON") + }) + + it("projects every match into a deterministic CloudEvent", () => { + const registry = CompiledProjectionRegistry.compile([projection()], sources(), projectors()) + const first = registry.evaluate(signal(), acceptedAt) + const second = registry.evaluate(signal(), acceptedAt) + expect(first.failures).toEqual([]) + expect(first.events).toEqual(second.events) + expect(first.events).toHaveLength(1) + expect(first.events[0]).toMatchObject({ + specversion: "1.0", + id: makeEventId({ + tenantId: "tenant-a", + sourceKind: "otel.log", + source: "urn:maple:source:otel:local", + occurrenceId: "event-123", + projectionId: "example-record-observed", + projectionRevision: 3, + }), + type: "dev.maple.example.record.observed.v1", + subject: "records/42", + projectionrevision: 3, + sourceoccurrenceid: "event-123", + sourceidentityquality: "source", + data: signal().data, + }) + }) + + it("validates historical CloudEvents that predate source identity extensions", () => { + const registry = CompiledProjectionRegistry.compile([projection()], sources(), projectors()) + const event = registry.evaluate(signal(), acceptedAt).events[0]! + const { sourceoccurrenceid: _occurrence, sourceidentityquality: _quality, ...historical } = event + const validated = validateMapleCloudEvent(historical).event + expect(validated.id).toBe(event.id) + expect(validated.sourceoccurrenceid).toBeUndefined() + expect(validated.sourceidentityquality).toBeUndefined() + }) + + it("runs every matching projection from one immutable registry snapshot", () => { + const registry = CompiledProjectionRegistry.compile( + [projection(), projection({ id: "example-record-observed-audit" })], + sources(), + projectors(), + ) + const result = registry.evaluate(signal(), acceptedAt) + expect(result.failures).toEqual([]) + expect(result.events.map(({ projectionid }) => projectionid)).toEqual([ + "example-record-observed", + "example-record-observed-audit", + ]) + }) + + it("runs all matching projections and isolates projector failures", () => { + const registryDefinitions = projectors().register({ + id: "broken", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.broken.v1", + dataSchema: "urn:maple:event-schema:broken:v1", + decodeOutput: decodeJsonOutput, + decodeConfig: () => ({}), + project: () => { + throw new Error("projector invariant failed") + }, + }) + registryDefinitions.register({ + id: "invalid-output", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.invalid-output.v1", + dataSchema: "urn:maple:event-schema:invalid-output:v1", + decodeConfig: () => ({}), + decodeOutput: () => { + throw new Error("projector output violated declared schema") + }, + project: () => ({ data: { invalid: true } }), + }) + const registry = CompiledProjectionRegistry.compile( + [ + projection(), + projection({ id: "broken-projection", projector: { id: "broken", version: 1, config: {} } }), + projection({ + id: "invalid-output-projection", + projector: { id: "invalid-output", version: 1, config: {} }, + }), + ], + sources(), + registryDefinitions, + ) + const result = registry.evaluate(signal(), acceptedAt) + expect(result.events).toHaveLength(1) + expect(result.failures).toEqual([ + expect.objectContaining({ + projectionId: "broken-projection", + message: "projector invariant failed", + }), + expect.objectContaining({ + projectionId: "invalid-output-projection", + message: "projector output violated declared schema", + }), + ]) + }) + + it("isolates complete-envelope schema and size failures from successful siblings", () => { + const registryDefinitions = projectors() + .register({ + id: "oversized", + version: 1, + sourceKinds: ["otel.log"], + outputType: "dev.maple.oversized.v1", + dataSchema: "urn:maple:event-schema:oversized:v1", + decodeOutput: decodeJsonOutput, + decodeConfig: () => ({}), + project: () => ({ data: { payload: "x".repeat(MAX_CLOUD_EVENT_BYTES) } }), + }) + .register({ + id: "invalid-envelope", + version: 1, + sourceKinds: ["otel.log"], + outputType: "x".repeat(257), + dataSchema: "urn:maple:event-schema:invalid-envelope:v1", + decodeOutput: decodeJsonOutput, + decodeConfig: () => ({}), + project: () => ({ data: {} }), + }) + const registry = CompiledProjectionRegistry.compile( + [ + projection(), + projection({ + id: "oversized-projection", + projector: { id: "oversized", version: 1, config: {} }, + }), + projection({ + id: "invalid-envelope-projection", + projector: { id: "invalid-envelope", version: 1, config: {} }, + }), + ], + sources(), + registryDefinitions, + ) + const result = registry.evaluate(signal(), acceptedAt) + expect(result.events.map(({ projectionid }) => projectionid)).toEqual(["example-record-observed"]) + expect(result.failures).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + projectionId: "oversized-projection", + message: expect.stringContaining("CloudEvent exceeds"), + }), + expect.objectContaining({ + projectionId: "invalid-envelope-projection", + }), + ]), + ) + }) + + it("isolates tenants, source kinds, activation time, and disabled revisions", () => { + const registry = CompiledProjectionRegistry.compile( + [ + projection(), + projection({ id: "future", revision: 1, activeFrom: "2026-08-08T00:00:00Z" }), + projection({ id: "disabled", revision: 1, enabled: false }), + projection({ id: "other-tenant", revision: 1, tenantId: "tenant-b" }), + ], + sources(), + projectors(), + ) + expect( + registry + .evaluate(signal({ observedAt: "1999-01-01T00:00:00Z" }), acceptedAt) + .events.map(({ projectionid }) => projectionid), + ).toEqual(["example-record-observed"]) + expect(registry.evaluate(signal({ sourceKind: "otel.span" }), acceptedAt).events).toEqual([]) + expect( + registry + .evaluate(signal({ observedAt: "2099-01-01T00:00:00Z" }), "2026-08-07T00:00:00Z") + .events.map(({ projectionid }) => projectionid), + ).toEqual(["example-record-observed"]) + }) + + it("requires occurrence identity for durable projection", () => { + const registry = CompiledProjectionRegistry.compile([projection()], sources(), projectors()) + const result = registry.evaluate(signal({ occurrenceId: null, identityQuality: "none" }), acceptedAt) + expect(result.events).toEqual([]) + expect(result.failures[0]?.message).toBe( + "durable event projection requires stable or derived occurrence identity", + ) + }) + + it("rejects duplicate registrations, projection revisions, and invalid projector bindings", () => { + const definitions = projectors() + expect(() => + definitions.register({ + id: "example.record", + version: 1, + sourceKinds: ["otel.log"], + outputType: "duplicate", + dataSchema: "duplicate", + decodeOutput: decodeJsonOutput, + decodeConfig: (value) => value, + project: () => ({ data: {} }), + }), + ).toThrow("duplicate projector registration") + expect(() => + CompiledProjectionRegistry.compile([projection(), projection()], sources(), projectors()), + ).toThrow("duplicate projection revision") + expect(() => + CompiledProjectionRegistry.compile( + [projection({ projector: { id: "missing", version: 1, config: {} } })], + sources(), + projectors(), + ), + ).toThrow("unregistered projector") + }) + + it("validates selector fields and operators against the source catalog", () => { + const closed = new SignalSourceRegistry().register({ + sourceKind: "otel.log", + fields: [ + { + field: { namespace: "attribute", key: "event.name", type: "string" }, + operators: ["eq"], + sensitivity: "public", + replay: "coerced", + }, + ], + }) + expect(() => + CompiledProjectionRegistry.compile( + [ + projection({ + selector: { + op: "contains", + field: { namespace: "attribute", key: "event.name", type: "string" }, + value: { type: "string", value: "example" }, + }, + }), + ], + closed, + projectors(), + ), + ).toThrow("contains is not allowed for catalog field") + expect(() => + CompiledProjectionRegistry.compile( + [ + projection({ + selector: { + op: "eq", + field: { namespace: "attribute", key: "unknown", type: "string" }, + value: { type: "string", value: "x" }, + }, + }), + ], + closed, + projectors(), + ), + ).toThrow("unknown field attribute:unknown") + }) +}) diff --git a/packages/eventing-core/src/registry.ts b/packages/eventing-core/src/registry.ts new file mode 100644 index 000000000..f53df875a --- /dev/null +++ b/packages/eventing-core/src/registry.ts @@ -0,0 +1,201 @@ +import { makeCloudEvent } from "./event" +import { Schema } from "effect" +import type { + JsonValue, + MapleCloudEvent, + NormalizedSignal, + ProjectedEventData, + SignalProjectionSpec, +} from "./model" +import { SignalProjectionSpecSchema } from "./model" +import { timestampToEpochNanos, compileSignalPredicate, validateSignalProjectionSpec } from "./predicate" +import { SignalSourceRegistry, validatePredicateAgainstSource } from "./source" + +// BOUNDARY: projector codecs intentionally own decoding of untrusted configuration and output values. +export interface SignalProjector { + readonly id: string + readonly version: number + readonly sourceKinds: readonly string[] + readonly outputType: string + readonly dataSchema: string + readonly decodeConfig: (value: unknown) => TConfig + readonly decodeOutput: (value: unknown) => TData + readonly project: (signal: NormalizedSignal, config: TConfig) => ProjectedEventData +} + +type ErasedSignalProjector = SignalProjector + +export class ProjectorRegistry { + readonly #projectors = new Map() + + register(projector: SignalProjector): this { + if (projector.id.trim().length === 0) throw new Error("projector ID must not be empty") + if (!Number.isSafeInteger(projector.version) || projector.version < 1) + throw new Error("projector version must be a positive safe integer") + if (projector.sourceKinds.length === 0) throw new Error("projector must accept a source kind") + if (projector.outputType.trim().length === 0) + throw new Error("projector output type must not be empty") + if (projector.dataSchema.trim().length === 0) + throw new Error("projector data schema must not be empty") + const key = ProjectorRegistry.key(projector.id, projector.version) + if (this.#projectors.has(key)) throw new Error(`duplicate projector registration: ${key}`) + this.#projectors.set(key, { + id: projector.id, + version: projector.version, + sourceKinds: projector.sourceKinds, + outputType: projector.outputType, + dataSchema: projector.dataSchema, + decodeConfig: projector.decodeConfig, + decodeOutput: projector.decodeOutput, + project: (signal, config) => { + // SAFETY: compiled projections pass only the value returned by this projector's decodeConfig. + return projector.project(signal, config as TConfig) + }, + }) + return this + } + + get(id: string, version: number): ErasedSignalProjector | undefined { + return this.#projectors.get(ProjectorRegistry.key(id, version)) + } + + static key(id: string, version: number): string { + return `${id}@${version}` + } +} + +interface CompiledProjection { + readonly spec: SignalProjectionSpec + readonly evaluate: ReturnType + readonly projector: ErasedSignalProjector + readonly config: unknown + readonly activeFromNanos: bigint +} + +export interface ProjectionFailure { + readonly projectionId: string + readonly projectionRevision: number + readonly occurrenceId: string | null + readonly message: string +} + +export interface ProjectionBatchResult { + readonly events: readonly MapleCloudEvent[] + readonly failures: readonly ProjectionFailure[] + readonly typeMismatchFields: readonly string[] +} + +const validateProjectedData = (value: ProjectedEventData): ProjectedEventData => { + if (value.time !== undefined && timestampToEpochNanos(value.time) === null) + throw new Error("projector returned an invalid event timestamp") + return value +} + +/** Immutable compiled snapshot. Hosts atomically replace the whole instance. */ +export class CompiledProjectionRegistry { + readonly #bySourceKind: ReadonlyMap + + private constructor(bySourceKind: ReadonlyMap) { + this.#bySourceKind = bySourceKind + } + + static compile( + specs: readonly SignalProjectionSpec[], + sources: SignalSourceRegistry, + projectors: ProjectorRegistry, + ): CompiledProjectionRegistry { + const bySourceKind = new Map() + const revisions = new Set() + + for (const candidate of specs) { + const spec = Schema.decodeUnknownSync(SignalProjectionSpecSchema)(candidate) + const source = sources.get(spec.sourceKind) + if (!source) + throw new Error( + `projection ${spec.id}@${spec.revision} references an unregistered source ${spec.sourceKind}`, + ) + const issues = [ + ...validateSignalProjectionSpec(spec), + ...validatePredicateAgainstSource(spec.selector, source), + ] + if (issues.length > 0) + throw new Error( + `invalid projection ${spec.id}@${spec.revision}: ${issues + .map(({ path, message }) => `${path}: ${message}`) + .join("; ")}`, + ) + const revisionKey = `${spec.tenantId}:${spec.id}@${spec.revision}` + if (revisions.has(revisionKey)) throw new Error(`duplicate projection revision: ${revisionKey}`) + revisions.add(revisionKey) + if (!spec.enabled) continue + + const projector = projectors.get(spec.projector.id, spec.projector.version) + if (!projector) + throw new Error( + `projection ${spec.id}@${spec.revision} references an unregistered projector ${spec.projector.id}@${spec.projector.version}`, + ) + if (!projector.sourceKinds.includes(spec.sourceKind)) + throw new Error( + `projector ${projector.id}@${projector.version} does not accept ${spec.sourceKind}`, + ) + + const compiled: CompiledProjection = { + spec, + evaluate: compileSignalPredicate(spec.selector), + projector, + config: projector.decodeConfig(spec.projector.config), + activeFromNanos: timestampToEpochNanos(spec.activeFrom)!, + } + const bucket = bySourceKind.get(spec.sourceKind) + if (bucket) bucket.push(compiled) + else bySourceKind.set(spec.sourceKind, [compiled]) + } + + return new CompiledProjectionRegistry(bySourceKind) + } + + evaluate(signal: NormalizedSignal, acceptedAt: string): ProjectionBatchResult { + const events: MapleCloudEvent[] = [] + const failures: ProjectionFailure[] = [] + const typeMismatchFields = new Set() + const acceptedAtNanos = timestampToEpochNanos(acceptedAt) + if (acceptedAtNanos === null) throw new Error("projection acceptance time must be a valid instant") + + for (const projection of this.#bySourceKind.get(signal.sourceKind) ?? []) { + if (projection.spec.tenantId !== signal.tenantId) continue + if (acceptedAtNanos < projection.activeFromNanos) continue + const evaluation = projection.evaluate(signal) + for (const field of evaluation.typeMismatches) + typeMismatchFields.add(`${field.namespace}:${field.key}`) + if (!evaluation.matches) continue + + try { + const projected = validateProjectedData( + projection.projector.project(signal, projection.config), + ) + events.push( + makeCloudEvent({ + signal, + projection: projection.spec, + projectorId: projection.projector.id, + projectorVersion: projection.projector.version, + outputType: projection.projector.outputType, + dataSchema: projection.projector.dataSchema, + subject: projected.subject, + time: projected.time, + data: projection.projector.decodeOutput(projected.data), + }), + ) + } catch (error) { + failures.push({ + projectionId: projection.spec.id, + projectionRevision: projection.spec.revision, + occurrenceId: signal.occurrenceId, + message: error instanceof Error ? error.message : String(error), + }) + } + } + + return { events, failures, typeMismatchFields: [...typeMismatchFields] } + } +} diff --git a/packages/eventing-core/src/source.ts b/packages/eventing-core/src/source.ts new file mode 100644 index 000000000..cc1a722f7 --- /dev/null +++ b/packages/eventing-core/src/source.ts @@ -0,0 +1,157 @@ +import type { FieldNamespace, FieldRef, NormalizedSignal, SignalPredicate, SignalScalarType } from "./model" +import { fieldKey } from "./model" +import type { ValidationIssue } from "./predicate" + +export type SignalLeafOperator = "exists" | "eq" | "neq" | "gt" | "gte" | "lt" | "lte" | "contains" | "in" +export type ReplayCapability = "exact" | "coerced" | "unavailable" + +interface SignalFieldCatalogEntryBase { + readonly operators: readonly SignalLeafOperator[] + readonly sensitivity: "public" | "sensitive" + readonly replay: ReplayCapability +} + +export type SignalFieldCatalogEntry = SignalFieldCatalogEntryBase & + ( + | { readonly field: FieldRef; readonly types?: never } + | { + readonly field: Pick + readonly types: readonly SignalScalarType[] + } + ) + +export interface OpenFieldNamespacePolicy { + readonly namespace: FieldNamespace + readonly types: readonly SignalScalarType[] + readonly operators: readonly SignalLeafOperator[] + readonly sensitivity: "public" | "sensitive" + readonly replay: ReplayCapability +} + +export interface SignalSourceDefinition { + readonly sourceKind: string + readonly fields: readonly SignalFieldCatalogEntry[] + readonly openFields?: readonly OpenFieldNamespacePolicy[] +} + +export interface SignalSourceAdapter { + readonly definition: SignalSourceDefinition + readonly normalize: (raw: TRaw, context: TContext) => readonly NormalizedSignal[] +} + +interface RegisteredSignalSource { + readonly definition: SignalSourceDefinition + readonly fields: ReadonlyMap + readonly openFields: ReadonlyMap +} + +const catalogEntryTypes = (entry: SignalFieldCatalogEntry): readonly SignalScalarType[] => { + if (entry.types !== undefined) return entry.types + return [entry.field.type] +} + +export class SignalSourceRegistry { + readonly #sources = new Map() + + register(definition: SignalSourceDefinition): this { + if (definition.sourceKind.trim().length === 0) throw new Error("source kind must not be empty") + if (this.#sources.has(definition.sourceKind)) + throw new Error(`duplicate source registration: ${definition.sourceKind}`) + + const fields = new Map() + for (const entry of definition.fields) { + const key = fieldKey(entry.field) + if (fields.has(key)) + throw new Error(`duplicate field catalog entry: ${definition.sourceKind}:${key}`) + if (entry.operators.length === 0) throw new Error(`field catalog entry has no operators: ${key}`) + if (entry.types !== undefined && entry.types.length === 0) + throw new Error(`field catalog entry has no types: ${key}`) + fields.set(key, entry) + } + + const openFields = new Map() + for (const policy of definition.openFields ?? []) { + if (openFields.has(policy.namespace)) + throw new Error(`duplicate open field policy: ${definition.sourceKind}:${policy.namespace}`) + if (policy.types.length === 0 || policy.operators.length === 0) + throw new Error(`open field policy must declare types and operators: ${policy.namespace}`) + openFields.set(policy.namespace, policy) + } + + this.#sources.set(definition.sourceKind, { definition, fields, openFields }) + return this + } + + get(sourceKind: string): RegisteredSignalSource | undefined { + return this.#sources.get(sourceKind) + } +} + +const leafFields = ( + predicate: SignalPredicate, +): ReadonlyArray<{ + readonly field: FieldRef + readonly operator: SignalLeafOperator + readonly path: string +}> => { + const fields: Array<{ field: FieldRef; operator: SignalLeafOperator; path: string }> = [] + const visit = (node: SignalPredicate, path: string): void => { + switch (node.op) { + case "all": + case "any": + for (let i = 0; i < node.clauses.length; i++) visit(node.clauses[i]!, `${path}.clauses[${i}]`) + break + case "not": + visit(node.clause, `${path}.clause`) + break + default: + fields.push({ field: node.field, operator: node.op, path }) + } + } + visit(predicate, "selector") + return fields +} + +export const validatePredicateAgainstSource = ( + predicate: SignalPredicate, + source: RegisteredSignalSource, +): readonly ValidationIssue[] => { + const issues: ValidationIssue[] = [] + for (const leaf of leafFields(predicate)) { + const catalogEntry = source.fields.get(fieldKey(leaf.field)) + if (catalogEntry) { + const catalogTypes = catalogEntryTypes(catalogEntry) + if (!catalogTypes.includes(leaf.field.type)) + issues.push({ + path: `${leaf.path}.field.type`, + message: `catalog field ${fieldKey(leaf.field)} allows ${catalogTypes.join(", ")}`, + }) + if (!catalogEntry.operators.includes(leaf.operator)) + issues.push({ + path: `${leaf.path}.op`, + message: `${leaf.operator} is not allowed for catalog field ${fieldKey(leaf.field)}`, + }) + continue + } + + const open = source.openFields.get(leaf.field.namespace) + if (!open) { + issues.push({ + path: `${leaf.path}.field`, + message: `unknown field ${fieldKey(leaf.field)} for source ${source.definition.sourceKind}`, + }) + continue + } + if (!open.types.includes(leaf.field.type)) + issues.push({ + path: `${leaf.path}.field.type`, + message: `${leaf.field.type} is not allowed for open ${leaf.field.namespace} fields`, + }) + if (!open.operators.includes(leaf.operator)) + issues.push({ + path: `${leaf.path}.op`, + message: `${leaf.operator} is not allowed for open ${leaf.field.namespace} fields`, + }) + } + return issues +} diff --git a/packages/eventing-core/tsconfig.json b/packages/eventing-core/tsconfig.json new file mode 100644 index 000000000..12d9920b4 --- /dev/null +++ b/packages/eventing-core/tsconfig.json @@ -0,0 +1,23 @@ +{ + "include": ["**/*.ts"], + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "lib": ["ES2022", "DOM"], + "types": ["node"], + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "noEmit": true, + "skipLibCheck": true, + "strict": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedSideEffectImports": true, + "plugins": [ + { + "name": "@effect/language-service", + "reportSuggestionsAsWarningsInTsc": true + } + ] + } +}