Observability: Prometheus /metrics + a /ready probe

README.md

@@ -38,7 +38,8 @@ That's it — no clone, no config, no database to provision. Open
 
 Plus a **zero-build browser SDK** (`/sdk.js`, typed), a **self-describing API**
 (`/v1` + `/llms.txt`) an AI can consume in one read, an **admin dashboard**
-(`/admin`), an ops endpoint (`/v1/stats`), per-IP rate limiting, and one-command
+(`/admin`), ops endpoints (`/v1/stats`, Prometheus `/metrics`, a `/ready` probe),
+per-IP rate limiting, and one-command
 deploys (npx · Docker Compose · systemd). TypeScript + Fastify, single process,
 SQLite + local disk by default — Postgres and S3 optional. CI, MIT.
 

package.json

@@ -63,7 +63,8 @@
     "better-sqlite3": "^12.10.0",
     "fastify": "^5.8.5",
     "ioredis": "^5.11.1",
-    "pg": "^8.21.0"
+    "pg": "^8.21.0",
+    "prom-client": "^15.1.3"
   },
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.13",

public/docs.html

@@ -300,7 +300,9 @@ <h2 id="rest-admin">REST API — Admin &amp; backup</h2>
           <h2 id="rest-meta">Meta &amp; health</h2>
           <table>
             <tbody>
-              <tr><td class="ep"><span class="m">GET</span>/health</td><td><code>{ ok, uptime }</code> — always public.</td></tr>
+              <tr><td class="ep"><span class="m">GET</span>/health</td><td><code>{ ok, uptime }</code> — liveness, always public.</td></tr>
+              <tr><td class="ep"><span class="m">GET</span>/ready</td><td>Readiness probe: <code>200 { ready: true }</code> when the DB is reachable, else <code>503</code>.</td></tr>
+              <tr><td class="ep"><span class="m">GET</span>/metrics</td><td>Prometheus exposition — request rate/latency, memory, realtime connections. No per-tenant labels; restrict at the proxy if needed.</td></tr>
               <tr><td class="ep"><span class="m">GET</span>/v1</td><td>Capabilities + endpoint map + mode + AI provider.</td></tr>
               <tr><td class="ep"><span class="m">GET</span>/v1/stats</td><td>Version, uptime, usage counts, AI provider readiness.</td></tr>
               <tr><td class="ep"><span class="m">GET</span>/sdk.js · /sdk.d.ts · /llms.txt</td><td>The browser client, its types, and the AI-readable guide.</td></tr>

src/metrics.ts

@@ -0,0 +1,72 @@
+import { Registry, Counter, Histogram, Gauge } from 'prom-client';
+import type { FastifyInstance } from 'fastify';
+
+// Paths that are themselves infra (probes/scrape) — excluded from request metrics
+// so they don't drown out real traffic.
+const SKIP = new Set(['/metrics', '/health', '/ready']);
+
+/**
+ * Operational endpoints for running the server under an orchestrator / scrape:
+ *   GET /metrics  — Prometheus exposition (request rate, latency, memory, realtime)
+ *   GET /ready    — readiness probe; 200 when the database is reachable, else 503
+ *
+ * Both are public (like /health) and carry no per-tenant labels, so nothing
+ * sensitive leaks — restrict at the proxy/network if you want them private.
+ */
+export function registerObservability(app: FastifyInstance): void {
+  // A per-instance registry (not the global default) so multiple servers in one
+  // process — e.g. the test suite — don't collide on metric registration.
+  const registry = new Registry();
+
+  const gauge = (name: string, help: string, collect: () => number): void => {
+    new Gauge({ name, help, registers: [registry], collect() {
+      this.set(collect());
+    } });
+  };
+  gauge('zero_uptime_seconds', 'Process uptime in seconds.', () => process.uptime());
+  gauge('zero_resident_memory_bytes', 'Resident set size in bytes.', () => process.memoryUsage().rss);
+  gauge('zero_heap_used_bytes', 'V8 heap used in bytes.', () => process.memoryUsage().heapUsed);
+  gauge('zero_realtime_connections', 'Open realtime websocket connections (this process).', () =>
+    app.realtime.channels().reduce((n, c) => n + c.clients, 0),
+  );
+  gauge('zero_realtime_channels', 'Active realtime channels (this process).', () => app.realtime.channels().length);
+
+  const requests = new Counter({
+    name: 'zero_http_requests_total',
+    help: 'HTTP requests handled, by method, route, and status.',
+    labelNames: ['method', 'route', 'status'] as const,
+    registers: [registry],
+  });
+  const duration = new Histogram({
+    name: 'zero_http_request_duration_seconds',
+    help: 'HTTP request duration in seconds, by method and route.',
+    labelNames: ['method', 'route'] as const,
+    buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
+    registers: [registry],
+  });
+
+  app.addHook('onResponse', async (req, reply) => {
+    if (SKIP.has(req.url.split('?')[0] ?? '')) return;
+    // routeOptions.url is the route *pattern* (e.g. /v1/data/:collection/:id), so
+    // labels stay low-cardinality — ids never become distinct label values.
+    const route = req.routeOptions?.url ?? 'unmatched';
+    const method = req.method;
+    requests.inc({ method, route, status: reply.statusCode });
+    duration.observe({ method, route }, reply.elapsedTime / 1000);
+  });
+
+  app.get('/metrics', async (_req, reply) => {
+    reply.header('content-type', registry.contentType);
+    return registry.metrics();
+  });
+
+  app.get('/ready', async (_req, reply) => {
+    try {
+      await app.db.get('SELECT 1 AS ok');
+      return { ready: true };
+    } catch (err) {
+      app.log.error({ err }, 'readiness check failed');
+      return reply.code(503).send({ ready: false, error: 'database_unreachable' });
+    }
+  });
+}

src/server.ts

@@ -18,6 +18,7 @@ import { getRealtimeTransport } from './modules/realtime/transport.js';
 import { metaRoutes } from './modules/meta/routes.js';
 import { backupRoutes } from './modules/backup/routes.js';
 import { registerRateLimit, type RateLimitOptions } from './rate-limit.js';
+import { registerObservability } from './metrics.js';
 import { AclStore } from './modules/acl/store.js';
 import { SchemaStore, SchemaValidationError } from './modules/data/schema.js';
 import { getBlobStore } from './modules/files/blob.js';
@@ -255,6 +256,9 @@ export async function buildServer(opts: BuildOptions = {}): Promise<FastifyInsta
 
   app.get('/health', async () => ({ ok: true, uptime: process.uptime() }));
 
+  // /metrics (Prometheus) + /ready (readiness probe) for scaled deployments.
+  registerObservability(app);
+
   await app.register(metaRoutes);
   await app.register(adminRoutes);
   await app.register(backupRoutes);

test/api.test.ts

@@ -32,6 +32,24 @@ test('health check', async () => {
   assert.equal((await json(res)).ok, true);
 });
 
+test('observability: /ready probes the DB and /metrics exposes Prometheus metrics', async () => {
+  const ready = await fetch(`${base}/ready`);
+  assert.equal(ready.status, 200);
+  assert.equal((await json(ready)).ready, true);
+
+  // Make a request that should be counted, then scrape.
+  await fetch(`${base}/v1`);
+  const res = await fetch(`${base}/metrics`);
+  assert.equal(res.status, 200);
+  assert.match(res.headers.get('content-type') ?? '', /text\/plain/);
+  const body = await res.text();
+  assert.match(body, /zero_uptime_seconds/);
+  assert.match(body, /zero_http_requests_total/);
+  assert.match(body, /zero_http_request_duration_seconds/);
+  // The earlier GET /v1 was recorded with its route pattern (not a raw path).
+  assert.match(body, /zero_http_requests_total\{[^}]*route="\/v1"[^}]*\}/);
+});
+
 test('observability: responses carry x-request-id, honoring an inbound one', async () => {
   const echoed = await fetch(`${base}/health`, { headers: { 'x-request-id': 'trace-abc-123' } });
   assert.equal(echoed.headers.get('x-request-id'), 'trace-abc-123');