perf(rpc): fix POST /links/create 3–6s latency during Redis instability

.cursor/rules/performance.mdc

@@ -38,6 +38,8 @@ When you discover a new performance improvement, optimization pattern, or fix a
 - Skip expensive middleware for routes that don't need it — `applyAuthWideEvent` does a session DB lookup on every request; skip it for anonymous `/public/` routes via URL check in `onBeforeHandle`
 - Use `@elysiajs/server-timing` to profile per-phase durations (CORS, beforeHandle, handle, afterHandle) without manual instrumentation
 - Elysia's JIT compiler is on by default (`precompile: true`) — no extra config needed
+- Redis clients fail fast after a failure instead of paying the timeout per call: `cacheable` skips Redis for 30s after a timeout, and `runLinkCacheCommand` rejects immediately for 5s after a failed operation. Sequential deadline penalties (1.5–2s per call) must never compound within one request
+- `runAuditedMutation` audit writes are fire-and-forget after success; `appendAuditEvent` has an outbox fallback so blocking the response on the INSERT is unnecessary latency
 
 ## Database queries
 

packages/redis/000-redis.test.ts

@@ -1,4 +1,4 @@
-import { afterAll, describe, expect, it } from "bun:test";
+import { afterAll, describe, expect, it, mock } from "bun:test";
 import {
 	createLinkCacheRedisConnectionOptions,
 	createRateLimitRedisConnectionOptions,
@@ -8,7 +8,9 @@ import {
 
 process.env.REDIS_URL = "redis://test-host:6379";
 
-const { getRedisCache, shutdownRedis } = await import("./redis");
+const { getRedisCache, runLinkCacheCommand, shutdownRedis } = await import(
+	"./redis"
+);
 
 describe("redis", () => {
 	describe("connection options", () => {
@@ -106,4 +108,33 @@ describe("redis", () => {
 			second.disconnect();
 		});
 	});
+
+	describe("link cache fail-fast", () => {
+		afterAll(async () => {
+			await shutdownRedis();
+		});
+
+		it("rejects immediately after a recent failure without running the operation", async () => {
+			await expect(
+				runLinkCacheCommand(async () => "unreachable")
+			).rejects.toThrow();
+
+			const operation = mock(async () => "value");
+			const startedAt = performance.now();
+			await expect(runLinkCacheCommand(operation)).rejects.toThrow(
+				"failing fast"
+			);
+			expect(performance.now() - startedAt).toBeLessThan(100);
+			expect(operation).not.toHaveBeenCalled();
+		});
+
+		it("probes again after shutdown resets the fail-fast window", async () => {
+			await shutdownRedis();
+			const error = await runLinkCacheCommand(async () => "value").catch(
+				(caught: Error) => caught
+			);
+			expect(error).toBeInstanceOf(Error);
+			expect(error.message).not.toContain("failing fast");
+		});
+	});
 });

packages/redis/redis.ts

@@ -15,9 +15,12 @@ let shutdownHooksRegistered = false;
 
 const LINK_CACHE_CONNECT_DEADLINE_MS = 1250;
 export const LINK_CACHE_OPERATION_DEADLINE_MS = 1500;
+const LINK_CACHE_FAIL_FAST_WINDOW_MS = 5000;
 const RATE_LIMIT_CONNECT_DEADLINE_MS = 1250;
 export const RATE_LIMIT_OPERATION_DEADLINE_MS = 1500;
 
+let linkCacheFailFastUntil = 0;
+
 function withDeadline<T>(
 	operation: Promise<T>,
 	timeoutMs: number,
@@ -153,19 +156,26 @@ export function runRateLimitCommand<T>(
 async function runLinkCacheRedisCommand<T>(
 	operation: (redis: Redis) => Promise<T>
 ): Promise<T> {
+	if (Date.now() < linkCacheFailFastUntil) {
+		throw new Error("Link cache is failing fast after a recent Redis failure");
+	}
+
 	let instance: Redis | null = null;
 	const command = getLinkCacheRedis().then((redis) => {
 		instance = redis;
 		return operation(redis);
 	});
 
 	try {
-		return await withDeadline(
+		const result = await withDeadline(
 			command,
 			LINK_CACHE_OPERATION_DEADLINE_MS,
 			`Link cache operation exceeded ${LINK_CACHE_OPERATION_DEADLINE_MS}ms`
 		);
+		linkCacheFailFastUntil = 0;
+		return result;
 	} catch (error) {
+		linkCacheFailFastUntil = Date.now() + LINK_CACHE_FAIL_FAST_WINDOW_MS;
 		if (instance) {
 			discardLinkCacheRedis(instance);
 		}
@@ -197,6 +207,7 @@ async function runRateLimitRedisCommand<T>(
 }
 
 export async function shutdownRedis() {
+	linkCacheFailFastUntil = 0;
 	const linkCacheInstance = linkCacheRedisInstance;
 	linkCacheRedisInstance = null;
 	linkCacheConnectPromise = null;

packages/rpc/src/middleware/audit-mutation.ts

@@ -123,10 +123,10 @@ export async function runAuditedMutation<T>(
 
 	try {
 		const result = await fn();
-		await writeMutationAudit(context, path, "success");
+		writeMutationAudit(context, path, "success");
 		return result;
 	} catch (error) {
-		await writeMutationAudit(context, path, getAuditOutcome(error), error);
+		writeMutationAudit(context, path, getAuditOutcome(error), error);
 		throw error;
 	}
 }

packages/rpc/src/routers/links.ts

@@ -19,7 +19,6 @@ import {
 	type CachedLinkMutationNext,
 	finishCachedLinkMutation,
 	invalidateAgentContextSnapshotsForOwner,
-	setCachedLinkIfAbsent,
 } from "@databuddy/redis";
 import { isDeepLinkTarget } from "@databuddy/shared/constants/deep-link-apps";
 import { randomUUIDv7 } from "bun";
@@ -281,27 +280,6 @@ async function finishLinkCacheMutation(
 	return false;
 }
 
-async function backfillLinkCache(
-	slug: string,
-	link: CacheableLink,
-	reason: string
-): Promise<void> {
-	try {
-		if (await setCachedLinkIfAbsent(slug, toCachedLink(link))) {
-			return;
-		}
-		logger.warn(
-			{ linkId: link.id, slug, reason },
-			"Link cache backfill did not replace an existing entry"
-		);
-	} catch (error) {
-		logger.error(
-			{ linkId: link.id, slug, reason, ...getErrorLogFields(error) },
-			"Failed to backfill link cache"
-		);
-	}
-}
-
 async function tombstoneLinkCacheMutations(
 	mutations: LinkCacheMutation[],
 	reason: string
@@ -669,12 +647,6 @@ export const linksRouter = {
 							{ link: toCachedLink(newLink), state: "link" },
 							"create persisted"
 						);
-					} else {
-						await backfillLinkCache(
-							slug,
-							newLink,
-							"create bypassed cache lease"
-						);
 					}
 					invalidateLinkAgentContext(organizationId);
 
@@ -734,12 +706,6 @@ export const linksRouter = {
 								{ link: toCachedLink(persistedLink), state: "link" },
 								"create reconciled after ambiguous database error"
 							);
-						} else {
-							await backfillLinkCache(
-								slug,
-								persistedLink,
-								"create reconciled after cache bypass"
-							);
 						}
 						invalidateLinkAgentContext(organizationId);
 						return persistedLink;