fix(run-store,webapp): correct split-database read routing, write residency, and batches list ordering (#4272)

## Summary

Correctness and performance fixes for deployments that split run data
across more than one database. Single-database / self-hosted deployments
are unaffected (they collapse to a single read/write path).

- **Batches list (dashboard):** for some organizations the Batches list
could hide older batches or show them out of order. It now orders and
paginates by creation time (with the id as a stable tiebreak), so every
batch appears exactly once, newest first. The pagination cursor format
changes; older in-flight cursors simply restart from the first page.
- **Reads:** waitpoint and snapshot lookups that are keyed by a single
run now read only the database that holds that run instead of querying
both, removing redundant queries on hot paths (unblock, snapshot reads).
- **Writes:** environment-scoped writes with no owning run (standalone
wait tokens, waitpoint tags, idempotency-key resets) now land in the
same database as that environment's runs, rather than defaulting to the
other one. An idempotency-key reset also falls back to the other
database when it matches nothing, so a reset still clears the key
wherever the run actually lives.

## Notes

Verified end-to-end against multi-database setups: run-keyed reads and
env-scoped writes land on the correct database with no cross-database
writes, and the batches list surfaces every batch in creation order. New
tests cover the batches ordering/reachability and the write-residency
routing.

Author: d-cs

apps/webapp/app/models/waitpointTag.server.ts

@@ -8,22 +8,30 @@ export async function createWaitpointTag({
   tag,
   environmentId,
   projectId,
+  residency,
 }: {
   tag: string;
   environmentId: string;
   projectId: string;
+  // Residency from the env mint kind: a tag has no owning run, so a minted-new env pins it to NEW
+  // instead of defaulting to the draining legacy DB.
+  residency?: "NEW" | "LEGACY";
 }) {
   if (tag.trim().length === 0) return;
 
   let attempts = 0;
 
   while (attempts < MAX_RETRIES) {
     try {
-      return await runStore.upsertWaitpointTag({
-        environmentId,
-        name: tag,
-        projectId,
-      });
+      return await runStore.upsertWaitpointTag(
+        {
+          environmentId,
+          name: tag,
+          projectId,
+        },
+        undefined,
+        residency
+      );
     } catch (error) {
       if (error instanceof Prisma.PrismaClientKnownRequestError && error.code === "P2002") {
         // Handle unique constraint violation (conflict)

apps/webapp/app/presenters/v3/BatchListPresenter.server.ts

@@ -43,6 +43,28 @@ type BatchRow = {
   batchVersion: string;
 };
 
+// Composite keyset cursor "<createdAt-epoch-ms>_<id>". Ordering is by createdAt then id: a batch id is
+// a cuid (legacy) OR a run-ops id (new), and the two schemes occupy different lexical ranges, so `id`
+// alone is not a valid chronological order across the residency split. `id` is the stable tiebreak.
+// Old plain-id cursors (no "_") decode to undefined and restart from page 1 (self-healing).
+type BatchCursor = { createdAt: Date; id: string };
+function encodeBatchCursor(row: BatchCursor): string {
+  return `${row.createdAt.getTime()}_${row.id}`;
+}
+function decodeBatchCursor(cursor: string | undefined): BatchCursor | undefined {
+  if (!cursor) return undefined;
+  const sep = cursor.indexOf("_");
+  if (sep === -1) return undefined;
+  const ms = Number(cursor.slice(0, sep));
+  const id = cursor.slice(sep + 1);
+  // Number.isFinite accepts e.g. 1e20, but new Date(1e20) is Invalid Date — reject it so a malformed
+  // URL cursor self-heals to page 1 instead of reaching Prisma with an invalid date.
+  const createdAt = new Date(ms);
+  if (!Number.isFinite(ms) || Number.isNaN(createdAt.getTime()) || id.length === 0)
+    return undefined;
+  return { createdAt, id };
+}
+
 export class BatchListPresenter extends BasePresenter {
   // Optional run-ops read-routing. Omitted (single-DB / self-host) => everything
   // reads from `_replica` exactly as today (passthrough). Field names are local to
@@ -86,17 +108,16 @@ export class BatchListPresenter extends BasePresenter {
       return scan(passthrough);
     }
 
-    const newRows = await scan(this.readRoute.runOpsNew ?? passthrough);
-
-    // New DB filled the page — skip the legacy read entirely; older rows fall on a later page.
-    if (newRows.length >= pageSize + 1) {
-      return newRows;
-    }
-
-    const legacyRows = await scan(this.readRoute.runOpsLegacyReplica ?? passthrough);
+    // Always read BOTH stores and merge. The old "skip legacy when new fills the page" shortcut is
+    // unsound across the residency split: legacy cuid ids ("c…") sort ABOVE new run-ops ids ("0…")
+    // under id order, so a new-only page can hide pre-flip legacy batches that belong ahead of it.
+    // Ordering is by createdAt (id tiebreak), which is chronologically correct across both schemes.
+    const [newRows, legacyRows] = await Promise.all([
+      scan(this.readRoute.runOpsNew ?? passthrough),
+      scan(this.readRoute.runOpsLegacyReplica ?? passthrough),
+    ]);
 
-    // De-dupe by id (new wins), re-sort under the page's keyset order, re-apply the over-fetch
-    // LIMIT — reproduces the pageSize+1 window a single union scan would return.
+    // De-dupe by id (new wins), re-sort under the page's keyset order, re-apply the over-fetch LIMIT.
     const byId = new Map<string, BatchRow>();
     for (const row of newRows) {
       byId.set(row.id, row);
@@ -107,10 +128,16 @@ export class BatchListPresenter extends BasePresenter {
       }
     }
 
-    // codepoint comparator (NEVER localeCompare): BatchTaskRun.id is ASCII (cuid or run-ops id).
-    const sign = direction === "forward" ? 1 : -1; // forward => DESC; backward => ASC
+    // forward => newest-first (createdAt DESC), backward => oldest-first (ASC); id is the stable
+    // tiebreak (ASCII codepoint, NEVER localeCompare).
+    const sign = direction === "forward" ? 1 : -1;
     return Array.from(byId.values())
-      .sort((a, b) => (a.id < b.id ? sign : a.id > b.id ? -sign : 0))
+      .sort((a, b) => {
+        const at = a.createdAt.getTime();
+        const bt = b.createdAt.getTime();
+        if (at !== bt) return at < bt ? sign : -sign;
+        return a.id < b.id ? sign : a.id > b.id ? -sign : 0;
+      })
       .slice(0, pageSize + 1);
   }
 
@@ -212,11 +239,28 @@ export class BatchListPresenter extends BasePresenter {
     }
     const createdAtLte: Date | undefined = time.to;
 
+    // Composite (createdAt, id) keyset — see encodeBatchCursor. An old plain-id cursor decodes to
+    // undefined and restarts from page 1.
+    const keyCursor = decodeBatchCursor(cursor);
+
     const batches = await this.#scanBatchTaskRun(pageSize, direction, (client) =>
       client.batchTaskRun.findMany({
         where: {
           runtimeEnvironmentId: environmentId,
-          ...(cursor ? { id: direction === "forward" ? { lt: cursor } : { gt: cursor } } : {}),
+          ...(keyCursor
+            ? {
+                OR:
+                  direction === "forward"
+                    ? [
+                        { createdAt: { lt: keyCursor.createdAt } },
+                        { createdAt: keyCursor.createdAt, id: { lt: keyCursor.id } },
+                      ]
+                    : [
+                        { createdAt: { gt: keyCursor.createdAt } },
+                        { createdAt: keyCursor.createdAt, id: { gt: keyCursor.id } },
+                      ],
+              }
+            : {}),
           ...(friendlyId ? { friendlyId } : {}),
           ...(statuses && statuses.length > 0
             ? { status: { in: statuses }, batchVersion: { not: "v1" } }
@@ -230,7 +274,10 @@ export class BatchListPresenter extends BasePresenter {
               }
             : {}),
         },
-        orderBy: { id: direction === "forward" ? "desc" : "asc" },
+        orderBy: [
+          { createdAt: direction === "forward" ? "desc" : "asc" },
+          { id: direction === "forward" ? "desc" : "asc" },
+        ],
         take: pageSize + 1,
         select: {
           id: true,
@@ -248,23 +295,24 @@ export class BatchListPresenter extends BasePresenter {
 
     const hasMore = batches.length > pageSize;
 
-    //get cursors for next and previous pages
+    //get cursors for next and previous pages (composite (createdAt, id) keyset)
+    const cur = (row?: BatchRow) => (row ? encodeBatchCursor(row) : undefined);
     let next: string | undefined;
     let previous: string | undefined;
     switch (direction) {
       case "forward":
-        previous = cursor ? batches.at(0)?.id : undefined;
+        previous = cursor ? cur(batches.at(0)) : undefined;
         if (hasMore) {
-          next = batches[pageSize - 1]?.id;
+          next = cur(batches[pageSize - 1]);
         }
         break;
       case "backward":
         batches.reverse();
         if (hasMore) {
-          previous = batches[1]?.id;
-          next = batches[pageSize]?.id;
+          previous = cur(batches[1]);
+          next = cur(batches[pageSize]);
         } else {
-          next = batches[pageSize - 1]?.id;
+          next = cur(batches[pageSize - 1]);
         }
         break;
     }

apps/webapp/app/routes/api.v1.waitpoints.tokens.ts

@@ -16,6 +16,7 @@ import {
   type PrismaClientOrTransaction,
 } from "~/db.server";
 import { type AuthenticatedEnvironment } from "~/services/apiAuth.server";
+import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
 import { logger } from "~/services/logger.server";
 import { generateHttpCallbackUrl } from "~/services/httpCallback.server";
 import {
@@ -58,6 +59,16 @@ const { action } = createActionApiRoute(
 
       const timeout = await parseDelay(body.timeout);
 
+      // A token (and its tags) has no owning run, so it can't co-locate. Resolve the env mint kind so a
+      // minted-new env creates them on the run-ops DB (NEW) instead of defaulting to the draining LEGACY
+      // DB by their cuid id-shape.
+      const mintKind = await resolveRunIdMintKind({
+        organizationId: authentication.environment.organizationId,
+        id: authentication.environment.id,
+        orgFeatureFlags: authentication.environment.organization.featureFlags,
+      });
+      const residency = mintKind === "runOpsId" ? "NEW" : "LEGACY";
+
       //upsert tags
       let tags: { id: string; name: string }[] = [];
       const bodyTags = typeof body.tags === "string" ? [body.tags] : body.tags;
@@ -74,6 +85,7 @@ const { action } = createActionApiRoute(
             tag,
             environmentId: authentication.environment.id,
             projectId: authentication.environment.projectId,
+            residency,
           });
           if (tagRecord) {
             tags.push(tagRecord);
@@ -88,6 +100,7 @@ const { action } = createActionApiRoute(
         idempotencyKeyExpiresAt,
         timeout,
         tags: bodyTags,
+        standaloneResidency: residency,
       });
 
       const $responseHeaders = await responseHeaders(authentication.environment);

apps/webapp/app/v3/services/resetIdempotencyKey.server.ts

@@ -2,19 +2,35 @@ import type { AuthenticatedEnvironment } from "~/services/apiAuth.server";
 import { BaseService, ServiceValidationError } from "./baseService.server";
 import { logger } from "~/services/logger.server";
 import { getMollifierBuffer } from "~/v3/mollifier/mollifierBuffer.server";
+import { resolveRunIdMintKind } from "~/v3/engineVersion.server";
 
 export class ResetIdempotencyKeyService extends BaseService {
   public async call(
     idempotencyKey: string,
     taskIdentifier: string,
     authenticatedEnv: AuthenticatedEnvironment
   ): Promise<{ id: string }> {
+    // The predicate has no run id to route by. When the env mints run-ops ids its runs live on NEW,
+    // so pin the reset to NEW and skip the wrong-DB (0-row) write to the draining legacy DB. Resolve
+    // this only when the org (and its flags) is loaded on the env — which the authenticated API path
+    // always provides; otherwise fall back to the two-store reset (correct, just not optimized).
+    let residency: "NEW" | "LEGACY" = "LEGACY";
+    if (authenticatedEnv.organization) {
+      const mintKind = await resolveRunIdMintKind({
+        organizationId: authenticatedEnv.organizationId,
+        id: authenticatedEnv.id,
+        orgFeatureFlags: authenticatedEnv.organization.featureFlags,
+      });
+      residency = mintKind === "runOpsId" ? "NEW" : "LEGACY";
+    }
+
     const { count: pgCount } = await this.runStore.clearIdempotencyKey(
       {
         byPredicate: {
           idempotencyKey,
           taskIdentifier,
           runtimeEnvironmentId: authenticatedEnv.id,
+          residency,
         },
       },
       this._prisma
@@ -80,6 +96,7 @@ export class ResetIdempotencyKeyService extends BaseService {
             idempotencyKey,
             taskIdentifier,
             runtimeEnvironmentId: authenticatedEnv.id,
+            residency,
           },
         },
         this._prisma