Data: cursor pagination for stable iteration over large collections

README.md

@@ -145,7 +145,7 @@ The server ships a zero-build client at **`/sdk.js`** — the intended way for a
 </script>
 ```
 
-Filter shorthand: `where: { age: { op: 'gte', value: 18 } }` (ops: `eq ne gt gte lt lte like in nin`); a bare value means equality, an array means `in`. Queries also take `or`, `select`, `limit`, `offset`, `sort`. The collection client adds `createMany`, `count`, `deleteWhere`, `getAcl`/`setAcl`, and optimistic-concurrency updates (`update(id, patch, { ifVersion })`); the top-level client adds `stats()`. Failed calls throw an `Error` with `.status` and `.body`.
+Filter shorthand: `where: { age: { op: 'gte', value: 18 } }` (ops: `eq ne gt gte lt lte like in nin`); a bare value means equality, an array means `in`. Queries also take `or`, `select`, `limit`, `offset`, `sort`, and `after` (cursor). The collection client adds `createMany`, `count`, `page` (returns `nextCursor`), `deleteWhere`, `getAcl`/`setAcl`, `getSchema`/`setSchema`/`clearSchema`, `getIndexes`/`setIndexes`, and optimistic-concurrency updates (`update(id, patch, { ifVersion })`); the top-level client adds `stats()`. Failed calls throw an `Error` with `.status` and `.body`.
 
 **TypeScript:** types are served at `/sdk.d.ts` — reference them for a fully typed client:
 
@@ -188,6 +188,7 @@ curl -X POST localhost:3737/v1/data/notes/bulk \
 
 # Query: filter + sort + paginate
 curl 'localhost:3737/v1/data/notes?f.done=false&sort=-createdAt&limit=20'
+curl 'localhost:3737/v1/data/notes?sort=-createdAt&limit=20&after=<nextCursor>' # stable cursor paging
 curl 'localhost:3737/v1/data/notes?f.priority=gte:3'   # ops: eq ne gt gte lt lte like in nin
 curl 'localhost:3737/v1/data/notes?f.status=in:active,pending'  # value in a set
 curl 'localhost:3737/v1/data/notes?f.createdAt=gte:1700000000000' # filter on timestamps too
@@ -239,6 +240,15 @@ curl -X POST localhost:3737/v1/data/members -H 'content-type: application/json'
 curl -X DELETE localhost:3737/v1/data/members/schema   # back to schemaless
 ```
 
+Lists support both **offset** (`?limit&offset`) and **cursor** paging: a list
+response includes a `nextCursor`; pass it back as `?after=<cursor>` to fetch the
+next page in stable `(sort, id)` order — no skips or repeats when rows are
+inserted mid-iteration, the way offset drifts. With the SDK:
+`const page = await api.data('notes').page({ sort: '-createdAt', after });`
+then follow `page.nextCursor` until it's `null`. Back a hot sort field with a
+[declared index](#data--schemaless-collections) and cursor paging stays fast at
+any size.
+
 Via the SDK: `api.data('members').setSchema(schema)` / `getSchema()` / `clearSchema()`.
 
 ### Access control — per-collection rules

public/docs.html

@@ -182,7 +182,7 @@ <h2 id="rest-data">REST API — Data</h2>
             <tbody>
               <tr><td class="ep"><span class="m">POST</span>/v1/data/:c</td><td>Create (body = document).</td></tr>
               <tr><td class="ep"><span class="m">POST</span>/v1/data/:c/bulk</td><td>Create many (array or <code>{ documents: [...] }</code>) in one transaction.</td></tr>
-              <tr><td class="ep"><span class="m">GET</span>/v1/data/:c</td><td>List/query — <code>?limit&amp;offset&amp;sort=-field&amp;f.field=op:value</code>. See <a href="#queries">queries</a>.</td></tr>
+              <tr><td class="ep"><span class="m">GET</span>/v1/data/:c</td><td>List/query — <code>?limit&amp;offset&amp;sort=-field&amp;f.field=op:value</code>. Response carries <code>nextCursor</code>; pass it as <code>?after=</code> for stable cursor paging. See <a href="#queries">queries</a>.</td></tr>
               <tr><td class="ep"><span class="m">GET</span>/v1/data/:c/count</td><td>Count matches → <code>{ count }</code>.</td></tr>
               <tr><td class="ep"><span class="m">GET</span>/v1/data/:c/aggregate</td><td><code>?op=count|sum|avg|min|max&amp;field=…</code> → <code>{ value }</code>.</td></tr>
               <tr><td class="ep"><span class="m">GET</span>/v1/data/:c/search</td><td>Full-text: <code>?q=terms&amp;limit</code> (ranked).</td></tr>
@@ -316,6 +316,7 @@ <h2 id="queries">Queries &amp; filters</h2>
               <tr><td class="ep">sort=-field</td><td>Sort ascending, or <code>-field</code> for descending.</td></tr>
               <tr><td class="ep">select=a,b</td><td>Return only these fields (plus id + timestamps).</td></tr>
               <tr><td class="ep">limit / offset</td><td>Pagination (limit bounded 1–1000).</td></tr>
+              <tr><td class="ep">after=&lt;cursor&gt;</td><td>Cursor paging: pass a response's <code>nextCursor</code> to continue in stable (sort, id) order — no skips/repeats under concurrent writes.</td></tr>
             </tbody>
           </table>
           <p>In the SDK: <code>where: { age: { op: 'gte', value: 18 } }</code>; a bare value means equality, an array means <code>in</code>. <code>id</code>, <code>createdAt</code>, <code>updatedAt</code> are filterable + sortable too.</p>

public/sdk.d.ts

@@ -35,6 +35,8 @@ declare namespace Zero {
     offset?: number;
     /** Field name, or `-field` for descending. */
     sort?: string;
+    /** A `nextCursor` from a previous page() for stable forward paging (replaces offset). */
+    after?: string;
     /** Equality (bare value), `[a, b]` for `in`, or `{ op, value }` for any operator. */
     where?: Record<string, string | number | boolean | (string | number)[] | Filter>;
     /** OR group: at least one must match, ANDed with `where` as a whole. */
@@ -57,6 +59,8 @@ declare namespace Zero {
     collection: string;
     total: number;
     documents: Doc[];
+    /** Pass to the next `page({ after })` to continue; null when there are no more. */
+    nextCursor: string | null;
   }
 
   interface FileMeta {

public/sdk.js

@@ -81,6 +81,8 @@
       if (q.limit != null) parts.push('limit=' + encodeURIComponent(q.limit));
       if (q.offset != null) parts.push('offset=' + encodeURIComponent(q.offset));
       if (q.sort) parts.push('sort=' + encodeURIComponent(q.sort));
+      if (q.after) parts.push('after=' + encodeURIComponent(q.after)); // cursor paging
+
       if (q.all) parts.push('all=true'); // only honored by deleteWhere()
       if (q.select) parts.push('select=' + encodeURIComponent([].concat(q.select).join(',')));
       // `where` → ANDed `f.` filters; `or` → an ORed `or.` group.

src/modules/data/routes.ts

@@ -115,10 +115,11 @@ export async function dataRoutes(app: FastifyInstance): Promise<void> {
     const query = (req.query ?? {}) as Record<string, unknown>;
     const filters = parseFilters(query, 'f.');
     const orFilters = parseFilters(query, 'or.');
-    const documents = await store.list(req.projectId, collection, {
+    const { documents, nextCursor } = await store.list(req.projectId, collection, {
       limit: typeof query.limit === 'string' ? Number(query.limit) : undefined,
       offset: typeof query.offset === 'string' ? Number(query.offset) : undefined,
       sort: typeof query.sort === 'string' ? query.sort : undefined,
+      after: typeof query.after === 'string' ? query.after : undefined,
       filters,
       orFilters,
       select: parseSelect(query),
@@ -128,6 +129,7 @@ export async function dataRoutes(app: FastifyInstance): Promise<void> {
       collection,
       total: await store.count(req.projectId, collection, filters, orFilters),
       documents: hidden.length ? documents.map((d) => redact(d, hidden)) : documents,
+      nextCursor,
     };
   });
 

test/api.test.ts

@@ -212,6 +212,41 @@ test('data: filtering, operators, and sorting', async () => {
   assert.deepEqual(inHits.map((d: any) => d.priority), [1, 5]);
 });
 
+test('data: cursor pagination walks a collection stably across inserts', async () => {
+  const api = Zero(base);
+  const col = api.data('ledger');
+  for (let i = 0; i < 5; i++) await col.create({ n: i });
+
+  // Page by 2, following nextCursor; collect everything.
+  const seen: number[] = [];
+  let cursor: string | undefined;
+  let pages = 0;
+  do {
+    const page = await col.page({ sort: 'n', limit: 2, after: cursor });
+    seen.push(...page.documents.map((d: any) => d.n));
+    cursor = page.nextCursor ?? undefined;
+    // Insert a *new* row mid-iteration: offset paging would skip/repeat, cursor won't.
+    if (pages === 0) await col.create({ n: 99 });
+    pages += 1;
+  } while (cursor && pages < 10);
+
+  // The original five are each seen exactly once, in order (the late n:99 sorts last).
+  assert.deepEqual(
+    seen.filter((n) => n < 10),
+    [0, 1, 2, 3, 4],
+  );
+  assert.equal(new Set(seen).size, seen.length, 'no duplicates across pages');
+
+  // A garbage cursor is a clean 400.
+  const bad = await fetch(`${base}/v1/data/ledger?after=not-a-cursor`);
+  assert.equal(bad.status, 400);
+
+  // A cursor taken under one sort can't be reused under a different sort.
+  const first = await col.page({ sort: 'n', limit: 1 });
+  const mismatch = await fetch(`${base}/v1/data/ledger?sort=-n&after=${encodeURIComponent(first.nextCursor)}`);
+  assert.equal(mismatch.status, 400);
+});
+
 test('data: aggregate computes sum/avg/min/max/count over a field', async () => {
   const api = Zero(base);
   await api.data('orders').createMany([