From 7607b79ccce4f7f3153e23dda5039ec68236088c Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 13 Jun 2026 08:56:59 +0000 Subject: [PATCH] Data: cursor pagination for stable iteration over large collections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add opt-in cursor paging alongside the existing limit/offset. A list response now carries a nextCursor; pass it back as ?after= to continue in stable (sort, id) order — no skips or repeats when rows are inserted mid-iteration, unlike offset. Pairs with declared indexes to stay fast at any size. - The cursor encodes { sortField, dir, value, id } (base64url). The next page adds a row-value predicate `(sortField, id) > (value, id)` (flipped for DESC), reusing the dialect's typed JSON comparison so it's correct on SQLite and Postgres. id is always the unique tiebreaker, and is appended to ORDER BY for a deterministic total order. - A malformed cursor, or one whose sort doesn't match the request, is a clean 400. - list() now returns { documents, nextCursor }; offset paging is unchanged when no cursor is given. SDK: page({ after }) + buildQuery emits after; Query.after and Page.nextCursor typed. Verification: 64/64 SQLite (+ a test that walks a collection by cursor while inserting a row mid-iteration — every original row seen once, no dupes; plus bad-cursor and sort-mismatch 400s), 46/46 Postgres 16 (exercises the jsonb comparison path via a JSON sort field). Docs updated (README, /docs). --- README.md | 12 +++++++++++- public/docs.html | 3 ++- public/sdk.d.ts | 4 ++++ public/sdk.js | 2 ++ src/modules/data/routes.ts | 4 +++- src/modules/data/store.ts | Bin 21997 -> 25319 bytes test/api.test.ts | 35 +++++++++++++++++++++++++++++++++++ 7 files changed, 57 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 88503f2..de434b4 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ The server ships a zero-build client at **`/sdk.js`** — the intended way for a ``` -Filter shorthand: `where: { age: { op: 'gte', value: 18 } }` (ops: `eq ne gt gte lt lte like in nin`); a bare value means equality, an array means `in`. Queries also take `or`, `select`, `limit`, `offset`, `sort`. The collection client adds `createMany`, `count`, `deleteWhere`, `getAcl`/`setAcl`, and optimistic-concurrency updates (`update(id, patch, { ifVersion })`); the top-level client adds `stats()`. Failed calls throw an `Error` with `.status` and `.body`. +Filter shorthand: `where: { age: { op: 'gte', value: 18 } }` (ops: `eq ne gt gte lt lte like in nin`); a bare value means equality, an array means `in`. Queries also take `or`, `select`, `limit`, `offset`, `sort`, and `after` (cursor). The collection client adds `createMany`, `count`, `page` (returns `nextCursor`), `deleteWhere`, `getAcl`/`setAcl`, `getSchema`/`setSchema`/`clearSchema`, `getIndexes`/`setIndexes`, and optimistic-concurrency updates (`update(id, patch, { ifVersion })`); the top-level client adds `stats()`. Failed calls throw an `Error` with `.status` and `.body`. **TypeScript:** types are served at `/sdk.d.ts` — reference them for a fully typed client: @@ -188,6 +188,7 @@ curl -X POST localhost:3737/v1/data/notes/bulk \ # Query: filter + sort + paginate curl 'localhost:3737/v1/data/notes?f.done=false&sort=-createdAt&limit=20' +curl 'localhost:3737/v1/data/notes?sort=-createdAt&limit=20&after=' # stable cursor paging curl 'localhost:3737/v1/data/notes?f.priority=gte:3' # ops: eq ne gt gte lt lte like in nin curl 'localhost:3737/v1/data/notes?f.status=in:active,pending' # value in a set curl 'localhost:3737/v1/data/notes?f.createdAt=gte:1700000000000' # filter on timestamps too @@ -239,6 +240,15 @@ curl -X POST localhost:3737/v1/data/members -H 'content-type: application/json' curl -X DELETE localhost:3737/v1/data/members/schema # back to schemaless ``` +Lists support both **offset** (`?limit&offset`) and **cursor** paging: a list +response includes a `nextCursor`; pass it back as `?after=` to fetch the +next page in stable `(sort, id)` order — no skips or repeats when rows are +inserted mid-iteration, the way offset drifts. With the SDK: +`const page = await api.data('notes').page({ sort: '-createdAt', after });` +then follow `page.nextCursor` until it's `null`. Back a hot sort field with a +[declared index](#data--schemaless-collections) and cursor paging stays fast at +any size. + Via the SDK: `api.data('members').setSchema(schema)` / `getSchema()` / `clearSchema()`. ### Access control — per-collection rules diff --git a/public/docs.html b/public/docs.html index f397661..e8fbfd5 100644 --- a/public/docs.html +++ b/public/docs.html @@ -182,7 +182,7 @@

REST API — Data

POST/v1/data/:cCreate (body = document). POST/v1/data/:c/bulkCreate many (array or { documents: [...] }) in one transaction. - GET/v1/data/:cList/query — ?limit&offset&sort=-field&f.field=op:value. See queries. + GET/v1/data/:cList/query — ?limit&offset&sort=-field&f.field=op:value. Response carries nextCursor; pass it as ?after= for stable cursor paging. See queries. GET/v1/data/:c/countCount matches → { count }. GET/v1/data/:c/aggregate?op=count|sum|avg|min|max&field=… → { value }. GET/v1/data/:c/searchFull-text: ?q=terms&limit (ranked). @@ -316,6 +316,7 @@

Queries & filters

sort=-fieldSort ascending, or -field for descending. select=a,bReturn only these fields (plus id + timestamps). limit / offsetPagination (limit bounded 1–1000). + after=<cursor>Cursor paging: pass a response's nextCursor to continue in stable (sort, id) order — no skips/repeats under concurrent writes.

In the SDK: where: { age: { op: 'gte', value: 18 } }; a bare value means equality, an array means in. id, createdAt, updatedAt are filterable + sortable too.

diff --git a/public/sdk.d.ts b/public/sdk.d.ts index 1fda4c7..4a99b8a 100644 --- a/public/sdk.d.ts +++ b/public/sdk.d.ts @@ -35,6 +35,8 @@ declare namespace Zero { offset?: number; /** Field name, or `-field` for descending. */ sort?: string; + /** A `nextCursor` from a previous page() for stable forward paging (replaces offset). */ + after?: string; /** Equality (bare value), `[a, b]` for `in`, or `{ op, value }` for any operator. */ where?: Record; /** OR group: at least one must match, ANDed with `where` as a whole. */ @@ -57,6 +59,8 @@ declare namespace Zero { collection: string; total: number; documents: Doc[]; + /** Pass to the next `page({ after })` to continue; null when there are no more. */ + nextCursor: string | null; } interface FileMeta { diff --git a/public/sdk.js b/public/sdk.js index 61ba287..359d79b 100644 --- a/public/sdk.js +++ b/public/sdk.js @@ -81,6 +81,8 @@ if (q.limit != null) parts.push('limit=' + encodeURIComponent(q.limit)); if (q.offset != null) parts.push('offset=' + encodeURIComponent(q.offset)); if (q.sort) parts.push('sort=' + encodeURIComponent(q.sort)); + if (q.after) parts.push('after=' + encodeURIComponent(q.after)); // cursor paging + if (q.all) parts.push('all=true'); // only honored by deleteWhere() if (q.select) parts.push('select=' + encodeURIComponent([].concat(q.select).join(','))); // `where` → ANDed `f.` filters; `or` → an ORed `or.` group. diff --git a/src/modules/data/routes.ts b/src/modules/data/routes.ts index a3503a1..181be20 100644 --- a/src/modules/data/routes.ts +++ b/src/modules/data/routes.ts @@ -115,10 +115,11 @@ export async function dataRoutes(app: FastifyInstance): Promise { const query = (req.query ?? {}) as Record; const filters = parseFilters(query, 'f.'); const orFilters = parseFilters(query, 'or.'); - const documents = await store.list(req.projectId, collection, { + const { documents, nextCursor } = await store.list(req.projectId, collection, { limit: typeof query.limit === 'string' ? Number(query.limit) : undefined, offset: typeof query.offset === 'string' ? Number(query.offset) : undefined, sort: typeof query.sort === 'string' ? query.sort : undefined, + after: typeof query.after === 'string' ? query.after : undefined, filters, orFilters, select: parseSelect(query), @@ -128,6 +129,7 @@ export async function dataRoutes(app: FastifyInstance): Promise { collection, total: await store.count(req.projectId, collection, filters, orFilters), documents: hidden.length ? documents.map((d) => redact(d, hidden)) : documents, + nextCursor, }; }); diff --git a/src/modules/data/store.ts b/src/modules/data/store.ts index 5f77b854743228d25c4c9ced6f104cf6641f6c5b..a259d0e05eb1d3a999a2c84221cc9595d5665c92 100644 GIT binary patch delta 3630 zcmbVP!EPKy5ETODU=f5w-~>`dGM*W0Gft#{(3*|e*ol-7;t(eY6j``E)3csrc6!~@ zvyR!UhJRRzKj6d#ksIQK#3%3t95}&&FW^=8%eZ|AUq8I^=Wnn4 z7@w<%HrCf^cO;HWNl~c_r76*BNP+`z}eBu=%AOsaC2jk~lLiA?Aw!1IfMACvj7KMyHt-0x*sK8$9eI#@t{Tftd zQQ;CGXgO@baJcMq-R+LtX+pj^9ZB#=g^5d=dzVLRYvijic8iXy&tO$ASkeY;g!7fV z07bpQyU(ZR2IpU6<(mEd*3K2Aoy*KJxkEUj;l1Z7S<6CYW4Q-$@zux_A8&!FD%E3VgwvF6AP~)|D0YMr+VI zHkwSc^jGERKA>DJ+h&`5vdnZEtE68glRd&Uj%u|nXKyX1NTyQ(1h8^(dpGdmK?$2h z+mV_`^WnZ@Yx{ynSvpKX9&md&x|P>upk}u(ynlWlce#DAdX{ak&ce&D&n~~$sa7X0 z@{d)T`yTc0?vNKSe#<}pV0Y7lPMvJ756(x2@X;XIce`~y+!GU?N_yP9if|}KzEL~% zv{9R(O!`z=Gt`NsbU4aVX@iX-9g0&r#2=;QCLM`9rb7`uL9%LgVi;Lx*n#BCxZ;|rYX`GmKV7>tvFvb( zkA@?}9vEEHunh%4_dd3SzUFPte!sJJjU%(9zZf&f4L|eN-L9AcYwu9xU`k9|GrITA zJ{(PH3q!Y412BBNkA~278WbOfaVj|A;p0N(+iD22xf09d$ayHMjTqomwkd z)-Y#dd_6;|cH_0|!+dyYKFxVS_X8f+INi-8wE16?=%(%GjR{R>-#u7;^DG-@KR@{X z-4~|-?yp%n%yH`Y;rf?zcisXS*^T6ywHNlA|Uw6x;X5(V4I&a$SHH>$f$wvv*c z%%j1@T3na|6pheSo7`o?<);7z;U2C*t_mvE$vKIo#i_Ll{y{FTK?+WhK%tV1%woOb z{Gt-qih?4I{DKk)PqS9R$J5s{M8RIc-_0%9705`KywFi~@&j8|d7vda3ekFcdIgC^ ziMho(3OSj%nI$?35Ld-ccJz>&+-YyTSw$(&PPd>avn;VBRRL}Xh@oJspsrh5kOGuQ zaV$~Sv{EQ8Day=CpX`>XGPx>osZavYvH3Zrxp}ofH>PA3)h0}~PSR3Thw6z>EKzWA i4R%(y<^p-KD7B=tC{F>XY4YAA1wptTkZdhiEf)X>J#L}^ diff --git a/test/api.test.ts b/test/api.test.ts index f021be4..cd31f8e 100644 --- a/test/api.test.ts +++ b/test/api.test.ts @@ -212,6 +212,41 @@ test('data: filtering, operators, and sorting', async () => { assert.deepEqual(inHits.map((d: any) => d.priority), [1, 5]); }); +test('data: cursor pagination walks a collection stably across inserts', async () => { + const api = Zero(base); + const col = api.data('ledger'); + for (let i = 0; i < 5; i++) await col.create({ n: i }); + + // Page by 2, following nextCursor; collect everything. + const seen: number[] = []; + let cursor: string | undefined; + let pages = 0; + do { + const page = await col.page({ sort: 'n', limit: 2, after: cursor }); + seen.push(...page.documents.map((d: any) => d.n)); + cursor = page.nextCursor ?? undefined; + // Insert a *new* row mid-iteration: offset paging would skip/repeat, cursor won't. + if (pages === 0) await col.create({ n: 99 }); + pages += 1; + } while (cursor && pages < 10); + + // The original five are each seen exactly once, in order (the late n:99 sorts last). + assert.deepEqual( + seen.filter((n) => n < 10), + [0, 1, 2, 3, 4], + ); + assert.equal(new Set(seen).size, seen.length, 'no duplicates across pages'); + + // A garbage cursor is a clean 400. + const bad = await fetch(`${base}/v1/data/ledger?after=not-a-cursor`); + assert.equal(bad.status, 400); + + // A cursor taken under one sort can't be reused under a different sort. + const first = await col.page({ sort: 'n', limit: 1 }); + const mismatch = await fetch(`${base}/v1/data/ledger?sort=-n&after=${encodeURIComponent(first.nextCursor)}`); + assert.equal(mismatch.status, 400); +}); + test('data: aggregate computes sum/avg/min/max/count over a field', async () => { const api = Zero(base); await api.data('orders').createMany([