diff --git a/README.md b/README.md index 88503f2..de434b4 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ The server ships a zero-build client at **`/sdk.js`** — the intended way for a ``` -Filter shorthand: `where: { age: { op: 'gte', value: 18 } }` (ops: `eq ne gt gte lt lte like in nin`); a bare value means equality, an array means `in`. Queries also take `or`, `select`, `limit`, `offset`, `sort`. The collection client adds `createMany`, `count`, `deleteWhere`, `getAcl`/`setAcl`, and optimistic-concurrency updates (`update(id, patch, { ifVersion })`); the top-level client adds `stats()`. Failed calls throw an `Error` with `.status` and `.body`. +Filter shorthand: `where: { age: { op: 'gte', value: 18 } }` (ops: `eq ne gt gte lt lte like in nin`); a bare value means equality, an array means `in`. Queries also take `or`, `select`, `limit`, `offset`, `sort`, and `after` (cursor). The collection client adds `createMany`, `count`, `page` (returns `nextCursor`), `deleteWhere`, `getAcl`/`setAcl`, `getSchema`/`setSchema`/`clearSchema`, `getIndexes`/`setIndexes`, and optimistic-concurrency updates (`update(id, patch, { ifVersion })`); the top-level client adds `stats()`. Failed calls throw an `Error` with `.status` and `.body`. **TypeScript:** types are served at `/sdk.d.ts` — reference them for a fully typed client: @@ -188,6 +188,7 @@ curl -X POST localhost:3737/v1/data/notes/bulk \ # Query: filter + sort + paginate curl 'localhost:3737/v1/data/notes?f.done=false&sort=-createdAt&limit=20' +curl 'localhost:3737/v1/data/notes?sort=-createdAt&limit=20&after=' # stable cursor paging curl 'localhost:3737/v1/data/notes?f.priority=gte:3' # ops: eq ne gt gte lt lte like in nin curl 'localhost:3737/v1/data/notes?f.status=in:active,pending' # value in a set curl 'localhost:3737/v1/data/notes?f.createdAt=gte:1700000000000' # filter on timestamps too @@ -239,6 +240,15 @@ curl -X POST localhost:3737/v1/data/members -H 'content-type: application/json' curl -X DELETE localhost:3737/v1/data/members/schema # back to schemaless ``` +Lists support both **offset** (`?limit&offset`) and **cursor** paging: a list +response includes a `nextCursor`; pass it back as `?after=` to fetch the +next page in stable `(sort, id)` order — no skips or repeats when rows are +inserted mid-iteration, the way offset drifts. With the SDK: +`const page = await api.data('notes').page({ sort: '-createdAt', after });` +then follow `page.nextCursor` until it's `null`. Back a hot sort field with a +[declared index](#data--schemaless-collections) and cursor paging stays fast at +any size. + Via the SDK: `api.data('members').setSchema(schema)` / `getSchema()` / `clearSchema()`. ### Access control — per-collection rules diff --git a/public/docs.html b/public/docs.html index f397661..e8fbfd5 100644 --- a/public/docs.html +++ b/public/docs.html @@ -182,7 +182,7 @@

REST API — Data

POST/v1/data/:cCreate (body = document). POST/v1/data/:c/bulkCreate many (array or { documents: [...] }) in one transaction. - GET/v1/data/:cList/query — ?limit&offset&sort=-field&f.field=op:value. See queries. + GET/v1/data/:cList/query — ?limit&offset&sort=-field&f.field=op:value. Response carries nextCursor; pass it as ?after= for stable cursor paging. See queries. GET/v1/data/:c/countCount matches → { count }. GET/v1/data/:c/aggregate?op=count|sum|avg|min|max&field=… → { value }. GET/v1/data/:c/searchFull-text: ?q=terms&limit (ranked). @@ -316,6 +316,7 @@

Queries & filters

sort=-fieldSort ascending, or -field for descending. select=a,bReturn only these fields (plus id + timestamps). limit / offsetPagination (limit bounded 1–1000). + after=<cursor>Cursor paging: pass a response's nextCursor to continue in stable (sort, id) order — no skips/repeats under concurrent writes.

In the SDK: where: { age: { op: 'gte', value: 18 } }; a bare value means equality, an array means in. id, createdAt, updatedAt are filterable + sortable too.

diff --git a/public/sdk.d.ts b/public/sdk.d.ts index 1fda4c7..4a99b8a 100644 --- a/public/sdk.d.ts +++ b/public/sdk.d.ts @@ -35,6 +35,8 @@ declare namespace Zero { offset?: number; /** Field name, or `-field` for descending. */ sort?: string; + /** A `nextCursor` from a previous page() for stable forward paging (replaces offset). */ + after?: string; /** Equality (bare value), `[a, b]` for `in`, or `{ op, value }` for any operator. */ where?: Record; /** OR group: at least one must match, ANDed with `where` as a whole. */ @@ -57,6 +59,8 @@ declare namespace Zero { collection: string; total: number; documents: Doc[]; + /** Pass to the next `page({ after })` to continue; null when there are no more. */ + nextCursor: string | null; } interface FileMeta { diff --git a/public/sdk.js b/public/sdk.js index 61ba287..359d79b 100644 --- a/public/sdk.js +++ b/public/sdk.js @@ -81,6 +81,8 @@ if (q.limit != null) parts.push('limit=' + encodeURIComponent(q.limit)); if (q.offset != null) parts.push('offset=' + encodeURIComponent(q.offset)); if (q.sort) parts.push('sort=' + encodeURIComponent(q.sort)); + if (q.after) parts.push('after=' + encodeURIComponent(q.after)); // cursor paging + if (q.all) parts.push('all=true'); // only honored by deleteWhere() if (q.select) parts.push('select=' + encodeURIComponent([].concat(q.select).join(','))); // `where` → ANDed `f.` filters; `or` → an ORed `or.` group. diff --git a/src/modules/data/routes.ts b/src/modules/data/routes.ts index a3503a1..181be20 100644 --- a/src/modules/data/routes.ts +++ b/src/modules/data/routes.ts @@ -115,10 +115,11 @@ export async function dataRoutes(app: FastifyInstance): Promise { const query = (req.query ?? {}) as Record; const filters = parseFilters(query, 'f.'); const orFilters = parseFilters(query, 'or.'); - const documents = await store.list(req.projectId, collection, { + const { documents, nextCursor } = await store.list(req.projectId, collection, { limit: typeof query.limit === 'string' ? Number(query.limit) : undefined, offset: typeof query.offset === 'string' ? Number(query.offset) : undefined, sort: typeof query.sort === 'string' ? query.sort : undefined, + after: typeof query.after === 'string' ? query.after : undefined, filters, orFilters, select: parseSelect(query), @@ -128,6 +129,7 @@ export async function dataRoutes(app: FastifyInstance): Promise { collection, total: await store.count(req.projectId, collection, filters, orFilters), documents: hidden.length ? documents.map((d) => redact(d, hidden)) : documents, + nextCursor, }; }); diff --git a/src/modules/data/store.ts b/src/modules/data/store.ts index 5f77b85..a259d0e 100644 Binary files a/src/modules/data/store.ts and b/src/modules/data/store.ts differ diff --git a/test/api.test.ts b/test/api.test.ts index f021be4..cd31f8e 100644 --- a/test/api.test.ts +++ b/test/api.test.ts @@ -212,6 +212,41 @@ test('data: filtering, operators, and sorting', async () => { assert.deepEqual(inHits.map((d: any) => d.priority), [1, 5]); }); +test('data: cursor pagination walks a collection stably across inserts', async () => { + const api = Zero(base); + const col = api.data('ledger'); + for (let i = 0; i < 5; i++) await col.create({ n: i }); + + // Page by 2, following nextCursor; collect everything. + const seen: number[] = []; + let cursor: string | undefined; + let pages = 0; + do { + const page = await col.page({ sort: 'n', limit: 2, after: cursor }); + seen.push(...page.documents.map((d: any) => d.n)); + cursor = page.nextCursor ?? undefined; + // Insert a *new* row mid-iteration: offset paging would skip/repeat, cursor won't. + if (pages === 0) await col.create({ n: 99 }); + pages += 1; + } while (cursor && pages < 10); + + // The original five are each seen exactly once, in order (the late n:99 sorts last). + assert.deepEqual( + seen.filter((n) => n < 10), + [0, 1, 2, 3, 4], + ); + assert.equal(new Set(seen).size, seen.length, 'no duplicates across pages'); + + // A garbage cursor is a clean 400. + const bad = await fetch(`${base}/v1/data/ledger?after=not-a-cursor`); + assert.equal(bad.status, 400); + + // A cursor taken under one sort can't be reused under a different sort. + const first = await col.page({ sort: 'n', limit: 1 }); + const mismatch = await fetch(`${base}/v1/data/ledger?sort=-n&after=${encodeURIComponent(first.nextCursor)}`); + assert.equal(mismatch.status, 400); +}); + test('data: aggregate computes sum/avg/min/max/count over a field', async () => { const api = Zero(base); await api.data('orders').createMany([