From 7d980cdd6f079f2658b449628fd8c6e787341885 Mon Sep 17 00:00:00 2001 From: betegon Date: Sun, 25 Oct 2026 22:24:11 +0100 Subject: [PATCH 1/3] feat(cli): expose api pagination and cursor continuation --- apps/cli-docs/src/fragments/commands/api.md | 17 +++ .../skills/sentry-cli/references/api.md | 7 + packages/cli/src/commands/api.ts | 65 ++++++++- packages/cli/test/e2e/api.test.ts | 126 +++++++++++++++++- packages/cli/test/e2e/library.test.ts | 44 ++++++ 5 files changed, 251 insertions(+), 8 deletions(-) diff --git a/apps/cli-docs/src/fragments/commands/api.md b/apps/cli-docs/src/fragments/commands/api.md index 852926a6b..e0f19512c 100644 --- a/apps/cli-docs/src/fragments/commands/api.md +++ b/apps/cli-docs/src/fragments/commands/api.md @@ -13,6 +13,23 @@ bodies; streaming responses such as SSE are excluded. When querying the Events API (`/events/` endpoint), valid dataset values are: `spans`, `logs`, `errors`, `tracemetrics`, `profile_functions`, and `uptime_results`. +## Pagination + +Each invocation fetches one page. With `--json`, successful textual responses +include `pagination.nextCursor` and/or `pagination.prevCursor` when the server's +`Link` header advertises a usable continuation. `--fields` filters only the +response body and preserves this metadata. + +Repeat the same GET request with `--cursor ` to continue, keeping its +filters, ordering, page size, and time range. The token is opaque: `next` and +`prev` do not refer to saved pages. `--cursor` replaces existing cursor parameters +from both the endpoint URL and request fields; all other parameters are retained. +It is only supported for GET requests, including `--dry-run` previews. + +When no usable cursor is advertised, `pagination` is omitted. This does not prove +that the underlying dataset was fully scanned or unsampled. The command does not +automatically follow pages. + ## Binary responses Endpoints that return binary data — image attachments, minidumps, debug files — are streamed to stdout as raw bytes, so you can redirect them straight to a file. diff --git a/packages/cli/plugins/sentry-cli/skills/sentry-cli/references/api.md b/packages/cli/plugins/sentry-cli/skills/sentry-cli/references/api.md index b907d639a..06fb5960e 100644 --- a/packages/cli/plugins/sentry-cli/skills/sentry-cli/references/api.md +++ b/packages/cli/plugins/sentry-cli/skills/sentry-cli/references/api.md @@ -23,6 +23,7 @@ Make an authenticated API request **Flags:** - `-X, --method - The HTTP method for the request - (default: "GET")` +- `--cursor - Pagination cursor for GET (replaces any existing cursor)` - `-d, --data - Inline JSON body for the request (like curl -d)` - `-F, --field ... - Add a typed parameter (key=value, key[sub]=value, key[]=value)` - `-f, --raw-field ... - Add a string parameter without JSON parsing` @@ -38,6 +39,12 @@ Make an authenticated API request # List organizations sentry api organizations/ +# Read a page and its pagination metadata +sentry api organizations/ -f per_page=2 --json + +# Continue using a cursor from the previous response +sentry api organizations/ -f per_page=2 --cursor '2:1:0' --json + # Get a specific issue sentry api issues/123456789/ diff --git a/packages/cli/src/commands/api.ts b/packages/cli/src/commands/api.ts index b00846361..a129fb7bd 100644 --- a/packages/cli/src/commands/api.ts +++ b/packages/cli/src/commands/api.ts @@ -9,7 +9,11 @@ import { access, readFile } from "node:fs/promises"; // oxlint-disable-next-line sentry-cli/no-namespace-import -- Sentry SDK recommends namespace import import * as Sentry from "@sentry/node-core/light"; import type { SentryContext } from "../context.js"; -import { appendSearchParams, rawApiRequest } from "../lib/api-client.js"; +import { + appendSearchParams, + parseLinkHeader, + rawApiRequest, +} from "../lib/api-client.js"; import { buildCommand } from "../lib/command.js"; import { OutputError, @@ -41,6 +45,7 @@ type HttpMethod = "GET" | "POST" | "PUT" | "DELETE" | "PATCH"; type ApiFlags = { readonly method: HttpMethod; + readonly cursor?: string; readonly data?: string; readonly field?: string[]; readonly "raw-field"?: string[]; @@ -1177,10 +1182,12 @@ function formatApiResponseOutput( if (!json || output instanceof Uint8Array) { return output; } + const pagination = parseLinkHeader(response.headers.get("link")); return { status: response.status, statusText: response.statusText, body: output, + ...(pagination.nextCursor || pagination.prevCursor ? { pagination } : {}), }; } @@ -1206,6 +1213,25 @@ function formatApiResponseJson(data: unknown, fields?: string[]): unknown { }; } +/** Compose query parameters, replacing all existing cursors when explicitly set. */ +function resolveRequestEndpoint( + endpoint: string, + params: Record | undefined, + cursor: string | undefined, +): string { + const composed = appendSearchParams(endpoint, params); + if (cursor === undefined) { + return composed; + } + const queryIndex = composed.indexOf("?"); + const path = queryIndex === -1 ? composed : composed.slice(0, queryIndex); + const query = new URLSearchParams( + queryIndex === -1 ? "" : composed.slice(queryIndex + 1), + ); + query.set("cursor", cursor); + return `${path}?${query}`; +} + /** * Resolve the full URL that rawApiRequest would use for a request. * @@ -1529,6 +1555,9 @@ export const apiCommand = buildCommand({ "Absolute HTTP(S) Sentry URLs are also accepted; their origin is " + "validated against your authenticated host. " + "Authentication is handled automatically using your stored credentials.\n\n" + + "Pagination: --json includes usable nextCursor/prevCursor values in " + + "pagination when the server advertises them. Repeat the same GET with " + + "--cursor to continue. Each invocation fetches one page.\n\n" + "Body options:\n" + ' --data/-d \'{"key":"value"}\' Inline JSON body (like curl -d)\n' + ' --input/-i file.json Read body from file (or "-" for stdin)\n\n' + @@ -1543,6 +1572,15 @@ export const apiCommand = buildCommand({ description: "List organizations", command: "sentry api organizations/", }, + { + description: "Read a page and its pagination metadata", + command: "sentry api organizations/ -f per_page=2 --json", + }, + { + description: "Continue using a cursor from the previous response", + command: + "sentry api organizations/ -f per_page=2 --cursor '2:1:0' --json", + }, { description: "Get a specific issue", command: "sentry api issues/123456789/", @@ -1619,6 +1657,13 @@ export const apiCommand = buildCommand({ default: "GET" as const, placeholder: "method", }, + cursor: { + kind: "parsed", + parse: String, + brief: "Pagination cursor for GET (replaces any existing cursor)", + optional: true, + placeholder: "token", + }, data: { kind: "parsed", parse: String, @@ -1682,6 +1727,12 @@ export const apiCommand = buildCommand({ }, async *func(this: SentryContext, flags: ApiFlags, endpoint: string) { const { stdin } = this; + if (flags.cursor !== undefined && flags.method !== "GET") { + throw new ValidationError( + "--cursor is only supported for GET requests", + "cursor", + ); + } const { normalizedEndpoint, requestBaseUrl, strippedApiPrefix } = resolveApiTarget(endpoint); @@ -1696,6 +1747,11 @@ export const apiCommand = buildCommand({ ); } const { body, params } = await resolveBody(flags, stdin); + const requestEndpoint = resolveRequestEndpoint( + normalizedEndpoint, + params, + flags.cursor, + ); const headers = flags.header && flags.header.length > 0 @@ -1706,7 +1762,7 @@ export const apiCommand = buildCommand({ if (flags["dry-run"]) { yield new CommandOutput({ method: flags.method, - url: resolveRequestUrl(normalizedEndpoint, params, requestBaseUrl), + url: resolveRequestUrl(requestEndpoint, undefined, requestBaseUrl), headers: resolveEffectiveHeaders(headers, body), body: body ?? null, }); @@ -1716,15 +1772,14 @@ export const apiCommand = buildCommand({ const verbose = flags.verbose && !flags.silent; if (verbose) { - logRequest(flags.method, normalizedEndpoint, headers); + logRequest(flags.method, requestEndpoint, headers); } // The shared authenticated fetch validates requestBaseUrl against the // token host and registered regional origins before adding credentials. - const response = await rawApiRequest(normalizedEndpoint, { + const response = await rawApiRequest(requestEndpoint, { method: flags.method, body, - params, headers, baseUrl: requestBaseUrl, }); diff --git a/packages/cli/test/e2e/api.test.ts b/packages/cli/test/e2e/api.test.ts index 8366db5cd..4035d10df 100644 --- a/packages/cli/test/e2e/api.test.ts +++ b/packages/cli/test/e2e/api.test.ts @@ -16,16 +16,50 @@ import { } from "vitest"; import { EXIT } from "../../src/lib/errors.js"; import { createE2EContext, type E2EContext } from "../fixture.js"; +import organizationsFixture from "../fixtures/organizations.json"; import { cleanupTestDir, createTestConfigDir } from "../helpers.js"; -import { createSentryMockServer, TEST_TOKEN } from "../mocks/routes.js"; -import type { MockServer } from "../mocks/server.js"; +import { apiRoutes, TEST_TOKEN } from "../mocks/routes.js"; +import { createMockServer, type MockServer } from "../mocks/server.js"; let testConfigDir: string; let mockServer: MockServer; let ctx: E2EContext; +const paginationRequests: URL[] = []; +const nextCursor = "100:1:0"; +const prevCursor = "100:0:1"; beforeAll(async () => { - mockServer = createSentryMockServer(); + mockServer = createMockServer( + [ + ...apiRoutes, + { + method: "GET", + path: "/api/0/api-pagination/", + response: (req, _params, serverUrl) => { + const url = new URL(req.url); + paginationRequests.push(url); + const isSecondPage = url.searchParams.get("cursor") === nextCursor; + return { + body: [organizationsFixture[isSecondPage ? 1 : 0]], + headers: { + Link: + `<${serverUrl}/api/0/api-pagination/?cursor=${nextCursor}>; rel="next"; results="${!isSecondPage}"; cursor="${nextCursor}", ` + + `<${serverUrl}/api/0/api-pagination/?cursor=${prevCursor}>; rel="previous"; results="${isSecondPage}"; cursor="${prevCursor}"`, + }, + }; + }, + }, + { + method: "POST", + path: "/api/0/api-pagination/", + response: (req) => { + paginationRequests.push(new URL(req.url)); + return { body: { accepted: true } }; + }, + }, + ], + { validTokens: [TEST_TOKEN] }, + ); await mockServer.start(); }); @@ -34,6 +68,7 @@ afterAll(() => { }); beforeEach(async () => { + paginationRequests.length = 0; testConfigDir = await createTestConfigDir("e2e-api-"); ctx = createE2EContext(testConfigDir, mockServer.url); }); @@ -124,6 +159,91 @@ describe("sentry api", () => { expect(JSON.parse(result.stdout)).toEqual({ method: "GET" }); }); + test("continues one page at a time and preserves pagination with --fields", async () => { + await ctx.setAuthToken(TEST_TOKEN); + const args = ["api", "api-pagination/", "--json", "--fields", "name"]; + + const first = await ctx.run(args); + expect(first.exitCode, first.stderr + first.stdout).toBe(0); + const firstPage = JSON.parse(first.stdout); + expect(firstPage).toEqual({ + status: 200, + statusText: "OK", + body: [{ name: organizationsFixture[0].name }], + pagination: { nextCursor }, + }); + expect(paginationRequests).toHaveLength(1); + + const second = await ctx.run([ + ...args, + "--cursor", + firstPage.pagination.nextCursor, + ]); + expect(second.exitCode, second.stderr + second.stdout).toBe(0); + expect(JSON.parse(second.stdout)).toEqual({ + status: 200, + statusText: "OK", + body: [{ name: organizationsFixture[1].name }], + pagination: { prevCursor }, + }); + expect(paginationRequests).toHaveLength(2); + expect(paginationRequests[1].searchParams.getAll("cursor")).toEqual([ + nextCursor, + ]); + }); + + test.each([ + { + name: "typed and raw fields", + flags: ["-F", "cursor=typed-old", "-f", "cursor=raw-old"], + }, + { name: "JSON data", flags: ["-d", '{"cursor":"data-old"}'] }, + { name: "form data", flags: ["-d", "cursor=data-old"] }, + ])( + "--cursor replaces $name and URL cursors in the preview and request", + async ({ flags }) => { + await ctx.setAuthToken(TEST_TOKEN); + const args = [ + "api", + `${mockServer.url}/api/0/api-pagination/?cursor=url-old&cursor=url-older&field=id&field=name&project=1&project=2`, + ...flags, + "--cursor", + nextCursor, + "--json", + ]; + + const preview = await ctx.run([...args, "--dry-run"]); + expect(preview.exitCode, preview.stderr + preview.stdout).toBe(0); + const request = JSON.parse(preview.stdout); + const url = new URL(request.url); + expect(url.searchParams.getAll("cursor")).toEqual([nextCursor]); + expect(url.searchParams.getAll("field")).toEqual(["id", "name"]); + expect(url.searchParams.getAll("project")).toEqual(["1", "2"]); + expect(paginationRequests).toHaveLength(0); + + const result = await ctx.run(args); + expect(result.exitCode, result.stderr + result.stdout).toBe(0); + expect(paginationRequests.map((sent) => sent.href)).toEqual([url.href]); + expect(JSON.parse(result.stdout).body).toEqual([organizationsFixture[1]]); + }, + ); + + test("rejects --cursor for non-GET requests before sending them", async () => { + await ctx.setAuthToken(TEST_TOKEN); + const result = await ctx.run([ + "api", + "api-pagination/", + "--method", + "POST", + "--cursor", + nextCursor, + ]); + + expect(result.exitCode).toBe(EXIT.VALIDATION); + expect(result.stdout + result.stderr).toMatch(/cursor.*GET|GET.*cursor/i); + expect(paginationRequests).toHaveLength(0); + }); + test("empty error bodies report the HTTP status and request", async () => { await ctx.setAuthToken(TEST_TOKEN); diff --git a/packages/cli/test/e2e/library.test.ts b/packages/cli/test/e2e/library.test.ts index c2a6f5392..075cd8a56 100644 --- a/packages/cli/test/e2e/library.test.ts +++ b/packages/cli/test/e2e/library.test.ts @@ -14,6 +14,7 @@ import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { beforeAll, describe, expect, test } from "vitest"; import { EXIT } from "../../src/lib/errors.js"; +import organizationsFixture from "../fixtures/organizations.json"; import { useTestConfigDir } from "../helpers.js"; import { BUNDLE_INDEX_PATH, @@ -205,6 +206,49 @@ describe("library mode (bundled)", () => { // --- Typed SDK --- + test("sdk.api passes an opaque cursor and returns page metadata", async () => { + const { stdout } = await runNodeScriptOk(` + const { createSentrySDK } = require('./dist/index.cjs'); + const requests = []; + globalThis.fetch = async (input) => { + requests.push(String(input)); + return Response.json(${JSON.stringify([organizationsFixture[1]])}, { + statusText: 'OK', + headers: { + Link: '; rel="next"; results="true"; cursor="100:2:0", ; rel="previous"; results="true"; cursor="100:0:1"', + }, + }); + }; + const sdk = createSentrySDK({ + token: 'test-sdk-token', + url: 'http://localhost:9000', + cwd: ${JSON.stringify(getConfigDir())}, + }); + sdk.api({ + endpoint: 'organizations/?cursor=old', + cursor: 'page-token+/=:1', + rawField: ['field=id', 'field=name'], + }).then(response => { + console.log(JSON.stringify({ requests, response })); + }).catch(error => { + console.error(error); + process.exitCode = 1; + }); + `); + + const result = JSON.parse(stdout.trim()); + expect(result.requests).toHaveLength(1); + const request = new URL(result.requests[0]); + expect(request.searchParams.getAll("cursor")).toEqual(["page-token+/=:1"]); + expect(request.searchParams.getAll("field")).toEqual(["id", "name"]); + expect(result.response).toEqual({ + status: 200, + statusText: "OK", + body: [organizationsFixture[1]], + pagination: { nextCursor: "100:2:0", prevCursor: "100:0:1" }, + }); + }); + test("sourcemap upload rejects a split token without leaking it or fetching", async () => { const directory = getConfigDir(); await writeFile( From 0ec43e21fa79f6aa5f105ce1df55614be21904dc Mon Sep 17 00:00:00 2001 From: betegon Date: Sun, 25 Oct 2026 23:05:02 +0100 Subject: [PATCH 2/3] fix(cli): exclude URL fragments from api pagination requests --- packages/cli/src/commands/api.ts | 8 ++++++-- packages/cli/test/e2e/api.test.ts | 24 ++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/commands/api.ts b/packages/cli/src/commands/api.ts index a129fb7bd..3335dc0b3 100644 --- a/packages/cli/src/commands/api.ts +++ b/packages/cli/src/commands/api.ts @@ -1213,13 +1213,17 @@ function formatApiResponseJson(data: unknown, fields?: string[]): unknown { }; } -/** Compose query parameters, replacing all existing cursors when explicitly set. */ +/** + * Compose query parameters without URL fragments, replacing all existing cursors + * when an explicit cursor is provided. + */ function resolveRequestEndpoint( endpoint: string, params: Record | undefined, cursor: string | undefined, ): string { - const composed = appendSearchParams(endpoint, params); + const withoutFragment = endpoint.split("#", 1)[0] ?? endpoint; + const composed = appendSearchParams(withoutFragment, params); if (cursor === undefined) { return composed; } diff --git a/packages/cli/test/e2e/api.test.ts b/packages/cli/test/e2e/api.test.ts index 4035d10df..a86810c68 100644 --- a/packages/cli/test/e2e/api.test.ts +++ b/packages/cli/test/e2e/api.test.ts @@ -244,6 +244,30 @@ describe("sentry api", () => { expect(paginationRequests).toHaveLength(0); }); + test.each([ + { endpoint: "api-pagination/?query=foo%23bar#anchor", query: "foo#bar" }, + { endpoint: "api-pagination/#anchor", query: null }, + ])( + "--cursor ignores URL fragments in $endpoint", + async ({ endpoint, query }) => { + await ctx.setAuthToken(TEST_TOKEN); + const args = ["api", endpoint, "--cursor", nextCursor, "--json"]; + + const preview = await ctx.run([...args, "--dry-run"]); + expect(preview.exitCode, preview.stderr + preview.stdout).toBe(0); + const url = new URL(JSON.parse(preview.stdout).url); + expect(url.hash).toBe(""); + expect(url.searchParams.get("query")).toBe(query); + expect(url.searchParams.getAll("cursor")).toEqual([nextCursor]); + expect(paginationRequests).toHaveLength(0); + + const result = await ctx.run(args); + expect(result.exitCode, result.stderr + result.stdout).toBe(0); + expect(paginationRequests.map((sent) => sent.href)).toEqual([url.href]); + expect(JSON.parse(result.stdout).body).toEqual([organizationsFixture[1]]); + }, + ); + test("empty error bodies report the HTTP status and request", async () => { await ctx.setAuthToken(TEST_TOKEN); From f2d8d61de133676eaee069ff78eb81219eec761a Mon Sep 17 00:00:00 2001 From: betegon Date: Tue, 27 Oct 2026 20:29:15 +0100 Subject: [PATCH 3/3] fix(cli): reuse shared api pagination and custom cursors --- apps/cli-docs/src/fragments/commands/api.md | 38 +- .../skills/sentry-cli/references/api.md | 9 +- packages/cli/src/commands/api.ts | 215 +++++++++--- packages/cli/src/lib/api-client.ts | 3 + packages/cli/src/lib/api/infrastructure.ts | 12 + packages/cli/src/lib/db/pagination.ts | 19 +- packages/cli/test/e2e/api.test.ts | 326 +++++++++++++++++- packages/cli/test/e2e/library.test.ts | 24 +- .../cli/test/lib/paginate.property.test.ts | 62 ++++ packages/toolkit-core/README.md | 4 + packages/toolkit-core/src/pagination.ts | 98 ++++-- 11 files changed, 692 insertions(+), 118 deletions(-) diff --git a/apps/cli-docs/src/fragments/commands/api.md b/apps/cli-docs/src/fragments/commands/api.md index e0f19512c..4e33faa53 100644 --- a/apps/cli-docs/src/fragments/commands/api.md +++ b/apps/cli-docs/src/fragments/commands/api.md @@ -15,20 +15,30 @@ When querying the Events API (`/events/` endpoint), valid dataset values are: `s ## Pagination -Each invocation fetches one page. With `--json`, successful textual responses -include `pagination.nextCursor` and/or `pagination.prevCursor` when the server's -`Link` header advertises a usable continuation. `--fields` filters only the -response body and preserves this metadata. - -Repeat the same GET request with `--cursor ` to continue, keeping its -filters, ordering, page size, and time range. The token is opaque: `next` and -`prev` do not refer to saved pages. `--cursor` replaces existing cursor parameters -from both the endpoint URL and request fields; all other parameters are retained. -It is only supported for GET requests, including `--dry-run` previews. - -When no usable cursor is advertised, `pagination` is omitted. This does not prove -that the underlying dataset was fully scanned or unsampled. The command does not -automatically follow pages. +Each invocation fetches one server page. Repeat the same GET endpoint and filters +with `--cursor next` (`-c next`) to continue, `--cursor prev` to revisit a page, +or `--cursor first` to start again. The shared history expires after five minutes +and is scoped to the request and authenticated identity. Keep the same query +parameters and headers when navigating. The endpoint remains required. + +Raw server cursors and custom cursors from the CLI's shared paginator are also +accepted. Custom cursors preserve the page size and skip rows already returned, +including when navigating back to a bookmarked page. A cursor's encoded page +size takes precedence over `per_page`. Mid-page bookmarks support array bodies +and responses with a `data` array, such as the Events API; envelope metadata is +preserved. Other response bodies remain unchanged for ordinary cursors. + +With `--json`, `pagination.nextCursor` is the CLI continuation and can retain a +custom bookmark; pass it back through `--cursor` to resume exactly. +`pagination.prevCursor` comes from the server, while `pagination.hasPrev` +indicates a previously visited page in local history. `--fields` filters only +the body and preserves this metadata. +The pagination object is omitted when no continuation or local previous page is +available. Its absence does not prove a complete or unsampled dataset. + +`--cursor` replaces cursor parameters from the endpoint and request fields. It is +only supported for GET requests. `--dry-run` previews the resolved request; +dry-runs and failed requests do not advance the saved history. ## Binary responses diff --git a/packages/cli/plugins/sentry-cli/skills/sentry-cli/references/api.md b/packages/cli/plugins/sentry-cli/skills/sentry-cli/references/api.md index 06fb5960e..2bbe4fe35 100644 --- a/packages/cli/plugins/sentry-cli/skills/sentry-cli/references/api.md +++ b/packages/cli/plugins/sentry-cli/skills/sentry-cli/references/api.md @@ -23,7 +23,7 @@ Make an authenticated API request **Flags:** - `-X, --method - The HTTP method for the request - (default: "GET")` -- `--cursor - Pagination cursor for GET (replaces any existing cursor)` +- `-c, --cursor - Navigate pages: "next", "prev", "first" (or raw cursor string)` - `-d, --data - Inline JSON body for the request (like curl -d)` - `-F, --field ... - Add a typed parameter (key=value, key[sub]=value, key[]=value)` - `-f, --raw-field ... - Add a string parameter without JSON parsing` @@ -42,8 +42,11 @@ sentry api organizations/ # Read a page and its pagination metadata sentry api organizations/ -f per_page=2 --json -# Continue using a cursor from the previous response -sentry api organizations/ -f per_page=2 --cursor '2:1:0' --json +# Continue the same query using shared pagination +sentry api organizations/ -f per_page=2 --cursor next --json + +# Return to the previously visited page +sentry api organizations/ -f per_page=2 --cursor prev --json # Get a specific issue sentry api issues/123456789/ diff --git a/packages/cli/src/commands/api.ts b/packages/cli/src/commands/api.ts index 3335dc0b3..89dec9a3c 100644 --- a/packages/cli/src/commands/api.ts +++ b/packages/cli/src/commands/api.ts @@ -11,10 +11,22 @@ import * as Sentry from "@sentry/node-core/light"; import type { SentryContext } from "../context.js"; import { appendSearchParams, + consumePage, + decodePageCursor, + type PagePosition, parseLinkHeader, rawApiRequest, } from "../lib/api-client.js"; import { buildCommand } from "../lib/command.js"; +import { applyCustomHeaders } from "../lib/custom-headers.js"; +import { getIdentityFingerprint } from "../lib/db/auth.js"; +import { + advancePaginationState, + buildPaginationContextKey, + type CursorDirection, + hasPreviousPage, + resolveCursor, +} from "../lib/db/pagination.js"; import { OutputError, stringifyUnknown, @@ -24,7 +36,9 @@ import { filterFields } from "../lib/formatters/json.js"; import { CommandOutput } from "../lib/formatters/output.js"; import { validateEndpoint } from "../lib/input-validation.js"; import { imageBytesToKitty } from "../lib/kitty-image.js"; +import { LIST_CURSOR_FLAG } from "../lib/list-command.js"; import { logger } from "../lib/logger.js"; +import { buildCacheKey } from "../lib/response-cache.js"; import { getDefaultSdkConfig, getSdkConfig } from "../lib/sentry-client.js"; import { canRenderKitty, @@ -40,11 +54,13 @@ const log = logger.withTag("api"); const LINE_BREAK_PATTERN = /[ \t]*[\r\n]+[ \t]*/g; const API_PREFIX = "/api/0"; const API_PREFIX_WITH_SLASH = `${API_PREFIX}/`; +const PAGINATION_KEY = "api"; type HttpMethod = "GET" | "POST" | "PUT" | "DELETE" | "PATCH"; type ApiFlags = { readonly method: HttpMethod; + /** Shared navigation keyword or opaque pagination cursor for GET requests. */ readonly cursor?: string; readonly data?: string; readonly field?: string[]; @@ -1173,21 +1189,29 @@ function throwApiResponseError( throw new OutputError(response.body); } -/** Add HTTP metadata to textual responses in JSON output mode. */ +/** Server continuations plus navigation available in the local cursor history. */ +type ApiPaginationMetadata = ReturnType & { + /** Whether this query has a previously visited page in the local history. */ + hasPrev?: boolean; +}; + +/** Add HTTP and pagination metadata to textual responses in JSON output mode. */ function formatApiResponseOutput( response: ApiResponseOutput, output: unknown, json: boolean, + pagination: ApiPaginationMetadata, ): unknown { if (!json || output instanceof Uint8Array) { return output; } - const pagination = parseLinkHeader(response.headers.get("link")); return { status: response.status, statusText: response.statusText, body: output, - ...(pagination.nextCursor || pagination.prevCursor ? { pagination } : {}), + ...(pagination.nextCursor || pagination.prevCursor || pagination.hasPrev + ? { pagination } + : {}), }; } @@ -1214,26 +1238,92 @@ function formatApiResponseJson(data: unknown, fields?: string[]): unknown { } /** - * Compose query parameters without URL fragments, replacing all existing cursors - * when an explicit cursor is provided. + * Compose query parameters without URL fragments. */ function resolveRequestEndpoint( endpoint: string, params: Record | undefined, - cursor: string | undefined, ): string { const withoutFragment = endpoint.split("#", 1)[0] ?? endpoint; - const composed = appendSearchParams(withoutFragment, params); - if (cursor === undefined) { - return composed; - } - const queryIndex = composed.indexOf("?"); - const path = queryIndex === -1 ? composed : composed.slice(0, queryIndex); + return appendSearchParams(withoutFragment, params); +} + +/** Pagination resolved for one GET, without mutating its saved history. */ +type ApiPagination = { + /** Relative endpoint with the resolved server cursor and page size. */ + endpoint: string; + /** Identity-scoped fingerprint of the caller's request, excluding cursor. */ + contextKey: string; + /** Navigation operation to commit after a successful response. */ + direction: CursorDirection; + /** Original cursor, including any bookmark, saved for backward navigation. */ + currentCursor: string | undefined; + /** Shared decoded position; skip counts rows already returned to the caller. */ + position: PagePosition; +}; + +/** + * Resolve shared navigation and custom cursors while retaining a single request. + * Context uses the caller's query before cursor-derived page-size overrides, so + * repeating the same arguments with `next` finds the same walk. Header/body + * values participate in an opaque fingerprint and are never stored as plaintext. + */ +async function resolveApiPagination( + endpoint: string, + cursorFlag: string | undefined, + request: NonNullable[1]>, +): Promise { + const queryIndex = endpoint.indexOf("?"); + const path = queryIndex === -1 ? endpoint : endpoint.slice(0, queryIndex); const query = new URLSearchParams( - queryIndex === -1 ? "" : composed.slice(queryIndex + 1), + queryIndex === -1 ? "" : endpoint.slice(queryIndex + 1), ); - query.set("cursor", cursor); - return `${path}?${query}`; + const inputCursor = query.getAll("cursor").at(-1); + query.delete("cursor"); + const contextUrl = resolveRequestUrl( + `${path}?${query}`, + undefined, + request.baseUrl, + ); + const effectiveHeaders = new Headers( + resolveEffectiveHeaders(request.headers, request.body), + ); + applyCustomHeaders(effectiveHeaders, contextUrl); + effectiveHeaders.delete("authorization"); + effectiveHeaders.delete("sentry-trace"); + effectiveHeaders.delete("baggage"); + const identity = JSON.stringify([ + getIdentityFingerprint(), + [...effectiveHeaders], + request.body, + ]); + const contextKey = buildPaginationContextKey( + "api", + await buildCacheKey("GET", contextUrl, identity), + ); + const { cursor, direction } = resolveCursor( + cursorFlag, + PAGINATION_KEY, + contextKey, + ); + const currentCursor = cursorFlag === undefined ? inputCursor : cursor; + const position = decodePageCursor(currentCursor); + if (position.cursor !== undefined) { + query.set("cursor", position.cursor); + } + if (position.perPage !== undefined) { + query.set("per_page", String(position.perPage)); + } + return { + endpoint: + cursorFlag === undefined && inputCursor === undefined + ? endpoint + : `${path}${query.size > 0 ? `?${query}` : ""}`, + contextKey, + direction, + currentCursor, + position, + }; } /** @@ -1559,9 +1649,11 @@ export const apiCommand = buildCommand({ "Absolute HTTP(S) Sentry URLs are also accepted; their origin is " + "validated against your authenticated host. " + "Authentication is handled automatically using your stored credentials.\n\n" + - "Pagination: --json includes usable nextCursor/prevCursor values in " + - "pagination when the server advertises them. Repeat the same GET with " + - "--cursor to continue. Each invocation fetches one page.\n\n" + + "Pagination: --json includes nextCursor/prevCursor and hasPrev in " + + "pagination when navigation is available. Repeat the same GET and " + + "filters with --cursor next, prev, or first, or pass a raw cursor. " + + "Shared custom cursors restore page size and skip rows already read. " + + "Each invocation fetches one page.\n\n" + "Body options:\n" + ' --data/-d \'{"key":"value"}\' Inline JSON body (like curl -d)\n' + ' --input/-i file.json Read body from file (or "-" for stdin)\n\n' + @@ -1581,9 +1673,12 @@ export const apiCommand = buildCommand({ command: "sentry api organizations/ -f per_page=2 --json", }, { - description: "Continue using a cursor from the previous response", - command: - "sentry api organizations/ -f per_page=2 --cursor '2:1:0' --json", + description: "Continue the same query using shared pagination", + command: "sentry api organizations/ -f per_page=2 --cursor next --json", + }, + { + description: "Return to the previously visited page", + command: "sentry api organizations/ -f per_page=2 --cursor prev --json", }, { description: "Get a specific issue", @@ -1661,13 +1756,7 @@ export const apiCommand = buildCommand({ default: "GET" as const, placeholder: "method", }, - cursor: { - kind: "parsed", - parse: String, - brief: "Pagination cursor for GET (replaces any existing cursor)", - optional: true, - placeholder: "token", - }, + cursor: LIST_CURSOR_FLAG, data: { kind: "parsed", parse: String, @@ -1722,6 +1811,7 @@ export const apiCommand = buildCommand({ }, aliases: { X: "method", + c: "cursor", d: "data", F: "field", f: "raw-field", @@ -1751,16 +1841,23 @@ export const apiCommand = buildCommand({ ); } const { body, params } = await resolveBody(flags, stdin); - const requestEndpoint = resolveRequestEndpoint( - normalizedEndpoint, - params, - flags.cursor, - ); + const composedEndpoint = resolveRequestEndpoint(normalizedEndpoint, params); const headers = flags.header && flags.header.length > 0 ? parseHeaders(flags.header) : undefined; + const request = { + method: flags.method, + body, + headers, + baseUrl: requestBaseUrl, + }; + const navigation = + flags.method === "GET" + ? await resolveApiPagination(composedEndpoint, flags.cursor, request) + : undefined; + const requestEndpoint = navigation?.endpoint ?? composedEndpoint; // Dry-run mode: preview the request that would be sent if (flags["dry-run"]) { @@ -1781,12 +1878,7 @@ export const apiCommand = buildCommand({ // The shared authenticated fetch validates requestBaseUrl against the // token host and registered regional origins before adding credentials. - const response = await rawApiRequest(requestEndpoint, { - method: flags.method, - body, - headers, - baseUrl: requestBaseUrl, - }); + const response = await rawApiRequest(requestEndpoint, request); const isError = isApiErrorStatus(response.status); @@ -1805,6 +1897,49 @@ export const apiCommand = buildCommand({ ); } + let pagination: ApiPaginationMetadata = parseLinkHeader( + response.headers.get("link"), + ); + if (navigation && !isError) { + const envelope = + response.body !== null && + typeof response.body === "object" && + "data" in response.body && + Array.isArray(response.body.data) + ? response.body + : undefined; + const rows = Array.isArray(response.body) + ? response.body + : envelope?.data; + if (Array.isArray(rows)) { + const page = consumePage( + { data: rows, ...pagination }, + navigation.position, + ); + response.body = envelope ? { ...envelope, data: page.data } : page.data; + pagination = { + nextCursor: page.nextCursor, + prevCursor: pagination.prevCursor, + }; + } else if (navigation.position.skip > 0) { + throw new ValidationError( + "Cannot resume a mid-page cursor: response body is not an array or a data envelope", + "cursor", + ); + } + advancePaginationState( + PAGINATION_KEY, + navigation.contextKey, + navigation.direction, + pagination.nextCursor, + navigation.currentCursor ?? "", + ); + pagination.hasPrev = hasPreviousPage( + PAGINATION_KEY, + navigation.contextKey, + ); + } + const output = resolveApiResponseOutput(response, { silent: flags.silent, isTTY: this.stdout.isTTY, @@ -1836,7 +1971,7 @@ export const apiCommand = buildCommand({ // formatter, no trailing newline). Textual JSON responses expose the HTTP // metadata so callers can distinguish an empty success from an error. return yield new CommandOutput( - formatApiResponseOutput(response, output, flags.json), + formatApiResponseOutput(response, output, flags.json, pagination), ); }, }); diff --git a/packages/cli/src/lib/api-client.ts b/packages/cli/src/lib/api-client.ts index ea0746030..84db4836d 100644 --- a/packages/cli/src/lib/api-client.ts +++ b/packages/cli/src/lib/api-client.ts @@ -88,8 +88,11 @@ export { appendSearchParams, autoPaginate, buildSearchParams, + consumePage, + decodePageCursor, isTextualContentType, ORG_FANOUT_CONCURRENCY, + type PagePosition, type PaginatedResponse, parseLinkHeader, rawApiRequest, diff --git a/packages/cli/src/lib/api/infrastructure.ts b/packages/cli/src/lib/api/infrastructure.ts index d3f122d13..0014477a7 100644 --- a/packages/cli/src/lib/api/infrastructure.ts +++ b/packages/cli/src/lib/api/infrastructure.ts @@ -16,8 +16,10 @@ import { import { buildSentryApiUrl } from "@sentry/toolkit-core/api-request"; import { API_MAX_PER_PAGE, + decodePageCursor as decodeCursorPosition, fetchAllPages as fetchEveryPage, type FetchAllPagesFailure, + type PagePosition, paginate as paginateFrom, } from "@sentry/toolkit-core/pagination"; // oxlint-disable-next-line sentry-cli/no-namespace-import -- Sentry SDK recommends namespace import @@ -410,6 +412,16 @@ export const MAX_PAGINATION_PAGES = Math.max( */ export { API_MAX_PER_PAGE }; +export { + consumePage, + type PagePosition, +} from "@sentry/toolkit-core/pagination"; + +/** Decode a shared cursor, reporting unreadable legacy bookmarks through the CLI logger. */ +export function decodePageCursor(raw: string | undefined): PagePosition { + return decodeCursorPosition(raw, (message) => logger.warn(message)); +} + /** * Maximum concurrent API requests when fanning out across organizations or regions. * diff --git a/packages/cli/src/lib/db/pagination.ts b/packages/cli/src/lib/db/pagination.ts index b56811e5d..0f70515cc 100644 --- a/packages/cli/src/lib/db/pagination.ts +++ b/packages/cli/src/lib/db/pagination.ts @@ -39,14 +39,14 @@ type PaginationCursorRow = { * Pagination state for a (command, context) pair. * * The `stack` array contains one entry per visited page: - * - `stack[0]` = `""` (first page, no cursor) + * - `stack[0]` = `""` (first page) or an explicit cursor that began the walk * - `stack[1]` = cursor for page 2 * - `stack[N]` = cursor for page N+1 * * `index` is the page the user is currently viewing (0-based). */ export type PaginationState = { - /** Array of page-start cursors. Index 0 is always `""` (first page). */ + /** Page-start cursors; index 0 is the first visited page, normally `""`. */ stack: string[]; /** Index of the currently viewed page (0-based). */ index: number; @@ -239,18 +239,22 @@ export function resolveCursor( * @param contextKey - Serialized query context * @param direction - Which direction the user navigated * @param nextCursor - Cursor for the page after the one just displayed (undefined if last page) + * @param currentCursor - Exact cursor used for this page, including any bookmark; + * `""` means the first page. Omit to retain the existing stack entry. */ export function advancePaginationState( commandKey: string, contextKey: string, direction: CursorDirection, nextCursor: string | undefined, + currentCursor?: string, ): void { const state = getPaginationState(commandKey, contextKey); if (!state) { // First page ever — initialise the stack - const stack = nextCursor ? ["", nextCursor] : [""]; + const first = currentCursor ?? ""; + const stack = nextCursor ? [first, nextCursor] : [first]; savePaginationState(commandKey, contextKey, { stack, index: 0 }); return; } @@ -260,6 +264,9 @@ export function advancePaginationState( const newIndex = state.index + 1; // Truncate anything beyond the new position (back-then-forward scenario) const stack = state.stack.slice(0, newIndex + 1); + if (currentCursor !== undefined) { + stack[newIndex] = currentCursor; + } if (nextCursor) { stack[newIndex + 1] = nextCursor; } @@ -274,6 +281,9 @@ export function advancePaginationState( const newIndex = Math.max(0, state.index - 1); // Refresh the "next" entry from this position in case data shifted const stack = [...state.stack]; + if (currentCursor !== undefined) { + stack[newIndex] = currentCursor; + } if (nextCursor) { stack[newIndex + 1] = nextCursor; } @@ -286,7 +296,8 @@ export function advancePaginationState( } case "first": { - const stack = nextCursor ? ["", nextCursor] : [""]; + const first = currentCursor ?? ""; + const stack = nextCursor ? [first, nextCursor] : [first]; savePaginationState(commandKey, contextKey, { stack, index: 0 }); break; } diff --git a/packages/cli/test/e2e/api.test.ts b/packages/cli/test/e2e/api.test.ts index a86810c68..8c3817a79 100644 --- a/packages/cli/test/e2e/api.test.ts +++ b/packages/cli/test/e2e/api.test.ts @@ -14,9 +14,11 @@ import { expect, test, } from "vitest"; +import { paginate } from "../../src/lib/api/infrastructure.js"; import { EXIT } from "../../src/lib/errors.js"; -import { createE2EContext, type E2EContext } from "../fixture.js"; +import { createE2EContext, type E2EContext, runCli } from "../fixture.js"; import organizationsFixture from "../fixtures/organizations.json"; +import transactionsFixture from "../fixtures/transactions.json"; import { cleanupTestDir, createTestConfigDir } from "../helpers.js"; import { apiRoutes, TEST_TOKEN } from "../mocks/routes.js"; import { createMockServer, type MockServer } from "../mocks/server.js"; @@ -25,8 +27,20 @@ let testConfigDir: string; let mockServer: MockServer; let ctx: E2EContext; const paginationRequests: URL[] = []; -const nextCursor = "100:1:0"; -const prevCursor = "100:0:1"; +const nextCursor = "1:1:0"; +const prevCursor = "1:0:1"; +const alternateToken = "test-api-other-identity"; +const cursorOrganizations = Array.from({ length: 6 }, (_, index) => ({ + ...organizationsFixture[index % organizationsFixture.length], + id: String(100_001 + index), + slug: `test-org-${index + 1}`, + name: `Test Organization ${index + 1}`, +})); +const cursorTransactions = Array.from({ length: 6 }, (_, index) => ({ + ...transactionsFixture.data[index % transactionsFixture.data.length], + id: String(index + 1).padStart(16, "0"), +})); +let failNextPage = false; beforeAll(async () => { mockServer = createMockServer( @@ -39,9 +53,14 @@ beforeAll(async () => { const url = new URL(req.url); paginationRequests.push(url); const isSecondPage = url.searchParams.get("cursor") === nextCursor; + if (isSecondPage && failNextPage) { + failNextPage = false; + return { status: 403, body: { detail: "Access denied" } }; + } return { body: [organizationsFixture[isSecondPage ? 1 : 0]], headers: { + "Cache-Control": "no-store", Link: `<${serverUrl}/api/0/api-pagination/?cursor=${nextCursor}>; rel="next"; results="${!isSecondPage}"; cursor="${nextCursor}", ` + `<${serverUrl}/api/0/api-pagination/?cursor=${prevCursor}>; rel="previous"; results="${isSecondPage}"; cursor="${prevCursor}"`, @@ -49,6 +68,36 @@ beforeAll(async () => { }; }, }, + { + method: "GET", + path: "/api/0/api-custom-cursor/", + response: (req, _params, serverUrl) => { + const url = new URL(req.url); + paginationRequests.push(url); + const perPage = Number(url.searchParams.get("per_page") ?? "2"); + const page = Number( + url.searchParams.get("cursor")?.split(":")[1] ?? "0", + ); + const start = page * perPage; + const next = `${perPage}:${page + 1}:0`; + const previous = `${perPage}:${page - 1}:1`; + const isEventsEnvelope = + url.searchParams.get("envelope") === "events"; + const rows = isEventsEnvelope + ? cursorTransactions + : cursorOrganizations; + const data = rows.slice(start, start + perPage); + return { + body: isEventsEnvelope ? { ...transactionsFixture, data } : data, + headers: { + "Cache-Control": "no-store", + Link: + `<${serverUrl}/api/0/api-custom-cursor/?cursor=${next}>; rel="next"; results="${start + perPage < rows.length}"; cursor="${next}", ` + + `<${serverUrl}/api/0/api-custom-cursor/?cursor=${previous}>; rel="previous"; results="${page > 0}"; cursor="${previous}"`, + }, + }; + }, + }, { method: "POST", path: "/api/0/api-pagination/", @@ -58,7 +107,7 @@ beforeAll(async () => { }, }, ], - { validTokens: [TEST_TOKEN] }, + { validTokens: [TEST_TOKEN, alternateToken] }, ); await mockServer.start(); }); @@ -69,6 +118,7 @@ afterAll(() => { beforeEach(async () => { paginationRequests.length = 0; + failNextPage = false; testConfigDir = await createTestConfigDir("e2e-api-"); ctx = createE2EContext(testConfigDir, mockServer.url); }); @@ -159,37 +209,279 @@ describe("sentry api", () => { expect(JSON.parse(result.stdout)).toEqual({ method: "GET" }); }); - test("continues one page at a time and preserves pagination with --fields", async () => { + test("navigates next, prev and first while preserving filters and --fields", async () => { await ctx.setAuthToken(TEST_TOKEN); - const args = ["api", "api-pagination/", "--json", "--fields", "name"]; + const args = [ + "api", + "api-pagination/?project=1&project=2", + "-f", + "query=level:error", + "-f", + "per_page=1", + "--json", + "--fields", + "name", + ]; const first = await ctx.run(args); expect(first.exitCode, first.stderr + first.stdout).toBe(0); const firstPage = JSON.parse(first.stdout); - expect(firstPage).toEqual({ + expect(firstPage).toMatchObject({ status: 200, statusText: "OK", - body: [{ name: organizationsFixture[0].name }], - pagination: { nextCursor }, + pagination: { nextCursor, hasPrev: false }, }); + expect(firstPage.body).toEqual([{ name: organizationsFixture[0].name }]); expect(paginationRequests).toHaveLength(1); - const second = await ctx.run([ - ...args, - "--cursor", - firstPage.pagination.nextCursor, - ]); + const second = await ctx.run([...args, "--cursor", "next"]); expect(second.exitCode, second.stderr + second.stdout).toBe(0); - expect(JSON.parse(second.stdout)).toEqual({ + const secondPage = JSON.parse(second.stdout); + expect(secondPage).toMatchObject({ status: 200, statusText: "OK", - body: [{ name: organizationsFixture[1].name }], - pagination: { prevCursor }, + pagination: { prevCursor, hasPrev: true }, }); + expect(secondPage.body).toEqual([{ name: organizationsFixture[1].name }]); + expect(secondPage.pagination.nextCursor).toBeUndefined(); expect(paginationRequests).toHaveLength(2); expect(paginationRequests[1].searchParams.getAll("cursor")).toEqual([ nextCursor, ]); + + for (const [cursor, organization] of [ + ["prev", organizationsFixture[0]], + ["next", organizationsFixture[1]], + ["first", organizationsFixture[0]], + ] as const) { + const page = await ctx.run([ + ...args, + ...(cursor === "first" ? ["-f", "cursor=stale"] : []), + "-c", + cursor, + ]); + expect(page.exitCode, page.stderr + page.stdout).toBe(0); + expect(JSON.parse(page.stdout).body).toEqual([ + { name: organization.name }, + ]); + } + expect(paginationRequests).toHaveLength(5); + expect( + paginationRequests.map((url) => url.searchParams.get("cursor")), + ).toEqual([null, nextCursor, null, nextCursor, null]); + for (const url of paginationRequests) { + expect(url.searchParams.get("query")).toBe("level:error"); + expect(url.searchParams.get("per_page")).toBe("1"); + expect(url.searchParams.getAll("project")).toEqual(["1", "2"]); + } + }); + + test("dry-runs and failed requests do not advance pagination history", async () => { + await ctx.setAuthToken(TEST_TOKEN); + const args = ["api", "api-pagination/", "-f", "per_page=1", "--json"]; + const preview = await ctx.run([...args, "--dry-run"]); + expect(preview.exitCode, preview.stderr + preview.stdout).toBe(0); + const missing = await ctx.run([...args, "--cursor", "next"]); + expect(missing.exitCode).toBe(EXIT.VALIDATION); + expect(paginationRequests).toHaveLength(0); + + const first = await ctx.run(args); + expect(first.exitCode, first.stderr + first.stdout).toBe(0); + const nextPreview = await ctx.run([ + ...args, + "--cursor", + "next", + "--dry-run", + ]); + expect(nextPreview.exitCode, nextPreview.stderr + nextPreview.stdout).toBe( + 0, + ); + expect( + new URL(JSON.parse(nextPreview.stdout).url).searchParams.get("cursor"), + ).toBe(nextCursor); + expect(paginationRequests).toHaveLength(1); + + failNextPage = true; + const failed = await ctx.run([...args, "--cursor", "next"]); + expect(failed.exitCode).toBe(EXIT.OUTPUT_ERROR); + expect(JSON.parse(failed.stdout).status).toBe(403); + const retried = await ctx.run([...args, "--cursor", "next"]); + expect(retried.exitCode, retried.stderr + retried.stdout).toBe(0); + expect(JSON.parse(retried.stdout).body).toEqual([organizationsFixture[1]]); + expect( + paginationRequests.map((url) => url.searchParams.get("cursor")), + ).toEqual([null, nextCursor, nextCursor]); + const exhausted = await ctx.run([...args, "--cursor", "next"]); + expect(exhausted.exitCode).toBe(EXIT.VALIDATION); + expect(paginationRequests).toHaveLength(3); + }); + + test.each(["human output", "silent output"])( + "saves pagination history with %s", + async (output) => { + await ctx.setAuthToken(TEST_TOKEN); + const args = ["api", "api-pagination/", "-f", "per_page=1"]; + const first = await ctx.run( + output === "silent output" ? [...args, "--silent"] : args, + ); + expect(first.exitCode, first.stderr + first.stdout).toBe(0); + if (output === "silent output") { + expect(first.stdout).toBe(""); + } else { + expect(JSON.parse(first.stdout)).toEqual([organizationsFixture[0]]); + } + const second = await ctx.run([...args, "--cursor", "next", "--json"]); + expect(second.exitCode, second.stderr + second.stdout).toBe(0); + expect(JSON.parse(second.stdout).body).toEqual([organizationsFixture[1]]); + expect(paginationRequests).toHaveLength(2); + }, + ); + + test.each(["query", "host", "identity"])( + "does not reuse pagination history after changing the %s", + async (change) => { + const args = ["api", "api-pagination/", "-f", "per_page=1", "--json"]; + const env = { + SENTRY_AUTH_TOKEN: TEST_TOKEN, + SENTRY_TOKEN: "", + SENTRY_CONFIG_DIR: testConfigDir, + SENTRY_URL: mockServer.url, + SENTRY_CLI_NO_TELEMETRY: "1", + }; + const first = await runCli(args, { env }); + expect(first.exitCode, first.stderr + first.stdout).toBe(0); + const changed = await runCli( + [ + ...args, + "--cursor", + "next", + ...(change === "query" ? ["-f", "query=other"] : []), + ], + { + env: { + ...env, + SENTRY_URL: + change === "host" + ? mockServer.url.replace("localhost", "127.0.0.1") + : mockServer.url, + SENTRY_AUTH_TOKEN: + change === "identity" ? alternateToken : TEST_TOKEN, + }, + }, + ); + expect(changed.exitCode).toBe(EXIT.VALIDATION); + expect(paginationRequests).toHaveLength(1); + const resumed = await runCli([...args, "--cursor", "next"], { env }); + expect(resumed.exitCode, resumed.stderr + resumed.stdout).toBe(0); + expect(JSON.parse(resumed.stdout).body).toEqual([ + organizationsFixture[1], + ]); + expect(paginationRequests).toHaveLength(2); + }, + ); + + test.each(["shared", "legacy"])( + "resumes a %s mid-page cursor without repeating or skipping rows", + async (format) => { + await ctx.setAuthToken(TEST_TOKEN); + const nativePage = await paginate({ limit: 1, cursor: "2:1:0" }, () => + Promise.resolve({ + data: cursorOrganizations.slice(2, 4), + nextCursor: "2:2:0", + }), + ); + expect(nativePage.nextCursor).toBeDefined(); + const cursor = + format === "legacy" ? "cli1~2~1~2:1:0" : String(nativePage.nextCursor); + const args = ["api", "api-custom-cursor/", "-f", "per_page=9", "--json"]; + const preview = await ctx.run([...args, "--cursor", cursor, "--dry-run"]); + expect(preview.exitCode, preview.stderr + preview.stdout).toBe(0); + const url = new URL(JSON.parse(preview.stdout).url); + expect(url.searchParams.get("per_page")).toBe("2"); + expect(url.searchParams.get("cursor")).toBe("2:1:0"); + expect(paginationRequests).toHaveLength(0); + + const resumed = await ctx.run([...args, "--cursor", cursor]); + expect(resumed.exitCode, resumed.stderr + resumed.stdout).toBe(0); + const resumedPage = JSON.parse(resumed.stdout); + expect(resumedPage.body).toEqual(cursorOrganizations.slice(3, 4)); + expect(resumedPage.pagination.nextCursor).toBe("2:2:0"); + expect(resumedPage.pagination.hasPrev).toBe(false); + const next = await ctx.run([...args, "--cursor", "next"]); + expect(next.exitCode, next.stderr + next.stdout).toBe(0); + const nextPage = JSON.parse(next.stdout); + expect(nextPage.body).toEqual(cursorOrganizations.slice(4)); + expect(nextPage.pagination.hasPrev).toBe(true); + expect([ + ...nativePage.data, + ...resumedPage.body, + ...nextPage.body, + ]).toEqual(cursorOrganizations.slice(2)); + + const previous = await ctx.run([...args, "--cursor", "prev"]); + expect(previous.exitCode, previous.stderr + previous.stdout).toBe(0); + const previousPage = JSON.parse(previous.stdout); + expect(previousPage.body).toEqual(cursorOrganizations.slice(3, 4)); + expect(previousPage.pagination.hasPrev).toBe(false); + expect( + paginationRequests.map((request) => request.searchParams.get("cursor")), + ).toEqual(["2:1:0", "2:2:0", "2:1:0"]); + expect( + paginationRequests.every( + (request) => request.searchParams.get("per_page") === "2", + ), + ).toBe(true); + }, + ); + + test("resumes a shared cursor in an Events data envelope while preserving metadata", async () => { + await ctx.setAuthToken(TEST_TOKEN); + const nativePage = await paginate({ limit: 1, cursor: "2:1:0" }, () => + Promise.resolve({ + data: cursorTransactions.slice(2, 4), + nextCursor: "2:2:0", + }), + ); + expect(nativePage.nextCursor).toBeDefined(); + const args = [ + "api", + "api-custom-cursor/?envelope=events", + "-f", + "per_page=9", + "--json", + ]; + + for (const [cursor, data, hasPrev] of [ + [String(nativePage.nextCursor), cursorTransactions.slice(3, 4), false], + ["next", cursorTransactions.slice(4), true], + ["prev", cursorTransactions.slice(3, 4), false], + ] as const) { + const result = await ctx.run([...args, "--cursor", cursor]); + expect(result.exitCode, result.stderr + result.stdout).toBe(0); + const page = JSON.parse(result.stdout); + expect(page.body).toEqual({ ...transactionsFixture, data }); + expect(page.pagination.hasPrev).toBe(hasPrev); + } + + expect( + paginationRequests.map((request) => request.searchParams.get("cursor")), + ).toEqual(["2:1:0", "2:2:0", "2:1:0"]); + expect( + paginationRequests.every( + (request) => request.searchParams.get("per_page") === "2", + ), + ).toBe(true); + }); + + test("rejects mid-page bookmarks on object responses without saving history", async () => { + await ctx.setAuthToken(TEST_TOKEN); + const args = ["api", "organizations/test-org/", "--json"]; + const result = await ctx.run([...args, "--cursor", "cli1~2~1~"]); + expect(result.exitCode).toBe(EXIT.VALIDATION); + expect(result.stderr + result.stdout).toContain("not an array"); + const next = await ctx.run([...args, "--cursor", "next"]); + expect(next.exitCode).toBe(EXIT.VALIDATION); + expect(next.stderr + next.stdout).toContain("No next page saved"); }); test.each([ diff --git a/packages/cli/test/e2e/library.test.ts b/packages/cli/test/e2e/library.test.ts index 075cd8a56..5f7e59fec 100644 --- a/packages/cli/test/e2e/library.test.ts +++ b/packages/cli/test/e2e/library.test.ts @@ -206,7 +206,7 @@ describe("library mode (bundled)", () => { // --- Typed SDK --- - test("sdk.api passes an opaque cursor and returns page metadata", async () => { + test("sdk.api accepts an opaque cursor and continues with shared navigation", async () => { const { stdout } = await runNodeScriptOk(` const { createSentrySDK } = require('./dist/index.cjs'); const requests = []; @@ -224,12 +224,13 @@ describe("library mode (bundled)", () => { url: 'http://localhost:9000', cwd: ${JSON.stringify(getConfigDir())}, }); - sdk.api({ + const options = { endpoint: 'organizations/?cursor=old', - cursor: 'page-token+/=:1', rawField: ['field=id', 'field=name'], - }).then(response => { - console.log(JSON.stringify({ requests, response })); + }; + sdk.api({ ...options, cursor: 'page-token+/=:1' }).then(async response => { + const next = await sdk.api({ ...options, cursor: 'next' }); + console.log(JSON.stringify({ requests, response, next })); }).catch(error => { console.error(error); process.exitCode = 1; @@ -237,7 +238,7 @@ describe("library mode (bundled)", () => { `); const result = JSON.parse(stdout.trim()); - expect(result.requests).toHaveLength(1); + expect(result.requests).toHaveLength(2); const request = new URL(result.requests[0]); expect(request.searchParams.getAll("cursor")).toEqual(["page-token+/=:1"]); expect(request.searchParams.getAll("field")).toEqual(["id", "name"]); @@ -245,8 +246,17 @@ describe("library mode (bundled)", () => { status: 200, statusText: "OK", body: [organizationsFixture[1]], - pagination: { nextCursor: "100:2:0", prevCursor: "100:0:1" }, + pagination: { + nextCursor: "100:2:0", + prevCursor: "100:0:1", + hasPrev: false, + }, }); + const nextRequest = new URL(result.requests[1]); + expect(nextRequest.searchParams.getAll("cursor")).toEqual(["100:2:0"]); + expect(nextRequest.searchParams.get("per_page")).toBe("100"); + expect(nextRequest.searchParams.getAll("field")).toEqual(["id", "name"]); + expect(result.next.pagination.hasPrev).toBe(true); }); test("sourcemap upload rejects a split token without leaking it or fetching", async () => { diff --git a/packages/cli/test/lib/paginate.property.test.ts b/packages/cli/test/lib/paginate.property.test.ts index b5e74a3c5..c522da139 100644 --- a/packages/cli/test/lib/paginate.property.test.ts +++ b/packages/cli/test/lib/paginate.property.test.ts @@ -20,6 +20,8 @@ import { import { describe, expect, test } from "vitest"; import { API_MAX_PER_PAGE, + consumePage, + decodePageCursor, type PaginatedResponse, paginate, } from "../../src/lib/api/infrastructure.js"; @@ -120,6 +122,66 @@ const limitsArb = array(integer({ min: 1, max: API_MAX_PER_PAGE * 4 }), { maxLength: 4, }); +test("property: a one-page reader carries a bookmark across smaller server pages", async () => { + await fcAssert( + asyncProperty( + integer({ min: 1, max: API_MAX_PER_PAGE - 1 }), + nat(API_MAX_PER_PAGE * 2), + async (skip, remaining) => { + const total = skip + remaining; + const fetchPage = fixedPageServer(total); + const seen: number[] = []; + let cursor: string | undefined = `cli1~100~${skip}~`; + for (let call = 0; call <= total; call += 1) { + const position = decodePageCursor(cursor); + const result = consumePage( + await fetchPage(position.perPage ?? 100, position.cursor), + position, + ); + seen.push(...result.data); + cursor = result.nextCursor; + if (!cursor) { + break; + } + } + expect(cursor).toBeUndefined(); + expect(seen).toEqual( + Array.from({ length: remaining }, (_, index) => skip + index), + ); + }, + ), + { numRuns: DEFAULT_NUM_RUNS }, + ); +}); + +test("property: a legacy bookmark keeps its skip across opaque server cursors", async () => { + await fcAssert( + asyncProperty( + integer({ min: 1, max: 20 }), + integer({ min: 1, max: 5 }), + integer({ min: 1, max: 20 }), + async (pageSize, skippedPages, limit) => { + const skip = pageSize * skippedPages + 1; + const result = await paginate( + { cursor: `cli1~${pageSize}~${skip}~page-0`, limit }, + (perPage, cursor) => { + expect(perPage).toBe(pageSize); + const offset = Number(cursor?.slice("page-".length)); + return Promise.resolve({ + data: Array.from({ length: pageSize }, (_, i) => offset + i), + nextCursor: `page-${offset + pageSize}`, + }); + }, + ); + expect(result.data).toEqual( + Array.from({ length: limit }, (_, i) => skip + i), + ); + }, + ), + { numRuns: DEFAULT_NUM_RUNS }, + ); +}); + describe.each(paginators)("property: paginate over $name", ({ make }) => { test("fills the limit on a fresh walk", async () => { await fcAssert( diff --git a/packages/toolkit-core/README.md b/packages/toolkit-core/README.md index 6216e65b5..d8c14966a 100644 --- a/packages/toolkit-core/README.md +++ b/packages/toolkit-core/README.md @@ -32,6 +32,10 @@ the page cap, receives warnings, and turns failures into its own errors. The CLI's `paginate` and `fetchAllPages` in `packages/cli/src/lib/api/infrastructure.ts` are thin wrappers over it. +`decodePageCursor` and `consumePage` expose the same resume logic for callers +that read one server page at a time, such as `sentry api`. They restore the +cursor's page size and remove rows already returned without fetching more pages. + ## Response cache `response-cache` holds the cache policy for Sentry API GET responses. diff --git a/packages/toolkit-core/src/pagination.ts b/packages/toolkit-core/src/pagination.ts index 604ed68cc..81e4606cd 100644 --- a/packages/toolkit-core/src/pagination.ts +++ b/packages/toolkit-core/src/pagination.ts @@ -40,9 +40,12 @@ const CLI_CURSOR_PREFIX = "cli1~"; * page, the page size it must be fetched with, and how many of that page's * rows were already returned. */ -type PagePosition = { +export type PagePosition = { + /** Server cursor to request; undefined starts at the first page. */ cursor: string | undefined; - perPage: number; + /** Required page size in rows, when encoded in the cursor; otherwise unspecified. */ + perPage?: number; + /** Number of rows already returned from this position; zero starts at its first row. */ skip: number; }; @@ -92,12 +95,12 @@ function encodePageCursor(cursor: string | undefined, skip: number): string { * A damaged `cli1~` cursor is ignored with a warning and the walk starts from * the first page, so it never fails a call or reaches the API. */ -function decodePageCursor( +export function decodePageCursor( raw: string | undefined, - warn: Warn, -): Partial & { cursor: string | undefined } { + warn: Warn = () => {}, +): PagePosition { if (!raw) { - return { cursor: undefined }; + return { cursor: undefined, skip: 0 }; } if (raw.startsWith(CLI_CURSOR_PREFIX)) { return decodeCliCursor(raw, warn); @@ -112,10 +115,7 @@ function decodePageCursor( return perPage === undefined ? position : { ...position, perPage }; } -function decodeCliCursor( - raw: string, - warn: Warn, -): Partial & { cursor: string | undefined } { +function decodeCliCursor(raw: string, warn: Warn): PagePosition { const [perPage, skip, ...server] = raw .slice(CLI_CURSOR_PREFIX.length) .split("~"); @@ -136,7 +136,44 @@ function decodeCliCursor( warn( "Ignoring an unreadable pagination cursor; starting from the first page.", ); - return { cursor: undefined }; + return { cursor: undefined, skip: 0 }; +} + +/** + * Consume one fetched page from a decoded cursor position without making requests. + * + * Removes rows already delivered by a previous call. Sentry next-page cursors + * record any unconsumed rows or remaining skip; other cursor shapes are returned + * unchanged and may replay rows on resume. + * + * @param page - Server page fetched using the position's cursor and page size + * @param position - Decoded continuation for this page + * @param limit - Maximum rows to return, a non-negative integer; omitted means all remaining rows + * @returns Remaining rows and continuation metadata + */ +export function consumePage( + page: Page, + position: PagePosition, + limit = Number.POSITIVE_INFINITY, +): Page { + const unseen = page.data.slice(position.skip); + if (unseen.length > limit) { + return { + ...page, + data: unseen.slice(0, limit), + nextCursor: encodePageCursor(position.cursor, position.skip + limit), + }; + } + return { + ...page, + data: unseen, + nextCursor: page.nextCursor + ? encodePageCursor( + page.nextCursor, + Math.max(0, position.skip - page.data.length), + ) + : undefined, + }; } /** @@ -221,41 +258,36 @@ export async function paginate( ) => Promise>, ): Promise> { const { limit, maxPages, warn = () => {} } = options; - const start = decodePageCursor(options.cursor, warn); - let cursor = start.cursor; - let skip = start.skip ?? 0; + let position = decodePageCursor(options.cursor, warn); // Cursors without a page size accept any, so fetch the skipped rows too. - const perPage = start.perPage ?? pageSizeForLimit(limit + skip); + const perPage = position.perPage ?? pageSizeForLimit(limit + position.skip); const rows: T[] = []; for (let page = 0; page < maxPages; page += 1) { - const { data, nextCursor } = await fetchPage(perPage, cursor); - const unseen = data.slice(skip); - const remaining = limit - rows.length; - - if (unseen.length > remaining) { - rows.push(...unseen.slice(0, remaining)); + const serverPage = await fetchPage(perPage, position.cursor); + const result = consumePage(serverPage, position, limit - rows.length); + rows.push(...result.data); + if (!result.nextCursor) { + return { data: rows, stoppedAtPageCap: false }; + } + if (rows.length === limit) { return { data: rows, - nextCursor: encodePageCursor(cursor, skip + remaining), + nextCursor: result.nextCursor, stoppedAtPageCap: false, }; } - rows.push(...unseen); - if (!nextCursor) { - return { data: rows, stoppedAtPageCap: false }; - } - cursor = nextCursor; - // A resumed walk can fetch pages smaller than the rows it must skip. - skip = Math.max(0, skip - data.length); - if (rows.length === limit) { - break; - } + // Opaque server cursors cannot encode a skip; keep it in memory while walking. + position = { + ...position, + cursor: serverPage.nextCursor, + skip: Math.max(0, position.skip - serverPage.data.length), + }; } return { data: rows, - nextCursor: encodePageCursor(cursor, skip), + nextCursor: encodePageCursor(position.cursor, position.skip), stoppedAtPageCap: rows.length < limit, }; }