diff --git a/.changeset/batch-storage-reads.md b/.changeset/batch-storage-reads.md new file mode 100644 index 0000000..8015d5d --- /dev/null +++ b/.changeset/batch-storage-reads.md @@ -0,0 +1,14 @@ +--- +'@thatopen/services': minor +--- + +Add batch read methods so a page of files, or a model's tiles, can be hydrated in one request instead of one per id. + +- `getHiddenFileSignedUrlsBatch(hiddenIds, expiresIn?)` — signs many hidden files at once. This is the call tile-based viewers (splats, point clouds) should use; minting one URL per tile as the camera moves is what pushes a single session past the rate limit. +- `listVersionsBatch(itemIds, { archived? })` — versions for many items. The records carry their metadata, so a list that only needs metadata does not need a second call. +- `getFileVersionMetadataBatch(entries, { withDraft? })` — metadata for many `{ itemId, versionTag }` pairs. +- `getFoldersBatch(folderIds)` — resolves a known set of folder ids. + +All four split inputs longer than `STORAGE_BATCH_MAX` (100) into several requests automatically, return entries in request order, and mark an id the caller cannot read with an `error` instead of failing the whole batch. + +Requires the matching backend endpoints (`POST /item/hidden/signed-url/batch`, `POST /item/batch/versions`, `POST /item/batch/version-metadata`, `POST /item/batch/folders`). diff --git a/docs/client/paths.json b/docs/client/paths.json index e4155b9..4c870ed 100644 --- a/docs/client/paths.json +++ b/docs/client/paths.json @@ -3,6 +3,10 @@ "path": "src/core/examples/apps.ts", "description": "App lifecycle — list, get with versions, create, download bundle, and archive." }, + { + "path": "src/core/examples/batch-reads.ts", + "description": "Batch reads — sign many hidden files, list versions and metadata for many files, and resolve many folders in one request each." + }, { "path": "src/core/examples/components.ts", "description": "Cloud component lifecycle — list, get with versions, create, update, download bundle, archive, and recover." diff --git a/src/core/client.test.ts b/src/core/client.test.ts index b6f0c18..bc5ef50 100644 --- a/src/core/client.test.ts +++ b/src/core/client.test.ts @@ -397,7 +397,9 @@ describe('EngineServicesClient — HTTP contract', () => { }); it('encodes URL-unsafe characters in itemId and versionTag', async () => { - fetchMock.mockResolvedValue(okResponse({ tag: 'v1?bug', archived: true })); + fetchMock.mockResolvedValue( + okResponse({ tag: 'v1?bug', archived: true }), + ); const client = new EngineServicesClient(TOKEN, API); await client.archiveVersion('item/with slash', 'v1?bug'); const { url } = getCall(fetchMock); @@ -437,4 +439,138 @@ describe('EngineServicesClient — HTTP contract', () => { }); }); }); + + describe('batch reads', () => { + it('getHiddenFileSignedUrlsBatch POSTs the ids and unwraps results', async () => { + const results = [ + { hiddenFileId: 'h1', url: 'https://s3/h1', expiresAt: 'later' }, + { hiddenFileId: 'h2', error: { status: 404, message: 'gone' } }, + ]; + fetchMock.mockResolvedValue(okResponse({ results })); + const client = new EngineServicesClient(TOKEN, API); + + const entries = await client.getHiddenFileSignedUrlsBatch( + ['h1', 'h2'], + 1200, + ); + + const { url, init } = getCall(fetchMock); + const { pathname } = parseUrl(url); + expect(init.method).toBe('POST'); + expect(pathname).toBe('/api/item/hidden/signed-url/batch'); + expect(JSON.parse(init.body as string)).toEqual({ + hiddenFileIds: ['h1', 'h2'], + expiresIn: 1200, + }); + expect(entries).toEqual(results); + }); + + it('getHiddenFileSignedUrlsBatch omits expiresIn when not given', async () => { + fetchMock.mockResolvedValue(okResponse({ results: [] })); + const client = new EngineServicesClient(TOKEN, API); + await client.getHiddenFileSignedUrlsBatch(['h1']); + const { init } = getCall(fetchMock); + expect(JSON.parse(init.body as string)).toEqual({ + hiddenFileIds: ['h1'], + }); + }); + + it('splits inputs above the batch ceiling into several requests', async () => { + const ids = Array.from({ length: 250 }, (_, index) => `h${index}`); + fetchMock.mockImplementation(async (_url: string, init: RequestInit) => { + const { hiddenFileIds } = JSON.parse(init.body as string); + return okResponse({ + results: hiddenFileIds.map((hiddenFileId: string) => ({ + hiddenFileId, + url: `https://s3/${hiddenFileId}`, + })), + }); + }); + const client = new EngineServicesClient(TOKEN, API); + + const entries = await client.getHiddenFileSignedUrlsBatch(ids); + + expect(fetchMock).toHaveBeenCalledTimes(3); + expect(entries).toHaveLength(250); + expect(entries.map((entry) => entry.hiddenFileId)).toEqual(ids); + const sizes = fetchMock.mock.calls.map( + (call) => + JSON.parse((call[1] as RequestInit).body as string).hiddenFileIds + .length, + ); + expect(sizes).toEqual([100, 100, 50]); + }); + + it('does not call the API for an empty input', async () => { + const client = new EngineServicesClient(TOKEN, API); + await expect(client.getHiddenFileSignedUrlsBatch([])).resolves.toEqual( + [], + ); + await expect(client.listVersionsBatch([])).resolves.toEqual([]); + await expect(client.getFileVersionMetadataBatch([])).resolves.toEqual([]); + await expect(client.getFoldersBatch([])).resolves.toEqual([]); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('listVersionsBatch POSTs /item/batch/versions and forwards archived', async () => { + fetchMock.mockResolvedValue(okResponse({ results: [] })); + const client = new EngineServicesClient(TOKEN, API); + + await client.listVersionsBatch(['item-1', 'item-2'], { archived: false }); + + const { url, init } = getCall(fetchMock); + expect(parseUrl(url).pathname).toBe('/api/item/batch/versions'); + expect(JSON.parse(init.body as string)).toEqual({ + itemIds: ['item-1', 'item-2'], + archived: false, + }); + }); + + it('listVersionsBatch omits archived when not provided', async () => { + fetchMock.mockResolvedValue(okResponse({ results: [] })); + const client = new EngineServicesClient(TOKEN, API); + await client.listVersionsBatch(['item-1']); + expect(JSON.parse(getCall(fetchMock).init.body as string)).toEqual({ + itemIds: ['item-1'], + }); + }); + + it('getFileVersionMetadataBatch POSTs the pairs', async () => { + fetchMock.mockResolvedValue(okResponse({ results: [] })); + const client = new EngineServicesClient(TOKEN, API); + + await client.getFileVersionMetadataBatch( + [{ itemId: 'item-1', versionTag: 'v1' }], + { withDraft: true }, + ); + + const { url, init } = getCall(fetchMock); + expect(parseUrl(url).pathname).toBe('/api/item/batch/version-metadata'); + expect(JSON.parse(init.body as string)).toEqual({ + entries: [{ itemId: 'item-1', versionTag: 'v1' }], + withDraft: true, + }); + }); + + it('getFoldersBatch POSTs the folder ids', async () => { + fetchMock.mockResolvedValue(okResponse({ results: [] })); + const client = new EngineServicesClient(TOKEN, API); + + await client.getFoldersBatch(['folder-1']); + + const { url, init } = getCall(fetchMock); + expect(parseUrl(url).pathname).toBe('/api/item/batch/folders'); + expect(JSON.parse(init.body as string)).toEqual({ + folderIds: ['folder-1'], + }); + }); + + it('propagates a failing chunk as a RequestError', async () => { + fetchMock.mockResolvedValue(errorResponse(429, 'Too Many Requests')); + const client = new EngineServicesClient(TOKEN, API); + await expect( + client.getHiddenFileSignedUrlsBatch(['h1']), + ).rejects.toMatchObject({ status: 429 }); + }); + }); }); diff --git a/src/core/client.ts b/src/core/client.ts index 1835771..755e371 100644 --- a/src/core/client.ts +++ b/src/core/client.ts @@ -22,6 +22,14 @@ import { HiddenFileEntity, Metadata, } from '../types/files'; +import { + HiddenFileSignedUrlBatchEntry, + ItemFoldersBatchEntry, + ItemVersionsBatchEntry, + STORAGE_BATCH_MAX, + VersionMetadataBatchEntry, + VersionMetadataBatchRequest, +} from '../types/batch'; import { ThatOpenContext } from '../types/context'; import { NpmCredentials } from '../types/npm'; import { RequestError } from './request-error'; @@ -203,10 +211,9 @@ export class EngineServicesClient { static fromPlatformContext( props?: Omit, ): EngineServicesClient { - const ctx: ThatOpenContext = - (typeof window !== 'undefined' - ? window.__THATOPEN_CONTEXT__ - : null) || { appId: '', projectId: '', accessToken: '', apiUrl: '' }; + const ctx: ThatOpenContext = (typeof window !== 'undefined' + ? window.__THATOPEN_CONTEXT__ + : null) || { appId: '', projectId: '', accessToken: '', apiUrl: '' }; const client = new EngineServicesClient(ctx.accessToken, ctx.apiUrl, { ...props, useBearer: true, @@ -582,6 +589,43 @@ export class EngineServicesClient { ); } + /** + * Batch variant of {@link getFileVersionMetadata}. Reads the metadata of + * many (file, version) pairs in one request instead of one request per pair, + * which is what a file list needs when it hydrates a whole page at once. + * + * Inputs longer than {@link STORAGE_BATCH_MAX} are split into several + * requests automatically. Entries come back in the order they were asked + * for; a pair the caller cannot read carries an `error` instead of + * `metadata`, so one bad id does not lose the whole page. + * + * @param entries - The `{ itemId, versionTag }` pairs to read. + * @param params - Optional flags such as `withDraft`. + */ + async getFileVersionMetadataBatch( + entries: VersionMetadataBatchRequest[], + params?: { withDraft?: boolean }, + ): Promise { + const { withDraft } = params || {}; + return await this.#requestBatches( + entries, + async (chunk) => + ( + await this.#requestApi<{ results: VersionMetadataBatchEntry[] }>( + 'POST', + `${ITEM_PATH}/batch/version-metadata`, + { + body: JSON.stringify({ + entries: chunk, + ...(withDraft !== undefined && { withDraft }), + }), + contentType: 'application/json', + }, + ) + ).results, + ); + } + // ─── Folders ───────────────────────────────────────────────────── /** @@ -619,6 +663,36 @@ export class EngineServicesClient { ); } + /** + * Batch variant of {@link getFolder}. Resolves a known set of folder ids in + * one request — for example the parents behind a file list's breadcrumbs. + * + * Reach for {@link listFolders} instead when the whole (project) tree is + * wanted; this is for the case where the ids are already known. + * + * Inputs longer than {@link STORAGE_BATCH_MAX} are split into several + * requests automatically. A folder the caller cannot read carries an + * `error` instead of `folder`. + * + * @param folderIds - The folder ids to resolve. + */ + async getFoldersBatch(folderIds: string[]): Promise { + return await this.#requestBatches( + folderIds, + async (chunk) => + ( + await this.#requestApi<{ results: ItemFoldersBatchEntry[] }>( + 'POST', + `${ITEM_PATH}/batch/folders`, + { + body: JSON.stringify({ folderIds: chunk }), + contentType: 'application/json', + }, + ) + ).results, + ); + } + /** * Creates a new folder. * @param name - Display name for the folder. @@ -924,7 +998,8 @@ export class EngineServicesClient { globals: Record, ...builtIns: { uuid: string }[] ): Promise<{ components: TComponents }> { - const OBC = globals.OBC as { Components?: new () => TComponents } | undefined; + const OBC = globals.OBC as + { Components?: new () => TComponents } | undefined; const BUI = globals.BUI as { Manager?: { init(): void } } | undefined; if (!OBC?.Components) throw new Error('globals.OBC must include Components'); @@ -1011,15 +1086,12 @@ export class EngineServicesClient { */ async downloadAppBundle(appId: string, params?: DownloadItemFileParams) { const { versionTag, withDraft } = params || {}; - return await this.#requestFile( - `${ITEM_PATH}/${appId}/download/bundle`, - { - query: { - ...(versionTag && { versionTag }), - ...(withDraft && { withDraft }), - }, + return await this.#requestFile(`${ITEM_PATH}/${appId}/download/bundle`, { + query: { + ...(versionTag && { versionTag }), + ...(withDraft && { withDraft }), }, - ); + }); } /** @@ -1077,7 +1149,9 @@ export class EngineServicesClient { }); if (!response.ok) { const text = await response.text().catch(() => ''); - throw new Error(`Local server request failed: ${response.status} - ${text}`); + throw new Error( + `Local server request failed: ${response.status} - ${text}`, + ); } return (await response.json()) as { executionId: string }; } @@ -1102,7 +1176,9 @@ export class EngineServicesClient { const response = await fetch(url, { method: 'POST' }); if (!response.ok) { const text = await response.text().catch(() => ''); - throw new Error(`Local server request failed: ${response.status} - ${text}`); + throw new Error( + `Local server request failed: ${response.status} - ${text}`, + ); } return (await response.json()) as ExecutionEntity; } @@ -1156,7 +1232,9 @@ export class EngineServicesClient { const response = await fetch(url); if (!response.ok) { const text = await response.text().catch(() => ''); - throw new Error(`Local server request failed: ${response.status} - ${text}`); + throw new Error( + `Local server request failed: ${response.status} - ${text}`, + ); } return (await response.json()) as ExecutionEntity; } @@ -1298,6 +1376,47 @@ export class EngineServicesClient { ); } + /** + * Batch variant of {@link getHiddenFileSignedUrl}. Mints presigned URLs for + * many hidden files in one request. + * + * This is the right call for tile-based viewers (splats, point clouds), + * which mint a URL per piece as the camera moves. One request here replaces + * up to {@link STORAGE_BATCH_MAX} single calls, which keeps a busy session + * well under the per-endpoint rate limit. + * + * Inputs longer than {@link STORAGE_BATCH_MAX} are split into several + * requests automatically. Entries come back in request order; a file that + * is gone or not accessible carries an `error` instead of a `url`, so the + * remaining tiles still load. Re-call to re-mint as `expiresAt` approaches. + * + * @param hiddenIds - The hidden files to sign. + * @param expiresIn - Desired URL lifetime in seconds (60–3600). Defaults to + * 900 (15 min) server-side; values are clamped to that range. + */ + async getHiddenFileSignedUrlsBatch( + hiddenIds: string[], + expiresIn?: number, + ): Promise { + return await this.#requestBatches( + hiddenIds, + async (chunk) => + ( + await this.#requestApi<{ results: HiddenFileSignedUrlBatchEntry[] }>( + 'POST', + `${ITEM_PATH}/${HIDDEN_PATH}/signed-url/batch`, + { + body: JSON.stringify({ + hiddenFileIds: chunk, + ...(expiresIn != null && { expiresIn }), + }), + contentType: 'application/json', + }, + ) + ).results, + ); + } + /** * Lists all hidden files attached to a parent item. * @param parentFileId - The parent item's unique identifier. @@ -1333,11 +1452,9 @@ export class EngineServicesClient { async uploadItemIcon(itemId: string, icon: File | Blob) { const formData = new FormData(); formData.append('icon', icon); - return await this.#requestApi( - 'PUT', - `${ITEM_PATH}/${itemId}/icon`, - { body: formData }, - ); + return await this.#requestApi('PUT', `${ITEM_PATH}/${itemId}/icon`, { + body: formData, + }); } /** @@ -1426,6 +1543,45 @@ export class EngineServicesClient { ); } + /** + * Batch variant of {@link listVersions}. Lists the versions of many items in + * one request, which is what a file panel needs to render a whole page. + * + * The returned versions are the full records, metadata included, so a list + * that only needs version metadata does not need + * {@link getFileVersionMetadataBatch} on top. + * + * Inputs longer than {@link STORAGE_BATCH_MAX} are split into several + * requests automatically. An item the caller cannot read carries an `error` + * instead of `versions`. + * + * @param itemIds - The items whose versions to list. + * @param params - Optional `{ archived }` filter, applied to every item. + */ + async listVersionsBatch( + itemIds: string[], + params: { archived?: boolean } = {}, + ): Promise { + const { archived } = params; + return await this.#requestBatches( + itemIds, + async (chunk) => + ( + await this.#requestApi<{ results: ItemVersionsBatchEntry[] }>( + 'POST', + `${ITEM_PATH}/batch/versions`, + { + body: JSON.stringify({ + itemIds: chunk, + ...(archived !== undefined && { archived }), + }), + contentType: 'application/json', + }, + ) + ).results, + ); + } + /** * Archives a version of an item. Archived versions remain available via * `listVersions({ archived: true })` and can be recovered or permanently @@ -1482,6 +1638,27 @@ export class EngineServicesClient { // ─── Private Helpers ───────────────────────────────────────────── + /** + * Splits `inputs` into chunks the API accepts and concatenates the results, + * so callers never have to think about the batch ceiling. Chunks go out + * together; an empty input skips the network entirely, since the API + * rejects an empty batch. + */ + async #requestBatches( + inputs: TInput[], + requestChunk: (chunk: TInput[]) => Promise, + ): Promise { + if (!inputs.length) return []; + + const chunks: TInput[][] = []; + for (let start = 0; start < inputs.length; start += STORAGE_BATCH_MAX) { + chunks.push(inputs.slice(start, start + STORAGE_BATCH_MAX)); + } + + const responses = await Promise.all(chunks.map(requestChunk)); + return responses.flat(); + } + async #downloadItem(itemId: string, params?: DownloadItemFileParams) { const { versionTag, withDraft } = params || {}; return await this.#requestFile(`${ITEM_PATH}/${itemId}/download`, { diff --git a/src/core/examples/batch-reads.ts b/src/core/examples/batch-reads.ts new file mode 100644 index 0000000..3787b40 --- /dev/null +++ b/src/core/examples/batch-reads.ts @@ -0,0 +1,99 @@ +// description: "Batch reads — sign many hidden files, list versions and metadata for many files, and resolve many folders in one request each." +import { config } from 'dotenv'; +import { resolve } from 'path'; +import { EngineServicesClient } from '../client'; + +config({ path: resolve(__dirname, '.env') }); + +const ACCESS_TOKEN = process.env.ACCESS_TOKEN; +const API_URL = process.env.API_URL; +const PROJECT_ID = process.env.PROJECT_ID; + +if (!ACCESS_TOKEN || !API_URL) { + throw new Error('ACCESS_TOKEN and API_URL are required in src/core/examples/.env'); +} + +async function main() { + const client = new EngineServicesClient(ACCESS_TOKEN!, API_URL!); + + if (!PROJECT_ID) { + console.log('Set PROJECT_ID in .env to run this example.'); + return; + } + + const files = await client.listFiles({ projectId: PROJECT_ID }); + if (!files.length) { + console.log('No files in this project — nothing to batch.'); + return; + } + const fileIds = files.map((file) => String(file._id)); + + // --- Versions for a whole page of files --- + // One request instead of one per file. Entries come back in request order, + // and a file the token cannot read carries an `error` instead of `versions`. + const versionEntries = await client.listVersionsBatch(fileIds); + console.log('\nVersions per file:'); + for (const entry of versionEntries) { + if (entry.error) { + console.log(` ${entry.itemId} → ${entry.error.status} ${entry.error.message}`); + continue; + } + console.log(` ${entry.itemId} → ${entry.versions?.length ?? 0} version(s)`); + } + + // --- Metadata for specific (file, version) pairs --- + // The versions above already carry their metadata, so this is only needed + // when the version list itself is not wanted. + const pairs = versionEntries + .filter((entry) => entry.versions?.length) + .map((entry) => ({ + itemId: entry.itemId, + versionTag: entry.versions![0].tag, + })); + const metadataEntries = await client.getFileVersionMetadataBatch(pairs); + console.log('\nLatest-version metadata:'); + for (const entry of metadataEntries) { + console.log( + ` ${entry.itemId}@${entry.versionTag} → ${JSON.stringify(entry.metadata ?? entry.error)}`, + ); + } + + // --- Folders by id --- + // Use listFolders({ projectId }) for the whole tree; this is for a known set + // of ids, such as the parents behind a breadcrumb. + const folderIds = [ + ...new Set(files.map((file) => file.folderId).filter(Boolean).map(String)), + ]; + if (folderIds.length) { + const folderEntries = await client.getFoldersBatch(folderIds); + console.log('\nFolders:'); + for (const entry of folderEntries) { + console.log(` ${entry.folderId} → ${entry.folder?.name ?? entry.error?.message}`); + } + } + + // --- Signed URLs for many hidden files --- + // This is the call tile-based viewers (splats, point clouds) should use. + // Minting one URL per tile as the camera moves is what pushes a single + // session past the rate limit; one request covers up to 100 tiles, and + // longer lists are split into several requests automatically. + const hiddenFiles = await client.getHiddenFilesByParent(fileIds[0]); + if (!hiddenFiles.length) { + console.log('\nNo hidden files on the first file — skipping signed URLs.'); + return; + } + const signedEntries = await client.getHiddenFileSignedUrlsBatch( + hiddenFiles.map((hiddenFile) => String(hiddenFile._id)), + 3600, + ); + console.log('\nSigned hidden file URLs:'); + for (const entry of signedEntries) { + if (entry.error) { + console.log(` ${entry.hiddenFileId} → ${entry.error.status} ${entry.error.message}`); + continue; + } + console.log(` ${entry.hiddenFileId} → expires ${entry.expiresAt}`); + } +} + +main().catch(console.error); diff --git a/src/index.ts b/src/index.ts index 6f6bbf0..788fdd1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -5,6 +5,7 @@ export * from './types/items'; export * from './types/base'; export * from './types/execution'; export * from './types/files'; +export * from './types/batch'; export * from './types/response'; export * from './types/item.dto'; export * from './types/projects'; diff --git a/src/types/batch.ts b/src/types/batch.ts new file mode 100644 index 0000000..9c16c3a --- /dev/null +++ b/src/types/batch.ts @@ -0,0 +1,42 @@ +import { Metadata } from './files'; +import { ItemFolder, ItemVersion } from './items'; + +/** + * Maximum number of ids the API accepts in one batch request. The client + * methods split larger inputs into chunks of this size automatically. + */ +export const STORAGE_BATCH_MAX = 100; + +export type BatchError = { + status: number; + message: string; +}; + +export type HiddenFileSignedUrlBatchEntry = { + hiddenFileId: string; + url?: string; + expiresAt?: string; + error?: BatchError; +}; + +export type ItemVersionsBatchEntry = { + itemId: string; + versions?: ItemVersion[]; + error?: BatchError; +}; + +export type VersionMetadataBatchRequest = { + itemId: string; + versionTag: string; +}; + +export type VersionMetadataBatchEntry = VersionMetadataBatchRequest & { + metadata?: Metadata; + error?: BatchError; +}; + +export type ItemFoldersBatchEntry = { + folderId: string; + folder?: ItemFolder; + error?: BatchError; +};