diff --git a/README.md b/README.md index eabe761ac0..1b65e0d83b 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,10 @@ Run `Deepnote: Preview App` from the Command Palette, or right-click a folder in - A run keeps going until its notebook finishes, even after you close the preview. Reloading or closing the VS Code window stops it. - For a legacy `.deepnote` file that holds several notebooks, the first run shows the prompt to split the file, as opening the file does. - `appData()` answers from that notebook's newest successful `run()` in the preview: the first JSON object the run displayed (an `application/json` output), the first agent block's answer, and when the run started. Until a run succeeds, it rejects with Deepnote's message. The preview keeps run results in memory, so reloading or closing the VS Code window clears them. +- `dataframe(name, { pageIndex, pageSize })` pages through the frame stored under that name in `deepnote_dataframes//` in the app's project folder, the folder of the `.deepnote` file nearest at or above the app folder. Pages and values match Deepnote's, but the preview does not apply Deepnote's limits on how much one page may hold. + - Deepnote's client does not have `dataframe()` yet, so the preview adds it to the bundled copy. A published app cannot call it until Deepnote's client does. + - The deepnote-toolkit version the extension installs does not store frames there yet: `store_dataframe(df, name)` writes them to `/work/deepnote_dataframes/`, which every project shares. The preview reads a frame from there when the project folder has none under that name. + - Reading a frame that deepnote-toolkit compressed with zstd needs an editor whose Node.js has zstd (22.15 or later). - Previewing needs a trusted workspace and the desktop app. ### Database integrations diff --git a/build/esbuild/build.ts b/build/esbuild/build.ts index 69097660dd..3e13c8f469 100644 --- a/build/esbuild/build.ts +++ b/build/esbuild/build.ts @@ -211,6 +211,7 @@ function createConfig( ): SameShape { const inject: string[] = []; const isWebTestSource = source.endsWith(path.join('web', 'index.ts')); + const isAppPreviewScript = source.endsWith(path.join('appPreview', 'dataframeShim.ts')); const plugins: Plugin[] = []; let define: SameShape['define'] = undefined; if (target === 'web') { @@ -286,14 +287,15 @@ function createConfig( ); } // Desktop builds use CommonJS for VS Code/Cursor compatibility - // Web builds use ESM for browser compatibility + // Web builds use ESM for browser compatibility, except the dataframe() shim: app pages load it as a classic script, + // where an ESM bundle's top-level names would become globals. const config: SameShape = { entryPoints: [source], outfile, bundle: true, external, alias, - format: target === 'desktop' ? 'cjs' : 'esm', + format: target === 'desktop' ? 'cjs' : isAppPreviewScript ? 'iife' : 'esm', metafile: isDevbuild && !watch, define, target: target === 'desktop' ? 'node18' : 'es2018', @@ -447,6 +449,11 @@ async function buildAll() { path.join(extensionFolder, 'dist', 'webviews', 'webview-side', 'bigNumberComparisonSettings', 'index.js'), { target: 'web', watch: watchAll } ), + build( + path.join(extensionFolder, 'src', 'webviews', 'webview-side', 'appPreview', 'dataframeShim.ts'), + path.join(extensionFolder, 'dist', 'webviews', 'webview-side', 'appPreview', 'dataframeShim.js'), + { target: 'web', watch: watchAll } + ), // Notebook renderers (integrated from jupyter-renderers) build( path.join(extensionFolder, 'src', 'renderers', 'client', 'index.tsx'), diff --git a/cspell.json b/cspell.json index c963a48b60..d30734c548 100644 --- a/cspell.json +++ b/cspell.json @@ -116,7 +116,8 @@ "xanchor", "xaxis", "yanchor", - "yaxis" + "yaxis", + "žluťoučký" ], "useGitignore": true } diff --git a/package-lock.json b/package-lock.json index 5df68e7792..92c9b2ddcc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,6 +40,7 @@ "@vscode/jupyter-ipywidgets8": "^1.0.9", "@vscode/python-extension": "^1.0.4", "ansi-to-html": "^0.6.7", + "apache-arrow": "^21.2.0", "bootstrap": "^5.0.0", "bootstrap-less": "^3.3.8", "clsx": "^2.1.1", @@ -10478,6 +10479,36 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/apache-arrow": { + "version": "21.2.0", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-21.2.0.tgz", + "integrity": "sha512-Hxe6Agq26gQOM954qpzYSllJBPJl+e16U5CkfuMUhLrNba+5nKkttIVlflaovN6oaTratqMGAO8H5u/aNhmHWQ==", + "license": "Apache-2.0", + "dependencies": { + "@types/node": "^25.2.0", + "flatbuffers": "^25.1.24", + "json-with-bigint": "^3.5.3", + "tslib": "^2.6.2" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.js" + } + }, + "node_modules/apache-arrow/node_modules/@types/node": { + "version": "25.9.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.9.9.tgz", + "integrity": "sha512-b4e2xxj/yMeT2hNlD6x7zNFiQK2Dlmkb9CQ6v3/1G2RezKN4YS2lcEv8lzGm1mhs4f2lM4yNAMI/zUDBZBBt7w==", + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/apache-arrow/node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "license": "MIT" + }, "node_modules/append-transform": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/append-transform/-/append-transform-2.0.0.tgz", @@ -17041,6 +17072,12 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, "node_modules/flatted": { "version": "3.4.4", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.4.4.tgz", @@ -20232,7 +20269,6 @@ "version": "3.5.7", "resolved": "https://registry.npmjs.org/json-with-bigint/-/json-with-bigint-3.5.7.tgz", "integrity": "sha512-7ei3MdAI5+fJPVnKlW77TKNKwQ5ppSzWvhPuSuINT/GYW9ZOC1eRKOuhV9yHG5aEsUPj9BBx5JIekkmoLHxZOw==", - "dev": true, "license": "MIT" }, "node_modules/json2csv": { diff --git a/package.json b/package.json index 73f39e1da2..71ff452dca 100644 --- a/package.json +++ b/package.json @@ -2818,6 +2818,7 @@ "@vscode/jupyter-ipywidgets8": "^1.0.9", "@vscode/python-extension": "^1.0.4", "ansi-to-html": "^0.6.7", + "apache-arrow": "^21.2.0", "bootstrap": "^5.0.0", "bootstrap-less": "^3.3.8", "clsx": "^2.1.1", diff --git a/src/notebooks/deepnote/appPreview/appPreviewApi.node.ts b/src/notebooks/deepnote/appPreview/appPreviewApi.node.ts index eb54e64fef..f12d1e6e64 100644 --- a/src/notebooks/deepnote/appPreview/appPreviewApi.node.ts +++ b/src/notebooks/deepnote/appPreview/appPreviewApi.node.ts @@ -13,7 +13,14 @@ import { isObject } from '../../../platform/common/utils/sysTypes'; import { generateUuid } from '../../../platform/common/uuid'; import type { DeepnoteNotebook } from '../../../platform/deepnote/deepnoteTypes'; import { logger } from '../../../platform/logging'; +import type { AppPreviewProject } from './appPreviewSession.node'; import { blocksInNotebookOrder, toNotebookInputs, type WorkspaceNotebook } from './appPreviewWorkspace.node'; +import { + DATAFRAME_NAME_PATTERN, + DATAFRAME_NOT_FOUND_MESSAGE, + DataframePageError, + readStoredDataframePage +} from './storedDataframePage.node'; import type { AppPreviewInputValue, AppPreviewRunOutcome, @@ -22,12 +29,11 @@ import type { SnapshotBlock } from './types'; -export type AppPreviewOperation = 'dataframe()'; - export interface AppPreviewApiOptions { token: string; findNotebook(notebookId: string): Promise; - onUnsupported(operation: AppPreviewOperation): void; + /** The project the previewed app belongs to, read again on every dataframe request. */ + getProject(): AppPreviewProject | undefined; runner: IAppPreviewRunner; } @@ -67,16 +73,22 @@ type AppViewerNotebook = operations['getNotebook']['responses'][200]['content'][ // Each finished run holds all its outputs, and a page may start one on every reload. const MAX_FINISHED_RUNS = 20; +const MAX_DATAFRAME_PAGE_SIZE = 1000; const createRunBodySchema = z.strictObject({ notebookId: z.string().min(1), inputs: z.record(z.string().min(1), z.union([z.string(), z.boolean(), z.array(z.string())])).optional(), detachedRunStorageMode: z.enum(['read_write', 'readonly']).optional() }); - -function notSupportedMessage(operation: AppPreviewOperation): string { - return `${operation} is not supported in the local preview yet.`; -} +// Deepnote also requires a UUID `projectId`, but a `.deepnote` file's project id can be any string. +const dataframeParamsSchema = z.object({ + projectId: z.string(), + name: z.string().regex(DATAFRAME_NAME_PATTERN) +}); +const dataframeQuerySchema = z.strictObject({ + pageIndex: z.coerce.number().int().min(0).default(0), + pageSize: z.coerce.number().int().min(1).max(MAX_DATAFRAME_PAGE_SIZE).default(10) +}); /** The 4xx status of an error body-parser raises for the request itself, such as malformed JSON. */ function clientErrorStatus(error: unknown): number | undefined { @@ -227,21 +239,17 @@ function toAppViewerNotebook({ file, notebook }: WorkspaceNotebook): AppViewerNo * The local stand-in for Deepnote's app viewer API, mounted at `/v2` on the preview server. * Requests without the preview's bearer token get 401. `GET /notebooks/:notebookId` answers from the workspace's * notebooks, and `POST /runs` runs one with `runner`, which `GET /runs/:runId` then reports on. - * `GET /notebooks/:notebookId/app-data` answers from the notebook's newest successful run. The dataframe route, which - * the version of Deepnote's client that adds `dataframe()` calls, gets 501, and every other route gets 403, as Deepnote - * answers app viewer tokens. A body that cannot be read, such as malformed JSON, gets its 4xx; any other failure is - * logged and gets a JSON 500. + * `GET /notebooks/:notebookId/app-data` answers from the notebook's newest successful run, and + * `GET /projects/:projectId/dataframes/:name` from the dataframes stored in the folder of `getProject()`'s project, or + * under `$HOME/work` for a name that folder has none under. + * Every other route gets 403, as Deepnote answers app viewer tokens. A body that cannot be read, such as malformed + * JSON, gets its 4xx; any other failure is logged and gets a JSON 500. */ export function createAppPreviewApi(options: AppPreviewApiOptions): AppPreviewApi { const router = express.Router(); const runs = new Map(); const unfinished = new Set(); - const unsupported = (operation: AppPreviewOperation) => (_req: Request, res: Response) => { - options.onUnsupported(operation); - res.status(501).json({ message: notSupportedMessage(operation) }); - }; - const startRun = (run: PreviewRun, request: AppPreviewRunRequest) => { const cancellation = new CancellationTokenSource(); unfinished.add(cancellation); @@ -358,7 +366,35 @@ export function createAppPreviewApi(options: AppPreviewApiOptions): AppPreviewAp } res.json({ run: { ...run, snapshotExpiresAt: null, snapshotContent: null, snapshotDownloadUrl: null } }); }); - router.get('/projects/:projectId/dataframes/:name', unsupported('dataframe()')); + router.get('/projects/:projectId/dataframes/:name', async (req: Request, res: Response) => { + const params = dataframeParamsSchema.safeParse(req.params); + if (!params.success) { + res.status(400).json({ message: readableErrors(params.error) }); + + return; + } + const query = dataframeQuerySchema.safeParse(req.query); + if (!query.success) { + res.status(400).json({ message: readableErrors(query.error) }); + + return; + } + const project = options.getProject(); + if (!project || project.id !== params.data.projectId) { + res.status(404).json({ message: DATAFRAME_NOT_FOUND_MESSAGE }); + + return; + } + + try { + res.json(await readStoredDataframePage(project.folder, params.data.name, query.data)); + } catch (error) { + if (!(error instanceof DataframePageError)) { + throw error; + } + res.status(error.status).json({ message: error.message }); + } + }); router.use((_req: Request, res: Response) => { res.status(403).json({ message: NOT_AVAILABLE_MESSAGE }); diff --git a/src/notebooks/deepnote/appPreview/appPreviewApi.node.unit.test.ts b/src/notebooks/deepnote/appPreview/appPreviewApi.node.unit.test.ts index ccb9bd43a1..6a015ac99e 100644 --- a/src/notebooks/deepnote/appPreview/appPreviewApi.node.unit.test.ts +++ b/src/notebooks/deepnote/appPreview/appPreviewApi.node.unit.test.ts @@ -1,27 +1,29 @@ import type { IDisplayData, IOutput, IStream } from '@deepnote/runtime-core'; import { assert } from 'chai'; import express from 'express'; +import * as fs from 'fs'; import * as http from 'http'; import type { AddressInfo } from 'net'; +import * as os from 'os'; import * as sinon from 'sinon'; import { anything, capture, instance, mock, verify, when } from 'ts-mockito'; import { Uri } from 'vscode'; +import zlib from 'zlib'; import { createDeferred, type Deferred } from '../../../platform/common/utils/async'; +import { platformUtils } from '../../../platform/common/utils/platform.node'; import { isUUID } from '../../../platform/common/uuid'; import { logger } from '../../../platform/logging'; +import * as path from '../../../platform/vscode-path/path'; +import { EXTENSION_ROOT_DIR_FOR_TESTS } from '../../../test/constants.node'; import { createDeepnoteBlock, createDeepnoteFile, createDeepnoteNotebook, createDeepnoteProject } from '../deepnoteTestHelpers'; -import { - createAppPreviewApi, - type AppPreviewApi, - type AppPreviewOperation, - type PreviewRun -} from './appPreviewApi.node'; +import { createAppPreviewApi, type AppPreviewApi, type PreviewRun } from './appPreviewApi.node'; +import type { AppPreviewProject } from './appPreviewSession.node'; import type { WorkspaceNotebook } from './appPreviewWorkspace.node'; import type { AppPreviewRunOutcome, IAppPreviewRunner, SnapshotBlock } from './types'; @@ -104,7 +106,7 @@ suite('createAppPreviewApi', () => { let api: AppPreviewApi; let runner: IAppPreviewRunner; let findNotebook: sinon.SinonStub<[string], Promise>; - let onUnsupported: sinon.SinonStub<[AppPreviewOperation], void>; + let getProject: sinon.SinonStub<[], AppPreviewProject | undefined>; setup(async () => { runner = mock(); @@ -118,8 +120,8 @@ suite('createAppPreviewApi', () => { findNotebook = sinon .stub<[string], Promise>() .callsFake(async (notebookId) => notebooks.get(notebookId)); - onUnsupported = sinon.stub<[AppPreviewOperation], void>(); - api = createAppPreviewApi({ token: TOKEN, findNotebook, onUnsupported, runner: instance(runner) }); + getProject = sinon.stub<[], AppPreviewProject | undefined>(); + api = createAppPreviewApi({ token: TOKEN, findNotebook, getProject, runner: instance(runner) }); const app = express(); app.use('/v2', api.router); server = http.createServer(app); @@ -199,7 +201,6 @@ suite('createAppPreviewApi', () => { ] } }); - sinon.assert.notCalled(onUnsupported); }); test('GET /notebooks/:notebookId leaves inputs out for a notebook without any, and marks the init notebook', async () => { @@ -220,7 +221,6 @@ suite('createAppPreviewApi', () => { test('GET /notebooks/:notebookId answers 404 for an ID no workspace notebook has', async () => { await assertJson(await call('GET', '/notebooks/nb-missing'), 404, { message: 'Notebook not found' }); - sinon.assert.notCalled(onUnsupported); }); test('answers a JSON 500 without the stack, and logs the error, when the notebook lookup fails', async () => { @@ -254,7 +254,6 @@ suite('createAppPreviewApi', () => { notebookId: 'nb-1', inputs: { region: 'Asia' } }); - sinon.assert.notCalled(onUnsupported); }); test('POST /runs runs the notebook without inputs when the body has none', async () => { @@ -299,7 +298,6 @@ suite('createAppPreviewApi', () => { snapshotBlocks: CHART_SNAPSHOT, completedAt: '2026-01-02T03:05:10.000Z' }); - sinon.assert.notCalled(onUnsupported); }); const outcomes: [AppPreviewRunOutcome, string][] = [ @@ -457,9 +455,8 @@ suite('createAppPreviewApi', () => { return call('GET', `/notebooks/${notebookId}/app-data`); } - test('answers 404 before the notebook has run, without a warning', async () => { + test('answers 404 before the notebook has run', async () => { await assertJson(await getAppData(), 404, NO_RUN_RESULTS); - sinon.assert.notCalled(onUnsupported); }); test("answers 404 when none of the notebook's runs succeeded, though another notebook's run did", async () => { @@ -500,7 +497,6 @@ suite('createAppPreviewApi', () => { agentBlockSummary: null, runCreatedAt: '2026-01-02T03:05:05.000Z' }); - sinon.assert.notCalled(onUnsupported); }); test('answers the newest started successful run, not the last one to finish, past a run still in flight', async () => { @@ -634,17 +630,147 @@ suite('createAppPreviewApi', () => { }); }); - const stubs: [string, string, AppPreviewOperation][] = [ - ['GET', '/projects/p-1/dataframes/sales%20by%20month', 'dataframe()'] - ]; - for (const [method, path, operation] of stubs) { - test(`${method} ${path} answers 501 for ${operation}`, async () => { - await assertJson(await call(method, path), 501, { - message: `${operation} is not supported in the local preview yet.` + suite('GET /projects/:projectId/dataframes/:name', () => { + const PROJECT_ID = '5b0d4a8e-2f1c-4e3b-9a6d-7c8e9f0a1b2c'; + const NOT_FOUND = { message: 'Dataframe not found' }; + // Holds deepnote_dataframes/toolkit_frame, three rows that deepnote-toolkit 2.9.1 stored. + const FRAMES_PROJECT: AppPreviewProject = { + id: PROJECT_ID, + folder: Uri.joinPath( + Uri.file(EXTENSION_ROOT_DIR_FOR_TESTS), + 'src', + 'test', + 'testFiles', + 'appPreview', + 'frames' + ) + }; + let emptyFolder: string; + + setup(() => { + emptyFolder = fs.mkdtempSync(path.join(os.tmpdir(), 'deepnote-app-preview-api-')); + getProject.returns(FRAMES_PROJECT); + // The reader falls back to $HOME/work, which must not hold the developer's own frames here. + sinon.stub(platformUtils, 'getEnvironmentVariable').callThrough().withArgs('HOME').returns(emptyFolder); + }); + + teardown(() => { + fs.rmSync(emptyFolder, { recursive: true, force: true }); + }); + + function getDataframe(query = '', projectId = PROJECT_ID): Promise { + return call('GET', `/projects/${projectId}/dataframes/toolkit_frame${query}`); + } + + test("answers a page of the frame stored in the project's folder, in Deepnote's shape and key order", async () => { + const response = await getDataframe('?pageIndex=1&pageSize=2'); + + assert.strictEqual(response.status, 200); + const body = await response.json(); + assert.deepStrictEqual(Object.keys(body), [ + 'referenceId', + 'rowCount', + 'columnCount', + 'pageIndex', + 'pageSize', + 'pageCount', + 'columns', + 'rows' + ]); + assert.deepStrictEqual(body, { + referenceId: '937dcee4-b07d-4ee0-9ee9-4793acaaf0a7', + rowCount: 3, + columnCount: 4, + pageIndex: 1, + pageSize: 2, + pageCount: 2, + columns: [ + { name: 'id', type: 'int64', timezone: null }, + { name: 'score', type: 'float', timezone: null }, + { name: 'seen_at', type: 'timestamp', timezone: 'Europe/Prague' }, + { name: 'label', type: 'string', timezone: null } + ], + rows: [['0', -0.25, null, 'žluťoučký']] }); - sinon.assert.calledOnceWithExactly(onUnsupported, operation); }); - } + + test('answers page 0 of 10 rows without a query', async () => { + const body = await (await getDataframe()).json(); + + assert.deepStrictEqual([body.pageIndex, body.pageSize, body.pageCount, body.rows.length], [0, 10, 1, 3]); + }); + + const invalidRequests: [string, string, string][] = [ + ['a pageSize above 1000', '?pageSize=1001', 'pageSize too big: expected number to be <=1000'], + [ + 'a pageSize that is not a number', + '?pageSize=ten', + 'pageSize invalid input: expected number, received nan' + ], + ['a fractional pageIndex', '?pageIndex=0.5', 'pageIndex invalid input: expected int, received number'], + ['a fractional pageSize', '?pageSize=2.5', 'pageSize invalid input: expected int, received number'], + ['a query key Deepnote does not know', '?pageIndex=0&limit=5', 'Unrecognized key: "limit"'], + [ + 'several invalid values', + '?pageIndex=-1&pageSize=0', + 'pageIndex too small: expected number to be >=0; pageSize too small: expected number to be >=1' + ], + [ + 'a pageIndex past the last page', + '?pageIndex=1', + 'pageIndex 1 is out of range: the dataframe has 1 page(s) of 10 rows.' + ] + ]; + for (const [condition, query, message] of invalidRequests) { + test(`answers 400 to ${condition}`, async () => { + await assertJson(await getDataframe(query), 400, { message }); + }); + } + + test('answers 400 to a name outside [A-Za-z0-9_-]', async () => { + await assertJson(await call('GET', `/projects/${PROJECT_ID}/dataframes/sales%20by%20month`), 400, { + message: 'name invalid string: must match pattern /^[a-za-z0-9_-]{1,128}$/' + }); + }); + + test('reads the frames of a project whose ID is not a UUID, as .deepnote files allow', async () => { + getProject.returns({ ...FRAMES_PROJECT, id: 'e2e-hello-world-project' }); + + assert.strictEqual((await getDataframe('', 'e2e-hello-world-project')).status, 200); + }); + + test("answers 404 for a project ID other than the previewed app's project", async () => { + await assertJson(await getDataframe('', 'a7c3e1f0-9b8d-4c6e-8f2a-1b3c5d7e9f00'), 404, NOT_FOUND); + }); + + test('answers 404 when the previewed app belongs to no project', async () => { + getProject.returns(undefined); + + await assertJson(await getDataframe(), 404, NOT_FOUND); + }); + + test('asks for the project again on every request', async () => { + getProject.returns(undefined); + await assertJson(await getDataframe(), 404, NOT_FOUND); + + getProject.returns(FRAMES_PROJECT); + assert.strictEqual((await getDataframe()).status, 200); + + getProject.returns({ id: PROJECT_ID, folder: Uri.file(emptyFolder) }); + await assertJson(await getDataframe(), 404, NOT_FOUND); + }); + + test("answers the reader's 422 with its message, without logging an error", async () => { + const logError = sinon.stub(logger, 'error'); + sinon.stub(zlib, 'zstdDecompressSync').value(undefined); + + await assertJson(await getDataframe(), 422, { + message: + "The editor's Node.js cannot decompress the stored dataframe, which is compressed with zstd. Update the editor to read it." + }); + sinon.assert.notCalled(logError); + }); + }); const rejectedAuthorizations: [string, string | null][] = [ ['no authorization header', null], @@ -657,7 +783,6 @@ suite('createAppPreviewApi', () => { await assertJson(await call('GET', '/notebooks/nb-1', authorization), 401, { message: 'Unauthorized' }); await assertJson(await call('POST', '/runs', authorization), 401, { message: 'Unauthorized' }); sinon.assert.notCalled(findNotebook); - sinon.assert.notCalled(onUnsupported); }); } @@ -666,6 +791,5 @@ suite('createAppPreviewApi', () => { await assertJson(await call('GET', '/projects/p-1'), 403, notAvailable); await assertJson(await call('DELETE', '/runs/run-1'), 403, notAvailable); await assertJson(await call('POST', '/notebooks/nb-1'), 403, notAvailable); - sinon.assert.notCalled(onUnsupported); }); }); diff --git a/src/notebooks/deepnote/appPreview/appPreviewCommandHandler.node.ts b/src/notebooks/deepnote/appPreview/appPreviewCommandHandler.node.ts index a463233faa..544069f049 100644 --- a/src/notebooks/deepnote/appPreview/appPreviewCommandHandler.node.ts +++ b/src/notebooks/deepnote/appPreview/appPreviewCommandHandler.node.ts @@ -6,8 +6,6 @@ import { Commands } from '../../../platform/common/constants'; import { IDisposableRegistry, IExtensionContext } from '../../../platform/common/types'; import { DeepnoteAppPreview } from '../../../platform/common/utils/localize'; import { logger } from '../../../platform/logging'; -import { basename } from '../../../platform/vscode-path/resources'; -import type { AppPreviewOperation } from './appPreviewApi.node'; import { AppPreviewSession } from './appPreviewSession.node'; import { resolvePreviewProject } from './appPreviewWorkspace.node'; import { IAppPreviewRunner } from './types'; @@ -134,21 +132,21 @@ export class AppPreviewCommandHandler implements IExtensionSyncActivationService } private async startSession(folder: Uri): Promise { - const warned = new Set(); const session = new AppPreviewSession({ folder, project: await resolvePreviewProject(folder), runner: this.runner, - // A copy of https://deepnote.com/static/app-client/v1.js, so that previews work offline. scripts: { - appClient: Uri.joinPath(this.extensionContext.extensionUri, 'resources', 'appPreview', 'v1.js') - }, - onUnsupported: (operation) => { - if (warned.has(operation)) { - return; - } - warned.add(operation); - void window.showWarningMessage(DeepnoteAppPreview.operationNotSupported(operation, basename(folder))); + // A copy of https://deepnote.com/static/app-client/v1.js, so that previews work offline. + appClient: Uri.joinPath(this.extensionContext.extensionUri, 'resources', 'appPreview', 'v1.js'), + dataframeShim: Uri.joinPath( + this.extensionContext.extensionUri, + 'dist', + 'webviews', + 'webview-side', + 'appPreview', + 'dataframeShim.js' + ) } }); diff --git a/src/notebooks/deepnote/appPreview/appPreviewCommandHandler.node.unit.test.ts b/src/notebooks/deepnote/appPreview/appPreviewCommandHandler.node.unit.test.ts index b82cc521c2..6dd0a808cc 100644 --- a/src/notebooks/deepnote/appPreview/appPreviewCommandHandler.node.unit.test.ts +++ b/src/notebooks/deepnote/appPreview/appPreviewCommandHandler.node.unit.test.ts @@ -276,21 +276,4 @@ suite('AppPreviewCommandHandler', () => { verify(commands().executeCommand(SIMPLE_BROWSER_OPEN, anything(), anything())).once(); assert.include(await page(), '

app

'); }); - - test('warns once per unsupported operation, naming the operation and the folder', async () => { - await previewApp(Uri.file(appFolder)); - const { token } = await requestToken(); - const api = (urlPath: string, method = 'GET') => - fetch(`${serverUrl()}v2${urlPath}`, { - method, - headers: { authorization: `Bearer ${token}` } - }); - - assert.strictEqual((await api('/projects/project-1/dataframes/sales')).status, 501); - assert.strictEqual((await api('/projects/project-1/dataframes/sales')).status, 501); - - verify( - win().showWarningMessage('The app in app called dataframe(), which the local preview does not support yet.') - ).once(); - }); }); diff --git a/src/notebooks/deepnote/appPreview/appPreviewSession.node.ts b/src/notebooks/deepnote/appPreview/appPreviewSession.node.ts index 4f6017c9b7..60fd4c9a72 100644 --- a/src/notebooks/deepnote/appPreview/appPreviewSession.node.ts +++ b/src/notebooks/deepnote/appPreview/appPreviewSession.node.ts @@ -7,7 +7,7 @@ import { IDisposable } from '../../../platform/common/types'; import { generateUuid } from '../../../platform/common/uuid'; import { logger } from '../../../platform/logging'; import { extname, extUri } from '../../../platform/vscode-path/resources'; -import { createAppPreviewApi, type AppPreviewApi, type AppPreviewOperation } from './appPreviewApi.node'; +import { createAppPreviewApi, type AppPreviewApi } from './appPreviewApi.node'; import { findNotebookById } from './appPreviewWorkspace.node'; import type { IAppPreviewRunner } from './types'; @@ -20,8 +20,7 @@ export interface AppPreviewSessionOptions { folder: Uri; project: AppPreviewProject | undefined; runner: IAppPreviewRunner; - scripts: { appClient: Uri }; - onUnsupported(operation: AppPreviewOperation): void; + scripts: { appClient: Uri; dataframeShim: Uri }; } const LOOPBACK_HOSTNAMES = ['127.0.0.1', 'localhost', '[::1]']; @@ -37,6 +36,10 @@ const FILE_HEADERS = { const HOSTED_CLIENT_PATH = '/static/app-client/v1.js'; const CLIENT_SRC_ATTRIBUTE = 'src="/__deepnote/client.js"'; +// The page's own closing tag closes the shim's. The shim needs `window.Deepnote` when it runs and adds dataframe() only +// to connections made after that, so a client tag with defer, async or type=module, or an onload that connects, +// leaves the app without it. +const DATAFRAME_SHIM_TAGS = '`, `${SWAPPED}`, - '', + ``, '', '', ``, @@ -486,7 +530,7 @@ suite('AppPreviewSession', () => { assert.strictEqual( await getText(session, '/page.html'), - ``, - ``, + ``, + ``, `${SWAPPED}` ].join('\n') ); diff --git a/src/notebooks/deepnote/appPreview/storedDataframePage.node.ts b/src/notebooks/deepnote/appPreview/storedDataframePage.node.ts new file mode 100644 index 0000000000..4ffde8dadb --- /dev/null +++ b/src/notebooks/deepnote/appPreview/storedDataframePage.node.ts @@ -0,0 +1,515 @@ +import { + CompressionType, + compressionRegistry, + DataType, + DateUnit, + Int64, + RecordBatchReader, + TimeUnit, + Type, + Vector, + type Data, + type Decimal, + type Field, + type FixedSizeList, + type LargeList, + type List, + type MapRow, + type RecordBatch, + type StructRow, + type Timestamp +} from 'apache-arrow'; +import fs, { type FileHandle } from 'fs/promises'; +import { homedir } from 'os'; +import { Uri } from 'vscode'; +import zlib from 'zlib'; +import { z } from 'zod'; + +import { isFileNotFoundError } from '../../../platform/common/platform/errors'; +import { DeepnoteAppPreview } from '../../../platform/common/utils/localize'; +import { getEnvironmentVariable } from '../../../platform/common/utils/platform.node'; + +export type DataframeColumnType = + | 'null' + | 'integer' + | 'int64' + | 'float' + | 'boolean' + | 'string' + | 'binary' + | 'decimal' + | 'date' + | 'timestamp' + | 'time' + | 'duration' + | 'list' + | 'struct' + | 'map'; + +export interface DataframeColumn { + name: string; + timezone: string | null; + type: DataframeColumnType; +} + +/** A page of a stored dataframe, as Deepnote's `GET /v2/projects/:projectId/dataframes/:name` answers it. */ +export interface DataframePage { + columnCount: number; + columns: DataframeColumn[]; + pageCount: number; + pageIndex: number; + pageSize: number; + referenceId: string; + rowCount: number; + rows: unknown[][]; +} + +/** A page that cannot be read, with the HTTP status Deepnote answers for it. */ +export class DataframePageError extends Error { + constructor( + public readonly status: 400 | 404 | 422, + message: string + ) { + super(message); + } +} + +/** The names deepnote-toolkit's `store_dataframe` accepts, so each is a single path segment. */ +export const DATAFRAME_NAME_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; + +// Protocol strings: app code receives them, so they stay English like Deepnote's API messages. +export const DATAFRAME_NOT_FOUND_MESSAGE = 'Dataframe not found'; +const DATE_OUT_OF_RANGE_MESSAGE = + 'The page holds a date or timestamp outside the years 0 to 9999, which the server cannot format.'; +const LZ4_MESSAGE = 'The stored dataframe is compressed with lz4, which the server cannot decode.'; + +// deepnote-toolkit's FRAMES_DIR, relative to the folder it stores frames under. +const DATAFRAMES_DIRECTORY = 'deepnote_dataframes'; +const MANIFEST_FILE = 'manifest.json'; +const FRAME_EXTENSION = '.arrow'; +const FRAME_FILE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.arrow$/; +const UUID_EXTENSION_KEY = 'ARROW:extension:name'; +const UUID_EXTENSION = 'arrow.uuid'; +const UUID_BYTES = 16; +// Past these years toISOString switches to signed six-digit years. +const MIN_FORMATTED_YEAR = 0; +const MAX_FORMATTED_YEAR = 9999; +const MILLISECONDS_PER_HOUR = 3_600_000; +const NANOSECONDS_PER_MILLISECOND = 1_000_000n; +const NANOSECONDS_PER_SECOND = 1_000_000_000n; +const NANOSECONDS_PER_UNIT: Record = { + [TimeUnit.SECOND]: NANOSECONDS_PER_SECOND, + [TimeUnit.MILLISECOND]: NANOSECONDS_PER_MILLISECOND, + [TimeUnit.MICROSECOND]: 1_000n, + [TimeUnit.NANOSECOND]: 1n +}; +const TEXT_TYPES = new Set([Type.Utf8, Type.LargeUtf8, Type.Utf8View]); +const BINARY_TYPES = new Set([Type.Binary, Type.LargeBinary, Type.BinaryView, Type.FixedSizeBinary]); +const LIST_TYPES = new Set([Type.List, Type.LargeList, Type.FixedSizeList]); + +const manifestSchema = z + .object({ + file: z.string().regex(FRAME_FILE), + rows: z.int().nonnegative(), + batch_rows: z.array(z.int().nonnegative()) + }) + .refine((manifest) => manifest.batch_rows.reduce((sum, rows) => sum + rows, 0) === manifest.rows); + +type Manifest = z.infer; + +interface PageRequest { + pageIndex: number; + pageSize: number; +} + +function columnType(field: Field): DataframeColumnType { + const type = valueType(field.type); + // `children` is declared as an array, yet it is null on a type that has none. + (type.children ?? []).forEach(columnType); + + if (DataType.isNull(type)) { + return 'null'; + } + if (DataType.isInt(type)) { + return type.bitWidth === 64 ? 'int64' : 'integer'; + } + if (DataType.isFloat(type)) { + return 'float'; + } + if (DataType.isBool(type)) { + return 'boolean'; + } + if (isText(type) || isUuid(field)) { + return 'string'; + } + if (isBinary(type)) { + return 'binary'; + } + if (DataType.isDecimal(type)) { + return 'decimal'; + } + if (DataType.isDate(type)) { + return 'date'; + } + if (DataType.isTimestamp(type)) { + return 'timestamp'; + } + if (DataType.isTime(type)) { + return 'time'; + } + if (DataType.isDuration(type)) { + return 'duration'; + } + if (isList(type)) { + return 'list'; + } + if (DataType.isStruct(type)) { + return 'struct'; + } + if (DataType.isMap(type)) { + return 'map'; + } + throw new DataframePageError(422, `The dataframe has an unsupported column type ${String(type)}.`); +} + +function decompressZstd(bytes: Uint8Array): Uint8Array { + // zlib has zstd from Node.js 22.15 on, and older editors run an older Node.js. + if (typeof zlib.zstdDecompressSync !== 'function') { + throw new DataframePageError(422, DeepnoteAppPreview.zstdUnavailable); + } + + return zlib.zstdDecompressSync(bytes); +} + +function describeColumn(field: Field): DataframeColumn { + const type = valueType(field.type); + + return { + name: field.name, + type: columnType(field), + timezone: DataType.isTimestamp(type) && type.timezone ? type.timezone : null + }; +} + +function encodeRows(batch: RecordBatch, fields: Field[], begin: number, end: number): unknown[][] { + const columns = batch.data.children.map((data) => new Vector([withInt64Times(data)])); + const rows: unknown[][] = []; + for (let row = begin; row < end; row++) { + rows.push(fields.map((field, column) => encodeValue(columns[column].get(row), field))); + } + + return rows; +} + +function encodeValue(value: unknown, field: Field): unknown { + if (value === null || value === undefined) { + return null; + } + + const type = valueType(field.type); + if (DataType.isInt(type)) { + return type.bitWidth === 64 ? String(value) : value; + } + if (DataType.isFloat(type)) { + return Number.isFinite(value) ? value : null; + } + if (isUuid(field)) { + return formatUuid(value as Uint8Array); + } + if (isBinary(type)) { + return Buffer.from(value as Uint8Array).toString('base64'); + } + if (DataType.isDecimal(type)) { + return formatDecimal(value as Uint32Array, type); + } + if (DataType.isDate(type)) { + return isoString(Number(value)).slice(0, 10); + } + if (DataType.isTimestamp(type)) { + return formatTimestamp(value as bigint, type); + } + if (DataType.isTime(type)) { + const nanoseconds = BigInt(value as number | bigint) * NANOSECONDS_PER_UNIT[type.unit]; + + return formatTime(Number(nanoseconds / NANOSECONDS_PER_MILLISECOND)); + } + if (DataType.isDuration(type)) { + return Number(value) / Number(NANOSECONDS_PER_SECOND / NANOSECONDS_PER_UNIT[type.unit]); + } + if (isList(type)) { + return Array.from(value as Vector, (item) => encodeValue(item, type.children[0])); + } + if (DataType.isStruct(type)) { + return Object.fromEntries( + Array.from(value as StructRow, ([name, child], index) => [name, encodeValue(child, type.children[index])]) + ); + } + if (DataType.isMap(type)) { + const [keyField, valueField] = type.children[0].type.children; + + return Array.from(value as MapRow, ([key, item]) => [ + encodeValue(key, keyField), + encodeValue(item, valueField) + ]); + } + + return value; +} + +function floorDivide(dividend: bigint, divisor: bigint): bigint { + const quotient = dividend / divisor; + + return dividend % divisor < 0n ? quotient - 1n : quotient; +} + +/** apache-arrow hands a decimal over as the 32-bit words, least significant first, of its unscaled two's complement. */ +function formatDecimal(words: Uint32Array, type: Decimal): string { + let bits = 0n; + for (let index = words.length - 1; index >= 0; index--) { + bits = (bits << 32n) + BigInt(words[index]); + } + const unscaled = BigInt.asIntN(type.bitWidth, bits); + const sign = unscaled < 0n ? '-' : ''; + const magnitude = (unscaled < 0n ? -unscaled : unscaled).toString(); + if (type.scale <= 0) { + return sign + magnitude + (unscaled === 0n ? '' : '0'.repeat(-type.scale)); + } + const whole = magnitude.length > type.scale ? magnitude.slice(0, magnitude.length - type.scale) : '0'; + const fraction = magnitude.slice(-type.scale).padStart(type.scale, '0'); + + return `${sign}${whole}.${fraction}`; +} + +function formatTime(milliseconds: number): string { + const hours = Math.floor(milliseconds / MILLISECONDS_PER_HOUR); + // A time of day can pass 24 hours, so a Date formats only what follows the hours: `MM:SS.sss`. + const rest = new Date(milliseconds % MILLISECONDS_PER_HOUR).toISOString().slice(14, 23); + + return `${String(hours).padStart(2, '0')}:${rest}`; +} + +function formatTimestamp(value: bigint, type: Timestamp): string { + const milliseconds = floorDivide(value * NANOSECONDS_PER_UNIT[type.unit], NANOSECONDS_PER_MILLISECOND); + const iso = isoString(Number(milliseconds)); + + return type.timezone ? iso : iso.slice(0, -1); +} + +function formatUuid(bytes: Uint8Array): string { + return Buffer.from(bytes) + .toString('hex') + .replace(/^(\w{8})(\w{4})(\w{4})(\w{4})(\w{12})$/, '$1-$2-$3-$4-$5'); +} + +/** + * Falls back to `$HOME/work` because deepnote-toolkit 2.9.1 stores frames there when nothing sets its notebook root, + * and the extension's servers set none. Drop the fallback once the toolkit reads a storage root the extension sets. + * HOME comes first, as in the toolkit, because Node's homedir() reads USERPROFILE on Windows. + */ +async function frameDirectory(projectFolder: Uri, name: string): Promise { + const inProject = Uri.joinPath(projectFolder, DATAFRAMES_DIRECTORY, name); + try { + await fs.stat(Uri.joinPath(inProject, MANIFEST_FILE).fsPath); + + return inProject; + } catch (error) { + if (!isFileNotFoundError(error)) { + throw error; + } + } + + return Uri.joinPath(Uri.file(getEnvironmentVariable('HOME') || homedir()), 'work', DATAFRAMES_DIRECTORY, name); +} + +function isBinary(type: DataType): boolean { + return BINARY_TYPES.has(type.typeId); +} + +function isList(type: DataType): type is List | LargeList | FixedSizeList { + return LIST_TYPES.has(type.typeId); +} + +function isoString(milliseconds: number): string { + const date = new Date(milliseconds); + const year = date.getUTCFullYear(); + if (!(year >= MIN_FORMATTED_YEAR && year <= MAX_FORMATTED_YEAR)) { + throw new DataframePageError(422, DATE_OUT_OF_RANGE_MESSAGE); + } + + return date.toISOString(); +} + +function isText(type: DataType): boolean { + return TEXT_TYPES.has(type.typeId); +} + +function isUuid(field: Field): boolean { + const type = valueType(field.type); + + return ( + field.metadata.get(UUID_EXTENSION_KEY) === UUID_EXTENSION && + DataType.isFixedSizeBinary(type) && + type.byteWidth === UUID_BYTES + ); +} + +function notFound(): DataframePageError { + return new DataframePageError(404, DATAFRAME_NOT_FOUND_MESSAGE); +} + +async function readCurrentFrame(directory: Uri, page: PageRequest): Promise { + const manifest = await readManifest(directory); + if (!manifest) { + throw notFound(); + } + const pageCount = Math.max(1, Math.ceil(manifest.rows / page.pageSize)); + if (page.pageIndex >= pageCount) { + throw new DataframePageError( + 400, + `pageIndex ${page.pageIndex} is out of range: the dataframe has ${pageCount} page(s) of ${page.pageSize} rows.` + ); + } + + let handle: FileHandle; + try { + handle = await fs.open(Uri.joinPath(directory, manifest.file).fsPath, 'r'); + } catch (error) { + if (isFileNotFoundError(error)) { + return undefined; + } + throw error; + } + try { + return await readFramePage(handle, manifest, page, pageCount); + } finally { + await handle.close(); + } +} + +async function readFramePage( + handle: FileHandle, + manifest: Manifest, + { pageIndex, pageSize }: PageRequest, + pageCount: number +): Promise { + const reader = await RecordBatchReader.from(handle); + if (!reader.isFile()) { + throw notFound(); + } + await reader.open(); + if (reader.numRecordBatches !== manifest.batch_rows.length) { + throw notFound(); + } + const fields = reader.schema.fields; + const columns = fields.map(describeColumn); + + const firstRow = pageIndex * pageSize; + const endRow = Math.min(firstRow + pageSize, manifest.rows); + const rows: unknown[][] = []; + let batchStart = 0; + for (const [index, batchRows] of manifest.batch_rows.entries()) { + const batchEnd = batchStart + batchRows; + if (batchRows > 0 && batchStart < endRow && batchEnd > firstRow) { + const batch = await reader.readRecordBatch(index); + if (batch?.numRows !== batchRows) { + throw notFound(); + } + const begin = Math.max(firstRow, batchStart) - batchStart; + rows.push(...encodeRows(batch, fields, begin, Math.min(endRow, batchEnd) - batchStart)); + } + batchStart = batchEnd; + } + + return { + referenceId: manifest.file.slice(0, -FRAME_EXTENSION.length), + rowCount: manifest.rows, + columnCount: fields.length, + pageIndex, + pageSize, + pageCount, + columns, + rows + }; +} + +async function readManifest(directory: Uri): Promise { + let text: string; + try { + text = await fs.readFile(Uri.joinPath(directory, MANIFEST_FILE).fsPath, 'utf8'); + } catch (error) { + if (isFileNotFoundError(error)) { + return undefined; + } + throw error; + } + + let json: unknown; + try { + json = JSON.parse(text); + } catch { + return undefined; + } + const parsed = manifestSchema.safeParse(json); + + return parsed.success ? parsed.data : undefined; +} + +function refuseLz4(): never { + throw new DataframePageError(422, LZ4_MESSAGE); +} + +function valueType(type: DataType): DataType { + return DataType.isDictionary(type) ? type.dictionary : type; +} + +/** + * apache-arrow reads a timestamp or a Date64 as float milliseconds, which rounds sub-millisecond units and throws past + * 2^53. Viewed as Int64, the same buffers read as the exact count of units. + */ +function withInt64Times(data: Data): Data { + const isInt64Time = + DataType.isTimestamp(data.type) || (DataType.isDate(data.type) && data.type.unit === DateUnit.MILLISECOND); + const copy = data.clone( + isInt64Time ? new Int64() : data.type, + undefined, + undefined, + undefined, + undefined, + data.children.map(withInt64Times) + ); + if (data.dictionary) { + copy.dictionary = new Vector(data.dictionary.data.map(withInt64Times)); + } + + return copy; +} + +/** + * Reads one page of the dataframe that deepnote-toolkit's `store_dataframe` stored under `name` in `projectFolder`, or, + * when that folder holds no manifest for `name`, in `$HOME/work`, where the toolkit stores frames by default: + * `deepnote_dataframes//manifest.json` names an Arrow IPC file beside it, of which only the record batches the + * page overlaps are decoded. Throws a {@link DataframePageError}: 400 for a `pageIndex` past the last page, 404 for a + * `name` outside {@link DATAFRAME_NAME_PATTERN} or a missing or invalid manifest or frame file, and 422 for a frame or + * page that cannot be decompressed or encoded. Any other error, such as a corrupt record batch, propagates as it is. + */ +export async function readStoredDataframePage( + projectFolder: Uri, + name: string, + page: PageRequest +): Promise { + if (!DATAFRAME_NAME_PATTERN.test(name)) { + throw notFound(); + } + // The registry is process-wide and `set` replaces a whole codec, so one registered elsewhere could stand in. + compressionRegistry.set(CompressionType.ZSTD, { decode: decompressZstd }); + compressionRegistry.set(CompressionType.LZ4_FRAME, { decode: refuseLz4 }); + + const directory = await frameDirectory(projectFolder, name); + // A newer store may have replaced the frame, deleting its file, between reading the manifest and opening the file. + const found = (await readCurrentFrame(directory, page)) ?? (await readCurrentFrame(directory, page)); + if (!found) { + throw notFound(); + } + + return found; +} diff --git a/src/notebooks/deepnote/appPreview/storedDataframePage.node.unit.test.ts b/src/notebooks/deepnote/appPreview/storedDataframePage.node.unit.test.ts new file mode 100644 index 0000000000..0ecb30bd61 --- /dev/null +++ b/src/notebooks/deepnote/appPreview/storedDataframePage.node.unit.test.ts @@ -0,0 +1,793 @@ +import { + Binary, + Bool, + CompressionType, + compressionRegistry, + DateDay, + DateMillisecond, + Decimal, + Dictionary, + Duration, + Field, + FixedSizeBinary, + FixedSizeList, + Float16, + Float64, + Int32, + Int64, + Int8, + IntervalDayTime, + LargeBinary, + LargeList, + LargeUtf8, + List, + makeData, + Map_, + Null, + RecordBatch, + RecordBatchFileWriter, + Schema, + Struct, + TimeMillisecond, + TimeNanosecond, + TimeSecond, + TimestampMicrosecond, + TimestampNanosecond, + TimestampSecond, + TimeUnit, + Uint64, + Utf8, + Vector, + vectorFromArray, + type Data, + type DataType +} from 'apache-arrow'; +import { assert } from 'chai'; +import * as fs from 'fs'; +import fsPromises from 'fs/promises'; +import { syncBuiltinESMExports } from 'module'; +import os from 'os'; +import * as sinon from 'sinon'; +import { Uri } from 'vscode'; +import zlib from 'zlib'; + +import { platformUtils } from '../../../platform/common/utils/platform.node'; +import * as path from '../../../platform/vscode-path/path'; +import { EXTENSION_ROOT_DIR_FOR_TESTS } from '../../../test/constants.node'; +import { DataframePageError, readStoredDataframePage } from './storedDataframePage.node'; + +const REFERENCE_ID = '0b5c1f2e-3d4a-4b6c-8d7e-9f0a1b2c3d4e'; +const LZ4_FRAME_HEADER = Uint8Array.of(0x04, 0x22, 0x4d, 0x18, 0x40, 0x40, 0xc0); +const OTHER_REFERENCE_ID = '7a8b9c0d-1e2f-4a3b-8c4d-5e6f7a8b9c0d'; +const FIXTURE_PROJECT = Uri.joinPath( + Uri.file(EXTENSION_ROOT_DIR_FOR_TESTS), + 'src', + 'test', + 'testFiles', + 'appPreview', + 'frames' +); + +function column(name: string, data: Data, metadata: Record = {}): [Field, Data] { + return [new Field(name, data.type, true, new Map(Object.entries(metadata))), data]; +} + +function vector(values: unknown[], type: DataType): Data { + return vectorFromArray(values, type).data[0]; +} + +function recordBatch(...columns: [Field, Data][]): RecordBatch { + const schema = new Schema(columns.map(([field]) => field)); + + return new RecordBatch( + schema, + makeData({ + type: new Struct(schema.fields), + length: columns[0][1].length, + children: columns.map(([, data]) => data) + }) + ); +} + +function numbers(...values: number[]): RecordBatch { + return recordBatch(column('n', vector(values, new Int32()))); +} + +/** The little-endian 32-bit words of each value's two's complement, as Arrow stores a decimal. */ +function decimalWords(values: bigint[], bitWidth: number): Uint32Array { + const words = new Uint32Array((values.length * bitWidth) / 32); + values.forEach((value, index) => { + let bits = BigInt.asUintN(bitWidth, value); + for (let word = 0; word < bitWidth / 32; word++) { + words[(index * bitWidth) / 32 + word] = Number(bits & 0xffffffffn); + bits >>= 32n; + } + }); + + return words; +} + +async function rejection(promise: Promise): Promise<{ status: number; message: string }> { + const error = await promise.then( + () => assert.fail('the read did not reject'), + (reason: unknown) => reason + ); + assert.instanceOf(error, DataframePageError); + + return { status: error.status, message: error.message }; +} + +suite('readStoredDataframePage', () => { + let projectFolder: string; + let home: string; + let getEnvironmentVariable: sinon.SinonStub; + + setup(() => { + projectFolder = fs.mkdtempSync(path.join(os.tmpdir(), 'deepnote-stored-dataframe-')); + home = fs.mkdtempSync(path.join(os.tmpdir(), 'deepnote-stored-dataframe-home-')); + getEnvironmentVariable = sinon.stub(platformUtils, 'getEnvironmentVariable').callThrough(); + getEnvironmentVariable.withArgs('HOME').returns(home); + }); + + teardown(() => { + sinon.restore(); + syncBuiltinESMExports(); + fs.rmSync(projectFolder, { recursive: true, force: true }); + fs.rmSync(home, { recursive: true, force: true }); + }); + + function frameDirectory(name: string): string { + return path.join(projectFolder, 'deepnote_dataframes', name); + } + + function moveFramesToHome(): void { + fs.mkdirSync(path.join(home, 'work')); + fs.renameSync(path.join(projectFolder, 'deepnote_dataframes'), path.join(home, 'work', 'deepnote_dataframes')); + } + + function writeManifest(name: string, manifest: unknown): void { + fs.mkdirSync(frameDirectory(name), { recursive: true }); + const text = typeof manifest === 'string' ? manifest : JSON.stringify(manifest); + fs.writeFileSync(path.join(frameDirectory(name), 'manifest.json'), text); + } + + function writeFrame( + name: string, + batches: RecordBatch[], + options: { compression?: CompressionType; manifest?: unknown; referenceId?: string } = {} + ): void { + const referenceId = options.referenceId ?? REFERENCE_ID; + if (options.compression === CompressionType.ZSTD) { + compressionRegistry.set(CompressionType.ZSTD, { encode: (bytes) => zlib.zstdCompressSync(bytes) }); + } + // The reader refuses lz4 before decoding, so a buffer need not be real lz4: only start like an lz4 frame, as + // apache-arrow checks, and be shorter than the data, or the writer stores the data uncompressed. + if (options.compression === CompressionType.LZ4_FRAME) { + compressionRegistry.set(CompressionType.LZ4_FRAME, { encode: () => LZ4_FRAME_HEADER }); + } + const writer = RecordBatchFileWriter.writeAll(batches, { compressionType: options.compression ?? null }); + fs.mkdirSync(frameDirectory(name), { recursive: true }); + fs.writeFileSync(path.join(frameDirectory(name), `${referenceId}.arrow`), writer.toUint8Array(true)); + writeManifest( + name, + options.manifest ?? { + file: `${referenceId}.arrow`, + rows: batches.reduce((rows, batch) => rows + batch.numRows, 0), + batch_rows: batches.map((batch) => batch.numRows) + } + ); + } + + function read(name: string, pageIndex = 0, pageSize = 10, folder = Uri.file(projectFolder)) { + return readStoredDataframePage(folder, name, { pageIndex, pageSize }); + } + + async function readColumnsAndRows(batch: RecordBatch): Promise<{ columns: unknown; rows: unknown }> { + writeFrame('frame', [batch]); + const { columns, rows } = await read('frame'); + + return { columns, rows }; + } + + suite('encoding', () => { + test('encodes 8- to 32-bit integers as numbers, 64-bit integers as decimal strings, and NaN and infinities as null', async () => { + const batch = recordBatch( + column('int8', vector([-128, 7, null], new Int8())), + column('int64', vector([9007199254740993n, -1n, null], new Int64())), + column( + 'uint64', + makeData({ type: new Uint64(), length: 3, data: BigUint64Array.of(2n ** 64n - 1n, 0n, 1n) }) + ), + column('float64', vector([1.5, NaN, null], new Float64())), + column('float16', vector([0.5, Infinity, -Infinity], new Float16())) + ); + + assert.deepStrictEqual(await readColumnsAndRows(batch), { + columns: [ + { name: 'int8', type: 'integer', timezone: null }, + { name: 'int64', type: 'int64', timezone: null }, + { name: 'uint64', type: 'int64', timezone: null }, + { name: 'float64', type: 'float', timezone: null }, + { name: 'float16', type: 'float', timezone: null } + ], + rows: [ + [-128, '9007199254740993', '18446744073709551615', 1.5, 0.5], + [7, '-1', '0', null, null], + [null, null, '1', null, null] + ] + }); + }); + + test('encodes booleans, strings, null columns, and dictionary columns by their values', async () => { + const batch = recordBatch( + column('bool', vector([true, false, null], new Bool())), + column('utf8', vector(['a', null, 'ž'], new Utf8())), + column('large_utf8', vector(['x', 'y', null], new LargeUtf8())), + column('null', makeData({ type: new Null(), length: 3 })), + column('dictionary', vector(['b', 'b', null], new Dictionary(new Utf8(), new Int32()))), + column( + 'dictionary_timestamp', + makeData({ + type: new Dictionary(new TimestampNanosecond(), new Int32()), + length: 3, + data: Int32Array.of(1, 0, 1), + dictionary: new Vector([ + makeData({ type: new TimestampNanosecond(), length: 2, data: BigInt64Array.of(-500n, 0n) }) + ]) + }) + ) + ); + + assert.deepStrictEqual(await readColumnsAndRows(batch), { + columns: [ + { name: 'bool', type: 'boolean', timezone: null }, + { name: 'utf8', type: 'string', timezone: null }, + { name: 'large_utf8', type: 'string', timezone: null }, + { name: 'null', type: 'null', timezone: null }, + { name: 'dictionary', type: 'string', timezone: null }, + { name: 'dictionary_timestamp', type: 'timestamp', timezone: null } + ], + rows: [ + [true, 'a', 'x', null, 'b', '1970-01-01T00:00:00.000'], + [false, null, 'y', null, 'b', '1969-12-31T23:59:59.999'], + [null, 'ž', null, null, null, '1970-01-01T00:00:00.000'] + ] + }); + }); + + test('encodes an arrow.uuid as a lowercase UUID, and other binary values as padded base64', async () => { + // A column of its own bytes each: apache-arrow's writer garbles a column that shares another's buffer. + const sixteenBytes = () => + makeData({ + type: new FixedSizeBinary(16), + length: 1, + data: Uint8Array.from(Buffer.from('0B5C1F2E3D4A4B6C8D7E9F0A1B2C3D4E', 'hex')) + }); + const batch = recordBatch( + column('uuid', sixteenBytes(), { 'ARROW:extension:name': 'arrow.uuid' }), + column('fixed', sixteenBytes()), + column('binary', vector([Uint8Array.of(0xfb, 0xff)], new Binary())), + column('large_binary', vector([Uint8Array.of(1)], new LargeBinary())) + ); + + assert.deepStrictEqual(await readColumnsAndRows(batch), { + columns: [ + { name: 'uuid', type: 'string', timezone: null }, + { name: 'fixed', type: 'binary', timezone: null }, + { name: 'binary', type: 'binary', timezone: null }, + { name: 'large_binary', type: 'binary', timezone: null } + ], + rows: [['0b5c1f2e-3d4a-4b6c-8d7e-9f0a1b2c3d4e', 'C1wfLj1KS2yNfp8KGyw9Tg==', '+/8=', 'AQ==']] + }); + }); + + test('encodes decimals as plain strings with exactly the scale of fraction digits', async () => { + const decimal = (scale: number, precision: number, bitWidth: number, values: bigint[]) => + makeData({ + type: new Decimal(scale, precision, bitWidth), + length: values.length, + data: decimalWords(values, bitWidth) + }); + const batch = recordBatch( + column('scale_2', decimal(2, 38, 128, [12345n, 0n, -1n])), + column('scale_7', decimal(7, 18, 64, [-5n, 10_000_000n, 1n])), + column('scale_minus_2', decimal(-2, 9, 32, [12n, 0n, -3n])), + column('wide', decimal(0, 76, 256, [2n ** 200n, -(2n ** 200n), 7n])) + ); + + assert.deepStrictEqual(await readColumnsAndRows(batch), { + columns: [ + { name: 'scale_2', type: 'decimal', timezone: null }, + { name: 'scale_7', type: 'decimal', timezone: null }, + { name: 'scale_minus_2', type: 'decimal', timezone: null }, + { name: 'wide', type: 'decimal', timezone: null } + ], + rows: [ + ['123.45', '-0.0000005', '1200', '1606938044258990275541962092341162602522202993782792835301376'], + ['0.00', '1.0000000', '0', '-1606938044258990275541962092341162602522202993782792835301376'], + ['-0.01', '0.0000001', '-300', '7'] + ] + }); + }); + + test('encodes dates as UTC days, and timestamps floored to milliseconds, with Z only when zoned', async () => { + const batch = recordBatch( + column('date32', makeData({ type: new DateDay(), length: 2, data: Int32Array.of(0, 19_723) })), + column( + 'date64', + makeData({ type: new DateMillisecond(), length: 2, data: BigInt64Array.of(-1n, 86_400_000n) }) + ), + column( + 'nanoseconds', + makeData({ + type: new TimestampNanosecond(), + length: 2, + data: BigInt64Array.of(-500n, 1_735_689_604_123_999_999n) + }) + ), + column( + 'microseconds_zoned', + makeData({ + type: new TimestampMicrosecond('America/New_York'), + length: 2, + data: BigInt64Array.of(0n, 253_402_300_799_999_999n) + }) + ), + column( + 'seconds', + makeData({ + type: new TimestampSecond(), + length: 2, + data: BigInt64Array.of(1_700_000_000n, -62_167_219_200n) + }) + ) + ); + + assert.deepStrictEqual(await readColumnsAndRows(batch), { + columns: [ + { name: 'date32', type: 'date', timezone: null }, + { name: 'date64', type: 'date', timezone: null }, + { name: 'nanoseconds', type: 'timestamp', timezone: null }, + { name: 'microseconds_zoned', type: 'timestamp', timezone: 'America/New_York' }, + { name: 'seconds', type: 'timestamp', timezone: null } + ], + rows: [ + [ + '1970-01-01', + '1969-12-31', + '1969-12-31T23:59:59.999', + '1970-01-01T00:00:00.000Z', + '2023-11-14T22:13:20.000' + ], + [ + '2024-01-01', + '1970-01-02', + '2025-01-01T00:00:04.123', + '9999-12-31T23:59:59.999Z', + '0000-01-01T00:00:00.000' + ] + ] + }); + }); + + test('encodes times truncated to milliseconds without wrapping the hours, and durations as seconds', async () => { + const batch = recordBatch( + column('time_seconds', makeData({ type: new TimeSecond(), length: 2, data: Int32Array.of(90_061, 0) })), + column( + 'time_milliseconds', + makeData({ type: new TimeMillisecond(), length: 2, data: Int32Array.of(45_296_789, 1) }) + ), + column( + 'time_nanoseconds', + makeData({ type: new TimeNanosecond(), length: 2, data: BigInt64Array.of(1_999_999n, 999_999n) }) + ), + column( + 'duration_milliseconds', + makeData({ + type: new Duration(TimeUnit.MILLISECOND), + length: 2, + data: BigInt64Array.of(1500n, -1n) + }) + ), + column( + 'duration_nanoseconds', + makeData({ type: new Duration(TimeUnit.NANOSECOND), length: 2, data: BigInt64Array.of(3n, 0n) }) + ) + ); + + assert.deepStrictEqual(await readColumnsAndRows(batch), { + columns: [ + { name: 'time_seconds', type: 'time', timezone: null }, + { name: 'time_milliseconds', type: 'time', timezone: null }, + { name: 'time_nanoseconds', type: 'time', timezone: null }, + { name: 'duration_milliseconds', type: 'duration', timezone: null }, + { name: 'duration_nanoseconds', type: 'duration', timezone: null } + ], + rows: [ + ['25:01:01.000', '12:34:56.789', '00:00:00.001', 1.5, 3e-9], + ['00:00:00.000', '00:00:00.001', '00:00:00.000', -0.001, 0] + ] + }); + }); + + test('encodes lists as arrays, structs as objects in field order, and maps as key-value pairs, with each value by its type', async () => { + const timestamps = new Field('item', new TimestampNanosecond('UTC'), true); + const structFields = [new Field('z', new Int64(), true), new Field('a', new List(timestamps), true)]; + const batch = recordBatch( + column('list', vector([[1, null], [], null], new List(new Field('item', new Int32(), true)))), + column( + 'large_list', + vector([['a'], ['b', 'c'], []], new LargeList(new Field('item', new Utf8(), true))) + ), + column( + 'fixed_list', + vector([[1.5, NaN], null, [0, 2]], new FixedSizeList(2, new Field('item', new Float64(), true))) + ), + column( + 'struct', + makeData({ + type: new Struct(structFields), + length: 3, + nullCount: 1, + nullBitmap: Uint8Array.of(0b011), + children: [ + vector([1n, null, 0n], new Int64()), + makeData({ + type: new List(timestamps), + length: 3, + valueOffsets: Int32Array.of(0, 1, 1, 1), + child: makeData({ + type: timestamps.type, + length: 1, + data: BigInt64Array.of(-500n) + }) + }) + ] + }) + ), + column( + 'map', + vector( + [ + new Map([ + [2n, 1.5], + [1n, NaN] + ]), + new Map(), + null + ], + new Map_( + new Field( + 'entries', + new Struct<{ key: Int64; value: Float64 }>([ + new Field('key', new Int64(), false), + new Field('value', new Float64(), true) + ]), + false + ) + ) + ) + ) + ); + + assert.deepStrictEqual(await readColumnsAndRows(batch), { + columns: [ + { name: 'list', type: 'list', timezone: null }, + { name: 'large_list', type: 'list', timezone: null }, + { name: 'fixed_list', type: 'list', timezone: null }, + { name: 'struct', type: 'struct', timezone: null }, + { name: 'map', type: 'map', timezone: null } + ], + rows: [ + [ + [1, null], + ['a'], + [1.5, null], + { z: '1', a: ['1969-12-31T23:59:59.999Z'] }, + [ + ['2', 1.5], + ['1', null] + ] + ], + [[], ['b', 'c'], null, { z: null, a: [] }, []], + [null, [], [0, 2], null, null] + ] + }); + }); + + test('answers 422 for a column type it has no encoding for, also nested in another type', async () => { + writeFrame('top', [recordBatch(column('interval', vector([], new IntervalDayTime())))]); + writeFrame('nested', [ + recordBatch( + column('list', vector([], new List(new Field('item', new IntervalDayTime(), true)))), + column('ok', vector([], new Int32())) + ) + ]); + + assert.deepStrictEqual(await rejection(read('top')), { + status: 422, + message: 'The dataframe has an unsupported column type Interval.' + }); + assert.deepStrictEqual(await rejection(read('nested')), { + status: 422, + message: 'The dataframe has an unsupported column type Interval.' + }); + }); + + const outOfRange: [string, Data, unknown[][]][] = [ + [ + 'a date in the year 10000', + makeData({ type: new DateDay(), length: 2, data: Int32Array.of(0, 2_932_897) }), + [['1970-01-01']] + ], + [ + 'a timestamp before the year 0', + makeData({ type: new TimestampSecond(), length: 2, data: BigInt64Array.of(0n, -62_167_219_201n) }), + [['1970-01-01T00:00:00.000']] + ] + ]; + for (const [condition, data, firstPage] of outOfRange) { + test(`answers 422 for a page that holds ${condition}, and serves the pages without it`, async () => { + writeFrame('frame', [recordBatch(column('when', data))]); + + assert.deepStrictEqual((await read('frame', 0, 1)).rows, firstPage); + assert.deepStrictEqual(await rejection(read('frame', 1, 1)), { + status: 422, + message: + 'The page holds a date or timestamp outside the years 0 to 9999, which the server cannot format.' + }); + }); + } + }); + + suite('paging', () => { + const FRAME = [numbers(0, 1, 2), numbers(), numbers(3, 4), numbers(5, 6, 7)]; + + test('answers the page in the response key order, reading across a record batch boundary', async () => { + writeFrame('frame', FRAME); + + const page = await read('frame', 1, 2); + + assert.deepStrictEqual(Object.keys(page), [ + 'referenceId', + 'rowCount', + 'columnCount', + 'pageIndex', + 'pageSize', + 'pageCount', + 'columns', + 'rows' + ]); + assert.deepStrictEqual(page, { + referenceId: REFERENCE_ID, + rowCount: 8, + columnCount: 1, + pageIndex: 1, + pageSize: 2, + pageCount: 4, + columns: [{ name: 'n', type: 'integer', timezone: null }], + rows: [[2], [3]] + }); + }); + + test('answers pages that end exactly on a record batch boundary, and a short last page', async () => { + writeFrame('frame', FRAME); + + assert.deepStrictEqual((await read('frame', 0, 3)).rows, [[0], [1], [2]]); + assert.deepStrictEqual((await read('frame', 1, 3)).rows, [[3], [4], [5]]); + assert.deepStrictEqual((await read('frame', 2, 3)).rows, [[6], [7]]); + assert.deepStrictEqual((await read('frame', 0, 1000)).rows, [[0], [1], [2], [3], [4], [5], [6], [7]]); + }); + + test('answers 400 for a pageIndex past the last page', async () => { + writeFrame('frame', FRAME); + + assert.deepStrictEqual(await rejection(read('frame', 2, 5)), { + status: 400, + message: 'pageIndex 2 is out of range: the dataframe has 2 page(s) of 5 rows.' + }); + }); + + test('answers an empty frame as one empty page', async () => { + writeFrame('empty', [numbers(), numbers()]); + + const page = await read('empty'); + + assert.deepStrictEqual(page, { + referenceId: REFERENCE_ID, + rowCount: 0, + columnCount: 1, + pageIndex: 0, + pageSize: 10, + pageCount: 1, + columns: [{ name: 'n', type: 'integer', timezone: null }], + rows: [] + }); + assert.deepStrictEqual(await rejection(read('empty', 1)), { + status: 400, + message: 'pageIndex 1 is out of range: the dataframe has 1 page(s) of 10 rows.' + }); + }); + + test('reads only the record batches the page overlaps', async () => { + writeFrame('frame', [numbers(0, 1, 2), numbers(3, 4), numbers(5)], { + manifest: { file: `${REFERENCE_ID}.arrow`, rows: 6, batch_rows: [2, 2, 2] } + }); + + assert.deepStrictEqual((await read('frame', 1, 2)).rows, [[3], [4]]); + assert.deepStrictEqual(await rejection(read('frame', 0, 2)), { + status: 404, + message: 'Dataframe not found' + }); + }); + }); + + suite('storage', () => { + // apache-arrow's writer stores a buffer uncompressed unless compressing makes it smaller. + const COMPRESSIBLE = new Array(1000).fill(7); + + test('reads a zstd frame written by deepnote-toolkit 2.9.1, ignoring the extra manifest keys', async () => { + const page = await read('toolkit_frame', 0, 10, FIXTURE_PROJECT); + + assert.deepStrictEqual(page, { + referenceId: '937dcee4-b07d-4ee0-9ee9-4793acaaf0a7', + rowCount: 3, + columnCount: 4, + pageIndex: 0, + pageSize: 10, + pageCount: 1, + columns: [ + { name: 'id', type: 'int64', timezone: null }, + { name: 'score', type: 'float', timezone: null }, + { name: 'seen_at', type: 'timestamp', timezone: 'Europe/Prague' }, + { name: 'label', type: 'string', timezone: null } + ], + rows: [ + ['9007199254740993', 1.5, '2024-01-02T03:04:05.678Z', 'alpha'], + ['-7', null, '1969-12-31T23:59:59.999Z', null], + ['0', -0.25, null, 'žluťoučký'] + ] + }); + }); + + const missing: [string, () => void][] = [ + ['nothing is stored under the name', () => undefined], + ['the manifest is not JSON', () => writeManifest('frame', '{"file":')], + [ + "the manifest's file is not a lowercase UUID with .arrow", + () => writeFrame('frame', [numbers(1)], { referenceId: REFERENCE_ID.toUpperCase() }) + ], + [ + "the manifest's file is a path out of the frame's folder", + () => { + writeFrame('other', [numbers(1)]); + writeManifest('frame', { file: `../other/${REFERENCE_ID}.arrow`, rows: 1, batch_rows: [1] }); + } + ], + [ + 'the manifest has no batch_rows', + () => writeFrame('frame', [numbers(1)], { manifest: { file: `${REFERENCE_ID}.arrow`, rows: 1 } }) + ], + [ + "the manifest's batch_rows do not add up to its rows", + () => + writeFrame('frame', [numbers(1)], { + manifest: { file: `${REFERENCE_ID}.arrow`, rows: 2, batch_rows: [1] } + }) + ], + [ + 'the file the manifest names is missing', + () => writeManifest('frame', { file: `${OTHER_REFERENCE_ID}.arrow`, rows: 1, batch_rows: [1] }) + ], + [ + 'the frame holds more record batches than batch_rows lists', + () => + writeFrame('frame', [numbers(1), numbers(2)], { + manifest: { file: `${REFERENCE_ID}.arrow`, rows: 1, batch_rows: [1] } + }) + ], + [ + 'a record batch holds another row count than its batch_rows entry', + () => + writeFrame('frame', [numbers(1)], { + manifest: { file: `${REFERENCE_ID}.arrow`, rows: 2, batch_rows: [2] } + }) + ] + ]; + for (const [condition, store] of missing) { + test(`answers 404 when ${condition}`, async () => { + store(); + + assert.deepStrictEqual(await rejection(read('frame')), { + status: 404, + message: 'Dataframe not found' + }); + }); + } + + test('reads a frame stored under $HOME/work when the project folder has none under its name', async () => { + writeFrame('frame', [numbers(1, 2)]); + moveFramesToHome(); + + const page = await read('frame'); + + assert.deepStrictEqual([page.referenceId, page.rows], [REFERENCE_ID, [[1], [2]]]); + }); + + test('reads a frame stored under the home folder Node reports when HOME is unset, as it usually is on Windows', async () => { + getEnvironmentVariable.withArgs('HOME').returns(undefined); + sinon.stub(os, 'homedir').returns(home); + // The reader's named import of homedir follows os.homedir only once synced, here and after the restore. + syncBuiltinESMExports(); + writeFrame('frame', [numbers(1, 2)]); + moveFramesToHome(); + + const page = await read('frame'); + + assert.deepStrictEqual([page.referenceId, page.rows], [REFERENCE_ID, [[1], [2]]]); + }); + + test("reads the project folder's frame over one of the same name under $HOME/work", async () => { + writeFrame('frame', [numbers(1)], { referenceId: OTHER_REFERENCE_ID }); + moveFramesToHome(); + writeFrame('frame', [numbers(2)]); + + const page = await read('frame'); + + assert.deepStrictEqual([page.referenceId, page.rows], [REFERENCE_ID, [[2]]]); + }); + + test('answers 404 for an invalid manifest in the project folder, even when $HOME/work has the frame', async () => { + writeFrame('frame', [numbers(1)]); + moveFramesToHome(); + writeManifest('frame', '{"file":'); + + assert.deepStrictEqual(await rejection(read('frame')), { status: 404, message: 'Dataframe not found' }); + }); + + test('answers 404 for a name that is not a single safe path segment', async () => { + writeFrame('frame', [numbers(1)]); + + assert.deepStrictEqual(await rejection(read('../deepnote_dataframes/frame')), { + status: 404, + message: 'Dataframe not found' + }); + }); + + test('reads the manifest again when the frame it named was replaced before the file was opened', async () => { + writeFrame('frame', [numbers(1, 2)], { referenceId: OTHER_REFERENCE_ID }); + writeManifest('frame', { file: `${REFERENCE_ID}.arrow`, rows: 1, batch_rows: [1] }); + const realOpen = fsPromises.open; + const open = sinon.stub(fsPromises, 'open').callThrough(); + open.onFirstCall().callsFake(async (...args: Parameters) => { + writeManifest('frame', { file: `${OTHER_REFERENCE_ID}.arrow`, rows: 2, batch_rows: [2] }); + + return realOpen(...args); + }); + + const page = await read('frame'); + + assert.deepStrictEqual([page.referenceId, page.rows], [OTHER_REFERENCE_ID, [[1], [2]]]); + }); + + test('answers 422 for a frame compressed with lz4', async () => { + writeFrame('frame', [numbers(...COMPRESSIBLE)], { compression: CompressionType.LZ4_FRAME }); + + assert.deepStrictEqual(await rejection(read('frame')), { + status: 422, + message: 'The stored dataframe is compressed with lz4, which the server cannot decode.' + }); + }); + + test("answers 422 for a zstd frame when the editor's Node.js has no zstd", async () => { + writeFrame('frame', [numbers(...COMPRESSIBLE)], { compression: CompressionType.ZSTD }); + sinon.stub(zlib, 'zstdDecompressSync').value(undefined); + + assert.deepStrictEqual(await rejection(read('frame')), { + status: 422, + message: + "The editor's Node.js cannot decompress the stored dataframe, which is compressed with zstd. Update the editor to read it." + }); + }); + }); +}); diff --git a/src/platform/common/utils/localize.ts b/src/platform/common/utils/localize.ts index 0d4e88719d..cc1bb1b82b 100644 --- a/src/platform/common/utils/localize.ts +++ b/src/platform/common/utils/localize.ts @@ -1417,8 +1417,6 @@ export namespace DeepnoteAppPreview { export const chooseFolderTitle = l10n.t('Choose the app folder to preview'); export const preview = l10n.t('Preview'); export const startFailed = (message: string) => l10n.t('Could not start the app preview: {0}', message); - export const operationNotSupported = (operation: string, folderName: string) => - l10n.t('The app in {1} called {0}, which the local preview does not support yet.', operation, folderName); export const noInterpreter = l10n.t( 'No Python interpreter is selected for the notebook. Select one in VS Code, then run the app again.' ); @@ -1428,6 +1426,9 @@ export namespace DeepnoteAppPreview { export const blockFailed = (blockId: string) => l10n.t('Block {0} failed.', blockId); export const notebookNotInFile = (notebookId: string) => l10n.t('The notebook {0} is no longer in its .deepnote file.', notebookId); + export const zstdUnavailable = l10n.t( + "The editor's Node.js cannot decompress the stored dataframe, which is compressed with zstd. Update the editor to read it." + ); } export namespace SelectInputSettings { diff --git a/src/test/testFiles/appPreview/frames/deepnote_dataframes/toolkit_frame/937dcee4-b07d-4ee0-9ee9-4793acaaf0a7.arrow b/src/test/testFiles/appPreview/frames/deepnote_dataframes/toolkit_frame/937dcee4-b07d-4ee0-9ee9-4793acaaf0a7.arrow new file mode 100644 index 0000000000..b5b01afeb9 Binary files /dev/null and b/src/test/testFiles/appPreview/frames/deepnote_dataframes/toolkit_frame/937dcee4-b07d-4ee0-9ee9-4793acaaf0a7.arrow differ diff --git a/src/test/testFiles/appPreview/frames/deepnote_dataframes/toolkit_frame/manifest.json b/src/test/testFiles/appPreview/frames/deepnote_dataframes/toolkit_frame/manifest.json new file mode 100644 index 0000000000..e5f969b6e1 --- /dev/null +++ b/src/test/testFiles/appPreview/frames/deepnote_dataframes/toolkit_frame/manifest.json @@ -0,0 +1 @@ +{"version": 1, "file": "937dcee4-b07d-4ee0-9ee9-4793acaaf0a7.arrow", "written_at": "2026-11-13T23:11:13Z", "rows": 3, "batch_rows": [3], "toolkit_version": "2.9.1"} \ No newline at end of file diff --git a/src/webviews/webview-side/appPreview/dataframeShim.ts b/src/webviews/webview-side/appPreview/dataframeShim.ts new file mode 100644 index 0000000000..5c649502d6 --- /dev/null +++ b/src/webviews/webview-side/appPreview/dataframeShim.ts @@ -0,0 +1,6 @@ +// Loaded right after the bundled copy of Deepnote's client, which has no `dataframe()` yet, so that apps can page +// through their stored dataframes in the preview. + +import { addDataframe, type DeepnoteGlobal } from './deepnoteDataframe'; + +addDataframe((window as unknown as { Deepnote: DeepnoteGlobal }).Deepnote, window.fetch.bind(window)); diff --git a/src/webviews/webview-side/appPreview/deepnoteDataframe.ts b/src/webviews/webview-side/appPreview/deepnoteDataframe.ts new file mode 100644 index 0000000000..b05aba5dd4 --- /dev/null +++ b/src/webviews/webview-side/appPreview/deepnoteDataframe.ts @@ -0,0 +1,74 @@ +const TOKEN_PATH = '/__deepnote/api-token'; +const REQUEST_TIMEOUT_MS = 30_000; +const NO_PROJECT_MESSAGE = + "This app folder is not next to or inside the folder of a Deepnote project's .deepnote files, so it has no stored dataframes. After adding one, run Deepnote: Preview App again."; + +export interface DataframePageOptions { + pageIndex?: number; + pageSize?: number; +} + +export interface DeepnoteConnection { + dataframe?(name: string, options?: DataframePageOptions): Promise; +} + +export interface DeepnoteGlobal { + connect(...args: unknown[]): DeepnoteConnection; +} + +interface TokenReply { + token: string; + apiOrigin: string; + projectId?: string; +} + +async function readJson(response: Response): Promise { + if (!response.ok) { + const body = (await response.json().catch(() => null)) as { message?: string } | null; + throw new Error(body?.message || `Request failed with status ${response.status}`); + } + + return (await response.json()) as T; +} + +/** + * Gives the connections of Deepnote's client a `dataframe(name, { pageIndex, pageSize })` when they have none. It asks + * the preview's token endpoint for the token, the API origin and the project, as the version of the client that adds + * `dataframe()` does. It rejects with a failed request's message, or its status when the answer has none, and rejects + * without asking the API when the preview has no project. + */ +export function addDataframe(deepnote: DeepnoteGlobal, fetchFn: typeof fetch): void { + async function dataframe(name: string, options: DataframePageOptions = {}): Promise { + const { token, apiOrigin, projectId } = await readJson( + await fetchFn(TOKEN_PATH, { method: 'POST' }) + ); + if (!projectId) { + throw new Error(NO_PROJECT_MESSAGE); + } + + const query = new URLSearchParams(); + if (options.pageIndex !== undefined) { + query.set('pageIndex', String(options.pageIndex)); + } + if (options.pageSize !== undefined) { + query.set('pageSize', String(options.pageSize)); + } + const search = query.toString(); + const path = `/v2/projects/${encodeURIComponent(projectId)}/dataframes/${encodeURIComponent(name)}`; + + return readJson( + await fetchFn(`${apiOrigin}${path}${search ? `?${search}` : ''}`, { + headers: { authorization: `Bearer ${token}` }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) + }) + ); + } + + const connect = deepnote.connect.bind(deepnote); + deepnote.connect = (...args) => { + const connection = connect(...args); + connection.dataframe ??= dataframe; + + return connection; + }; +} diff --git a/src/webviews/webview-side/appPreview/deepnoteDataframe.unit.test.ts b/src/webviews/webview-side/appPreview/deepnoteDataframe.unit.test.ts new file mode 100644 index 0000000000..e64ae6e4b6 --- /dev/null +++ b/src/webviews/webview-side/appPreview/deepnoteDataframe.unit.test.ts @@ -0,0 +1,135 @@ +import { assert } from 'chai'; + +import { addDataframe, type DeepnoteConnection, type DeepnoteGlobal } from './deepnoteDataframe'; + +const NO_PROJECT_MESSAGE = + "This app folder is not next to or inside the folder of a Deepnote project's .deepnote files, so it has no stored dataframes. After adding one, run Deepnote: Preview App again."; + +interface FetchCall { + url: string; + method: string | undefined; + authorization: string | undefined; +} + +function jsonResponse(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } }); +} + +suite('addDataframe', () => { + let calls: FetchCall[]; + let replies: Response[]; + + const fakeFetch: typeof fetch = async (input, init) => { + const headers = new Headers(init?.headers); + calls.push({ + url: String(input), + method: init?.method, + authorization: headers.get('authorization') ?? undefined + }); + const reply = replies.shift(); + if (!reply) { + throw new Error(`Unexpected request to ${String(input)}`); + } + + return reply; + }; + + function tokenReply(projectId: string | null = 'project 1'): Response { + return jsonResponse(200, { + token: 'token-1', + apiOrigin: 'http://127.0.0.1:5000', + expiresAtSeconds: 2_000_000_000, + ...(projectId === null ? {} : { projectId }) + }); + } + + function connectWith(connection: DeepnoteConnection = {}): DeepnoteConnection { + const deepnote: DeepnoteGlobal = { connect: () => connection }; + addDataframe(deepnote, fakeFetch); + + return deepnote.connect(); + } + + setup(() => { + calls = []; + replies = []; + }); + + test("pages through the frame with the token, API origin and project of the preview's token endpoint", async () => { + const page = { rows: [{ id: 1 }], pageIndex: 2, pageSize: 50 }; + replies = [tokenReply(), jsonResponse(200, page)]; + + const result = await connectWith().dataframe!('sales frame', { pageIndex: 2, pageSize: 50 }); + + assert.deepStrictEqual(result, page); + assert.deepStrictEqual(calls, [ + { url: '/__deepnote/api-token', method: 'POST', authorization: undefined }, + { + url: 'http://127.0.0.1:5000/v2/projects/project%201/dataframes/sales%20frame?pageIndex=2&pageSize=50', + method: undefined, + authorization: 'Bearer token-1' + } + ]); + }); + + test('leaves the page to the API when no page is given', async () => { + replies = [tokenReply(), jsonResponse(200, { rows: [] })]; + + await connectWith().dataframe!('sales'); + + assert.strictEqual(calls[1].url, 'http://127.0.0.1:5000/v2/projects/project%201/dataframes/sales'); + }); + + test("passes connect()'s options to the client and keeps the connection's own methods", () => { + const notebook = () => undefined; + const seen: unknown[][] = []; + const deepnote: DeepnoteGlobal = { + connect: (...args) => { + seen.push(args); + + return { notebook } as DeepnoteConnection; + } + }; + addDataframe(deepnote, fakeFetch); + const options = { onAuthExpired: () => undefined }; + + const connection = deepnote.connect(options) as DeepnoteConnection & { notebook: unknown }; + + assert.deepStrictEqual(seen, [[options]]); + assert.strictEqual(connection.notebook, notebook); + assert.isFunction(connection.dataframe); + }); + + test("leaves a dataframe() that the client's connection has in place", () => { + const own = async () => 'own page'; + + const connection = connectWith({ dataframe: own }); + + assert.strictEqual(connection.dataframe, own); + }); + + test("rejects with the API's message", async () => { + replies = [tokenReply(), jsonResponse(404, { message: 'Dataframe not found' })]; + + await assert.isRejected(connectWith().dataframe!('sales'), /^Dataframe not found$/); + }); + + test('rejects with the status when the answer has no message', async () => { + replies = [tokenReply(), new Response('Internal error', { status: 500 })]; + + await assert.isRejected(connectWith().dataframe!('sales'), /^Request failed with status 500$/); + }); + + test('rejects with the status when the token endpoint refuses', async () => { + replies = [new Response('Forbidden', { status: 403 })]; + + await assert.isRejected(connectWith().dataframe!('sales'), /^Request failed with status 403$/); + }); + + test('rejects without asking the API when the app has no project', async () => { + replies = [tokenReply(null)]; + + await assert.isRejected(connectWith().dataframe!('sales'), NO_PROJECT_MESSAGE); + assert.lengthOf(calls, 1); + }); +}); diff --git a/test/e2e/fixtures/app-preview-dataframes/3f2b8c1e-6d4a-4e9b-9a7c-0d5e1f2a3b4c.arrow b/test/e2e/fixtures/app-preview-dataframes/3f2b8c1e-6d4a-4e9b-9a7c-0d5e1f2a3b4c.arrow new file mode 100644 index 0000000000..ee0d0cf35c Binary files /dev/null and b/test/e2e/fixtures/app-preview-dataframes/3f2b8c1e-6d4a-4e9b-9a7c-0d5e1f2a3b4c.arrow differ diff --git a/test/e2e/fixtures/app-preview-dataframes/index.html b/test/e2e/fixtures/app-preview-dataframes/index.html new file mode 100644 index 0000000000..05faa7f5a1 --- /dev/null +++ b/test/e2e/fixtures/app-preview-dataframes/index.html @@ -0,0 +1,241 @@ + + + + + + Customers + + + + +
+

Customers

+

Loading the stored dataframe…

+
+ + + + + +
+ +
+ +
+ + + diff --git a/test/e2e/fixtures/app-preview/app.js b/test/e2e/fixtures/app-preview/app.js index 8073ba9ce6..979bf176bb 100644 --- a/test/e2e/fixtures/app-preview/app.js +++ b/test/e2e/fixtures/app-preview/app.js @@ -69,6 +69,7 @@ const notebook = connection.notebook('e2e-preview-notebook'); connection.notebook('e2e-missing-notebook').inputs() ); await call('app-data', 'appData()', () => notebook.appData()); + await call('dataframe', 'dataframe()', () => connection.dataframe('sales')); const deepnoteRequests = performance .getEntriesByType('resource') .filter((entry) => new URL(entry.name).hostname.endsWith('deepnote.com')); diff --git a/test/e2e/fixtures/app-preview/index.html b/test/e2e/fixtures/app-preview/index.html index 3037d9f1ec..c3f013155c 100644 --- a/test/e2e/fixtures/app-preview/index.html +++ b/test/e2e/fixtures/app-preview/index.html @@ -16,6 +16,7 @@

Sales dashboard

  • inputs(): pending
  • inputs() of an unknown notebook: pending
  • appData(): pending
  • +
  • dataframe(): pending
  • requests to deepnote.com: pending
  • diff --git a/test/e2e/helpers/venv.ts b/test/e2e/helpers/venv.ts index b1d234d8c1..1edb76b6ae 100644 --- a/test/e2e/helpers/venv.ts +++ b/test/e2e/helpers/venv.ts @@ -19,7 +19,7 @@ const SETTINGS_SOURCE = path.join(REPO_ROOT, 'test', 'e2e', 'settings.json'); const SETTINGS_TARGET = path.join(REPO_ROOT, 'test', 'e2e', 'settings.generated.json'); /** The interpreter inside the pre-baked venv. Suites run against it as the active interpreter. */ -function venvPython(): string { +export function venvPython(): string { return process.platform === 'win32' ? path.join(VENV_DIR, 'Scripts', 'python.exe') : path.join(VENV_DIR, 'bin', 'python'); diff --git a/test/e2e/suite/appPreview/previewApp.e2e.test.ts b/test/e2e/suite/appPreview/previewApp.e2e.test.ts index 4d629a6d68..8ad68f50c3 100644 --- a/test/e2e/suite/appPreview/previewApp.e2e.test.ts +++ b/test/e2e/suite/appPreview/previewApp.e2e.test.ts @@ -6,10 +6,20 @@ */ import { expect } from 'chai'; +import { execFileSync } from 'child_process'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; -import { EditorView, InputBox, SideBarView, VSBrowser, Workbench, type ViewItem } from 'vscode-extension-tester'; +import { + By, + EditorView, + InputBox, + SideBarView, + VSBrowser, + Workbench, + type TreeItem, + type ViewItem +} from 'vscode-extension-tester'; import { KERNEL_CONNECT_TIMEOUT, @@ -23,12 +33,15 @@ import { openFolderViaDialog, selectDeepnoteContextMenu } from '../../helpers'; +import { venvPython } from '../../helpers/venv'; const FIXTURE_APP = path.resolve(process.cwd(), 'test', 'e2e', 'fixtures', 'app-preview'); const RUNS_FIXTURE_APP = path.resolve(process.cwd(), 'test', 'e2e', 'fixtures', 'app-preview-runs'); const APP_DATA_FIXTURE_APP = path.resolve(process.cwd(), 'test', 'e2e', 'fixtures', 'app-preview-app-data'); +const DATAFRAMES_FIXTURE = path.resolve(process.cwd(), 'test', 'e2e', 'fixtures', 'app-preview-dataframes'); const INPUTS_NOTEBOOK_FIXTURE = 'app-preview-inputs.deepnote'; const RUNS_NOTEBOOK_FIXTURES = ['app-preview-runs.deepnote', 'app-preview-runs-init.deepnote']; +const PROJECT_FIXTURE = 'hello-world.deepnote'; const SYNCED_APP = path.join('Sales', '.files', '_deepnote_static'); const SYNCED_TITLE = 'Synced fixture'; const RELOAD_TITLE = 'Reload fixture'; @@ -36,12 +49,16 @@ const REUSE_TITLE = 'Reuse fixture'; const INPUTS_TITLE = 'Inputs fixture'; const RUNS_TITLE = 'Runs fixture'; const APP_DATA_TITLE = 'App data fixture'; +const DATAFRAMES_APP = 'app'; +const DATAFRAMES_TITLE = 'Customers'; const PREVIEW_URL = /^http:\/\/127\.0\.0\.1:\d+\//; const INPUTS_RESOLVED = 'inputs() resolved: '; const RUN_RESULT = ' result(): '; const APP_DATA_AFTER_RUN = 'appData() after the run: '; const NO_RUN_RESULTS = 'You have no run results for this notebook yet. Run the notebook to load them.'; +const NO_PROJECT_DATAFRAME = + "This app folder is not next to or inside the folder of a Deepnote project's .deepnote files, so it has no stored dataframes. After adding one, run Deepnote: Preview App again."; const NOTEBOOK_INPUTS = [ { blockId: 'e2e-region-input', name: 'region', type: 'input-text', value: 'Europe', label: 'Region' }, @@ -66,6 +83,43 @@ const NOTEBOOK_INPUTS = [ } ]; +// The frame is stored in the developer's own home folder on a local run, so under a name no notebook of theirs uses. +const FRAME_NAME = 'app_preview_e2e_customers'; +const FRAME_FILE = '3f2b8c1e-6d4a-4e9b-9a7c-0d5e1f2a3b4c.arrow'; +const STORE_FRAME_SCRIPT = [ + 'import sys', + 'import pyarrow as pa', + 'from deepnote_toolkit.artifacts import store_dataframe', + 'store_dataframe(pa.ipc.open_file(sys.argv[1]).read_all().to_pandas(), sys.argv[2])' +].join('\n'); +const STORE_FRAME_TIMEOUT = 60_000; +const FIRST_ROW = [ + '0', + 'customer-000039', + '58.88', + 'silver', + 'false', + '2025-01-01T00:00:04.000', + 'left voicemail (ticket 34)' +]; +const LAST_ROW = [ + '199999', + 'customer-000167', + '68.8', + 'diamond', + 'true', + '2025-01-06T19:13:25.000', + 'called back (ticket 33)' +]; + +interface DataframeSitePage { + error: string; + indicator: string; + isNextDisabled: boolean; + rows: string[][]; + summary: string; +} + function expectedReport(title: string): unknown[] { return [ title, @@ -75,6 +129,7 @@ function expectedReport(title: string): unknown[] { NOTEBOOK_INPUTS, 'inputs() of an unknown notebook rejected: Notebook not found', `appData() rejected: ${NO_RUN_RESULTS}`, + `dataframe() rejected: ${NO_PROJECT_DATAFRAME}`, 'requests to deepnote.com: 0' ]; } @@ -109,6 +164,42 @@ async function readReport(timeout = WORKBENCH_TIMEOUT): Promise { return withParsedInputs(report); } +function consecutiveIds(first: number, count: number): string[] { + return Array.from({ length: count }, (_, offset) => String(first + offset)); +} + +async function readDataframePage(indicator: string): Promise { + const driver = VSBrowser.instance.driver; + let page: DataframeSitePage | undefined; + try { + await driver.wait( + async () => { + page = await driver + .executeScript( + `return { + error: document.querySelector('[role="alert"]').textContent, + indicator: document.querySelector('[role="status"]').textContent, + isNextDisabled: document.getElementById('next').disabled, + rows: Array.from(document.querySelectorAll('tbody tr'), (row) => + Array.from(row.cells, (cell) => cell.textContent) + ), + summary: document.getElementById('summary').textContent + };` + ) + .catch(() => undefined); + + return page?.indicator === indicator; + }, + WORKBENCH_TIMEOUT, + `the dataframe site did not show "${indicator}"` + ); + } catch (error) { + throw new Error(`${(error as Error).message}. Last read: ${JSON.stringify(page)}`); + } + + return page!; +} + async function onIntegratedBrowserPage(title: string, read: () => Promise): Promise { const driver = VSBrowser.instance.driver; const workbench = await driver.getWindowHandle(); @@ -156,6 +247,23 @@ async function findRootExplorerItem(label: string): Promise { return found!; } +/** Looks the parent up again on every try, so a row the Explorer has re-rendered is not held stale. */ +async function findExplorerChildItem(parentLabel: string, label: string): Promise { + let found: TreeItem | undefined; + await VSBrowser.instance.driver.wait( + async () => { + const parent = (await findRootExplorerItem(parentLabel)) as TreeItem; + found = await parent.findChildItem(label).catch(() => undefined); + + return found !== undefined; + }, + WORKBENCH_TIMEOUT, + `"${label}" did not appear under "${parentLabel}" in the file explorer` + ); + + return found!; +} + /** In the simple folder dialog, OK first navigates toward the typed path, so it is clicked until the dialog closes. */ async function previewFromCommandPalette(folder: string): Promise { const driver = VSBrowser.instance.driver; @@ -220,6 +328,39 @@ async function closeAllEditors(): Promise { ); } +/** + * Stores the fixture frame with the venv's deepnote-toolkit, which puts it under `$HOME/work`, as it does for a + * notebook on the extension's servers. Returns the function that removes the frame and the folders the store created. + */ +function storeFrameWithToolkit(): () => void { + const work = path.join(process.env.HOME || os.homedir(), 'work'); + const frames = path.join(work, 'deepnote_dataframes'); + const created = [frames, work].filter((folder) => !fs.existsSync(folder)); + const removeFrame = () => { + fs.rmSync(path.join(frames, FRAME_NAME), { recursive: true, force: true }); + // Only while empty: a kernel of another test may have written to them since. + for (const folder of created.filter((candidate) => fs.existsSync(candidate))) { + try { + fs.rmdirSync(folder); + } catch (error) { + console.warn(`[deepnote-e2e] kept ${folder}:`, error); + } + } + }; + try { + execFileSync( + venvPython(), + ['-I', '-c', STORE_FRAME_SCRIPT, path.join(DATAFRAMES_FIXTURE, FRAME_FILE), FRAME_NAME], + { timeout: STORE_FRAME_TIMEOUT } + ); + } catch (error) { + removeFrame(); + throw error; + } + + return removeFrame; +} + describe('Deepnote: Preview App — serve a data app folder in the built-in browser', function () { this.timeout(SUITE_TIMEOUT); @@ -238,6 +379,15 @@ describe('Deepnote: Preview App — serve a data app folder in the built-in brow return appDir; } + function addDataframesProject(): string { + const project = fs.mkdtempSync(path.join(workspaceDir, 'dataframes-')); + fs.mkdirSync(path.join(project, DATAFRAMES_APP)); + fs.copyFileSync(path.join(DATAFRAMES_FIXTURE, 'index.html'), path.join(project, DATAFRAMES_APP, 'index.html')); + copyFixtureIntoDir(project, PROJECT_FIXTURE); + + return path.basename(project); + } + before(async function () { workspaceDir = fs.mkdtempSync(path.join(os.tmpdir(), 'deepnote-e2e-preview-')); fs.mkdirSync(path.join(workspaceDir, '.vscode')); @@ -401,6 +551,48 @@ describe('Deepnote: Preview App — serve a data app folder in the built-in brow expect(Date.parse(createdAt ?? ''), `the run's createdAt, ${createdAt}`).to.be.within(startedAt, Date.now()); }); + it('pages through a dataframe deepnote-toolkit stored under $HOME/work, its default, across its record batches', async function () { + const screenshot = createScreenshotter(this); + const removeFrame = storeFrameWithToolkit(); + try { + const folder = addDataframesProject(); + await selectDeepnoteContextMenu(await findExplorerChildItem(folder, DATAFRAMES_APP), 'Preview App'); + + await onIntegratedBrowserPage(DATAFRAMES_TITLE, async () => { + const driver = VSBrowser.instance.driver; + const clickButton = (name: string) => + driver.findElement(By.xpath(`//button[normalize-space()='${name}']`)).click(); + const ids = (page: DataframeSitePage) => page.rows.map(([id]) => id); + + const first = await readDataframePage('Page 1 of 20,000'); + expect(first.summary).to.include('200,000 rows · 7 columns'); + expect(first.rows[0]).to.deep.equal(FIRST_ROW); + expect(ids(first)).to.deep.equal(consecutiveIds(0, 10)); + await screenshot('first-page'); + + await clickButton('Next'); + expect(ids(await readDataframePage('Page 2 of 20,000'))).to.deep.equal(consecutiveIds(10, 10)); + await screenshot('next-page'); + + // deepnote-toolkit caps a record batch at 65,536 rows, so rows 65,530 to 65,539 straddle two batches. + const pageNumber = driver.findElement(By.css('input[type="number"]')); + await pageNumber.clear(); + await pageNumber.sendKeys('6554'); + await clickButton('Go'); + expect(ids(await readDataframePage('Page 6,554 of 20,000'))).to.deep.equal(consecutiveIds(65_530, 10)); + await screenshot('across-record-batches'); + + await clickButton('Last'); + const last = await readDataframePage('Page 20,000 of 20,000'); + expect(last.rows.at(-1)).to.deep.equal(LAST_ROW); + expect(last.isNextDisabled).to.equal(true); + await screenshot('last-page'); + }); + } finally { + removeFrame(); + } + }); + it('switches to the open tab and reloads it when the same folder is previewed again', async function () { const screenshot = createScreenshotter(this); const driver = VSBrowser.instance.driver;