Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,13 @@ Run `Deepnote: Preview App` from the Command Palette, or right-click a folder in
- The folder is served the way Deepnote serves a published app, from a local address. Edit a file and reload the page to see the change.
- The page's script tag for Deepnote's client (`https://deepnote.com/static/app-client/v1.js`) loads a copy of that client bundled with the extension instead, so the preview works without network access.
- `inputs()` returns the input blocks of the notebook with that ID in the workspace's `.deepnote` files, as saved on disk, so save the notebook and reload the page to see a change. An ID that no workspace notebook has rejects with `Notebook not found`.
- The page's other calls to Deepnote reach a local stand-in that does not support them yet: `appData()` and `run()` reject with a message saying so, and VS Code shows a warning the first time the app calls each of them.
- `run()` runs that notebook, as saved on disk, in a new kernel on the Deepnote server the extension runs for its `.deepnote` file: the project's init notebook first, then the notebook with the page's input values.
- It uses the notebook's Python interpreter. Its SQL blocks use the integrations configured in VS Code, except BigQuery integrations that use Google OAuth.
- Agent blocks run only when the model provider's API key, such as `OPENAI_API_KEY`, is set in VS Code's own environment. The key saved for agent cells is not used.
- Like a run in Deepnote, it is not read-only: what its code writes, such as files or stored dataframes, is written.
- A run keeps going until its notebook finishes, even after you close the preview. Reloading or closing the VS Code window stops it.
- For a legacy `.deepnote` file that holds several notebooks, the first run shows the prompt to split the file, as opening the file does.
- The page's other calls to Deepnote reach a local stand-in that does not support them yet: `appData()` rejects with a message saying so, and VS Code shows a warning the first time the app calls it.
- Previewing needs a trusted workspace and the desktop app.

### Database integrations
Expand Down
202 changes: 195 additions & 7 deletions src/notebooks/deepnote/appPreview/appPreviewApi.node.ts
Original file line number Diff line number Diff line change
@@ -1,32 +1,121 @@
import { getInputBlockValueOverrideValidationError, isInputBlock } from '@deepnote/blocks';
// TODO: Import from `@deepnote/cloud` once it exports the API's generated types. The mock package is only a
// devDependency, which is fine for a type-only import because the build erases it.
import type { operations } from '@deepnote/cloud-mock';
import { timingSafeEqual } from 'crypto';
import express, { type NextFunction, type Request, type Response, type Router } from 'express';
import { CancellationTokenSource } from 'vscode';
import { z } from 'zod';

import { generateUuid } from '../../../platform/common/uuid';
import type { DeepnoteNotebook } from '../../../platform/deepnote/deepnoteTypes';
import { logger } from '../../../platform/logging';
import { blocksInNotebookOrder, toNotebookInputs, type WorkspaceNotebook } from './appPreviewWorkspace.node';
import type { AppPreviewInputValue, AppPreviewRunOutcome, AppPreviewRunRequest, IAppPreviewRunner } from './types';

export type AppPreviewOperation = 'appData()' | 'run()' | 'dataframe()';
export type AppPreviewOperation = 'appData()' | 'dataframe()';

export interface AppPreviewApiOptions {
token: string;
findNotebook(notebookId: string): Promise<WorkspaceNotebook | undefined>;
onUnsupported(operation: AppPreviewOperation): void;
runner: IAppPreviewRunner;
}

/** A run as Deepnote's `GET /v2/runs/:runId` answers it, without the snapshot fields a local run never has. */
export interface PreviewRun {
completedAt: string | null;
createdAt: string;
error: string | null;
notebookId: string;
runId: string;
snapshotBlocks: AppPreviewRunOutcome['snapshotBlocks'];
snapshotStatus: 'pending' | 'available' | 'unavailable';
status: 'pending' | 'running' | AppPreviewRunOutcome['status'];
}

export interface AppPreviewApi {
router: Router;
/**
* The preview's runs by ID, in the order they started. Besides unfinished runs and each notebook's newest
* successful run, it keeps only the newest {@link MAX_FINISHED_RUNS} finished runs.
*/
runs: Map<string, PreviewRun>;
/** Cancels every unfinished run, which stops its kernel once it has one. */
dispose(): void;
}

// Protocol strings: app code receives them, so they stay English like Deepnote's API messages.
const UNAUTHORIZED_MESSAGE = 'Unauthorized';
const NOT_AVAILABLE_MESSAGE = 'This endpoint is not available to app viewer tokens';
const NOTEBOOK_NOT_FOUND_MESSAGE = 'Notebook not found';
const RUN_NOT_FOUND_MESSAGE = 'Run not found';
const INTERNAL_ERROR_MESSAGE = 'Internal server error';

type AppViewerNotebook = operations['getNotebook']['responses'][200]['content']['application/json']['notebook'];

// Each finished run holds all its outputs, and a page may start one on every reload.
const MAX_FINISHED_RUNS = 20;

const createRunBodySchema = z.strictObject({
notebookId: z.string().min(1),
inputs: z.record(z.string().min(1), z.union([z.string(), z.boolean(), z.array(z.string())])).optional(),
detachedRunStorageMode: z.enum(['read_write', 'readonly']).optional()
});

function notSupportedMessage(operation: AppPreviewOperation): string {
return `${operation} is not supported in the local preview yet.`;
}

/** The 4xx status of an error body-parser raises for the request itself, such as malformed JSON. */
function clientErrorStatus(error: unknown): number | undefined {
if (typeof error !== 'object' || error === null || !('status' in error) || !('expose' in error)) {
return undefined;
}

return typeof error.status === 'number' && error.status < 500 && error.expose === true ? error.status : undefined;
}

function errorMessage(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}

/**
* Drops the oldest finished runs beyond {@link MAX_FINISHED_RUNS}. Unfinished runs stay, and so does each notebook's
* newest successful run, which `appData()` answers from.
*/
function evictFinishedRuns(runs: Map<string, PreviewRun>): void {
const newestSuccess = new Map<string, string>();
for (const run of runs.values()) {
if (run.status === 'success') {
newestSuccess.set(run.notebookId, run.runId);
}
}
const removable = [...runs.values()].filter(
(run) => run.completedAt !== null && newestSuccess.get(run.notebookId) !== run.runId
);
removable.slice(0, Math.max(0, removable.length - MAX_FINISHED_RUNS)).forEach((run) => runs.delete(run.runId));
}

/** The input check Deepnote's `POST /v2/runs` makes before it accepts a run. */
function inputsError(notebook: DeepnoteNotebook, inputs: Record<string, AppPreviewInputValue>): string | undefined {
const inputBlocks = notebook.blocks.filter(isInputBlock);
for (const [name, value] of Object.entries(inputs)) {
const blocks = inputBlocks.filter((block) => block.metadata.deepnote_variable_name === name);
if (blocks.length === 0) {
return `Input "${name}" is not defined for this notebook`;
}
for (const block of blocks) {
const error = getInputBlockValueOverrideValidationError(block, value);
if (error) {
return `Input "${name}" ${error}`;
}
}
}

return undefined;
}

function isAuthorized(authorizationHeader: string | undefined, token: string): boolean {
if (authorizationHeader === undefined) {
return false;
Expand All @@ -38,6 +127,17 @@ function isAuthorized(authorizationHeader: string | undefined, token: string): b
return actual.length === expected.length && timingSafeEqual(actual, expected);
}

/** Deepnote's 400 message for a body that fails its schema. */
function readableErrors(error: z.ZodError): string {
return error.issues
.map((issue) =>
issue.path.length === 0
? issue.message
: `${issue.path.map(String).join('.')} ${issue.message.toLowerCase()}`
)
.join('; ');
}

/**
* The `notebook` of Deepnote's `GET /v2/notebooks/:notebookId` answer. Notebooks carry no timestamps in a `.deepnote`
* file, so `createdAt` is the file's, and a local notebook has no Deepnote schedule or runs.
Expand All @@ -60,17 +160,48 @@ function toAppViewerNotebook({ file, notebook }: WorkspaceNotebook): AppViewerNo
/**
* The local stand-in for Deepnote's app viewer API, mounted at `/v2` on the preview server.
* Requests without the preview's bearer token get 401. `GET /notebooks/:notebookId` answers from the workspace's
* notebooks, the other routes Deepnote's client calls or is about to call get 501, and every other route gets 403, as
* Deepnote answers app viewer tokens. A handler that fails is logged and gets a JSON 500.
* notebooks, and `POST /runs` runs one with `runner`, which `GET /runs/:runId` then reports on. The other routes
* Deepnote's client calls or is about to call get 501, and every other route gets 403, as Deepnote answers app viewer
* tokens. A body that cannot be read, such as malformed JSON, gets its 4xx; any other failure is logged and gets a JSON
* 500.
*/
export function createAppPreviewApi(options: AppPreviewApiOptions): Router {
export function createAppPreviewApi(options: AppPreviewApiOptions): AppPreviewApi {
const router = express.Router();
const runs = new Map<string, PreviewRun>();
const unfinished = new Set<CancellationTokenSource>();

const unsupported = (operation: AppPreviewOperation) => (_req: Request, res: Response) => {
options.onUnsupported(operation);
res.status(501).json({ message: notSupportedMessage(operation) });
};

const startRun = (run: PreviewRun, request: AppPreviewRunRequest) => {
const cancellation = new CancellationTokenSource();
unfinished.add(cancellation);
void options.runner
.run(request, cancellation.token, () => {
run.status = 'running';
})
.catch((error): AppPreviewRunOutcome => {
logger.error(`AppPreviewApi: Run ${run.runId} failed`, error);

return { status: 'internal_error', error: errorMessage(error), snapshotBlocks: null };
})
.then((outcome) => {
run.status = outcome.status;
run.error = outcome.error;
run.snapshotBlocks = outcome.snapshotBlocks;
run.snapshotStatus =
outcome.status === 'success' || outcome.status === 'error' ? 'available' : 'unavailable';
run.completedAt = new Date().toISOString();
evictFinishedRuns(runs);
})
.finally(() => {
unfinished.delete(cancellation);
cancellation.dispose();
});
};

router.use((req: Request, res: Response, next: NextFunction) => {
if (!isAuthorized(req.headers.authorization, options.token)) {
res.status(401).json({ message: UNAUTHORIZED_MESSAGE });
Expand All @@ -90,18 +221,75 @@ export function createAppPreviewApi(options: AppPreviewApiOptions): Router {
res.json({ notebook: toAppViewerNotebook(found) });
});
router.get('/notebooks/:notebookId/app-data', unsupported('appData()'));
router.post('/runs', unsupported('run()'));
router.get('/runs/:runId', unsupported('run()'));
router.post('/runs', express.json(), async (req: Request, res: Response) => {
const parsed = createRunBodySchema.safeParse(req.body, {
// Deepnote's error map, so a missing field reads `is required`.
error: (issue) => (issue.code === 'invalid_type' && issue.input === undefined ? 'is required' : undefined)
});
if (!parsed.success) {
res.status(400).json({ message: readableErrors(parsed.error) });

return;
}

const { notebookId, inputs = {} } = parsed.data;
const found = await options.findNotebook(notebookId);
// Unlike its GET routes, Deepnote answers 400 here.
if (!found) {
res.status(400).json({ message: NOTEBOOK_NOT_FOUND_MESSAGE });

return;
}
const inputError = inputsError(found.notebook, inputs);
if (inputError) {
res.status(400).json({ message: inputError });

return;
}

const run: PreviewRun = {
runId: generateUuid(),
notebookId,
status: 'pending',
snapshotStatus: 'pending',
snapshotBlocks: null,
error: null,
createdAt: new Date().toISOString(),
completedAt: null
};
runs.set(run.runId, run);
startRun(run, { fileUri: found.fileUri, notebookId, inputs });
res.status(202).json({ runId: run.runId, status: run.status, createdAt: run.createdAt });
});
router.get('/runs/:runId', (req: Request<{ runId: string }>, res: Response) => {
const run = runs.get(req.params.runId);
if (!run) {
res.status(404).json({ message: RUN_NOT_FOUND_MESSAGE });

return;
}
res.json({ run: { ...run, snapshotExpiresAt: null, snapshotContent: null, snapshotDownloadUrl: null } });
});
router.get('/projects/:projectId/dataframes/:name', unsupported('dataframe()'));

router.use((_req: Request, res: Response) => {
res.status(403).json({ message: NOT_AVAILABLE_MESSAGE });
});
// Express's own handler would answer an HTML page that holds the stack, and log it outside the extension's logger.
router.use((error: unknown, req: Request, res: Response, _next: NextFunction) => {
const status = clientErrorStatus(error);
if (status !== undefined) {
res.status(status).json({ message: errorMessage(error) });

return;
}
logger.error(`AppPreviewApi: ${req.method} ${req.path} failed`, error);
res.status(500).json({ message: INTERNAL_ERROR_MESSAGE });
});

return router;
return {
router,
runs,
dispose: () => unfinished.forEach((cancellation) => cancellation.cancel())
};
}
Loading
Loading