diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index accff02..ce7925d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,10 @@ jobs: steps: - uses: actions/checkout@v5 - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v6 + with: + node-version: 24 + - run: echo "VANA_PDPP_NODE=$(command -v node)" >> "$GITHUB_ENV" - uses: actions/setup-node@v6 with: node-version: ${{ matrix.node-version }} diff --git a/scripts/prepare-vhs-fixtures.mjs b/scripts/prepare-vhs-fixtures.mjs index 4ba58fe..0e3b1e2 100644 --- a/scripts/prepare-vhs-fixtures.mjs +++ b/scripts/prepare-vhs-fixtures.mjs @@ -1,4 +1,5 @@ import fs from "node:fs/promises"; +import { createHash } from "node:crypto"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -117,6 +118,34 @@ async function seedDemoHome() { await fs.writeFile(browserPath, "", "utf8"); await fs.chmod(browserPath, 0o755); + const demoAddress = "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const githubResultPath = path.join(dataConnectRoot, "last-result.json"); + const githubResult = `${JSON.stringify( + { + profile: { username: "tnunamak" }, + repositories: [{ name: "vana-connect" }, { name: "data-connectors" }], + starred: [], + }, + null, + 2, + )}\n`; + await fs.writeFile( + path.join(dataConnectRoot, "auth.json"), + `${JSON.stringify( + { + account: { + address: demoAddress, + session_token: "expired-demo-token", + expires_at: "2000-01-01T00:00:00.000Z", + }, + personal_server: null, + }, + null, + 2, + )}\n`, + "utf8", + ); + const state = { version: 1, sources: { @@ -125,7 +154,17 @@ async function seedDemoHome() { lastRunAt: "2026-03-14T13:10:03.677Z", lastRunOutcome: "connected_local_only", dataState: "collected_local", - lastResultPath: path.join(dataConnectRoot, "last-result.json"), + lastResultPath: githubResultPath, + exportReceipt: { + version: 1, + id: "demo-github-export", + owner: { + accountUrl: "https://account.vana.org", + address: demoAddress, + }, + path: githubResultPath, + sha256: createHash("sha256").update(githubResult).digest("hex"), + }, lastLogPath: path.join(dataConnectRoot, "logs", "run-github-demo.log"), }, shop: { @@ -156,19 +195,7 @@ async function seedDemoHome() { "utf8", ); - await fs.writeFile( - path.join(dataConnectRoot, "last-result.json"), - `${JSON.stringify( - { - profile: { username: "tnunamak" }, - repositories: [{ name: "vana-connect" }, { name: "data-connectors" }], - starred: [], - }, - null, - 2, - )}\n`, - "utf8", - ); + await fs.writeFile(githubResultPath, githubResult, "utf8"); await fs.writeFile( path.join(dataConnectRoot, "spotify-result.json"), `${JSON.stringify( diff --git a/src/cli/auth.ts b/src/cli/auth.ts index d2d7708..f1da477 100644 --- a/src/cli/auth.ts +++ b/src/cli/auth.ts @@ -12,6 +12,7 @@ import fsp from "node:fs/promises"; import path from "node:path"; import { getVanaHome } from "../core/paths.js"; import { formatDisplayPath } from "./render/format.js"; +import type { ExportOwner } from "../core/source-export.js"; export interface VanaCredentials { account: { @@ -175,6 +176,14 @@ export function loadAccountAddressForOwnership(): string | null { return process.env.VANA_SESSION_TOKEN ? null : readStoredAccountAddress(); } +/** Capture identity independently of token expiry, never from a destination server. */ +export function loadExportOwner(): ExportOwner { + const address = loadAccountAddressForOwnership(); + if (!address || !/^0x[a-f0-9]{40}$/i.test(address)) return null; + const accountUrl = new URL(getAccountUrl()).toString().replace(/\/+$/, ""); + return { accountUrl, address: address.toLowerCase() }; +} + /** * Raw peek at auth.json ignoring expiry: the previous address and the * previous personal_server block. Login uses it to detect an account switch diff --git a/src/cli/index.ts b/src/cli/index.ts index a616957..5064d48 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -5,6 +5,14 @@ import { createRequire } from "node:module"; import { fileURLToPath } from "node:url"; import { spawn, execSync } from "node:child_process"; import os from "node:os"; +import crypto from "node:crypto"; +import { + exportDigest, + hasUnresolvedExport, + pendingExportBlockReason, + readExportReceipt, +} from "../core/source-export.js"; +import type { SourceExportReceipt } from "../core/source-export.js"; import { confirm, input, isPromptInputClosed, password } from "./prompts.js"; import { searchSelect } from "./search-select.js"; @@ -48,6 +56,7 @@ import { getLogsDir, getSessionsDir, getSourceResultPath, + getResultsDir, readCliState, readCliConfig, recordScheduledRun, @@ -171,6 +180,7 @@ import { } from "./update-check.js"; import { loadAccountAddressForOwnership, + loadExportOwner, loadCredentials, loadPersonalServerSession, readStoredAuthFile, @@ -1801,6 +1811,14 @@ async function runConnect( connectOptions: ConnectOptions = {}, ): Promise { const source = rawSource.toLowerCase(); + const collectionOwner = loadExportOwner(); + const exportId = crypto.randomUUID(); + const resultKey = path.basename(getSourceResultPath(source), ".json"); + const runResultPath = path.join( + getResultsDir(), + resultKey, + `${exportId}.json`, + ); const pdppSource = await resolvePdppSource(source, { from: connectOptions.from, }); @@ -1822,6 +1840,24 @@ async function runConnect( // Title renderer?.title(displayName); + const beforeCollection = await readCliState(); + const pendingBlock = pendingExportBlockReason( + beforeCollection.sources[source], + collectionOwner, + ); + if (pendingBlock) { + renderer?.fail(pendingBlock); + if (!renderer && !options.json) process.stderr.write(`${pendingBlock}\n`); + emit.event({ + type: "outcome", + status: CliOutcomeStatus.PERSONAL_SERVER_UNAVAILABLE, + source, + reason: "pending_export_account_mismatch", + message: pendingBlock, + }); + return CliExitCode.SERVER_UNAVAILABLE; + } + const target = await withPhaseProgress( renderer, "Finding your Personal Server", @@ -1831,9 +1867,10 @@ async function runConnect( // Collecting into a server owned by another identity looks like success // and is not undoable, so settle ownership before touching the source. const credentials = loadCredentials(); + let allowForeignServer = false; const mismatch = personalServerOwnerMismatch( target.health?.owner, - loadAccountAddressForOwnership(), + collectionOwner?.address, ); if (mismatch) { // An expired login names whoever signed in last, which may be long @@ -1875,6 +1912,7 @@ async function runConnect( ); return CliExitCode.CONFIRMATION_REQUIRED; } + allowForeignServer = true; } } @@ -1883,7 +1921,10 @@ async function runConnect( const runtime = isPdpp ? new PdppRuntime({ from: connectOptions.from, - owner: target.health?.owner ?? loadCredentials()?.account?.address, + owner: + collectionOwner?.address ?? + target.health?.owner ?? + loadCredentials()?.account?.address, }) : new ManagedPlaywrightRuntime(); @@ -2243,9 +2284,11 @@ async function runConnect( }; const shownBrowserPrompts = new Set(); + let acceptedReceipt: SourceExportReceipt | undefined; for await (const event of runtime.runConnector({ connectorPath: resolution.connectorPath, source: resolution.source, + resultPath: runResultPath, noInput: options.ipc ? false : options.noInput, onNeedInput: interactiveCallback, })) { @@ -2395,10 +2438,10 @@ async function runConnect( continue; } - // Check if the result is actually an error object + let exportBytes: Buffer; try { - const raw = await fsp.readFile(event.resultPath, "utf8"); - const parsed = JSON.parse(raw); + exportBytes = await fsp.readFile(event.resultPath); + const parsed = JSON.parse(exportBytes.toString("utf8")); const errorOnly = parsed && typeof parsed === "object" && @@ -2447,14 +2490,52 @@ async function runConnect( await updateSourceState(source, { lastError: `Failed to parse result file (${event.resultPath}): ${msg}`, }); + pendingExitCode = CliExitCode.FAILURE; + continue; } - collectedResult = true; resultPath = event.resultPath; + const receipt: SourceExportReceipt = { + version: 1, + id: exportId, + owner: collectionOwner, + path: resultPath, + sha256: exportDigest(exportBytes), + }; + acceptedReceipt = receipt; + const admitted = await updateSourceState( + resolution.source, + { + exportReceipt: receipt, + lastResultPath: resultPath, + dataState: "collected_local", + ingestScopes: undefined, + lastCollectedAt: new Date().toISOString(), + }, + { publish: receipt }, + ); + if (!admitted) { + const message = `Another account's pending export occupies ${source}. This run was kept locally at ${resultPath}; the accepted export was preserved. Run \`vana data show ${source}\` to inspect the accepted export.`; + renderer?.fail(message); + if (!renderer && !options.json) process.stderr.write(`${message}\n`); + emit.event({ + type: "outcome", + source, + status: CliOutcomeStatus.PERSONAL_SERVER_UNAVAILABLE, + reason: "pending_export_changed", + message, + resultPath, + }); + return CliExitCode.SERVER_UNAVAILABLE; + } + collectedResult = true; const ingestEvents = await withPhaseProgress( renderer, "Saving to your Personal Server", - () => ingestResult(resolution.source, resultPath, target), + () => + ingestResult(resolution.source, resultPath, target, { + export: { kind: "fresh", receipt, allowForeignServer }, + }), ); for (const ingestEvent of ingestEvents) { emit.event(ingestEvent); @@ -2481,6 +2562,11 @@ async function runConnect( ingestEvent.type === "ingest-skipped" && ingestEvent.reason === "personal_server_unavailable", ); + const blockedExport = ingestEvents.find( + (ingestEvent) => + ingestEvent.type === "ingest-skipped" && + ingestEvent.reason?.startsWith("export_"), + ); if (ingestCompleted) { finalStatus = CliOutcomeStatus.CONNECTED_AND_INGESTED; finalDataState = "ingested_personal_server"; @@ -2492,6 +2578,11 @@ async function runConnect( finalDataState = "ingest_failed"; ingestFailureMessage = ingestFailedEvent.message ?? "Personal Server sync failed."; + } else if (blockedExport) { + finalStatus = CliOutcomeStatus.PERSONAL_SERVER_UNAVAILABLE; + finalDataState = "collected_local"; + ingestFailureMessage = blockedExport.message ?? null; + pendingExitCode = CliExitCode.SERVER_UNAVAILABLE; } else if (ingestSkippedUnavailable) { finalStatus = CliOutcomeStatus.CONNECTED_LOCAL_ONLY; finalDataState = "ingest_unavailable"; @@ -2517,7 +2608,7 @@ async function runConnect( return pendingExitCode; } - if (!collectedResult) { + if (!collectedResult || !acceptedReceipt) { await updateSourceState(resolution.source, { connectorInstalled: true, sessionPresent: fs.existsSync(profilePath), @@ -2537,27 +2628,48 @@ async function runConnect( return 1; } - await updateSourceState(resolution.source, { - connectorInstalled: true, - connectorVersion: fetched.version, - exportFrequency: fetched.exportFrequency, - sessionPresent: true, - lastRunAt: new Date().toISOString(), - lastCollectedAt: new Date().toISOString(), - lastRunOutcome: finalStatus, - dataState: finalDataState, - lastError: ingestFailureMessage, - lastResultPath: resultPath, - lastLogPath: runLogPath ?? fetchLogPath ?? setupLogPath ?? null, - connectionHealth: pendingExitCode !== null ? undefined : "healthy", - connectionHealthChangedAt: - pendingExitCode !== null ? undefined : new Date().toISOString(), - connectionHealthReason: - pendingExitCode !== null ? undefined : "collection-complete", - connectionHealthRetryable: undefined, - ingestScopes: ingestScopeResults, - skippedStreams: skippedStreams.length > 0 ? skippedStreams : undefined, - }); + const settled = await updateSourceState( + resolution.source, + { + connectorInstalled: true, + connectorVersion: fetched.version, + exportFrequency: fetched.exportFrequency, + sessionPresent: true, + lastRunAt: new Date().toISOString(), + lastCollectedAt: new Date().toISOString(), + lastRunOutcome: finalStatus, + dataState: finalDataState, + lastError: ingestFailureMessage, + lastResultPath: resultPath, + lastLogPath: runLogPath ?? fetchLogPath ?? setupLogPath ?? null, + connectionHealth: pendingExitCode !== null ? undefined : "healthy", + connectionHealthChangedAt: + pendingExitCode !== null ? undefined : new Date().toISOString(), + connectionHealthReason: + pendingExitCode !== null ? undefined : "collection-complete", + connectionHealthRetryable: undefined, + ingestScopes: ingestScopeResults, + skippedStreams: skippedStreams.length > 0 ? skippedStreams : undefined, + }, + { settle: acceptedReceipt.id }, + ); + if (!settled || pendingExitCode === CliExitCode.SERVER_UNAVAILABLE) { + const message = !settled + ? `A newer export replaced this run's state. This run remains at ${resultPath}; its sync did not change the newer export.` + : (ingestFailureMessage ?? + "Export kept locally; sync requires the collecting account."); + renderer?.fail(message); + if (!renderer && !options.json) process.stderr.write(`${message}\n`); + emit.event({ + type: "outcome", + source, + status: CliOutcomeStatus.PERSONAL_SERVER_UNAVAILABLE, + reason: !settled ? "export_changed" : "export_ownership_unverified", + message, + resultPath, + }); + return CliExitCode.SERVER_UNAVAILABLE; + } // Build scope-aware success summary const storedCount = @@ -4538,6 +4650,7 @@ interface SyncPendingResult { syncedCount: number; sourceResults: Array<{ source: string; + blocked?: string; scopeResults?: Array<{ scope: string; status: string; error?: string }>; }>; } @@ -4573,7 +4686,7 @@ function shouldRetryPendingSource( function buildRetryOptions( stored: StoredSourceState | undefined, mode: SyncRetryMode, -): IngestResultOptions | undefined { +): Pick | undefined { if (!stored) { return undefined; } @@ -4669,8 +4782,12 @@ async function syncPendingSources( mode: SyncRetryMode, ): Promise { const state = await readCliState(); - const pendingSources = Object.entries(state.sources).filter(([, stored]) => - shouldRetryPendingSource(stored, mode), + const selectedOwner = loadExportOwner(); + const pendingSources = Object.entries(state.sources).filter( + ([, stored]) => + shouldRetryPendingSource(stored, mode) || + (hasUnresolvedExport(stored) && + Boolean(pendingExportBlockReason(stored, selectedOwner))), ); if (pendingSources.length === 0) { @@ -4686,12 +4803,29 @@ async function syncPendingSources( } const ingestOptions = buildRetryOptions(stored, mode); + const receipt = readExportReceipt(stored.exportReceipt); const ingestEvents = await ingestResult( source, stored.lastResultPath, target, - ingestOptions, + { + ...ingestOptions, + export: { kind: "retry", receipt: stored.exportReceipt }, + }, + ); + const blockedExport = ingestEvents.find( + (event) => + event.type === "ingest-skipped" && event.reason?.startsWith("export_"), ); + if (blockedExport) { + sourceResults.push({ + source, + blocked: + blockedExport.message ?? + "Export remains local until its collecting account and server are verified.", + }); + continue; + } const resultEvent = ingestEvents.find( (event) => @@ -4716,17 +4850,32 @@ async function syncPendingSources( mergedScopes, stored.dataState as SourceStatus["dataState"] | null | undefined, ); - await updateSourceState(source, { - dataState, - ingestScopes: mergedScopes, - lastError: summarizeSyncError(mergedScopes), - // The run's failed sync is now done; leaving the outcome would keep - // the source under "Needs attention". - ...(stored.lastRunOutcome === CliOutcomeStatus.INGEST_FAILED && - dataState === "ingested_personal_server" - ? { lastRunOutcome: CliOutcomeStatus.CONNECTED_AND_INGESTED } - : {}), - }); + const settled = + receipt && + (await updateSourceState( + source, + { + dataState, + ingestScopes: mergedScopes, + lastError: summarizeSyncError(mergedScopes), + // The run's failed sync is now done; leaving the outcome would keep + // the source under "Needs attention". + ...(stored.lastRunOutcome === CliOutcomeStatus.INGEST_FAILED && + dataState === "ingested_personal_server" + ? { lastRunOutcome: CliOutcomeStatus.CONNECTED_AND_INGESTED } + : {}), + }, + { settle: receipt.id }, + )); + if (!settled) { + syncedCount--; + sourceResults.push({ + source, + blocked: + "A newer export replaced this queued export while syncing; the newer state was preserved.", + }); + continue; + } } sourceResults.push({ source, scopeResults: resultEvent?.scopeResults }); @@ -4755,9 +4904,15 @@ async function runCollectAll(options: GlobalOptions): Promise { connectExitCode = result; } const after = await readCliState(); + const blocked = + result === CliExitCode.SERVER_UNAVAILABLE + ? pendingExportBlockReason(after.sources[source], loadExportOwner()) + : null; results.set( source, - classifyCollectedSource(source, result, after.sources[source]), + blocked + ? { source, outcome: "sync_pending", error: blocked } + : classifyCollectedSource(source, result, after.sources[source]), ); } @@ -4774,7 +4929,7 @@ async function runCollectAll(options: GlobalOptions): Promise { const after = await readCliState(); const error = `Pending data not synced: the Personal Server at ${target.url} belongs to ${formatAddress(foreign.owner)}, not to you. Start yours with \`vana server start\`.`; for (const [source, stored] of Object.entries(after.sources)) { - if (!shouldRetryPendingSource(stored, "automatic")) continue; + if (!hasUnresolvedExport(stored)) continue; blockedPendingCount++; const prior = results.get(source)?.outcome; if (prior === "collect_failed" || prior === "sync_failed") continue; @@ -4789,6 +4944,15 @@ async function runCollectAll(options: GlobalOptions): Promise { syncedPendingCount = synced.syncedCount; for (const entry of synced.sourceResults) { if (results.get(entry.source)?.outcome === "collect_failed") continue; + if (entry.blocked) { + blockedPendingCount++; + results.set(entry.source, { + source: entry.source, + outcome: "sync_pending", + error: entry.blocked, + }); + continue; + } const failed = entry.scopeResults?.find((r) => r.status === "failed"); results.set( entry.source, @@ -4818,7 +4982,7 @@ async function runCollectAll(options: GlobalOptions): Promise { syncedPendingCount > 0 ? `Synced ${syncedPendingCount} pending dataset(s).` : blockedPendingCount > 0 - ? `${blockedPendingCount} pending dataset(s) not synced: the Personal Server belongs to another account.` + ? `${blockedPendingCount} pending dataset(s) kept locally; the collection account or server could not be verified.` : "No sources are due for collection."; process.stdout.write( `${JSON.stringify({ message, count: 0, syncedPendingCount, sources: sourceResults })}\n`, @@ -4915,6 +5079,27 @@ async function runServerSync(options: GlobalOptions): Promise { } const syncResult = await syncPendingSources(target, "manual"); + const blockedExports = syncResult.sourceResults.filter( + (entry) => entry.blocked, + ); + if (blockedExports.length > 0) { + const message = blockedExports + .map((entry) => `${entry.source}: ${entry.blocked}`) + .join("\n"); + if (options.json) { + process.stdout.write( + `${JSON.stringify({ + error: "pending_exports_blocked", + message, + syncedCount: syncResult.syncedCount, + sources: syncResult.sourceResults, + })}\n`, + ); + } else { + process.stderr.write(`${message}\n`); + } + return CliExitCode.SERVER_UNAVAILABLE; + } const storedScopeCount = syncResult.sourceResults.reduce( (total, entry) => total + diff --git a/src/core/source-export.ts b/src/core/source-export.ts new file mode 100644 index 0000000..df55b35 --- /dev/null +++ b/src/core/source-export.ts @@ -0,0 +1,90 @@ +import crypto from "node:crypto"; + +import type { StoredSourceState } from "./state-store.js"; + +/** The Account deployment and wallet that collected an export; null is unknown. */ +export type ExportOwner = { accountUrl: string; address: string } | null; + +/** Accepted bytes and collection identity, independent of the current login. */ +export interface SourceExportReceipt { + version: 1; + id: string; + owner: ExportOwner; + path: string; + sha256: string; +} + +/** Persisted receipts are untrusted; legacy or malformed records remain local. */ +export function readExportReceipt(value: unknown): SourceExportReceipt | null { + if (!value || typeof value !== "object") return null; + const receipt = value as Partial; + if ( + receipt.version !== 1 || + typeof receipt.id !== "string" || + !receipt.id || + typeof receipt.path !== "string" || + !receipt.path || + typeof receipt.sha256 !== "string" || + !/^[a-f0-9]{64}$/.test(receipt.sha256) + ) + return null; + const owner = receipt.owner; + if ( + owner !== null && + (!owner || + typeof owner.accountUrl !== "string" || + !owner.accountUrl || + typeof owner.address !== "string" || + !/^0x[a-f0-9]{40}$/i.test(owner.address)) + ) + return null; + return receipt as SourceExportReceipt; +} + +/** SHA-256 of the exact accepted bytes, before parsing or upload projection. */ +export function exportDigest(bytes: Uint8Array): string { + return crypto.createHash("sha256").update(bytes).digest("hex"); +} + +/** Unknown identities never match, even when both exports are unattributed. */ +export function sameExportOwner(a: ExportOwner, b: ExportOwner): boolean { + return Boolean( + a && + b && + a.accountUrl === b.accountUrl && + a.address.toLowerCase() === b.address.toLowerCase(), + ); +} + +/** Failed and partially stored exports still occupy the accepted source slot. */ +export function hasUnresolvedExport( + source: StoredSourceState | undefined, +): boolean { + return Boolean( + source?.lastResultPath && + (source.dataState === "collected_local" || + source.dataState === "ingest_unavailable" || + source.dataState === "ingest_failed" || + source.ingestScopes?.some((scope) => scope.status === "failed")), + ); +} + +/** A pending slot can only be replaced by its known collecting account. */ +export function pendingExportBlockReason( + source: StoredSourceState | undefined, + selected: ExportOwner, +): string | null { + if (!hasUnresolvedExport(source)) return null; + const receipt = readExportReceipt(source?.exportReceipt); + if ( + receipt && + receipt.path === source?.lastResultPath && + sameExportOwner(receipt.owner, selected) + ) + return null; + const location = `Kept locally at ${source?.lastResultPath}. Use \`vana data show\` to inspect it.`; + if (!receipt?.owner || receipt.path !== source?.lastResultPath) { + return `The pending export's collection account cannot be verified. Automatic sync and replacement are disabled. ${location}`; + } + return `Pending export belongs to Account ${receipt.owner.address} at ${receipt.owner.accountUrl}. ${location} Sign in to that collecting account and run \`vana server sync\`.`; +} diff --git a/src/core/state-store.ts b/src/core/state-store.ts index 8de1c2c..b44b264 100644 --- a/src/core/state-store.ts +++ b/src/core/state-store.ts @@ -3,6 +3,11 @@ import path from "node:path"; import crypto from "node:crypto"; import { getCliStatePath, getVanaHome } from "./paths.js"; +import { + pendingExportBlockReason, + readExportReceipt, +} from "./source-export.js"; +import type { SourceExportReceipt } from "./source-export.js"; const STATE_LOCK_TIMEOUT_MS = 5_000; const STATE_LOCK_RETRY_MS = 25; @@ -16,6 +21,7 @@ let testHooks: | undefined; export interface StoredSourceState { + exportReceipt?: SourceExportReceipt; connectorInstalled?: boolean; connectorVersion?: string; exportFrequency?: string; @@ -111,21 +117,44 @@ export async function readCliState(): Promise { } } +type SourceExportGuard = { publish: SourceExportReceipt } | { settle: string }; + +export function updateSourceState( + source: string, + patch: StoredSourceState, +): Promise; +export function updateSourceState( + source: string, + patch: StoredSourceState, + exportGuard: SourceExportGuard, +): Promise; export async function updateSourceState( source: string, patch: StoredSourceState, -): Promise { + exportGuard?: SourceExportGuard, +): Promise { await fs.mkdir(getVanaHome(), { recursive: true }); - await withStateFileLock(async () => { + return withStateFileLock(async () => { await testHooks?.beforeRead?.(); const state = await readCliState(); const current = state.sources[source] ?? {}; + if (exportGuard) { + if ("publish" in exportGuard) { + if (pendingExportBlockReason(current, exportGuard.publish.owner)) + return false; + } else if ( + readExportReceipt(current.exportReceipt)?.id !== exportGuard.settle + ) { + return false; + } + } state.sources[source] = { ...current, ...patch }; await testHooks?.beforeWrite?.(); await atomicWriteFile( getCliStatePath(), `${JSON.stringify(state, null, 2)}\n`, ); + return exportGuard ? true : undefined; }); } diff --git a/src/pdpp/runtime.ts b/src/pdpp/runtime.ts index 84d4633..4daef8f 100644 --- a/src/pdpp/runtime.ts +++ b/src/pdpp/runtime.ts @@ -411,7 +411,7 @@ export class PdppRuntime { records, checkpoints, }); - const resultPath = getSourceResultPath(source); + const resultPath = options.resultPath ?? getSourceResultPath(source); await writeResult( resultPath, projectResult(source, launch.displayName, merged, completed), diff --git a/src/personal-server/index.ts b/src/personal-server/index.ts index e308a61..7634bc3 100644 --- a/src/personal-server/index.ts +++ b/src/personal-server/index.ts @@ -6,8 +6,15 @@ import { resolveScopes } from "./scope-resolver.js"; import { createPersonalServerClient } from "./client.js"; import { readCachedConnectorMetadata } from "../connectors/registry.js"; import { getConnectorCacheDir } from "../core/paths.js"; +import { + exportDigest, + readExportReceipt, + sameExportOwner, +} from "../core/source-export.js"; +import type { SourceExportReceipt } from "../core/source-export.js"; import { loadAccountAddressForOwnership, + loadExportOwner, loadPersonalServerSession, } from "../cli/auth.js"; @@ -50,6 +57,14 @@ export type PersonalServerAuthConfig = export interface IngestResultOptions { scopes?: string[]; + export: + | { kind: "retry"; receipt: unknown } + | { + kind: "fresh"; + receipt: SourceExportReceipt; + /** A person's destination consent applies to this collection only. */ + allowForeignServer: boolean; + }; } /** @@ -152,7 +167,7 @@ export async function ingestResult( source: string, resultPath: string, target: PersonalServerTarget, - options?: IngestResultOptions, + options: IngestResultOptions, ): Promise { if (target.state !== "available" || !target.url) { return [ @@ -164,13 +179,90 @@ export async function ingestResult( ]; } - const raw = await fs.readFile(resultPath, "utf8"); - const result = JSON.parse(raw) as Record; + const receipt = readExportReceipt(options.export.receipt); + const selected = loadExportOwner(); + const blocked = ( + reason: string, + detail: string, + remedy = "Use `vana data show` to inspect it.", + ): CliEvent[] => [ + { + type: "ingest-skipped", + source, + reason, + message: `${detail} Export kept locally at ${resultPath}. ${remedy}`, + }, + ]; + if (!receipt || receipt.path !== resultPath) { + return blocked( + "export_unattributed", + "The export has no valid collection account receipt.", + ); + } + if (options.export.kind === "retry") { + if (!receipt.owner) + return blocked( + "export_unattributed", + "The export's collection account is unknown.", + ); + if (!sameExportOwner(receipt.owner, selected)) { + return blocked( + "export_owner_mismatch", + "The selected Account deployment or account does not own this export.", + `Sign in as ${receipt.owner.address} at ${receipt.owner.accountUrl}, then run \`vana server sync\`.`, + ); + } + if ( + !target.health?.owner || + target.health.owner.toLowerCase() !== receipt.owner.address.toLowerCase() + ) { + return blocked( + "export_server_owner_mismatch", + "The Personal Server has not reported this export's collecting account as its owner.", + "Select the collecting account's Personal Server, then run `vana server sync`.", + ); + } + } else if (receipt.owner) { + if (!sameExportOwner(receipt.owner, selected)) { + return blocked( + "export_owner_mismatch", + "The collecting account changed before this export could be uploaded.", + ); + } + if ( + personalServerOwnerMismatch( + target.health?.owner, + receipt.owner.address, + ) && + !options.export.allowForeignServer + ) { + return blocked( + "export_server_owner_mismatch", + "The Personal Server belongs to another account.", + ); + } + } + let raw: Buffer; + try { + raw = await fs.readFile(resultPath); + } catch { + return blocked( + "export_unreadable", + "The accepted export file cannot be read; no data was uploaded.", + ); + } + if (exportDigest(raw) !== receipt.sha256) { + return blocked( + "export_bytes_changed", + "The export bytes changed after collection and cannot be safely synced.", + ); + } + const result = JSON.parse(raw.toString("utf8")) as Record; const metadata = await readCachedConnectorMetadata( source, getConnectorCacheDir(), ); - const selectedScopes = options?.scopes ? new Set(options.scopes) : null; + const selectedScopes = options.scopes ? new Set(options.scopes) : null; const scopeMappings = resolveScopes(source, result, metadata).filter( (mapping) => !selectedScopes || selectedScopes.has(mapping.scope), ); diff --git a/src/runtime/core/contracts.ts b/src/runtime/core/contracts.ts index 69e509f..1db83ad 100644 --- a/src/runtime/core/contracts.ts +++ b/src/runtime/core/contracts.ts @@ -86,6 +86,8 @@ export interface RuntimeInputRequest { export interface ConnectorRunRequest { connectorPath: string; source: string; + /** Output path for this run; defaults to the source's standard result path. */ + resultPath?: string; noInput?: boolean; signal?: AbortSignal; onNeedInput?: (event: RuntimeInputRequest) => Promise>; diff --git a/src/runtime/managed-playwright.ts b/src/runtime/managed-playwright.ts index 8bd88f2..dffe599 100644 --- a/src/runtime/managed-playwright.ts +++ b/src/runtime/managed-playwright.ts @@ -34,6 +34,8 @@ export interface RuntimeInstallResult { export interface RunConnectorOptions { connectorPath: string; source: string; + /** Output path for this run; defaults to the source's standard result path. */ + resultPath?: string; pretty?: boolean; noInput?: boolean; signal?: AbortSignal; diff --git a/src/runtime/playwright/in-process-run.ts b/src/runtime/playwright/in-process-run.ts index ea7d477..2400bc1 100644 --- a/src/runtime/playwright/in-process-run.ts +++ b/src/runtime/playwright/in-process-run.ts @@ -429,9 +429,10 @@ export function startInProcessConnectorRun({ "data" in result ? (result as { data: unknown }).data : result; - const resultPath = getSourceResultPath(request.source); + const resultPath = + request.resultPath ?? getSourceResultPath(request.source); await ensureParentDir(resultPath); - await rotateResult(request.source); + if (!request.resultPath) await rotateResult(request.source); await fsp.writeFile( resultPath, `${JSON.stringify(exportData, null, 2)}\n`, @@ -780,9 +781,10 @@ function createPageApi({ } if (key === "result") { if (!runState.hasResult) { - const resultPath = getSourceResultPath(request.source); + const resultPath = + request.resultPath ?? getSourceResultPath(request.source); await ensureParentDir(resultPath); - await rotateResult(request.source); + if (!request.resultPath) await rotateResult(request.source); await fsp.writeFile( resultPath, `${JSON.stringify(value, null, 2)}\n`, diff --git a/test/cli/export-owner.test.ts b/test/cli/export-owner.test.ts new file mode 100644 index 0000000..514cc97 --- /dev/null +++ b/test/cli/export-owner.test.ts @@ -0,0 +1,528 @@ +import fs from "node:fs/promises"; +import http from "node:http"; +import os from "node:os"; +import path from "node:path"; +import { execFileSync, spawn } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; + +import { findCompatibleNodeSync } from "../../src/pdpp/host.js"; + +const repo = fileURLToPath(new URL("../../", import.meta.url)); +const A = "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const B = "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const cleanEnv = Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => + !key.startsWith("VANA_") && + !key.startsWith("PDPP_") && + key !== "NODE_OPTIONS", + ), +); +let connectorNode: string; + +beforeAll(() => { + const node = findCompatibleNodeSync(">=24.15.0 <25"); + if (!node) throw new Error("Set VANA_PDPP_NODE to an installed Node 24."); + connectorNode = node.path; + execFileSync( + process.execPath, + ["node_modules/typescript/bin/tsc", "--build"], + { + cwd: repo, + env: cleanEnv, + stdio: "pipe", + }, + ); + execFileSync(process.execPath, ["scripts/copy-vendor.mjs"], { + cwd: repo, + env: cleanEnv, + stdio: "pipe", + }); +}, 60_000); + +interface Upload { + owner: string | null; + authorization: string | undefined; + body: unknown; +} + +interface CommandResult { + code: number | null; + stdout: string; + stderr: string; +} + +describe("account-bound queued exports at the CLI boundary", () => { + let root: string; + let home: string; + let checkout: string; + let executionFile: string; + let server: http.Server; + let url: string; + let owner: string | null; + let rejectUploads: boolean; + let uploads: Upload[]; + let stateAtUpload: any; + let rejectScope: string | undefined; + let uploadGate: (() => Promise) | undefined; + + async function write(file: string, value: unknown) { + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile( + file, + typeof value === "string" ? value : JSON.stringify(value), + ); + } + + async function login(address: string, expired = false) { + owner = address; + await write(path.join(home, "auth.json"), { + account: { + address, + session_token: `fake-account-${address}`, + expires_at: expired ? "2000-01-01T00:00:00Z" : "2099-01-01T00:00:00Z", + }, + personal_server: { + url, + session_token: `fake-ps-${address}`, + expires_at: "2099-01-01T00:00:00Z", + }, + }); + } + + function run( + args: string[], + extra: NodeJS.ProcessEnv = {}, + ): Promise { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, ["dist/cli/bin.js", ...args], { + cwd: repo, + env: { + ...cleanEnv, + HOME: root, + USERPROFILE: root, + VANA_HOME: home, + VANA_DATA_CONNECTORS_DIR: checkout, + VANA_PERSONAL_SERVER_URL: url, + VANA_TELEMETRY_DISABLED: "1", + VANA_PDPP_NODE: connectorNode, + FAKE_EXECUTION_FILE: executionFile, + FAKE_RECORD_OWNER: owner ?? "unknown", + ...extra, + }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + const timeout = setTimeout(() => { + child.kill("SIGKILL"); + reject(new Error(`CLI timed out: ${stdout}\n${stderr}`)); + }, 30_000); + child.on("error", (error) => { + clearTimeout(timeout); + reject(error); + }); + child.on("close", (code) => { + clearTimeout(timeout); + resolve({ code, stdout, stderr }); + }); + }); + } + + async function state() { + return JSON.parse( + await fs.readFile(path.join(home, "vana-connect-state.json"), "utf8"), + ); + } + + async function collectA() { + await login(A); + rejectUploads = true; + await run(["connect", "fake", "--from", checkout, "--no-input", "--json"]); + expect(uploads).toHaveLength(1); + expect(uploads[0]).toMatchObject({ + owner: A, + authorization: `Bearer fake-ps-${A}`, + }); + expect(stateAtUpload.sources.fake).toMatchObject({ + dataState: "collected_local", + exportReceipt: { + version: 1, + owner: { accountUrl: "https://account.vana.org", address: A }, + }, + }); + const before = await state(); + expect(before.sources.fake.dataState).toBe("ingest_failed"); + const resultPath = before.sources.fake.lastResultPath as string; + const bytes = await fs.readFile(resultPath); + expect(bytes.toString()).toContain(A); + const executions = await fs.readFile(executionFile, "utf8"); + rejectUploads = false; + uploads = []; + return { + resultPath, + bytes, + executions, + receipt: before.sources.fake.exportReceipt, + }; + } + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), "vana-export-owner-")); + home = path.join(root, "vana"); + checkout = path.join(root, "connector"); + executionFile = path.join(root, "executions.log"); + uploads = []; + rejectUploads = false; + rejectScope = undefined; + uploadGate = undefined; + stateAtUpload = undefined; + server = http.createServer(async (request, response) => { + response.setHeader("content-type", "application/json"); + if (request.url === "/health") { + response.end( + JSON.stringify({ status: "healthy", owner, version: "fake" }), + ); + } else if ( + request.method === "POST" && + request.url?.startsWith("/v1/data/") + ) { + let body = ""; + for await (const chunk of request) body += chunk; + uploads.push({ + owner, + authorization: request.headers.authorization, + body: JSON.parse(body), + }); + stateAtUpload = await state(); + const rejected = rejectUploads || request.url === rejectScope; + await uploadGate?.(); + response.writeHead(rejected ? 503 : 201); + response.end(rejected ? '{"error":"interrupted upload"}' : "{}"); + } else { + response.writeHead(404); + response.end("{}"); + } + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("No fake server port"); + url = `http://127.0.0.1:${address.port}`; + await write( + path.join(checkout, "connectors/fake/index.ts"), + ` +import fs from "node:fs"; +import readline from "node:readline"; +fs.appendFileSync(process.env.FAKE_EXECUTION_FILE, "run\\n"); +const input = readline.createInterface({ input: process.stdin }); +await input[Symbol.asyncIterator]().next(); +if (process.env.FAKE_WAIT_FILE) { + while (!fs.existsSync(process.env.FAKE_WAIT_FILE)) await new Promise((resolve) => setTimeout(resolve, 10)); +} +const emit = (event) => process.stdout.write(JSON.stringify(event) + "\\n"); +emit({ type: "RECORD", stream: "profile", key: "fake", data: { owner: process.env.FAKE_RECORD_OWNER, value: process.env.FAKE_RECORD_VALUE }, emitted_at: "2026-01-01T00:00:00Z" }); +if (process.env.FAKE_SECOND_STREAM) emit({ type: "RECORD", stream: "other", key: "other", data: { owner: process.env.FAKE_RECORD_OWNER }, emitted_at: "2026-01-01T00:00:00Z" }); +emit({ type: "DONE", status: "succeeded", records_emitted: process.env.FAKE_SECOND_STREAM ? 2 : 1 }); +await new Promise((resolve) => input.once("close", resolve)); +`, + ); + await write(path.join(checkout, "connectors/fake/manifest.json"), { + connector_key: "fake", + display_name: "Fake", + version: "0.0.1", + streams: [{ name: "profile" }], + export_frequency: "daily", + }); + await write(path.join(checkout, "package.json"), { type: "module" }); + await write(path.join(checkout, "node_modules/tsx/package.json"), { + type: "module", + exports: "./index.mjs", + }); + await write(path.join(checkout, "node_modules/tsx/index.mjs"), ""); + await write(path.join(checkout, "registry.json"), { connectors: [] }); + await write(path.join(checkout, "skills/vana-connect/scripts/.keep"), ""); + }); + + afterEach(async () => { + server?.closeAllConnections(); + if (server?.listening) + await new Promise((resolve) => server.close(() => resolve())); + await fs.rm(root, { recursive: true, force: true }); + }); + + it("blocks syncing A's interrupted export after account and server switch to B", async () => { + const queued = await collectA(); + await login(B); + const synced = await run(["server", "sync", "--no-input", "--json"]); + expect.soft(synced.code, synced.stdout + synced.stderr).toBe(5); + expect.soft(uploads).toEqual([]); + expect.soft(await fs.readFile(queued.resultPath)).toEqual(queued.bytes); + expect.soft((await state()).sources.fake.dataState).toBe("ingest_failed"); + expect + .soft((await state()).sources.fake.exportReceipt) + .toEqual(queued.receipt); + expect + .soft(JSON.parse(synced.stdout).error) + .toBe("pending_exports_blocked"); + }); + + it.each([ + ["connect", ["connect", "fake", "--no-input", "--json"]], + ["IPC connect", ["connect", "fake", "--ipc", "--json"]], + ["due collect", ["collect", "--all", "--no-input", "--json"]], + ])( + "blocks B's %s before replacing A's queued bytes or starting the connector", + async (_name, args) => { + const queued = await collectA(); + const before = await state(); + before.config = { localConnectors: { fake: { path: checkout } } }; + before.sources.fake.exportFrequency = "daily"; + before.sources.fake.lastCollectedAt = "2000-01-01T00:00:00Z"; + await write(path.join(home, "vana-connect-state.json"), before); + await login(B); + const collected = await run(args); + expect.soft(collected.code, collected.stdout + collected.stderr).toBe(5); + expect.soft(uploads).toEqual([]); + expect.soft(await fs.readFile(queued.resultPath)).toEqual(queued.bytes); + expect + .soft(await fs.readFile(executionFile, "utf8")) + .toBe(queued.executions); + expect + .soft((await state()).sources.fake.lastResultPath) + .toBe(queued.resultPath); + }, + ); + + it("recovers A's original export with its live server token after Account login expires without recollection", async () => { + const queued = await collectA(); + await login(B); + await login(A, true); + const synced = await run(["server", "sync", "--no-input", "--json"]); + expect(synced.code, synced.stdout + synced.stderr).toBe(0); + expect(uploads).toHaveLength(1); + expect(uploads[0]).toMatchObject({ + owner: A, + authorization: `Bearer fake-ps-${A}`, + }); + expect(JSON.stringify(uploads[0].body)).toContain(A); + expect(await fs.readFile(queued.resultPath)).toEqual(queued.bytes); + expect(await fs.readFile(executionFile, "utf8")).toBe(queued.executions); + expect((await state()).sources.fake.dataState).toBe( + "ingested_personal_server", + ); + }); + + it.each(["missing receipt", "malformed receipt"])( + "keeps a legacy export local and inspectable with %s", + async (kind) => { + const queued = await collectA(); + const saved = await state(); + saved.sources.fake.exportReceipt = + kind === "missing receipt" ? undefined : { owner: A }; + await write(path.join(home, "vana-connect-state.json"), saved); + const synced = await run(["server", "sync", "--no-input", "--json"]); + expect(synced.code).toBe(5); + // copy-assertion-ok: re-login must not be offered as a repair for missing provenance. + expect(synced.stdout).toContain("receipt"); + expect(synced.stdout).not.toContain("Sign in"); + const inspected = await run(["data", "show", "fake", "--json"]); + expect(inspected.code).toBe(0); + expect(JSON.parse(inspected.stdout).path).toBe(queued.resultPath); + expect(inspected.stdout).toContain(A); + const collected = await run([ + "connect", + "fake", + "--from", + checkout, + "--no-input", + "--json", + ]); + expect(collected.code).toBe(5); + expect(uploads).toEqual([]); + expect(await fs.readFile(queued.resultPath)).toEqual(queued.bytes); + expect(await fs.readFile(executionFile, "utf8")).toBe(queued.executions); + }, + ); + + it.each([ + "environment token", + "missing health owner", + "changed bytes", + "different deployment", + ])("refuses deferred upload with %s", async (kind) => { + const queued = await collectA(); + let extra: NodeJS.ProcessEnv = {}; + if (kind === "environment token") + extra = { VANA_SESSION_TOKEN: "fake-env-token" }; + if (kind === "missing health owner") owner = null; + if (kind === "changed bytes") await fs.appendFile(queued.resultPath, " "); + if (kind === "different deployment") { + await fs.copyFile( + path.join(home, "auth.json"), + path.join(home, "auth.account-dev.vana.org.json"), + ); + extra = { VANA_ENV: "dev" }; + } + const beforeBytes = await fs.readFile(queued.resultPath); + const synced = await run(["server", "sync", "--no-input", "--json"], extra); + expect(synced.code, synced.stdout + synced.stderr).toBe(5); + expect(uploads).toEqual([]); + expect(await fs.readFile(queued.resultPath)).toEqual(beforeBytes); + expect((await state()).sources.fake.exportReceipt).toEqual(queued.receipt); + }); + + it("preserves fresh accountless upload to an explicit destination without assigning its owner to later retry", async () => { + owner = A; + const extra = { VANA_PS_TOKEN: "fake-explicit-ps-token" }; + const collected = await run( + ["connect", "fake", "--from", checkout, "--no-input", "--json"], + extra, + ); + expect(collected.code, collected.stdout + collected.stderr).toBe(0); + expect(uploads).toHaveLength(1); + expect(uploads[0].authorization).toBe("Bearer fake-explicit-ps-token"); + expect((await state()).sources.fake.exportReceipt.owner).toBeNull(); + rejectUploads = true; + uploads = []; + await run( + ["connect", "fake", "--from", checkout, "--no-input", "--json"], + extra, + ); + expect(uploads).toHaveLength(1); + expect((await state()).sources.fake.dataState).toBe("ingest_failed"); + await login(A); + rejectUploads = false; + uploads = []; + const synced = await run(["server", "sync", "--no-input", "--json"]); + expect(synced.code).toBe(5); + expect(JSON.parse(synced.stdout).error).toBe("pending_exports_blocked"); + expect(uploads).toEqual([]); + }); + + it("reports B's automatic and scheduled exclusion of a partially synced A export", async () => { + await login(A); + await write(path.join(checkout, "connectors/fake/manifest.json"), { + connector_key: "fake", + display_name: "Fake", + version: "0.0.1", + streams: [{ name: "profile" }, { name: "other" }], + }); + rejectScope = "/v1/data/fake.other"; + const partial = await run( + ["connect", "fake", "--from", checkout, "--no-input", "--json"], + { + FAKE_SECOND_STREAM: "1", + }, + ); + expect(partial.code, partial.stdout + partial.stderr).toBe(0); + const saved = await state(); + expect(saved.sources.fake.dataState).toBe("ingested_personal_server"); + expect(saved.sources.fake.ingestScopes).toEqual( + expect.arrayContaining([ + expect.objectContaining({ scope: "fake.profile", status: "stored" }), + expect.objectContaining({ scope: "fake.other", status: "failed" }), + ]), + ); + saved.sources.fake.exportFrequency = "daily"; + await write(path.join(home, "vana-connect-state.json"), saved); + const bytes = await fs.readFile(saved.sources.fake.lastResultPath); + await login(B); + uploads = []; + for (const scheduled of [false, true]) { + const collected = await run( + ["collect", "--all", "--no-input", "--json"], + scheduled ? { VANA_SCHEDULED_RUN: "1" } : {}, + ); + expect(collected.code).toBe(5); + expect(JSON.parse(collected.stdout).sources).toEqual([ + expect.objectContaining({ + source: "fake", + outcome: "sync_pending", + error: expect.stringContaining(A), + }), + ]); + } + expect((await state()).lastScheduledRun).toMatchObject({ + exitCode: 5, + sources: [{ source: "fake", outcome: "sync_pending" }], + }); + expect(uploads).toEqual([]); + expect(await fs.readFile(saved.sources.fake.lastResultPath)).toEqual(bytes); + }); + + it("preserves A's admitted bytes when B's already-running collection loses publication", async () => { + await login(B); + const gate = path.join(root, "release-b"); + const competing = run( + ["connect", "fake", "--from", checkout, "--no-input", "--json"], + { FAKE_WAIT_FILE: gate }, + ); + while ( + !(await fs.readFile(executionFile, "utf8").catch(() => "")).includes( + "run", + ) + ) + await new Promise((resolve) => setTimeout(resolve, 10)); + const queued = await collectA(); + await login(B); + await write(gate, "release"); + const rejected = await competing; + expect(rejected.code, rejected.stdout + rejected.stderr).toBe(5); + const outcome = rejected.stdout + .trim() + .split("\n") + .map((line) => JSON.parse(line)) + .find((event) => event.reason === "pending_export_changed"); + expect(outcome.resultPath).not.toBe(queued.resultPath); + expect(await fs.readFile(outcome.resultPath, "utf8")).toContain(B); + expect(await fs.readFile(queued.resultPath)).toEqual(queued.bytes); + expect((await state()).sources.fake.exportReceipt).toEqual(queued.receipt); + expect(uploads).toEqual([]); + }); + + it("settles only the accepted generation when an earlier upload finishes after a newer collection", async () => { + await login(A); + rejectUploads = true; + let release!: () => void; + let posted!: () => void; + const firstPosted = new Promise((resolve) => { + posted = resolve; + }); + const held = new Promise((resolve) => { + release = resolve; + }); + uploadGate = async () => { + uploadGate = undefined; + posted(); + await held; + }; + const first = run( + ["connect", "fake", "--from", checkout, "--no-input", "--json"], + { FAKE_RECORD_VALUE: "first" }, + ); + await firstPosted; + const firstReceipt = (await state()).sources.fake.exportReceipt; + rejectUploads = false; + const second = await run( + ["connect", "fake", "--from", checkout, "--no-input", "--json"], + { FAKE_RECORD_VALUE: "second" }, + ); + expect(second.code, second.stdout + second.stderr).toBe(0); + const accepted = (await state()).sources.fake; + release(); + expect((await first).code).toBe(5); + expect(accepted.exportReceipt.id).not.toBe(firstReceipt.id); + expect(await fs.readFile(firstReceipt.path, "utf8")).toContain("first"); + expect(await fs.readFile(accepted.lastResultPath, "utf8")).toContain( + "second", + ); + expect((await state()).sources.fake).toEqual(accepted); + }); +}); diff --git a/test/cli/index.test.ts b/test/cli/index.test.ts index 5dd3089..8ff17fa 100644 --- a/test/cli/index.test.ts +++ b/test/cli/index.test.ts @@ -28,6 +28,28 @@ const mockReadCliState = vi.fn(); const mockReadCliConfig = vi.fn(); const mockUpdateCliConfig = vi.fn(); const mockUpdateSourceState = vi.fn(); +const exportAccount = "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +function ownedReceipt(resultPath: string, address = exportAccount) { + return { + version: 1, + id: "accepted-export", + path: resultPath, + sha256: "a".repeat(64), + owner: { accountUrl: "https://account.vana.org", address }, + }; +} +function selectExportAccount() { + mockReadFileSync.mockReturnValue( + JSON.stringify({ + account: { + address: exportAccount, + session_token: "fake-account-token", + expires_at: "2999-01-01T00:00:00Z", + }, + personal_server: null, + }), + ); +} const mockRecordScheduledRun = vi.fn(); const mockConfirm = vi.fn(); const mockInput = vi.fn(); @@ -405,6 +427,7 @@ describe("runCli", () => { mockResolvePersonalServerAuthConfig.mockReset(); mockCreatePersonalServerClient.mockReset(); mockUpdateSourceState.mockReset(); + mockUpdateSourceState.mockResolvedValue(true); mockRecordScheduledRun.mockReset(); mockRecordScheduledRun.mockResolvedValue(undefined); mockUpdateCliConfig.mockReset(); @@ -2866,9 +2889,15 @@ describe("runCli", () => { mockConfirm.mockResolvedValueOnce(true); mockIngestResult.mockResolvedValue([ - { type: "ingest-complete", source: "github" }, + { + type: "ingest-skipped", + source: "github", + reason: "export_unattributed", + message: + "The legacy export has no collection account receipt and stays local.", + }, ]); - expect(await runCli(["node", "vana", "server", "sync"])).toBe(0); + expect(await runCli(["node", "vana", "server", "sync"])).toBe(5); expect(mockIngestResult).toHaveBeenCalledTimes(1); }, ); @@ -2884,6 +2913,10 @@ describe("runCli", () => { sources: { github: { lastResultPath: "/tmp/results/github.json", + exportReceipt: ownedReceipt( + "/tmp/results/github.json", + signedIn.account.address, + ), dataState: "collected_local", }, }, @@ -2907,7 +2940,7 @@ describe("runCli", () => { }); it.each(["env", "accountless"])( - "preserves %s identity's unattended sync", + "keeps %s identity's unattributed pending export local", async (identity) => { if (identity === "env") { selectLogin("expired"); @@ -2923,7 +2956,13 @@ describe("runCli", () => { }, }); mockIngestResult.mockResolvedValue([ - { type: "ingest-complete", source: "github" }, + { + type: "ingest-skipped", + source: "github", + reason: "export_unattributed", + message: + "The export's collecting account is unknown; it stays local.", + }, ]); const { runCli } = await import("../../src/cli/index.js"); expect( @@ -2935,13 +2974,13 @@ describe("runCli", () => { "--no-input", "--json", ]), - ).toBe(0); + ).toBe(5); expect(mockIngestResult).toHaveBeenCalledTimes(1); expect(mockConfirm).not.toHaveBeenCalled(); }, ); - it("collect --all succeeds when a foreign server has no automatically retryable pending work", async () => { + it("collect --all reports a retained failed export even when it is not automatically retried", async () => { selectLogin("expired"); mockReadCliState.mockResolvedValue({ version: 1, @@ -2963,11 +3002,16 @@ describe("runCli", () => { "--no-input", "--json", ]), - ).toBe(0); + ).toBe(5); expect(JSON.parse(stdout.trim())).toMatchObject({ count: 0, syncedPendingCount: 0, - sources: [], + sources: [ + expect.objectContaining({ + source: "github", + outcome: "sync_pending", + }), + ], }); expect(mockIngestResult).not.toHaveBeenCalled(); expect(stderr).toBe(""); @@ -4222,6 +4266,7 @@ describe("runCli", () => { dataState: "ingest_failed", lastError: "server exploded", }), + { settle: expect.any(String) }, ); }); @@ -4276,6 +4321,7 @@ describe("runCli", () => { lastRunOutcome: "connected_and_ingested", dataState: "ingested_personal_server", }), + { settle: expect.any(String) }, ); }); @@ -4518,6 +4564,10 @@ describe("runCli", () => { }); it("allows collect when previously connected", async () => { + mockReadFile.mockResolvedValue( + JSON.stringify({ profile: { id: "fixture" } }), + ); + selectExportAccount(); mockListAvailableSources.mockResolvedValue([ { id: "github", name: "GitHub", authMode: "interactive" }, ]); @@ -4531,6 +4581,7 @@ describe("runCli", () => { lastRunOutcome: "connected_local_only", dataState: "collected_local", lastResultPath: "/tmp/results/github.json", + exportReceipt: ownedReceipt("/tmp/results/github.json"), }, }, }); @@ -4553,6 +4604,10 @@ describe("runCli", () => { }); it("titles a collect run Collect, not Connect", async () => { + mockReadFile.mockResolvedValue( + JSON.stringify({ profile: { id: "fixture" } }), + ); + selectExportAccount(); mockListAvailableSources.mockResolvedValue([ { id: "github", name: "GitHub", authMode: "interactive" }, ]); @@ -4565,6 +4620,7 @@ describe("runCli", () => { lastRunOutcome: "connected_local_only", dataState: "collected_local", lastResultPath: "/tmp/results/github.json", + exportReceipt: ownedReceipt("/tmp/results/github.json"), }, }, }); @@ -4669,9 +4725,11 @@ describe("runCli", () => { }); it("server sync settles the failed outcome it fixed", async () => { + selectExportAccount(); mockDetectPersonalServerTarget.mockResolvedValue({ state: "available", url: "http://localhost:8080", + health: { owner: exportAccount }, }); mockReadCliState.mockResolvedValue({ version: 1, @@ -4680,6 +4738,7 @@ describe("runCli", () => { connectorInstalled: true, lastRunOutcome: "ingest_failed", lastResultPath: "/tmp/results/github.json", + exportReceipt: ownedReceipt("/tmp/results/github.json"), dataState: "ingest_failed", ingestScopes: [ { scope: "github.repos", status: "failed", error: "HTTP 401" }, @@ -4705,6 +4764,7 @@ describe("runCli", () => { dataState: "ingested_personal_server", lastRunOutcome: "connected_and_ingested", }), + { settle: "accepted-export" }, ); }); }); @@ -5152,6 +5212,10 @@ describe("runCli", () => { }); it("collect runs a saved local connector, not the legacy runtime", async () => { + mockReadFile.mockResolvedValue( + JSON.stringify({ profile: { id: "fixture" } }), + ); + selectExportAccount(); // The entry needs a manual browser step; a headless Linux runner (CI) // has no display and would stop before the runtime is chosen. vi.stubEnv("DISPLAY", ":0"); @@ -5178,6 +5242,7 @@ describe("runCli", () => { lastRunOutcome: "connected_local_only", dataState: "collected_local", lastResultPath: "/tmp/results/slack_browser.json", + exportReceipt: ownedReceipt("/tmp/results/slack_browser.json"), }, }, }); @@ -5209,7 +5274,7 @@ describe("runCli", () => { expect(managedFetchCalls).toBe(0); // No --from: the runtime looks the saved entry up itself. expect(pdppRuntimeOptions).toEqual([ - { from: undefined, owner: undefined }, + { from: undefined, owner: exportAccount }, ]); const events = stdout .trim() @@ -5228,6 +5293,9 @@ describe("runCli", () => { }); it("connect prints where a saved local connector runs from", async () => { + mockReadFile.mockResolvedValue( + JSON.stringify({ profile: { id: "fixture" } }), + ); mockReadCliConfig.mockResolvedValue({ localConnectors: { slack_browser: entry }, }); @@ -5324,9 +5392,11 @@ describe("runCli", () => { }); it("server sync succeeds with pending datasets", async () => { + selectExportAccount(); mockDetectPersonalServerTarget.mockResolvedValue({ state: "available", url: "http://localhost:8080", + health: { owner: exportAccount }, }); mockReadCliState.mockResolvedValue({ version: 1, @@ -5334,6 +5404,7 @@ describe("runCli", () => { github: { connectorInstalled: true, lastResultPath: "/tmp/results/github.json", + exportReceipt: ownedReceipt("/tmp/results/github.json"), dataState: "collected_local", }, }, @@ -5376,9 +5447,11 @@ describe("runCli", () => { }); it("collect --all flushes pending sync work even when no sources are due", async () => { + selectExportAccount(); mockDetectPersonalServerTarget.mockResolvedValue({ state: "available", url: "http://localhost:8080", + health: { owner: exportAccount }, }); mockReadCliState.mockResolvedValue({ version: 1, @@ -5388,6 +5461,7 @@ describe("runCli", () => { exportFrequency: "1d", lastCollectedAt: new Date().toISOString(), lastResultPath: "/tmp/results/github.json", + exportReceipt: ownedReceipt("/tmp/results/github.json"), dataState: "ingest_unavailable", }, }, @@ -5417,7 +5491,12 @@ describe("runCli", () => { state: "available", url: "http://localhost:8080", }), - undefined, + { + export: { + kind: "retry", + receipt: ownedReceipt("/tmp/results/github.json"), + }, + }, ); const parsed = JSON.parse(stdout.trim()); @@ -5433,15 +5512,18 @@ describe("runCli", () => { }), ]), }), + { settle: "accepted-export" }, ); }); it("collect --all exits non-zero and says so on stderr when sync fails, even with --quiet", async () => { + selectExportAccount(); // A scheduled run once printed nothing and exited 0 every night while // every source's sync was answered with a 401. mockDetectPersonalServerTarget.mockResolvedValue({ state: "available", url: "http://localhost:8080", + health: { owner: exportAccount }, }); mockReadCliState.mockResolvedValue({ version: 1, @@ -5451,6 +5533,7 @@ describe("runCli", () => { exportFrequency: "1d", lastCollectedAt: new Date().toISOString(), lastResultPath: "/tmp/results/github.json", + exportReceipt: ownedReceipt("/tmp/results/github.json"), dataState: "ingest_unavailable", }, }, @@ -5682,6 +5765,7 @@ describe("runCli", () => { }), ]), }), + { settle: expect.any(String) }, ); }); @@ -5781,6 +5865,7 @@ describe("runCli", () => { dataState: "ingest_unavailable", lastResultPath: "/tmp/.vana/github-result.json", }), + { settle: expect.any(String) }, ); expect(stderr).toContain("Personal Server sync is pending."); }); @@ -5833,9 +5918,11 @@ describe("runCli", () => { }); it("server sync re-ingests failed scopes", async () => { + selectExportAccount(); mockDetectPersonalServerTarget.mockResolvedValue({ state: "available", url: "http://localhost:8080", + health: { owner: exportAccount }, }); mockReadCliState.mockResolvedValue({ version: 1, @@ -5843,6 +5930,7 @@ describe("runCli", () => { github: { connectorInstalled: true, lastResultPath: "/tmp/results/github.json", + exportReceipt: ownedReceipt("/tmp/results/github.json"), dataState: "ingested_personal_server", ingestScopes: [ { scope: "github.profile", status: "stored" }, @@ -5870,7 +5958,13 @@ describe("runCli", () => { state: "available", url: "http://localhost:8080", }), - { scopes: ["github.starred"] }, + { + scopes: ["github.starred"], + export: { + kind: "retry", + receipt: ownedReceipt("/tmp/results/github.json"), + }, + }, ); const lines = stdout.trim().split("\n"); const lastLine = JSON.parse(lines[lines.length - 1]); @@ -5890,6 +5984,7 @@ describe("runCli", () => { }), ]), }), + { settle: "accepted-export" }, ); }); @@ -6370,9 +6465,11 @@ describe("runCli", () => { }); it("server sync shows themed scope results with next step guidance", async () => { + selectExportAccount(); mockDetectPersonalServerTarget.mockResolvedValue({ state: "available", url: "http://localhost:8080", + health: { owner: exportAccount }, }); mockReadCliState.mockResolvedValue({ version: 1, @@ -6380,6 +6477,7 @@ describe("runCli", () => { github: { connectorInstalled: true, lastResultPath: "/tmp/results/github.json", + exportReceipt: ownedReceipt("/tmp/results/github.json"), dataState: "collected_local", }, }, @@ -6741,9 +6839,11 @@ describe("runCli", () => { }); it("server sync suggests retry when some scopes fail", async () => { + selectExportAccount(); mockDetectPersonalServerTarget.mockResolvedValue({ state: "available", url: "http://localhost:8080", + health: { owner: exportAccount }, }); mockReadCliState.mockResolvedValue({ version: 1, @@ -6751,6 +6851,7 @@ describe("runCli", () => { github: { connectorInstalled: true, lastResultPath: "/tmp/results/github.json", + exportReceipt: ownedReceipt("/tmp/results/github.json"), dataState: "collected_local", }, }, @@ -6783,9 +6884,11 @@ describe("runCli", () => { }); it("server sync humanizes transport and scope validation errors", async () => { + selectExportAccount(); mockDetectPersonalServerTarget.mockResolvedValue({ state: "available", url: "http://localhost:8080", + health: { owner: exportAccount }, }); mockReadCliState.mockResolvedValue({ version: 1, @@ -6793,6 +6896,7 @@ describe("runCli", () => { youtube: { connectorInstalled: true, lastResultPath: "/tmp/results/youtube.json", + exportReceipt: ownedReceipt("/tmp/results/youtube.json"), dataState: "collected_local", }, }, diff --git a/test/runtime/in-process-run.test.ts b/test/runtime/in-process-run.test.ts index 731913c..6bb50de 100644 --- a/test/runtime/in-process-run.test.ts +++ b/test/runtime/in-process-run.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { getSourceResultPath } from "../../src/core/paths.js"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -743,6 +744,14 @@ describe("startInProcessConnectorRun", () => { it("writes a result and emits collection-complete", async () => { createFakeRuntime(); + const acceptedPath = getSourceResultPath("github"); + const isolatedPath = path.join( + path.dirname(acceptedPath), + "github", + "return-run.json", + ); + await fs.mkdir(path.dirname(acceptedPath), { recursive: true }); + await fs.writeFile(acceptedPath, '{"accepted":"A"}'); const connectorPath = await writeConnector(` (async () => { await page.setData("status", "Collecting"); @@ -761,6 +770,7 @@ describe("startInProcessConnectorRun", () => { connectorPath, source: "github", noInput: true, + resultPath: isolatedPath, }, logPath: path.join(os.tmpdir(), "vana-connect-collection.log"), }); @@ -781,6 +791,8 @@ describe("startInProcessConnectorRun", () => { ); const resultPath = (completion as { resultPath: string }).resultPath; + expect(resultPath).toBe(isolatedPath); + expect(await fs.readFile(acceptedPath, "utf8")).toBe('{"accepted":"A"}'); const result = JSON.parse(await fs.readFile(resultPath, "utf8")); expect(result).toEqual({ profile: { username: "tester" }, @@ -790,6 +802,14 @@ describe("startInProcessConnectorRun", () => { it("treats setData('result', ...) as collection completion", async () => { createFakeRuntime(); + const acceptedPath = getSourceResultPath("github"); + const isolatedPath = path.join( + path.dirname(acceptedPath), + "github", + "set-data-run.json", + ); + await fs.mkdir(path.dirname(acceptedPath), { recursive: true }); + await fs.writeFile(acceptedPath, '{"accepted":"A"}'); const connectorPath = await writeConnector(` (async () => { await page.setData("status", "Collecting"); @@ -809,6 +829,7 @@ describe("startInProcessConnectorRun", () => { connectorPath, source: "github", noInput: false, + resultPath: isolatedPath, }, logPath: path.join(os.tmpdir(), "vana-connect-setdata-result.log"), }); @@ -829,6 +850,8 @@ describe("startInProcessConnectorRun", () => { ); const resultPath = (completion as { resultPath: string }).resultPath; + expect(resultPath).toBe(isolatedPath); + expect(await fs.readFile(acceptedPath, "utf8")).toBe('{"accepted":"A"}'); const result = JSON.parse(await fs.readFile(resultPath, "utf8")); expect(result).toEqual({ profile: { username: "tester" },