diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 48b0f4180..bf68f193d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -37,6 +37,9 @@ jobs: - name: Run Tests run: npx turbo run test --filter=roam --ui stream + - name: Run Package Unit Tests + run: npx turbo run test:unit --ui stream + lint-changed-files: if: github.event_name == 'pull_request' runs-on: ubuntu-latest diff --git a/apps/obsidian/src/utils/importNodes.ts b/apps/obsidian/src/utils/importNodes.ts index 06ba93edd..70c7860f8 100644 --- a/apps/obsidian/src/utils/importNodes.ts +++ b/apps/obsidian/src/utils/importNodes.ts @@ -2,6 +2,7 @@ import type { Json } from "@repo/database/dbTypes"; import matter from "gray-matter"; import { App, Notice, TFile } from "obsidian"; import type { DGSupabaseClient } from "@repo/database/lib/client"; +import { listGroupSharedNodes } from "@repo/database/lib/sharedNodes"; import type DiscourseGraphPlugin from "~/index"; import { getLoggedInClient, getSupabaseContext } from "./supabaseContext"; import type { DiscourseNode, ImportableNode } from "~/types"; @@ -40,84 +41,34 @@ export const getPublishedNodesForGroups = async ({ groupIds: string[]; currentSpaceId: number; }): Promise> => { - if (groupIds.length === 0) { - return []; - } - - // Query my_contents (RLS applied); exclude current space. Get both variants so we can use - // the latest last_modified per node and prefer "direct" for text (title). - const { data, error } = await client - .from("my_contents") - .select( - "source_local_id, space_id, text, created, last_modified, variant, metadata, author_id", - ) - .neq("space_id", currentSpaceId); - - if (error) { + const candidates = await listGroupSharedNodes({ + client, + currentSpaceId, + groupIds, + }).catch((error: { message?: string }) => { console.error("Error fetching published nodes:", error); throw new Error(`Failed to fetch published nodes: ${error.message}`); - } - - if (!data || data.length === 0) { - return []; - } - - type Row = { - source_local_id: string | null; - space_id: number | null; - text: string | null; - created: string | null; - last_modified: string | null; - variant: string | null; - author_id: number | null; - metadata: Json; - }; - - const key = (r: Row) => `${r.space_id ?? ""}\t${r.source_local_id ?? ""}`; - const groups = new Map(); - for (const row of data as Row[]) { - if (row.source_local_id == null || row.space_id == null) continue; - const k = key(row); - if (!groups.has(k)) groups.set(k, []); - groups.get(k)!.push(row); - } - - const nodes: Array = []; + }); - for (const rows of groups.values()) { - const withDate = rows.filter( - (r) => r.last_modified != null && r.text != null, - ); - if (withDate.length === 0) continue; - const latest = withDate.reduce((a, b) => - (a.last_modified ?? "") >= (b.last_modified ?? "") ? a : b, - ); - const direct = rows.find((r) => r.variant === "direct"); - const text = direct?.text ?? latest.text ?? ""; - const createdAt = latest.created - ? new Date(latest.created + "Z").valueOf() - : 0; - const modifiedAt = latest.last_modified - ? new Date(latest.last_modified + "Z").valueOf() - : 0; + return candidates.map((candidate) => { + const metadata = candidate.directMetadata; const filePath: string | undefined = - direct && - typeof direct.metadata === "object" && - typeof (direct.metadata as Record).filePath === "string" - ? (direct.metadata as Record).filePath + metadata !== null && + typeof metadata === "object" && + !Array.isArray(metadata) && + typeof metadata.filePath === "string" + ? metadata.filePath : undefined; - nodes.push({ - source_local_id: latest.source_local_id!, - space_id: latest.space_id!, - text, - createdAt, - modifiedAt, + return { + source_local_id: candidate.sourceLocalId, + space_id: candidate.spaceId, + text: candidate.title, + createdAt: candidate.created ? new Date(candidate.created).valueOf() : 0, + modifiedAt: new Date(candidate.lastModified).valueOf(), filePath, - authorId: latest.author_id ?? undefined, - }); - } - - return nodes; + authorId: candidate.authorId, + }; + }); }; export const getLocalNodeInstanceIds = ( diff --git a/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts b/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts index e52696b3a..4a3a9637e 100644 --- a/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts +++ b/apps/roam/src/utils/__tests__/discoverSharedNodes.test.ts @@ -1,82 +1,35 @@ import { describe, expect, it } from "vitest"; -import { buildDiscoveredSharedNodes } from "~/utils/discoverSharedNodes"; - -type BuildArgs = Parameters[0]; - -const resources: BuildArgs["resources"] = [ - { space_id: 20, source_local_id: "node-1" }, - { space_id: 20, source_local_id: "schema-1" }, -]; -const spaces: BuildArgs["spaces"] = [ - { - id: 20, - name: "Research vault", - platform: "Obsidian", - url: "obsidian:vault-a", - }, -]; -const concepts: BuildArgs["concepts"] = [ - { - is_schema: false, - last_modified: "2026-06-14T12:00:00", - schema_id: 200, - source_local_id: "node-1", - space_id: 20, - }, -]; -const contents: BuildArgs["contents"] = [ - { - content_type: "text/plain", - last_modified: "2026-06-14T13:00:00", - source_local_id: "node-1", - space_id: 20, - text: "EVD - REM sleep and recall", - variant: "direct", - }, - { - content_type: "text/markdown", - last_modified: "2026-06-14T15:00:00", - source_local_id: "node-1", - space_id: 20, - text: "# EVD - REM sleep and recall", - variant: "full", - }, -]; -const sourceNodeRid = "orn:obsidian.note:vault-a/node-1"; - -const build = ({ - conceptsOverride = concepts, - contentsOverride = contents, - currentSpaceId = 10, - importedSourceRids = new Set(), - resourcesOverride = resources, - spacesOverride = spaces, -}: { - conceptsOverride?: typeof concepts; - contentsOverride?: typeof contents; - currentSpaceId?: number; - importedSourceRids?: ReadonlySet; - resourcesOverride?: typeof resources; - spacesOverride?: typeof spaces; -} = {}) => - buildDiscoveredSharedNodes({ - concepts: conceptsOverride, - contents: contentsOverride, - currentSpaceId, - importedSourceRids, - resources: resourcesOverride, - spaces: spacesOverride, - }); - -describe("buildDiscoveredSharedNodes", () => { - it("builds a group-shared contract node with stable source identity", () => { - expect(build({ importedSourceRids: new Set([sourceNodeRid]) })).toEqual([ +import { toDiscoveredSharedNodes } from "~/utils/discoverSharedNodes"; +import type { SharedNodeCandidate } from "@repo/database/lib/sharedNodes"; + +const candidate: SharedNodeCandidate = { + rid: "orn:obsidian.note:vault-a/node-1", + sourceLocalId: "node-1", + spaceId: 20, + spaceName: "Research vault", + spaceUri: "obsidian:vault-a", + platform: "Obsidian", + title: "EVD - REM sleep and recall", + created: "2026-06-14T12:30:00.000Z", + lastModified: "2026-06-14T15:00:00.000Z", + authorId: 7, + directMetadata: null, +}; + +describe("toDiscoveredSharedNodes", () => { + it("maps a candidate to the exact discovered shared node shape", () => { + expect( + toDiscoveredSharedNodes({ + candidates: [candidate], + importedSourceRids: new Set([candidate.rid]), + }), + ).toEqual([ { alreadyImported: true, modifiedAt: "2026-06-14T15:00:00.000Z", sourceApp: "Obsidian", sourceNodeId: "node-1", - sourceNodeRid, + sourceNodeRid: "orn:obsidian.note:vault-a/node-1", sourceSpaceId: "obsidian:vault-a", sourceSpaceName: "Research vault", title: "EVD - REM sleep and recall", @@ -84,96 +37,12 @@ describe("buildDiscoveredSharedNodes", () => { ]); }); - it("does not discover shared resources from the current space", () => { - expect(build({ currentSpaceId: 20 })).toEqual([]); - }); - - it("requires the exact shared resource identity", () => { - expect( - build({ - resourcesOverride: [{ space_id: 21, source_local_id: "node-1" }], - }), - ).toEqual([]); - }); - - it.each([ - { - name: "schema concept", - conceptsOverride: [{ ...concepts[0], is_schema: true }], - contentsOverride: contents, - }, - { - name: "missing node type", - conceptsOverride: [{ ...concepts[0], schema_id: null }], - contentsOverride: contents, - }, - { - name: "missing direct content", - conceptsOverride: concepts, - contentsOverride: [contents[1]], - }, - { - name: "missing full content", - conceptsOverride: concepts, - contentsOverride: [contents[0]], - }, - { - name: "untyped full content", - conceptsOverride: concepts, - contentsOverride: [contents[0], { ...contents[1], content_type: null }], - }, - ])("filters a node with $name", ({ conceptsOverride, contentsOverride }) => { - expect(build({ conceptsOverride, contentsOverride })).toEqual([]); - }); - it("matches imports by RID rather than source-local ID alone", () => { expect( - build({ + toDiscoveredSharedNodes({ + candidates: [candidate], importedSourceRids: new Set(["orn:obsidian.note:another-vault/node-1"]), })[0]?.alreadyImported, ).toBe(false); }); - - it.each([ - "orn:obsidian.note:vault-a/node-1", - "https://example.com/shared/node-1", - ])("preserves a RID-shaped source-local ID: %s", (rid) => { - expect( - build({ - conceptsOverride: [{ ...concepts[0], source_local_id: rid }], - contentsOverride: contents.map((content) => ({ - ...content, - source_local_id: rid, - })), - resourcesOverride: [{ space_id: 20, source_local_id: rid }], - })[0]?.sourceNodeRid, - ).toBe(rid); - }); - - it("sorts newest nodes first", () => { - const olderConcept = { - ...concepts[0], - last_modified: "2026-06-10T12:00:00", - source_local_id: "node-2", - }; - const olderContents = contents.map((content) => ({ - ...content, - last_modified: "2026-06-10T12:00:00", - source_local_id: "node-2", - text: - content.variant === "direct" - ? "Older shared node" - : "# Older shared node", - })); - expect( - build({ - conceptsOverride: [olderConcept, concepts[0]], - contentsOverride: [...olderContents, ...contents], - resourcesOverride: [ - ...resources, - { space_id: 20, source_local_id: "node-2" }, - ], - }).map((node) => node.sourceNodeId), - ).toEqual(["node-1", "node-2"]); - }); }); diff --git a/apps/roam/src/utils/discoverSharedNodes.ts b/apps/roam/src/utils/discoverSharedNodes.ts index f6b01a8cd..1dfea1acd 100644 --- a/apps/roam/src/utils/discoverSharedNodes.ts +++ b/apps/roam/src/utils/discoverSharedNodes.ts @@ -1,45 +1,17 @@ import type { DGSupabaseClient } from "@repo/database/lib/client"; -import { getAvailableGroupIds } from "@repo/database/lib/groups"; -import { getAllPages } from "@repo/database/lib/pagination"; -import { isRid, spaceUriAndLocalIdToRid } from "@repo/database/lib/rid"; -import type { Tables } from "@repo/database/dbTypes"; +import { + listGroupSharedNodes, + type SharedNodeCandidate, +} from "@repo/database/lib/sharedNodes"; +import type { Enums } from "@repo/database/dbTypes"; import { DISCOURSE_GRAPH_PROP_NAME } from "./createReifiedBlock"; const IMPORTED_FROM_PROP_KEY = "importedFrom"; -const PAGE_SIZE = 1000; -const RESOURCE_ID_CHUNK_SIZE = 100; - -type ResourceAccess = Pick< - Tables<"ResourceAccess">, - "space_id" | "source_local_id" ->; -type SharedConcept = Pick< - Tables<"my_concepts">, - "is_schema" | "last_modified" | "schema_id" | "source_local_id" | "space_id" ->; -type SharedContent = Pick< - Tables<"my_contents">, - | "content_type" - | "last_modified" - | "source_local_id" - | "space_id" - | "text" - | "variant" ->; -type SharedSpace = Pick< - Tables<"my_spaces">, - "id" | "name" | "platform" | "url" ->; -type ValidSharedSpace = { - name: string; - platform: "Roam" | "Obsidian"; - url: string; -}; export type DiscoveredSharedNode = { alreadyImported: boolean; modifiedAt: string; - sourceApp: "Roam" | "Obsidian"; + sourceApp: Enums<"Platform">; sourceNodeId?: string; sourceNodeRid: string; sourceSpaceId: string; @@ -47,263 +19,23 @@ export type DiscoveredSharedNode = { title: string; }; -type SharedNodeRows = { - concepts: SharedConcept[]; - contents: SharedContent[]; - resources: ResourceAccess[]; - spaces: SharedSpace[]; -}; - -const getResourceKey = ({ - sourceLocalId, - spaceId, -}: { - sourceLocalId: string; - spaceId: number; -}): string => `${spaceId}:${sourceLocalId}`; - -const normalizeUtcTimestamp = (timestamp: string | null): string | null => { - if (!timestamp) return null; - const date = new Date(`${timestamp}Z`); - return Number.isNaN(date.getTime()) ? null : date.toISOString(); -}; - -const getLatestTimestamp = (timestamps: (string | null)[]): string | null => { - const validTimestamps = timestamps - .map(normalizeUtcTimestamp) - .filter((timestamp): timestamp is string => typeof timestamp === "string"); - if (validTimestamps.length === 0) return null; - return validTimestamps.reduce((latest, timestamp) => - timestamp > latest ? timestamp : latest, - ); -}; - -export const buildDiscoveredSharedNodes = ({ - concepts, - contents, - currentSpaceId, +export const toDiscoveredSharedNodes = ({ + candidates, importedSourceRids, - resources, - spaces, -}: SharedNodeRows & { - currentSpaceId: number; - importedSourceRids: ReadonlySet; -}): DiscoveredSharedNode[] => { - const sharedResourceKeys = new Set( - resources - .filter((resource) => resource.space_id !== currentSpaceId) - .map((resource) => - getResourceKey({ - sourceLocalId: resource.source_local_id, - spaceId: resource.space_id, - }), - ), - ); - const spacesById = new Map( - spaces.flatMap((space): [number, ValidSharedSpace][] => { - if ( - typeof space.id !== "number" || - typeof space.name !== "string" || - (space.platform !== "Roam" && space.platform !== "Obsidian") || - typeof space.url !== "string" - ) - return []; - return [ - [ - space.id, - { - name: space.name, - platform: space.platform, - url: space.url, - }, - ], - ]; - }), - ); - const contentByResource = new Map< - string, - Partial> - >(); - - contents.forEach((content) => { - if ( - typeof content.space_id !== "number" || - typeof content.source_local_id !== "string" || - (content.variant !== "direct" && content.variant !== "full") - ) - return; - const key = getResourceKey({ - sourceLocalId: content.source_local_id, - spaceId: content.space_id, - }); - const variants = contentByResource.get(key) ?? {}; - variants[content.variant] = content; - contentByResource.set(key, variants); - }); - - return concepts - .flatMap((concept): DiscoveredSharedNode[] => { - if ( - concept.is_schema !== false || - concept.schema_id === null || - typeof concept.space_id !== "number" || - typeof concept.source_local_id !== "string" - ) - return []; - - const resourceKey = getResourceKey({ - sourceLocalId: concept.source_local_id, - spaceId: concept.space_id, - }); - if (!sharedResourceKeys.has(resourceKey)) return []; - - const space = spacesById.get(concept.space_id); - const variants = contentByResource.get(resourceKey); - const direct = variants?.direct; - const full = variants?.full; - if ( - !space || - typeof direct?.text !== "string" || - typeof full?.text !== "string" || - typeof full.content_type !== "string" - ) - return []; - - const modifiedAt = getLatestTimestamp([ - concept.last_modified, - direct.last_modified, - full.last_modified, - ]); - if (!modifiedAt) return []; - - let sourceNodeRid: string; - try { - sourceNodeRid = isRid(concept.source_local_id) - ? concept.source_local_id - : spaceUriAndLocalIdToRid( - space.url, - concept.source_local_id, - space.platform === "Obsidian" ? "note" : undefined, - ); - } catch { - return []; - } - - return [ - { - alreadyImported: importedSourceRids.has(sourceNodeRid), - modifiedAt, - sourceApp: space.platform, - sourceNodeId: concept.source_local_id || undefined, - sourceNodeRid, - sourceSpaceId: space.url, - sourceSpaceName: space.name, - title: direct.text, - }, - ]; - }) - .sort( - (left, right) => - Date.parse(right.modifiedAt) - Date.parse(left.modifiedAt) || - left.title.localeCompare(right.title), - ); -}; - -const getGroupSharedResources = async ( - client: DGSupabaseClient, -): Promise => { - const groupIds = await getAvailableGroupIds(client); - if (groupIds.length === 0) return []; - - const resources = await getAllPages( - client - .from("ResourceAccess") - .select("space_id, source_local_id") - .in("account_uid", groupIds) - .order("space_id") - .order("source_local_id"), - PAGE_SIZE, - ); - if (!Array.isArray(resources)) throw resources; - - return [ - ...new Map( - resources.map((resource) => [ - getResourceKey({ - sourceLocalId: resource.source_local_id, - spaceId: resource.space_id, - }), - resource, - ]), - ).values(), - ]; -}; - -const chunk = (values: T[], size: number): T[][] => - Array.from({ length: Math.ceil(values.length / size) }, (_, index) => - values.slice(index * size, (index + 1) * size), - ); - -const getSharedNodeRows = async ({ - client, - currentSpaceId, }: { - client: DGSupabaseClient; - currentSpaceId: number; -}): Promise => { - const resources = (await getGroupSharedResources(client)).filter( - (resource) => resource.space_id !== currentSpaceId, - ); - if (resources.length === 0) - return { concepts: [], contents: [], resources, spaces: [] }; - - const spaceIds = [...new Set(resources.map((resource) => resource.space_id))]; - const spacesResponse = await client - .from("my_spaces") - .select("id, name, platform, url") - .in("id", spaceIds); - if (spacesResponse.error) throw spacesResponse.error; - - const concepts: SharedConcept[] = []; - const contents: SharedContent[] = []; - for (const spaceId of spaceIds) { - const sourceLocalIds = resources - .filter((resource) => resource.space_id === spaceId) - .map((resource) => resource.source_local_id); - for (const ids of chunk(sourceLocalIds, RESOURCE_ID_CHUNK_SIZE)) { - const [conceptsResponse, contentsResponse] = await Promise.all([ - client - .from("my_concepts") - .select( - "is_schema, last_modified, schema_id, source_local_id, space_id", - ) - .eq("space_id", spaceId) - .eq("is_schema", false) - .eq("arity", 0) - .in("source_local_id", ids), - client - .from("my_contents") - .select( - "content_type, last_modified, source_local_id, space_id, text, variant", - ) - .eq("space_id", spaceId) - .in("source_local_id", ids) - .in("variant", ["direct", "full"]), - ]); - if (conceptsResponse.error) throw conceptsResponse.error; - if (contentsResponse.error) throw contentsResponse.error; - concepts.push(...conceptsResponse.data); - contents.push(...contentsResponse.data); - } - } - - return { - concepts, - contents, - resources, - spaces: spacesResponse.data, - }; -}; + candidates: SharedNodeCandidate[]; + importedSourceRids: ReadonlySet; +}): DiscoveredSharedNode[] => + candidates.map((candidate) => ({ + alreadyImported: importedSourceRids.has(candidate.rid), + modifiedAt: candidate.lastModified, + sourceApp: candidate.platform, + sourceNodeId: candidate.sourceLocalId || undefined, + sourceNodeRid: candidate.rid, + sourceSpaceId: candidate.spaceUri, + sourceSpaceName: candidate.spaceName, + title: candidate.title, + })); const getImportedSourceRids = async (): Promise> => { const query = `[:find [?rid ...] @@ -325,13 +57,9 @@ export const discoverSharedNodes = async ({ client: DGSupabaseClient; currentSpaceId: number; }): Promise => { - const [rows, importedSourceRids] = await Promise.all([ - getSharedNodeRows({ client, currentSpaceId }), + const [candidates, importedSourceRids] = await Promise.all([ + listGroupSharedNodes({ client, currentSpaceId }), getImportedSourceRids(), ]); - return buildDiscoveredSharedNodes({ - ...rows, - currentSpaceId, - importedSourceRids, - }); + return toDiscoveredSharedNodes({ candidates, importedSourceRids }); }; diff --git a/packages/database/package.json b/packages/database/package.json index 5653b1eda..879730574 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -34,6 +34,7 @@ "lint:fix": "eslint --fix . && tsx scripts/lintSchemas.ts -f && tsx scripts/lintFunctions.ts", "migrate": "tsx scripts/migrate.ts", "test": "pnpm run build && cucumber-js", + "test:unit": "vitest run", "test:withserve": "pnpm run build && tsx scripts/serveAndTest.ts", "genenv": "tsx scripts/createEnv.mts", "gentypes": "tsx scripts/genTypes.ts", @@ -65,7 +66,8 @@ "ts-node-maintained": "^10.9.5", "tsx": "4.20.6", "typescript": "5.9.2", - "vercel": "53.1.0" + "vercel": "53.1.0", + "vitest": "catalog:" }, "prettier": { "plugins": [ diff --git a/packages/database/src/lib/__tests__/sharedNodes.test.ts b/packages/database/src/lib/__tests__/sharedNodes.test.ts new file mode 100644 index 000000000..3804b4222 --- /dev/null +++ b/packages/database/src/lib/__tests__/sharedNodes.test.ts @@ -0,0 +1,171 @@ +import { describe, expect, it } from "vitest"; +import { buildSharedNodeCandidates } from "../sharedNodes"; + +type BuildArgs = Parameters[0]; + +const resources: BuildArgs["resources"] = [ + { space_id: 20, source_local_id: "node-1" }, + { space_id: 20, source_local_id: "schema-1" }, +]; +const spaces: BuildArgs["spaces"] = [ + { + id: 20, + name: "Research vault", + platform: "Obsidian", + url: "obsidian:vault-a", + }, +]; +const concepts: BuildArgs["concepts"] = [ + { + is_schema: false, + last_modified: "2026-06-14T12:00:00", + schema_id: 200, + source_local_id: "node-1", + space_id: 20, + }, +]; +const directContents: BuildArgs["directContents"] = [ + { + author_id: 42, + created: "2026-06-14T11:00:00", + last_modified: "2026-06-14T13:00:00", + metadata: { source: "obsidian" }, + source_local_id: "node-1", + space_id: 20, + text: "EVD - REM sleep and recall", + variant: "direct", + }, +]; +const fullContentSummaries: BuildArgs["fullContentSummaries"] = [ + { + last_modified: "2026-06-14T15:00:00", + source_local_id: "node-1", + space_id: 20, + }, +]; +const rid = "orn:obsidian.note:vault-a/node-1"; + +const build = ({ + conceptsOverride = concepts, + currentSpaceId = 10, + directOverride = directContents, + fullOverride = fullContentSummaries, + resourcesOverride = resources, + spacesOverride = spaces, +}: { + conceptsOverride?: typeof concepts; + currentSpaceId?: number; + directOverride?: typeof directContents; + fullOverride?: typeof fullContentSummaries; + resourcesOverride?: typeof resources; + spacesOverride?: typeof spaces; +} = {}) => + buildSharedNodeCandidates({ + concepts: conceptsOverride, + currentSpaceId, + directContents: directOverride, + fullContentSummaries: fullOverride, + resources: resourcesOverride, + spaces: spacesOverride, + }); + +describe("buildSharedNodeCandidates", () => { + it("builds a group-shared contract node with stable source identity", () => { + expect(build()).toEqual([ + { + rid, + sourceLocalId: "node-1", + spaceId: 20, + spaceName: "Research vault", + spaceUri: "obsidian:vault-a", + platform: "Obsidian", + title: "EVD - REM sleep and recall", + created: "2026-06-14T11:00:00.000Z", + lastModified: "2026-06-14T15:00:00.000Z", + authorId: 42, + directMetadata: { source: "obsidian" }, + }, + ]); + }); + + it("does not discover shared resources from the current space", () => { + expect(build({ currentSpaceId: 20 })).toEqual([]); + }); + + it("requires the exact shared resource identity", () => { + expect( + build({ + resourcesOverride: [{ space_id: 21, source_local_id: "node-1" }], + }), + ).toEqual([]); + }); + + it.each([ + "orn:obsidian.note:vault-a/node-1", + "https://example.com/shared/node-1", + ])("preserves a RID-shaped source-local ID: %s", (sourceLocalId) => { + expect( + build({ + conceptsOverride: [{ ...concepts[0]!, source_local_id: sourceLocalId }], + directOverride: [ + { ...directContents[0]!, source_local_id: sourceLocalId }, + ], + fullOverride: [ + { ...fullContentSummaries[0]!, source_local_id: sourceLocalId }, + ], + resourcesOverride: [{ space_id: 20, source_local_id: sourceLocalId }], + })[0]?.rid, + ).toBe(sourceLocalId); + }); + + it("discovers a node without full content", () => { + expect(build({ fullOverride: [] })[0]?.lastModified).toBe( + "2026-06-14T13:00:00.000Z", + ); + }); + + it.each([ + { + name: "schema concept", + conceptsOverride: [{ ...concepts[0]!, is_schema: true }], + directOverride: directContents, + }, + { + name: "missing node type", + conceptsOverride: [{ ...concepts[0]!, schema_id: null }], + directOverride: directContents, + }, + { + name: "missing direct content", + conceptsOverride: concepts, + directOverride: [], + }, + ])("filters a node with $name", ({ conceptsOverride, directOverride }) => { + expect(build({ conceptsOverride, directOverride })).toEqual([]); + }); + + it("sorts newest nodes first", () => { + const olderConcept = { + ...concepts[0]!, + last_modified: "2026-06-10T12:00:00", + source_local_id: "node-2", + }; + const olderDirect = { + ...directContents[0]!, + last_modified: "2026-06-10T12:00:00", + source_local_id: "node-2", + text: "Older shared node", + }; + expect( + build({ + conceptsOverride: [olderConcept, concepts[0]!], + directOverride: [olderDirect, ...directContents], + fullOverride: fullContentSummaries, + resourcesOverride: [ + ...resources, + { space_id: 20, source_local_id: "node-2" }, + ], + }).map((node) => node.sourceLocalId), + ).toEqual(["node-1", "node-2"]); + }); +}); diff --git a/packages/database/src/lib/sharedNodes.ts b/packages/database/src/lib/sharedNodes.ts new file mode 100644 index 000000000..2f918c29f --- /dev/null +++ b/packages/database/src/lib/sharedNodes.ts @@ -0,0 +1,360 @@ +import type { DGSupabaseClient } from "./client"; +import { getAvailableGroupIds } from "./groups"; +import { getAllPages } from "./pagination"; +import { isRid, spaceUriAndLocalIdToRid } from "./rid"; +import type { Enums, Json, Tables } from "../dbTypes"; + +const PAGE_SIZE = 1000; +const RESOURCE_ID_CHUNK_SIZE = 100; + +type ResourceAccess = Pick< + Tables<"ResourceAccess">, + "space_id" | "source_local_id" +>; +type SharedConcept = Pick< + Tables<"my_concepts">, + "is_schema" | "last_modified" | "schema_id" | "source_local_id" | "space_id" +>; +type SharedContent = Pick< + Tables<"my_contents">, + | "author_id" + | "created" + | "last_modified" + | "metadata" + | "source_local_id" + | "space_id" + | "text" + | "variant" +>; +type SharedContentSummary = Pick< + Tables<"my_contents">, + "last_modified" | "source_local_id" | "space_id" +>; +type SharedSpace = Pick< + Tables<"my_spaces">, + "id" | "name" | "platform" | "url" +>; +type Platform = Enums<"Platform">; + +type ValidSharedSpace = { + name: string; + platform: Platform; + url: string; +}; + +export type SharedNodeCandidate = { + rid: string; + sourceLocalId: string; + spaceId: number; + spaceName: string; + spaceUri: string; + platform: Platform; + title: string; + created: string | null; + lastModified: string; + authorId?: number; + directMetadata: Json; +}; + +export type SharedNodeRows = { + concepts: SharedConcept[]; + directContents: SharedContent[]; + fullContentSummaries: SharedContentSummary[]; + resources: ResourceAccess[]; + spaces: SharedSpace[]; +}; + +const getResourceKey = ({ + sourceLocalId, + spaceId, +}: { + sourceLocalId: string; + spaceId: number; +}): string => `${spaceId}:${sourceLocalId}`; + +const normalizeUtcTimestamp = (timestamp: string | null): string | null => { + if (!timestamp) return null; + const date = new Date(`${timestamp}Z`); + return Number.isNaN(date.getTime()) ? null : date.toISOString(); +}; + +const getLatestTimestamp = (timestamps: (string | null)[]): string | null => { + const validTimestamps = timestamps + .map(normalizeUtcTimestamp) + .filter((timestamp): timestamp is string => typeof timestamp === "string"); + if (validTimestamps.length === 0) return null; + return validTimestamps.reduce((latest, timestamp) => + timestamp > latest ? timestamp : latest, + ); +}; + +export const buildSharedNodeCandidates = ({ + concepts, + currentSpaceId, + directContents, + fullContentSummaries, + resources, + spaces, +}: SharedNodeRows & { + currentSpaceId: number; +}): SharedNodeCandidate[] => { + const sharedResourceKeys = new Set( + resources + .filter((resource) => resource.space_id !== currentSpaceId) + .map((resource) => + getResourceKey({ + sourceLocalId: resource.source_local_id, + spaceId: resource.space_id, + }), + ), + ); + const spacesById = new Map( + spaces.flatMap((space): [number, ValidSharedSpace][] => { + if ( + typeof space.id !== "number" || + typeof space.name !== "string" || + space.platform === null || + typeof space.url !== "string" + ) + return []; + return [ + [ + space.id, + { + name: space.name, + platform: space.platform, + url: space.url, + }, + ], + ]; + }), + ); + const directByResource = new Map(); + directContents.forEach((content) => { + if ( + typeof content.space_id !== "number" || + typeof content.source_local_id !== "string" || + content.variant !== "direct" + ) + return; + directByResource.set( + getResourceKey({ + sourceLocalId: content.source_local_id, + spaceId: content.space_id, + }), + content, + ); + }); + + const fullModifiedByResource = new Map(); + fullContentSummaries.forEach((summary) => { + if ( + typeof summary.space_id !== "number" || + typeof summary.source_local_id !== "string" + ) + return; + fullModifiedByResource.set( + getResourceKey({ + sourceLocalId: summary.source_local_id, + spaceId: summary.space_id, + }), + summary.last_modified, + ); + }); + + return concepts + .flatMap((concept): SharedNodeCandidate[] => { + if ( + concept.is_schema !== false || + concept.schema_id === null || + typeof concept.space_id !== "number" || + typeof concept.source_local_id !== "string" + ) + return []; + + const resourceKey = getResourceKey({ + sourceLocalId: concept.source_local_id, + spaceId: concept.space_id, + }); + if (!sharedResourceKeys.has(resourceKey)) return []; + + const space = spacesById.get(concept.space_id); + const direct = directByResource.get(resourceKey); + if (!space || typeof direct?.text !== "string") return []; + + const lastModified = getLatestTimestamp([ + concept.last_modified, + direct.last_modified, + fullModifiedByResource.get(resourceKey) ?? null, + ]); + if (!lastModified) return []; + + let rid: string; + try { + rid = isRid(concept.source_local_id) + ? concept.source_local_id + : spaceUriAndLocalIdToRid( + space.url, + concept.source_local_id, + space.platform === "Obsidian" ? "note" : undefined, + ); + } catch { + return []; + } + + return [ + { + rid, + sourceLocalId: concept.source_local_id, + spaceId: concept.space_id, + spaceName: space.name, + spaceUri: space.url, + platform: space.platform, + title: direct.text, + created: normalizeUtcTimestamp(direct.created), + lastModified, + authorId: direct.author_id ?? undefined, + directMetadata: direct.metadata, + }, + ]; + }) + .sort( + (left, right) => + Date.parse(right.lastModified) - Date.parse(left.lastModified) || + left.title.localeCompare(right.title), + ); +}; + +const getGroupSharedResources = async ( + client: DGSupabaseClient, + currentSpaceId: number, + groupIds?: string[], +): Promise => { + const availableGroupIds = groupIds ?? (await getAvailableGroupIds(client)); + if (availableGroupIds.length === 0) return []; + + const resources = await getAllPages( + client + .from("ResourceAccess") + .select("space_id, source_local_id") + .in("account_uid", availableGroupIds) + .neq("space_id", currentSpaceId) + .order("space_id") + .order("source_local_id") + .order("account_uid"), + PAGE_SIZE, + ); + if (!Array.isArray(resources)) throw resources; + + return [ + ...new Map( + resources.map((resource) => [ + getResourceKey({ + sourceLocalId: resource.source_local_id, + spaceId: resource.space_id, + }), + resource, + ]), + ).values(), + ]; +}; + +const chunk = (values: T[], size: number): T[][] => + Array.from({ length: Math.ceil(values.length / size) }, (_, index) => + values.slice(index * size, (index + 1) * size), + ); + +const getSharedNodeRows = async ({ + client, + currentSpaceId, + groupIds, +}: { + client: DGSupabaseClient; + currentSpaceId: number; + groupIds?: string[]; +}): Promise => { + const resources = await getGroupSharedResources( + client, + currentSpaceId, + groupIds, + ); + if (resources.length === 0) + return { + concepts: [], + directContents: [], + fullContentSummaries: [], + resources, + spaces: [], + }; + + const spaceIds = [...new Set(resources.map((resource) => resource.space_id))]; + const spacesResponse = await client + .from("my_spaces") + .select("id, name, platform, url") + .in("id", spaceIds); + if (spacesResponse.error) throw spacesResponse.error; + + const concepts: SharedConcept[] = []; + const directContents: SharedContent[] = []; + const fullContentSummaries: SharedContentSummary[] = []; + for (const spaceId of spaceIds) { + const sourceLocalIds = resources + .filter((resource) => resource.space_id === spaceId) + .map((resource) => resource.source_local_id); + for (const ids of chunk(sourceLocalIds, RESOURCE_ID_CHUNK_SIZE)) { + const [conceptsResponse, directResponse, fullResponse] = + await Promise.all([ + client + .from("my_concepts") + .select( + "is_schema, last_modified, schema_id, source_local_id, space_id", + ) + .eq("space_id", spaceId) + .eq("is_schema", false) + .eq("arity", 0) + .in("source_local_id", ids), + client + .from("my_contents") + .select( + "author_id, created, last_modified, metadata, source_local_id, space_id, text, variant", + ) + .eq("space_id", spaceId) + .in("source_local_id", ids) + .eq("variant", "direct"), + client + .from("my_contents") + .select("last_modified, source_local_id, space_id") + .eq("space_id", spaceId) + .in("source_local_id", ids) + .eq("variant", "full"), + ]); + if (conceptsResponse.error) throw conceptsResponse.error; + if (directResponse.error) throw directResponse.error; + if (fullResponse.error) throw fullResponse.error; + concepts.push(...conceptsResponse.data); + directContents.push(...directResponse.data); + fullContentSummaries.push(...fullResponse.data); + } + } + + return { + concepts, + directContents, + fullContentSummaries, + resources, + spaces: spacesResponse.data, + }; +}; + +export const listGroupSharedNodes = async ({ + client, + currentSpaceId, + groupIds, +}: { + client: DGSupabaseClient; + currentSpaceId: number; + groupIds?: string[]; +}): Promise => { + const rows = await getSharedNodeRows({ client, currentSpaceId, groupIds }); + return buildSharedNodeCandidates({ ...rows, currentSpaceId }); +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d3d0aeba4..ba8f12ba6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -642,6 +642,9 @@ importers: vercel: specifier: 53.1.0 version: 53.1.0(rollup@4.60.3)(typescript@5.9.2) + vitest: + specifier: 'catalog:' + version: 4.1.6(@edge-runtime/vm@3.2.0)(@opentelemetry/api@1.9.0)(@types/node@22.20.0)(jsdom@20.0.3)(msw@2.11.1(@types/node@22.20.0)(typescript@5.5.4))(vite@7.3.3(@types/node@22.20.0)(jiti@1.21.7)(tsx@4.21.0)(yaml@2.8.2)) packages/eslint-config: devDependencies: diff --git a/turbo.json b/turbo.json index 28a36e4ad..b96290138 100644 --- a/turbo.json +++ b/turbo.json @@ -75,6 +75,9 @@ "test": { "outputs": [] }, + "test:unit": { + "outputs": [] + }, "dev": { "passThroughEnv": ["OBSIDIAN_PLUGIN_PATH"], "cache": false,