diff --git a/packages/database/features/addContent.feature b/packages/database/features/addContent.feature index 35e8b237a..46c6c5c8f 100644 --- a/packages/database/features/addContent.feature +++ b/packages/database/features/addContent.feature @@ -175,7 +175,8 @@ Feature: Content access "created": "2000/01/01", "last_modified": "2001/01/02", "text": "{\"type\":\"root\",\"children\":[]}", - "content_type": "application/vnd.discourse-graph.atjson+json; version=1" + "content_type": "application/vnd.discourse-graph.atjson+json; version=1", + "original": false } ] """ diff --git a/packages/database/src/dbTypes.ts b/packages/database/src/dbTypes.ts index a94c55b25..4a501846f 100644 --- a/packages/database/src/dbTypes.ts +++ b/packages/database/src/dbTypes.ts @@ -264,6 +264,7 @@ export type Database = { id: number last_modified: string metadata: Json + original: boolean | null part_of_id: number | null scale: Database["public"]["Enums"]["Scale"] source_local_id: string | null @@ -280,6 +281,7 @@ export type Database = { id?: number last_modified: string metadata?: Json + original?: boolean | null part_of_id?: number | null scale: Database["public"]["Enums"]["Scale"] source_local_id?: string | null @@ -296,6 +298,7 @@ export type Database = { id?: number last_modified?: string metadata?: Json + original?: boolean | null part_of_id?: number | null scale?: Database["public"]["Enums"]["Scale"] source_local_id?: string | null @@ -587,31 +590,31 @@ export type Database = { } FileReference: { Row: { - content_type: string created: string filehash: string filepath: string last_modified: string + original: boolean | null source_local_id: string space_id: number variant: Database["public"]["Enums"]["ContentVariant"] | null } Insert: { - content_type?: string created: string filehash: string filepath: string last_modified: string + original?: boolean | null source_local_id: string space_id: number variant?: Database["public"]["Enums"]["ContentVariant"] | null } Update: { - content_type?: string created?: string filehash?: string filepath?: string last_modified?: string + original?: boolean | null source_local_id?: string space_id?: number variant?: Database["public"]["Enums"]["ContentVariant"] | null @@ -619,38 +622,38 @@ export type Database = { Relationships: [ { foreignKeyName: "FileReference_content_fkey" - columns: ["space_id", "source_local_id", "variant", "content_type"] + columns: ["space_id", "source_local_id", "variant", "original"] isOneToOne: false referencedRelation: "Content" referencedColumns: [ "space_id", "source_local_id", "variant", - "content_type", + "original", ] }, { foreignKeyName: "FileReference_content_fkey" - columns: ["space_id", "source_local_id", "variant", "content_type"] + columns: ["space_id", "source_local_id", "variant", "original"] isOneToOne: false referencedRelation: "my_contents" referencedColumns: [ "space_id", "source_local_id", "variant", - "content_type", + "original", ] }, { foreignKeyName: "FileReference_content_fkey" - columns: ["space_id", "source_local_id", "variant", "content_type"] + columns: ["space_id", "source_local_id", "variant", "original"] isOneToOne: false referencedRelation: "my_contents_with_embedding_openai_text_embedding_3_small_1536" referencedColumns: [ "space_id", "source_local_id", "variant", - "content_type", + "original", ] }, ] @@ -1064,6 +1067,7 @@ export type Database = { id: number | null last_modified: string | null metadata: Json | null + original: boolean | null part_of_id: number | null scale: Database["public"]["Enums"]["Scale"] | null source_local_id: string | null @@ -1168,6 +1172,7 @@ export type Database = { my_contents_with_embedding_openai_text_embedding_3_small_1536: { Row: { author_id: number | null + content_type: string | null created: string | null creator_id: number | null document_id: number | null @@ -1175,6 +1180,7 @@ export type Database = { last_modified: string | null metadata: Json | null model: Database["public"]["Enums"]["EmbeddingName"] | null + original: boolean | null part_of_id: number | null scale: Database["public"]["Enums"]["Scale"] | null source_local_id: string | null @@ -1182,7 +1188,6 @@ export type Database = { text: string | null variant: Database["public"]["Enums"]["ContentVariant"] | null vector: string | null - content_type: string | null } Relationships: [ { @@ -1455,6 +1460,7 @@ export type Database = { id: number last_modified: string metadata: Json + original: boolean | null part_of_id: number | null scale: Database["public"]["Enums"]["Scale"] source_local_id: string | null @@ -1628,6 +1634,7 @@ export type Database = { id: number | null last_modified: string | null metadata: Json | null + original: boolean | null part_of_id: number | null scale: Database["public"]["Enums"]["Scale"] | null source_local_id: string | null @@ -2026,6 +2033,7 @@ export type Database = { | null variant: Database["public"]["Enums"]["ContentVariant"] | null content_type: string | null + original: boolean | null } document_local_input: { space_id: number | null diff --git a/packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql b/packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql index b459a6f90..06c58970a 100644 --- a/packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql +++ b/packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql @@ -1,15 +1,13 @@ +ALTER TABLE ONLY public."Content" +ADD COLUMN original BOOLEAN DEFAULT true; + +ALTER TYPE public."content_local_input" ADD ATTRIBUTE original BOOLEAN; + ALTER TABLE ONLY public."FileReference" DROP CONSTRAINT IF EXISTS "FileReference_content_fkey"; ALTER TABLE public."FileReference" -ADD COLUMN IF NOT EXISTS content_type character varying NOT NULL DEFAULT 'text/obsidian+markdown'; - -UPDATE public."FileReference" AS fr -SET content_type = ct.content_type -FROM public."Content" AS ct -WHERE fr.space_id = ct.space_id - AND fr.source_local_id = ct.source_local_id - AND fr.variant = ct.variant; +ADD COLUMN original BOOLEAN GENERATED ALWAYS AS (true) STORED; DROP INDEX IF EXISTS public.content_space_local_id_variant_idx; @@ -17,10 +15,14 @@ CREATE UNIQUE INDEX IF NOT EXISTS content_space_local_id_variant_content_type_id space_id, source_local_id, variant, content_type ) NULLS DISTINCT; +CREATE UNIQUE INDEX IF NOT EXISTS content_space_local_id_variant_content_type_originals_idx ON public."Content" USING btree ( + space_id, source_local_id, variant, original +); + ALTER TABLE ONLY public."FileReference" ADD CONSTRAINT "FileReference_content_fkey" FOREIGN KEY ( - space_id, source_local_id, variant, content_type -) REFERENCES public."Content" (space_id, source_local_id, variant, content_type) ON DELETE CASCADE; + space_id, source_local_id, variant, original +) REFERENCES public."Content"(space_id, source_local_id, variant, original) ON DELETE CASCADE; CREATE OR REPLACE FUNCTION public._local_content_to_db_content(data public.content_local_input) RETURNS public."Content" STABLE @@ -61,6 +63,7 @@ BEGIN SELECT id FROM public."Space" WHERE url = data.space_url INTO content.space_id; END IF; + content.original := CASE WHEN data.original IS false THEN NULL ELSE true END; -- now avoid null defaults IF content.metadata IS NULL then content.metadata := '{}'; @@ -74,7 +77,7 @@ $$; COMMENT ON FUNCTION public._local_content_to_db_content IS 'utility function so we have the option to use platform identifiers for content upsert'; -CREATE OR REPLACE FUNCTION public.upsert_content(v_space_id bigint, data jsonb, v_creator_id BIGINT, content_as_document boolean DEFAULT TRUE) +CREATE OR REPLACE FUNCTION public.upsert_content(v_space_id bigint, data jsonb, v_creator_id BIGINT, content_as_document boolean DEFAULT true) RETURNS SETOF BIGINT SET search_path = '' LANGUAGE plpgsql @@ -176,7 +179,8 @@ BEGIN space_id, last_modified, part_of_id, - content_type + content_type, + original ) VALUES ( db_content.document_id, db_content.source_local_id, @@ -190,7 +194,8 @@ BEGIN db_content.space_id, db_content.last_modified, db_content.part_of_id, - db_content.content_type + db_content.content_type, + db_content.original ) ON CONFLICT (space_id, source_local_id, variant, content_type) DO UPDATE SET document_id = COALESCE(db_content.document_id, EXCLUDED.document_id), @@ -201,7 +206,8 @@ BEGIN metadata = COALESCE(db_content.metadata, EXCLUDED.metadata), scale = COALESCE(db_content.scale, EXCLUDED.scale), last_modified = COALESCE(db_content.last_modified, EXCLUDED.last_modified), - part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id) + part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id), + original = db_content.original RETURNING id INTO STRICT upsert_id; IF model(embedding_inline(local_content)) IS NOT NULL THEN PERFORM public.upsert_content_embedding(upsert_id, model(embedding_inline(local_content)), vector(embedding_inline(local_content))); @@ -213,27 +219,56 @@ $$; COMMENT ON FUNCTION public.upsert_content IS 'batch content upsert'; -CREATE OR REPLACE VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536 AS +DROP VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536; + +CREATE VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536 AS SELECT -ct.id, -ct.document_id, -ct.source_local_id, -ct.variant, -ct.author_id, -ct.creator_id, -ct.created, -ct.text, -ct.metadata, -ct.scale, -ct.space_id, -ct.last_modified, -ct.part_of_id, -emb.model, -emb.vector, -ct.content_type + ct.id, + ct.document_id, + ct.source_local_id, + ct.variant, + ct.author_id, + ct.creator_id, + ct.created, + ct.text, + ct.metadata, + ct.scale, + ct.space_id, + ct.last_modified, + ct.part_of_id, + ct.content_type, + ct.original, + emb.model, + emb.vector FROM public."Content" AS ct -JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id) -LEFT OUTER JOIN public.my_accessible_resources () AS ra USING (space_id, source_local_id) -WHERE (ct.space_id = any (public.my_space_ids ('reader')) -OR (ct.space_id = any (public.my_space_ids ('partial')) AND ra.space_id IS NOT NULL)) + JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id) + LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id) +WHERE ( + ct.space_id = any(public.my_space_ids('reader')) + OR (ct.space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT null) +) AND NOT emb.obsolete; + +CREATE OR REPLACE VIEW public.my_contents AS +SELECT + id, + document_id, + source_local_id, + variant, + author_id, + creator_id, + created, + text, + metadata, + scale, + space_id, + last_modified, + part_of_id, + content_type, + original +FROM public."Content" + LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id) +WHERE ( + space_id = any(public.my_space_ids('reader')) + OR (space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT null) +); diff --git a/packages/database/supabase/schemas/assets.sql b/packages/database/supabase/schemas/assets.sql index 2e5003be3..9510b4111 100644 --- a/packages/database/supabase/schemas/assets.sql +++ b/packages/database/supabase/schemas/assets.sql @@ -5,17 +5,17 @@ CREATE TABLE IF NOT EXISTS public."FileReference" ( filehash character varying NOT NULL, -- or binary? "created" timestamp without time zone NOT NULL, last_modified timestamp without time zone NOT NULL, - content_type character varying NOT NULL DEFAULT 'text/obsidian+markdown', -- not allowed virtual with user types - variant public."ContentVariant" GENERATED ALWAYS AS ('full') STORED + variant public."ContentVariant" GENERATED ALWAYS AS ('full') STORED, + original BOOLEAN GENERATED ALWAYS AS (true) STORED ); ALTER TABLE ONLY public."FileReference" ADD CONSTRAINT "FileReference_pkey" PRIMARY KEY (source_local_id, space_id, filepath); ALTER TABLE ONLY public."FileReference" ADD CONSTRAINT "FileReference_content_fkey" FOREIGN KEY ( - space_id, source_local_id, variant, content_type -) REFERENCES public."Content" (space_id, source_local_id, variant, content_type) ON DELETE CASCADE; + space_id, source_local_id, variant, original +) REFERENCES public."Content"(space_id, source_local_id, variant, original) ON DELETE CASCADE; -- note the absence of on update ; the generated column forbids cascade, so it will error -- However, update on those columns should never happen. diff --git a/packages/database/supabase/schemas/content.sql b/packages/database/supabase/schemas/content.sql index a41846c6e..b70f099a1 100644 --- a/packages/database/supabase/schemas/content.sql +++ b/packages/database/supabase/schemas/content.sql @@ -88,7 +88,9 @@ CREATE TABLE IF NOT EXISTS public."Content" ( space_id bigint, last_modified timestamp without time zone NOT NULL, part_of_id bigint, - content_type character varying NOT NULL DEFAULT 'text/plain' + content_type character varying NOT NULL DEFAULT 'text/plain', + -- Use null, never false for the non-original rows. + original BOOLEAN DEFAULT true ); ALTER TABLE ONLY public."Content" @@ -135,6 +137,10 @@ CREATE UNIQUE INDEX content_space_local_id_variant_content_type_idx ON public."C space_id, source_local_id, variant, content_type ) NULLS DISTINCT; +CREATE UNIQUE INDEX IF NOT EXISTS content_space_local_id_variant_content_type_originals_idx ON public."Content" USING btree ( + space_id, source_local_id, variant, original +); + CREATE INDEX "Content_text" ON public."Content" USING pgroonga (text); ALTER TABLE public."Content" OWNER TO "postgres"; @@ -265,7 +271,8 @@ SELECT space_id, last_modified, part_of_id, - content_type + content_type, + original FROM public."Content" LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id) WHERE ( @@ -332,6 +339,7 @@ CREATE TYPE public.content_local_input AS ( last_modified timestamp without time zone, part_of_id bigint, content_type character varying, + original boolean, -- this should be true or false, will be translated to true or null -- local values document_local_id character varying, creator_local_id character varying, @@ -425,6 +433,7 @@ BEGIN SELECT id FROM public."Space" WHERE url = data.space_url INTO content.space_id; END IF; + content.original := CASE WHEN data.original IS false THEN NULL ELSE true END; -- now avoid null defaults IF content.metadata IS NULL then content.metadata := '{}'; @@ -631,7 +640,8 @@ BEGIN space_id, last_modified, part_of_id, - content_type + content_type, + original ) VALUES ( db_content.document_id, db_content.source_local_id, @@ -645,7 +655,8 @@ BEGIN db_content.space_id, db_content.last_modified, db_content.part_of_id, - db_content.content_type + db_content.content_type, + db_content.original ) ON CONFLICT (space_id, source_local_id, variant, content_type) DO UPDATE SET document_id = COALESCE(db_content.document_id, EXCLUDED.document_id), @@ -656,7 +667,8 @@ BEGIN metadata = COALESCE(db_content.metadata, EXCLUDED.metadata), scale = COALESCE(db_content.scale, EXCLUDED.scale), last_modified = COALESCE(db_content.last_modified, EXCLUDED.last_modified), - part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id) + part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id), + original = db_content.original RETURNING id INTO STRICT upsert_id; IF model(embedding_inline(local_content)) IS NOT NULL THEN PERFORM public.upsert_content_embedding(upsert_id, model(embedding_inline(local_content)), vector(embedding_inline(local_content))); diff --git a/packages/database/supabase/schemas/embedding.sql b/packages/database/supabase/schemas/embedding.sql index 43801cf5d..eb129fe47 100644 --- a/packages/database/supabase/schemas/embedding.sql +++ b/packages/database/supabase/schemas/embedding.sql @@ -43,9 +43,10 @@ ct.scale, ct.space_id, ct.last_modified, ct.part_of_id, +ct.content_type, +ct.original, emb.model, -emb.vector, -ct.content_type +emb.vector FROM public."Content" AS ct JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id) LEFT OUTER JOIN public.my_accessible_resources () AS ra USING (space_id, source_local_id)