Skip to content

ENG-1907 adjustments #1236

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
maparent wants to merge 1 commit into
base: eng-1907-update-content-uniqueness-and-upsert-surfaces
Choose a base branch
from
Open

ENG-1907 adjustments #1236

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion packages/database/features/addContent.feature
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,8 @@ Feature: Content access
"created": "2000/01/01",
"last_modified": "2001/01/02",
"text": "{\"type\":\"root\",\"children\":[]}",
"content_type": "application/vnd.discourse-graph.atjson+json; version=1"
"content_type": "application/vnd.discourse-graph.atjson+json; version=1",
"original": false
}
]
"""
Expand Down
28 changes: 18 additions & 10 deletions packages/database/src/dbTypes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,7 @@ export type Database = {
id: number
last_modified: string
metadata: Json
original: boolean | null
part_of_id: number | null
scale: Database["public"]["Enums"]["Scale"]
source_local_id: string | null
Expand All @@ -280,6 +281,7 @@ export type Database = {
id?: number
last_modified: string
metadata?: Json
original?: boolean | null
part_of_id?: number | null
scale: Database["public"]["Enums"]["Scale"]
source_local_id?: string | null
Expand All @@ -296,6 +298,7 @@ export type Database = {
id?: number
last_modified?: string
metadata?: Json
original?: boolean | null
part_of_id?: number | null
scale?: Database["public"]["Enums"]["Scale"]
source_local_id?: string | null
Expand Down Expand Up @@ -587,70 +590,70 @@ export type Database = {
}
FileReference: {
Row: {
content_type: string
created: string
filehash: string
filepath: string
last_modified: string
original: boolean | null
source_local_id: string
space_id: number
variant: Database["public"]["Enums"]["ContentVariant"] | null
}
Insert: {
content_type?: string
created: string
filehash: string
filepath: string
last_modified: string
original?: boolean | null
source_local_id: string
space_id: number
variant?: Database["public"]["Enums"]["ContentVariant"] | null
}
Update: {
content_type?: string
created?: string
filehash?: string
filepath?: string
last_modified?: string
original?: boolean | null
source_local_id?: string
space_id?: number
variant?: Database["public"]["Enums"]["ContentVariant"] | null
}
Relationships: [
{
foreignKeyName: "FileReference_content_fkey"
columns: ["space_id", "source_local_id", "variant", "content_type"]
columns: ["space_id", "source_local_id", "variant", "original"]
isOneToOne: false
referencedRelation: "Content"
referencedColumns: [
"space_id",
"source_local_id",
"variant",
"content_type",
"original",
]
},
{
foreignKeyName: "FileReference_content_fkey"
columns: ["space_id", "source_local_id", "variant", "content_type"]
columns: ["space_id", "source_local_id", "variant", "original"]
isOneToOne: false
referencedRelation: "my_contents"
referencedColumns: [
"space_id",
"source_local_id",
"variant",
"content_type",
"original",
]
},
{
foreignKeyName: "FileReference_content_fkey"
columns: ["space_id", "source_local_id", "variant", "content_type"]
columns: ["space_id", "source_local_id", "variant", "original"]
isOneToOne: false
referencedRelation: "my_contents_with_embedding_openai_text_embedding_3_small_1536"
referencedColumns: [
"space_id",
"source_local_id",
"variant",
"content_type",
"original",
]
},
]
Expand Down Expand Up @@ -1064,6 +1067,7 @@ export type Database = {
id: number | null
last_modified: string | null
metadata: Json | null
original: boolean | null
part_of_id: number | null
scale: Database["public"]["Enums"]["Scale"] | null
source_local_id: string | null
Expand Down Expand Up @@ -1168,21 +1172,22 @@ export type Database = {
my_contents_with_embedding_openai_text_embedding_3_small_1536: {
Row: {
author_id: number | null
content_type: string | null
created: string | null
creator_id: number | null
document_id: number | null
id: number | null
last_modified: string | null
metadata: Json | null
model: Database["public"]["Enums"]["EmbeddingName"] | null
original: boolean | null
part_of_id: number | null
scale: Database["public"]["Enums"]["Scale"] | null
source_local_id: string | null
space_id: number | null
text: string | null
variant: Database["public"]["Enums"]["ContentVariant"] | null
vector: string | null
content_type: string | null
}
Relationships: [
{
Expand Down Expand Up @@ -1455,6 +1460,7 @@ export type Database = {
id: number
last_modified: string
metadata: Json
original: boolean | null
part_of_id: number | null
scale: Database["public"]["Enums"]["Scale"]
source_local_id: string | null
Expand Down Expand Up @@ -1628,6 +1634,7 @@ export type Database = {
id: number | null
last_modified: string | null
metadata: Json | null
original: boolean | null
part_of_id: number | null
scale: Database["public"]["Enums"]["Scale"] | null
source_local_id: string | null
Expand Down Expand Up @@ -2026,6 +2033,7 @@ export type Database = {
| null
variant: Database["public"]["Enums"]["ContentVariant"] | null
content_type: string | null
original: boolean | null
}
document_local_input: {
space_id: number | null
Expand Down
105 changes: 70 additions & 35 deletions packages/database/supabase/migrations/20260704120000_update_content_uniqueness.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,28 @@
ALTER TABLE ONLY public."Content"
ADD COLUMN original BOOLEAN DEFAULT true;

ALTER TYPE public."content_local_input" ADD ATTRIBUTE original BOOLEAN;

ALTER TABLE ONLY public."FileReference"
DROP CONSTRAINT IF EXISTS "FileReference_content_fkey";

ALTER TABLE public."FileReference"
ADD COLUMN IF NOT EXISTS content_type character varying NOT NULL DEFAULT 'text/obsidian+markdown';

UPDATE public."FileReference" AS fr
SET content_type = ct.content_type
FROM public."Content" AS ct
WHERE fr.space_id = ct.space_id
AND fr.source_local_id = ct.source_local_id
AND fr.variant = ct.variant;
ADD COLUMN original BOOLEAN GENERATED ALWAYS AS (true) STORED;

DROP INDEX IF EXISTS public.content_space_local_id_variant_idx;

CREATE UNIQUE INDEX IF NOT EXISTS content_space_local_id_variant_content_type_idx ON public."Content" USING btree (
space_id, source_local_id, variant, content_type
) NULLS DISTINCT;

CREATE UNIQUE INDEX IF NOT EXISTS content_space_local_id_variant_content_type_originals_idx ON public."Content" USING btree (
space_id, source_local_id, variant, original
);

ALTER TABLE ONLY public."FileReference"
ADD CONSTRAINT "FileReference_content_fkey" FOREIGN KEY (
space_id, source_local_id, variant, content_type
) REFERENCES public."Content" (space_id, source_local_id, variant, content_type) ON DELETE CASCADE;
space_id, source_local_id, variant, original
) REFERENCES public."Content"(space_id, source_local_id, variant, original) ON DELETE CASCADE;

CREATE OR REPLACE FUNCTION public._local_content_to_db_content(data public.content_local_input)
RETURNS public."Content" STABLE
Expand Down Expand Up @@ -61,6 +63,7 @@ BEGIN
SELECT id FROM public."Space"
WHERE url = data.space_url INTO content.space_id;
END IF;
content.original := CASE WHEN data.original IS false THEN NULL ELSE true END;
-- now avoid null defaults
IF content.metadata IS NULL then
content.metadata := '{}';
Expand All @@ -74,7 +77,7 @@ $$;

COMMENT ON FUNCTION public._local_content_to_db_content IS 'utility function so we have the option to use platform identifiers for content upsert';

CREATE OR REPLACE FUNCTION public.upsert_content(v_space_id bigint, data jsonb, v_creator_id BIGINT, content_as_document boolean DEFAULT TRUE)
CREATE OR REPLACE FUNCTION public.upsert_content(v_space_id bigint, data jsonb, v_creator_id BIGINT, content_as_document boolean DEFAULT true)
RETURNS SETOF BIGINT
SET search_path = ''
LANGUAGE plpgsql
Expand Down Expand Up @@ -176,7 +179,8 @@ BEGIN
space_id,
last_modified,
part_of_id,
content_type
content_type,
original
) VALUES (
db_content.document_id,
db_content.source_local_id,
Expand All @@ -190,7 +194,8 @@ BEGIN
db_content.space_id,
db_content.last_modified,
db_content.part_of_id,
db_content.content_type
db_content.content_type,
db_content.original
)
ON CONFLICT (space_id, source_local_id, variant, content_type) DO UPDATE SET
document_id = COALESCE(db_content.document_id, EXCLUDED.document_id),
Expand All @@ -201,7 +206,8 @@ BEGIN
metadata = COALESCE(db_content.metadata, EXCLUDED.metadata),
scale = COALESCE(db_content.scale, EXCLUDED.scale),
last_modified = COALESCE(db_content.last_modified, EXCLUDED.last_modified),
part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id)
part_of_id = COALESCE(db_content.part_of_id, EXCLUDED.part_of_id),
original = db_content.original
RETURNING id INTO STRICT upsert_id;
IF model(embedding_inline(local_content)) IS NOT NULL THEN
PERFORM public.upsert_content_embedding(upsert_id, model(embedding_inline(local_content)), vector(embedding_inline(local_content)));
Expand All @@ -213,27 +219,56 @@ $$;

COMMENT ON FUNCTION public.upsert_content IS 'batch content upsert';

CREATE OR REPLACE VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536 AS
DROP VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536;

CREATE VIEW public.my_contents_with_embedding_openai_text_embedding_3_small_1536 AS
SELECT
ct.id,
ct.document_id,
ct.source_local_id,
ct.variant,
ct.author_id,
ct.creator_id,
ct.created,
ct.text,
ct.metadata,
ct.scale,
ct.space_id,
ct.last_modified,
ct.part_of_id,
emb.model,
emb.vector,
ct.content_type
ct.id,
ct.document_id,
ct.source_local_id,
ct.variant,
ct.author_id,
ct.creator_id,
ct.created,
ct.text,
ct.metadata,
ct.scale,
ct.space_id,
ct.last_modified,
ct.part_of_id,
ct.content_type,
ct.original,
emb.model,
emb.vector
FROM public."Content" AS ct
JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id)
LEFT OUTER JOIN public.my_accessible_resources () AS ra USING (space_id, source_local_id)
WHERE (ct.space_id = any (public.my_space_ids ('reader'))
OR (ct.space_id = any (public.my_space_ids ('partial')) AND ra.space_id IS NOT NULL))
JOIN public."ContentEmbedding_openai_text_embedding_3_small_1536" AS emb ON (ct.id = emb.target_id)
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
WHERE (
ct.space_id = any(public.my_space_ids('reader'))
OR (ct.space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT null)
)
AND NOT emb.obsolete;

CREATE OR REPLACE VIEW public.my_contents AS
SELECT
id,
document_id,
source_local_id,
variant,
author_id,
creator_id,
created,
text,
metadata,
scale,
space_id,
last_modified,
part_of_id,
content_type,
original
FROM public."Content"
LEFT OUTER JOIN public.my_accessible_resources() AS ra USING (space_id, source_local_id)
WHERE (
space_id = any(public.my_space_ids('reader'))
OR (space_id = any(public.my_space_ids('partial')) AND ra.space_id IS NOT null)
);
8 changes: 4 additions & 4 deletions packages/database/supabase/schemas/assets.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,17 @@ CREATE TABLE IF NOT EXISTS public."FileReference" (
filehash character varying NOT NULL, -- or binary?
"created" timestamp without time zone NOT NULL,
last_modified timestamp without time zone NOT NULL,
content_type character varying NOT NULL DEFAULT 'text/obsidian+markdown',
-- not allowed virtual with user types
variant public."ContentVariant" GENERATED ALWAYS AS ('full') STORED
variant public."ContentVariant" GENERATED ALWAYS AS ('full') STORED,
original BOOLEAN GENERATED ALWAYS AS (true) STORED
);
ALTER TABLE ONLY public."FileReference"
ADD CONSTRAINT "FileReference_pkey" PRIMARY KEY (source_local_id, space_id, filepath);

ALTER TABLE ONLY public."FileReference"
ADD CONSTRAINT "FileReference_content_fkey" FOREIGN KEY (
space_id, source_local_id, variant, content_type
) REFERENCES public."Content" (space_id, source_local_id, variant, content_type) ON DELETE CASCADE;
space_id, source_local_id, variant, original
) REFERENCES public."Content"(space_id, source_local_id, variant, original) ON DELETE CASCADE;
-- note the absence of on update ; the generated column forbids cascade, so it will error
-- However, update on those columns should never happen.

Expand Down
Loading