From 47dd6a4ded6aabc790604bdac8a5f27c8f84ef96 Mon Sep 17 00:00:00 2001 From: Shawn Zivontsis Date: Thu, 23 Jul 2026 11:40:31 -0400 Subject: [PATCH 1/3] Update moz_origins index column order to match desktop This change ports the fix from https://bugzilla.mozilla.org/show_bug.cgi?id=2025999 which replaces the (prefix, host) unique index on moz_origins with a (host, prefix) index instead. This improves the performance of the index by putting the higher-cardinality column first, and also makes queries which don't filter on prefix eligible for the index. Similar to the fix on desktop, we create a new table and copy the data over due to sqlite limitations on modifying constraints. However, we can't use defer_foreign_keys like we do on desktop, since the foreign key here is ON DELETE CASCADE -- therefore we have to also temporarily null out the foreign key references and copy them back afterward. --- .../places/sql/create_shared_schema.sql | 2 +- .../places/sql/create_shared_triggers.sql | 6 +- components/places/src/db/schema.rs | 278 +++++++++++++++++- 3 files changed, 281 insertions(+), 5 deletions(-) diff --git a/components/places/sql/create_shared_schema.sql b/components/places/sql/create_shared_schema.sql index 7073204fa61..64352931f1b 100644 --- a/components/places/sql/create_shared_schema.sql +++ b/components/places/sql/create_shared_schema.sql @@ -152,7 +152,7 @@ CREATE TABLE IF NOT EXISTS moz_origins ( host TEXT NOT NULL, rev_host TEXT NOT NULL, frecency INTEGER NOT NULL, -- XXX - why not default of -1 like in moz_places? - UNIQUE (prefix, host) + UNIQUE (host, prefix) ); CREATE INDEX IF NOT EXISTS hostindex ON moz_origins(rev_host); diff --git a/components/places/sql/create_shared_triggers.sql b/components/places/sql/create_shared_triggers.sql index e9232cfd6ed..067e15405df 100644 --- a/components/places/sql/create_shared_triggers.sql +++ b/components/places/sql/create_shared_triggers.sql @@ -184,7 +184,7 @@ BEGIN OLD.rev_host, MAX(OLD.frecency, 0) ) - ON CONFLICT(prefix, host) DO UPDATE + ON CONFLICT(host, prefix) DO UPDATE SET frecency = frecency + OLD.frecency WHERE OLD.frecency > 0; @@ -211,7 +211,7 @@ BEGIN get_host_and_port(OLD.url), -MAX(OLD.frecency, 0) ) - ON CONFLICT(prefix, host) DO UPDATE + ON CONFLICT(host, prefix) DO UPDATE SET frecency_delta = frecency_delta - OLD.frecency WHERE OLD.frecency > 0; END; @@ -250,7 +250,7 @@ BEGIN get_host_and_port(NEW.url), MAX(NEW.frecency, 0) - MAX(OLD.frecency, 0) ) - ON CONFLICT(prefix, host) DO UPDATE + ON CONFLICT(host, prefix) DO UPDATE SET frecency_delta = frecency_delta + EXCLUDED.frecency_delta; END; diff --git a/components/places/src/db/schema.rs b/components/places/src/db/schema.rs index 502b492e2d3..866705671cf 100644 --- a/components/places/src/db/schema.rs +++ b/components/places/src/db/schema.rs @@ -14,7 +14,7 @@ use sql_support::ConnExt; use super::db::{Pragma, PragmaGuard}; -pub const VERSION: u32 = 20; +pub const VERSION: u32 = 21; // Shared schema and temp tables for the read-write and Sync connections. const CREATE_SHARED_SCHEMA_SQL: &str = include_str!("../../sql/create_shared_schema.sql"); @@ -341,6 +341,70 @@ pub fn upgrade_from(db: &Connection, from: u32) -> rusqlite::Result<()> { db.execute("ANALYZE moz_places", [])?; db.execute("ANALYZE moz_historyvisits", [])?; } + 20 => { + // Invert the moz_origins UNIQUE constraint to (host, prefix), so the + // higher cardinality column comes first and queries only filtering on + // host, like the address bar ones, can use the index. + + // Skip the rebuild if the constraint is already inverted. + let already_inverted = db.exists( + "SELECT 1 FROM sqlite_schema + WHERE type = 'table' AND name = 'moz_origins' + AND sql LIKE '%UNIQUE (host, prefix)%'", + [], + )?; + if !already_inverted { + // The table must be rebuilt, and PRAGMA foreign_keys is a no-op + // inside the migration transaction, so the moz_places.origin_id + // foreign key stays enforced throughout. Since origin_id is + // nullable, we stash it and null it out, so that nothing + // references moz_origins while it's swapped out. + // The stash must be keyed, or the restore below would scan it + // for every row. + db.execute_batch( + "CREATE TEMP TABLE moz_places_origin_id_stash ( + id INTEGER PRIMARY KEY, + origin_id INTEGER NOT NULL + ); + + INSERT INTO moz_places_origin_id_stash (id, origin_id) + SELECT id, origin_id FROM moz_places WHERE origin_id IS NOT NULL; + + CREATE TABLE moz_origins_new ( + id INTEGER PRIMARY KEY, + prefix TEXT NOT NULL, + host TEXT NOT NULL, + rev_host TEXT NOT NULL, + frecency INTEGER NOT NULL, + UNIQUE (host, prefix) + ); + + INSERT INTO moz_origins_new (id, prefix, host, rev_host, frecency) + SELECT id, prefix, host, rev_host, frecency FROM moz_origins; + + UPDATE moz_places SET origin_id = NULL WHERE origin_id IS NOT NULL; + + -- A rename would rewrite the REFERENCES clause in moz_places, while a + -- drop leaves it dangling until the new table takes over the name. + DROP TABLE moz_origins; + + ALTER TABLE moz_origins_new RENAME TO moz_origins; + + UPDATE moz_places + SET origin_id = stash.origin_id + FROM moz_places_origin_id_stash AS stash + WHERE moz_places.id = stash.id; + + DROP TABLE moz_places_origin_id_stash;", + )?; + // Recreate hostindex, which was dropped along with the old table, + // by calling the shared schema file + db.execute_batch(CREATE_SHARED_SCHEMA_SQL)?; + // Manually call analyze so the planner has statistics for the + // rebuilt table + db.execute("ANALYZE moz_origins", [])?; + } + } // Add more migrations here... // Any other from value indicates that something very wrong happened @@ -1173,6 +1237,218 @@ mod tests { ); } + #[test] + fn test_upgrade_schema_20_21() { + use std::sync::Arc; + let db_file = MigratedDatabaseFile::new(PlacesInitializer::new_for_test(), CREATE_V17_DB); + db_file.upgrade_to(20); + + // Seed origins, plus pages pointing at them, so the migration has both rows to + // rebuild and foreign keys to keep intact. + let conn = db_file.open(); + conn.execute_batch( + "INSERT INTO moz_origins(id, prefix, host, rev_host, frecency) + VALUES (1, 'https://', 'example.com', 'moc.elpmaxe.', 100), + (2, 'http://', 'example.com', 'moc.elpmaxe.', 50), + (3, 'https://', 'mozilla.org', 'gro.allizom.', 75); + + UPDATE moz_places SET origin_id = 1 WHERE id = 1; + + INSERT INTO moz_places(id, guid, url, origin_id, frecency) + VALUES (2, 'page_guid__2', 'http://example.com/', 2, -1), + (3, 'page_guid__3', 'https://mozilla.org/', 3, -1), + (4, 'page_guid__4', 'https://unvisited.com/', NULL, -1);", + ) + .expect("should seed origins and places"); + + fn unique_index_columns(conn: &Connection) -> Vec { + let indexes = conn + .query_rows_and_then( + "SELECT name FROM pragma_index_list('moz_origins') WHERE origin = 'u'", + [], + |row| row.get::<_, String>(0), + ) + .expect("should query the unique indexes"); + assert_eq!( + indexes.len(), + 1, + "moz_origins should have a single unique index" + ); + conn.query_rows_and_then( + "SELECT name FROM pragma_index_info(?) ORDER BY seqno", + (indexes[0].as_str(),), + |row| row.get::<_, String>(0), + ) + .expect("should query the unique index columns") + } + + // moz_origins should be keyed on (prefix, host) before the migration. The + // upgrades above replay the current shared schema, so check they left the + // constraint alone. + assert_eq!(unique_index_columns(&conn), &["prefix", "host"]); + drop(conn); + + // Open through `PlacesDb`, so the migration runs with foreign keys enforced; + // otherwise the null-out step it relies on goes untested. + let db = PlacesDb::open( + &db_file.path, + ConnectionType::ReadWrite, + 0, + Arc::new(parking_lot::Mutex::new(())), + ) + .expect("should upgrade"); + + // The unique index should now lead with the higher cardinality column. + assert_eq!(unique_index_columns(&db), &["host", "prefix"]); + + // The origins themselves should be untouched, ids included, since moz_places + // references them. + #[derive(Eq, PartialEq, Debug)] + struct OriginRow { + id: i64, + prefix: String, + host: String, + rev_host: String, + frecency: i64, + } + let origins = db + .query_rows_and_then( + "SELECT id, prefix, host, rev_host, frecency FROM moz_origins ORDER BY id", + [], + |row| -> rusqlite::Result<_> { + Ok(OriginRow { + id: row.get("id")?, + prefix: row.get("prefix")?, + host: row.get("host")?, + rev_host: row.get("rev_host")?, + frecency: row.get("frecency")?, + }) + }, + ) + .expect("should query all origins"); + assert_eq!( + origins, + &[ + OriginRow { + id: 1, + prefix: "https://".into(), + host: "example.com".into(), + rev_host: "moc.elpmaxe.".into(), + frecency: 100, + }, + OriginRow { + id: 2, + prefix: "http://".into(), + host: "example.com".into(), + rev_host: "moc.elpmaxe.".into(), + frecency: 50, + }, + OriginRow { + id: 3, + prefix: "https://".into(), + host: "mozilla.org".into(), + rev_host: "gro.allizom.".into(), + frecency: 75, + }, + ] + ); + + // ...And every page should still point at the origin it did before. + let pages = db + .query_rows_and_then( + "SELECT id, origin_id FROM moz_places ORDER BY id", + [], + |row| -> rusqlite::Result<_> { + Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) + }, + ) + .expect("should query all places"); + assert_eq!( + pages, + &[(1, Some(1)), (2, Some(2)), (3, Some(3)), (4, None)] + ); + + // hostindex should have been recreated, since rebuilding the table dropped it. + assert!(db + .exists( + "SELECT 1 FROM sqlite_schema WHERE type = 'index' AND name = 'hostindex'", + [] + ) + .expect("should look for hostindex")); + + // The table used to rebuild moz_origins should have been removed. + assert!(!db + .exists( + "SELECT 1 FROM sqlite_schema WHERE name = 'moz_origins_new'", + [] + ) + .expect("should look for moz_origins_new")); + + // moz_places should still reference the rebuilt moz_origins. + let foreign_key = db + .query_row( + r#"SELECT "table", "from", "to" FROM pragma_foreign_key_list('moz_places')"#, + [], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }, + ) + .expect("should query the foreign key"); + assert_eq!( + foreign_key, + ("moz_origins".into(), "origin_id".into(), "id".into()) + ); + + let integrity_ok = db + .query_row("PRAGMA integrity_check", [], |row| { + Ok(row.get::<_, String>(0)? == "ok") + }) + .expect("should perform integrity check"); + assert!(integrity_ok); + + let foreign_keys_ok = db + .prepare("PRAGMA foreign_key_check") + .and_then(|mut statement| Ok(statement.query([])?.next()?.is_none())) + .expect("should perform foreign key check"); + assert!(foreign_keys_ok); + + // The origin-creation trigger should still upsert against the rebuilt + // moz_origins. + db.execute( + "INSERT INTO moz_places(guid, url, url_hash) + VALUES ('page_guid__5', 'https://example.com/new-page', + hash('https://example.com/new-page')), + ('page_guid__6', 'https://example.org/', + hash('https://example.org/'))", + [], + ) + .expect("should insert pages"); + // origins are maintained via triggers, so make sure they are done. + crate::storage::delete_pending_temp_tables(&db).expect("should update origins"); + + // Adding a page for a known origin should update it rather than add a new one... + assert_eq!( + db.conn_ext_query_one::( + "SELECT origin_id FROM moz_places WHERE guid = 'page_guid__5'" + ) + .expect("should query the known origin"), + 1 + ); + // ...And a page for an unknown origin should add one. + assert_eq!( + db.conn_ext_query_one::( + "SELECT COUNT(*) FROM moz_origins + WHERE prefix = 'https://' AND host = 'example.org'" + ) + .expect("should query the new origin"), + 1 + ); + } + #[test] fn test_all_upgrades() { // Test the migration process in general: open a fresh DB and a DB that's gone through the migration From e157a2b8d4d92cd665275d50fa3746ebcc74f29b Mon Sep 17 00:00:00 2001 From: Shawn Zivontsis Date: Sat, 1 Aug 2026 20:09:51 -0400 Subject: [PATCH 2/3] Revise migration 20 to directly modify sqlite_schema Instead of rebuilding the entire moz_origins table, which requires nulling out and then restoring the entire origin_id column in the large moz_places table, we can just rewrite the schema in sqlite_schema and then trigger a REINDEX. This is not as safe, and has the risk of causing silent data corruption if the schema is modified incorrectly (like for example if it was unknowingly modified by application code). But, it is significantly faster and generates significantly less WAL. --- components/places/src/db/schema.rs | 85 +++++++++++++----------------- 1 file changed, 37 insertions(+), 48 deletions(-) diff --git a/components/places/src/db/schema.rs b/components/places/src/db/schema.rs index 866705671cf..4218c11db8f 100644 --- a/components/places/src/db/schema.rs +++ b/components/places/src/db/schema.rs @@ -354,54 +354,43 @@ pub fn upgrade_from(db: &Connection, from: u32) -> rusqlite::Result<()> { [], )?; if !already_inverted { - // The table must be rebuilt, and PRAGMA foreign_keys is a no-op - // inside the migration transaction, so the moz_places.origin_id - // foreign key stays enforced throughout. Since origin_id is - // nullable, we stash it and null it out, so that nothing - // references moz_origins while it's swapped out. - // The stash must be keyed, or the restore below would scan it - // for every row. - db.execute_batch( - "CREATE TEMP TABLE moz_places_origin_id_stash ( - id INTEGER PRIMARY KEY, - origin_id INTEGER NOT NULL - ); - - INSERT INTO moz_places_origin_id_stash (id, origin_id) - SELECT id, origin_id FROM moz_places WHERE origin_id IS NOT NULL; - - CREATE TABLE moz_origins_new ( - id INTEGER PRIMARY KEY, - prefix TEXT NOT NULL, - host TEXT NOT NULL, - rev_host TEXT NOT NULL, - frecency INTEGER NOT NULL, - UNIQUE (host, prefix) - ); - - INSERT INTO moz_origins_new (id, prefix, host, rev_host, frecency) - SELECT id, prefix, host, rev_host, frecency FROM moz_origins; - - UPDATE moz_places SET origin_id = NULL WHERE origin_id IS NOT NULL; - - -- A rename would rewrite the REFERENCES clause in moz_places, while a - -- drop leaves it dangling until the new table takes over the name. - DROP TABLE moz_origins; - - ALTER TABLE moz_origins_new RENAME TO moz_origins; - - UPDATE moz_places - SET origin_id = stash.origin_id - FROM moz_places_origin_id_stash AS stash - WHERE moz_places.id = stash.id; - - DROP TABLE moz_places_origin_id_stash;", - )?; - // Recreate hostindex, which was dropped along with the old table, - // by calling the shared schema file - db.execute_batch(CREATE_SHARED_SCHEMA_SQL)?; - // Manually call analyze so the planner has statistics for the - // rebuilt table + // PRAGMA foreign_keys is a no-op inside the migration transaction, + // so dropping moz_origins to rebuild it would cascade to every + // page; we rewrite the stored schema in place instead. + // Must not change anything but the constraints; changing the column + // list will silently corrupt existing data. + const NEW_SQL: &str = "CREATE TABLE moz_origins ( \ + id INTEGER PRIMARY KEY, \ + prefix TEXT NOT NULL, \ + host TEXT NOT NULL, \ + rev_host TEXT NOT NULL, \ + frecency INTEGER NOT NULL, \ + UNIQUE (host, prefix))"; + + let schema_version: i64 = + db.query_row("PRAGMA schema_version", [], |row| row.get(0))?; + + { + let _w = PragmaGuard::new(db, Pragma::WritableSchema, true)?; + db.execute( + "UPDATE sqlite_schema SET + sql = ? + WHERE type = 'table' AND name = 'moz_origins'", + // _Must_ be valid SQL; updating `sqlite_schema.sql` with + // invalid SQL will corrupt the database. + rusqlite::params![NEW_SQL], + )?; + } + + // Reload the schema and rebuild the index with the new column order + db.execute_one("PRAGMA writable_schema = RESET")?; + db.execute("REINDEX moz_origins", [])?; + + // Increment the schema version like an ALTER TABLE would, so that + // other connections reload the schema + db.execute_one(&format!("PRAGMA schema_version = {}", schema_version + 1))?; + + // Manually call analyze so the planner can start using the index immediately db.execute("ANALYZE moz_origins", [])?; } } From 83aad889d634f020a3e787d3190e2a78dc25bef9 Mon Sep 17 00:00:00 2001 From: Shawn Zivontsis Date: Sun, 9 Aug 2026 22:00:47 -0400 Subject: [PATCH 3/3] Add a frozen copy of places schema v20 for old migrations Several places DB migrations invoke the shared schema SQL to create/refresh indexes. In order to avoid older migrations being impacted by schema changes, add a frozen copy of the places schema at version 20 and update past migrations to invoke that instead. In future migrations, we can either avoid this pattern or add new frozen copies of the schema at the time of those migrations, following the same pattern. --- .../sql/legacy/create_shared_schema_v20.sql | 283 ++++++++++++++++++ components/places/src/db/schema.rs | 25 +- 2 files changed, 298 insertions(+), 10 deletions(-) create mode 100644 components/places/sql/legacy/create_shared_schema_v20.sql diff --git a/components/places/sql/legacy/create_shared_schema_v20.sql b/components/places/sql/legacy/create_shared_schema_v20.sql new file mode 100644 index 00000000000..7073204fa61 --- /dev/null +++ b/components/places/sql/legacy/create_shared_schema_v20.sql @@ -0,0 +1,283 @@ +-- This Source Code Form is subject to the terms of the Mozilla Public +-- License, v. 2.0. If a copy of the MPL was not distributed with this +-- file, You can obtain one at http://mozilla.org/MPL/2.0/. + +-- XXX - TODO - moz_annos +-- XXX - TODO - moz_anno_attributes +-- XXX - TODO - moz_items_annos + +CREATE TABLE IF NOT EXISTS moz_places ( + id INTEGER PRIMARY KEY, + url LONGVARCHAR NOT NULL, + title LONGVARCHAR, + -- note - desktop has rev_host here - that's now in moz_origin. + visit_count_local INTEGER NOT NULL DEFAULT 0, + visit_count_remote INTEGER NOT NULL DEFAULT 0, + hidden INTEGER DEFAULT 0 NOT NULL, + typed INTEGER DEFAULT 0 NOT NULL, -- XXX - is 'typed' ok? Note also we want this as a *count*, not a bool. + frecency INTEGER DEFAULT -1 NOT NULL, + -- XXX - splitting last visit into local and remote correct? + last_visit_date_local INTEGER NOT NULL DEFAULT 0, + last_visit_date_remote INTEGER NOT NULL DEFAULT 0, + guid TEXT NOT NULL UNIQUE, + foreign_count INTEGER DEFAULT 0 NOT NULL, + url_hash INTEGER DEFAULT 0 NOT NULL, + description TEXT, -- XXXX - title above? + preview_image_url TEXT, + -- origin_id would ideally be NOT NULL, but we use a trigger to keep + -- it up to date, so do perform the initial insert with a null. + origin_id INTEGER, + -- a couple of sync-related fields. + sync_status TINYINT NOT NULL DEFAULT 1, -- 1 is SyncStatus::New + sync_change_counter INTEGER NOT NULL DEFAULT 0, -- adding visits will increment this + unknown_fields TEXT, + + FOREIGN KEY(origin_id) REFERENCES moz_origins(id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS url_hashindex ON moz_places(url_hash); +CREATE INDEX IF NOT EXISTS visitcountlocal ON moz_places(visit_count_local); +CREATE INDEX IF NOT EXISTS visitcountremote ON moz_places(visit_count_remote); +CREATE INDEX IF NOT EXISTS frecencyindex ON moz_places(frecency); +CREATE INDEX IF NOT EXISTS lastvisitdatelocalindex ON moz_places(last_visit_date_local); +CREATE INDEX IF NOT EXISTS lastvisitdateremoteindex ON moz_places(last_visit_date_remote); +CREATE UNIQUE INDEX IF NOT EXISTS guid_uniqueindex ON moz_places(guid); +CREATE INDEX IF NOT EXISTS originidindex ON moz_places(origin_id); + +-- partial index to help speed up the fetch_outgoing query in history.rs +CREATE INDEX IF NOT EXISTS idx_places_outgoing_by_frecency +ON moz_places(frecency DESC) +WHERE hidden = 0 AND (sync_change_counter > 0 OR sync_status != 2); -- 2 is SyncStatus::Normal + +-- partial index for get_top_frecent_site_infos +CREATE INDEX IF NOT EXISTS top_frecent_cover_idx +ON moz_places(frecency DESC, id DESC) +WHERE hidden = 0 + AND (last_visit_date_local + last_visit_date_remote) != 0 + AND (url GLOB 'http:*' OR url GLOB 'https:*'); + +CREATE TABLE IF NOT EXISTS moz_places_tombstones ( + guid TEXT PRIMARY KEY +) WITHOUT ROWID; + + +-- This table stores Place IDs with stale frecencies, along with the time they +-- were marked as stale. Maintenance and Sync periodically recalculate +-- frecencies for Place IDs in this table. +CREATE TABLE IF NOT EXISTS moz_places_stale_frecencies ( + place_id INTEGER PRIMARY KEY NOT NULL REFERENCES moz_places(id) + ON DELETE CASCADE, + stale_at INTEGER NOT NULL -- In milliseconds. +); + + +CREATE TABLE IF NOT EXISTS moz_historyvisits ( + id INTEGER PRIMARY KEY, + is_local INTEGER NOT NULL, -- XXX - not in desktop - will always be true for visits added locally, always false visits added by sync. + from_visit INTEGER, -- XXX - self-reference? + place_id INTEGER NOT NULL, + visit_date INTEGER NOT NULL, + visit_type INTEGER NOT NULL, + -- session INTEGER, -- XXX - what is 'session'? Appears unused. + unknown_fields TEXT, + + FOREIGN KEY(place_id) REFERENCES moz_places(id) ON DELETE CASCADE, + FOREIGN KEY(from_visit) REFERENCES moz_historyvisits(id) +); + +CREATE INDEX IF NOT EXISTS placedateindex ON moz_historyvisits(place_id, visit_date); +CREATE INDEX IF NOT EXISTS fromindex ON moz_historyvisits(from_visit); +CREATE INDEX IF NOT EXISTS dateindex ON moz_historyvisits(visit_date); +CREATE INDEX IF NOT EXISTS islocalindex ON moz_historyvisits(is_local); + +-- Speeds up queries frecency queries, specifically get_top_frecent_site_infos +CREATE INDEX IF NOT EXISTS idx_visits_place_type ON moz_historyvisits(place_id, visit_type); + +-- Greatly helps the multi-join query in frecency. +CREATE INDEX IF NOT EXISTS visits_from_type_idx ON moz_historyvisits(from_visit, visit_type); + +CREATE TABLE IF NOT EXISTS moz_historyvisit_tombstones ( + place_id INTEGER NOT NULL, + visit_date INTEGER NOT NULL, + FOREIGN KEY(place_id) REFERENCES moz_places(id) ON DELETE CASCADE, + PRIMARY KEY(place_id, visit_date) +); + + +CREATE TABLE IF NOT EXISTS moz_inputhistory ( + place_id INTEGER NOT NULL, + input LONGVARCHAR NOT NULL, + use_count INTEGER, + + PRIMARY KEY (place_id, input), + FOREIGN KEY(place_id) REFERENCES moz_places(id) ON DELETE CASCADE +); + + +CREATE TABLE IF NOT EXISTS moz_bookmarks ( + id INTEGER PRIMARY KEY, + fk INTEGER DEFAULT NULL, -- place_id + type INTEGER NOT NULL, + parent INTEGER, + position INTEGER NOT NULL, + title TEXT, -- a'la bug 1356159, NULL is special here - it means 'not edited' + dateAdded INTEGER NOT NULL DEFAULT 0, + lastModified INTEGER NOT NULL DEFAULT 0, + guid TEXT NOT NULL UNIQUE, + + syncStatus INTEGER NOT NULL DEFAULT 0, + syncChangeCounter INTEGER NOT NULL DEFAULT 1, + + FOREIGN KEY(fk) REFERENCES moz_places(id) ON DELETE RESTRICT +); + +-- CREATE INDEX IF NOT EXISTS itemindex ON moz_bookmarks(fk, type); +-- CREATE INDEX IF NOT EXISTS parentindex ON moz_bookmarks(parent, position); +CREATE INDEX IF NOT EXISTS itemlastmodifiedindex ON moz_bookmarks(fk, lastModified); +-- CREATE INDEX IF NOT EXISTS dateaddedindex ON moz_bookmarks(dateAdded); +CREATE UNIQUE INDEX IF NOT EXISTS guid_uniqueindex ON moz_bookmarks(guid); + + +CREATE TABLE IF NOT EXISTS moz_bookmarks_deleted ( + guid TEXT PRIMARY KEY, + dateRemoved INTEGER NOT NULL +) WITHOUT ROWID; + +-- Note: desktop has/had a 'keywords' table, but we intentionally do not. + + +CREATE TABLE IF NOT EXISTS moz_origins ( + id INTEGER PRIMARY KEY, + prefix TEXT NOT NULL, + host TEXT NOT NULL, + rev_host TEXT NOT NULL, + frecency INTEGER NOT NULL, -- XXX - why not default of -1 like in moz_places? + UNIQUE (prefix, host) +); + +CREATE INDEX IF NOT EXISTS hostindex ON moz_origins(rev_host); + + +-- This table holds key-value metadata for Places and its consumers. Sync stores +-- the sync IDs for the bookmarks and history collections in this table, and the +-- last sync time for history. +CREATE TABLE IF NOT EXISTS moz_meta ( + key TEXT PRIMARY KEY, + value NOT NULL +) WITHOUT ROWID; + +-- Support for tags. +CREATE TABLE IF NOT EXISTS moz_tags( + id INTEGER PRIMARY KEY, + tag TEXT UNIQUE NOT NULL, + lastModified INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS moz_tags_relation( + tag_id INTEGER NOT NULL REFERENCES moz_tags(id) ON DELETE CASCADE, + place_id INTEGER NOT NULL REFERENCES moz_places(id) ON DELETE CASCADE, + PRIMARY KEY(tag_id, place_id) +) WITHOUT ROWID; + +-- This table holds synced items, including tombstones. It's unused if Sync +-- isn't configured. At the end of a sync, this table's contents should match +-- both what's on the server, and the local tree in `moz_bookmarks`. +CREATE TABLE IF NOT EXISTS moz_bookmarks_synced( + id INTEGER PRIMARY KEY, + -- We intentionally don't validate GUIDs, as we allow and fix up invalid + -- ones. + guid TEXT UNIQUE NOT NULL, + -- The `parentid` from the record. + parentGuid TEXT, + -- The server modified time, in milliseconds. This is *not* a + -- ServerTimestamp, which is in fractional seconds. + serverModified INTEGER NOT NULL DEFAULT 0, + needsMerge BOOLEAN NOT NULL DEFAULT 0, + validity INTEGER NOT NULL DEFAULT 1, -- SyncValidity::Valid + isDeleted BOOLEAN NOT NULL DEFAULT 0, + kind INTEGER NOT NULL DEFAULT -1, + -- The creation date, in milliseconds. + dateAdded INTEGER NOT NULL DEFAULT 0, + title TEXT, + placeId INTEGER REFERENCES moz_places(id) + ON DELETE SET NULL, + keyword TEXT, + description TEXT, + loadInSidebar BOOLEAN, + smartBookmarkName TEXT, + feedURL TEXT, + siteURL TEXT, + -- All unknown fields from the server record, encoded as a JSON object. + unknownFields TEXT +); + +CREATE INDEX IF NOT EXISTS moz_bookmarks_synced_urls ON moz_bookmarks_synced(placeId); +CREATE INDEX IF NOT EXISTS moz_bookmarks_synced_keywords ON moz_bookmarks_synced(keyword) + WHERE keyword NOT NULL; + +-- This table holds parent-child relationships and positions for synced items, +-- from each folder's `children`. Unlike `moz_bookmarks`, this is stored +-- separately because we might see an incoming folder before its children. This +-- also lets us catch disagreements between a folder's `children` and its +-- childrens' `parentid`. +CREATE TABLE IF NOT EXISTS moz_bookmarks_synced_structure( + guid TEXT, + parentGuid TEXT REFERENCES moz_bookmarks_synced(guid) + ON DELETE CASCADE, + position INTEGER NOT NULL, + PRIMARY KEY(parentGuid, guid) +) WITHOUT ROWID; + +-- This table holds tags for synced items. +CREATE TABLE IF NOT EXISTS moz_bookmarks_synced_tag_relation( + itemId INTEGER NOT NULL REFERENCES moz_bookmarks_synced(id) + ON DELETE CASCADE, + tagId INTEGER NOT NULL REFERENCES moz_tags(id) + ON DELETE CASCADE, + PRIMARY KEY(itemId, tagId) +) WITHOUT ROWID; + +-- This table holds search keywords for URLs. Desktop would like to replace +-- these with custom search engines eventually (bug 648398); however, we +-- must still round-trip keywords imported via Sync or migrated from Fennec. +-- Since none of the `moz_bookmarks_synced_*` tables are durable, we store +-- keywords for URLs in a separate table. Unlike Desktop, we don't support +-- custom POST data, since we don't sync it (bug 1345417), and Fennec +-- doesn't write it. +CREATE TABLE IF NOT EXISTS moz_keywords( + place_id INTEGER PRIMARY KEY REFERENCES moz_places(id) + ON DELETE RESTRICT, + keyword TEXT NOT NULL UNIQUE +); + +---------------------------------------------------------------------- +--------------------History Metadata---------------------------------- +---------------------------------------------------------------------- + +-- These tables store metadata information related to moz_places. +-- None of this data is synced for now. +CREATE TABLE IF NOT EXISTS moz_places_metadata ( + id INTEGER PRIMARY KEY, + created_at INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL DEFAULT 0, + + place_id INTEGER NOT NULL, + + total_view_time INTEGER NOT NULL DEFAULT 0, -- a rolling aggregate + search_query_id INTEGER, + referrer_place_id INTEGER, + document_type INTEGER NOT NULL DEFAULT 0, -- 0=generic, 1=media + typing_time INTEGER NOT NULL DEFAULT 0, + key_presses INTEGER NOT NULL DEFAULT 0, + + FOREIGN KEY(place_id) REFERENCES moz_places(id) ON DELETE CASCADE, + FOREIGN KEY(search_query_id) REFERENCES moz_places_metadata_search_queries(id) ON DELETE CASCADE, + FOREIGN KEY(referrer_place_id) REFERENCES moz_places(id) ON DELETE CASCADE + + CHECK(place_id != referrer_place_id) +); + +CREATE TABLE IF NOT EXISTS moz_places_metadata_search_queries ( + id INTEGER PRIMARY KEY, + term TEXT NOT NULL UNIQUE +); diff --git a/components/places/src/db/schema.rs b/components/places/src/db/schema.rs index 4218c11db8f..8805965be43 100644 --- a/components/places/src/db/schema.rs +++ b/components/places/src/db/schema.rs @@ -38,6 +38,11 @@ lazy_static::lazy_static! { }; } +// Historical versions of the shared schema for migrations which apply it as +// part of the migration operations. +const CREATE_SHARED_SCHEMA_V20_SQL: &str = + include_str!("../../sql/legacy/create_shared_schema_v20.sql"); + // Keys in the moz_meta table. pub(crate) static MOZ_META_KEY_ORIGIN_FRECENCY_COUNT: &str = "origin_frecency_count"; pub(crate) static MOZ_META_KEY_ORIGIN_FRECENCY_SUM: &str = "origin_frecency_sum"; @@ -145,7 +150,7 @@ pub fn upgrade_from(db: &Connection, from: u32) -> rusqlite::Result<()> { // Old-style migrations - migration(db, from, 2, &[CREATE_SHARED_SCHEMA_SQL], || Ok(()))?; + migration(db, from, 2, &[CREATE_SHARED_SCHEMA_V20_SQL], || Ok(()))?; migration( db, from, @@ -153,13 +158,13 @@ pub fn upgrade_from(db: &Connection, from: u32) -> rusqlite::Result<()> { &[ // Previous versions had an incomplete version of moz_bookmarks. "DROP TABLE moz_bookmarks", - CREATE_SHARED_SCHEMA_SQL, + CREATE_SHARED_SCHEMA_V20_SQL, ], || create_bookmark_roots(db.conn()), )?; - migration(db, from, 4, &[CREATE_SHARED_SCHEMA_SQL], || Ok(()))?; - migration(db, from, 5, &[CREATE_SHARED_SCHEMA_SQL], || Ok(()))?; // new tags tables. - migration(db, from, 6, &[CREATE_SHARED_SCHEMA_SQL], || Ok(()))?; // bookmark syncing. + migration(db, from, 4, &[CREATE_SHARED_SCHEMA_V20_SQL], || Ok(()))?; + migration(db, from, 5, &[CREATE_SHARED_SCHEMA_V20_SQL], || Ok(()))?; // new tags tables. + migration(db, from, 6, &[CREATE_SHARED_SCHEMA_V20_SQL], || Ok(()))?; // bookmark syncing. migration( db, from, @@ -170,7 +175,7 @@ pub fn upgrade_from(db: &Connection, from: u32) -> rusqlite::Result<()> { &format!("DELETE FROM moz_meta WHERE key = '{}'", LAST_SYNC_META_KEY), "DROP TABLE moz_bookmarks_synced", "DROP TABLE moz_bookmarks_synced_structure", - CREATE_SHARED_SCHEMA_SQL, + CREATE_SHARED_SCHEMA_V20_SQL, ], || Ok(()), )?; @@ -252,7 +257,7 @@ pub fn upgrade_from(db: &Connection, from: u32) -> rusqlite::Result<()> { ], || Ok(()), )?; - migration(db, from, 13, &[CREATE_SHARED_SCHEMA_SQL], || Ok(()))?; // moz_places_metadata. + migration(db, from, 13, &[CREATE_SHARED_SCHEMA_V20_SQL], || Ok(()))?; // moz_places_metadata. migration( db, from, @@ -260,7 +265,7 @@ pub fn upgrade_from(db: &Connection, from: u32) -> rusqlite::Result<()> { &[ // Changing `moz_places_metadata` structure, drop and recreate it. "DROP TABLE moz_places_metadata", - CREATE_SHARED_SCHEMA_SQL, + CREATE_SHARED_SCHEMA_V20_SQL, ], || Ok(()), )?; @@ -329,14 +334,14 @@ pub fn upgrade_from(db: &Connection, from: u32) -> rusqlite::Result<()> { 18 => { // Create the new indexes by just calling the shared schema file // idx_places_outgoing_by_frecency - db.execute_batch(CREATE_SHARED_SCHEMA_SQL)?; + db.execute_batch(CREATE_SHARED_SCHEMA_V20_SQL)?; // Manually call analyze so the planner can start using the indexes immediately db.execute("ANALYZE moz_places", [])?; } 19 => { // Create the new indexes by just calling the shared schema file // top_frecent_cover_idx, idx_visits_place_type - db.execute_batch(CREATE_SHARED_SCHEMA_SQL)?; + db.execute_batch(CREATE_SHARED_SCHEMA_V20_SQL)?; // Manually call analyze so the planner can start using the indexes immediately db.execute("ANALYZE moz_places", [])?; db.execute("ANALYZE moz_historyvisits", [])?;