diff --git a/duckdb/src/catalog/duckdb_catalog.cpp b/duckdb/src/catalog/duckdb_catalog.cpp index d7a09bae..6d6b8db2 100644 --- a/duckdb/src/catalog/duckdb_catalog.cpp +++ b/duckdb/src/catalog/duckdb_catalog.cpp @@ -1,10 +1,14 @@ #include "catalog/duckdb_catalog.h" +#include + #include "binder/bound_attach_info.h" #include "catalog/catalog_entry/node_table_catalog_entry.h" +#include "catalog/catalog_entry/rel_group_catalog_entry.h" #include "catalog/duckdb_table_catalog_entry.h" #include "common/exception/binder.h" #include "common/exception/runtime.h" +#include "common/string_utils.h" #include "connector/duckdb_type_converter.h" #include "function/duckdb_scan.h" #include "storage/buffer_manager/memory_manager.h" @@ -55,7 +59,13 @@ void DuckDBCatalog::init() { conversionFunc(resultChunk->data[0], tableNamesVector, resultChunk->size()); for (auto i = 0u; i < resultChunk->size(); i++) { auto tableName = tableNamesVector.getValue(i).getAsString(); - createForeignTable(tableName); + auto lowerName = tableName; + common::StringUtils::toLower(lowerName); + if (lowerName.rfind("rel_", 0) == 0) { + createForeignRelTable(tableName); + } else { + createForeignTable(tableName); + } } } @@ -118,6 +128,111 @@ void DuckDBCatalog::createForeignTable(const std::string& tableName) { lbug::storage::StorageManager::Get(*context_)->createTable(mainEntry); } +void DuckDBCatalog::createForeignRelTable(const std::string& tableName) { + // Query foreign key info to find src/dst node tables + auto fkQuery = std::format("SELECT kcu.column_name, ccu.table_name " + "FROM information_schema.table_constraints tc " + "JOIN information_schema.key_column_usage kcu " + " ON tc.constraint_name = kcu.constraint_name " + " AND tc.table_schema = kcu.table_schema " + "JOIN information_schema.constraint_column_usage ccu " + " ON ccu.constraint_name = tc.constraint_name " + " AND ccu.table_schema = tc.table_schema " + "WHERE tc.constraint_type = 'FOREIGN KEY' " + " AND tc.table_name = '{}'", + tableName); + auto fkResult = connector.executeQuery(fkQuery); + + std::string srcTableName, dstTableName; + for (auto i = 0u; i < fkResult->RowCount(); i++) { + auto colName = fkResult->GetValue(0, i).GetValue(); + auto refTable = fkResult->GetValue(1, i).GetValue(); + auto lowerCol = colName; + common::StringUtils::toLower(lowerCol); + if (lowerCol == "src_id" || lowerCol.find("src") == 0) { + srcTableName = refTable; + } else if (lowerCol == "dst_id" || lowerCol.find("dst") == 0 || + lowerCol.find("dest") == 0) { + dstTableName = refTable; + } + } + + if (srcTableName.empty() || dstTableName.empty()) { + createForeignTable(tableName); + return; + } + + // Build property definitions + std::vector propertyDefinitions; + bindPropertyDefinitions(tableName, propertyDefinitions); + + // Determine the node table IDs from the main catalog + auto* catalog = context_->getDatabase()->getCatalog(); + auto* srcEntry = catalog->getTableCatalogEntry(&transaction::DUMMY_TRANSACTION, srcTableName); + auto* dstEntry = catalog->getTableCatalogEntry(&transaction::DUMMY_TRANSACTION, dstTableName); + if (srcEntry == nullptr || dstEntry == nullptr) { + createForeignTable(tableName); + return; + } + + common::table_id_t srcTableID = srcEntry->getTableID(); + common::table_id_t dstTableID = dstEntry->getTableID(); + + // Build query and scan info + std::vector columnTypes; + std::vector columnNames; + for (auto& def : propertyDefinitions) { + columnNames.push_back(def.getName()); + columnTypes.push_back(def.getType().copy()); + } + + auto queryStr = + std::format("SELECT * FROM \"{}\".{}.{}", catalogName, defaultSchemaName, tableName); + auto duckdbTableInfo = std::make_shared(queryStr, std::move(columnTypes), + columnNames, connector); + auto scanFunc = getScanFunction(duckdbTableInfo); + + // Create DuckDB table catalog entry + auto tableEntry = + std::make_unique(tableName, scanFunc, duckdbTableInfo); + for (auto& def : propertyDefinitions) { + tableEntry->addProperty(def); + } + tables->createEntry(&transaction::DUMMY_TRANSACTION, std::move(tableEntry)); + + // Create bind data for the scan function + binder::expression_vector emptyColumns; + auto bindData = + std::make_shared(queryStr, columnNames, connector, emptyColumns); + + // Create RelGroupCatalogEntry + auto foreignDatabaseName = std::format("{}.{}", catalogName, tableName); + + std::vector relTableInfos; + auto info = bindCreateTableInfo(tableName); + common::oid_t relOID = tables->getNextOID(); + relTableInfos.emplace_back(catalog::NodeTableIDPair{srcTableID, dstTableID}, relOID, + common::RelMultiplicity::MANY, common::RelMultiplicity::MANY); + + auto relGroupEntry = + std::make_unique(tableName, common::RelMultiplicity::MANY, + common::RelMultiplicity::MANY, common::ExtendDirection::BOTH, std::move(relTableInfos), + "", // storage + common::StorageFormat::NONE, scanFunc, bindData, std::move(foreignDatabaseName)); + + for (auto& def : propertyDefinitions) { + relGroupEntry->addProperty(def); + } + + context_->getDatabase()->getCatalog()->addTableEntry(std::move(relGroupEntry)); + + auto mainEntry = context_->getDatabase()->getCatalog()->getTableCatalogEntry( + &transaction::DUMMY_TRANSACTION, tableName); + if (mainEntry) { + storage::StorageManager::Get(*context_)->createTable(mainEntry); + } +} + static bool getTableInfo(const DuckDBConnector& connector, const std::string& tableName, const std::string& schemaName, const std::string& catalogName, std::vector& columnTypes, std::vector& columnNames, diff --git a/duckdb/src/include/catalog/duckdb_catalog.h b/duckdb/src/include/catalog/duckdb_catalog.h index 2cdc49d5..6161fe89 100644 --- a/duckdb/src/include/catalog/duckdb_catalog.h +++ b/duckdb/src/include/catalog/duckdb_catalog.h @@ -50,6 +50,7 @@ class DuckDBCatalog : public extension::CatalogExtension { private: void createForeignTable(const std::string& tableName); + void createForeignRelTable(const std::string& tableName); protected: std::string dbPath; diff --git a/pg_client/src/catalog/pg_client_catalog.cpp b/pg_client/src/catalog/pg_client_catalog.cpp index c11fb749..86290251 100644 --- a/pg_client/src/catalog/pg_client_catalog.cpp +++ b/pg_client/src/catalog/pg_client_catalog.cpp @@ -71,11 +71,11 @@ void PgClientCatalog::init() { std::string tableName = row.cells[0].value; auto lowerName = tableName; common::StringUtils::toLower(lowerName); - if (lowerName.rfind("fkrel_", 0) == 0) { + if (lowerName.rfind("rel_", 0) == 0) { // Foreign-key-based rel table: scan-driven, optimizer generates a join. // No CSR columns; backed by a ForeignRelTable. createForeignRelTable(tableName); - } else if (lowerName.rfind("rel_", 0) == 0) { + } else if (lowerName.rfind("csr_rel_", 0) == 0) { // CSR-based rel table: materialized into a local on-disk CSR rel table. // TODO: COPY data from PostgreSQL into a local RelTable. createForeignRelTable(tableName); @@ -274,7 +274,7 @@ void PgClientCatalog::createForeignRelTable(const std::string& tableName) { auto columnTypes = getColumnTypes(columnInfo); // Look up src/dst node tables in the attached catalog (the foreign PgClientTableCatalogEntry - // entries), not the main catalog shadows. fkrel_ tables join against the foreign node + // entries), not the main catalog shadows. rel_ tables join against the foreign node // entries directly, so the rel's src/dst table IDs must match the entries that // `testdb.node_person` (and bare `node_person` via shadow) resolve to. auto* srcEntry = tables->getEntry(&transaction::DUMMY_TRANSACTION, srcTableName); @@ -316,7 +316,7 @@ void PgClientCatalog::createForeignRelTable(const std::string& tableName) { // and ForeignRelTable now owns the shared state and serializes offset // advancement with its own mutex, matching the morsel-driven model. // The rel group's foreignDatabaseName must be the lbug attached-database - // name -- NOT the schema-qualified PG name ("public.fkrel_knows") -- because + // name -- NOT the schema-qualified PG name ("public.rel_knows") -- because // the join-push-down optimizer uses it as a lookup key into // DatabaseManager::getAttachedDatabase(). auto foreignDatabaseName = attachedDbName; diff --git a/pg_client/src/include/storage/attached_pg_client_database.h b/pg_client/src/include/storage/attached_pg_client_database.h index 2d394c2c..176ffcf0 100644 --- a/pg_client/src/include/storage/attached_pg_client_database.h +++ b/pg_client/src/include/storage/attached_pg_client_database.h @@ -22,7 +22,7 @@ class AttachedPgClientDatabase final : public main::AttachedDatabase { std::vector getTableColumnNames(const std::string& tableName) const override { // The foreign join push-down optimizer expects the first two columns // to be the src and dst FK columns respectively. For FK-based rel - // tables (fkrel_*), the FK columns may not be the first two columns + // tables (rel_*), the FK columns may not be the first two columns // in ordinal_position order (the PK often comes first). // // To satisfy this contract, query FK constraints and put any detected diff --git a/pg_client/test/test_files/create_pg_client_test_db.sql b/pg_client/test/test_files/create_pg_client_test_db.sql index f269f857..62d14d17 100644 --- a/pg_client/test/test_files/create_pg_client_test_db.sql +++ b/pg_client/test/test_files/create_pg_client_test_db.sql @@ -30,10 +30,10 @@ CREATE TABLE public.node_person ( ALTER TABLE public.node_person OWNER TO ci; -- --- Table: fkrel_knows +-- Table: rel_knows -- -CREATE TABLE public.fkrel_knows ( +CREATE TABLE public.rel_knows ( id integer NOT NULL, from_id integer NOT NULL REFERENCES public.node_person(id), to_id integer NOT NULL REFERENCES public.node_person(id), @@ -41,7 +41,7 @@ CREATE TABLE public.fkrel_knows ( ); -ALTER TABLE public.fkrel_knows OWNER TO ci; +ALTER TABLE public.rel_knows OWNER TO ci; -- -- Data for node_person @@ -57,14 +57,14 @@ INSERT INTO public.node_person (id, name, age) VALUES SELECT pg_catalog.setval(pg_catalog.pg_get_serial_sequence('public.node_person', 'id'), 5, true); -- --- Data for fkrel_knows +-- Data for rel_knows -- -INSERT INTO public.fkrel_knows (id, from_id, to_id, since) VALUES +INSERT INTO public.rel_knows (id, from_id, to_id, since) VALUES (1, 1, 2, '2020-01-15'), (2, 1, 3, '2021-03-20'), (3, 2, 4, '2022-06-10'), (4, 3, 4, '2023-08-05'), (5, 4, 5, '2023-12-01'); -SELECT pg_catalog.setval(pg_catalog.pg_get_serial_sequence('public.fkrel_knows', 'id'), 5, true); +SELECT pg_catalog.setval(pg_catalog.pg_get_serial_sequence('public.rel_knows', 'id'), 5, true); diff --git a/pg_client/test/test_files/pg_client.test b/pg_client/test/test_files/pg_client.test index 343b9900..401deea8 100644 --- a/pg_client/test/test_files/pg_client.test +++ b/pg_client/test/test_files/pg_client.test @@ -44,13 +44,13 @@ Attached database successfully. Charlie|35 Eve|32 --CASE ScanFkRelTable +-CASE ScanRelTable -SKIP_FSM_LEAK_CHECK -LOAD_DYNAMIC_EXTENSION pg_client -STATEMENT ATTACH '${PG_CLIENT_CONNECTION_STRING}' as testdb (dbtype PG_CLIENT); ---- 1 Attached database successfully. --STATEMENT LOAD FROM testdb.fkrel_knows RETURN from_id, to_id, since ORDER BY id; +-STATEMENT LOAD FROM testdb.rel_knows RETURN from_id, to_id, since ORDER BY id; ---- 5 1|2|2020-01-15 1|3|2021-03-20 @@ -72,14 +72,14 @@ Alice|30 Eve|32 Charlie|35 --CASE MatchOnFkRelTable +-CASE MatchOnRelTable -SKIP -SKIP_FSM_LEAK_CHECK -LOAD_DYNAMIC_EXTENSION pg_client -STATEMENT ATTACH '${PG_CLIENT_CONNECTION_STRING}' as testdb (dbtype PG_CLIENT); ---- 1 Attached database successfully. --STATEMENT MATCH (a:testdb.node_person)-[k:fkrel_knows]->(b:testdb.node_person) RETURN count(*); +-STATEMENT MATCH (a:testdb.node_person)-[k:rel_knows]->(b:testdb.node_person) RETURN count(*); ---- 1 5 @@ -93,5 +93,5 @@ Attached database successfully. ---- 4 0|node_person|ATTACHED|testdb(PG_CLIENT)| 0|node_person|NODE|shadow(graph)| -1|fkrel_knows|ATTACHED|testdb(PG_CLIENT)| -1|fkrel_knows|REL|testdb| +1|rel_knows|ATTACHED|testdb(PG_CLIENT)| +1|rel_knows|REL|testdb| diff --git a/pg_client/test/test_pg_client.py b/pg_client/test/test_pg_client.py index 75830bad..c16bc769 100644 --- a/pg_client/test/test_pg_client.py +++ b/pg_client/test/test_pg_client.py @@ -89,10 +89,10 @@ def setUpClass(cls): ('Initech', 500000.00) """)) - # Create fkrel_knows table with FK columns named src_id/dst_id + # Create rel_knows table with FK columns named src_id/dst_id # Prefix rel_ + FK constraints → auto-register as relationship table conn.execute(sa.text(""" - CREATE TABLE fkrel_knows ( + CREATE TABLE rel_knows ( id SERIAL PRIMARY KEY, src_id INTEGER NOT NULL, dst_id INTEGER NOT NULL, @@ -102,13 +102,13 @@ def setUpClass(cls): # Add actual FOREIGN KEY constraints so the FK query detects src/dst tables conn.execute(sa.text(""" - ALTER TABLE fkrel_knows + ALTER TABLE rel_knows ADD CONSTRAINT fk_src FOREIGN KEY (src_id) REFERENCES node_person(id), ADD CONSTRAINT fk_dst FOREIGN KEY (dst_id) REFERENCES node_person(id) """)) conn.execute(sa.text(""" - INSERT INTO fkrel_knows (src_id, dst_id, since) VALUES + INSERT INTO rel_knows (src_id, dst_id, since) VALUES (1, 2, '2020-01-15'), (1, 3, '2021-03-20'), (2, 4, '2022-06-10'), @@ -116,9 +116,9 @@ def setUpClass(cls): (4, 5, '2023-12-01') """)) - # Create fkrel_works_at table with FK constraints + # Create rel_works_at table with FK constraints conn.execute(sa.text(""" - CREATE TABLE fkrel_works_at ( + CREATE TABLE rel_works_at ( id SERIAL PRIMARY KEY, src_id INTEGER NOT NULL REFERENCES node_person(id), dst_id INTEGER NOT NULL REFERENCES node_company(id) @@ -126,7 +126,7 @@ def setUpClass(cls): """)) conn.execute(sa.text(""" - INSERT INTO fkrel_works_at (src_id, dst_id) VALUES + INSERT INTO rel_works_at (src_id, dst_id) VALUES (1, 1), (2, 2), (3, 1), @@ -199,7 +199,7 @@ def test_06_load_rel_table(self): script = f""" LOAD EXTENSION '{PG_CLIENT_EXT}'; ATTACH '{self.conn_str}' AS testdb (DBTYPE PG_CLIENT); - LOAD FROM testdb.fkrel_knows RETURN src_id, dst_id, since ORDER BY id; + LOAD FROM testdb.rel_knows RETURN src_id, dst_id, since ORDER BY id; """ stdout, stderr = run_lbug(script) self.assertIn("2020-01-15", stdout, f"Rel load failed: {stderr}") @@ -226,7 +226,7 @@ def test_06b_match_rel_table(self): script = f""" LOAD EXTENSION '{PG_CLIENT_EXT}'; ATTACH '{self.conn_str}' AS testdb (DBTYPE PG_CLIENT); - MATCH (a:node_person)-[k:fkrel_knows]->(b:node_person) + MATCH (a:node_person)-[k:rel_knows]->(b:node_person) RETURN count(*); """ stdout, stderr = run_lbug(script) @@ -244,7 +244,7 @@ def test_06c_match_rel_count_parallel(self): script = f""" LOAD EXTENSION '{PG_CLIENT_EXT}'; ATTACH '{self.conn_str}' AS testdb (DBTYPE PG_CLIENT); - MATCH (a:node_person)-[k:fkrel_knows]->(b:node_person) + MATCH (a:node_person)-[k:rel_knows]->(b:node_person) RETURN count(*); """ stdout, stderr = run_lbug(script) @@ -255,7 +255,7 @@ def test_07_scan_rel_table_filter(self): script = f""" LOAD EXTENSION '{PG_CLIENT_EXT}'; ATTACH '{self.conn_str}' AS testdb (DBTYPE PG_CLIENT); - LOAD FROM testdb.fkrel_knows WHERE src_id = 1 RETURN dst_id, since ORDER BY dst_id; + LOAD FROM testdb.rel_knows WHERE src_id = 1 RETURN dst_id, since ORDER BY dst_id; """ stdout, stderr = run_lbug(script) self.assertIn("2020-01-15", stdout, f"Rel filter failed: {stderr}") @@ -270,7 +270,7 @@ def test_09_show_tables(self): """ stdout, stderr = run_lbug(script) self.assertIn("node_person", stdout, f"SHOW_TABLES missing node_person: {stderr}") - self.assertIn("fkrel_knows", stdout, f"SHOW_TABLES missing fkrel_knows: {stderr}") + self.assertIn("rel_knows", stdout, f"SHOW_TABLES missing rel_knows: {stderr}") def test_10_table_info(self): """Test TABLE_INFO on attached table."""