From a8e5f53190d43ffac648c648c275e5d24d04e3a8 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Thu, 2 Jul 2026 10:12:23 +0800 Subject: [PATCH 01/29] catalog: add pg_foreign_catalog, pg_foreign_volume, pg_lake_table Introduce the three system catalogs backing Iceberg lake-table DDL: - pg_foreign_catalog: named foreign catalog bound to a foreign server - pg_foreign_volume: named foreign volume bound to a foreign server - pg_lake_table: per-relation lake-table metadata (type, catalog, volume, options) Register the headers in the catalog Makefile and bump CATALOG_VERSION_NO. No code references the catalogs yet; DDL/commands land in later commits. OIDs 8549-8558 / 9901-9902 verified free via unused_oids; duplicate_oids clean. --- src/backend/catalog/Makefile | 1 + src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_foreign_catalog.h | 55 ++++++++++++++++++++ src/include/catalog/pg_foreign_volume.h | 55 ++++++++++++++++++++ src/include/catalog/pg_lake_table.h | 66 ++++++++++++++++++++++++ 5 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 src/include/catalog/pg_foreign_catalog.h create mode 100644 src/include/catalog/pg_foreign_volume.h create mode 100644 src/include/catalog/pg_lake_table.h diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 5679abf152a..44598e25cff 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -102,6 +102,7 @@ CATALOG_HEADERS := \ pg_subscription_rel.h gp_partition_template.h pg_task.h pg_task_run_history.h \ pg_profile.h pg_password_history.h pg_directory_table.h gp_storage_server.h \ gp_storage_user_mapping.h pg_tag.h pg_tag_description.h \ + pg_foreign_catalog.h pg_foreign_volume.h pg_lake_table.h \ gp_matview_aux.h \ gp_matview_tables.h diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 3de2e549f4c..3d5e3915585 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -60,6 +60,6 @@ */ /* 3yyymmddN */ -#define CATALOG_VERSION_NO 302512051 +#define CATALOG_VERSION_NO 302607021 #endif diff --git a/src/include/catalog/pg_foreign_catalog.h b/src/include/catalog/pg_foreign_catalog.h new file mode 100644 index 00000000000..7623cd9e807 --- /dev/null +++ b/src/include/catalog/pg_foreign_catalog.h @@ -0,0 +1,55 @@ +/*------------------------------------------------------------------------- + * + * pg_foreign_catalog.h + * definition of the "foreign catalog" system catalog (pg_foreign_catalog) + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/catalog/pg_foreign_catalog.h + * + * NOTES + * The Catalog.pm module reads this file and derives schema + * information. + * + *------------------------------------------------------------------------- + */ +#ifndef PG_FOREIGN_CATALOG_H +#define PG_FOREIGN_CATALOG_H + +#include "catalog/genbki.h" +#include "catalog/pg_foreign_catalog_d.h" + +/* ---------------- + * pg_foreign_catalog definition. cpp turns this into + * typedef struct FormData_pg_foreign_catalog + * ---------------- + */ +CATALOG(pg_foreign_catalog,8549,ForeignCatalogRelationId) +{ + Oid oid; /* oid */ + + NameData fcname; /* foreign catalog name */ + + Oid fcowner BKI_LOOKUP(pg_authid); /* owner of the foreign catalog */ + + Oid fcserver BKI_LOOKUP(pg_foreign_server); /* foreign server this catalog belongs to */ + +#ifdef CATALOG_VARLEN /* variable-length fields start here */ + text fcoptions[1]; /* foreign catalog options */ +#endif +} FormData_pg_foreign_catalog; + +/* ---------------- + * Form_pg_foreign_catalog corresponds to a pointer to a tuple with + * the format of pg_foreign_catalog relation. + * ---------------- + */ +typedef FormData_pg_foreign_catalog *Form_pg_foreign_catalog; + +DECLARE_TOAST(pg_foreign_catalog, 8550, 8551); + +DECLARE_UNIQUE_INDEX_PKEY(pg_foreign_catalog_oid_index, 8552, ForeignCatalogOidIndexId, on pg_foreign_catalog using btree(oid oid_ops)); +DECLARE_UNIQUE_INDEX(pg_foreign_catalog_name_index, 8553, ForeignCatalogNameIndexId, on pg_foreign_catalog using btree(fcname name_ops)); + +#endif /* PG_FOREIGN_CATALOG_H */ diff --git a/src/include/catalog/pg_foreign_volume.h b/src/include/catalog/pg_foreign_volume.h new file mode 100644 index 00000000000..669bb1246d0 --- /dev/null +++ b/src/include/catalog/pg_foreign_volume.h @@ -0,0 +1,55 @@ +/*------------------------------------------------------------------------- + * + * pg_foreign_volume.h + * definition of the "foreign volume" system catalog (pg_foreign_volume) + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/catalog/pg_foreign_volume.h + * + * NOTES + * The Catalog.pm module reads this file and derives schema + * information. + * + *------------------------------------------------------------------------- + */ +#ifndef PG_FOREIGN_VOLUME_H +#define PG_FOREIGN_VOLUME_H + +#include "catalog/genbki.h" +#include "catalog/pg_foreign_volume_d.h" + +/* ---------------- + * pg_foreign_volume definition. cpp turns this into + * typedef struct FormData_pg_foreign_volume + * ---------------- + */ +CATALOG(pg_foreign_volume,8554,ForeignVolumeRelationId) +{ + Oid oid; /* oid */ + + NameData fvname; /* foreign volume name */ + + Oid fvowner BKI_LOOKUP(pg_authid); /* owner of the foreign volume */ + + Oid fvserver BKI_LOOKUP(pg_foreign_server); /* foreign server this volume belongs to */ + +#ifdef CATALOG_VARLEN /* variable-length fields start here */ + text fvoptions[1]; /* foreign volume options */ +#endif +} FormData_pg_foreign_volume; + +/* ---------------- + * Form_pg_foreign_volume corresponds to a pointer to a tuple with + * the format of pg_foreign_volume relation. + * ---------------- + */ +typedef FormData_pg_foreign_volume *Form_pg_foreign_volume; + +DECLARE_TOAST(pg_foreign_volume, 8555, 8556); + +DECLARE_UNIQUE_INDEX_PKEY(pg_foreign_volume_oid_index, 8557, ForeignVolumeOidIndexId, on pg_foreign_volume using btree(oid oid_ops)); +DECLARE_UNIQUE_INDEX(pg_foreign_volume_name_index, 8558, ForeignVolumeNameIndexId, on pg_foreign_volume using btree(fvname name_ops)); + +#endif /* PG_FOREIGN_VOLUME_H */ diff --git a/src/include/catalog/pg_lake_table.h b/src/include/catalog/pg_lake_table.h new file mode 100644 index 00000000000..37c8a208d31 --- /dev/null +++ b/src/include/catalog/pg_lake_table.h @@ -0,0 +1,66 @@ +/*------------------------------------------------------------------------- + * + * pg_lake_table.h + * definition of the "lake table" system catalog (pg_lake_table) + * + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/catalog/pg_lake_table.h + * + * NOTES + * The Catalog.pm module reads this file and derives schema + * information. + * + *------------------------------------------------------------------------- + */ +#ifndef PG_LAKE_TABLE_H +#define PG_LAKE_TABLE_H + +#include "catalog/genbki.h" +#include "catalog/pg_lake_table_d.h" +#include "nodes/pg_list.h" + +/* ---------------- + * pg_lake_table definition. cpp turns this into + * typedef struct FormData_pg_lake_table + * ---------------- + */ +CATALOG(pg_lake_table,9901,LakeTableRelationId) +{ + Oid ltrelid BKI_LOOKUP(pg_class); /* OID of the lake table relation */ + Oid ltforeign_catalog BKI_LOOKUP_OPT(pg_foreign_catalog); /* OID of foreign catalog */ + Oid ltforeign_volume BKI_LOOKUP_OPT(pg_foreign_volume); /* OID of foreign volume */ + +#ifdef CATALOG_VARLEN /* variable-length fields start here */ + text lttable_type; /* table type: ICEBERG, etc. */ + text ltoptions[1]; /* lake table options */ +#endif +} FormData_pg_lake_table; + +/* ---------------- + * Form_pg_lake_table corresponds to a pointer to a tuple with + * the format of pg_lake_table relation. + * ---------------- + */ +typedef FormData_pg_lake_table *Form_pg_lake_table; + +DECLARE_TOAST(pg_lake_table, 9903, 9904); + +DECLARE_UNIQUE_INDEX_PKEY(pg_lake_table_relid_index, 9902, LakeTableRelidIndexId, on pg_lake_table using btree(ltrelid oid_ops)); + +/* ---------------- + * Lake table structure for caching + * ---------------- + */ +typedef struct LakeTable +{ + Oid relid; /* OID of the lake table relation */ + char *table_type; /* table type: ICEBERG, etc. */ + char *foreign_catalog; /* foreign catalog name */ + char *foreign_volume; /* foreign volume name */ + List *options; /* lake table options */ +} LakeTable; + +#endif /* PG_LAKE_TABLE_H */ From c2bb7c0e6cc6bb5d37755d2e875403cf7b50581f Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Thu, 2 Jul 2026 10:50:06 +0800 Subject: [PATCH 02/29] nodes: add parse nodes for Iceberg lake-table DDL Add three statement parse nodes and their hand-maintained node-support plumbing (copy/equal/out/read + fast serialization), mirroring the existing CreateDirectoryTableStmt and CreateForeignServerStmt patterns: - CreateLakeTableStmt (CREATE ICEBERG TABLE ...; embeds CreateStmt) - CreateForeignCatalogStmt (CREATE FOREIGN CATALOG ...) - CreateForeignVolumeStmt (CREATE FOREIGN VOLUME ...) Nodes are not yet produced by the grammar or dispatched; grammar and command handling land in later commits. ObjectType additions are deferred to the command commit to keep exhaustive switches complete. --- src/backend/nodes/copyfuncs.funcs.c | 65 ++++++++++++++++++++++++++++ src/backend/nodes/copyfuncs.switch.c | 9 ++++ src/backend/nodes/equalfuncs.c | 46 ++++++++++++++++++++ src/backend/nodes/outfast.c | 31 +++++++++++++ src/backend/nodes/outfuncs.c | 15 +++++++ src/backend/nodes/readfast.c | 50 +++++++++++++++++++++ src/include/nodes/nodes.h | 3 ++ src/include/nodes/parsenodes.h | 27 ++++++++++++ 8 files changed, 246 insertions(+) diff --git a/src/backend/nodes/copyfuncs.funcs.c b/src/backend/nodes/copyfuncs.funcs.c index 6f312d17574..2b7cd7cfe3e 100644 --- a/src/backend/nodes/copyfuncs.funcs.c +++ b/src/backend/nodes/copyfuncs.funcs.c @@ -2748,6 +2748,32 @@ _copyCreateForeignServerStmt(const CreateForeignServerStmt *from) return newnode; } +static CreateForeignCatalogStmt * +_copyCreateForeignCatalogStmt(const CreateForeignCatalogStmt *from) +{ + CreateForeignCatalogStmt *newnode = makeNode(CreateForeignCatalogStmt); + + COPY_STRING_FIELD(catalogname); + COPY_STRING_FIELD(servername); + COPY_SCALAR_FIELD(if_not_exists); + COPY_NODE_FIELD(options); + + return newnode; +} + +static CreateForeignVolumeStmt * +_copyCreateForeignVolumeStmt(const CreateForeignVolumeStmt *from) +{ + CreateForeignVolumeStmt *newnode = makeNode(CreateForeignVolumeStmt); + + COPY_STRING_FIELD(volumename); + COPY_STRING_FIELD(servername); + COPY_SCALAR_FIELD(if_not_exists); + COPY_NODE_FIELD(options); + + return newnode; +} + static AlterForeignServerStmt * _copyAlterForeignServerStmt(const AlterForeignServerStmt *from) { @@ -3358,6 +3384,45 @@ _copyCreateDirectoryTableStmt(const CreateDirectoryTableStmt *from) return newnode; } +static CreateLakeTableStmt * +_copyCreateLakeTableStmt(const CreateLakeTableStmt *from) +{ + CreateLakeTableStmt *newnode = makeNode(CreateLakeTableStmt); + + COPY_NODE_FIELD(base.relation); + COPY_NODE_FIELD(base.tableElts); + COPY_NODE_FIELD(base.inhRelations); + COPY_NODE_FIELD(base.partbound); + COPY_NODE_FIELD(base.partspec); + COPY_NODE_FIELD(base.ofTypename); + COPY_NODE_FIELD(base.constraints); + COPY_NODE_FIELD(base.options); + COPY_SCALAR_FIELD(base.oncommit); + COPY_STRING_FIELD(base.tablespacename); + COPY_STRING_FIELD(base.accessMethod); + COPY_SCALAR_FIELD(base.if_not_exists); + COPY_SCALAR_FIELD(base.gp_style_alter_part); + COPY_NODE_FIELD(base.distributedBy); + COPY_NODE_FIELD(base.partitionBy); + COPY_SCALAR_FIELD(base.relKind); + COPY_SCALAR_FIELD(base.ownerid); + COPY_SCALAR_FIELD(base.buildAoBlkdir); + COPY_NODE_FIELD(base.attr_encodings); + COPY_SCALAR_FIELD(base.isCtas); + COPY_NODE_FIELD(base.intoQuery); + COPY_NODE_FIELD(base.intoPolicy); + COPY_NODE_FIELD(base.part_idx_oids); + COPY_NODE_FIELD(base.part_idx_names); + COPY_NODE_FIELD(base.tags); + COPY_SCALAR_FIELD(base.origin); + COPY_STRING_FIELD(table_type); + COPY_STRING_FIELD(foreign_catalog); + COPY_STRING_FIELD(foreign_volume); + COPY_NODE_FIELD(options); + + return newnode; +} + static AlterDirectoryTableStmt * _copyAlterDirectoryTableStmt(const AlterDirectoryTableStmt *from) { diff --git a/src/backend/nodes/copyfuncs.switch.c b/src/backend/nodes/copyfuncs.switch.c index 69dcef19150..585e90be3c1 100644 --- a/src/backend/nodes/copyfuncs.switch.c +++ b/src/backend/nodes/copyfuncs.switch.c @@ -567,6 +567,12 @@ case T_CreateForeignServerStmt: retval = _copyCreateForeignServerStmt(from); break; + case T_CreateForeignCatalogStmt: + retval = _copyCreateForeignCatalogStmt(from); + break; + case T_CreateForeignVolumeStmt: + retval = _copyCreateForeignVolumeStmt(from); + break; case T_AlterForeignServerStmt: retval = _copyAlterForeignServerStmt(from); break; @@ -699,6 +705,9 @@ case T_CreateDirectoryTableStmt: retval = _copyCreateDirectoryTableStmt(from); break; + case T_CreateLakeTableStmt: + retval = _copyCreateLakeTableStmt(from); + break; case T_AlterDirectoryTableStmt: retval = _copyAlterDirectoryTableStmt(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index eb77ea9169f..8a8984237dd 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -2163,6 +2163,28 @@ _equalCreateForeignServerStmt(const CreateForeignServerStmt *a, const CreateFore return true; } +static bool +_equalCreateForeignCatalogStmt(const CreateForeignCatalogStmt *a, const CreateForeignCatalogStmt *b) +{ + COMPARE_STRING_FIELD(catalogname); + COMPARE_STRING_FIELD(servername); + COMPARE_SCALAR_FIELD(if_not_exists); + COMPARE_NODE_FIELD(options); + + return true; +} + +static bool +_equalCreateForeignVolumeStmt(const CreateForeignVolumeStmt *a, const CreateForeignVolumeStmt *b) +{ + COMPARE_STRING_FIELD(volumename); + COMPARE_STRING_FIELD(servername); + COMPARE_SCALAR_FIELD(if_not_exists); + COMPARE_NODE_FIELD(options); + + return true; +} + static bool _equalAddForeignSegStmt(const AddForeignSegStmt *a, const AddForeignSegStmt *b) { @@ -3486,6 +3508,20 @@ _equalCreateDirectoryTableStmt(const CreateDirectoryTableStmt *a, const CreateDi return true; } +static bool +_equalCreateLakeTableStmt(const CreateLakeTableStmt *a, const CreateLakeTableStmt *b) +{ + if (!_equalCreateStmt(&a->base, &b->base)) + return false; + + COMPARE_STRING_FIELD(table_type); + COMPARE_STRING_FIELD(foreign_catalog); + COMPARE_STRING_FIELD(foreign_volume); + COMPARE_NODE_FIELD(options); + + return true; +} + static bool _equalAlterDirectoryTableStmt(const AlterDirectoryTableStmt *a, const AlterDirectoryTableStmt *b) { @@ -4247,6 +4283,12 @@ equal(const void *a, const void *b) case T_CreateForeignServerStmt: retval = _equalCreateForeignServerStmt(a, b); break; + case T_CreateForeignCatalogStmt: + retval = _equalCreateForeignCatalogStmt(a, b); + break; + case T_CreateForeignVolumeStmt: + retval = _equalCreateForeignVolumeStmt(a, b); + break; case T_AddForeignSegStmt: retval = _equalAddForeignSegStmt(a, b); break; @@ -4602,6 +4644,10 @@ equal(const void *a, const void *b) retval = _equalCreateDirectoryTableStmt(a, b); break; + case T_CreateLakeTableStmt: + retval = _equalCreateLakeTableStmt(a, b); + break; + case T_AlterDirectoryTableStmt: retval = _equalAlterDirectoryTableStmt(a, b); break; diff --git a/src/backend/nodes/outfast.c b/src/backend/nodes/outfast.c index f31bfa87045..42907eccab3 100644 --- a/src/backend/nodes/outfast.c +++ b/src/backend/nodes/outfast.c @@ -673,6 +673,28 @@ _outCreateForeignServerStmt(StringInfo str, CreateForeignServerStmt *node) WRITE_NODE_FIELD(options); } +static void +_outCreateForeignCatalogStmt(StringInfo str, CreateForeignCatalogStmt *node) +{ + WRITE_NODE_TYPE("CREATEFOREIGNCATALOGSTMT"); + + WRITE_STRING_FIELD(catalogname); + WRITE_STRING_FIELD(servername); + WRITE_BOOL_FIELD(if_not_exists); + WRITE_NODE_FIELD(options); +} + +static void +_outCreateForeignVolumeStmt(StringInfo str, CreateForeignVolumeStmt *node) +{ + WRITE_NODE_TYPE("CREATEFOREIGNVOLUMESTMT"); + + WRITE_STRING_FIELD(volumename); + WRITE_STRING_FIELD(servername); + WRITE_BOOL_FIELD(if_not_exists); + WRITE_NODE_FIELD(options); +} + static void _outAddForeignSegstmt(StringInfo str, AddForeignSegStmt *node) { @@ -1818,6 +1840,12 @@ _outNode(StringInfo str, void *obj) case T_CreateForeignServerStmt: _outCreateForeignServerStmt(str, obj); break; + case T_CreateForeignCatalogStmt: + _outCreateForeignCatalogStmt(str, obj); + break; + case T_CreateForeignVolumeStmt: + _outCreateForeignVolumeStmt(str, obj); + break; case T_AddForeignSegStmt: _outAddForeignSegstmt(str, obj); break; @@ -1940,6 +1968,9 @@ _outNode(StringInfo str, void *obj) case T_CreateDirectoryTableStmt: _outCreateDirectoryTableStmt(str, obj); break; + case T_CreateLakeTableStmt: + _outCreateLakeTableStmt(str, obj); + break; case T_AlterDirectoryTableStmt: _outAlterDirectoryTableStmt(str, obj); break; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index c48ded5a813..e8ea5b7bafc 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -4270,6 +4270,18 @@ _outCreateDirectoryTableStmt(StringInfo str, const CreateDirectoryTableStmt *nod WRITE_STRING_FIELD(location); } +static void +_outCreateLakeTableStmt(StringInfo str, const CreateLakeTableStmt *node) +{ + WRITE_NODE_TYPE("CREATELAKETABLESTMT"); + + _outCreateStmtInfo(str, (const CreateStmt *) node); + WRITE_STRING_FIELD(table_type); + WRITE_STRING_FIELD(foreign_catalog); + WRITE_STRING_FIELD(foreign_volume); + WRITE_NODE_FIELD(options); +} + static void _outAlterDirectoryTableStmt(StringInfo str, const AlterDirectoryTableStmt *node) { @@ -5612,6 +5624,9 @@ outNode(StringInfo str, const void *obj) case T_CreateDirectoryTableStmt: _outCreateDirectoryTableStmt(str, obj); break; + case T_CreateLakeTableStmt: + _outCreateLakeTableStmt(str, obj); + break; case T_AlterDirectoryTableStmt: _outAlterDirectoryTableStmt(str, obj); break; diff --git a/src/backend/nodes/readfast.c b/src/backend/nodes/readfast.c index ff3cb5eaddf..adb7d9e4811 100644 --- a/src/backend/nodes/readfast.c +++ b/src/backend/nodes/readfast.c @@ -1598,6 +1598,32 @@ _readCreateForeignServerStmt(void) READ_DONE(); } +static CreateForeignCatalogStmt * +_readCreateForeignCatalogStmt(void) +{ + READ_LOCALS(CreateForeignCatalogStmt); + + READ_STRING_FIELD(catalogname); + READ_STRING_FIELD(servername); + READ_BOOL_FIELD(if_not_exists); + READ_NODE_FIELD(options); + + READ_DONE(); +} + +static CreateForeignVolumeStmt * +_readCreateForeignVolumeStmt(void) +{ + READ_LOCALS(CreateForeignVolumeStmt); + + READ_STRING_FIELD(volumename); + READ_STRING_FIELD(servername); + READ_BOOL_FIELD(if_not_exists); + READ_NODE_FIELD(options); + + READ_DONE(); +} + static AddForeignSegStmt * _readAddForeignSegStmt(void) { @@ -1910,6 +1936,21 @@ _readCreateDirectoryTableStmt(void) READ_DONE(); } +static CreateLakeTableStmt * +_readCreateLakeTableStmt(void) +{ + READ_LOCALS(CreateLakeTableStmt); + + _readCreateStmt_common(&local_node->base); + + READ_STRING_FIELD(table_type); + READ_STRING_FIELD(foreign_catalog); + READ_STRING_FIELD(foreign_volume); + READ_NODE_FIELD(options); + + READ_DONE(); +} + static AlterDirectoryTableStmt * _readAlterDirectoryTableStmt(void) { @@ -2881,6 +2922,12 @@ readNodeBinary(void) case T_CreateForeignServerStmt: return_value = _readCreateForeignServerStmt(); break; + case T_CreateForeignCatalogStmt: + return_value = _readCreateForeignCatalogStmt(); + break; + case T_CreateForeignVolumeStmt: + return_value = _readCreateForeignVolumeStmt(); + break; case T_AddForeignSegStmt: return_value = _readAddForeignSegStmt(); break; @@ -2991,6 +3038,9 @@ readNodeBinary(void) case T_CreateDirectoryTableStmt: return_value = _readCreateDirectoryTableStmt(); break; + case T_CreateLakeTableStmt: + return_value = _readCreateLakeTableStmt(); + break; case T_AlterDirectoryTableStmt: return_value = _readAlterDirectoryTableStmt(); break; diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index bd2c1bcf58c..9efc253e44b 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -526,6 +526,8 @@ typedef enum NodeTag T_AlterFdwStmt, T_CreateForeignServerStmt, T_AlterForeignServerStmt, + T_CreateForeignCatalogStmt, + T_CreateForeignVolumeStmt, T_CreateStorageServerStmt, T_AlterStorageServerStmt, T_DropStorageServerStmt, @@ -572,6 +574,7 @@ typedef enum NodeTag T_CreateDirectoryTableStmt, T_AlterDirectoryTableStmt, T_DropDirectoryTableStmt, + T_CreateLakeTableStmt, T_CreateFileSpaceStmt, T_FileSpaceEntry, T_DropFileSpaceStmt, diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index b79846b3d6d..956e90f115e 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3271,6 +3271,24 @@ typedef struct CreateForeignServerStmt List *options; /* generic options to server */ } CreateForeignServerStmt; +typedef struct CreateForeignCatalogStmt +{ + NodeTag type; + char *catalogname; /* foreign catalog name */ + char *servername; /* server name */ + bool if_not_exists; /* just do nothing if it already exists? */ + List *options; /* generic options to catalog */ +} CreateForeignCatalogStmt; + +typedef struct CreateForeignVolumeStmt +{ + NodeTag type; + char *volumename; /* foreign volume name */ + char *servername; /* server name */ + bool if_not_exists; /* just do nothing if it already exists? */ + List *options; /* generic options to volume */ +} CreateForeignVolumeStmt; + typedef struct AlterForeignServerStmt { NodeTag type; @@ -3778,6 +3796,15 @@ typedef struct CreateDirectoryTableStmt char *location; /* dtlocation for pg_directory_table */ } CreateDirectoryTableStmt; +typedef struct CreateLakeTableStmt +{ + CreateStmt base; /* base table creation info */ + char *table_type; /* lake table type, e.g. "ICEBERG" */ + char *foreign_catalog; /* foreign catalog name, or NULL */ + char *foreign_volume; /* foreign volume name, or NULL */ + List *options; /* lake-table-specific options */ +} CreateLakeTableStmt; + typedef struct AlterDirectoryTableStmt { NodeTag type; From b028fb8479c2b9843fafa44e6a524452aa65fd03 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Thu, 2 Jul 2026 11:05:08 +0800 Subject: [PATCH 03/29] parser: add grammar for CREATE ICEBERG TABLE / FOREIGN CATALOG / VOLUME Add the ICEBERG and VOLUME unreserved keywords and the CREATE-side grammar productions that build the parse nodes from the previous commit: - CREATE ICEBERG TABLE name (cols) [FOREIGN CATALOG c] [FOREIGN VOLUME v] OPTIONS (...) -> CreateLakeTableStmt (forced DISTRIBUTED RANDOMLY) - CREATE FOREIGN CATALOG name SERVER s OPTIONS (...) -> CreateForeignCatalogStmt - CREATE FOREIGN VOLUME name SERVER s OPTIONS (...) -> CreateForeignVolumeStmt Statements parse but are not yet dispatched; command handling, ObjectType entries and DROP support land in the next commit. Verified: bison reports no grammar conflicts; the statements parse and reach ProcessUtility. --- src/backend/parser/gram.y | 156 +++++++++++++++++++++++++++++++++++- src/include/parser/kwlist.h | 2 + 2 files changed, 156 insertions(+), 2 deletions(-) diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index bc657554219..c2bf21f7898 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -313,6 +313,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ CreateSchemaStmt CreateSeqStmt CreateStmt CreateStatsStmt CreateStorageServerStmt CreateStorageUserMappingStmt CreateTableSpaceStmt CreateFdwStmt CreateForeignServerStmt CreateForeignTableStmt CreateDirectoryTableStmt + CreateLakeTableStmt CreateForeignCatalogStmt CreateForeignVolumeStmt CreateAssertionStmt CreateTransformStmt CreateTrigStmt CreateEventTrigStmt CreateUserStmt CreateUserMappingStmt CreateRoleStmt CreatePolicyStmt CreatedbStmt CreateWarehouseStmt DeclareCursorStmt DefineStmt DeleteStmt DiscardStmt DoStmt @@ -410,6 +411,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ %type OptProfileElem %type opt_type +%type OptForeignCatalog OptForeignVolume %type foreign_server_version opt_foreign_server_version %type opt_in_database @@ -830,7 +832,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ HANDLER HAVING HEADER_P HOLD HOUR_P - IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE + ICEBERG IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION @@ -883,7 +885,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ UNLISTEN UNLOGGED UNTIL UPDATE USER USING VACUUM VALID VALIDATE VALIDATOR VALUE_P VALUES VARCHAR VARIADIC VARYING - VERBOSE VERSION_P VIEW VIEWS VOLATILE + VERBOSE VERSION_P VIEW VIEWS VOLATILE VOLUME WHEN WHERE WHITESPACE_P WINDOW WITH WITHIN WITHOUT WORK WRAPPER WRITE @@ -1535,6 +1537,9 @@ stmt: | CreateConversionStmt | CreateDomainStmt | CreateDirectoryTableStmt + | CreateLakeTableStmt + | CreateForeignCatalogStmt + | CreateForeignVolumeStmt | CreateExtensionStmt | CreateExternalStmt | CreateFdwStmt @@ -9093,6 +9098,149 @@ CreateDirectoryTableStmt: } ; +/***************************************************************************** + * + * QUERY: + * CREATE FOREIGN CATALOG name SERVER server_name OPTIONS (...) + * + *****************************************************************************/ + +CreateForeignCatalogStmt: + CREATE FOREIGN CATALOG_P name SERVER name create_generic_options + { + CreateForeignCatalogStmt *n = makeNode(CreateForeignCatalogStmt); + n->catalogname = $4; + n->servername = $6; + n->options = $7; + n->if_not_exists = false; + $$ = (Node *) n; + } + | CREATE FOREIGN CATALOG_P IF_P NOT EXISTS name SERVER name create_generic_options + { + CreateForeignCatalogStmt *n = makeNode(CreateForeignCatalogStmt); + n->catalogname = $7; + n->servername = $9; + n->options = $10; + n->if_not_exists = true; + $$ = (Node *) n; + } + ; + +/***************************************************************************** + * + * QUERY: + * CREATE FOREIGN VOLUME name SERVER server_name OPTIONS (...) + * + *****************************************************************************/ + +CreateForeignVolumeStmt: + CREATE FOREIGN VOLUME name SERVER name create_generic_options + { + CreateForeignVolumeStmt *n = makeNode(CreateForeignVolumeStmt); + n->volumename = $4; + n->servername = $6; + n->options = $7; + n->if_not_exists = false; + $$ = (Node *) n; + } + | CREATE FOREIGN VOLUME IF_P NOT EXISTS name SERVER name create_generic_options + { + CreateForeignVolumeStmt *n = makeNode(CreateForeignVolumeStmt); + n->volumename = $7; + n->servername = $9; + n->options = $10; + n->if_not_exists = true; + $$ = (Node *) n; + } + ; + +OptForeignCatalog: + CATALOG_P name { $$ = $2; } + | /*EMPTY*/ { $$ = NULL; } + ; + +OptForeignVolume: + VOLUME name { $$ = $2; } + | /*EMPTY*/ { $$ = NULL; } + ; + +/***************************************************************************** + * + * QUERY: + * CREATE ICEBERG TABLE relname (columns) + * [FOREIGN CATALOG cat] [FOREIGN VOLUME vol] OPTIONS (...) + * + * A lake table stores its data on external object storage; fragments are + * not hash-distributed across segments, so the distribution policy is + * forced to RANDOM to keep UPDATE/DELETE correct. + * + *****************************************************************************/ + +CreateLakeTableStmt: + CREATE ICEBERG TABLE qualified_name '(' OptTableElementList ')' + OptForeignCatalog OptForeignVolume create_generic_options + OptDistributedBy table_access_method_clause + { + CreateLakeTableStmt *n = makeNode(CreateLakeTableStmt); + $4->relpersistence = RELPERSISTENCE_PERMANENT; + n->base.relation = $4; + n->base.tableElts = $6; + n->base.inhRelations = NIL; + n->base.ofTypename = NULL; + n->base.constraints = NIL; + n->base.options = NIL; + n->base.oncommit = ONCOMMIT_NOOP; + n->base.tablespacename = NULL; + n->base.accessMethod = $12 ? $12 : pstrdup("iceberg"); + n->base.if_not_exists = false; + n->base.relKind = RELKIND_RELATION; + n->table_type = pstrdup("ICEBERG"); + n->foreign_catalog = $8 ? pstrdup($8) : NULL; + n->foreign_volume = $9 ? pstrdup($9) : NULL; + n->options = $10; + if ($11 != NULL) + ereport(WARNING, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("DISTRIBUTED clause has no effect for lake tables, using DISTRIBUTED RANDOMLY"))); + n->base.distributedBy = makeNode(DistributedBy); + n->base.distributedBy->ptype = POLICYTYPE_PARTITIONED; + n->base.distributedBy->keyCols = NIL; + n->base.distributedBy->numsegments = -1; + $$ = (Node *) n; + } + | CREATE ICEBERG TABLE IF_P NOT EXISTS qualified_name '(' OptTableElementList ')' + OptForeignCatalog OptForeignVolume create_generic_options + OptDistributedBy table_access_method_clause + { + CreateLakeTableStmt *n = makeNode(CreateLakeTableStmt); + $7->relpersistence = RELPERSISTENCE_PERMANENT; + n->base.relation = $7; + n->base.tableElts = $9; + n->base.inhRelations = NIL; + n->base.ofTypename = NULL; + n->base.constraints = NIL; + n->base.options = NIL; + n->base.oncommit = ONCOMMIT_NOOP; + n->base.tablespacename = NULL; + n->base.accessMethod = $15 ? $15 : pstrdup("iceberg"); + n->base.if_not_exists = true; + n->base.relKind = RELKIND_RELATION; + n->table_type = pstrdup("ICEBERG"); + n->foreign_catalog = $11 ? pstrdup($11) : NULL; + n->foreign_volume = $12 ? pstrdup($12) : NULL; + n->options = $13; + if ($14 != NULL) + ereport(WARNING, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("DISTRIBUTED clause has no effect for lake tables, using DISTRIBUTED RANDOMLY"))); + n->base.distributedBy = makeNode(DistributedBy); + n->base.distributedBy->ptype = POLICYTYPE_PARTITIONED; + n->base.distributedBy->keyCols = NIL; + n->base.distributedBy->numsegments = -1; + $$ = (Node *) n; + } + ; + /***************************************************************************** * * QUERY: @@ -21175,6 +21323,7 @@ unreserved_keyword: | HOLD | HOST | HOUR_P + | ICEBERG | IDENTITY_P | IF_P | IGNORE_P @@ -21415,6 +21564,7 @@ unreserved_keyword: | VIEW | VIEWS | VOLATILE + | VOLUME | WAREHOUSE | WAREHOUSE_SIZE | WEB /* gp */ @@ -22162,6 +22312,7 @@ bare_label_keyword: | HEADER_P | HOLD | HOST + | ICEBERG | IDENTITY_P | IF_P | IGNORE_P @@ -22465,6 +22616,7 @@ bare_label_keyword: | VIEW | VIEWS | VOLATILE + | VOLUME | WAREHOUSE | WAREHOUSE_SIZE | WEB diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 24b6936bd46..795b317f175 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -225,6 +225,7 @@ PG_KEYWORD("header", HEADER_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("hold", HOLD, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("host", HOST, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("hour", HOUR_P, UNRESERVED_KEYWORD, AS_LABEL) +PG_KEYWORD("iceberg", ICEBERG, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("identity", IDENTITY_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("if", IF_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("ignore", IGNORE_P, UNRESERVED_KEYWORD, BARE_LABEL) @@ -548,6 +549,7 @@ PG_KEYWORD("version", VERSION_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("view", VIEW, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("views", VIEWS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("volatile", VOLATILE, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("volume", VOLUME, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("warehouse", WAREHOUSE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("warehouse_size", WAREHOUSE_SIZE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("web", WEB, UNRESERVED_KEYWORD, BARE_LABEL) From 5a679b20f8d6bc870ccfbc1aac9c23ae81d42d7b Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Thu, 2 Jul 2026 12:32:13 +0800 Subject: [PATCH 04/29] commands: implement CREATE/DROP FOREIGN CATALOG and FOREIGN VOLUME Make the foreign catalog and foreign volume DDL commands functional on top of the previously added grammar, parse nodes and system catalogs: * CreateForeignCatalog()/CreateForeignVolume() insert into pg_foreign_catalog/pg_foreign_volume, check server existence and USAGE privilege, record dependencies on the server and owner, and dispatch to segments with preassigned OIDs. * Lookup helpers get_foreign_catalog_oid(), get_foreign_volume_oid() and GetForeignVolumeByName(); both objects are unique on (name, server). * New object infrastructure: OBJECT_FOREIGN_CATALOG/OBJECT_FOREIGN_VOLUME and OCLASS_FOREIGN_CATALOG/OCLASS_FOREIGN_VOLUME with handlers in all exhaustive switches (objectaddress, dependency, aclchk, event trigger, seclabel, dropcmds, alter). Ownership checks go through the generic object_ownercheck() via the new ObjectProperty entries. * Four new syscaches: FOREIGNCATALOGNAME/FOREIGNCATALOGOID and FOREIGNVOLUMENAMESERVER/FOREIGNVOLUMEOID. * DROP CATALOG / DROP VOLUME grammar via drop_type_name, going through the regular RemoveObjects() path with dependency handling, so DROP SERVER ... CASCADE also removes dependent catalogs and volumes. * utility.c dispatch and CREATE/DROP FOREIGN CATALOG|VOLUME command tags. --- src/backend/catalog/aclchk.c | 12 ++ src/backend/catalog/dependency.c | 12 ++ src/backend/catalog/objectaddress.c | 154 ++++++++++++++++ src/backend/catalog/oid_dispatch.c | 36 ++++ src/backend/commands/alter.c | 2 + src/backend/commands/dropcmds.c | 8 + src/backend/commands/event_trigger.c | 12 ++ src/backend/commands/foreigncmds.c | 262 +++++++++++++++++++++++++++ src/backend/commands/seclabel.c | 2 + src/backend/foreign/foreign.c | 94 ++++++++++ src/backend/parser/gram.y | 2 + src/backend/tcop/utility.c | 26 +++ src/backend/utils/cache/syscache.c | 26 +++ src/include/catalog/dependency.h | 4 +- src/include/catalog/oid_dispatch.h | 4 + src/include/commands/defrem.h | 2 + src/include/foreign/foreign.h | 12 ++ src/include/nodes/parsenodes.h | 2 + src/include/tcop/cmdtaglist.h | 4 + src/include/utils/syscache.h | 4 + 20 files changed, 679 insertions(+), 1 deletion(-) diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index 28e37f72ba1..573310a42e1 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -3050,6 +3050,12 @@ aclcheck_error(AclResult aclerr, ObjectType objtype, case OBJECT_FOREIGN_SERVER: msg = gettext_noop("permission denied for foreign server %s"); break; + case OBJECT_FOREIGN_CATALOG: + msg = gettext_noop("permission denied for foreign catalog %s"); + break; + case OBJECT_FOREIGN_VOLUME: + msg = gettext_noop("permission denied for foreign volume %s"); + break; case OBJECT_FOREIGN_TABLE: msg = gettext_noop("permission denied for foreign table %s"); break; @@ -3198,6 +3204,12 @@ aclcheck_error(AclResult aclerr, ObjectType objtype, case OBJECT_FOREIGN_SERVER: msg = gettext_noop("must be owner of foreign server %s"); break; + case OBJECT_FOREIGN_CATALOG: + msg = gettext_noop("must be owner of foreign catalog %s"); + break; + case OBJECT_FOREIGN_VOLUME: + msg = gettext_noop("must be owner of foreign volume %s"); + break; case OBJECT_FOREIGN_TABLE: msg = gettext_noop("must be owner of foreign table %s"); break; diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index f48824d9edc..2621f12ed19 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -42,8 +42,10 @@ #include "catalog/pg_directory_table.h" #include "catalog/pg_event_trigger.h" #include "catalog/pg_extension.h" +#include "catalog/pg_foreign_catalog.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" +#include "catalog/pg_foreign_volume.h" #include "catalog/pg_init_privs.h" #include "catalog/pg_language.h" #include "catalog/pg_largeobject.h" @@ -226,6 +228,8 @@ static const Oid object_classes[] = { ExtprotocolRelationId, /* OCLASS_EXTPROTOCOL */ GpMatviewAuxId, /* OCLASS_MATVIEW_AUX */ TaskRelationId, /* OCLASS_TASK */ + ForeignCatalogRelationId, /* OCLASS_FOREIGN_CATALOG */ + ForeignVolumeRelationId, /* OCLASS_FOREIGN_VOLUME */ }; /* @@ -1629,6 +1633,8 @@ doDeletion(const ObjectAddress *object, int flags) case OCLASS_TSTEMPLATE: case OCLASS_FDW: case OCLASS_FOREIGN_SERVER: + case OCLASS_FOREIGN_CATALOG: + case OCLASS_FOREIGN_VOLUME: case OCLASS_USER_MAPPING: case OCLASS_DEFACL: case OCLASS_EVENT_TRIGGER: @@ -3141,6 +3147,12 @@ getObjectClass(const ObjectAddress *object) case TagDescriptionRelationId: return OCLASS_TAG_DESCRIPTION; + case ForeignCatalogRelationId: + return OCLASS_FOREIGN_CATALOG; + + case ForeignVolumeRelationId: + return OCLASS_FOREIGN_VOLUME; + default: { struct CustomObjectClass *coc; diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index db5294c89e9..3432197bfb7 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -40,8 +40,10 @@ #include "catalog/pg_event_trigger.h" #include "catalog/pg_extension.h" #include "catalog/pg_extprotocol.h" +#include "catalog/pg_foreign_catalog.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" +#include "catalog/pg_foreign_volume.h" #include "catalog/pg_language.h" #include "catalog/pg_largeobject.h" #include "catalog/pg_largeobject_metadata.h" @@ -305,6 +307,34 @@ static const ObjectPropertyType ObjectProperty[] = OBJECT_FOREIGN_SERVER, true }, + { + "foreign catalog", + ForeignCatalogRelationId, + ForeignCatalogOidIndexId, + FOREIGNCATALOGOID, + FOREIGNCATALOGNAME, + Anum_pg_foreign_catalog_oid, + Anum_pg_foreign_catalog_fcname, + InvalidAttrNumber, + Anum_pg_foreign_catalog_fcowner, + InvalidAttrNumber, + OBJECT_FOREIGN_CATALOG, + true + }, + { + "foreign volume", + ForeignVolumeRelationId, + ForeignVolumeOidIndexId, + FOREIGNVOLUMEOID, + FOREIGNVOLUMENAME, + Anum_pg_foreign_volume_oid, + Anum_pg_foreign_volume_fvname, + InvalidAttrNumber, + Anum_pg_foreign_volume_fvowner, + InvalidAttrNumber, + OBJECT_FOREIGN_VOLUME, + true + }, { "storage server", StorageServerRelationId, @@ -995,6 +1025,14 @@ static const struct object_type_map /* OCLASS_TAG */ { "tag", OBJECT_TAG + }, + /* OCLASS_FOREIGN_CATALOG */ + { + "catalog", OBJECT_FOREIGN_CATALOG + }, + /* OCLASS_FOREIGN_VOLUME */ + { + "volume", OBJECT_FOREIGN_VOLUME } }; @@ -1165,6 +1203,8 @@ get_object_address(ObjectType objtype, Node *object, case OBJECT_LANGUAGE: case OBJECT_FDW: case OBJECT_FOREIGN_SERVER: + case OBJECT_FOREIGN_CATALOG: + case OBJECT_FOREIGN_VOLUME: case OBJECT_EVENT_TRIGGER: case OBJECT_EXTPROTOCOL: case OBJECT_PARAMETER_ACL: @@ -1472,6 +1512,16 @@ get_object_address_unqualified(ObjectType objtype, address.objectId = get_foreign_server_oid(name, missing_ok); address.objectSubId = 0; break; + case OBJECT_FOREIGN_CATALOG: + address.classId = ForeignCatalogRelationId; + address.objectId = get_foreign_catalog_oid(name, missing_ok); + address.objectSubId = 0; + break; + case OBJECT_FOREIGN_VOLUME: + address.classId = ForeignVolumeRelationId; + address.objectId = get_foreign_volume_oid(name, missing_ok); + address.objectSubId = 0; + break; case OBJECT_EVENT_TRIGGER: address.classId = EventTriggerRelationId; address.objectId = get_event_trigger_oid(name, missing_ok); @@ -2506,6 +2556,8 @@ pg_get_object_address(PG_FUNCTION_ARGS) case OBJECT_EXTENSION: case OBJECT_FDW: case OBJECT_FOREIGN_SERVER: + case OBJECT_FOREIGN_CATALOG: + case OBJECT_FOREIGN_VOLUME: case OBJECT_STORAGE_SERVER: case OBJECT_LANGUAGE: case OBJECT_PARAMETER_ACL: @@ -2663,6 +2715,8 @@ check_object_ownership(Oid roleid, ObjectType objtype, ObjectAddress address, case OBJECT_EXTENSION: case OBJECT_FDW: case OBJECT_FOREIGN_SERVER: + case OBJECT_FOREIGN_CATALOG: + case OBJECT_FOREIGN_VOLUME: case OBJECT_LANGUAGE: case OBJECT_PUBLICATION: case OBJECT_SCHEMA: @@ -3955,6 +4009,50 @@ getObjectDescription(const ObjectAddress *object, bool missing_ok) break; } + case OCLASS_FOREIGN_CATALOG: + { + HeapTuple catTup; + Form_pg_foreign_catalog catForm; + + catTup = SearchSysCache1(FOREIGNCATALOGOID, + ObjectIdGetDatum(object->objectId)); + if (!HeapTupleIsValid(catTup)) + { + if (!missing_ok) + elog(ERROR, "cache lookup failed for foreign catalog %u", + object->objectId); + break; + } + + catForm = (Form_pg_foreign_catalog) GETSTRUCT(catTup); + appendStringInfo(&buffer, _("catalog %s"), + NameStr(catForm->fcname)); + ReleaseSysCache(catTup); + break; + } + + case OCLASS_FOREIGN_VOLUME: + { + HeapTuple volTup; + Form_pg_foreign_volume volForm; + + volTup = SearchSysCache1(FOREIGNVOLUMEOID, + ObjectIdGetDatum(object->objectId)); + if (!HeapTupleIsValid(volTup)) + { + if (!missing_ok) + elog(ERROR, "cache lookup failed for foreign volume %u", + object->objectId); + break; + } + + volForm = (Form_pg_foreign_volume) GETSTRUCT(volTup); + appendStringInfo(&buffer, _("volume %s"), + NameStr(volForm->fvname)); + ReleaseSysCache(volTup); + break; + } + case OCLASS_USER_MAPPING: { HeapTuple tup; @@ -4949,6 +5047,14 @@ getObjectTypeDescription(const ObjectAddress *object, bool missing_ok) appendStringInfoString(&buffer, "server"); break; + case OCLASS_FOREIGN_CATALOG: + appendStringInfoString(&buffer, "catalog"); + break; + + case OCLASS_FOREIGN_VOLUME: + appendStringInfoString(&buffer, "volume"); + break; + case OCLASS_USER_MAPPING: appendStringInfoString(&buffer, "user mapping"); break; @@ -6043,6 +6149,54 @@ getObjectIdentityParts(const ObjectAddress *object, break; } + case OCLASS_FOREIGN_CATALOG: + { + HeapTuple catTup; + Form_pg_foreign_catalog catForm; + + catTup = SearchSysCache1(FOREIGNCATALOGOID, + ObjectIdGetDatum(object->objectId)); + if (!HeapTupleIsValid(catTup)) + { + if (!missing_ok) + elog(ERROR, "cache lookup failed for foreign catalog %u", + object->objectId); + break; + } + + catForm = (Form_pg_foreign_catalog) GETSTRUCT(catTup); + appendStringInfoString(&buffer, + quote_identifier(NameStr(catForm->fcname))); + if (objname) + *objname = list_make1(pstrdup(NameStr(catForm->fcname))); + ReleaseSysCache(catTup); + break; + } + + case OCLASS_FOREIGN_VOLUME: + { + HeapTuple volTup; + Form_pg_foreign_volume volForm; + + volTup = SearchSysCache1(FOREIGNVOLUMEOID, + ObjectIdGetDatum(object->objectId)); + if (!HeapTupleIsValid(volTup)) + { + if (!missing_ok) + elog(ERROR, "cache lookup failed for foreign volume %u", + object->objectId); + break; + } + + volForm = (Form_pg_foreign_volume) GETSTRUCT(volTup); + appendStringInfoString(&buffer, + quote_identifier(NameStr(volForm->fvname))); + if (objname) + *objname = list_make1(pstrdup(NameStr(volForm->fvname))); + ReleaseSysCache(volTup); + break; + } + case OCLASS_STORAGE_SERVER: { StorageServer *srv; diff --git a/src/backend/catalog/oid_dispatch.c b/src/backend/catalog/oid_dispatch.c index 888c66b6a73..5271c65cc9f 100644 --- a/src/backend/catalog/oid_dispatch.c +++ b/src/backend/catalog/oid_dispatch.c @@ -98,8 +98,10 @@ #include "catalog/pg_extension.h" #include "catalog/pg_extprotocol.h" #include "catalog/pg_event_trigger.h" +#include "catalog/pg_foreign_catalog.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" +#include "catalog/pg_foreign_volume.h" #include "catalog/pg_language.h" #include "catalog/pg_largeobject_metadata.h" #include "catalog/pg_namespace.h" @@ -880,6 +882,40 @@ GetNewOidForForeignServer(Relation relation, Oid indexId, AttrNumber oidcolumn, } +Oid +GetNewOidForForeignCatalog(Relation relation, Oid indexId, AttrNumber oidcolumn, + char *catname) +{ + OidAssignment key; + + Assert(RelationGetRelid(relation) == ForeignCatalogRelationId); + Assert(indexId == ForeignCatalogOidIndexId); + Assert(oidcolumn == Anum_pg_foreign_catalog_oid); + + memset(&key, 0, sizeof(OidAssignment)); + key.type = T_OidAssignment; + key.objname = catname; + return GetNewOrPreassignedOid(relation, indexId, oidcolumn, &key); + +} + +Oid +GetNewOidForForeignVolume(Relation relation, Oid indexId, AttrNumber oidcolumn, + char *volumename) +{ + OidAssignment key; + + Assert(RelationGetRelid(relation) == ForeignVolumeRelationId); + Assert(indexId == ForeignVolumeOidIndexId); + Assert(oidcolumn == Anum_pg_foreign_volume_oid); + + memset(&key, 0, sizeof(OidAssignment)); + key.type = T_OidAssignment; + key.objname = volumename; + return GetNewOrPreassignedOid(relation, indexId, oidcolumn, &key); + +} + Oid GetNewOidForStorageServer(Relation relation, Oid indexId, AttrNumber oidcolumn, char *srvname) diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index 31c290530d7..b125d7b560b 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -774,6 +774,8 @@ AlterObjectNamespace_oid(Oid classId, Oid objid, Oid nspOid, case OCLASS_STORAGE_USER_MAPPING: case OCLASS_TAG: case OCLASS_TAG_DESCRIPTION: + case OCLASS_FOREIGN_CATALOG: + case OCLASS_FOREIGN_VOLUME: /* ignore object types that don't have schema-qualified names */ break; diff --git a/src/backend/commands/dropcmds.c b/src/backend/commands/dropcmds.c index 9bed9866aac..1a6a3e38ef4 100644 --- a/src/backend/commands/dropcmds.c +++ b/src/backend/commands/dropcmds.c @@ -456,6 +456,14 @@ does_not_exist_skipping(ObjectType objtype, Node *object) msg = gettext_noop("server \"%s\" does not exist, skipping"); name = strVal(object); break; + case OBJECT_FOREIGN_CATALOG: + msg = gettext_noop("foreign catalog \"%s\" does not exist, skipping"); + name = strVal(object); + break; + case OBJECT_FOREIGN_VOLUME: + msg = gettext_noop("foreign volume \"%s\" does not exist, skipping"); + name = strVal(object); + break; case OBJECT_STORAGE_SERVER: msg = gettext_noop("storage server \"%s\" does not exist, skipping"); name = strVal(object); diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index f46567a5b0c..e609f9bb0f1 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -965,6 +965,8 @@ EventTriggerSupportsObjectType(ObjectType obtype) case OBJECT_EXTENSION: case OBJECT_FDW: case OBJECT_FOREIGN_SERVER: + case OBJECT_FOREIGN_CATALOG: + case OBJECT_FOREIGN_VOLUME: case OBJECT_FOREIGN_TABLE: case OBJECT_FUNCTION: case OBJECT_INDEX: @@ -1067,6 +1069,8 @@ EventTriggerSupportsObjectClass(ObjectClass objclass) case OCLASS_TSCONFIG: case OCLASS_FDW: case OCLASS_FOREIGN_SERVER: + case OCLASS_FOREIGN_CATALOG: + case OCLASS_FOREIGN_VOLUME: case OCLASS_USER_MAPPING: case OCLASS_DEFACL: case OCLASS_EXTENSION: @@ -2063,6 +2067,10 @@ stringify_grant_objtype(ObjectType objtype) return "FOREIGN DATA WRAPPER"; case OBJECT_FOREIGN_SERVER: return "FOREIGN SERVER"; + case OBJECT_FOREIGN_CATALOG: + return "FOREIGN CATALOG"; + case OBJECT_FOREIGN_VOLUME: + return "FOREIGN VOLUME"; case OBJECT_STORAGE_SERVER: return "STORAGE SERVER"; case OBJECT_FUNCTION: @@ -2157,6 +2165,10 @@ stringify_adefprivs_objtype(ObjectType objtype) return "FOREIGN DATA WRAPPERS"; case OBJECT_FOREIGN_SERVER: return "FOREIGN SERVERS"; + case OBJECT_FOREIGN_CATALOG: + return "FOREIGN CATALOGS"; + case OBJECT_FOREIGN_VOLUME: + return "FOREIGN VOLUMES"; case OBJECT_STORAGE_SERVER: return "STORAGE SERVERS"; case OBJECT_FUNCTION: diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index e260af42188..c9a1e53787e 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -23,10 +23,12 @@ #include "catalog/indexing.h" #include "catalog/objectaccess.h" #include "catalog/oid_dispatch.h" +#include "catalog/pg_foreign_catalog.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" #include "catalog/pg_foreign_table_seg.h" +#include "catalog/pg_foreign_volume.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "catalog/pg_user_mapping.h" @@ -1026,6 +1028,266 @@ CreateForeignServer(CreateForeignServerStmt *stmt) } +/* + * Create a foreign catalog + */ +ObjectAddress +CreateForeignCatalog(CreateForeignCatalogStmt *stmt) +{ + Relation rel; + Datum catalogoptions; + Datum values[Natts_pg_foreign_catalog]; + bool nulls[Natts_pg_foreign_catalog]; + HeapTuple tuple; + Oid catalogId; + Oid ownerId; + AclResult aclresult; + ObjectAddress myself; + ObjectAddress referenced; + ForeignServer *server; + + rel = table_open(ForeignCatalogRelationId, RowExclusiveLock); + + /* For now the owner cannot be specified on create. Use effective user ID. */ + ownerId = GetUserId(); + + /* + * Check that there is no other foreign catalog by this name. Catalog + * names are global (like server names): every reference syntax (DROP + * CATALOG, the CATALOG clause of CREATE ICEBERG TABLE, GUCs) identifies + * a catalog by bare name, so the name alone must be unique. If there is + * one, do nothing if IF NOT EXISTS was specified. + */ + catalogId = get_foreign_catalog_oid(stmt->catalogname, true); + if (OidIsValid(catalogId)) + { + if (stmt->if_not_exists) + { + /* + * If we are in an extension script, insist that the pre-existing + * object be a member of the extension, to avoid security risks. + */ + ObjectAddressSet(myself, ForeignCatalogRelationId, catalogId); + checkMembershipInCurrentExtension(&myself); + + /* OK to skip */ + ereport(NOTICE, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("foreign catalog \"%s\" already exists, skipping", + stmt->catalogname))); + table_close(rel, RowExclusiveLock); + return InvalidObjectAddress; + } + else + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("foreign catalog \"%s\" already exists", + stmt->catalogname))); + } + + /* + * Check that the server exists and that we have USAGE on it. + */ + server = GetForeignServerByName(stmt->servername, false); + + aclresult = object_aclcheck(ForeignServerRelationId, server->serverid, ownerId, ACL_USAGE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, OBJECT_FOREIGN_SERVER, server->servername); + + /* + * Insert tuple into pg_foreign_catalog. + */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + + catalogId = GetNewOidForForeignCatalog(rel, ForeignCatalogOidIndexId, + Anum_pg_foreign_catalog_oid, + stmt->catalogname); + values[Anum_pg_foreign_catalog_oid - 1] = ObjectIdGetDatum(catalogId); + values[Anum_pg_foreign_catalog_fcname - 1] = + DirectFunctionCall1(namein, CStringGetDatum(stmt->catalogname)); + values[Anum_pg_foreign_catalog_fcowner - 1] = ObjectIdGetDatum(ownerId); + values[Anum_pg_foreign_catalog_fcserver - 1] = ObjectIdGetDatum(server->serverid); + + /* Add catalog options; there is no validator for them */ + catalogoptions = transformGenericOptions(ForeignCatalogRelationId, + PointerGetDatum(NULL), + stmt->options, + InvalidOid); + + if (PointerIsValid(DatumGetPointer(catalogoptions))) + values[Anum_pg_foreign_catalog_fcoptions - 1] = catalogoptions; + else + nulls[Anum_pg_foreign_catalog_fcoptions - 1] = true; + + tuple = heap_form_tuple(rel->rd_att, values, nulls); + + CatalogTupleInsert(rel, tuple); + + heap_freetuple(tuple); + + /* record dependencies */ + myself.classId = ForeignCatalogRelationId; + myself.objectId = catalogId; + myself.objectSubId = 0; + + referenced.classId = ForeignServerRelationId; + referenced.objectId = server->serverid; + referenced.objectSubId = 0; + recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + + recordDependencyOnOwner(ForeignCatalogRelationId, catalogId, ownerId); + + /* dependency on extension */ + recordDependencyOnCurrentExtension(&myself, false); + + /* Post creation hook for new foreign catalog */ + InvokeObjectPostCreateHook(ForeignCatalogRelationId, catalogId, 0); + + if (Gp_role == GP_ROLE_DISPATCH) + { + CdbDispatchUtilityStatement((Node *) stmt, + DF_WITH_SNAPSHOT | DF_CANCEL_ON_ERROR | DF_NEED_TWO_PHASE, + GetAssignedOidsForDispatch(), + NULL); + } + + table_close(rel, RowExclusiveLock); + + return myself; +} + + +/* + * Create a foreign volume + */ +ObjectAddress +CreateForeignVolume(CreateForeignVolumeStmt *stmt) +{ + Relation rel; + Datum volumeoptions; + Datum values[Natts_pg_foreign_volume]; + bool nulls[Natts_pg_foreign_volume]; + HeapTuple tuple; + Oid volumeId; + Oid ownerId; + AclResult aclresult; + ObjectAddress myself; + ObjectAddress referenced; + ForeignServer *server; + + rel = table_open(ForeignVolumeRelationId, RowExclusiveLock); + + /* For now the owner cannot be specified on create. Use effective user ID. */ + ownerId = GetUserId(); + + /* + * Check that there is no other foreign volume by this name. Volume + * names are global (like server names): every reference syntax (DROP + * VOLUME, the VOLUME clause of CREATE ICEBERG TABLE, GUCs) identifies + * a volume by bare name, so the name alone must be unique. If there is + * one, do nothing if IF NOT EXISTS was specified. + */ + volumeId = get_foreign_volume_oid(stmt->volumename, true); + if (OidIsValid(volumeId)) + { + if (stmt->if_not_exists) + { + /* + * If we are in an extension script, insist that the pre-existing + * object be a member of the extension, to avoid security risks. + */ + ObjectAddressSet(myself, ForeignVolumeRelationId, volumeId); + checkMembershipInCurrentExtension(&myself); + + /* OK to skip */ + ereport(NOTICE, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("foreign volume \"%s\" already exists, skipping", + stmt->volumename))); + table_close(rel, RowExclusiveLock); + return InvalidObjectAddress; + } + else + ereport(ERROR, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("foreign volume \"%s\" already exists", + stmt->volumename))); + } + + /* + * Check that the server exists and that we have USAGE on it. + */ + server = GetForeignServerByName(stmt->servername, false); + + aclresult = object_aclcheck(ForeignServerRelationId, server->serverid, ownerId, ACL_USAGE); + if (aclresult != ACLCHECK_OK) + aclcheck_error(aclresult, OBJECT_FOREIGN_SERVER, server->servername); + + /* + * Insert tuple into pg_foreign_volume. + */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + + volumeId = GetNewOidForForeignVolume(rel, ForeignVolumeOidIndexId, + Anum_pg_foreign_volume_oid, + stmt->volumename); + values[Anum_pg_foreign_volume_oid - 1] = ObjectIdGetDatum(volumeId); + values[Anum_pg_foreign_volume_fvname - 1] = + DirectFunctionCall1(namein, CStringGetDatum(stmt->volumename)); + values[Anum_pg_foreign_volume_fvowner - 1] = ObjectIdGetDatum(ownerId); + values[Anum_pg_foreign_volume_fvserver - 1] = ObjectIdGetDatum(server->serverid); + + /* Add volume options; there is no validator for them */ + volumeoptions = transformGenericOptions(ForeignVolumeRelationId, + PointerGetDatum(NULL), + stmt->options, + InvalidOid); + + if (PointerIsValid(DatumGetPointer(volumeoptions))) + values[Anum_pg_foreign_volume_fvoptions - 1] = volumeoptions; + else + nulls[Anum_pg_foreign_volume_fvoptions - 1] = true; + + tuple = heap_form_tuple(rel->rd_att, values, nulls); + + CatalogTupleInsert(rel, tuple); + + heap_freetuple(tuple); + + /* record dependencies */ + myself.classId = ForeignVolumeRelationId; + myself.objectId = volumeId; + myself.objectSubId = 0; + + referenced.classId = ForeignServerRelationId; + referenced.objectId = server->serverid; + referenced.objectSubId = 0; + recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + + recordDependencyOnOwner(ForeignVolumeRelationId, volumeId, ownerId); + + /* dependency on extension */ + recordDependencyOnCurrentExtension(&myself, false); + + /* Post creation hook for new foreign volume */ + InvokeObjectPostCreateHook(ForeignVolumeRelationId, volumeId, 0); + + if (Gp_role == GP_ROLE_DISPATCH) + { + CdbDispatchUtilityStatement((Node *) stmt, + DF_WITH_SNAPSHOT | DF_CANCEL_ON_ERROR | DF_NEED_TWO_PHASE, + GetAssignedOidsForDispatch(), + NULL); + } + + table_close(rel, RowExclusiveLock); + + return myself; +} + + /* * Alter foreign server */ diff --git a/src/backend/commands/seclabel.c b/src/backend/commands/seclabel.c index d4018ac0348..cd64f90f1d7 100644 --- a/src/backend/commands/seclabel.c +++ b/src/backend/commands/seclabel.c @@ -76,6 +76,8 @@ SecLabelSupportsObjectType(ObjectType objtype) case OBJECT_EXTENSION: case OBJECT_FDW: case OBJECT_FOREIGN_SERVER: + case OBJECT_FOREIGN_CATALOG: + case OBJECT_FOREIGN_VOLUME: case OBJECT_INDEX: case OBJECT_OPCLASS: case OBJECT_OPERATOR: diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c index 0fadf562ee4..331ef9bc40b 100644 --- a/src/backend/foreign/foreign.c +++ b/src/backend/foreign/foreign.c @@ -15,10 +15,12 @@ #include "access/htup_details.h" #include "access/reloptions.h" #include "access/table.h" +#include "catalog/pg_foreign_catalog.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" #include "catalog/pg_foreign_table_seg.h" +#include "catalog/pg_foreign_volume.h" #include "catalog/pg_user_mapping.h" #include "cdb/cdbgang.h" #include "cdb/cdbutil.h" @@ -982,6 +984,98 @@ get_foreign_server_oid(const char *servername, bool missing_ok) return oid; } +/* + * get_foreign_catalog_oid - given a foreign catalog name, look up the OID + * + * If missing_ok is false, throw an error if name not found. If true, just + * return InvalidOid. + */ +Oid +get_foreign_catalog_oid(const char *catalogname, bool missing_ok) +{ + Oid oid; + + oid = GetSysCacheOid1(FOREIGNCATALOGNAME, + Anum_pg_foreign_catalog_oid, + CStringGetDatum(catalogname)); + if (!OidIsValid(oid) && !missing_ok) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("foreign catalog \"%s\" does not exist", + catalogname))); + + return oid; +} + +/* + * get_foreign_volume_oid - given a foreign volume name, look up the OID + * + * If missing_ok is false, throw an error if name not found. If true, just + * return InvalidOid. + */ +Oid +get_foreign_volume_oid(const char *volumename, bool missing_ok) +{ + Oid oid; + + oid = GetSysCacheOid1(FOREIGNVOLUMENAME, + Anum_pg_foreign_volume_oid, + CStringGetDatum(volumename)); + if (!OidIsValid(oid) && !missing_ok) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("foreign volume \"%s\" does not exist", + volumename))); + + return oid; +} + +/* + * GetForeignVolumeByName - look up a foreign volume by name + */ +ForeignVolume * +GetForeignVolumeByName(const char *volumename, bool missing_ok) +{ + HeapTuple tp; + Form_pg_foreign_volume fvform; + ForeignVolume *volume; + Datum datum; + bool isnull; + + tp = SearchSysCache1(FOREIGNVOLUMENAME, + PointerGetDatum(volumename)); + if (!HeapTupleIsValid(tp)) + { + if (!missing_ok) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("foreign volume \"%s\" does not exist", + volumename))); + return NULL; + } + + fvform = (Form_pg_foreign_volume) GETSTRUCT(tp); + + volume = (ForeignVolume *) palloc(sizeof(ForeignVolume)); + volume->volumeid = fvform->oid; + volume->serverid = fvform->fvserver; + volume->volumename = pstrdup(NameStr(fvform->fvname)); + + /* Extract the volume options */ + datum = SysCacheGetAttr(FOREIGNVOLUMENAME, + tp, + Anum_pg_foreign_volume_fvoptions, + &isnull); + if (isnull) + volume->options = NIL; + else + volume->options = untransformRelOptions(datum); + + ReleaseSysCache(tp); + + return volume; +} + /* * Get a copy of an existing local path for a given join relation. * diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index c2bf21f7898..ae79cb403de 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -10577,6 +10577,8 @@ drop_type_name: | PUBLICATION { $$ = OBJECT_PUBLICATION; } | SCHEMA { $$ = OBJECT_SCHEMA; } | SERVER { $$ = OBJECT_FOREIGN_SERVER; } + | CATALOG_P { $$ = OBJECT_FOREIGN_CATALOG; } + | VOLUME { $$ = OBJECT_FOREIGN_VOLUME; } | PROTOCOL { $$ = OBJECT_EXTPROTOCOL; } ; diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 021c69ad031..f439fdc1582 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -210,6 +210,8 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree) case T_CreateExtensionStmt: case T_CreateFdwStmt: case T_CreateForeignServerStmt: + case T_CreateForeignCatalogStmt: + case T_CreateForeignVolumeStmt: case T_CreateForeignTableStmt: case T_AddForeignSegStmt: case T_CreateFunctionStmt: @@ -2183,6 +2185,14 @@ ProcessUtilitySlow(ParseState *pstate, address = CreateForeignServer((CreateForeignServerStmt *) parsetree); break; + case T_CreateForeignCatalogStmt: + address = CreateForeignCatalog((CreateForeignCatalogStmt *) parsetree); + break; + + case T_CreateForeignVolumeStmt: + address = CreateForeignVolume((CreateForeignVolumeStmt *) parsetree); + break; + case T_AlterForeignServerStmt: address = AlterForeignServer((AlterForeignServerStmt *) parsetree); break; @@ -3255,6 +3265,14 @@ CreateCommandTag(Node *parsetree) tag = CMDTAG_CREATE_SERVER; break; + case T_CreateForeignCatalogStmt: + tag = CMDTAG_CREATE_FOREIGN_CATALOG; + break; + + case T_CreateForeignVolumeStmt: + tag = CMDTAG_CREATE_FOREIGN_VOLUME; + break; + case T_AlterForeignServerStmt: tag = CMDTAG_ALTER_SERVER; break; @@ -3421,6 +3439,12 @@ CreateCommandTag(Node *parsetree) case OBJECT_FOREIGN_SERVER: tag = CMDTAG_DROP_SERVER; break; + case OBJECT_FOREIGN_CATALOG: + tag = CMDTAG_DROP_FOREIGN_CATALOG; + break; + case OBJECT_FOREIGN_VOLUME: + tag = CMDTAG_DROP_FOREIGN_VOLUME; + break; case OBJECT_STORAGE_SERVER: tag = CMDTAG_DROP_STORAGE_SERVER; break; @@ -4199,6 +4223,8 @@ GetCommandLogLevel(Node *parsetree) case T_CreateFdwStmt: case T_AlterFdwStmt: case T_CreateForeignServerStmt: + case T_CreateForeignCatalogStmt: + case T_CreateForeignVolumeStmt: case T_AlterForeignServerStmt: case T_CreateStorageServerStmt: case T_AlterStorageServerStmt: diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c index a8901a957eb..556c7536205 100644 --- a/src/backend/utils/cache/syscache.c +++ b/src/backend/utils/cache/syscache.c @@ -44,9 +44,11 @@ #include "catalog/pg_enum.h" #include "catalog/pg_event_trigger.h" #include "catalog/pg_extension.h" +#include "catalog/pg_foreign_catalog.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" +#include "catalog/pg_foreign_volume.h" #include "catalog/pg_language.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" @@ -363,6 +365,18 @@ static const struct cachedesc cacheinfo[] = { KEY(Anum_pg_foreign_data_wrapper_oid), 2 }, + [FOREIGNCATALOGNAME] = { + ForeignCatalogRelationId, + ForeignCatalogNameIndexId, + KEY(Anum_pg_foreign_catalog_fcname), + 2 + }, + [FOREIGNCATALOGOID] = { + ForeignCatalogRelationId, + ForeignCatalogOidIndexId, + KEY(Anum_pg_foreign_catalog_oid), + 2 + }, [FOREIGNSERVERNAME] = { ForeignServerRelationId, ForeignServerNameIndexId, @@ -393,6 +407,18 @@ static const struct cachedesc cacheinfo[] = { KEY(Anum_pg_foreign_table_ftrelid), 4 }, + [FOREIGNVOLUMENAME] = { + ForeignVolumeRelationId, + ForeignVolumeNameIndexId, + KEY(Anum_pg_foreign_volume_fvname), + 2 + }, + [FOREIGNVOLUMEOID] = { + ForeignVolumeRelationId, + ForeignVolumeOidIndexId, + KEY(Anum_pg_foreign_volume_oid), + 2 + }, [GPPOLICYID] = { GpPolicyRelationId, GpPolicyLocalOidIndexId, diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index 6a7ae2abea9..ac2ccb596b1 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -151,9 +151,11 @@ typedef enum ObjectClass OCLASS_EXTPROTOCOL, /* pg_extprotocol */ OCLASS_MATVIEW_AUX, /* gp_matview_aux */ OCLASS_TASK, /* pg_task */ + OCLASS_FOREIGN_CATALOG, /* pg_foreign_catalog */ + OCLASS_FOREIGN_VOLUME, /* pg_foreign_volume */ } ObjectClass; -#define LAST_OCLASS OCLASS_TASK +#define LAST_OCLASS OCLASS_FOREIGN_VOLUME /* flag bits for performDeletion/performMultipleDeletions: */ #define PERFORM_DELETION_INTERNAL 0x0001 /* internal action */ diff --git a/src/include/catalog/oid_dispatch.h b/src/include/catalog/oid_dispatch.h index 6ccd20ac4bd..9c0574faf38 100644 --- a/src/include/catalog/oid_dispatch.h +++ b/src/include/catalog/oid_dispatch.h @@ -65,6 +65,10 @@ extern Oid GetNewOidForForeignDataWrapper(Relation relation, Oid indexId, AttrNu char *fdwname); extern Oid GetNewOidForForeignServer(Relation relation, Oid indexId, AttrNumber oidcolumn, char *srvname); +extern Oid GetNewOidForForeignCatalog(Relation relation, Oid indexId, AttrNumber oidcolumn, + char *catname); +extern Oid GetNewOidForForeignVolume(Relation relation, Oid indexId, AttrNumber oidcolumn, + char *volumename); extern Oid GetNewOidForStorageServer(Relation relation, Oid indexId, AttrNumber oidcolumn, char *srvname); extern Oid GetNewOidForLanguage(Relation relation, Oid indexId, AttrNumber oidcolumn, diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index 7af15a37f52..72f3562a033 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -133,6 +133,8 @@ extern ObjectAddress CreateForeignDataWrapper(ParseState *pstate, CreateFdwStmt extern ObjectAddress AlterForeignDataWrapper(ParseState *pstate, AlterFdwStmt *stmt); extern ObjectAddress CreateForeignServer(CreateForeignServerStmt *stmt); extern ObjectAddress AlterForeignServer(AlterForeignServerStmt *stmt); +extern ObjectAddress CreateForeignCatalog(CreateForeignCatalogStmt *stmt); +extern ObjectAddress CreateForeignVolume(CreateForeignVolumeStmt *stmt); extern ObjectAddress CreateStorageServer(CreateStorageServerStmt *stmt); extern ObjectAddress AlterStorageServer(AlterStorageServerStmt *stmt); extern Oid RemoveStorageServer(DropStorageServerStmt *stmt); diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h index f95f0d331e7..417ac80d5fb 100644 --- a/src/include/foreign/foreign.h +++ b/src/include/foreign/foreign.h @@ -62,6 +62,14 @@ typedef struct ForeignTable int32 num_segments; /* the number of segments of the foreign table */ } ForeignTable; +typedef struct ForeignVolume +{ + Oid volumeid; /* volume Oid */ + Oid serverid; /* server Oid */ + char *volumename; /* name of the volume */ + List *options; /* fvoptions as DefElem list */ +} ForeignVolume; + /* Flags for GetForeignServerExtended */ #define FSV_MISSING_OK 0x01 @@ -84,11 +92,15 @@ extern ForeignDataWrapper *GetForeignDataWrapperByName(const char *fdwname, bool missing_ok); extern ForeignTable *GetForeignTable(Oid relid); extern bool rel_is_external_table(Oid relid); +extern ForeignVolume *GetForeignVolumeByName(const char *volumename, + bool missing_ok); extern List *GetForeignColumnOptions(Oid relid, AttrNumber attnum); extern Oid get_foreign_data_wrapper_oid(const char *fdwname, bool missing_ok); extern Oid get_foreign_server_oid(const char *servername, bool missing_ok); +extern Oid get_foreign_catalog_oid(const char *catalogname, bool missing_ok); +extern Oid get_foreign_volume_oid(const char *volumename, bool missing_ok); extern Oid GetForeignServerSegByRelid(Oid tableOid); extern List *GetForeignServerSegsByRelId(Oid relid); diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 956e90f115e..bac420b1d58 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2260,6 +2260,8 @@ typedef enum ObjectType OBJECT_EXTENSION, OBJECT_FDW, OBJECT_FOREIGN_SERVER, + OBJECT_FOREIGN_CATALOG, + OBJECT_FOREIGN_VOLUME, OBJECT_STORAGE_SERVER, OBJECT_FOREIGN_TABLE, OBJECT_FUNCTION, diff --git a/src/include/tcop/cmdtaglist.h b/src/include/tcop/cmdtaglist.h index 21db8a4d19d..a1cc6615a7d 100644 --- a/src/include/tcop/cmdtaglist.h +++ b/src/include/tcop/cmdtaglist.h @@ -106,8 +106,10 @@ PG_CMDTAG(CMDTAG_CREATE_DYNAMIC_TABLE, "CREATE DYNAMIC TABLE", true, false, fals PG_CMDTAG(CMDTAG_CREATE_EVENT_TRIGGER, "CREATE EVENT TRIGGER", false, false, false) PG_CMDTAG(CMDTAG_CREATE_EXTENSION, "CREATE EXTENSION", true, false, false) PG_CMDTAG(CMDTAG_CREATE_EXTERNAL, "CREATE EXTERNAL TABLE", true, false, false) +PG_CMDTAG(CMDTAG_CREATE_FOREIGN_CATALOG, "CREATE FOREIGN CATALOG", true, false, false) PG_CMDTAG(CMDTAG_CREATE_FOREIGN_DATA_WRAPPER, "CREATE FOREIGN DATA WRAPPER", true, false, false) PG_CMDTAG(CMDTAG_CREATE_FOREIGN_TABLE, "CREATE FOREIGN TABLE", true, false, false) +PG_CMDTAG(CMDTAG_CREATE_FOREIGN_VOLUME, "CREATE FOREIGN VOLUME", true, false, false) PG_CMDTAG(CMDTAG_CREATE_FUNCTION, "CREATE FUNCTION", true, false, false) PG_CMDTAG(CMDTAG_CREATE_INDEX, "CREATE INDEX", true, false, false) PG_CMDTAG(CMDTAG_CREATE_LANGUAGE, "CREATE LANGUAGE", true, false, false) @@ -176,8 +178,10 @@ PG_CMDTAG(CMDTAG_DROP_DOMAIN, "DROP DOMAIN", true, false, false) PG_CMDTAG(CMDTAG_DROP_DYNAMIC_TABLE, "DROP DYNAMIC TABLE", true, false, false) PG_CMDTAG(CMDTAG_DROP_EVENT_TRIGGER, "DROP EVENT TRIGGER", false, false, false) PG_CMDTAG(CMDTAG_DROP_EXTENSION, "DROP EXTENSION", true, false, false) +PG_CMDTAG(CMDTAG_DROP_FOREIGN_CATALOG, "DROP FOREIGN CATALOG", true, false, false) PG_CMDTAG(CMDTAG_DROP_FOREIGN_DATA_WRAPPER, "DROP FOREIGN DATA WRAPPER", true, false, false) PG_CMDTAG(CMDTAG_DROP_FOREIGN_TABLE, "DROP FOREIGN TABLE", true, false, false) +PG_CMDTAG(CMDTAG_DROP_FOREIGN_VOLUME, "DROP FOREIGN VOLUME", true, false, false) PG_CMDTAG(CMDTAG_DROP_FUNCTION, "DROP FUNCTION", true, false, false) PG_CMDTAG(CMDTAG_DROP_INDEX, "DROP INDEX", true, false, false) PG_CMDTAG(CMDTAG_DROP_LANGUAGE, "DROP LANGUAGE", true, false, false) diff --git a/src/include/utils/syscache.h b/src/include/utils/syscache.h index e790dfe2af5..5ed42d82b15 100644 --- a/src/include/utils/syscache.h +++ b/src/include/utils/syscache.h @@ -60,6 +60,8 @@ enum SysCacheIdentifier EVENTTRIGGEROID, EXTPROTOCOLOID, EXTPROTOCOLNAME, + FOREIGNCATALOGNAME, + FOREIGNCATALOGOID, FOREIGNDATAWRAPPERNAME, FOREIGNDATAWRAPPEROID, FOREIGNSERVERNAME, @@ -67,6 +69,8 @@ enum SysCacheIdentifier STORAGESERVERNAME, STORAGESERVEROID, FOREIGNTABLEREL, + FOREIGNVOLUMENAME, + FOREIGNVOLUMEOID, GPPOLICYID, AORELID, INDEXRELID, From b4704be776f814b46a5a59eb543df80fad32a864 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Thu, 2 Jul 2026 13:04:41 +0800 Subject: [PATCH 05/29] commands: implement CREATE ICEBERG TABLE Make CREATE ICEBERG TABLE functional on top of the existing grammar, parse nodes and pg_lake_table catalog: * laketablecmds.c: CreateLakeTable() inserts the pg_lake_table entry after DefineRelation and records dependencies on the table's foreign catalog and volume; ValidateLakeTableOptions() runs the same resolution on the QD before DefineRelation so validation failures don't surface as QE-annotated errors; RemoveLakeTableEntry() cleans up on drop (hooked into heap_drop_with_catalog). * iceberg_default_catalog / iceberg_default_volume GUCs (synchronized to QEs) provide defaults when CREATE ICEBERG TABLE has no CATALOG or VOLUME clause; their check hooks verify the object exists. * The iceberg table access method is resolved strictly by name (get_table_am_oid) and is expected to be provided by a datalake extension; without it, CREATE ICEBERG TABLE fails up front with a hint. The kernel does not hardcode any extension name or AM OID. * Guard rails: reject the iceberg AM for every creation path other than CreateLakeTableStmt (CREATE TABLE ... USING iceberg, CTAS, matview, default_table_access_method, partition children), reject ALTER TABLE ... SET ACCESS METHOD to or from iceberg, and reject SET DISTRIBUTED BY on lake tables, which must stay DISTRIBUTED RANDOMLY. A relation created with the iceberg AM but without its pg_lake_table metadata would be unusable and undroppable. * utility.c dispatch (transformCreateStmt works on the embedded CreateStmt) and the CREATE LAKE TABLE command tag. --- src/backend/catalog/heap.c | 5 + src/backend/commands/Makefile | 1 + src/backend/commands/laketablecmds.c | 469 +++++++++++++++++++++++++++ src/backend/commands/tablecmds.c | 46 +++ src/backend/tcop/utility.c | 72 ++++ src/backend/utils/misc/guc_tables.c | 23 ++ src/include/commands/laketablecmds.h | 47 +++ src/include/tcop/cmdtaglist.h | 1 + src/include/utils/sync_guc_name.h | 2 + 9 files changed, 666 insertions(+) create mode 100644 src/backend/commands/laketablecmds.c create mode 100644 src/include/commands/laketablecmds.h diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 0b62b174afa..b6171061700 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -72,6 +72,7 @@ #include "catalog/storage.h" #include "catalog/storage_directory_table.h" #include "catalog/storage_xlog.h" +#include "commands/laketablecmds.h" #include "commands/tablecmds.h" #include "commands/typecmds.h" #include "miscadmin.h" @@ -2322,6 +2323,10 @@ heap_drop_with_catalog(Oid relid) */ CheckTableForSerializableConflictIn(rel); + /* If this is a lake table, remove its pg_lake_table entry */ + if (RelationIsIcebergTable(rel)) + RemoveLakeTableEntry(relid); + /* * Delete pg_foreign_table tuple first. */ diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile index 3451b45d115..1a7d2d51db1 100644 --- a/src/backend/commands/Makefile +++ b/src/backend/commands/Makefile @@ -42,6 +42,7 @@ OBJS = \ foreigncmds.o \ functioncmds.o \ indexcmds.o \ + laketablecmds.o \ lockcmds.o \ matview.o \ opclasscmds.o \ diff --git a/src/backend/commands/laketablecmds.c b/src/backend/commands/laketablecmds.c new file mode 100644 index 00000000000..80771f0b1aa --- /dev/null +++ b/src/backend/commands/laketablecmds.c @@ -0,0 +1,469 @@ +/*------------------------------------------------------------------------- + * + * laketablecmds.c + * lake table creation/manipulation commands + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/commands/laketablecmds.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/genam.h" +#include "access/htup_details.h" +#include "access/reloptions.h" +#include "access/table.h" +#include "access/xact.h" +#include "catalog/catalog.h" +#include "catalog/dependency.h" +#include "catalog/indexing.h" +#include "catalog/objectaccess.h" +#include "catalog/pg_foreign_catalog.h" +#include "catalog/pg_foreign_volume.h" +#include "catalog/pg_lake_table.h" +#include "commands/defrem.h" +#include "commands/laketablecmds.h" +#include "foreign/foreign.h" +#include "miscadmin.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/rel.h" + +/* GUC variables for default Iceberg catalog and volume */ +char *iceberg_default_catalog = NULL; +char *iceberg_default_volume = NULL; + +/* + * check_iceberg_default_catalog: validate new iceberg_default_catalog GUC value + */ +bool +check_iceberg_default_catalog(char **newval, void **extra, GucSource source) +{ + /* + * If we aren't inside a transaction, or connected to a database, we + * cannot do the catalog accesses necessary to verify the name. Must + * accept the value on faith. + */ + if (IsTransactionState() && MyDatabaseId != InvalidOid) + { + if (**newval != '\0') + { + Oid catalog_oid = get_foreign_catalog_oid(*newval, true); + + if (!OidIsValid(catalog_oid)) + { + /* + * When source == PGC_S_TEST, don't throw a hard error for a + * nonexistent catalog, only a NOTICE. See comments in guc.h. + */ + if (source == PGC_S_TEST) + { + ereport(NOTICE, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("foreign catalog \"%s\" does not exist", + *newval))); + } + else + { + GUC_check_errdetail("Foreign catalog \"%s\" does not exist.", + *newval); + return false; + } + } + } + } + + return true; +} + +/* + * check_iceberg_default_volume: validate new iceberg_default_volume GUC value + */ +bool +check_iceberg_default_volume(char **newval, void **extra, GucSource source) +{ + /* + * If we aren't inside a transaction, or connected to a database, we + * cannot do the catalog accesses necessary to verify the name. Must + * accept the value on faith. + */ + if (IsTransactionState() && MyDatabaseId != InvalidOid) + { + if (**newval != '\0') + { + Oid volume_oid = get_foreign_volume_oid(*newval, true); + + if (!OidIsValid(volume_oid)) + { + /* + * When source == PGC_S_TEST, don't throw a hard error for a + * nonexistent volume, only a NOTICE. See comments in guc.h. + */ + if (source == PGC_S_TEST) + { + ereport(NOTICE, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("foreign volume \"%s\" does not exist", + *newval))); + } + else + { + GUC_check_errdetail("Foreign volume \"%s\" does not exist.", + *newval); + return false; + } + } + } + } + + return true; +} + +/* + * GetDefaultIcebergCatalog -- get the name of the current default Iceberg catalog + * + * Returns NULL if no default catalog is set. + * This function hides the iceberg_default_catalog GUC variable. + */ +const char * +GetDefaultIcebergCatalog(void) +{ + if (iceberg_default_catalog == NULL || iceberg_default_catalog[0] == '\0') + return NULL; + + /* + * Verify that the catalog still exists. We don't cache this because + * the catalog could be dropped after the GUC was set. + */ + if (!OidIsValid(get_foreign_catalog_oid(iceberg_default_catalog, true))) + return NULL; + + return iceberg_default_catalog; +} + +/* + * GetDefaultIcebergVolume -- get the name of the current default Iceberg volume + * + * Returns NULL if no default volume is set. + * This function hides the iceberg_default_volume GUC variable. + */ +const char * +GetDefaultIcebergVolume(void) +{ + if (iceberg_default_volume == NULL || iceberg_default_volume[0] == '\0') + return NULL; + + /* + * Verify that the volume still exists. We don't cache this because + * the volume could be dropped after the GUC was set. + */ + if (!OidIsValid(get_foreign_volume_oid(iceberg_default_volume, true))) + return NULL; + + return iceberg_default_volume; +} + +/* + * GetIcebergTableAmOid + * + * Look up the OID of the iceberg table access method, which is provided by + * a datalake extension rather than the kernel. Returns InvalidOid if the + * access method is not installed and missing_ok is true. + */ +Oid +GetIcebergTableAmOid(bool missing_ok) +{ + return get_table_am_oid(ICEBERG_TABLE_AM_NAME, missing_ok); +} + +/* + * RelationIsIcebergTable + * + * True iff the relation uses the iceberg table access method. Resolved by + * access method name so the kernel does not depend on any particular + * extension's OID assignments. + */ +bool +RelationIsIcebergTable(Relation rel) +{ + Oid iceberg_amoid; + + if (!OidIsValid(rel->rd_rel->relam)) + return false; + + iceberg_amoid = GetIcebergTableAmOid(true); + + return OidIsValid(iceberg_amoid) && rel->rd_rel->relam == iceberg_amoid; +} + +/* + * Validate table type + */ +static void +validate_table_type(const char *table_type) +{ + if (!table_type) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("table type cannot be NULL"))); + + if (strcmp(table_type, "ICEBERG") != 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unsupported table type \"%s\"", table_type), + errhint("The only supported table type is ICEBERG."))); +} + +/* + * Validate foreign catalog exists + */ +static Oid +validate_foreign_catalog(const char *catalog_name) +{ + if (!catalog_name || catalog_name[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("no foreign catalog specified"), + errhint("Specify CATALOG in CREATE ICEBERG TABLE or set iceberg_default_catalog."))); + + return get_foreign_catalog_oid(catalog_name, false); +} + +/* + * Validate foreign volume exists + */ +static Oid +validate_foreign_volume(const char *volume_name) +{ + if (!volume_name || volume_name[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("no foreign volume specified"), + errhint("Specify VOLUME in CREATE ICEBERG TABLE or set iceberg_default_volume."))); + + return get_foreign_volume_oid(volume_name, false); +} + +/* + * ResolveLakeTableOptions + * + * Resolve and validate the table type, catalog and volume of a + * CreateLakeTableStmt, returning the catalog/volume OIDs. + * + * Also exposed (via ValidateLakeTableOptions) so ProcessUtilitySlow can run + * the validation on the QD before DefineRelation: DefineRelation dispatches + * the statement to the QEs, so a validation failure raised only inside + * CreateLakeTable() would surface as a confusing QE-annotated error. + */ +static void +ResolveLakeTableOptions(CreateLakeTableStmt *stmt, + Oid *catalog_oid_out, Oid *volume_oid_out) +{ + const char *catalog_name; + const char *volume_name; + + /* + * Lake tables are unusable without an extension providing the iceberg + * table access method; check it first so the install hint takes + * precedence over catalog/volume resolution errors. + */ + if (!OidIsValid(GetIcebergTableAmOid(true))) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("table access method \"%s\" does not exist", + ICEBERG_TABLE_AM_NAME), + errhint("CREATE ICEBERG TABLE requires an extension that provides the \"%s\" table access method.", + ICEBERG_TABLE_AM_NAME))); + + /* + * The grammar accepts a USING clause, but an iceberg table must use the + * iceberg access method: a lake table created with another AM would get + * pg_lake_table metadata without lake-table semantics (and DROP TABLE, + * which detects lake tables by their AM, would leave that metadata + * behind as an orphaned row). + */ + if (stmt->base.accessMethod != NULL && + strcmp(stmt->base.accessMethod, ICEBERG_TABLE_AM_NAME) != 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("access method \"%s\" is not supported for iceberg tables", + stmt->base.accessMethod), + errhint("Omit the USING clause; CREATE ICEBERG TABLE always uses the \"%s\" access method.", + ICEBERG_TABLE_AM_NAME))); + + /* Validate table type */ + validate_table_type(stmt->table_type); + + /* + * Determine catalog name: use explicit value if provided, otherwise + * fall back to the iceberg_default_catalog GUC. When the GUC is set + * but its catalog has been dropped, say so instead of the generic + * "no foreign catalog specified". + */ + catalog_name = stmt->foreign_catalog; + if (catalog_name == NULL || catalog_name[0] == '\0') + { + catalog_name = GetDefaultIcebergCatalog(); + if (catalog_name == NULL && + iceberg_default_catalog != NULL && iceberg_default_catalog[0] != '\0') + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("default iceberg catalog \"%s\" does not exist", + iceberg_default_catalog), + errhint("Set iceberg_default_catalog to an existing foreign catalog."))); + } + + /* + * Determine volume name: use explicit value if provided, otherwise + * fall back to the iceberg_default_volume GUC. + */ + volume_name = stmt->foreign_volume; + if (volume_name == NULL || volume_name[0] == '\0') + { + volume_name = GetDefaultIcebergVolume(); + if (volume_name == NULL && + iceberg_default_volume != NULL && iceberg_default_volume[0] != '\0') + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("default iceberg volume \"%s\" does not exist", + iceberg_default_volume), + errhint("Set iceberg_default_volume to an existing foreign volume."))); + } + + *catalog_oid_out = validate_foreign_catalog(catalog_name); + + /* + * A volume is required for every lake table, even when the catalog + * vends the table's physical location: the QEs still read and write + * the data files through the volume's storage endpoint and + * credentials. Without this check the missing volume only surfaces + * later, deep in the access method's create path. + */ + *volume_oid_out = validate_foreign_volume(volume_name); +} + +/* + * ValidateLakeTableOptions + * + * QD-side pre-DefineRelation validation wrapper; see ResolveLakeTableOptions. + */ +void +ValidateLakeTableOptions(CreateLakeTableStmt *stmt) +{ + Oid catalog_oid; + Oid volume_oid; + + ResolveLakeTableOptions(stmt, &catalog_oid, &volume_oid); +} + +/* + * CreateLakeTable + * + * Create a lake table entry in pg_lake_table after the base table has been + * created. + */ +void +CreateLakeTable(CreateLakeTableStmt *stmt, Oid relId) +{ + Relation lake_rel; + Datum values[Natts_pg_lake_table]; + bool nulls[Natts_pg_lake_table]; + HeapTuple tuple; + Oid catalog_oid; + Oid volume_oid; + ObjectAddress myself; + ObjectAddress referenced; + + ResolveLakeTableOptions(stmt, &catalog_oid, &volume_oid); + + /* + * Make the just-created base relation (from DefineRelation) visible to + * this command before we record dependencies on it. + */ + CommandCounterIncrement(); + + lake_rel = table_open(LakeTableRelationId, RowExclusiveLock); + + /* + * Insert tuple into pg_lake_table. + */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + + values[Anum_pg_lake_table_ltrelid - 1] = ObjectIdGetDatum(relId); + values[Anum_pg_lake_table_ltforeign_catalog - 1] = ObjectIdGetDatum(catalog_oid); + values[Anum_pg_lake_table_ltforeign_volume - 1] = ObjectIdGetDatum(volume_oid); + values[Anum_pg_lake_table_lttable_type - 1] = CStringGetTextDatum(stmt->table_type); + + if (stmt->options) + { + Datum options_datum; + + /* Build standard text[] reloptions from DefElem list */ + options_datum = transformRelOptions((Datum) 0, stmt->options, + NULL, NULL, false, false); + + if (options_datum != (Datum) 0) + values[Anum_pg_lake_table_ltoptions - 1] = options_datum; + else + nulls[Anum_pg_lake_table_ltoptions - 1] = true; + } + else + { + nulls[Anum_pg_lake_table_ltoptions - 1] = true; + } + + tuple = heap_form_tuple(lake_rel->rd_att, values, nulls); + + CatalogTupleInsert(lake_rel, tuple); + + /* Record dependencies on the foreign catalog and volume */ + myself.classId = RelationRelationId; + myself.objectId = relId; + myself.objectSubId = 0; + + referenced.classId = ForeignCatalogRelationId; + referenced.objectId = catalog_oid; + referenced.objectSubId = 0; + recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + + referenced.classId = ForeignVolumeRelationId; + referenced.objectId = volume_oid; + referenced.objectSubId = 0; + recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); + + heap_freetuple(tuple); + table_close(lake_rel, RowExclusiveLock); + + CommandCounterIncrement(); + InvokeObjectPostCreateHook(LakeTableRelationId, relId, 0); +} + +/* + * RemoveLakeTableEntry + * + * Remove the pg_lake_table entry for the given relation. + */ +void +RemoveLakeTableEntry(Oid relid) +{ + Relation ltRel; + HeapTuple tup; + ScanKeyData skey; + SysScanDesc scan; + + ltRel = table_open(LakeTableRelationId, RowExclusiveLock); + ScanKeyInit(&skey, + Anum_pg_lake_table_ltrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + scan = systable_beginscan(ltRel, LakeTableRelidIndexId, true, NULL, 1, &skey); + while (HeapTupleIsValid(tup = systable_getnext(scan))) + CatalogTupleDelete(ltRel, &tup->t_self); + systable_endscan(scan); + table_close(ltRel, RowExclusiveLock); +} diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index b8b69336b89..3786ca44e1a 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -69,6 +69,7 @@ #include "commands/comment.h" #include "commands/createas.h" #include "commands/defrem.h" +#include "commands/laketablecmds.h" #include "commands/matview.h" #include "commands/event_trigger.h" #include "commands/policy.h" @@ -938,6 +939,24 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, amHandlerOid = get_table_am_handler_oid(accessMethod, false); } + /* + * Lake tables must be created through CREATE ICEBERG TABLE, which also + * creates the pg_lake_table catalog entries the iceberg access method + * relies on. A relation created with the iceberg AM through any other + * path (CREATE TABLE ... USING iceberg, CTAS, matview, + * default_table_access_method, partition child) would be unusable and + * undroppable, so reject it up front. CreateLakeTableStmt embeds + * CreateStmt as its first member, so nodeTag() distinguishes the paths. + */ + if (OidIsValid(accessMethodId) && + accessMethodId == GetIcebergTableAmOid(true) && + nodeTag(stmt) != T_CreateLakeTableStmt) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot create table \"%s\" with access method \"%s\"", + stmt->relation->relname, ICEBERG_TABLE_AM_NAME), + errhint("Use CREATE ICEBERG TABLE instead."))); + /* * GPDB: for partitioned tables, inherit reloptions from the parent. * Note this is applicable only if the parent has the same AM as the child. @@ -17059,6 +17078,7 @@ static void ATPrepSetAccessMethod(AlteredTableInfo *tab, Relation rel, const char *amname) { Oid amoid; + Oid iceberg_amoid; /* Check that the table access method exists */ amoid = get_table_am_oid(amname, false); @@ -17066,6 +17086,24 @@ ATPrepSetAccessMethod(AlteredTableInfo *tab, Relation rel, const char *amname) if (rel->rd_rel->relam == amoid) return; + /* + * The iceberg AM relies on catalog entries that only the CREATE/DROP + * ICEBERG TABLE paths manage, so a table cannot be converted to or from + * it with SET ACCESS METHOD. + */ + iceberg_amoid = GetIcebergTableAmOid(true); + if (OidIsValid(iceberg_amoid) && amoid == iceberg_amoid) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot change access method of table \"%s\" to \"%s\"", + RelationGetRelationName(rel), ICEBERG_TABLE_AM_NAME), + errhint("Use CREATE ICEBERG TABLE to create an iceberg table."))); + if (OidIsValid(iceberg_amoid) && rel->rd_rel->relam == iceberg_amoid) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot change access method of lake table \"%s\"", + RelationGetRelationName(rel)))); + /* Save info for Phase 3 to do the real work */ tab->rewrite |= AT_REWRITE_ACCESS_METHOD; tab->newAccessMethod = amoid; @@ -19759,6 +19797,14 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("SET DISTRIBUTED REPLICATED is not supported for external table"))); } + + /* Lake tables must remain DISTRIBUTED RANDOMLY */ + if (RelationIsIcebergTable(rel)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot change distribution policy of lake table \"%s\"", + RelationGetRelationName(rel)), + errhint("Lake tables must use DISTRIBUTED RANDOMLY because data is stored on object storage."))); } if (Gp_role == GP_ROLE_DISPATCH) diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index f439fdc1582..300c018ba8a 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -38,6 +38,7 @@ #include "commands/dbcommands.h" #include "commands/defrem.h" #include "commands/dirtablecmds.h" +#include "commands/laketablecmds.h" #include "commands/discard.h" #include "commands/event_trigger.h" #include "commands/explain.h" @@ -264,6 +265,7 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree) case T_AlterResourceGroupStmt: case T_AlterTagStmt: case T_CreateDirectoryTableStmt: + case T_CreateLakeTableStmt: case T_AlterDirectoryTableStmt: case T_DropDirectoryTableStmt: case T_CreateProfileStmt: @@ -1450,6 +1452,7 @@ ProcessUtilitySlow(ParseState *pstate, case T_CreateStmt: case T_CreateForeignTableStmt: case T_CreateDirectoryTableStmt: + case T_CreateLakeTableStmt: { List *stmts; RangeVar *table_rv = NULL; @@ -1657,6 +1660,70 @@ ProcessUtilitySlow(ParseState *pstate, secondaryObject, stmt); } + else if (IsA(stmt, CreateLakeTableStmt)) + { + CreateLakeTableStmt *cstmt = (CreateLakeTableStmt *) stmt; + Datum toast_options; + static char *validnsps[] = HEAP_RELOPT_NAMESPACES; + + /* Remember transformed RangeVar for LIKE */ + table_rv = cstmt->base.relation; + + /* + * Validate catalog/volume resolution up front: + * the statement is dispatched to the QEs, so a + * failure raised only later inside + * CreateLakeTable() would surface as a confusing + * QE-annotated error. + */ + if (Gp_role == GP_ROLE_DISPATCH) + ValidateLakeTableOptions(cstmt); + + /* + * Create the table itself. Dispatch manually + * below (like the plain CreateStmt path above) + * so that the TOAST table exists before the + * statement is sent and its OID is included in + * the dispatched OID list. + */ + address = DefineRelation(&cstmt->base, + RELKIND_RELATION, + InvalidOid, NULL, + queryString, + false, + true, + NULL); + /* Create the lake table metadata entry */ + CreateLakeTable(cstmt, address.objectId); + EventTriggerCollectSimpleCommand(address, + secondaryObject, + stmt); + + /* + * Lake tables are backed by a real table access + * method, so let NewRelationCreateToastTable + * decide whether a secondary relation is needed, + * just like plain CREATE TABLE. + */ + CommandCounterIncrement(); + + toast_options = transformRelOptions((Datum) 0, + cstmt->base.options, + "toast", + validnsps, + true, + false); + NewRelationCreateToastTable(address.objectId, + toast_options); + + if (Gp_role == GP_ROLE_DISPATCH && ENABLE_DISPATCH()) + CdbDispatchUtilityStatement((Node *) stmt, + DF_CANCEL_ON_ERROR | + DF_NEED_TWO_PHASE | + DF_WITH_SNAPSHOT, + GetAssignedOidsForDispatch(), + NULL); + } else if (IsA(stmt, TableLikeClause)) { /* @@ -3329,6 +3396,10 @@ CreateCommandTag(Node *parsetree) tag = CMDTAG_CREATE_DIRECTORY_TABLE; break; + case T_CreateLakeTableStmt: + tag = CMDTAG_CREATE_LAKE_TABLE; + break; + case T_AlterDirectoryTableStmt: tag = CMDTAG_ALTER_DIRECTORY_TABLE; break; @@ -4237,6 +4308,7 @@ GetCommandLogLevel(Node *parsetree) case T_DropStorageUserMappingStmt: case T_ImportForeignSchemaStmt: case T_CreateDirectoryTableStmt: + case T_CreateLakeTableStmt: case T_AlterDirectoryTableStmt: case T_DropDirectoryTableStmt: lev = LOGSTMT_DDL; diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index 1ff92994527..c4498a09fc4 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -39,6 +39,7 @@ #include "catalog/storage_directory_table.h" #include "commands/async.h" #include "commands/tablespace.h" +#include "commands/laketablecmds.h" #include "commands/trigger.h" #include "commands/user.h" #include "commands/vacuum.h" @@ -4119,6 +4120,28 @@ struct config_string ConfigureNamesString[] = check_temp_tablespaces, assign_temp_tablespaces, NULL }, + { + {"iceberg_default_catalog", PGC_USERSET, CLIENT_CONN_STATEMENT, + gettext_noop("Sets the default foreign catalog to create Iceberg tables in."), + gettext_noop("An empty string means no default catalog."), + GUC_IS_NAME + }, + &iceberg_default_catalog, + "", + check_iceberg_default_catalog, NULL, NULL + }, + + { + {"iceberg_default_volume", PGC_USERSET, CLIENT_CONN_STATEMENT, + gettext_noop("Sets the default foreign volume to create Iceberg tables in."), + gettext_noop("An empty string means no default volume."), + GUC_IS_NAME + }, + &iceberg_default_volume, + "", + check_iceberg_default_volume, NULL, NULL + }, + { {"createrole_self_grant", PGC_USERSET, CLIENT_CONN_STATEMENT, gettext_noop("Sets whether a CREATEROLE user automatically grants " diff --git a/src/include/commands/laketablecmds.h b/src/include/commands/laketablecmds.h new file mode 100644 index 00000000000..05564a79a98 --- /dev/null +++ b/src/include/commands/laketablecmds.h @@ -0,0 +1,47 @@ +/*------------------------------------------------------------------------- + * + * laketablecmds.h + * prototypes for laketablecmds.c. + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/commands/laketablecmds.h + * + *------------------------------------------------------------------------- + */ +#ifndef LAKETABLECMDS_H +#define LAKETABLECMDS_H + +#include "catalog/pg_lake_table.h" +#include "nodes/parsenodes.h" +#include "utils/guc.h" +#include "utils/rel.h" + +/* + * Name of the table access method lake tables are created with. The + * kernel only provides the DDL scaffolding; the access method itself is + * provided by a datalake extension. + */ +#define ICEBERG_TABLE_AM_NAME "iceberg" + +/* GUC variables */ +extern char *iceberg_default_catalog; +extern char *iceberg_default_volume; + +/* GUC check hooks */ +extern bool check_iceberg_default_catalog(char **newval, void **extra, GucSource source); +extern bool check_iceberg_default_volume(char **newval, void **extra, GucSource source); + +/* Functions to get default values */ +extern const char *GetDefaultIcebergCatalog(void); +extern const char *GetDefaultIcebergVolume(void); + +/* Lake table management */ +extern Oid GetIcebergTableAmOid(bool missing_ok); +extern bool RelationIsIcebergTable(Relation rel); +extern void ValidateLakeTableOptions(CreateLakeTableStmt *stmt); +extern void CreateLakeTable(CreateLakeTableStmt *stmt, Oid relId); +extern void RemoveLakeTableEntry(Oid relid); + +#endif /* LAKETABLECMDS_H */ diff --git a/src/include/tcop/cmdtaglist.h b/src/include/tcop/cmdtaglist.h index a1cc6615a7d..fec05c924ad 100644 --- a/src/include/tcop/cmdtaglist.h +++ b/src/include/tcop/cmdtaglist.h @@ -112,6 +112,7 @@ PG_CMDTAG(CMDTAG_CREATE_FOREIGN_TABLE, "CREATE FOREIGN TABLE", true, false, fals PG_CMDTAG(CMDTAG_CREATE_FOREIGN_VOLUME, "CREATE FOREIGN VOLUME", true, false, false) PG_CMDTAG(CMDTAG_CREATE_FUNCTION, "CREATE FUNCTION", true, false, false) PG_CMDTAG(CMDTAG_CREATE_INDEX, "CREATE INDEX", true, false, false) +PG_CMDTAG(CMDTAG_CREATE_LAKE_TABLE, "CREATE LAKE TABLE", true, false, false) PG_CMDTAG(CMDTAG_CREATE_LANGUAGE, "CREATE LANGUAGE", true, false, false) PG_CMDTAG(CMDTAG_CREATE_MATERIALIZED_VIEW, "CREATE MATERIALIZED VIEW", true, false, false) PG_CMDTAG(CMDTAG_CREATE_OPERATOR, "CREATE OPERATOR", true, false, false) diff --git a/src/include/utils/sync_guc_name.h b/src/include/utils/sync_guc_name.h index dfb6e946bac..6c829ace83a 100644 --- a/src/include/utils/sync_guc_name.h +++ b/src/include/utils/sync_guc_name.h @@ -122,6 +122,8 @@ "gp_workfile_limit_per_query", "gp_write_shared_snapshot", "hash_mem_multiplier", + "iceberg_default_catalog", + "iceberg_default_volume", "ignore_system_indexes", "ignore_checksum_failure", "IntervalStyle", From a81bd6b626348b8b1bea760d51a6c8d6342306a0 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Thu, 2 Jul 2026 13:22:20 +0800 Subject: [PATCH 06/29] bin: pg_dump skip and psql tab completion for lake-table DDL pg_dump cannot reproduce lake tables (their data lives in external object storage managed through the iceberg access method's foreign catalog and volume), so skip them with a warning, matching how other unsupported access methods are handled. The access method is matched by name, not by a hardcoded OID. psql tab completion learns CREATE FOREIGN CATALOG/VOLUME ... SERVER ... OPTIONS, CREATE ICEBERG TABLE, DROP CATALOG/VOLUME with CASCADE/ RESTRICT, and completes foreign catalog and volume names after the CATALOG and VOLUME keywords. --- src/bin/pg_dump/pg_dump.c | 13 ++++++++++ src/bin/psql/tab-complete.c | 51 +++++++++++++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 7604417e262..1f091d26f83 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -2134,6 +2134,19 @@ selectDumpableTable(TableInfo *tbinfo, Archive *fout) pg_log_warning("unsupport am pax yet, current relation \"%s\" will be ignore", tbinfo->dobj.name); } + + /* + * Lake tables cannot be reproduced by pg_dump: their data lives in + * external object storage managed through the iceberg access method's + * foreign catalog and volume. Skip them. + */ + if (tbinfo->amname && strcmp(tbinfo->amname, "iceberg") == 0) + { + tbinfo->dobj.dump = DUMP_COMPONENT_NONE; + + pg_log_warning("iceberg table \"%s\" is not supported by pg_dump and will be ignored", + tbinfo->dobj.name); + } } /* diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 81a81578c0b..48903015c2f 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1077,6 +1077,16 @@ static const SchemaQuery Query_for_trigger_of_table = { " FROM pg_catalog.pg_foreign_server "\ " WHERE srvname LIKE '%s'" +#define Query_for_list_of_foreign_catalogs \ +" SELECT fcname "\ +" FROM pg_catalog.pg_foreign_catalog "\ +" WHERE fcname LIKE '%s'" + +#define Query_for_list_of_foreign_volumes \ +" SELECT fvname "\ +" FROM pg_catalog.pg_foreign_volume "\ +" WHERE fvname LIKE '%s'" + #define Query_for_list_of_user_mappings \ " SELECT usename "\ " FROM pg_catalog.pg_user_mappings "\ @@ -1257,6 +1267,7 @@ static const pgsql_thing_t words_after_create[] = { {"AGGREGATE", NULL, NULL, Query_for_list_of_aggregates}, {"CAST", NULL, NULL, NULL}, /* Casts have complex structures for names, so * skip it */ + {"CATALOG", Query_for_list_of_foreign_catalogs, NULL, NULL, NULL, THING_NO_CREATE | THING_NO_ALTER}, {"COLLATION", NULL, NULL, &Query_for_list_of_collations}, /* @@ -1274,10 +1285,13 @@ static const pgsql_thing_t words_after_create[] = { {"EVENT TRIGGER", NULL, NULL, NULL}, {"EXTENSION", Query_for_list_of_extensions}, {"EXTERNAL TABLE", NULL, NULL, NULL}, + {"FOREIGN CATALOG", NULL, NULL, NULL, NULL, THING_NO_DROP | THING_NO_ALTER}, {"FOREIGN DATA WRAPPER", NULL, NULL, NULL}, {"FOREIGN TABLE", NULL, NULL, NULL}, + {"FOREIGN VOLUME", NULL, NULL, NULL, NULL, THING_NO_DROP | THING_NO_ALTER}, {"FUNCTION", NULL, NULL, Query_for_list_of_functions}, {"GROUP", Query_for_list_of_roles}, + {"ICEBERG TABLE", NULL, NULL, NULL, NULL, THING_NO_DROP | THING_NO_ALTER}, {"INCREMENTAL MATERIALIZED VIEW", NULL, NULL, &Query_for_list_of_matviews, NULL, THING_NO_DROP | THING_NO_ALTER}, {"INDEX", NULL, NULL, &Query_for_list_of_indexes}, {"LANGUAGE", Query_for_list_of_languages}, @@ -1324,6 +1338,7 @@ static const pgsql_thing_t words_after_create[] = { {"USER MAPPING FOR", NULL, NULL, NULL}, {"STORAGE USER MAPPING FOR", NULL, NULL, NULL}, {"VIEW", NULL, NULL, &Query_for_list_of_views}, + {"VOLUME", Query_for_list_of_foreign_volumes, NULL, NULL, NULL, THING_NO_CREATE | THING_NO_ALTER}, {"WAREHOUSE", NULL}, {NULL} /* end of list */ }; @@ -3042,7 +3057,27 @@ psql_completion(const char *text, int start, int end) /* CREATE FOREIGN */ else if (Matches("CREATE", "FOREIGN")) - COMPLETE_WITH("DATA WRAPPER", "TABLE"); + COMPLETE_WITH("CATALOG", "DATA WRAPPER", "TABLE", "VOLUME"); + + /* CREATE FOREIGN CATALOG */ + else if (Matches("CREATE", "FOREIGN", "CATALOG", MatchAny)) + COMPLETE_WITH("SERVER"); + else if (Matches("CREATE", "FOREIGN", "CATALOG", MatchAny, "SERVER")) + COMPLETE_WITH_QUERY(Query_for_list_of_servers); + else if (Matches("CREATE", "FOREIGN", "CATALOG", MatchAny, "SERVER", MatchAny)) + COMPLETE_WITH("OPTIONS"); + + /* CREATE FOREIGN VOLUME */ + else if (Matches("CREATE", "FOREIGN", "VOLUME", MatchAny)) + COMPLETE_WITH("SERVER"); + else if (Matches("CREATE", "FOREIGN", "VOLUME", MatchAny, "SERVER")) + COMPLETE_WITH_QUERY(Query_for_list_of_servers); + else if (Matches("CREATE", "FOREIGN", "VOLUME", MatchAny, "SERVER", MatchAny)) + COMPLETE_WITH("OPTIONS"); + + /* CREATE ICEBERG */ + else if (Matches("CREATE", "ICEBERG")) + COMPLETE_WITH("TABLE"); /* CREATE FOREIGN DATA WRAPPER */ else if (Matches("CREATE", "FOREIGN", "DATA", "WRAPPER", MatchAny)) @@ -3801,7 +3836,7 @@ psql_completion(const char *text, int start, int end) /* DROP */ /* Complete DROP object with CASCADE / RESTRICT */ else if (Matches("DROP", - "COLLATION|CONVERSION|DOMAIN|EXTENSION|LANGUAGE|PUBLICATION|SCHEMA|SEQUENCE|SERVER|SUBSCRIPTION|STATISTICS|TABLE|TYPE|VIEW", + "CATALOG|COLLATION|CONVERSION|DOMAIN|EXTENSION|LANGUAGE|PUBLICATION|SCHEMA|SEQUENCE|SERVER|SUBSCRIPTION|STATISTICS|TABLE|TYPE|VIEW|VOLUME", MatchAny) || Matches("DROP", "ACCESS", "METHOD", MatchAny) || (Matches("DROP", "AGGREGATE|FUNCTION|PROCEDURE|ROUTINE", MatchAny, MatchAny) && @@ -4040,6 +4075,18 @@ psql_completion(const char *text, int start, int end) else if (TailMatches("FOREIGN", "SERVER")) COMPLETE_WITH_QUERY(Query_for_list_of_servers); +/* CATALOG, e.g. the CATALOG clause of CREATE ICEBERG TABLE */ + else if (TailMatches("CATALOG") && + !TailMatches("CREATE", MatchAny, MatchAny) && + !TailMatches("FOREIGN", MatchAny)) + COMPLETE_WITH_QUERY(Query_for_list_of_foreign_catalogs); + +/* VOLUME, e.g. the VOLUME clause of CREATE ICEBERG TABLE */ + else if (TailMatches("VOLUME") && + !TailMatches("CREATE", MatchAny, MatchAny) && + !TailMatches("FOREIGN", MatchAny)) + COMPLETE_WITH_QUERY(Query_for_list_of_foreign_volumes); + /* STORAGE SERVER */ else if (TailMatches("ALTER", "STORAGE", "SERVER")) COMPLETE_WITH_QUERY(Query_for_list_of_storage_servers); From 73172e8933a70826d4eb66bd436487786364d3c0 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Thu, 2 Jul 2026 16:08:25 +0800 Subject: [PATCH 07/29] tests: add lake_table regression test, refresh catalog expecteds Add a lake_table test to the greenplum_schedule covering the new DDL end to end: CREATE/DROP FOREIGN CATALOG and FOREIGN VOLUME (duplicates, IF NOT EXISTS/IF EXISTS, missing server), segment dispatch, object descriptions, CREATE ICEBERG TABLE with explicit CATALOG/VOLUME clauses and via the iceberg_default_catalog/volume GUCs, the forced random distribution policy, every iceberg-AM misuse guard, ownership checks, and dependency behavior (RESTRICT errors, CASCADE, pg_lake_table cleanup on drop). The iceberg access method is simulated with a heap-backed CREATE ACCESS METHOD, since the kernel resolves it by name and the real AM comes from a datalake extension. Refresh expected output of tests that enumerate system catalogs for the three new ones: misc_sanity (pg_lake_table's toast-less varlena columns), sanity_check (catalog list), and oidjoins (BKI_LOOKUP references, including pg_lake_table.ltforeign_catalog pointing at pg_foreign_catalog). --- .../src/test/regress/expected/oidjoins.out | 7 + .../test/regress/expected/sanity_check.out | 3 + src/test/regress/expected/lake_table.out | 288 ++++++++++++++++++ src/test/regress/expected/oidjoins.out | 7 + src/test/regress/expected/sanity_check.out | 3 + src/test/regress/greenplum_schedule | 3 + src/test/regress/sql/lake_table.sql | 142 +++++++++ .../singlenode_regress/expected/oidjoins.out | 7 + .../expected/sanity_check.out | 3 + 9 files changed, 463 insertions(+) create mode 100644 src/test/regress/expected/lake_table.out create mode 100644 src/test/regress/sql/lake_table.sql diff --git a/contrib/pax_storage/src/test/regress/expected/oidjoins.out b/contrib/pax_storage/src/test/regress/expected/oidjoins.out index 19094e111dc..6dc91c68aec 100644 --- a/contrib/pax_storage/src/test/regress/expected/oidjoins.out +++ b/contrib/pax_storage/src/test/regress/expected/oidjoins.out @@ -235,6 +235,10 @@ NOTICE: checking pg_foreign_data_wrapper {fdwhandler} => pg_proc {oid} NOTICE: checking pg_foreign_data_wrapper {fdwvalidator} => pg_proc {oid} NOTICE: checking pg_foreign_server {srvowner} => pg_authid {oid} NOTICE: checking pg_foreign_server {srvfdw} => pg_foreign_data_wrapper {oid} +NOTICE: checking pg_foreign_catalog {fcowner} => pg_authid {oid} +NOTICE: checking pg_foreign_catalog {fcserver} => pg_foreign_server {oid} +NOTICE: checking pg_foreign_volume {fvowner} => pg_authid {oid} +NOTICE: checking pg_foreign_volume {fvserver} => pg_foreign_server {oid} NOTICE: checking pg_user_mapping {umuser} => pg_authid {oid} NOTICE: checking pg_user_mapping {umserver} => pg_foreign_server {oid} NOTICE: checking pg_compression {compconstructor} => pg_proc {oid} @@ -247,6 +251,9 @@ NOTICE: checking pg_foreign_table {ftrelid} => pg_class {oid} NOTICE: checking pg_foreign_table {ftserver} => pg_foreign_server {oid} NOTICE: checking pg_foreign_table_seg {ftsrelid} => pg_class {oid} NOTICE: checking pg_foreign_table_seg {ftsserver} => pg_foreign_server {oid} +NOTICE: checking pg_lake_table {ltrelid} => pg_class {oid} +NOTICE: checking pg_lake_table {ltforeign_catalog} => pg_foreign_catalog {oid} +NOTICE: checking pg_lake_table {ltforeign_volume} => pg_foreign_volume {oid} NOTICE: checking pg_policy {polrelid} => pg_class {oid} NOTICE: checking pg_policy {polroles} => pg_authid {oid} NOTICE: checking pg_default_acl {defaclrole} => pg_authid {oid} diff --git a/contrib/pax_storage/src/test/regress/expected/sanity_check.out b/contrib/pax_storage/src/test/regress/expected/sanity_check.out index b61eee481bd..2e36e3abd35 100644 --- a/contrib/pax_storage/src/test/regress/expected/sanity_check.out +++ b/contrib/pax_storage/src/test/regress/expected/sanity_check.out @@ -112,13 +112,16 @@ pg_enum|t pg_event_trigger|t pg_extension|t pg_extprotocol|t +pg_foreign_catalog|t pg_foreign_data_wrapper|t pg_foreign_server|t pg_foreign_table|t pg_foreign_table_seg|t +pg_foreign_volume|t pg_index|t pg_inherits|t pg_init_privs|t +pg_lake_table|t pg_language|t pg_largeobject|t pg_largeobject_metadata|t diff --git a/src/test/regress/expected/lake_table.out b/src/test/regress/expected/lake_table.out new file mode 100644 index 00000000000..a97e854c27c --- /dev/null +++ b/src/test/regress/expected/lake_table.out @@ -0,0 +1,288 @@ +-- +-- Test lake table DDL: FOREIGN CATALOG, FOREIGN VOLUME, ICEBERG TABLE +-- +-- Display the lake table catalogs +\d+ pg_foreign_catalog + Table "pg_catalog.pg_foreign_catalog" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +-----------+--------+-----------+----------+---------+----------+--------------+------------- + oid | oid | | not null | | plain | | + fcname | name | | not null | | plain | | + fcowner | oid | | not null | | plain | | + fcserver | oid | | not null | | plain | | + fcoptions | text[] | C | | | extended | | +Indexes: + "pg_foreign_catalog_oid_index" PRIMARY KEY, btree (oid) + "pg_foreign_catalog_name_index" UNIQUE CONSTRAINT, btree (fcname) + +\d+ pg_foreign_volume + Table "pg_catalog.pg_foreign_volume" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +-----------+--------+-----------+----------+---------+----------+--------------+------------- + oid | oid | | not null | | plain | | + fvname | name | | not null | | plain | | + fvowner | oid | | not null | | plain | | + fvserver | oid | | not null | | plain | | + fvoptions | text[] | C | | | extended | | +Indexes: + "pg_foreign_volume_oid_index" PRIMARY KEY, btree (oid) + "pg_foreign_volume_name_index" UNIQUE CONSTRAINT, btree (fvname) + +\d+ pg_lake_table + Table "pg_catalog.pg_lake_table" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +-------------------+--------+-----------+----------+---------+----------+--------------+------------- + ltrelid | oid | | not null | | plain | | + ltforeign_catalog | oid | | not null | | plain | | + ltforeign_volume | oid | | not null | | plain | | + lttable_type | text | C | | | extended | | + ltoptions | text[] | C | | | extended | | +Indexes: + "pg_lake_table_relid_index" PRIMARY KEY, btree (ltrelid) + +-- Setup: foreign servers for the catalogs and volumes to hang off +CREATE FOREIGN DATA WRAPPER lake_test_fdw; +CREATE SERVER lake_test_srv FOREIGN DATA WRAPPER lake_test_fdw; +CREATE SERVER lake_test_srv2 FOREIGN DATA WRAPPER lake_test_fdw; +-- CREATE FOREIGN CATALOG +CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv OPTIONS (type 'hive', uri 'thrift://localhost:9083'); +CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv; -- fail, duplicate +ERROR: foreign catalog "lake_test_cat" already exists +CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv; -- skip with notice +NOTICE: foreign catalog "lake_test_cat" already exists, skipping +-- catalog names are global: the same name on another server is still a duplicate +CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv2; -- fail, duplicate +ERROR: foreign catalog "lake_test_cat" already exists +CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv2; -- skip with notice +NOTICE: foreign catalog "lake_test_cat" already exists, skipping +CREATE FOREIGN CATALOG lake_test_bad SERVER no_such_server; -- fail, no server +ERROR: server "no_such_server" does not exist +SELECT fcname, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%' ORDER BY 1; + fcname | fcoptions +---------------+----------------------------------------- + lake_test_cat | {type=hive,uri=thrift://localhost:9083} +(1 row) + +-- CREATE FOREIGN VOLUME +CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv OPTIONS (path 's3://bucket/prefix'); +CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv; -- fail, duplicate +ERROR: foreign volume "lake_test_vol" already exists +CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv; -- skip with notice +NOTICE: foreign volume "lake_test_vol" already exists, skipping +-- volume names are global: the same name on another server is still a duplicate +CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv2; -- fail, duplicate +ERROR: foreign volume "lake_test_vol" already exists +CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv2; -- skip with notice +NOTICE: foreign volume "lake_test_vol" already exists, skipping +CREATE FOREIGN VOLUME lake_test_bad SERVER no_such_server; -- fail, no server +ERROR: server "no_such_server" does not exist +SELECT fvname, fvoptions FROM pg_foreign_volume WHERE fvname LIKE 'lake\_test%' ORDER BY 1; + fvname | fvoptions +---------------+--------------------------- + lake_test_vol | {path=s3://bucket/prefix} +(1 row) + +-- Object descriptions +SELECT pg_catalog.pg_describe_object('pg_foreign_catalog'::regclass, oid, 0) + FROM pg_foreign_catalog WHERE fcname = 'lake_test_cat'; + pg_describe_object +----------------------- + catalog lake_test_cat +(1 row) + +SELECT pg_catalog.pg_describe_object('pg_foreign_volume'::regclass, oid, 0) + FROM pg_foreign_volume WHERE fvname = 'lake_test_vol'; + pg_describe_object +---------------------- + volume lake_test_vol +(1 row) + +-- Catalog and volume rows are dispatched to all segments +SELECT count(DISTINCT gp_segment_id) > 1 AS on_all_segments + FROM gp_dist_random('pg_foreign_catalog') WHERE fcname = 'lake_test_cat'; + on_all_segments +----------------- + t +(1 row) + +SELECT count(DISTINCT gp_segment_id) > 1 AS on_all_segments + FROM gp_dist_random('pg_foreign_volume') WHERE fvname = 'lake_test_vol'; + on_all_segments +----------------- + t +(1 row) + +-- Without a provider extension there is no iceberg table AM +CREATE ICEBERG TABLE lake_test_t0 (a int) CATALOG lake_test_cat VOLUME lake_test_vol; -- fail with hint +ERROR: table access method "iceberg" does not exist +HINT: CREATE ICEBERG TABLE requires an extension that provides the "iceberg" table access method. +-- The default catalog/volume GUCs verify that the object exists +SET iceberg_default_catalog = 'no_such_catalog'; -- fail +ERROR: invalid value for parameter "iceberg_default_catalog": "no_such_catalog" +DETAIL: Foreign catalog "no_such_catalog" does not exist. +SET iceberg_default_volume = 'no_such_volume'; -- fail +ERROR: invalid value for parameter "iceberg_default_volume": "no_such_volume" +DETAIL: Foreign volume "no_such_volume" does not exist. +-- Simulate a datalake provider with a heap-backed iceberg AM +CREATE ACCESS METHOD iceberg TYPE TABLE HANDLER heap_tableam_handler; +-- CREATE ICEBERG TABLE with explicit catalog and volume +CREATE ICEBERG TABLE lake_test_t1 (a int, b text) CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (fileformat 'parquet'); +SELECT c.relname, lt.lttable_type, lt.ltoptions, fc.fcname, fv.fvname + FROM pg_lake_table lt + JOIN pg_class c ON c.oid = lt.ltrelid + JOIN pg_foreign_catalog fc ON fc.oid = lt.ltforeign_catalog + JOIN pg_foreign_volume fv ON fv.oid = lt.ltforeign_volume + ORDER BY 1; + relname | lttable_type | ltoptions | fcname | fvname +--------------+--------------+----------------------+---------------+--------------- + lake_test_t1 | ICEBERG | {fileformat=parquet} | lake_test_cat | lake_test_vol +(1 row) + +SELECT a.amname FROM pg_am a JOIN pg_class c ON c.relam = a.oid WHERE c.relname = 'lake_test_t1'; + amname +--------- + iceberg +(1 row) + +-- Lake tables are always DISTRIBUTED RANDOMLY (policytype 'p', no distkey) +SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_test_t1'::regclass; + policytype | distkey +------------+--------- + p | +(1 row) + +-- The (heap-backed) table is usable +INSERT INTO lake_test_t1 VALUES (1, 'x'), (2, 'y'); +SELECT count(*) FROM lake_test_t1; + count +------- + 2 +(1 row) + +-- Lake tables get a TOAST table like plain tables, so wide values work +SELECT reltoastrelid <> 0 AS has_toast FROM pg_class WHERE relname = 'lake_test_t1'; + has_toast +----------- + t +(1 row) + +INSERT INTO lake_test_t1 VALUES (3, repeat('x', 500000)); +SELECT a, length(b) FROM lake_test_t1 WHERE a = 3; + a | length +---+-------- + 3 | 500000 +(1 row) + +-- Catalog and volume are both required +CREATE ICEBERG TABLE lake_test_t2 (a int) VOLUME lake_test_vol; -- fail, no catalog +ERROR: no foreign catalog specified +HINT: Specify CATALOG in CREATE ICEBERG TABLE or set iceberg_default_catalog. +CREATE ICEBERG TABLE lake_test_t2 (a int) CATALOG lake_test_cat; -- fail, no volume +ERROR: no foreign volume specified +HINT: Specify VOLUME in CREATE ICEBERG TABLE or set iceberg_default_volume. +-- ... unless the GUCs provide defaults +SET iceberg_default_catalog = 'lake_test_cat'; +SET iceberg_default_volume = 'lake_test_vol'; +CREATE ICEBERG TABLE lake_test_t2 (a int); +RESET iceberg_default_catalog; +RESET iceberg_default_volume; +-- A DISTRIBUTED clause is ignored with a warning +CREATE ICEBERG TABLE lake_test_t3 (a int) CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); +WARNING: DISTRIBUTED clause has no effect for lake tables, using DISTRIBUTED RANDOMLY +SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_test_t3'::regclass; + policytype | distkey +------------+--------- + p | +(1 row) + +-- CREATE ICEBERG TABLE only accepts the iceberg access method +CREATE ICEBERG TABLE lake_test_bad0 (a int) CATALOG lake_test_cat VOLUME lake_test_vol USING heap; -- fail +ERROR: access method "heap" is not supported for iceberg tables +HINT: Omit the USING clause; CREATE ICEBERG TABLE always uses the "iceberg" access method. +CREATE ICEBERG TABLE lake_test_t4 (a int) CATALOG lake_test_cat VOLUME lake_test_vol USING iceberg; -- explicit iceberg is fine +-- The iceberg AM is rejected for every path other than CREATE ICEBERG TABLE +CREATE TABLE lake_test_bad1 (a int) USING iceberg DISTRIBUTED RANDOMLY; -- fail +ERROR: cannot create table "lake_test_bad1" with access method "iceberg" +HINT: Use CREATE ICEBERG TABLE instead. +CREATE TABLE lake_test_bad2 USING iceberg AS SELECT 1 AS a DISTRIBUTED RANDOMLY; -- fail +ERROR: cannot create table "lake_test_bad2" with access method "iceberg" +HINT: Use CREATE ICEBERG TABLE instead. +SET default_table_access_method = iceberg; +CREATE TABLE lake_test_bad3 (a int) DISTRIBUTED RANDOMLY; -- fail +ERROR: cannot create table "lake_test_bad3" with access method "iceberg" +HINT: Use CREATE ICEBERG TABLE instead. +RESET default_table_access_method; +CREATE TABLE lake_test_heap (a int) DISTRIBUTED RANDOMLY; +ALTER TABLE lake_test_heap SET ACCESS METHOD iceberg; -- fail +ERROR: cannot change access method of table "lake_test_heap" to "iceberg" +HINT: Use CREATE ICEBERG TABLE to create an iceberg table. +ALTER TABLE lake_test_t1 SET ACCESS METHOD heap; -- fail +ERROR: cannot change access method of lake table "lake_test_t1" +ALTER TABLE lake_test_t1 SET DISTRIBUTED BY (a); -- fail +ERROR: cannot change distribution policy of lake table "lake_test_t1" +HINT: Lake tables must use DISTRIBUTED RANDOMLY because data is stored on object storage. +-- Only the owner can drop a catalog or volume +CREATE ROLE regress_lake_user; +NOTICE: resource queue required -- using default resource queue "pg_default" +SET ROLE regress_lake_user; +DROP CATALOG lake_test_cat; -- fail, not owner +ERROR: must be owner of foreign catalog lake_test_cat +DROP VOLUME lake_test_vol; -- fail, not owner +ERROR: must be owner of foreign volume lake_test_vol +RESET ROLE; +-- Dependencies: the server holds the catalog/volume, which hold the tables +DROP SERVER lake_test_srv; -- fail, catalog and volume depend on it +ERROR: cannot drop server lake_test_srv because other objects depend on it +DETAIL: catalog lake_test_cat depends on server lake_test_srv +volume lake_test_vol depends on server lake_test_srv +table lake_test_t1 depends on volume lake_test_vol +table lake_test_t2 depends on volume lake_test_vol +table lake_test_t3 depends on volume lake_test_vol +table lake_test_t4 depends on volume lake_test_vol +HINT: Use DROP ... CASCADE to drop the dependent objects too. +DROP CATALOG lake_test_cat; -- fail, tables depend on it +ERROR: cannot drop catalog lake_test_cat because other objects depend on it +DETAIL: table lake_test_t1 depends on catalog lake_test_cat +table lake_test_t2 depends on catalog lake_test_cat +table lake_test_t3 depends on catalog lake_test_cat +table lake_test_t4 depends on catalog lake_test_cat +HINT: Use DROP ... CASCADE to drop the dependent objects too. +-- Dropping a lake table removes its pg_lake_table entry +DROP TABLE lake_test_t1; +SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid + WHERE c.relname = 'lake_test_t1'; + count +------- + 0 +(1 row) + +-- DROP CATALOG ... CASCADE takes the remaining tables with it +DROP CATALOG lake_test_cat CASCADE; +NOTICE: drop cascades to 3 other objects +DETAIL: drop cascades to table lake_test_t2 +drop cascades to table lake_test_t3 +drop cascades to table lake_test_t4 +SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid + WHERE c.relname LIKE 'lake\_test%'; + count +------- + 0 +(1 row) + +-- DROP variants +DROP CATALOG lake_test_cat; -- fail, already gone +ERROR: foreign catalog "lake_test_cat" does not exist +DROP CATALOG IF EXISTS lake_test_cat; -- skip with notice +NOTICE: foreign catalog "lake_test_cat" does not exist, skipping +DROP VOLUME lake_test_vol; +DROP VOLUME lake_test_vol; -- fail, already gone +ERROR: foreign volume "lake_test_vol" does not exist +DROP VOLUME IF EXISTS lake_test_vol; -- skip with notice +NOTICE: foreign volume "lake_test_vol" does not exist, skipping +-- Cleanup +DROP TABLE lake_test_heap; +DROP ROLE regress_lake_user; +DROP SERVER lake_test_srv; +DROP SERVER lake_test_srv2; +DROP FOREIGN DATA WRAPPER lake_test_fdw; +DROP ACCESS METHOD iceberg; diff --git a/src/test/regress/expected/oidjoins.out b/src/test/regress/expected/oidjoins.out index 19094e111dc..6dc91c68aec 100644 --- a/src/test/regress/expected/oidjoins.out +++ b/src/test/regress/expected/oidjoins.out @@ -235,6 +235,10 @@ NOTICE: checking pg_foreign_data_wrapper {fdwhandler} => pg_proc {oid} NOTICE: checking pg_foreign_data_wrapper {fdwvalidator} => pg_proc {oid} NOTICE: checking pg_foreign_server {srvowner} => pg_authid {oid} NOTICE: checking pg_foreign_server {srvfdw} => pg_foreign_data_wrapper {oid} +NOTICE: checking pg_foreign_catalog {fcowner} => pg_authid {oid} +NOTICE: checking pg_foreign_catalog {fcserver} => pg_foreign_server {oid} +NOTICE: checking pg_foreign_volume {fvowner} => pg_authid {oid} +NOTICE: checking pg_foreign_volume {fvserver} => pg_foreign_server {oid} NOTICE: checking pg_user_mapping {umuser} => pg_authid {oid} NOTICE: checking pg_user_mapping {umserver} => pg_foreign_server {oid} NOTICE: checking pg_compression {compconstructor} => pg_proc {oid} @@ -247,6 +251,9 @@ NOTICE: checking pg_foreign_table {ftrelid} => pg_class {oid} NOTICE: checking pg_foreign_table {ftserver} => pg_foreign_server {oid} NOTICE: checking pg_foreign_table_seg {ftsrelid} => pg_class {oid} NOTICE: checking pg_foreign_table_seg {ftsserver} => pg_foreign_server {oid} +NOTICE: checking pg_lake_table {ltrelid} => pg_class {oid} +NOTICE: checking pg_lake_table {ltforeign_catalog} => pg_foreign_catalog {oid} +NOTICE: checking pg_lake_table {ltforeign_volume} => pg_foreign_volume {oid} NOTICE: checking pg_policy {polrelid} => pg_class {oid} NOTICE: checking pg_policy {polroles} => pg_authid {oid} NOTICE: checking pg_default_acl {defaclrole} => pg_authid {oid} diff --git a/src/test/regress/expected/sanity_check.out b/src/test/regress/expected/sanity_check.out index 4625d100bf0..f5d55362906 100644 --- a/src/test/regress/expected/sanity_check.out +++ b/src/test/regress/expected/sanity_check.out @@ -124,13 +124,16 @@ pg_enum|t pg_event_trigger|t pg_extension|t pg_extprotocol|t +pg_foreign_catalog|t pg_foreign_data_wrapper|t pg_foreign_server|t pg_foreign_table|t pg_foreign_table_seg|t +pg_foreign_volume|t pg_index|t pg_inherits|t pg_init_privs|t +pg_lake_table|t pg_language|t pg_largeobject|t pg_largeobject_metadata|t diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index 84e8766844b..44a916cc218 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -358,6 +358,9 @@ test: am_encoding # tests of directory table test: directory_table +# tests of lake table DDL (foreign catalog, foreign volume, iceberg table) +test: lake_table + # test if motion sockets are created with the gp_segment_configuration.address test: motion_socket diff --git a/src/test/regress/sql/lake_table.sql b/src/test/regress/sql/lake_table.sql new file mode 100644 index 00000000000..f9683baf604 --- /dev/null +++ b/src/test/regress/sql/lake_table.sql @@ -0,0 +1,142 @@ +-- +-- Test lake table DDL: FOREIGN CATALOG, FOREIGN VOLUME, ICEBERG TABLE +-- + +-- Display the lake table catalogs +\d+ pg_foreign_catalog +\d+ pg_foreign_volume +\d+ pg_lake_table + +-- Setup: foreign servers for the catalogs and volumes to hang off +CREATE FOREIGN DATA WRAPPER lake_test_fdw; +CREATE SERVER lake_test_srv FOREIGN DATA WRAPPER lake_test_fdw; +CREATE SERVER lake_test_srv2 FOREIGN DATA WRAPPER lake_test_fdw; + +-- CREATE FOREIGN CATALOG +CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv OPTIONS (type 'hive', uri 'thrift://localhost:9083'); +CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv; -- fail, duplicate +CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv; -- skip with notice +-- catalog names are global: the same name on another server is still a duplicate +CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv2; -- fail, duplicate +CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv2; -- skip with notice +CREATE FOREIGN CATALOG lake_test_bad SERVER no_such_server; -- fail, no server +SELECT fcname, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%' ORDER BY 1; + +-- CREATE FOREIGN VOLUME +CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv OPTIONS (path 's3://bucket/prefix'); +CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv; -- fail, duplicate +CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv; -- skip with notice +-- volume names are global: the same name on another server is still a duplicate +CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv2; -- fail, duplicate +CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv2; -- skip with notice +CREATE FOREIGN VOLUME lake_test_bad SERVER no_such_server; -- fail, no server +SELECT fvname, fvoptions FROM pg_foreign_volume WHERE fvname LIKE 'lake\_test%' ORDER BY 1; + +-- Object descriptions +SELECT pg_catalog.pg_describe_object('pg_foreign_catalog'::regclass, oid, 0) + FROM pg_foreign_catalog WHERE fcname = 'lake_test_cat'; +SELECT pg_catalog.pg_describe_object('pg_foreign_volume'::regclass, oid, 0) + FROM pg_foreign_volume WHERE fvname = 'lake_test_vol'; + +-- Catalog and volume rows are dispatched to all segments +SELECT count(DISTINCT gp_segment_id) > 1 AS on_all_segments + FROM gp_dist_random('pg_foreign_catalog') WHERE fcname = 'lake_test_cat'; +SELECT count(DISTINCT gp_segment_id) > 1 AS on_all_segments + FROM gp_dist_random('pg_foreign_volume') WHERE fvname = 'lake_test_vol'; + +-- Without a provider extension there is no iceberg table AM +CREATE ICEBERG TABLE lake_test_t0 (a int) CATALOG lake_test_cat VOLUME lake_test_vol; -- fail with hint + +-- The default catalog/volume GUCs verify that the object exists +SET iceberg_default_catalog = 'no_such_catalog'; -- fail +SET iceberg_default_volume = 'no_such_volume'; -- fail + +-- Simulate a datalake provider with a heap-backed iceberg AM +CREATE ACCESS METHOD iceberg TYPE TABLE HANDLER heap_tableam_handler; + +-- CREATE ICEBERG TABLE with explicit catalog and volume +CREATE ICEBERG TABLE lake_test_t1 (a int, b text) CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (fileformat 'parquet'); +SELECT c.relname, lt.lttable_type, lt.ltoptions, fc.fcname, fv.fvname + FROM pg_lake_table lt + JOIN pg_class c ON c.oid = lt.ltrelid + JOIN pg_foreign_catalog fc ON fc.oid = lt.ltforeign_catalog + JOIN pg_foreign_volume fv ON fv.oid = lt.ltforeign_volume + ORDER BY 1; +SELECT a.amname FROM pg_am a JOIN pg_class c ON c.relam = a.oid WHERE c.relname = 'lake_test_t1'; +-- Lake tables are always DISTRIBUTED RANDOMLY (policytype 'p', no distkey) +SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_test_t1'::regclass; + +-- The (heap-backed) table is usable +INSERT INTO lake_test_t1 VALUES (1, 'x'), (2, 'y'); +SELECT count(*) FROM lake_test_t1; + +-- Lake tables get a TOAST table like plain tables, so wide values work +SELECT reltoastrelid <> 0 AS has_toast FROM pg_class WHERE relname = 'lake_test_t1'; +INSERT INTO lake_test_t1 VALUES (3, repeat('x', 500000)); +SELECT a, length(b) FROM lake_test_t1 WHERE a = 3; + +-- Catalog and volume are both required +CREATE ICEBERG TABLE lake_test_t2 (a int) VOLUME lake_test_vol; -- fail, no catalog +CREATE ICEBERG TABLE lake_test_t2 (a int) CATALOG lake_test_cat; -- fail, no volume + +-- ... unless the GUCs provide defaults +SET iceberg_default_catalog = 'lake_test_cat'; +SET iceberg_default_volume = 'lake_test_vol'; +CREATE ICEBERG TABLE lake_test_t2 (a int); +RESET iceberg_default_catalog; +RESET iceberg_default_volume; + +-- A DISTRIBUTED clause is ignored with a warning +CREATE ICEBERG TABLE lake_test_t3 (a int) CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); +SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_test_t3'::regclass; + +-- CREATE ICEBERG TABLE only accepts the iceberg access method +CREATE ICEBERG TABLE lake_test_bad0 (a int) CATALOG lake_test_cat VOLUME lake_test_vol USING heap; -- fail +CREATE ICEBERG TABLE lake_test_t4 (a int) CATALOG lake_test_cat VOLUME lake_test_vol USING iceberg; -- explicit iceberg is fine + +-- The iceberg AM is rejected for every path other than CREATE ICEBERG TABLE +CREATE TABLE lake_test_bad1 (a int) USING iceberg DISTRIBUTED RANDOMLY; -- fail +CREATE TABLE lake_test_bad2 USING iceberg AS SELECT 1 AS a DISTRIBUTED RANDOMLY; -- fail +SET default_table_access_method = iceberg; +CREATE TABLE lake_test_bad3 (a int) DISTRIBUTED RANDOMLY; -- fail +RESET default_table_access_method; +CREATE TABLE lake_test_heap (a int) DISTRIBUTED RANDOMLY; +ALTER TABLE lake_test_heap SET ACCESS METHOD iceberg; -- fail +ALTER TABLE lake_test_t1 SET ACCESS METHOD heap; -- fail +ALTER TABLE lake_test_t1 SET DISTRIBUTED BY (a); -- fail + +-- Only the owner can drop a catalog or volume +CREATE ROLE regress_lake_user; +SET ROLE regress_lake_user; +DROP CATALOG lake_test_cat; -- fail, not owner +DROP VOLUME lake_test_vol; -- fail, not owner +RESET ROLE; + +-- Dependencies: the server holds the catalog/volume, which hold the tables +DROP SERVER lake_test_srv; -- fail, catalog and volume depend on it +DROP CATALOG lake_test_cat; -- fail, tables depend on it + +-- Dropping a lake table removes its pg_lake_table entry +DROP TABLE lake_test_t1; +SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid + WHERE c.relname = 'lake_test_t1'; + +-- DROP CATALOG ... CASCADE takes the remaining tables with it +DROP CATALOG lake_test_cat CASCADE; +SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid + WHERE c.relname LIKE 'lake\_test%'; + +-- DROP variants +DROP CATALOG lake_test_cat; -- fail, already gone +DROP CATALOG IF EXISTS lake_test_cat; -- skip with notice +DROP VOLUME lake_test_vol; +DROP VOLUME lake_test_vol; -- fail, already gone +DROP VOLUME IF EXISTS lake_test_vol; -- skip with notice + +-- Cleanup +DROP TABLE lake_test_heap; +DROP ROLE regress_lake_user; +DROP SERVER lake_test_srv; +DROP SERVER lake_test_srv2; +DROP FOREIGN DATA WRAPPER lake_test_fdw; +DROP ACCESS METHOD iceberg; diff --git a/src/test/singlenode_regress/expected/oidjoins.out b/src/test/singlenode_regress/expected/oidjoins.out index b1dca18dc97..b6bb1a499ce 100644 --- a/src/test/singlenode_regress/expected/oidjoins.out +++ b/src/test/singlenode_regress/expected/oidjoins.out @@ -235,6 +235,10 @@ NOTICE: checking pg_foreign_data_wrapper {fdwhandler} => pg_proc {oid} NOTICE: checking pg_foreign_data_wrapper {fdwvalidator} => pg_proc {oid} NOTICE: checking pg_foreign_server {srvowner} => pg_authid {oid} NOTICE: checking pg_foreign_server {srvfdw} => pg_foreign_data_wrapper {oid} +NOTICE: checking pg_foreign_catalog {fcowner} => pg_authid {oid} +NOTICE: checking pg_foreign_catalog {fcserver} => pg_foreign_server {oid} +NOTICE: checking pg_foreign_volume {fvowner} => pg_authid {oid} +NOTICE: checking pg_foreign_volume {fvserver} => pg_foreign_server {oid} NOTICE: checking pg_user_mapping {umuser} => pg_authid {oid} NOTICE: checking pg_user_mapping {umserver} => pg_foreign_server {oid} NOTICE: checking pg_compression {compconstructor} => pg_proc {oid} @@ -247,6 +251,9 @@ NOTICE: checking pg_foreign_table {ftrelid} => pg_class {oid} NOTICE: checking pg_foreign_table {ftserver} => pg_foreign_server {oid} NOTICE: checking pg_foreign_table_seg {ftsrelid} => pg_class {oid} NOTICE: checking pg_foreign_table_seg {ftsserver} => pg_foreign_server {oid} +NOTICE: checking pg_lake_table {ltrelid} => pg_class {oid} +NOTICE: checking pg_lake_table {ltforeign_catalog} => pg_foreign_catalog {oid} +NOTICE: checking pg_lake_table {ltforeign_volume} => pg_foreign_volume {oid} NOTICE: checking pg_policy {polrelid} => pg_class {oid} NOTICE: checking pg_policy {polroles} => pg_authid {oid} NOTICE: checking pg_default_acl {defaclrole} => pg_authid {oid} diff --git a/src/test/singlenode_regress/expected/sanity_check.out b/src/test/singlenode_regress/expected/sanity_check.out index a5a122ac32d..844f094fa57 100644 --- a/src/test/singlenode_regress/expected/sanity_check.out +++ b/src/test/singlenode_regress/expected/sanity_check.out @@ -135,13 +135,16 @@ pg_enum|t pg_event_trigger|t pg_extension|t pg_extprotocol|t +pg_foreign_catalog|t pg_foreign_data_wrapper|t pg_foreign_server|t pg_foreign_table|t pg_foreign_table_seg|t +pg_foreign_volume|t pg_index|t pg_inherits|t pg_init_privs|t +pg_lake_table|t pg_language|t pg_largeobject|t pg_largeobject_metadata|t From 5be5f9ffd3515bcb7e376faf1ae9260212cdf201 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Mon, 6 Jul 2026 17:50:11 +0800 Subject: [PATCH 08/29] Use ASF license header for new lake-table source files Address review: new community-authored files should carry the standard Apache-2.0 header instead of the PostgreSQL boilerplate. --- src/backend/commands/laketablecmds.c | 16 +++++++++++++++- src/include/catalog/pg_foreign_catalog.h | 18 ++++++++++++++++-- src/include/catalog/pg_foreign_volume.h | 18 ++++++++++++++++-- src/include/catalog/pg_lake_table.h | 17 +++++++++++++++-- src/include/commands/laketablecmds.h | 18 ++++++++++++++++-- 5 files changed, 78 insertions(+), 9 deletions(-) diff --git a/src/backend/commands/laketablecmds.c b/src/backend/commands/laketablecmds.c index 80771f0b1aa..02037e39baa 100644 --- a/src/backend/commands/laketablecmds.c +++ b/src/backend/commands/laketablecmds.c @@ -3,8 +3,22 @@ * laketablecmds.c * lake table creation/manipulation commands * - * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. * * IDENTIFICATION * src/backend/commands/laketablecmds.c diff --git a/src/include/catalog/pg_foreign_catalog.h b/src/include/catalog/pg_foreign_catalog.h index 7623cd9e807..1f3df631447 100644 --- a/src/include/catalog/pg_foreign_catalog.h +++ b/src/include/catalog/pg_foreign_catalog.h @@ -3,8 +3,22 @@ * pg_foreign_catalog.h * definition of the "foreign catalog" system catalog (pg_foreign_catalog) * - * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. * * src/include/catalog/pg_foreign_catalog.h * diff --git a/src/include/catalog/pg_foreign_volume.h b/src/include/catalog/pg_foreign_volume.h index 669bb1246d0..ab44d6a0a18 100644 --- a/src/include/catalog/pg_foreign_volume.h +++ b/src/include/catalog/pg_foreign_volume.h @@ -3,8 +3,22 @@ * pg_foreign_volume.h * definition of the "foreign volume" system catalog (pg_foreign_volume) * - * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. * * src/include/catalog/pg_foreign_volume.h * diff --git a/src/include/catalog/pg_lake_table.h b/src/include/catalog/pg_lake_table.h index 37c8a208d31..7d6da077ce3 100644 --- a/src/include/catalog/pg_lake_table.h +++ b/src/include/catalog/pg_lake_table.h @@ -3,9 +3,22 @@ * pg_lake_table.h * definition of the "lake table" system catalog (pg_lake_table) * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. * * src/include/catalog/pg_lake_table.h * diff --git a/src/include/commands/laketablecmds.h b/src/include/commands/laketablecmds.h index 05564a79a98..98904902073 100644 --- a/src/include/commands/laketablecmds.h +++ b/src/include/commands/laketablecmds.h @@ -3,8 +3,22 @@ * laketablecmds.h * prototypes for laketablecmds.c. * - * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. * * src/include/commands/laketablecmds.h * From 272d910f26107d1e328d3fe49596faef0d093056 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Mon, 6 Jul 2026 17:58:21 +0800 Subject: [PATCH 09/29] Move ASF license header to top of file header block Match the community convention (see contrib/pax_storage pax_gbench.cc): the Apache-2.0 license block comes first, followed by the file name and IDENTIFICATION. --- src/backend/commands/laketablecmds.c | 6 +++--- src/include/catalog/pg_foreign_catalog.h | 6 +++--- src/include/catalog/pg_foreign_volume.h | 6 +++--- src/include/catalog/pg_lake_table.h | 6 +++--- src/include/commands/laketablecmds.h | 6 +++--- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/backend/commands/laketablecmds.c b/src/backend/commands/laketablecmds.c index 02037e39baa..90cfb6fa4b5 100644 --- a/src/backend/commands/laketablecmds.c +++ b/src/backend/commands/laketablecmds.c @@ -1,7 +1,4 @@ /*------------------------------------------------------------------------- - * - * laketablecmds.c - * lake table creation/manipulation commands * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -20,6 +17,9 @@ * specific language governing permissions and limitations * under the License. * + * laketablecmds.c + * lake table creation/manipulation commands + * * IDENTIFICATION * src/backend/commands/laketablecmds.c * diff --git a/src/include/catalog/pg_foreign_catalog.h b/src/include/catalog/pg_foreign_catalog.h index 1f3df631447..b6325915b22 100644 --- a/src/include/catalog/pg_foreign_catalog.h +++ b/src/include/catalog/pg_foreign_catalog.h @@ -1,7 +1,4 @@ /*------------------------------------------------------------------------- - * - * pg_foreign_catalog.h - * definition of the "foreign catalog" system catalog (pg_foreign_catalog) * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -20,6 +17,9 @@ * specific language governing permissions and limitations * under the License. * + * pg_foreign_catalog.h + * definition of the "foreign catalog" system catalog (pg_foreign_catalog) + * * src/include/catalog/pg_foreign_catalog.h * * NOTES diff --git a/src/include/catalog/pg_foreign_volume.h b/src/include/catalog/pg_foreign_volume.h index ab44d6a0a18..d3e377f1939 100644 --- a/src/include/catalog/pg_foreign_volume.h +++ b/src/include/catalog/pg_foreign_volume.h @@ -1,7 +1,4 @@ /*------------------------------------------------------------------------- - * - * pg_foreign_volume.h - * definition of the "foreign volume" system catalog (pg_foreign_volume) * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -20,6 +17,9 @@ * specific language governing permissions and limitations * under the License. * + * pg_foreign_volume.h + * definition of the "foreign volume" system catalog (pg_foreign_volume) + * * src/include/catalog/pg_foreign_volume.h * * NOTES diff --git a/src/include/catalog/pg_lake_table.h b/src/include/catalog/pg_lake_table.h index 7d6da077ce3..d22ce5a077f 100644 --- a/src/include/catalog/pg_lake_table.h +++ b/src/include/catalog/pg_lake_table.h @@ -1,7 +1,4 @@ /*------------------------------------------------------------------------- - * - * pg_lake_table.h - * definition of the "lake table" system catalog (pg_lake_table) * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -20,6 +17,9 @@ * specific language governing permissions and limitations * under the License. * + * pg_lake_table.h + * definition of the "lake table" system catalog (pg_lake_table) + * * src/include/catalog/pg_lake_table.h * * NOTES diff --git a/src/include/commands/laketablecmds.h b/src/include/commands/laketablecmds.h index 98904902073..3b7b949a7fa 100644 --- a/src/include/commands/laketablecmds.h +++ b/src/include/commands/laketablecmds.h @@ -1,7 +1,4 @@ /*------------------------------------------------------------------------- - * - * laketablecmds.h - * prototypes for laketablecmds.c. * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file @@ -20,6 +17,9 @@ * specific language governing permissions and limitations * under the License. * + * laketablecmds.h + * prototypes for laketablecmds.c. + * * src/include/commands/laketablecmds.h * *------------------------------------------------------------------------- From c70fbf9ac729f75b6cb8d736f2a88b7a031aecea Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Tue, 7 Jul 2026 15:38:13 +0800 Subject: [PATCH 10/29] foreigncmds: flatten IF NOT EXISTS duplicate handling Address review: invert the if_not_exists check and error out early so the skip path is no longer nested in an else branch, in both CreateForeignCatalog and CreateForeignVolume. --- src/backend/commands/foreigncmds.c | 68 ++++++++++++++---------------- 1 file changed, 32 insertions(+), 36 deletions(-) diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index c9a1e53787e..b5e6adc95f1 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -1061,28 +1061,26 @@ CreateForeignCatalog(CreateForeignCatalogStmt *stmt) catalogId = get_foreign_catalog_oid(stmt->catalogname, true); if (OidIsValid(catalogId)) { - if (stmt->if_not_exists) - { - /* - * If we are in an extension script, insist that the pre-existing - * object be a member of the extension, to avoid security risks. - */ - ObjectAddressSet(myself, ForeignCatalogRelationId, catalogId); - checkMembershipInCurrentExtension(&myself); - - /* OK to skip */ - ereport(NOTICE, - (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("foreign catalog \"%s\" already exists, skipping", - stmt->catalogname))); - table_close(rel, RowExclusiveLock); - return InvalidObjectAddress; - } - else + if (!stmt->if_not_exists) ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("foreign catalog \"%s\" already exists", stmt->catalogname))); + + /* + * If we are in an extension script, insist that the pre-existing + * object be a member of the extension, to avoid security risks. + */ + ObjectAddressSet(myself, ForeignCatalogRelationId, catalogId); + checkMembershipInCurrentExtension(&myself); + + /* OK to skip */ + ereport(NOTICE, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("foreign catalog \"%s\" already exists, skipping", + stmt->catalogname))); + table_close(rel, RowExclusiveLock); + return InvalidObjectAddress; } /* @@ -1191,28 +1189,26 @@ CreateForeignVolume(CreateForeignVolumeStmt *stmt) volumeId = get_foreign_volume_oid(stmt->volumename, true); if (OidIsValid(volumeId)) { - if (stmt->if_not_exists) - { - /* - * If we are in an extension script, insist that the pre-existing - * object be a member of the extension, to avoid security risks. - */ - ObjectAddressSet(myself, ForeignVolumeRelationId, volumeId); - checkMembershipInCurrentExtension(&myself); - - /* OK to skip */ - ereport(NOTICE, - (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("foreign volume \"%s\" already exists, skipping", - stmt->volumename))); - table_close(rel, RowExclusiveLock); - return InvalidObjectAddress; - } - else + if (!stmt->if_not_exists) ereport(ERROR, (errcode(ERRCODE_DUPLICATE_OBJECT), errmsg("foreign volume \"%s\" already exists", stmt->volumename))); + + /* + * If we are in an extension script, insist that the pre-existing + * object be a member of the extension, to avoid security risks. + */ + ObjectAddressSet(myself, ForeignVolumeRelationId, volumeId); + checkMembershipInCurrentExtension(&myself); + + /* OK to skip */ + ereport(NOTICE, + (errcode(ERRCODE_DUPLICATE_OBJECT), + errmsg("foreign volume \"%s\" already exists, skipping", + stmt->volumename))); + table_close(rel, RowExclusiveLock); + return InvalidObjectAddress; } /* From 5105d024172b5eb674ca5fdf97f301dfca18433b Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Fri, 10 Jul 2026 10:12:02 +0800 Subject: [PATCH 11/29] parser/commands: unify lake-table DROP syntax with CREATE Mirror the CREATE side so DROP matches: - DROP CATALOG / DROP VOLUME -> DROP FOREIGN CATALOG / DROP FOREIGN VOLUME (via drop_type_name, same FOREIGN prefix as FOREIGN DATA WRAPPER) - add DROP ICEBERG TABLE [IF EXISTS] as the pair for CREATE ICEBERG TABLE DROP ICEBERG TABLE carries a new DropStmt.isiceberg flag and validates in RangeVarCallbackForDropRelation that the target actually uses the iceberg access method, erroring '"%s" is not an iceberg table' otherwise (mirrors DROP FOREIGN TABLE). Plain DROP TABLE still removes an iceberg table. Adds CMDTAG_DROP_LAKE_TABLE, psql tab completion for the new forms, and refreshes the lake_table regression with new and negative cases. Addresses review feedback on PR #1842. --- src/backend/commands/tablecmds.c | 20 ++++++++++++ src/backend/nodes/copyfuncs.funcs.c | 2 ++ src/backend/nodes/equalfuncs.c | 1 + src/backend/nodes/outfuncs_common.c | 2 ++ src/backend/nodes/readfuncs_common.c | 2 ++ src/backend/parser/gram.y | 29 +++++++++++++++-- src/backend/tcop/utility.c | 5 ++- src/bin/psql/tab-complete.c | 22 ++++++++----- src/include/nodes/parsenodes.h | 1 + src/include/tcop/cmdtaglist.h | 1 + src/test/regress/expected/lake_table.out | 40 +++++++++++++++--------- src/test/regress/sql/lake_table.sql | 29 ++++++++++------- 12 files changed, 119 insertions(+), 35 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 3786ca44e1a..cb63d10a8fa 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -256,6 +256,7 @@ struct DropRelationCallbackState { /* These fields are set by RemoveRelations: */ char expected_relkind; + bool iceberg_only; /* DROP ICEBERG TABLE: require iceberg AM */ LOCKMODE heap_lockmode; /* These fields are state to track which subsidiary locks are held: */ Oid heapOid; @@ -1998,6 +1999,7 @@ RemoveRelations(DropStmt *drop) /* Look up the appropriate relation using namespace search. */ state.expected_relkind = relkind; + state.iceberg_only = drop->isiceberg; state.heap_lockmode = drop->concurrent ? ShareUpdateExclusiveLock : AccessExclusiveLock; /* We must initialize these fields to show that no locks are held: */ @@ -2219,6 +2221,24 @@ RangeVarCallbackForDropRelation(const RangeVar *rel, Oid relOid, Oid oldRelOid, DropErrorMsgWrongType(rel->relname, classform->relkind, state->expected_relkind); + /* + * DROP ICEBERG TABLE must target an iceberg (lake) table. Iceberg tables + * share RELKIND_RELATION with ordinary tables, so the relkind check above + * cannot tell them apart; verify the access method here (same rule as + * RelationIsIcebergTable). If no provider installed the iceberg AM, no + * relation can be an iceberg table, so this always rejects. + */ + if (state->iceberg_only) + { + Oid iceberg_amoid = GetIcebergTableAmOid(true); + + if (!OidIsValid(iceberg_amoid) || classform->relam != iceberg_amoid) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not an iceberg table", rel->relname), + errhint("Use DROP TABLE to remove a table."))); + } + /* Allow DROP to either table owner or schema owner */ if (!object_ownercheck(RelationRelationId, relOid, GetUserId()) && !object_ownercheck(NamespaceRelationId, classform->relnamespace, GetUserId())) diff --git a/src/backend/nodes/copyfuncs.funcs.c b/src/backend/nodes/copyfuncs.funcs.c index 2b7cd7cfe3e..81fbaa2330b 100644 --- a/src/backend/nodes/copyfuncs.funcs.c +++ b/src/backend/nodes/copyfuncs.funcs.c @@ -3446,6 +3446,7 @@ _copyDropStmt(const DropStmt *from) COPY_SCALAR_FIELD(missing_ok); COPY_SCALAR_FIELD(concurrent); COPY_SCALAR_FIELD(isdynamic); + COPY_SCALAR_FIELD(isiceberg); return newnode; } @@ -3461,6 +3462,7 @@ _copyDropDirectoryTableStmt(const DropDirectoryTableStmt *from) COPY_SCALAR_FIELD(base.missing_ok); COPY_SCALAR_FIELD(base.concurrent); COPY_SCALAR_FIELD(base.isdynamic); + COPY_SCALAR_FIELD(base.isiceberg); COPY_SCALAR_FIELD(with_content); return newnode; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 8a8984237dd..107b762ae0e 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1493,6 +1493,7 @@ _equalDropStmt(const DropStmt *a, const DropStmt *b) COMPARE_SCALAR_FIELD(missing_ok); COMPARE_SCALAR_FIELD(concurrent); COMPARE_SCALAR_FIELD(isdynamic); + COMPARE_SCALAR_FIELD(isiceberg); return true; } diff --git a/src/backend/nodes/outfuncs_common.c b/src/backend/nodes/outfuncs_common.c index c518e38db0d..e7d6abd13fe 100644 --- a/src/backend/nodes/outfuncs_common.c +++ b/src/backend/nodes/outfuncs_common.c @@ -668,6 +668,7 @@ _outDropStmt(StringInfo str, const DropStmt *node) WRITE_BOOL_FIELD(missing_ok); WRITE_BOOL_FIELD(concurrent); WRITE_BOOL_FIELD(isdynamic); + WRITE_BOOL_FIELD(isiceberg); } static void @@ -1809,6 +1810,7 @@ _outDropStmtInfo(StringInfo str, const DropStmt *node) WRITE_BOOL_FIELD(missing_ok); WRITE_BOOL_FIELD(concurrent); WRITE_BOOL_FIELD(isdynamic); + WRITE_BOOL_FIELD(isiceberg); } static void diff --git a/src/backend/nodes/readfuncs_common.c b/src/backend/nodes/readfuncs_common.c index f895e4a4468..882a38628be 100644 --- a/src/backend/nodes/readfuncs_common.c +++ b/src/backend/nodes/readfuncs_common.c @@ -738,6 +738,7 @@ _readDropStmt_common(DropStmt *local_node) READ_BOOL_FIELD(missing_ok); READ_BOOL_FIELD(concurrent); READ_BOOL_FIELD(isdynamic); + READ_BOOL_FIELD(isiceberg); /* Force 'missing_ok' in QEs */ #ifdef COMPILING_BINARY_FUNCS @@ -1148,6 +1149,7 @@ _readDropStmt(void) READ_BOOL_FIELD(missing_ok); READ_BOOL_FIELD(concurrent); READ_BOOL_FIELD(isdynamic); + READ_BOOL_FIELD(isiceberg); /* Force 'missing_ok' in QEs */ #ifdef COMPILING_BINARY_FUNCS diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index ae79cb403de..39f95ab3842 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -10530,6 +10530,31 @@ DropStmt: DROP object_type_any_name IF_P EXISTS any_name_list opt_drop_behavior n->isdynamic = true; $$ = (Node *)n; } +/* DROP ICEBERG TABLE */ + | DROP ICEBERG TABLE IF_P EXISTS any_name_list opt_drop_behavior + { + DropStmt *n = makeNode(DropStmt); + n->removeType = OBJECT_TABLE; + n->missing_ok = true; + n->objects = $6; + n->behavior = $7; + n->concurrent = false; + n->isdynamic = false; + n->isiceberg = true; + $$ = (Node *)n; + } + | DROP ICEBERG TABLE any_name_list opt_drop_behavior + { + DropStmt *n = makeNode(DropStmt); + n->removeType = OBJECT_TABLE; + n->missing_ok = false; + n->objects = $4; + n->behavior = $5; + n->concurrent = false; + n->isdynamic = false; + n->isiceberg = true; + $$ = (Node *)n; + } ; /* object types taking any_name/any_name_list */ @@ -10577,8 +10602,8 @@ drop_type_name: | PUBLICATION { $$ = OBJECT_PUBLICATION; } | SCHEMA { $$ = OBJECT_SCHEMA; } | SERVER { $$ = OBJECT_FOREIGN_SERVER; } - | CATALOG_P { $$ = OBJECT_FOREIGN_CATALOG; } - | VOLUME { $$ = OBJECT_FOREIGN_VOLUME; } + | FOREIGN CATALOG_P { $$ = OBJECT_FOREIGN_CATALOG; } + | FOREIGN VOLUME { $$ = OBJECT_FOREIGN_VOLUME; } | PROTOCOL { $$ = OBJECT_EXTPROTOCOL; } ; diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 300c018ba8a..3795a189072 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -3412,7 +3412,10 @@ CreateCommandTag(Node *parsetree) switch (((DropStmt *) parsetree)->removeType) { case OBJECT_TABLE: - tag = CMDTAG_DROP_TABLE; + if (((DropStmt *) parsetree)->isiceberg) + tag = CMDTAG_DROP_LAKE_TABLE; + else + tag = CMDTAG_DROP_TABLE; break; case OBJECT_SEQUENCE: tag = CMDTAG_DROP_SEQUENCE; diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 48903015c2f..2235f292804 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1267,7 +1267,6 @@ static const pgsql_thing_t words_after_create[] = { {"AGGREGATE", NULL, NULL, Query_for_list_of_aggregates}, {"CAST", NULL, NULL, NULL}, /* Casts have complex structures for names, so * skip it */ - {"CATALOG", Query_for_list_of_foreign_catalogs, NULL, NULL, NULL, THING_NO_CREATE | THING_NO_ALTER}, {"COLLATION", NULL, NULL, &Query_for_list_of_collations}, /* @@ -1285,13 +1284,13 @@ static const pgsql_thing_t words_after_create[] = { {"EVENT TRIGGER", NULL, NULL, NULL}, {"EXTENSION", Query_for_list_of_extensions}, {"EXTERNAL TABLE", NULL, NULL, NULL}, - {"FOREIGN CATALOG", NULL, NULL, NULL, NULL, THING_NO_DROP | THING_NO_ALTER}, + {"FOREIGN CATALOG", Query_for_list_of_foreign_catalogs, NULL, NULL, NULL, THING_NO_ALTER}, {"FOREIGN DATA WRAPPER", NULL, NULL, NULL}, {"FOREIGN TABLE", NULL, NULL, NULL}, - {"FOREIGN VOLUME", NULL, NULL, NULL, NULL, THING_NO_DROP | THING_NO_ALTER}, + {"FOREIGN VOLUME", Query_for_list_of_foreign_volumes, NULL, NULL, NULL, THING_NO_ALTER}, {"FUNCTION", NULL, NULL, Query_for_list_of_functions}, {"GROUP", Query_for_list_of_roles}, - {"ICEBERG TABLE", NULL, NULL, NULL, NULL, THING_NO_DROP | THING_NO_ALTER}, + {"ICEBERG TABLE", NULL, NULL, &Query_for_list_of_tables, NULL, THING_NO_ALTER}, {"INCREMENTAL MATERIALIZED VIEW", NULL, NULL, &Query_for_list_of_matviews, NULL, THING_NO_DROP | THING_NO_ALTER}, {"INDEX", NULL, NULL, &Query_for_list_of_indexes}, {"LANGUAGE", Query_for_list_of_languages}, @@ -1338,7 +1337,6 @@ static const pgsql_thing_t words_after_create[] = { {"USER MAPPING FOR", NULL, NULL, NULL}, {"STORAGE USER MAPPING FOR", NULL, NULL, NULL}, {"VIEW", NULL, NULL, &Query_for_list_of_views}, - {"VOLUME", Query_for_list_of_foreign_volumes, NULL, NULL, NULL, THING_NO_CREATE | THING_NO_ALTER}, {"WAREHOUSE", NULL}, {NULL} /* end of list */ }; @@ -3836,7 +3834,7 @@ psql_completion(const char *text, int start, int end) /* DROP */ /* Complete DROP object with CASCADE / RESTRICT */ else if (Matches("DROP", - "CATALOG|COLLATION|CONVERSION|DOMAIN|EXTENSION|LANGUAGE|PUBLICATION|SCHEMA|SEQUENCE|SERVER|SUBSCRIPTION|STATISTICS|TABLE|TYPE|VIEW|VOLUME", + "COLLATION|CONVERSION|DOMAIN|EXTENSION|LANGUAGE|PUBLICATION|SCHEMA|SEQUENCE|SERVER|SUBSCRIPTION|STATISTICS|TABLE|TYPE|VIEW", MatchAny) || Matches("DROP", "ACCESS", "METHOD", MatchAny) || (Matches("DROP", "AGGREGATE|FUNCTION|PROCEDURE|ROUTINE", MatchAny, MatchAny) && @@ -3844,6 +3842,8 @@ psql_completion(const char *text, int start, int end) Matches("DROP", "EVENT", "TRIGGER", MatchAny) || Matches("DROP", "FOREIGN", "DATA", "WRAPPER", MatchAny) || Matches("DROP", "FOREIGN", "TABLE", MatchAny) || + Matches("DROP", "FOREIGN", "CATALOG|VOLUME", MatchAny) || + Matches("DROP", "ICEBERG", "TABLE", MatchAny) || Matches("DROP", "DIRECTORY", "TABLE", MatchAny) || Matches("DROP", "TEXT", "SEARCH", "CONFIGURATION|DICTIONARY|PARSER|TEMPLATE", MatchAny)) COMPLETE_WITH("CASCADE", "RESTRICT"); @@ -3854,7 +3854,15 @@ psql_completion(const char *text, int start, int end) else if (Matches("DROP", "AGGREGATE|FUNCTION|PROCEDURE|ROUTINE", MatchAny, "(")) COMPLETE_WITH_FUNCTION_ARG(prev2_wd); else if (Matches("DROP", "FOREIGN")) - COMPLETE_WITH("DATA WRAPPER", "TABLE"); + COMPLETE_WITH("CATALOG", "DATA WRAPPER", "TABLE", "VOLUME"); + else if (Matches("DROP", "FOREIGN", "CATALOG")) + COMPLETE_WITH_QUERY(Query_for_list_of_foreign_catalogs); + else if (Matches("DROP", "FOREIGN", "VOLUME")) + COMPLETE_WITH_QUERY(Query_for_list_of_foreign_volumes); + else if (Matches("DROP", "ICEBERG")) + COMPLETE_WITH("TABLE"); + else if (Matches("DROP", "ICEBERG", "TABLE")) + COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_tables); else if (Matches("DROP", "DATABASE", MatchAny)) COMPLETE_WITH("WITH ("); else if (HeadMatches("DROP", "DATABASE") && (ends_with(prev_wd, '('))) diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index bac420b1d58..5ebc3142d4b 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3832,6 +3832,7 @@ typedef struct DropStmt bool missing_ok; /* skip error if object is missing? */ bool concurrent; /* drop index concurrently? */ bool isdynamic; /* drop a dynamic table? */ + bool isiceberg; /* drop an iceberg (lake) table? */ } DropStmt; /* ---------------------- diff --git a/src/include/tcop/cmdtaglist.h b/src/include/tcop/cmdtaglist.h index fec05c924ad..66684f8d6f9 100644 --- a/src/include/tcop/cmdtaglist.h +++ b/src/include/tcop/cmdtaglist.h @@ -185,6 +185,7 @@ PG_CMDTAG(CMDTAG_DROP_FOREIGN_TABLE, "DROP FOREIGN TABLE", true, false, false) PG_CMDTAG(CMDTAG_DROP_FOREIGN_VOLUME, "DROP FOREIGN VOLUME", true, false, false) PG_CMDTAG(CMDTAG_DROP_FUNCTION, "DROP FUNCTION", true, false, false) PG_CMDTAG(CMDTAG_DROP_INDEX, "DROP INDEX", true, false, false) +PG_CMDTAG(CMDTAG_DROP_LAKE_TABLE, "DROP LAKE TABLE", true, false, false) PG_CMDTAG(CMDTAG_DROP_LANGUAGE, "DROP LANGUAGE", true, false, false) PG_CMDTAG(CMDTAG_DROP_MATERIALIZED_VIEW, "DROP MATERIALIZED VIEW", true, false, false) PG_CMDTAG(CMDTAG_DROP_OPERATOR, "DROP OPERATOR", true, false, false) diff --git a/src/test/regress/expected/lake_table.out b/src/test/regress/expected/lake_table.out index a97e854c27c..b28d6a185bb 100644 --- a/src/test/regress/expected/lake_table.out +++ b/src/test/regress/expected/lake_table.out @@ -225,9 +225,9 @@ HINT: Lake tables must use DISTRIBUTED RANDOMLY because data is stored on objec CREATE ROLE regress_lake_user; NOTICE: resource queue required -- using default resource queue "pg_default" SET ROLE regress_lake_user; -DROP CATALOG lake_test_cat; -- fail, not owner +DROP FOREIGN CATALOG lake_test_cat; -- fail, not owner ERROR: must be owner of foreign catalog lake_test_cat -DROP VOLUME lake_test_vol; -- fail, not owner +DROP FOREIGN VOLUME lake_test_vol; -- fail, not owner ERROR: must be owner of foreign volume lake_test_vol RESET ROLE; -- Dependencies: the server holds the catalog/volume, which hold the tables @@ -240,7 +240,7 @@ table lake_test_t2 depends on volume lake_test_vol table lake_test_t3 depends on volume lake_test_vol table lake_test_t4 depends on volume lake_test_vol HINT: Use DROP ... CASCADE to drop the dependent objects too. -DROP CATALOG lake_test_cat; -- fail, tables depend on it +DROP FOREIGN CATALOG lake_test_cat; -- fail, tables depend on it ERROR: cannot drop catalog lake_test_cat because other objects depend on it DETAIL: table lake_test_t1 depends on catalog lake_test_cat table lake_test_t2 depends on catalog lake_test_cat @@ -248,7 +248,7 @@ table lake_test_t3 depends on catalog lake_test_cat table lake_test_t4 depends on catalog lake_test_cat HINT: Use DROP ... CASCADE to drop the dependent objects too. -- Dropping a lake table removes its pg_lake_table entry -DROP TABLE lake_test_t1; +DROP ICEBERG TABLE lake_test_t1; SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid WHERE c.relname = 'lake_test_t1'; count @@ -256,11 +256,23 @@ SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid 0 (1 row) --- DROP CATALOG ... CASCADE takes the remaining tables with it -DROP CATALOG lake_test_cat CASCADE; -NOTICE: drop cascades to 3 other objects -DETAIL: drop cascades to table lake_test_t2 -drop cascades to table lake_test_t3 +-- DROP ICEBERG TABLE rejects a non-iceberg table ... +DROP ICEBERG TABLE lake_test_heap; -- fail, not an iceberg table +ERROR: "lake_test_heap" is not an iceberg table +HINT: Use DROP TABLE to remove a table. +-- ... while DROP TABLE still removes an iceberg table (no reverse restriction) +DROP TABLE lake_test_t2; +SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid + WHERE c.relname = 'lake_test_t2'; + count +------- + 0 +(1 row) + +-- DROP FOREIGN CATALOG ... CASCADE takes the remaining tables with it +DROP FOREIGN CATALOG lake_test_cat CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table lake_test_t3 drop cascades to table lake_test_t4 SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid WHERE c.relname LIKE 'lake\_test%'; @@ -270,14 +282,14 @@ SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid (1 row) -- DROP variants -DROP CATALOG lake_test_cat; -- fail, already gone +DROP FOREIGN CATALOG lake_test_cat; -- fail, already gone ERROR: foreign catalog "lake_test_cat" does not exist -DROP CATALOG IF EXISTS lake_test_cat; -- skip with notice +DROP FOREIGN CATALOG IF EXISTS lake_test_cat; -- skip with notice NOTICE: foreign catalog "lake_test_cat" does not exist, skipping -DROP VOLUME lake_test_vol; -DROP VOLUME lake_test_vol; -- fail, already gone +DROP FOREIGN VOLUME lake_test_vol; +DROP FOREIGN VOLUME lake_test_vol; -- fail, already gone ERROR: foreign volume "lake_test_vol" does not exist -DROP VOLUME IF EXISTS lake_test_vol; -- skip with notice +DROP FOREIGN VOLUME IF EXISTS lake_test_vol; -- skip with notice NOTICE: foreign volume "lake_test_vol" does not exist, skipping -- Cleanup DROP TABLE lake_test_heap; diff --git a/src/test/regress/sql/lake_table.sql b/src/test/regress/sql/lake_table.sql index f9683baf604..623f39d841f 100644 --- a/src/test/regress/sql/lake_table.sql +++ b/src/test/regress/sql/lake_table.sql @@ -108,30 +108,37 @@ ALTER TABLE lake_test_t1 SET DISTRIBUTED BY (a); -- fail -- Only the owner can drop a catalog or volume CREATE ROLE regress_lake_user; SET ROLE regress_lake_user; -DROP CATALOG lake_test_cat; -- fail, not owner -DROP VOLUME lake_test_vol; -- fail, not owner +DROP FOREIGN CATALOG lake_test_cat; -- fail, not owner +DROP FOREIGN VOLUME lake_test_vol; -- fail, not owner RESET ROLE; -- Dependencies: the server holds the catalog/volume, which hold the tables DROP SERVER lake_test_srv; -- fail, catalog and volume depend on it -DROP CATALOG lake_test_cat; -- fail, tables depend on it +DROP FOREIGN CATALOG lake_test_cat; -- fail, tables depend on it -- Dropping a lake table removes its pg_lake_table entry -DROP TABLE lake_test_t1; +DROP ICEBERG TABLE lake_test_t1; SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid WHERE c.relname = 'lake_test_t1'; --- DROP CATALOG ... CASCADE takes the remaining tables with it -DROP CATALOG lake_test_cat CASCADE; +-- DROP ICEBERG TABLE rejects a non-iceberg table ... +DROP ICEBERG TABLE lake_test_heap; -- fail, not an iceberg table +-- ... while DROP TABLE still removes an iceberg table (no reverse restriction) +DROP TABLE lake_test_t2; +SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid + WHERE c.relname = 'lake_test_t2'; + +-- DROP FOREIGN CATALOG ... CASCADE takes the remaining tables with it +DROP FOREIGN CATALOG lake_test_cat CASCADE; SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid WHERE c.relname LIKE 'lake\_test%'; -- DROP variants -DROP CATALOG lake_test_cat; -- fail, already gone -DROP CATALOG IF EXISTS lake_test_cat; -- skip with notice -DROP VOLUME lake_test_vol; -DROP VOLUME lake_test_vol; -- fail, already gone -DROP VOLUME IF EXISTS lake_test_vol; -- skip with notice +DROP FOREIGN CATALOG lake_test_cat; -- fail, already gone +DROP FOREIGN CATALOG IF EXISTS lake_test_cat; -- skip with notice +DROP FOREIGN VOLUME lake_test_vol; +DROP FOREIGN VOLUME lake_test_vol; -- fail, already gone +DROP FOREIGN VOLUME IF EXISTS lake_test_vol; -- skip with notice -- Cleanup DROP TABLE lake_test_heap; From f21763c6f90d8f24bd8e438a15341e2e7e0ca86b Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Fri, 10 Jul 2026 11:38:21 +0800 Subject: [PATCH 12/29] commands: make foreign catalog type a required first-class column Promote the catalog type from a free-form OPTION to a required TYPE clause backed by a new pg_foreign_catalog.fctype column, per review on PR #1842. Every foreign catalog has a type (hive, hdfs, polaris, ...), so it is a property rather than an option; the value is stored verbatim as an open string and validated by the datalake provider, keeping the kernel provider-agnostic. Bumps CATALOG_VERSION_NO for the new column. --- src/backend/commands/foreigncmds.c | 13 ++++++++++++ src/backend/nodes/copyfuncs.funcs.c | 1 + src/backend/nodes/equalfuncs.c | 1 + src/backend/nodes/outfast.c | 1 + src/backend/nodes/readfast.c | 1 + src/backend/parser/gram.y | 10 +++++---- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_foreign_catalog.h | 1 + src/include/nodes/parsenodes.h | 1 + src/test/regress/expected/lake_table.out | 27 ++++++++++++++---------- src/test/regress/sql/lake_table.sql | 17 ++++++++------- 11 files changed, 51 insertions(+), 24 deletions(-) diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index b5e6adc95f1..46a90b88ef7 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -1107,6 +1107,19 @@ CreateForeignCatalog(CreateForeignCatalogStmt *stmt) values[Anum_pg_foreign_catalog_fcowner - 1] = ObjectIdGetDatum(ownerId); values[Anum_pg_foreign_catalog_fcserver - 1] = ObjectIdGetDatum(server->serverid); + /* + * The catalog type is a required property (every catalog has a type such + * as 'hive', 'hdfs', ...). The grammar enforces the TYPE clause, so this + * is just a defensive check; the value is stored verbatim and validated by + * the datalake provider rather than the kernel. + */ + if (stmt->catalogtype == NULL || stmt->catalogtype[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), + errmsg("foreign catalog type cannot be empty"))); + values[Anum_pg_foreign_catalog_fctype - 1] = + CStringGetTextDatum(stmt->catalogtype); + /* Add catalog options; there is no validator for them */ catalogoptions = transformGenericOptions(ForeignCatalogRelationId, PointerGetDatum(NULL), diff --git a/src/backend/nodes/copyfuncs.funcs.c b/src/backend/nodes/copyfuncs.funcs.c index 81fbaa2330b..d6597996a99 100644 --- a/src/backend/nodes/copyfuncs.funcs.c +++ b/src/backend/nodes/copyfuncs.funcs.c @@ -2755,6 +2755,7 @@ _copyCreateForeignCatalogStmt(const CreateForeignCatalogStmt *from) COPY_STRING_FIELD(catalogname); COPY_STRING_FIELD(servername); + COPY_STRING_FIELD(catalogtype); COPY_SCALAR_FIELD(if_not_exists); COPY_NODE_FIELD(options); diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 107b762ae0e..0150a714f65 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -2169,6 +2169,7 @@ _equalCreateForeignCatalogStmt(const CreateForeignCatalogStmt *a, const CreateFo { COMPARE_STRING_FIELD(catalogname); COMPARE_STRING_FIELD(servername); + COMPARE_STRING_FIELD(catalogtype); COMPARE_SCALAR_FIELD(if_not_exists); COMPARE_NODE_FIELD(options); diff --git a/src/backend/nodes/outfast.c b/src/backend/nodes/outfast.c index 42907eccab3..90d72d34559 100644 --- a/src/backend/nodes/outfast.c +++ b/src/backend/nodes/outfast.c @@ -680,6 +680,7 @@ _outCreateForeignCatalogStmt(StringInfo str, CreateForeignCatalogStmt *node) WRITE_STRING_FIELD(catalogname); WRITE_STRING_FIELD(servername); + WRITE_STRING_FIELD(catalogtype); WRITE_BOOL_FIELD(if_not_exists); WRITE_NODE_FIELD(options); } diff --git a/src/backend/nodes/readfast.c b/src/backend/nodes/readfast.c index adb7d9e4811..b9fbff9226e 100644 --- a/src/backend/nodes/readfast.c +++ b/src/backend/nodes/readfast.c @@ -1605,6 +1605,7 @@ _readCreateForeignCatalogStmt(void) READ_STRING_FIELD(catalogname); READ_STRING_FIELD(servername); + READ_STRING_FIELD(catalogtype); READ_BOOL_FIELD(if_not_exists); READ_NODE_FIELD(options); diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 39f95ab3842..fc28d5ce5bf 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -9106,21 +9106,23 @@ CreateDirectoryTableStmt: *****************************************************************************/ CreateForeignCatalogStmt: - CREATE FOREIGN CATALOG_P name SERVER name create_generic_options + CREATE FOREIGN CATALOG_P name SERVER name TYPE_P Sconst create_generic_options { CreateForeignCatalogStmt *n = makeNode(CreateForeignCatalogStmt); n->catalogname = $4; n->servername = $6; - n->options = $7; + n->catalogtype = $8; + n->options = $9; n->if_not_exists = false; $$ = (Node *) n; } - | CREATE FOREIGN CATALOG_P IF_P NOT EXISTS name SERVER name create_generic_options + | CREATE FOREIGN CATALOG_P IF_P NOT EXISTS name SERVER name TYPE_P Sconst create_generic_options { CreateForeignCatalogStmt *n = makeNode(CreateForeignCatalogStmt); n->catalogname = $7; n->servername = $9; - n->options = $10; + n->catalogtype = $11; + n->options = $12; n->if_not_exists = true; $$ = (Node *) n; } diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 3d5e3915585..53258aca56e 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -60,6 +60,6 @@ */ /* 3yyymmddN */ -#define CATALOG_VERSION_NO 302607021 +#define CATALOG_VERSION_NO 302607101 #endif diff --git a/src/include/catalog/pg_foreign_catalog.h b/src/include/catalog/pg_foreign_catalog.h index b6325915b22..547f01c6a9c 100644 --- a/src/include/catalog/pg_foreign_catalog.h +++ b/src/include/catalog/pg_foreign_catalog.h @@ -50,6 +50,7 @@ CATALOG(pg_foreign_catalog,8549,ForeignCatalogRelationId) Oid fcserver BKI_LOOKUP(pg_foreign_server); /* foreign server this catalog belongs to */ #ifdef CATALOG_VARLEN /* variable-length fields start here */ + text fctype BKI_FORCE_NOT_NULL; /* catalog type, e.g. 'hive' */ text fcoptions[1]; /* foreign catalog options */ #endif } FormData_pg_foreign_catalog; diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 5ebc3142d4b..8f4b9aa5c27 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3278,6 +3278,7 @@ typedef struct CreateForeignCatalogStmt NodeTag type; char *catalogname; /* foreign catalog name */ char *servername; /* server name */ + char *catalogtype; /* catalog type, e.g. 'hive' */ bool if_not_exists; /* just do nothing if it already exists? */ List *options; /* generic options to catalog */ } CreateForeignCatalogStmt; diff --git a/src/test/regress/expected/lake_table.out b/src/test/regress/expected/lake_table.out index b28d6a185bb..b2baf847936 100644 --- a/src/test/regress/expected/lake_table.out +++ b/src/test/regress/expected/lake_table.out @@ -10,6 +10,7 @@ fcname | name | | not null | | plain | | fcowner | oid | | not null | | plain | | fcserver | oid | | not null | | plain | | + fctype | text | C | not null | | extended | | fcoptions | text[] | C | | | extended | | Indexes: "pg_foreign_catalog_oid_index" PRIMARY KEY, btree (oid) @@ -44,23 +45,27 @@ Indexes: CREATE FOREIGN DATA WRAPPER lake_test_fdw; CREATE SERVER lake_test_srv FOREIGN DATA WRAPPER lake_test_fdw; CREATE SERVER lake_test_srv2 FOREIGN DATA WRAPPER lake_test_fdw; --- CREATE FOREIGN CATALOG -CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv OPTIONS (type 'hive', uri 'thrift://localhost:9083'); -CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv; -- fail, duplicate +-- CREATE FOREIGN CATALOG: TYPE is a required first-class property +CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv TYPE 'hive' OPTIONS (uri 'thrift://localhost:9083'); +CREATE FOREIGN CATALOG lake_test_notype SERVER lake_test_srv; -- fail, TYPE is required +ERROR: syntax error at or near ";" +LINE 1: ...REATE FOREIGN CATALOG lake_test_notype SERVER lake_test_srv; + ^ +CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv TYPE 'hive'; -- fail, duplicate ERROR: foreign catalog "lake_test_cat" already exists -CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv; -- skip with notice +CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv TYPE 'hive'; -- skip with notice NOTICE: foreign catalog "lake_test_cat" already exists, skipping -- catalog names are global: the same name on another server is still a duplicate -CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv2; -- fail, duplicate +CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv2 TYPE 'hive'; -- fail, duplicate ERROR: foreign catalog "lake_test_cat" already exists -CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv2; -- skip with notice +CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv2 TYPE 'hive'; -- skip with notice NOTICE: foreign catalog "lake_test_cat" already exists, skipping -CREATE FOREIGN CATALOG lake_test_bad SERVER no_such_server; -- fail, no server +CREATE FOREIGN CATALOG lake_test_bad SERVER no_such_server TYPE 'hive'; -- fail, no server ERROR: server "no_such_server" does not exist -SELECT fcname, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%' ORDER BY 1; - fcname | fcoptions ----------------+----------------------------------------- - lake_test_cat | {type=hive,uri=thrift://localhost:9083} +SELECT fcname, fctype, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%' ORDER BY 1; + fcname | fctype | fcoptions +---------------+--------+------------------------------- + lake_test_cat | hive | {uri=thrift://localhost:9083} (1 row) -- CREATE FOREIGN VOLUME diff --git a/src/test/regress/sql/lake_table.sql b/src/test/regress/sql/lake_table.sql index 623f39d841f..6f9c7af2e7d 100644 --- a/src/test/regress/sql/lake_table.sql +++ b/src/test/regress/sql/lake_table.sql @@ -12,15 +12,16 @@ CREATE FOREIGN DATA WRAPPER lake_test_fdw; CREATE SERVER lake_test_srv FOREIGN DATA WRAPPER lake_test_fdw; CREATE SERVER lake_test_srv2 FOREIGN DATA WRAPPER lake_test_fdw; --- CREATE FOREIGN CATALOG -CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv OPTIONS (type 'hive', uri 'thrift://localhost:9083'); -CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv; -- fail, duplicate -CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv; -- skip with notice +-- CREATE FOREIGN CATALOG: TYPE is a required first-class property +CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv TYPE 'hive' OPTIONS (uri 'thrift://localhost:9083'); +CREATE FOREIGN CATALOG lake_test_notype SERVER lake_test_srv; -- fail, TYPE is required +CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv TYPE 'hive'; -- fail, duplicate +CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv TYPE 'hive'; -- skip with notice -- catalog names are global: the same name on another server is still a duplicate -CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv2; -- fail, duplicate -CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv2; -- skip with notice -CREATE FOREIGN CATALOG lake_test_bad SERVER no_such_server; -- fail, no server -SELECT fcname, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%' ORDER BY 1; +CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv2 TYPE 'hive'; -- fail, duplicate +CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv2 TYPE 'hive'; -- skip with notice +CREATE FOREIGN CATALOG lake_test_bad SERVER no_such_server TYPE 'hive'; -- fail, no server +SELECT fcname, fctype, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%' ORDER BY 1; -- CREATE FOREIGN VOLUME CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv OPTIONS (path 's3://bucket/prefix'); From 7e6e7ca21fd049b9a9a7dc813afa9a444ee38d64 Mon Sep 17 00:00:00 2001 From: liuxiaoyu <45345701+MisterRaindrop@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:51:26 +0800 Subject: [PATCH 13/29] Update src/backend/foreign/foreign.c Co-authored-by: Andrey Sokolov --- src/backend/foreign/foreign.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c index 331ef9bc40b..aff82399f59 100644 --- a/src/backend/foreign/foreign.c +++ b/src/backend/foreign/foreign.c @@ -1066,10 +1066,7 @@ GetForeignVolumeByName(const char *volumename, bool missing_ok) tp, Anum_pg_foreign_volume_fvoptions, &isnull); - if (isnull) - volume->options = NIL; - else - volume->options = untransformRelOptions(datum); + volume->options = (isnull) ? NIL : untransformRelOptions(datum); ReleaseSysCache(tp); From 70b451aab4dd17576199738301ad9ed0c83198b8 Mon Sep 17 00:00:00 2001 From: liuxiaoyu <45345701+MisterRaindrop@users.noreply.github.com> Date: Fri, 10 Jul 2026 15:52:02 +0800 Subject: [PATCH 14/29] Update src/include/commands/laketablecmds.h Co-authored-by: Andrey Sokolov --- src/include/commands/laketablecmds.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/include/commands/laketablecmds.h b/src/include/commands/laketablecmds.h index 3b7b949a7fa..980d25784d8 100644 --- a/src/include/commands/laketablecmds.h +++ b/src/include/commands/laketablecmds.h @@ -54,7 +54,7 @@ extern const char *GetDefaultIcebergVolume(void); /* Lake table management */ extern Oid GetIcebergTableAmOid(bool missing_ok); extern bool RelationIsIcebergTable(Relation rel); -extern void ValidateLakeTableOptions(CreateLakeTableStmt *stmt); +extern void ValidateLakeTableStmt(CreateLakeTableStmt *stmt); extern void CreateLakeTable(CreateLakeTableStmt *stmt, Oid relId); extern void RemoveLakeTableEntry(Oid relid); From 32bda146412fc27bc92fbf90b441a160d4a6b9d4 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Fri, 10 Jul 2026 20:37:08 +0800 Subject: [PATCH 15/29] psql: complete only iceberg tables for DROP ICEBERG TABLE Tab completion for DROP ICEBERG TABLE previously offered every ordinary table. Add Query_for_list_of_iceberg_tables, which filters to relations whose access method is the iceberg AM -- the same rule the backend validates DROP ICEBERG TABLE against -- so completion only suggests tables the command will actually accept. When no provider installed the iceberg AM the scalar subquery is NULL and the list is empty. Addresses review on PR #1842. --- src/bin/psql/tab-complete.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 2235f292804..ead1098f979 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -678,6 +678,23 @@ static const SchemaQuery Query_for_list_of_tables = { .result = "c.relname", }; +/* + * An iceberg (lake) table is an ordinary relation whose access method is the + * "iceberg" AM -- the same rule DROP ICEBERG TABLE validates against. When no + * provider installed that AM the scalar subquery yields NULL and the list is + * empty, which is correct (no relation can be an iceberg table). + */ +static const SchemaQuery Query_for_list_of_iceberg_tables = { + .catname = "pg_catalog.pg_class c", + .selcondition = + "c.relkind IN (" CppAsString2(RELKIND_RELATION) ") AND " + "c.relam = (SELECT oid FROM pg_catalog.pg_am " + "WHERE amname = 'iceberg' AND amtype = 't')", + .viscondition = "pg_catalog.pg_table_is_visible(c.oid)", + .namespace = "c.relnamespace", + .result = "c.relname", +}; + static const SchemaQuery Query_for_list_of_directory_tables = { .catname = "pg_catalog.pg_class c", .selcondition = "c.relkind IN (" CppAsString2(RELKIND_DIRECTORY_TABLE) ")", @@ -1290,7 +1307,7 @@ static const pgsql_thing_t words_after_create[] = { {"FOREIGN VOLUME", Query_for_list_of_foreign_volumes, NULL, NULL, NULL, THING_NO_ALTER}, {"FUNCTION", NULL, NULL, Query_for_list_of_functions}, {"GROUP", Query_for_list_of_roles}, - {"ICEBERG TABLE", NULL, NULL, &Query_for_list_of_tables, NULL, THING_NO_ALTER}, + {"ICEBERG TABLE", NULL, NULL, &Query_for_list_of_iceberg_tables, NULL, THING_NO_ALTER}, {"INCREMENTAL MATERIALIZED VIEW", NULL, NULL, &Query_for_list_of_matviews, NULL, THING_NO_DROP | THING_NO_ALTER}, {"INDEX", NULL, NULL, &Query_for_list_of_indexes}, {"LANGUAGE", Query_for_list_of_languages}, @@ -3862,7 +3879,7 @@ psql_completion(const char *text, int start, int end) else if (Matches("DROP", "ICEBERG")) COMPLETE_WITH("TABLE"); else if (Matches("DROP", "ICEBERG", "TABLE")) - COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_tables); + COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_iceberg_tables); else if (Matches("DROP", "DATABASE", MatchAny)) COMPLETE_WITH("WITH ("); else if (HeadMatches("DROP", "DATABASE") && (ends_with(prev_wd, '('))) From f6d32869e76b44b0310ab663ff4076413b86f434 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Fri, 10 Jul 2026 20:46:26 +0800 Subject: [PATCH 16/29] commands: finish ValidateLakeTableOptions -> ValidateLakeTableStmt rename The header prototype was renamed to ValidateLakeTableStmt but the definition and its caller still used the old name, breaking the build (-Werror=missing-prototypes). Rename the definition, caller and comments to match. --- src/backend/commands/laketablecmds.c | 6 +++--- src/backend/tcop/utility.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/commands/laketablecmds.c b/src/backend/commands/laketablecmds.c index 90cfb6fa4b5..d5c9274b049 100644 --- a/src/backend/commands/laketablecmds.c +++ b/src/backend/commands/laketablecmds.c @@ -268,7 +268,7 @@ validate_foreign_volume(const char *volume_name) * Resolve and validate the table type, catalog and volume of a * CreateLakeTableStmt, returning the catalog/volume OIDs. * - * Also exposed (via ValidateLakeTableOptions) so ProcessUtilitySlow can run + * Also exposed (via ValidateLakeTableStmt) so ProcessUtilitySlow can run * the validation on the QD before DefineRelation: DefineRelation dispatches * the statement to the QEs, so a validation failure raised only inside * CreateLakeTable() would surface as a confusing QE-annotated error. @@ -361,12 +361,12 @@ ResolveLakeTableOptions(CreateLakeTableStmt *stmt, } /* - * ValidateLakeTableOptions + * ValidateLakeTableStmt * * QD-side pre-DefineRelation validation wrapper; see ResolveLakeTableOptions. */ void -ValidateLakeTableOptions(CreateLakeTableStmt *stmt) +ValidateLakeTableStmt(CreateLakeTableStmt *stmt) { Oid catalog_oid; Oid volume_oid; diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 3795a189072..530078e0799 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -1677,7 +1677,7 @@ ProcessUtilitySlow(ParseState *pstate, * QE-annotated error. */ if (Gp_role == GP_ROLE_DISPATCH) - ValidateLakeTableOptions(cstmt); + ValidateLakeTableStmt(cstmt); /* * Create the table itself. Dispatch manually From afe4142ba7f061c2dec7f5f57c33525f957b9ce6 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Tue, 14 Jul 2026 10:41:00 +0800 Subject: [PATCH 17/29] commands: reject plain DROP TABLE on an iceberg table Iceberg (lake) tables share RELKIND_RELATION with ordinary tables, so plain DROP TABLE used to drop them and they appeared in DROP TABLE tab completion -- unlike foreign tables. Align them with the foreign-table behavior: - RangeVarCallbackForDropRelation() now rejects an iceberg table on a non-iceberg DROP TABLE with errhint "Use DROP ICEBERG TABLE to remove an iceberg table" (fires under IF EXISTS too); DROP ICEBERG TABLE still rejects ordinary tables. - psql's Query_for_list_of_tables excludes iceberg tables via NOT EXISTS, so ordinary tables stay listed even when no iceberg AM is installed. DROP ICEBERG TABLE continues to complete iceberg tables via its own query. --- src/backend/commands/tablecmds.c | 30 +++++++++++++++++------- src/bin/psql/tab-complete.c | 9 ++++++- src/test/regress/expected/lake_table.out | 16 ++++++++++++- src/test/regress/sql/lake_table.sql | 7 +++++- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index cb63d10a8fa..641bb653a22 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -2222,21 +2222,33 @@ RangeVarCallbackForDropRelation(const RangeVar *rel, Oid relOid, Oid oldRelOid, state->expected_relkind); /* - * DROP ICEBERG TABLE must target an iceberg (lake) table. Iceberg tables - * share RELKIND_RELATION with ordinary tables, so the relkind check above - * cannot tell them apart; verify the access method here (same rule as - * RelationIsIcebergTable). If no provider installed the iceberg AM, no - * relation can be an iceberg table, so this always rejects. + * Iceberg (lake) tables share RELKIND_RELATION with ordinary tables and are + * told apart only by their access method. DROP ICEBERG TABLE must target one; + * plain DROP TABLE must NOT (mirrors the foreign-table rule) -- direct the user + * to the matching command in each case. */ - if (state->iceberg_only) + if (state->expected_relkind == RELKIND_RELATION) { Oid iceberg_amoid = GetIcebergTableAmOid(true); + bool is_iceberg = classform->relkind == RELKIND_RELATION && + OidIsValid(iceberg_amoid) && + classform->relam == iceberg_amoid; - if (!OidIsValid(iceberg_amoid) || classform->relam != iceberg_amoid) + if (state->iceberg_only) + { + if (!is_iceberg) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not an iceberg table", rel->relname), + errhint("Use DROP TABLE to remove a table."))); + } + else if (is_iceberg) + { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("\"%s\" is not an iceberg table", rel->relname), - errhint("Use DROP TABLE to remove a table."))); + errmsg("\"%s\" is not a table", rel->relname), + errhint("Use DROP ICEBERG TABLE to remove an iceberg table."))); + } } /* Allow DROP to either table owner or schema owner */ diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index ead1098f979..b7c83596f79 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -668,11 +668,18 @@ static const SchemaQuery Query_for_list_of_foreign_tables = { .result = "c.relname", }; +/* + * Exclude iceberg tables; Query_for_list_of_iceberg_tables serves DROP ICEBERG + * TABLE. With no iceberg AM, the NOT EXISTS subquery finds no match, so + * ordinary tables remain listed. + */ static const SchemaQuery Query_for_list_of_tables = { .catname = "pg_catalog.pg_class c", .selcondition = "c.relkind IN (" CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_PARTITIONED_TABLE) ")", + CppAsString2(RELKIND_PARTITIONED_TABLE) ") AND " + "NOT EXISTS (SELECT 1 FROM pg_catalog.pg_am a " + "WHERE a.oid = c.relam AND a.amname = 'iceberg' AND a.amtype = 't')", .viscondition = "pg_catalog.pg_table_is_visible(c.oid)", .namespace = "c.relnamespace", .result = "c.relname", diff --git a/src/test/regress/expected/lake_table.out b/src/test/regress/expected/lake_table.out index b2baf847936..b3ca57ea1d2 100644 --- a/src/test/regress/expected/lake_table.out +++ b/src/test/regress/expected/lake_table.out @@ -265,8 +265,22 @@ SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid DROP ICEBERG TABLE lake_test_heap; -- fail, not an iceberg table ERROR: "lake_test_heap" is not an iceberg table HINT: Use DROP TABLE to remove a table. --- ... while DROP TABLE still removes an iceberg table (no reverse restriction) +-- plain DROP TABLE must reject an iceberg table (mirrors foreign-table behavior) DROP TABLE lake_test_t2; +ERROR: "lake_test_t2" is not a table +HINT: Use DROP ICEBERG TABLE to remove an iceberg table. +DROP TABLE IF EXISTS lake_test_t2; -- IF EXISTS does not suppress wrong-type errors +ERROR: "lake_test_t2" is not a table +HINT: Use DROP ICEBERG TABLE to remove an iceberg table. +-- the rejected drops must have left the table and its lake metadata intact +SELECT count(*) FROM pg_lake_table WHERE ltrelid = 'lake_test_t2'::regclass; + count +------- + 1 +(1 row) + +-- the correct command still works +DROP ICEBERG TABLE lake_test_t2; SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid WHERE c.relname = 'lake_test_t2'; count diff --git a/src/test/regress/sql/lake_table.sql b/src/test/regress/sql/lake_table.sql index 6f9c7af2e7d..3cfc4173a74 100644 --- a/src/test/regress/sql/lake_table.sql +++ b/src/test/regress/sql/lake_table.sql @@ -124,8 +124,13 @@ SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid -- DROP ICEBERG TABLE rejects a non-iceberg table ... DROP ICEBERG TABLE lake_test_heap; -- fail, not an iceberg table --- ... while DROP TABLE still removes an iceberg table (no reverse restriction) +-- plain DROP TABLE must reject an iceberg table (mirrors foreign-table behavior) DROP TABLE lake_test_t2; +DROP TABLE IF EXISTS lake_test_t2; -- IF EXISTS does not suppress wrong-type errors +-- the rejected drops must have left the table and its lake metadata intact +SELECT count(*) FROM pg_lake_table WHERE ltrelid = 'lake_test_t2'::regclass; +-- the correct command still works +DROP ICEBERG TABLE lake_test_t2; SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid WHERE c.relname = 'lake_test_t2'; From 9e4e0c206382305f2ba0c5b71e976a304ede480f Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Tue, 14 Jul 2026 10:41:11 +0800 Subject: [PATCH 18/29] doc: add reference pages for the iceberg lake-table DDL commands Add SGML ref pages for CREATE/DROP FOREIGN CATALOG, CREATE/DROP FOREIGN VOLUME and CREATE/DROP ICEBERG TABLE, and register them in allfiles.sgml and reference.sgml. This makes "\h " in psql show their synopsis and adds them to the reference manual. sql_help.{c,h} are generated from these by create_help.pl, so they are not edited by hand. --- doc/src/sgml/ref/allfiles.sgml | 6 + doc/src/sgml/ref/create_foreign_catalog.sgml | 144 ++++++++++++++ doc/src/sgml/ref/create_foreign_volume.sgml | 133 +++++++++++++ doc/src/sgml/ref/create_iceberg_table.sgml | 193 +++++++++++++++++++ doc/src/sgml/ref/drop_foreign_catalog.sgml | 117 +++++++++++ doc/src/sgml/ref/drop_foreign_volume.sgml | 117 +++++++++++ doc/src/sgml/ref/drop_iceberg_table.sgml | 119 ++++++++++++ doc/src/sgml/reference.sgml | 6 + 8 files changed, 835 insertions(+) create mode 100644 doc/src/sgml/ref/create_foreign_catalog.sgml create mode 100644 doc/src/sgml/ref/create_foreign_volume.sgml create mode 100644 doc/src/sgml/ref/create_iceberg_table.sgml create mode 100644 doc/src/sgml/ref/drop_foreign_catalog.sgml create mode 100644 doc/src/sgml/ref/drop_foreign_volume.sgml create mode 100644 doc/src/sgml/ref/drop_iceberg_table.sgml diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index 3d47bff7fef..41a275686a1 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -69,10 +69,13 @@ Complete list of usable sgml source files in this directory. + + + @@ -118,10 +121,13 @@ Complete list of usable sgml source files in this directory. + + + diff --git a/doc/src/sgml/ref/create_foreign_catalog.sgml b/doc/src/sgml/ref/create_foreign_catalog.sgml new file mode 100644 index 00000000000..ee32abf6b47 --- /dev/null +++ b/doc/src/sgml/ref/create_foreign_catalog.sgml @@ -0,0 +1,144 @@ + + + + + CREATE FOREIGN CATALOG + + + + CREATE FOREIGN CATALOG + 7 + SQL - Language Statements + + + + CREATE FOREIGN CATALOG + define a new foreign catalog + + + + +CREATE FOREIGN CATALOG [ IF NOT EXISTS ] catalog_name + SERVER server_name + TYPE 'catalog_type' + [ OPTIONS ( option 'value' [, ...] ) ] + + + + + Description + + + CREATE FOREIGN CATALOG defines a new foreign catalog. + Foreign catalog names are global within a database. The user who creates + the catalog becomes its owner. + + + + Creating a foreign catalog requires USAGE privilege on + the referenced foreign server. + + + + The required catalog type is stored verbatim. Provider-specific validation + of the catalog type and options is outside the kernel, and generic options + currently have no kernel validator. + + + + + Parameters + + + + IF NOT EXISTS + + + Do not throw an error if a foreign catalog with the same name already + exists. A notice is issued in this case. + + + + + + catalog_name + + + The database-global name of the foreign catalog to be created. + + + + + + server_name + + + The name of an existing foreign server for the catalog. + + + + + + catalog_type + + + The required provider-specific catalog type. The value is stored + verbatim. + + + + + + option + + + The name of a provider-specific option for the catalog. + + + + + + value + + + The value of a provider-specific catalog option. + + + + + + + + Examples + + + Create a Hive catalog that uses the foreign server + hive_srv: + +CREATE FOREIGN CATALOG hive_cat SERVER hive_srv TYPE 'hive' OPTIONS (uri 'thrift://localhost:9083'); + + + + + Compatibility + + + CREATE FOREIGN CATALOG is an + Apache Cloudberry extension and is not defined in + the SQL standard. + + + + + See Also + + + + + + + + diff --git a/doc/src/sgml/ref/create_foreign_volume.sgml b/doc/src/sgml/ref/create_foreign_volume.sgml new file mode 100644 index 00000000000..7e81a4c134f --- /dev/null +++ b/doc/src/sgml/ref/create_foreign_volume.sgml @@ -0,0 +1,133 @@ + + + + + CREATE FOREIGN VOLUME + + + + CREATE FOREIGN VOLUME + 7 + SQL - Language Statements + + + + CREATE FOREIGN VOLUME + define a new foreign volume + + + + +CREATE FOREIGN VOLUME [ IF NOT EXISTS ] volume_name + SERVER server_name + [ OPTIONS ( option 'value' [, ...] ) ] + + + + + Description + + + CREATE FOREIGN VOLUME defines a new foreign volume. + Foreign volume names are global within a database. The user who creates + the volume becomes its owner. + + + + Creating a foreign volume requires USAGE privilege on + the referenced foreign server. + + + + Volume options are provider-specific. Provider-specific option validation + is outside the kernel, and generic options currently have no kernel + validator. + + + + + Parameters + + + + IF NOT EXISTS + + + Do not throw an error if a foreign volume with the same name already + exists. A notice is issued in this case. + + + + + + volume_name + + + The database-global name of the foreign volume to be created. + + + + + + server_name + + + The name of an existing foreign server for the volume. + + + + + + option + + + The name of a provider-specific option for the volume. + + + + + + value + + + The value of a provider-specific volume option. + + + + + + + + Examples + + + Create a volume for an object storage prefix using the foreign server + s3_srv: + +CREATE FOREIGN VOLUME s3_vol SERVER s3_srv OPTIONS (path 's3://bucket/prefix'); + + + + + Compatibility + + + CREATE FOREIGN VOLUME is an + Apache Cloudberry extension and is not defined in + the SQL standard. + + + + + See Also + + + + + + + + diff --git a/doc/src/sgml/ref/create_iceberg_table.sgml b/doc/src/sgml/ref/create_iceberg_table.sgml new file mode 100644 index 00000000000..dcff972e45b --- /dev/null +++ b/doc/src/sgml/ref/create_iceberg_table.sgml @@ -0,0 +1,193 @@ + + + + + CREATE ICEBERG TABLE + + + + CREATE ICEBERG TABLE + 7 + SQL - Language Statements + + + + CREATE ICEBERG TABLE + define a new Iceberg table + + + + +CREATE ICEBERG TABLE [ IF NOT EXISTS ] table_name ( [ + column_name data_type [, ... ] + ] ) + [ CATALOG catalog_name ] + [ VOLUME volume_name ] + [ OPTIONS ( option 'value' [, ...] ) ] + [ DISTRIBUTED BY ( column [, ... ] ) | DISTRIBUTED RANDOMLY | DISTRIBUTED REPLICATED ] + [ USING access_method ] + + + + + Description + + + CREATE ICEBERG TABLE defines a new Iceberg table. The + parenthesized element list is required, but it can be empty. The command + requires an extension that provides the iceberg table + access method. + + + + The foreign catalog and volume can be specified by the + CATALOG and VOLUME clauses. If either + clause is omitted, the corresponding iceberg_default_catalog + or iceberg_default_volume configuration parameter is used. + + + + Iceberg tables are always distributed randomly. A + DISTRIBUTED clause is accepted, but a warning is issued + and DISTRIBUTED RANDOMLY is used. The optional + USING clause accepts only iceberg and + is normally omitted. + + + + + Parameters + + + + IF NOT EXISTS + + + Do not throw an error if a relation with the same name already exists. + A notice is issued in this case. + + + + + + table_name + + + The name, optionally schema-qualified, of the Iceberg table to be + created. + + + + + + column_name + + + The name of a column in the new table. + + + + + + data_type + + + The data type of a column in the new table. + + + + + + catalog_name + + + The name of the foreign catalog to use. If omitted, the value of + iceberg_default_catalog is used. + + + + + + volume_name + + + The name of the foreign volume to use. If omitted, the value of + iceberg_default_volume is used. + + + + + + option + + + The name of an option for the Iceberg table. + + + + + + value + + + The value of an Iceberg table option. + + + + + + column + + + A column named in a DISTRIBUTED BY clause. The clause + is accepted with a warning; the table is distributed randomly instead. + + + + + + access_method + + + The table access method. The only accepted value is + iceberg. + + + + + + + + Examples + + + Create an Iceberg table using an explicit foreign catalog and volume: + +CREATE ICEBERG TABLE t (a int, b text) CATALOG hive_cat VOLUME s3_vol OPTIONS (fileformat 'parquet'); + + + + + Compatibility + + + CREATE ICEBERG TABLE is an + Apache Cloudberry extension and is not defined in + the SQL standard. + + + + + See Also + + + + + + + + + diff --git a/doc/src/sgml/ref/drop_foreign_catalog.sgml b/doc/src/sgml/ref/drop_foreign_catalog.sgml new file mode 100644 index 00000000000..330425abb61 --- /dev/null +++ b/doc/src/sgml/ref/drop_foreign_catalog.sgml @@ -0,0 +1,117 @@ + + + + + DROP FOREIGN CATALOG + + + + DROP FOREIGN CATALOG + 7 + SQL - Language Statements + + + + DROP FOREIGN CATALOG + remove a foreign catalog + + + + +DROP FOREIGN CATALOG [ IF EXISTS ] name [, ...] [ CASCADE | RESTRICT ] + + + + + Description + + + DROP FOREIGN CATALOG removes one or more foreign + catalogs. Foreign catalog names are global within a database. To execute + this command, the current user must own each catalog. + + + + Using CASCADE can also drop dependent Iceberg tables. + + + + + Parameters + + + + IF EXISTS + + + Do not throw an error if a foreign catalog does not exist. A notice is + issued in this case. + + + + + + name + + + The database-global name of a foreign catalog to drop. + + + + + + CASCADE + + + Automatically drop objects that depend on the catalog, including + dependent Iceberg tables, and in turn all objects that depend on those + objects (see ). + + + + + + RESTRICT + + + Refuse to drop the catalog if any objects depend on it. This is the + default. + + + + + + + + Examples + + + Drop the foreign catalog hive_cat: + +DROP FOREIGN CATALOG hive_cat; + + + + + Compatibility + + + DROP FOREIGN CATALOG is an + Apache Cloudberry extension and is not defined in + the SQL standard. + + + + + See Also + + + + + + + + diff --git a/doc/src/sgml/ref/drop_foreign_volume.sgml b/doc/src/sgml/ref/drop_foreign_volume.sgml new file mode 100644 index 00000000000..d387708aec2 --- /dev/null +++ b/doc/src/sgml/ref/drop_foreign_volume.sgml @@ -0,0 +1,117 @@ + + + + + DROP FOREIGN VOLUME + + + + DROP FOREIGN VOLUME + 7 + SQL - Language Statements + + + + DROP FOREIGN VOLUME + remove a foreign volume + + + + +DROP FOREIGN VOLUME [ IF EXISTS ] name [, ...] [ CASCADE | RESTRICT ] + + + + + Description + + + DROP FOREIGN VOLUME removes one or more foreign volumes. + Foreign volume names are global within a database. To execute this + command, the current user must own each volume. + + + + Using CASCADE can also drop dependent Iceberg tables. + + + + + Parameters + + + + IF EXISTS + + + Do not throw an error if a foreign volume does not exist. A notice is + issued in this case. + + + + + + name + + + The database-global name of a foreign volume to drop. + + + + + + CASCADE + + + Automatically drop objects that depend on the volume, including + dependent Iceberg tables, and in turn all objects that depend on those + objects (see ). + + + + + + RESTRICT + + + Refuse to drop the volume if any objects depend on it. This is the + default. + + + + + + + + Examples + + + Drop the foreign volume s3_vol: + +DROP FOREIGN VOLUME s3_vol; + + + + + Compatibility + + + DROP FOREIGN VOLUME is an + Apache Cloudberry extension and is not defined in + the SQL standard. + + + + + See Also + + + + + + + + diff --git a/doc/src/sgml/ref/drop_iceberg_table.sgml b/doc/src/sgml/ref/drop_iceberg_table.sgml new file mode 100644 index 00000000000..caba9cd06c7 --- /dev/null +++ b/doc/src/sgml/ref/drop_iceberg_table.sgml @@ -0,0 +1,119 @@ + + + + + DROP ICEBERG TABLE + + + + DROP ICEBERG TABLE + 7 + SQL - Language Statements + + + + DROP ICEBERG TABLE + remove an Iceberg table + + + + +DROP ICEBERG TABLE [ IF EXISTS ] name [, ...] [ CASCADE | RESTRICT ] + + + + + Description + + + DROP ICEBERG TABLE removes one or more Iceberg tables. + Only the owner of an Iceberg table can remove it. + + + + This command only drops Iceberg tables. Plain + DROP TABLE rejects Iceberg tables and is used for + ordinary tables. + + + + + Parameters + + + + IF EXISTS + + + Do not throw an error if the Iceberg table does not exist. A notice is + issued in this case. + + + + + + name + + + The name, optionally schema-qualified, of an Iceberg table to drop. + + + + + + CASCADE + + + Automatically drop objects that depend on the Iceberg table, and in + turn all objects that depend on those objects (see ). + + + + + + RESTRICT + + + Refuse to drop the Iceberg table if any objects depend on it. This is + the default. + + + + + + + + Examples + + + Drop the Iceberg table t: + +DROP ICEBERG TABLE t; + + + + + Compatibility + + + DROP ICEBERG TABLE is an + Apache Cloudberry extension and is not defined in + the SQL standard. + + + + + See Also + + + + + + + + + diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index a251363388e..6c374160d73 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -96,10 +96,13 @@ &createDynamicTable; &createEventTrigger; &createExtension; + &createForeignCatalog; &createForeignDataWrapper; &createForeignTable; + &createForeignVolume; &createFunction; &createGroup; + &createIcebergTable; &createIndex; &createLanguage; &createMaterializedView; @@ -144,10 +147,13 @@ &dropDynamicTable; &dropEventTrigger; &dropExtension; + &dropForeignCatalog; &dropForeignDataWrapper; &dropForeignTable; + &dropForeignVolume; &dropFunction; &dropGroup; + &dropIcebergTable; &dropIndex; &dropLanguage; &dropMaterializedView; From 199a0270ccd182d9ccd88b53add214b03ca2968b Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Mon, 20 Jul 2026 12:03:54 +0800 Subject: [PATCH 19/29] parser/commands: switch lake-table DDL to CREATE LAKE TABLE ... USING Per the PR #1842 review (leborchuk, andr-sokolov both voted for it), rename the user-facing syntax from CREATE/DROP ICEBERG TABLE to CREATE/DROP LAKE TABLE, with the table format given by a required USING clause, to leave room for non-Iceberg lake formats. The command tags, catalog (pg_lake_table), and internal names were already "lake"; this aligns the grammar with them. - gram.y/kwlist.h: drop the ICEBERG keyword, add a LAKE keyword. CREATE LAKE TABLE takes USING right after the column list (format is any identifier, upper-cased into table_type); DROP LAKE TABLE. - laketablecmds.c: the USING format now determines both table_type and the access method, so remove the redundant access-method guard; unsupported formats are rejected by validate_table_type; refresh the hints. - tablecmds/foreigncmds/pg_dump/tab-complete: user-facing messages, the DROP-completion + object-type list, and the pg_dump skip warning now say "lake table" / "CREATE LAKE TABLE ... USING ICEBERG". - docs: rename create/drop_iceberg_table.sgml -> *_lake_table.sgml and drop the meaningless DISTRIBUTED line from the CREATE synopsis (andr-sokolov). - regress: update lake_table for the new syntax and drop the pointless ORDER BY 1 single-row sorts (andr-sokolov). --- doc/src/sgml/ref/allfiles.sgml | 4 +- doc/src/sgml/ref/create_foreign_catalog.sgml | 2 +- doc/src/sgml/ref/create_foreign_volume.sgml | 2 +- ...berg_table.sgml => create_lake_table.sgml} | 91 +++++++++---------- doc/src/sgml/ref/drop_foreign_catalog.sgml | 6 +- doc/src/sgml/ref/drop_foreign_volume.sgml | 6 +- ...ceberg_table.sgml => drop_lake_table.sgml} | 46 +++++----- doc/src/sgml/reference.sgml | 4 +- src/backend/commands/foreigncmds.c | 4 +- src/backend/commands/laketablecmds.c | 30 ++---- src/backend/commands/tablecmds.c | 16 ++-- src/backend/parser/gram.y | 68 ++++++++------ src/bin/pg_dump/pg_dump.c | 2 +- src/bin/psql/tab-complete.c | 29 +++--- src/include/parser/kwlist.h | 2 +- src/test/regress/expected/lake_table.out | 65 +++++++------ src/test/regress/sql/lake_table.sql | 41 ++++----- 17 files changed, 206 insertions(+), 212 deletions(-) rename doc/src/sgml/ref/{create_iceberg_table.sgml => create_lake_table.sgml} (61%) rename doc/src/sgml/ref/{drop_iceberg_table.sgml => drop_lake_table.sgml} (56%) diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index 41a275686a1..8819f0545b2 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -75,8 +75,8 @@ Complete list of usable sgml source files in this directory. - + @@ -127,8 +127,8 @@ Complete list of usable sgml source files in this directory. - + diff --git a/doc/src/sgml/ref/create_foreign_catalog.sgml b/doc/src/sgml/ref/create_foreign_catalog.sgml index ee32abf6b47..771503a8bed 100644 --- a/doc/src/sgml/ref/create_foreign_catalog.sgml +++ b/doc/src/sgml/ref/create_foreign_catalog.sgml @@ -137,7 +137,7 @@ CREATE FOREIGN CATALOG hive_cat SERVER hive_srv TYPE 'hive' OPTIONS (uri 'thrift - + diff --git a/doc/src/sgml/ref/create_foreign_volume.sgml b/doc/src/sgml/ref/create_foreign_volume.sgml index 7e81a4c134f..eb41c65705e 100644 --- a/doc/src/sgml/ref/create_foreign_volume.sgml +++ b/doc/src/sgml/ref/create_foreign_volume.sgml @@ -126,7 +126,7 @@ CREATE FOREIGN VOLUME s3_vol SERVER s3_srv OPTIONS (path 's3://bucket/prefix'); - + diff --git a/doc/src/sgml/ref/create_iceberg_table.sgml b/doc/src/sgml/ref/create_lake_table.sgml similarity index 61% rename from doc/src/sgml/ref/create_iceberg_table.sgml rename to doc/src/sgml/ref/create_lake_table.sgml index dcff972e45b..043c4d5e750 100644 --- a/doc/src/sgml/ref/create_iceberg_table.sgml +++ b/doc/src/sgml/ref/create_lake_table.sgml @@ -1,45 +1,49 @@ - - - CREATE ICEBERG TABLE + + + CREATE LAKE TABLE - CREATE ICEBERG TABLE + CREATE LAKE TABLE 7 SQL - Language Statements - CREATE ICEBERG TABLE - define a new Iceberg table + CREATE LAKE TABLE + define a new lake table -CREATE ICEBERG TABLE [ IF NOT EXISTS ] table_name ( [ +CREATE LAKE TABLE [ IF NOT EXISTS ] table_name ( [ column_name data_type [, ... ] ] ) + USING format [ CATALOG catalog_name ] [ VOLUME volume_name ] [ OPTIONS ( option 'value' [, ...] ) ] - [ DISTRIBUTED BY ( column [, ... ] ) | DISTRIBUTED RANDOMLY | DISTRIBUTED REPLICATED ] - [ USING access_method ] - + Description - CREATE ICEBERG TABLE defines a new Iceberg table. The - parenthesized element list is required, but it can be empty. The command - requires an extension that provides the iceberg table - access method. + CREATE LAKE TABLE defines a new lake table: a table whose + data lives in external object storage and is described by an open table + format. The parenthesized element list is required, but it can be empty. + + + + The USING clause names the table format. The only + supported format is ICEBERG, which requires an extension + that provides the iceberg table access method. @@ -50,11 +54,9 @@ CREATE ICEBERG TABLE [ IF NOT EXISTS ] table_name - Iceberg tables are always distributed randomly. A - DISTRIBUTED clause is accepted, but a warning is issued - and DISTRIBUTED RANDOMLY is used. The optional - USING clause accepts only iceberg and - is normally omitted. + Lake tables are always distributed randomly, because their fragments are not + hash-distributed across segments. A DISTRIBUTED clause, + if given, is accepted with a warning and has no effect. @@ -76,8 +78,7 @@ CREATE ICEBERG TABLE [ IF NOT EXISTS ] table_name table_name - The name, optionally schema-qualified, of the Iceberg table to be - created. + The name, optionally schema-qualified, of the lake table to be created. @@ -100,6 +101,16 @@ CREATE ICEBERG TABLE [ IF NOT EXISTS ] table_name + + format + + + The lake table format. The only accepted value is + ICEBERG. + + + + catalog_name @@ -124,7 +135,7 @@ CREATE ICEBERG TABLE [ IF NOT EXISTS ] table_name option - The name of an option for the Iceberg table. + The name of an option for the lake table. @@ -133,48 +144,28 @@ CREATE ICEBERG TABLE [ IF NOT EXISTS ] table_name value - The value of an Iceberg table option. - - - - - - column - - - A column named in a DISTRIBUTED BY clause. The clause - is accepted with a warning; the table is distributed randomly instead. - - - - - - access_method - - - The table access method. The only accepted value is - iceberg. + The value of a lake table option. - + Examples - Create an Iceberg table using an explicit foreign catalog and volume: + Create an Iceberg lake table using an explicit foreign catalog and volume: -CREATE ICEBERG TABLE t (a int, b text) CATALOG hive_cat VOLUME s3_vol OPTIONS (fileformat 'parquet'); +CREATE LAKE TABLE t (a int, b text) USING ICEBERG CATALOG hive_cat VOLUME s3_vol OPTIONS (fileformat 'parquet'); - + Compatibility - CREATE ICEBERG TABLE is an + CREATE LAKE TABLE is an Apache Cloudberry extension and is not defined in the SQL standard. @@ -184,7 +175,7 @@ CREATE ICEBERG TABLE t (a int, b text) CATALOG hive_cat VOLUME s3_vol OPTIONS (f See Also - + diff --git a/doc/src/sgml/ref/drop_foreign_catalog.sgml b/doc/src/sgml/ref/drop_foreign_catalog.sgml index 330425abb61..91c2a0d70cd 100644 --- a/doc/src/sgml/ref/drop_foreign_catalog.sgml +++ b/doc/src/sgml/ref/drop_foreign_catalog.sgml @@ -35,7 +35,7 @@ DROP FOREIGN CATALOG [ IF EXISTS ] name - Using CASCADE can also drop dependent Iceberg tables. + Using CASCADE can also drop dependent lake tables. @@ -67,7 +67,7 @@ DROP FOREIGN CATALOG [ IF EXISTS ] name Automatically drop objects that depend on the catalog, including - dependent Iceberg tables, and in turn all objects that depend on those + dependent lake tables, and in turn all objects that depend on those objects (see ). @@ -110,7 +110,7 @@ DROP FOREIGN CATALOG hive_cat; - + diff --git a/doc/src/sgml/ref/drop_foreign_volume.sgml b/doc/src/sgml/ref/drop_foreign_volume.sgml index d387708aec2..7e8783cdeb9 100644 --- a/doc/src/sgml/ref/drop_foreign_volume.sgml +++ b/doc/src/sgml/ref/drop_foreign_volume.sgml @@ -35,7 +35,7 @@ DROP FOREIGN VOLUME [ IF EXISTS ] name - Using CASCADE can also drop dependent Iceberg tables. + Using CASCADE can also drop dependent lake tables. @@ -67,7 +67,7 @@ DROP FOREIGN VOLUME [ IF EXISTS ] name Automatically drop objects that depend on the volume, including - dependent Iceberg tables, and in turn all objects that depend on those + dependent lake tables, and in turn all objects that depend on those objects (see ). @@ -110,7 +110,7 @@ DROP FOREIGN VOLUME s3_vol; - + diff --git a/doc/src/sgml/ref/drop_iceberg_table.sgml b/doc/src/sgml/ref/drop_lake_table.sgml similarity index 56% rename from doc/src/sgml/ref/drop_iceberg_table.sgml rename to doc/src/sgml/ref/drop_lake_table.sgml index caba9cd06c7..0847f7efbbb 100644 --- a/doc/src/sgml/ref/drop_iceberg_table.sgml +++ b/doc/src/sgml/ref/drop_lake_table.sgml @@ -1,41 +1,41 @@ - - - DROP ICEBERG TABLE + + + DROP LAKE TABLE - DROP ICEBERG TABLE + DROP LAKE TABLE 7 SQL - Language Statements - DROP ICEBERG TABLE - remove an Iceberg table + DROP LAKE TABLE + remove a lake table -DROP ICEBERG TABLE [ IF EXISTS ] name [, ...] [ CASCADE | RESTRICT ] +DROP LAKE TABLE [ IF EXISTS ] name [, ...] [ CASCADE | RESTRICT ] - + Description - DROP ICEBERG TABLE removes one or more Iceberg tables. - Only the owner of an Iceberg table can remove it. + DROP LAKE TABLE removes one or more lake tables. + Only the owner of a lake table can remove it. - This command only drops Iceberg tables. Plain - DROP TABLE rejects Iceberg tables and is used for + This command only drops lake tables. Plain + DROP TABLE rejects lake tables and is used for ordinary tables. @@ -48,7 +48,7 @@ DROP ICEBERG TABLE [ IF EXISTS ] nameIF EXISTS - Do not throw an error if the Iceberg table does not exist. A notice is + Do not throw an error if the lake table does not exist. A notice is issued in this case. @@ -58,7 +58,7 @@ DROP ICEBERG TABLE [ IF EXISTS ] namename - The name, optionally schema-qualified, of an Iceberg table to drop. + The name, optionally schema-qualified, of a lake table to drop. @@ -67,7 +67,7 @@ DROP ICEBERG TABLE [ IF EXISTS ] nameCASCADE - Automatically drop objects that depend on the Iceberg table, and in + Automatically drop objects that depend on the lake table, and in turn all objects that depend on those objects (see ). @@ -78,7 +78,7 @@ DROP ICEBERG TABLE [ IF EXISTS ] nameRESTRICT - Refuse to drop the Iceberg table if any objects depend on it. This is + Refuse to drop the lake table if any objects depend on it. This is the default. @@ -86,21 +86,21 @@ DROP ICEBERG TABLE [ IF EXISTS ] name - + Examples - Drop the Iceberg table t: + Drop the lake table t: -DROP ICEBERG TABLE t; +DROP LAKE TABLE t; - + Compatibility - DROP ICEBERG TABLE is an + DROP LAKE TABLE is an Apache Cloudberry extension and is not defined in the SQL standard. @@ -110,7 +110,7 @@ DROP ICEBERG TABLE t; See Also - + diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index 6c374160d73..3aaf0601793 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -102,8 +102,8 @@ &createForeignVolume; &createFunction; &createGroup; - &createIcebergTable; &createIndex; + &createLakeTable; &createLanguage; &createMaterializedView; &createOperator; @@ -153,8 +153,8 @@ &dropForeignVolume; &dropFunction; &dropGroup; - &dropIcebergTable; &dropIndex; + &dropLakeTable; &dropLanguage; &dropMaterializedView; &dropOperator; diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index 46a90b88ef7..892871ef0cf 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -1054,7 +1054,7 @@ CreateForeignCatalog(CreateForeignCatalogStmt *stmt) /* * Check that there is no other foreign catalog by this name. Catalog * names are global (like server names): every reference syntax (DROP - * CATALOG, the CATALOG clause of CREATE ICEBERG TABLE, GUCs) identifies + * CATALOG, the CATALOG clause of CREATE LAKE TABLE, GUCs) identifies * a catalog by bare name, so the name alone must be unique. If there is * one, do nothing if IF NOT EXISTS was specified. */ @@ -1195,7 +1195,7 @@ CreateForeignVolume(CreateForeignVolumeStmt *stmt) /* * Check that there is no other foreign volume by this name. Volume * names are global (like server names): every reference syntax (DROP - * VOLUME, the VOLUME clause of CREATE ICEBERG TABLE, GUCs) identifies + * VOLUME, the VOLUME clause of CREATE LAKE TABLE, GUCs) identifies * a volume by bare name, so the name alone must be unique. If there is * one, do nothing if IF NOT EXISTS was specified. */ diff --git a/src/backend/commands/laketablecmds.c b/src/backend/commands/laketablecmds.c index d5c9274b049..3d00077b407 100644 --- a/src/backend/commands/laketablecmds.c +++ b/src/backend/commands/laketablecmds.c @@ -223,13 +223,13 @@ validate_table_type(const char *table_type) if (!table_type) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("table type cannot be NULL"))); + errmsg("lake table format cannot be NULL"))); if (strcmp(table_type, "ICEBERG") != 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("unsupported table type \"%s\"", table_type), - errhint("The only supported table type is ICEBERG."))); + errmsg("unsupported lake table format \"%s\"", table_type), + errhint("The only supported format is ICEBERG (USING ICEBERG)."))); } /* @@ -242,7 +242,7 @@ validate_foreign_catalog(const char *catalog_name) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no foreign catalog specified"), - errhint("Specify CATALOG in CREATE ICEBERG TABLE or set iceberg_default_catalog."))); + errhint("Specify CATALOG in CREATE LAKE TABLE or set iceberg_default_catalog."))); return get_foreign_catalog_oid(catalog_name, false); } @@ -257,7 +257,7 @@ validate_foreign_volume(const char *volume_name) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("no foreign volume specified"), - errhint("Specify VOLUME in CREATE ICEBERG TABLE or set iceberg_default_volume."))); + errhint("Specify VOLUME in CREATE LAKE TABLE or set iceberg_default_volume."))); return get_foreign_volume_oid(volume_name, false); } @@ -290,26 +290,10 @@ ResolveLakeTableOptions(CreateLakeTableStmt *stmt, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("table access method \"%s\" does not exist", ICEBERG_TABLE_AM_NAME), - errhint("CREATE ICEBERG TABLE requires an extension that provides the \"%s\" table access method.", + errhint("CREATE LAKE TABLE ... USING ICEBERG requires an extension that provides the \"%s\" table access method.", ICEBERG_TABLE_AM_NAME))); - /* - * The grammar accepts a USING clause, but an iceberg table must use the - * iceberg access method: a lake table created with another AM would get - * pg_lake_table metadata without lake-table semantics (and DROP TABLE, - * which detects lake tables by their AM, would leave that metadata - * behind as an orphaned row). - */ - if (stmt->base.accessMethod != NULL && - strcmp(stmt->base.accessMethod, ICEBERG_TABLE_AM_NAME) != 0) - ereport(ERROR, - (errcode(ERRCODE_INVALID_TABLE_DEFINITION), - errmsg("access method \"%s\" is not supported for iceberg tables", - stmt->base.accessMethod), - errhint("Omit the USING clause; CREATE ICEBERG TABLE always uses the \"%s\" access method.", - ICEBERG_TABLE_AM_NAME))); - - /* Validate table type */ + /* Validate the table format named in the USING clause */ validate_table_type(stmt->table_type); /* diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 641bb653a22..a8d745cd798 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -256,7 +256,7 @@ struct DropRelationCallbackState { /* These fields are set by RemoveRelations: */ char expected_relkind; - bool iceberg_only; /* DROP ICEBERG TABLE: require iceberg AM */ + bool iceberg_only; /* DROP LAKE TABLE: require iceberg AM */ LOCKMODE heap_lockmode; /* These fields are state to track which subsidiary locks are held: */ Oid heapOid; @@ -941,7 +941,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, } /* - * Lake tables must be created through CREATE ICEBERG TABLE, which also + * Lake tables must be created through CREATE LAKE TABLE, which also * creates the pg_lake_table catalog entries the iceberg access method * relies on. A relation created with the iceberg AM through any other * path (CREATE TABLE ... USING iceberg, CTAS, matview, @@ -956,7 +956,7 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot create table \"%s\" with access method \"%s\"", stmt->relation->relname, ICEBERG_TABLE_AM_NAME), - errhint("Use CREATE ICEBERG TABLE instead."))); + errhint("Use CREATE LAKE TABLE ... USING ICEBERG instead."))); /* * GPDB: for partitioned tables, inherit reloptions from the parent. @@ -2223,7 +2223,7 @@ RangeVarCallbackForDropRelation(const RangeVar *rel, Oid relOid, Oid oldRelOid, /* * Iceberg (lake) tables share RELKIND_RELATION with ordinary tables and are - * told apart only by their access method. DROP ICEBERG TABLE must target one; + * told apart only by their access method. DROP LAKE TABLE must target one; * plain DROP TABLE must NOT (mirrors the foreign-table rule) -- direct the user * to the matching command in each case. */ @@ -2239,7 +2239,7 @@ RangeVarCallbackForDropRelation(const RangeVar *rel, Oid relOid, Oid oldRelOid, if (!is_iceberg) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("\"%s\" is not an iceberg table", rel->relname), + errmsg("\"%s\" is not a lake table", rel->relname), errhint("Use DROP TABLE to remove a table."))); } else if (is_iceberg) @@ -2247,7 +2247,7 @@ RangeVarCallbackForDropRelation(const RangeVar *rel, Oid relOid, Oid oldRelOid, ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a table", rel->relname), - errhint("Use DROP ICEBERG TABLE to remove an iceberg table."))); + errhint("Use DROP LAKE TABLE to remove a lake table."))); } } @@ -17120,7 +17120,7 @@ ATPrepSetAccessMethod(AlteredTableInfo *tab, Relation rel, const char *amname) /* * The iceberg AM relies on catalog entries that only the CREATE/DROP - * ICEBERG TABLE paths manage, so a table cannot be converted to or from + * LAKE TABLE paths manage, so a table cannot be converted to or from * it with SET ACCESS METHOD. */ iceberg_amoid = GetIcebergTableAmOid(true); @@ -17129,7 +17129,7 @@ ATPrepSetAccessMethod(AlteredTableInfo *tab, Relation rel, const char *amname) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot change access method of table \"%s\" to \"%s\"", RelationGetRelationName(rel), ICEBERG_TABLE_AM_NAME), - errhint("Use CREATE ICEBERG TABLE to create an iceberg table."))); + errhint("Use CREATE LAKE TABLE ... USING ICEBERG to create a lake table."))); if (OidIsValid(iceberg_amoid) && rel->rd_rel->relam == iceberg_amoid) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index fc28d5ce5bf..b34d0e88caf 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -832,7 +832,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ HANDLER HAVING HEADER_P HOLD HOUR_P - ICEBERG IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE + IDENTITY_P IF_P ILIKE IMMEDIATE IMMUTABLE IMPLICIT_P IMPORT_P IN_P INCLUDE INCLUDING INCREMENT INCREMENTAL INDENT INDEX INDEXES INHERIT INHERITS INITIALLY INLINE_P INNER_P INOUT INPUT_P INSENSITIVE INSERT INSTEAD INT_P INTEGER INTERSECT INTERVAL INTO INVOKER IS ISNULL ISOLATION @@ -841,7 +841,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ KEY KEYS - LABEL LANGUAGE LARGE_P LAST_P LATERAL_P + LABEL LAKE LANGUAGE LARGE_P LAST_P LATERAL_P LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOCUS LOGGED @@ -9169,21 +9169,30 @@ OptForeignVolume: /***************************************************************************** * * QUERY: - * CREATE ICEBERG TABLE relname (columns) - * [FOREIGN CATALOG cat] [FOREIGN VOLUME vol] OPTIONS (...) + * CREATE LAKE TABLE relname (columns) USING format + * [CATALOG cat] [VOLUME vol] OPTIONS (...) * * A lake table stores its data on external object storage; fragments are * not hash-distributed across segments, so the distribution policy is - * forced to RANDOM to keep UPDATE/DELETE correct. + * forced to RANDOM to keep UPDATE/DELETE correct. The USING clause names + * the table format (currently only ICEBERG); the format also determines the + * table access method the relation is created with. * *****************************************************************************/ CreateLakeTableStmt: - CREATE ICEBERG TABLE qualified_name '(' OptTableElementList ')' + CREATE LAKE TABLE qualified_name '(' OptTableElementList ')' + USING name OptForeignCatalog OptForeignVolume create_generic_options - OptDistributedBy table_access_method_clause + OptDistributedBy { CreateLakeTableStmt *n = makeNode(CreateLakeTableStmt); + char *table_type = pstrdup($9); + char *p; + + for (p = table_type; *p; p++) + *p = pg_toupper((unsigned char) *p); + $4->relpersistence = RELPERSISTENCE_PERMANENT; n->base.relation = $4; n->base.tableElts = $6; @@ -9193,14 +9202,14 @@ CreateLakeTableStmt: n->base.options = NIL; n->base.oncommit = ONCOMMIT_NOOP; n->base.tablespacename = NULL; - n->base.accessMethod = $12 ? $12 : pstrdup("iceberg"); + n->base.accessMethod = $9; n->base.if_not_exists = false; n->base.relKind = RELKIND_RELATION; - n->table_type = pstrdup("ICEBERG"); - n->foreign_catalog = $8 ? pstrdup($8) : NULL; - n->foreign_volume = $9 ? pstrdup($9) : NULL; - n->options = $10; - if ($11 != NULL) + n->table_type = table_type; + n->foreign_catalog = $10 ? pstrdup($10) : NULL; + n->foreign_volume = $11 ? pstrdup($11) : NULL; + n->options = $12; + if ($13 != NULL) ereport(WARNING, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("DISTRIBUTED clause has no effect for lake tables, using DISTRIBUTED RANDOMLY"))); @@ -9210,11 +9219,18 @@ CreateLakeTableStmt: n->base.distributedBy->numsegments = -1; $$ = (Node *) n; } - | CREATE ICEBERG TABLE IF_P NOT EXISTS qualified_name '(' OptTableElementList ')' + | CREATE LAKE TABLE IF_P NOT EXISTS qualified_name '(' OptTableElementList ')' + USING name OptForeignCatalog OptForeignVolume create_generic_options - OptDistributedBy table_access_method_clause + OptDistributedBy { CreateLakeTableStmt *n = makeNode(CreateLakeTableStmt); + char *table_type = pstrdup($12); + char *p; + + for (p = table_type; *p; p++) + *p = pg_toupper((unsigned char) *p); + $7->relpersistence = RELPERSISTENCE_PERMANENT; n->base.relation = $7; n->base.tableElts = $9; @@ -9224,14 +9240,14 @@ CreateLakeTableStmt: n->base.options = NIL; n->base.oncommit = ONCOMMIT_NOOP; n->base.tablespacename = NULL; - n->base.accessMethod = $15 ? $15 : pstrdup("iceberg"); + n->base.accessMethod = $12; n->base.if_not_exists = true; n->base.relKind = RELKIND_RELATION; - n->table_type = pstrdup("ICEBERG"); - n->foreign_catalog = $11 ? pstrdup($11) : NULL; - n->foreign_volume = $12 ? pstrdup($12) : NULL; - n->options = $13; - if ($14 != NULL) + n->table_type = table_type; + n->foreign_catalog = $13 ? pstrdup($13) : NULL; + n->foreign_volume = $14 ? pstrdup($14) : NULL; + n->options = $15; + if ($16 != NULL) ereport(WARNING, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("DISTRIBUTED clause has no effect for lake tables, using DISTRIBUTED RANDOMLY"))); @@ -10532,8 +10548,8 @@ DropStmt: DROP object_type_any_name IF_P EXISTS any_name_list opt_drop_behavior n->isdynamic = true; $$ = (Node *)n; } -/* DROP ICEBERG TABLE */ - | DROP ICEBERG TABLE IF_P EXISTS any_name_list opt_drop_behavior +/* DROP LAKE TABLE */ + | DROP LAKE TABLE IF_P EXISTS any_name_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); n->removeType = OBJECT_TABLE; @@ -10545,7 +10561,7 @@ DropStmt: DROP object_type_any_name IF_P EXISTS any_name_list opt_drop_behavior n->isiceberg = true; $$ = (Node *)n; } - | DROP ICEBERG TABLE any_name_list opt_drop_behavior + | DROP LAKE TABLE any_name_list opt_drop_behavior { DropStmt *n = makeNode(DropStmt); n->removeType = OBJECT_TABLE; @@ -21352,7 +21368,6 @@ unreserved_keyword: | HOLD | HOST | HOUR_P - | ICEBERG | IDENTITY_P | IF_P | IGNORE_P @@ -21383,6 +21398,7 @@ unreserved_keyword: | KEY | KEYS | LABEL + | LAKE | LANGUAGE | LARGE_P | LAST_P @@ -22341,7 +22357,6 @@ bare_label_keyword: | HEADER_P | HOLD | HOST - | ICEBERG | IDENTITY_P | IF_P | IGNORE_P @@ -22386,6 +22401,7 @@ bare_label_keyword: | KEY | KEYS | LABEL + | LAKE | LANGUAGE | LARGE_P | LAST_P diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 1f091d26f83..d7857a6abad 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -2144,7 +2144,7 @@ selectDumpableTable(TableInfo *tbinfo, Archive *fout) { tbinfo->dobj.dump = DUMP_COMPONENT_NONE; - pg_log_warning("iceberg table \"%s\" is not supported by pg_dump and will be ignored", + pg_log_warning("lake table \"%s\" is not supported by pg_dump and will be ignored", tbinfo->dobj.name); } } diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index b7c83596f79..65bc6727a41 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -669,7 +669,7 @@ static const SchemaQuery Query_for_list_of_foreign_tables = { }; /* - * Exclude iceberg tables; Query_for_list_of_iceberg_tables serves DROP ICEBERG + * Exclude lake tables; Query_for_list_of_iceberg_tables serves DROP LAKE * TABLE. With no iceberg AM, the NOT EXISTS subquery finds no match, so * ordinary tables remain listed. */ @@ -686,10 +686,10 @@ static const SchemaQuery Query_for_list_of_tables = { }; /* - * An iceberg (lake) table is an ordinary relation whose access method is the - * "iceberg" AM -- the same rule DROP ICEBERG TABLE validates against. When no + * A lake table is an ordinary relation whose access method is the + * "iceberg" AM -- the same rule DROP LAKE TABLE validates against. When no * provider installed that AM the scalar subquery yields NULL and the list is - * empty, which is correct (no relation can be an iceberg table). + * empty, which is correct (no relation can be a lake table). */ static const SchemaQuery Query_for_list_of_iceberg_tables = { .catname = "pg_catalog.pg_class c", @@ -1314,9 +1314,9 @@ static const pgsql_thing_t words_after_create[] = { {"FOREIGN VOLUME", Query_for_list_of_foreign_volumes, NULL, NULL, NULL, THING_NO_ALTER}, {"FUNCTION", NULL, NULL, Query_for_list_of_functions}, {"GROUP", Query_for_list_of_roles}, - {"ICEBERG TABLE", NULL, NULL, &Query_for_list_of_iceberg_tables, NULL, THING_NO_ALTER}, {"INCREMENTAL MATERIALIZED VIEW", NULL, NULL, &Query_for_list_of_matviews, NULL, THING_NO_DROP | THING_NO_ALTER}, {"INDEX", NULL, NULL, &Query_for_list_of_indexes}, + {"LAKE TABLE", NULL, NULL, &Query_for_list_of_iceberg_tables, NULL, THING_NO_ALTER}, {"LANGUAGE", Query_for_list_of_languages}, {"LARGE OBJECT", NULL, NULL, NULL, NULL, THING_NO_CREATE | THING_NO_DROP}, {"MATERIALIZED VIEW", NULL, NULL, &Query_for_list_of_matviews}, @@ -3097,9 +3097,14 @@ psql_completion(const char *text, int start, int end) else if (Matches("CREATE", "FOREIGN", "VOLUME", MatchAny, "SERVER", MatchAny)) COMPLETE_WITH("OPTIONS"); - /* CREATE ICEBERG */ - else if (Matches("CREATE", "ICEBERG")) + /* CREATE LAKE TABLE */ + else if (Matches("CREATE", "LAKE")) COMPLETE_WITH("TABLE"); + /* CREATE LAKE TABLE ... USING */ + else if (HeadMatches("CREATE", "LAKE", "TABLE") && TailMatches("USING")) + COMPLETE_WITH("ICEBERG"); + else if (HeadMatches("CREATE", "LAKE", "TABLE") && TailMatches("USING", "ICEBERG")) + COMPLETE_WITH("CATALOG", "VOLUME", "OPTIONS"); /* CREATE FOREIGN DATA WRAPPER */ else if (Matches("CREATE", "FOREIGN", "DATA", "WRAPPER", MatchAny)) @@ -3867,7 +3872,7 @@ psql_completion(const char *text, int start, int end) Matches("DROP", "FOREIGN", "DATA", "WRAPPER", MatchAny) || Matches("DROP", "FOREIGN", "TABLE", MatchAny) || Matches("DROP", "FOREIGN", "CATALOG|VOLUME", MatchAny) || - Matches("DROP", "ICEBERG", "TABLE", MatchAny) || + Matches("DROP", "LAKE", "TABLE", MatchAny) || Matches("DROP", "DIRECTORY", "TABLE", MatchAny) || Matches("DROP", "TEXT", "SEARCH", "CONFIGURATION|DICTIONARY|PARSER|TEMPLATE", MatchAny)) COMPLETE_WITH("CASCADE", "RESTRICT"); @@ -3883,9 +3888,9 @@ psql_completion(const char *text, int start, int end) COMPLETE_WITH_QUERY(Query_for_list_of_foreign_catalogs); else if (Matches("DROP", "FOREIGN", "VOLUME")) COMPLETE_WITH_QUERY(Query_for_list_of_foreign_volumes); - else if (Matches("DROP", "ICEBERG")) + else if (Matches("DROP", "LAKE")) COMPLETE_WITH("TABLE"); - else if (Matches("DROP", "ICEBERG", "TABLE")) + else if (Matches("DROP", "LAKE", "TABLE")) COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_iceberg_tables); else if (Matches("DROP", "DATABASE", MatchAny)) COMPLETE_WITH("WITH ("); @@ -4107,13 +4112,13 @@ psql_completion(const char *text, int start, int end) else if (TailMatches("FOREIGN", "SERVER")) COMPLETE_WITH_QUERY(Query_for_list_of_servers); -/* CATALOG, e.g. the CATALOG clause of CREATE ICEBERG TABLE */ +/* CATALOG, e.g. the CATALOG clause of CREATE LAKE TABLE */ else if (TailMatches("CATALOG") && !TailMatches("CREATE", MatchAny, MatchAny) && !TailMatches("FOREIGN", MatchAny)) COMPLETE_WITH_QUERY(Query_for_list_of_foreign_catalogs); -/* VOLUME, e.g. the VOLUME clause of CREATE ICEBERG TABLE */ +/* VOLUME, e.g. the VOLUME clause of CREATE LAKE TABLE */ else if (TailMatches("VOLUME") && !TailMatches("CREATE", MatchAny, MatchAny) && !TailMatches("FOREIGN", MatchAny)) diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 795b317f175..60315094de6 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -225,7 +225,6 @@ PG_KEYWORD("header", HEADER_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("hold", HOLD, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("host", HOST, UNRESERVED_KEYWORD, BARE_LABEL) /* GPDB */ PG_KEYWORD("hour", HOUR_P, UNRESERVED_KEYWORD, AS_LABEL) -PG_KEYWORD("iceberg", ICEBERG, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("identity", IDENTITY_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("if", IF_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("ignore", IGNORE_P, UNRESERVED_KEYWORD, BARE_LABEL) @@ -273,6 +272,7 @@ PG_KEYWORD("json_objectagg", JSON_OBJECTAGG, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("keys", KEYS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("label", LABEL, UNRESERVED_KEYWORD, BARE_LABEL) +PG_KEYWORD("lake", LAKE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("language", LANGUAGE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("large", LARGE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("last", LAST_P, UNRESERVED_KEYWORD, BARE_LABEL) diff --git a/src/test/regress/expected/lake_table.out b/src/test/regress/expected/lake_table.out index b3ca57ea1d2..e310c3a9351 100644 --- a/src/test/regress/expected/lake_table.out +++ b/src/test/regress/expected/lake_table.out @@ -1,5 +1,5 @@ -- --- Test lake table DDL: FOREIGN CATALOG, FOREIGN VOLUME, ICEBERG TABLE +-- Test lake table DDL: FOREIGN CATALOG, FOREIGN VOLUME, LAKE TABLE -- -- Display the lake table catalogs \d+ pg_foreign_catalog @@ -62,7 +62,7 @@ CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv2 TYPE 'h NOTICE: foreign catalog "lake_test_cat" already exists, skipping CREATE FOREIGN CATALOG lake_test_bad SERVER no_such_server TYPE 'hive'; -- fail, no server ERROR: server "no_such_server" does not exist -SELECT fcname, fctype, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%' ORDER BY 1; +SELECT fcname, fctype, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%'; fcname | fctype | fcoptions ---------------+--------+------------------------------- lake_test_cat | hive | {uri=thrift://localhost:9083} @@ -81,7 +81,7 @@ CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv2; -- skip NOTICE: foreign volume "lake_test_vol" already exists, skipping CREATE FOREIGN VOLUME lake_test_bad SERVER no_such_server; -- fail, no server ERROR: server "no_such_server" does not exist -SELECT fvname, fvoptions FROM pg_foreign_volume WHERE fvname LIKE 'lake\_test%' ORDER BY 1; +SELECT fvname, fvoptions FROM pg_foreign_volume WHERE fvname LIKE 'lake\_test%'; fvname | fvoptions ---------------+--------------------------- lake_test_vol | {path=s3://bucket/prefix} @@ -118,9 +118,9 @@ SELECT count(DISTINCT gp_segment_id) > 1 AS on_all_segments (1 row) -- Without a provider extension there is no iceberg table AM -CREATE ICEBERG TABLE lake_test_t0 (a int) CATALOG lake_test_cat VOLUME lake_test_vol; -- fail with hint +CREATE LAKE TABLE lake_test_t0 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol; -- fail with hint ERROR: table access method "iceberg" does not exist -HINT: CREATE ICEBERG TABLE requires an extension that provides the "iceberg" table access method. +HINT: CREATE LAKE TABLE ... USING ICEBERG requires an extension that provides the "iceberg" table access method. -- The default catalog/volume GUCs verify that the object exists SET iceberg_default_catalog = 'no_such_catalog'; -- fail ERROR: invalid value for parameter "iceberg_default_catalog": "no_such_catalog" @@ -130,14 +130,13 @@ ERROR: invalid value for parameter "iceberg_default_volume": "no_such_volume" DETAIL: Foreign volume "no_such_volume" does not exist. -- Simulate a datalake provider with a heap-backed iceberg AM CREATE ACCESS METHOD iceberg TYPE TABLE HANDLER heap_tableam_handler; --- CREATE ICEBERG TABLE with explicit catalog and volume -CREATE ICEBERG TABLE lake_test_t1 (a int, b text) CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (fileformat 'parquet'); +-- CREATE LAKE TABLE with explicit catalog and volume +CREATE LAKE TABLE lake_test_t1 (a int, b text) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (fileformat 'parquet'); SELECT c.relname, lt.lttable_type, lt.ltoptions, fc.fcname, fv.fvname FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid JOIN pg_foreign_catalog fc ON fc.oid = lt.ltforeign_catalog - JOIN pg_foreign_volume fv ON fv.oid = lt.ltforeign_volume - ORDER BY 1; + JOIN pg_foreign_volume fv ON fv.oid = lt.ltforeign_volume; relname | lttable_type | ltoptions | fcname | fvname --------------+--------------+----------------------+---------------+--------------- lake_test_t1 | ICEBERG | {fileformat=parquet} | lake_test_cat | lake_test_vol @@ -179,20 +178,20 @@ SELECT a, length(b) FROM lake_test_t1 WHERE a = 3; (1 row) -- Catalog and volume are both required -CREATE ICEBERG TABLE lake_test_t2 (a int) VOLUME lake_test_vol; -- fail, no catalog +CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG VOLUME lake_test_vol; -- fail, no catalog ERROR: no foreign catalog specified -HINT: Specify CATALOG in CREATE ICEBERG TABLE or set iceberg_default_catalog. -CREATE ICEBERG TABLE lake_test_t2 (a int) CATALOG lake_test_cat; -- fail, no volume +HINT: Specify CATALOG in CREATE LAKE TABLE or set iceberg_default_catalog. +CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG CATALOG lake_test_cat; -- fail, no volume ERROR: no foreign volume specified -HINT: Specify VOLUME in CREATE ICEBERG TABLE or set iceberg_default_volume. +HINT: Specify VOLUME in CREATE LAKE TABLE or set iceberg_default_volume. -- ... unless the GUCs provide defaults SET iceberg_default_catalog = 'lake_test_cat'; SET iceberg_default_volume = 'lake_test_vol'; -CREATE ICEBERG TABLE lake_test_t2 (a int); +CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG; RESET iceberg_default_catalog; RESET iceberg_default_volume; -- A DISTRIBUTED clause is ignored with a warning -CREATE ICEBERG TABLE lake_test_t3 (a int) CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); +CREATE LAKE TABLE lake_test_t3 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); WARNING: DISTRIBUTED clause has no effect for lake tables, using DISTRIBUTED RANDOMLY SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_test_t3'::regclass; policytype | distkey @@ -200,27 +199,27 @@ SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_te p | (1 row) --- CREATE ICEBERG TABLE only accepts the iceberg access method -CREATE ICEBERG TABLE lake_test_bad0 (a int) CATALOG lake_test_cat VOLUME lake_test_vol USING heap; -- fail -ERROR: access method "heap" is not supported for iceberg tables -HINT: Omit the USING clause; CREATE ICEBERG TABLE always uses the "iceberg" access method. -CREATE ICEBERG TABLE lake_test_t4 (a int) CATALOG lake_test_cat VOLUME lake_test_vol USING iceberg; -- explicit iceberg is fine --- The iceberg AM is rejected for every path other than CREATE ICEBERG TABLE +-- The USING clause names the table format; only ICEBERG is supported (any case) +CREATE LAKE TABLE lake_test_bad0 (a int) USING heap CATALOG lake_test_cat VOLUME lake_test_vol; -- fail, unsupported format +ERROR: unsupported lake table format "HEAP" +HINT: The only supported format is ICEBERG (USING ICEBERG). +CREATE LAKE TABLE lake_test_t4 (a int) USING iceberg CATALOG lake_test_cat VOLUME lake_test_vol; -- lower-case format is fine +-- The iceberg AM is rejected for every path other than CREATE LAKE TABLE CREATE TABLE lake_test_bad1 (a int) USING iceberg DISTRIBUTED RANDOMLY; -- fail ERROR: cannot create table "lake_test_bad1" with access method "iceberg" -HINT: Use CREATE ICEBERG TABLE instead. +HINT: Use CREATE LAKE TABLE ... USING ICEBERG instead. CREATE TABLE lake_test_bad2 USING iceberg AS SELECT 1 AS a DISTRIBUTED RANDOMLY; -- fail ERROR: cannot create table "lake_test_bad2" with access method "iceberg" -HINT: Use CREATE ICEBERG TABLE instead. +HINT: Use CREATE LAKE TABLE ... USING ICEBERG instead. SET default_table_access_method = iceberg; CREATE TABLE lake_test_bad3 (a int) DISTRIBUTED RANDOMLY; -- fail ERROR: cannot create table "lake_test_bad3" with access method "iceberg" -HINT: Use CREATE ICEBERG TABLE instead. +HINT: Use CREATE LAKE TABLE ... USING ICEBERG instead. RESET default_table_access_method; CREATE TABLE lake_test_heap (a int) DISTRIBUTED RANDOMLY; ALTER TABLE lake_test_heap SET ACCESS METHOD iceberg; -- fail ERROR: cannot change access method of table "lake_test_heap" to "iceberg" -HINT: Use CREATE ICEBERG TABLE to create an iceberg table. +HINT: Use CREATE LAKE TABLE ... USING ICEBERG to create a lake table. ALTER TABLE lake_test_t1 SET ACCESS METHOD heap; -- fail ERROR: cannot change access method of lake table "lake_test_t1" ALTER TABLE lake_test_t1 SET DISTRIBUTED BY (a); -- fail @@ -253,7 +252,7 @@ table lake_test_t3 depends on catalog lake_test_cat table lake_test_t4 depends on catalog lake_test_cat HINT: Use DROP ... CASCADE to drop the dependent objects too. -- Dropping a lake table removes its pg_lake_table entry -DROP ICEBERG TABLE lake_test_t1; +DROP LAKE TABLE lake_test_t1; SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid WHERE c.relname = 'lake_test_t1'; count @@ -261,17 +260,17 @@ SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid 0 (1 row) --- DROP ICEBERG TABLE rejects a non-iceberg table ... -DROP ICEBERG TABLE lake_test_heap; -- fail, not an iceberg table -ERROR: "lake_test_heap" is not an iceberg table +-- DROP LAKE TABLE rejects a non-lake table ... +DROP LAKE TABLE lake_test_heap; -- fail, not a lake table +ERROR: "lake_test_heap" is not a lake table HINT: Use DROP TABLE to remove a table. --- plain DROP TABLE must reject an iceberg table (mirrors foreign-table behavior) +-- plain DROP TABLE must reject a lake table (mirrors foreign-table behavior) DROP TABLE lake_test_t2; ERROR: "lake_test_t2" is not a table -HINT: Use DROP ICEBERG TABLE to remove an iceberg table. +HINT: Use DROP LAKE TABLE to remove a lake table. DROP TABLE IF EXISTS lake_test_t2; -- IF EXISTS does not suppress wrong-type errors ERROR: "lake_test_t2" is not a table -HINT: Use DROP ICEBERG TABLE to remove an iceberg table. +HINT: Use DROP LAKE TABLE to remove a lake table. -- the rejected drops must have left the table and its lake metadata intact SELECT count(*) FROM pg_lake_table WHERE ltrelid = 'lake_test_t2'::regclass; count @@ -280,7 +279,7 @@ SELECT count(*) FROM pg_lake_table WHERE ltrelid = 'lake_test_t2'::regclass; (1 row) -- the correct command still works -DROP ICEBERG TABLE lake_test_t2; +DROP LAKE TABLE lake_test_t2; SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid WHERE c.relname = 'lake_test_t2'; count diff --git a/src/test/regress/sql/lake_table.sql b/src/test/regress/sql/lake_table.sql index 3cfc4173a74..794aa2db37d 100644 --- a/src/test/regress/sql/lake_table.sql +++ b/src/test/regress/sql/lake_table.sql @@ -1,5 +1,5 @@ -- --- Test lake table DDL: FOREIGN CATALOG, FOREIGN VOLUME, ICEBERG TABLE +-- Test lake table DDL: FOREIGN CATALOG, FOREIGN VOLUME, LAKE TABLE -- -- Display the lake table catalogs @@ -21,7 +21,7 @@ CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv TYPE 'hi CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv2 TYPE 'hive'; -- fail, duplicate CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv2 TYPE 'hive'; -- skip with notice CREATE FOREIGN CATALOG lake_test_bad SERVER no_such_server TYPE 'hive'; -- fail, no server -SELECT fcname, fctype, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%' ORDER BY 1; +SELECT fcname, fctype, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%'; -- CREATE FOREIGN VOLUME CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv OPTIONS (path 's3://bucket/prefix'); @@ -31,7 +31,7 @@ CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv; -- skip CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv2; -- fail, duplicate CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv2; -- skip with notice CREATE FOREIGN VOLUME lake_test_bad SERVER no_such_server; -- fail, no server -SELECT fvname, fvoptions FROM pg_foreign_volume WHERE fvname LIKE 'lake\_test%' ORDER BY 1; +SELECT fvname, fvoptions FROM pg_foreign_volume WHERE fvname LIKE 'lake\_test%'; -- Object descriptions SELECT pg_catalog.pg_describe_object('pg_foreign_catalog'::regclass, oid, 0) @@ -46,7 +46,7 @@ SELECT count(DISTINCT gp_segment_id) > 1 AS on_all_segments FROM gp_dist_random('pg_foreign_volume') WHERE fvname = 'lake_test_vol'; -- Without a provider extension there is no iceberg table AM -CREATE ICEBERG TABLE lake_test_t0 (a int) CATALOG lake_test_cat VOLUME lake_test_vol; -- fail with hint +CREATE LAKE TABLE lake_test_t0 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol; -- fail with hint -- The default catalog/volume GUCs verify that the object exists SET iceberg_default_catalog = 'no_such_catalog'; -- fail @@ -55,14 +55,13 @@ SET iceberg_default_volume = 'no_such_volume'; -- fail -- Simulate a datalake provider with a heap-backed iceberg AM CREATE ACCESS METHOD iceberg TYPE TABLE HANDLER heap_tableam_handler; --- CREATE ICEBERG TABLE with explicit catalog and volume -CREATE ICEBERG TABLE lake_test_t1 (a int, b text) CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (fileformat 'parquet'); +-- CREATE LAKE TABLE with explicit catalog and volume +CREATE LAKE TABLE lake_test_t1 (a int, b text) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (fileformat 'parquet'); SELECT c.relname, lt.lttable_type, lt.ltoptions, fc.fcname, fv.fvname FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid JOIN pg_foreign_catalog fc ON fc.oid = lt.ltforeign_catalog - JOIN pg_foreign_volume fv ON fv.oid = lt.ltforeign_volume - ORDER BY 1; + JOIN pg_foreign_volume fv ON fv.oid = lt.ltforeign_volume; SELECT a.amname FROM pg_am a JOIN pg_class c ON c.relam = a.oid WHERE c.relname = 'lake_test_t1'; -- Lake tables are always DISTRIBUTED RANDOMLY (policytype 'p', no distkey) SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_test_t1'::regclass; @@ -77,25 +76,25 @@ INSERT INTO lake_test_t1 VALUES (3, repeat('x', 500000)); SELECT a, length(b) FROM lake_test_t1 WHERE a = 3; -- Catalog and volume are both required -CREATE ICEBERG TABLE lake_test_t2 (a int) VOLUME lake_test_vol; -- fail, no catalog -CREATE ICEBERG TABLE lake_test_t2 (a int) CATALOG lake_test_cat; -- fail, no volume +CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG VOLUME lake_test_vol; -- fail, no catalog +CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG CATALOG lake_test_cat; -- fail, no volume -- ... unless the GUCs provide defaults SET iceberg_default_catalog = 'lake_test_cat'; SET iceberg_default_volume = 'lake_test_vol'; -CREATE ICEBERG TABLE lake_test_t2 (a int); +CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG; RESET iceberg_default_catalog; RESET iceberg_default_volume; -- A DISTRIBUTED clause is ignored with a warning -CREATE ICEBERG TABLE lake_test_t3 (a int) CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); +CREATE LAKE TABLE lake_test_t3 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_test_t3'::regclass; --- CREATE ICEBERG TABLE only accepts the iceberg access method -CREATE ICEBERG TABLE lake_test_bad0 (a int) CATALOG lake_test_cat VOLUME lake_test_vol USING heap; -- fail -CREATE ICEBERG TABLE lake_test_t4 (a int) CATALOG lake_test_cat VOLUME lake_test_vol USING iceberg; -- explicit iceberg is fine +-- The USING clause names the table format; only ICEBERG is supported (any case) +CREATE LAKE TABLE lake_test_bad0 (a int) USING heap CATALOG lake_test_cat VOLUME lake_test_vol; -- fail, unsupported format +CREATE LAKE TABLE lake_test_t4 (a int) USING iceberg CATALOG lake_test_cat VOLUME lake_test_vol; -- lower-case format is fine --- The iceberg AM is rejected for every path other than CREATE ICEBERG TABLE +-- The iceberg AM is rejected for every path other than CREATE LAKE TABLE CREATE TABLE lake_test_bad1 (a int) USING iceberg DISTRIBUTED RANDOMLY; -- fail CREATE TABLE lake_test_bad2 USING iceberg AS SELECT 1 AS a DISTRIBUTED RANDOMLY; -- fail SET default_table_access_method = iceberg; @@ -118,19 +117,19 @@ DROP SERVER lake_test_srv; -- fail, catalog and volume depend on it DROP FOREIGN CATALOG lake_test_cat; -- fail, tables depend on it -- Dropping a lake table removes its pg_lake_table entry -DROP ICEBERG TABLE lake_test_t1; +DROP LAKE TABLE lake_test_t1; SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid WHERE c.relname = 'lake_test_t1'; --- DROP ICEBERG TABLE rejects a non-iceberg table ... -DROP ICEBERG TABLE lake_test_heap; -- fail, not an iceberg table --- plain DROP TABLE must reject an iceberg table (mirrors foreign-table behavior) +-- DROP LAKE TABLE rejects a non-lake table ... +DROP LAKE TABLE lake_test_heap; -- fail, not a lake table +-- plain DROP TABLE must reject a lake table (mirrors foreign-table behavior) DROP TABLE lake_test_t2; DROP TABLE IF EXISTS lake_test_t2; -- IF EXISTS does not suppress wrong-type errors -- the rejected drops must have left the table and its lake metadata intact SELECT count(*) FROM pg_lake_table WHERE ltrelid = 'lake_test_t2'::regclass; -- the correct command still works -DROP ICEBERG TABLE lake_test_t2; +DROP LAKE TABLE lake_test_t2; SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid WHERE c.relname = 'lake_test_t2'; From 710af2c58d4a6d761232fa3a29bb778dd96b799a Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Mon, 20 Jul 2026 17:07:02 +0800 Subject: [PATCH 20/29] laketablecmds: case-normalize the USING format; use base_path in the volume example - gram.y: lower-case the USING format for the access method name (it was only upper-cased for table_type), so a quoted "ICEBERG" / "IceBerg" resolves the iceberg AM instead of failing late with "access method ... does not exist". - regress: add a quoted mixed-case format case to lock this in; rename the volume path option in the example from `path` to `base_path` to match the agreed name (discussion #1683). Volume options are free-form (no validator), so this is example-only. - create_foreign_volume.sgml: same base_path example. --- doc/src/sgml/ref/create_foreign_volume.sgml | 2 +- src/backend/parser/gram.y | 22 +++++++++++++++++++-- src/test/regress/expected/lake_table.out | 10 +++++----- src/test/regress/sql/lake_table.sql | 4 ++-- 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/doc/src/sgml/ref/create_foreign_volume.sgml b/doc/src/sgml/ref/create_foreign_volume.sgml index eb41c65705e..6bc3df3d28e 100644 --- a/doc/src/sgml/ref/create_foreign_volume.sgml +++ b/doc/src/sgml/ref/create_foreign_volume.sgml @@ -107,7 +107,7 @@ CREATE FOREIGN VOLUME [ IF NOT EXISTS ] volume_na Create a volume for an object storage prefix using the foreign server s3_srv: -CREATE FOREIGN VOLUME s3_vol SERVER s3_srv OPTIONS (path 's3://bucket/prefix'); +CREATE FOREIGN VOLUME s3_vol SERVER s3_srv OPTIONS (base_path 's3://bucket/prefix'); diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index b34d0e88caf..865bd7f8709 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -9188,10 +9188,19 @@ CreateLakeTableStmt: { CreateLakeTableStmt *n = makeNode(CreateLakeTableStmt); char *table_type = pstrdup($9); + char *access_method = pstrdup($9); char *p; + /* + * The USING format is case-insensitive: upper-case it for + * the stored table_type and lower-case it for the access + * method name, so a quoted "ICEBERG" resolves the same AM + * as an unquoted iceberg. + */ for (p = table_type; *p; p++) *p = pg_toupper((unsigned char) *p); + for (p = access_method; *p; p++) + *p = pg_tolower((unsigned char) *p); $4->relpersistence = RELPERSISTENCE_PERMANENT; n->base.relation = $4; @@ -9202,7 +9211,7 @@ CreateLakeTableStmt: n->base.options = NIL; n->base.oncommit = ONCOMMIT_NOOP; n->base.tablespacename = NULL; - n->base.accessMethod = $9; + n->base.accessMethod = access_method; n->base.if_not_exists = false; n->base.relKind = RELKIND_RELATION; n->table_type = table_type; @@ -9226,10 +9235,19 @@ CreateLakeTableStmt: { CreateLakeTableStmt *n = makeNode(CreateLakeTableStmt); char *table_type = pstrdup($12); + char *access_method = pstrdup($12); char *p; + /* + * The USING format is case-insensitive: upper-case it for + * the stored table_type and lower-case it for the access + * method name, so a quoted "ICEBERG" resolves the same AM + * as an unquoted iceberg. + */ for (p = table_type; *p; p++) *p = pg_toupper((unsigned char) *p); + for (p = access_method; *p; p++) + *p = pg_tolower((unsigned char) *p); $7->relpersistence = RELPERSISTENCE_PERMANENT; n->base.relation = $7; @@ -9240,7 +9258,7 @@ CreateLakeTableStmt: n->base.options = NIL; n->base.oncommit = ONCOMMIT_NOOP; n->base.tablespacename = NULL; - n->base.accessMethod = $12; + n->base.accessMethod = access_method; n->base.if_not_exists = true; n->base.relKind = RELKIND_RELATION; n->table_type = table_type; diff --git a/src/test/regress/expected/lake_table.out b/src/test/regress/expected/lake_table.out index e310c3a9351..39ee59b4fa2 100644 --- a/src/test/regress/expected/lake_table.out +++ b/src/test/regress/expected/lake_table.out @@ -69,7 +69,7 @@ SELECT fcname, fctype, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake (1 row) -- CREATE FOREIGN VOLUME -CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv OPTIONS (path 's3://bucket/prefix'); +CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv OPTIONS (base_path 's3://bucket/prefix'); CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv; -- fail, duplicate ERROR: foreign volume "lake_test_vol" already exists CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv; -- skip with notice @@ -82,9 +82,9 @@ NOTICE: foreign volume "lake_test_vol" already exists, skipping CREATE FOREIGN VOLUME lake_test_bad SERVER no_such_server; -- fail, no server ERROR: server "no_such_server" does not exist SELECT fvname, fvoptions FROM pg_foreign_volume WHERE fvname LIKE 'lake\_test%'; - fvname | fvoptions ----------------+--------------------------- - lake_test_vol | {path=s3://bucket/prefix} + fvname | fvoptions +---------------+-------------------------------- + lake_test_vol | {base_path=s3://bucket/prefix} (1 row) -- Object descriptions @@ -203,7 +203,7 @@ SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_te CREATE LAKE TABLE lake_test_bad0 (a int) USING heap CATALOG lake_test_cat VOLUME lake_test_vol; -- fail, unsupported format ERROR: unsupported lake table format "HEAP" HINT: The only supported format is ICEBERG (USING ICEBERG). -CREATE LAKE TABLE lake_test_t4 (a int) USING iceberg CATALOG lake_test_cat VOLUME lake_test_vol; -- lower-case format is fine +CREATE LAKE TABLE lake_test_t4 (a int) USING "IceBerg" CATALOG lake_test_cat VOLUME lake_test_vol; -- quoted mixed-case format resolves the iceberg AM -- The iceberg AM is rejected for every path other than CREATE LAKE TABLE CREATE TABLE lake_test_bad1 (a int) USING iceberg DISTRIBUTED RANDOMLY; -- fail ERROR: cannot create table "lake_test_bad1" with access method "iceberg" diff --git a/src/test/regress/sql/lake_table.sql b/src/test/regress/sql/lake_table.sql index 794aa2db37d..77e778161a6 100644 --- a/src/test/regress/sql/lake_table.sql +++ b/src/test/regress/sql/lake_table.sql @@ -24,7 +24,7 @@ CREATE FOREIGN CATALOG lake_test_bad SERVER no_such_server TYPE 'hive'; -- fail SELECT fcname, fctype, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%'; -- CREATE FOREIGN VOLUME -CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv OPTIONS (path 's3://bucket/prefix'); +CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv OPTIONS (base_path 's3://bucket/prefix'); CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv; -- fail, duplicate CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv; -- skip with notice -- volume names are global: the same name on another server is still a duplicate @@ -92,7 +92,7 @@ SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_te -- The USING clause names the table format; only ICEBERG is supported (any case) CREATE LAKE TABLE lake_test_bad0 (a int) USING heap CATALOG lake_test_cat VOLUME lake_test_vol; -- fail, unsupported format -CREATE LAKE TABLE lake_test_t4 (a int) USING iceberg CATALOG lake_test_cat VOLUME lake_test_vol; -- lower-case format is fine +CREATE LAKE TABLE lake_test_t4 (a int) USING "IceBerg" CATALOG lake_test_cat VOLUME lake_test_vol; -- quoted mixed-case format resolves the iceberg AM -- The iceberg AM is rejected for every path other than CREATE LAKE TABLE CREATE TABLE lake_test_bad1 (a int) USING iceberg DISTRIBUTED RANDOMLY; -- fail From 20a2d0007fa3d79870423257012fae2425846d49 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Wed, 22 Jul 2026 10:09:25 +0800 Subject: [PATCH 21/29] commands: slim pg_lake_table to the catalog/volume binding; reject DISTRIBUTED Address PR review (andr-sokolov): - pg_lake_table drops lttable_type and ltoptions. A lake table's format is its access method (pg_class.relam) and its options are the relation's reloptions (pg_class.reloptions), validated by the access method, so the catalog now records only the {relation, foreign catalog, foreign volume} binding -- and no longer needs a TOAST table. CREATE LAKE TABLE routes its OPTIONS into the base relation's reloptions; the now-dead CreateLakeTableStmt.options node field and its node-support handlers are removed. - CREATE LAKE TABLE ... DISTRIBUTED now raises an error instead of a warning, since lake tables are always distributed randomly. - Bump CATALOG_VERSION_NO for the pg_lake_table layout change. - regress: read pg_class.relam/reloptions instead of the dropped columns; add a case showing OPTIONS become AM-validated reloptions (accepted vs rejected); the DISTRIBUTED case is now a failure. --- src/backend/commands/laketablecmds.c | 19 ------ src/backend/nodes/copyfuncs.funcs.c | 1 - src/backend/nodes/equalfuncs.c | 1 - src/backend/nodes/outfuncs.c | 1 - src/backend/nodes/readfast.c | 1 - src/backend/parser/gram.y | 16 ++--- src/include/catalog/catversion.h | 2 +- src/include/catalog/pg_lake_table.h | 13 ++-- src/include/nodes/parsenodes.h | 3 +- src/test/regress/expected/lake_table.out | 76 ++++++++++-------------- src/test/regress/sql/lake_table.sql | 25 ++++---- 11 files changed, 58 insertions(+), 100 deletions(-) diff --git a/src/backend/commands/laketablecmds.c b/src/backend/commands/laketablecmds.c index 3d00077b407..ae260c3711d 100644 --- a/src/backend/commands/laketablecmds.c +++ b/src/backend/commands/laketablecmds.c @@ -395,25 +395,6 @@ CreateLakeTable(CreateLakeTableStmt *stmt, Oid relId) values[Anum_pg_lake_table_ltrelid - 1] = ObjectIdGetDatum(relId); values[Anum_pg_lake_table_ltforeign_catalog - 1] = ObjectIdGetDatum(catalog_oid); values[Anum_pg_lake_table_ltforeign_volume - 1] = ObjectIdGetDatum(volume_oid); - values[Anum_pg_lake_table_lttable_type - 1] = CStringGetTextDatum(stmt->table_type); - - if (stmt->options) - { - Datum options_datum; - - /* Build standard text[] reloptions from DefElem list */ - options_datum = transformRelOptions((Datum) 0, stmt->options, - NULL, NULL, false, false); - - if (options_datum != (Datum) 0) - values[Anum_pg_lake_table_ltoptions - 1] = options_datum; - else - nulls[Anum_pg_lake_table_ltoptions - 1] = true; - } - else - { - nulls[Anum_pg_lake_table_ltoptions - 1] = true; - } tuple = heap_form_tuple(lake_rel->rd_att, values, nulls); diff --git a/src/backend/nodes/copyfuncs.funcs.c b/src/backend/nodes/copyfuncs.funcs.c index d6597996a99..dcb93a84b3d 100644 --- a/src/backend/nodes/copyfuncs.funcs.c +++ b/src/backend/nodes/copyfuncs.funcs.c @@ -3419,7 +3419,6 @@ _copyCreateLakeTableStmt(const CreateLakeTableStmt *from) COPY_STRING_FIELD(table_type); COPY_STRING_FIELD(foreign_catalog); COPY_STRING_FIELD(foreign_volume); - COPY_NODE_FIELD(options); return newnode; } diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 0150a714f65..b56e94873a2 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -3519,7 +3519,6 @@ _equalCreateLakeTableStmt(const CreateLakeTableStmt *a, const CreateLakeTableStm COMPARE_STRING_FIELD(table_type); COMPARE_STRING_FIELD(foreign_catalog); COMPARE_STRING_FIELD(foreign_volume); - COMPARE_NODE_FIELD(options); return true; } diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index e8ea5b7bafc..cea71a8aa01 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -4279,7 +4279,6 @@ _outCreateLakeTableStmt(StringInfo str, const CreateLakeTableStmt *node) WRITE_STRING_FIELD(table_type); WRITE_STRING_FIELD(foreign_catalog); WRITE_STRING_FIELD(foreign_volume); - WRITE_NODE_FIELD(options); } static void diff --git a/src/backend/nodes/readfast.c b/src/backend/nodes/readfast.c index b9fbff9226e..031c9d062f4 100644 --- a/src/backend/nodes/readfast.c +++ b/src/backend/nodes/readfast.c @@ -1947,7 +1947,6 @@ _readCreateLakeTableStmt(void) READ_STRING_FIELD(table_type); READ_STRING_FIELD(foreign_catalog); READ_STRING_FIELD(foreign_volume); - READ_NODE_FIELD(options); READ_DONE(); } diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 865bd7f8709..56434969f05 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -9208,7 +9208,7 @@ CreateLakeTableStmt: n->base.inhRelations = NIL; n->base.ofTypename = NULL; n->base.constraints = NIL; - n->base.options = NIL; + n->base.options = $12; /* OPTIONS become reloptions, validated by the AM */ n->base.oncommit = ONCOMMIT_NOOP; n->base.tablespacename = NULL; n->base.accessMethod = access_method; @@ -9217,11 +9217,11 @@ CreateLakeTableStmt: n->table_type = table_type; n->foreign_catalog = $10 ? pstrdup($10) : NULL; n->foreign_volume = $11 ? pstrdup($11) : NULL; - n->options = $12; if ($13 != NULL) - ereport(WARNING, + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("DISTRIBUTED clause has no effect for lake tables, using DISTRIBUTED RANDOMLY"))); + errmsg("DISTRIBUTED clause is not supported for lake tables"), + errhint("Lake tables are always distributed randomly; omit the DISTRIBUTED clause."))); n->base.distributedBy = makeNode(DistributedBy); n->base.distributedBy->ptype = POLICYTYPE_PARTITIONED; n->base.distributedBy->keyCols = NIL; @@ -9255,7 +9255,7 @@ CreateLakeTableStmt: n->base.inhRelations = NIL; n->base.ofTypename = NULL; n->base.constraints = NIL; - n->base.options = NIL; + n->base.options = $15; /* OPTIONS become reloptions, validated by the AM */ n->base.oncommit = ONCOMMIT_NOOP; n->base.tablespacename = NULL; n->base.accessMethod = access_method; @@ -9264,11 +9264,11 @@ CreateLakeTableStmt: n->table_type = table_type; n->foreign_catalog = $13 ? pstrdup($13) : NULL; n->foreign_volume = $14 ? pstrdup($14) : NULL; - n->options = $15; if ($16 != NULL) - ereport(WARNING, + ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("DISTRIBUTED clause has no effect for lake tables, using DISTRIBUTED RANDOMLY"))); + errmsg("DISTRIBUTED clause is not supported for lake tables"), + errhint("Lake tables are always distributed randomly; omit the DISTRIBUTED clause."))); n->base.distributedBy = makeNode(DistributedBy); n->base.distributedBy->ptype = POLICYTYPE_PARTITIONED; n->base.distributedBy->keyCols = NIL; diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 53258aca56e..79fea91acf3 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -60,6 +60,6 @@ */ /* 3yyymmddN */ -#define CATALOG_VERSION_NO 302607101 +#define CATALOG_VERSION_NO 302607221 #endif diff --git a/src/include/catalog/pg_lake_table.h b/src/include/catalog/pg_lake_table.h index d22ce5a077f..e4d4ff48699 100644 --- a/src/include/catalog/pg_lake_table.h +++ b/src/include/catalog/pg_lake_table.h @@ -45,22 +45,19 @@ CATALOG(pg_lake_table,9901,LakeTableRelationId) Oid ltrelid BKI_LOOKUP(pg_class); /* OID of the lake table relation */ Oid ltforeign_catalog BKI_LOOKUP_OPT(pg_foreign_catalog); /* OID of foreign catalog */ Oid ltforeign_volume BKI_LOOKUP_OPT(pg_foreign_volume); /* OID of foreign volume */ - -#ifdef CATALOG_VARLEN /* variable-length fields start here */ - text lttable_type; /* table type: ICEBERG, etc. */ - text ltoptions[1]; /* lake table options */ -#endif } FormData_pg_lake_table; /* ---------------- * Form_pg_lake_table corresponds to a pointer to a tuple with * the format of pg_lake_table relation. + * + * A lake table's format is its access method (pg_class.relam) and its + * options are the relation's reloptions (pg_class.reloptions), validated by + * the access method; pg_lake_table only records the catalog/volume binding. * ---------------- */ typedef FormData_pg_lake_table *Form_pg_lake_table; -DECLARE_TOAST(pg_lake_table, 9903, 9904); - DECLARE_UNIQUE_INDEX_PKEY(pg_lake_table_relid_index, 9902, LakeTableRelidIndexId, on pg_lake_table using btree(ltrelid oid_ops)); /* ---------------- @@ -70,10 +67,8 @@ DECLARE_UNIQUE_INDEX_PKEY(pg_lake_table_relid_index, 9902, LakeTableRelidIndexId typedef struct LakeTable { Oid relid; /* OID of the lake table relation */ - char *table_type; /* table type: ICEBERG, etc. */ char *foreign_catalog; /* foreign catalog name */ char *foreign_volume; /* foreign volume name */ - List *options; /* lake table options */ } LakeTable; #endif /* PG_LAKE_TABLE_H */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 8f4b9aa5c27..d43898765b0 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3802,10 +3802,9 @@ typedef struct CreateDirectoryTableStmt typedef struct CreateLakeTableStmt { CreateStmt base; /* base table creation info */ - char *table_type; /* lake table type, e.g. "ICEBERG" */ + char *table_type; /* lake table format, e.g. "ICEBERG" (validation only) */ char *foreign_catalog; /* foreign catalog name, or NULL */ char *foreign_volume; /* foreign volume name, or NULL */ - List *options; /* lake-table-specific options */ } CreateLakeTableStmt; typedef struct AlterDirectoryTableStmt diff --git a/src/test/regress/expected/lake_table.out b/src/test/regress/expected/lake_table.out index 39ee59b4fa2..2b446154e99 100644 --- a/src/test/regress/expected/lake_table.out +++ b/src/test/regress/expected/lake_table.out @@ -30,14 +30,12 @@ Indexes: "pg_foreign_volume_name_index" UNIQUE CONSTRAINT, btree (fvname) \d+ pg_lake_table - Table "pg_catalog.pg_lake_table" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description --------------------+--------+-----------+----------+---------+----------+--------------+------------- - ltrelid | oid | | not null | | plain | | - ltforeign_catalog | oid | | not null | | plain | | - ltforeign_volume | oid | | not null | | plain | | - lttable_type | text | C | | | extended | | - ltoptions | text[] | C | | | extended | | + Table "pg_catalog.pg_lake_table" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +-------------------+------+-----------+----------+---------+---------+--------------+------------- + ltrelid | oid | | not null | | plain | | + ltforeign_catalog | oid | | not null | | plain | | + ltforeign_volume | oid | | not null | | plain | | Indexes: "pg_lake_table_relid_index" PRIMARY KEY, btree (ltrelid) @@ -130,22 +128,19 @@ ERROR: invalid value for parameter "iceberg_default_volume": "no_such_volume" DETAIL: Foreign volume "no_such_volume" does not exist. -- Simulate a datalake provider with a heap-backed iceberg AM CREATE ACCESS METHOD iceberg TYPE TABLE HANDLER heap_tableam_handler; --- CREATE LAKE TABLE with explicit catalog and volume -CREATE LAKE TABLE lake_test_t1 (a int, b text) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (fileformat 'parquet'); -SELECT c.relname, lt.lttable_type, lt.ltoptions, fc.fcname, fv.fvname +-- CREATE LAKE TABLE with explicit catalog and volume. The format is the +-- table's access method (pg_class.relam) and its options are the relation's +-- reloptions; pg_lake_table records only the catalog/volume binding. +CREATE LAKE TABLE lake_test_t1 (a int, b text) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol; +SELECT c.relname, am.amname, c.reloptions, fc.fcname, fv.fvname FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid + JOIN pg_am am ON am.oid = c.relam JOIN pg_foreign_catalog fc ON fc.oid = lt.ltforeign_catalog JOIN pg_foreign_volume fv ON fv.oid = lt.ltforeign_volume; - relname | lttable_type | ltoptions | fcname | fvname ---------------+--------------+----------------------+---------------+--------------- - lake_test_t1 | ICEBERG | {fileformat=parquet} | lake_test_cat | lake_test_vol -(1 row) - -SELECT a.amname FROM pg_am a JOIN pg_class c ON c.relam = a.oid WHERE c.relname = 'lake_test_t1'; - amname ---------- - iceberg + relname | amname | reloptions | fcname | fvname +--------------+---------+------------+---------------+--------------- + lake_test_t1 | iceberg | | lake_test_cat | lake_test_vol (1 row) -- Lake tables are always DISTRIBUTED RANDOMLY (policytype 'p', no distkey) @@ -163,20 +158,18 @@ SELECT count(*) FROM lake_test_t1; 2 (1 row) --- Lake tables get a TOAST table like plain tables, so wide values work -SELECT reltoastrelid <> 0 AS has_toast FROM pg_class WHERE relname = 'lake_test_t1'; - has_toast ------------ - t -(1 row) - -INSERT INTO lake_test_t1 VALUES (3, repeat('x', 500000)); -SELECT a, length(b) FROM lake_test_t1 WHERE a = 3; - a | length ----+-------- - 3 | 500000 +-- OPTIONS become the relation's reloptions and are validated by the access +-- method: a value the AM accepts is stored, an unknown one is rejected. +CREATE LAKE TABLE lake_test_opt (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (fillfactor '70'); +SELECT reloptions FROM pg_class WHERE relname = 'lake_test_opt'; + reloptions +----------------- + {fillfactor=70} (1 row) +CREATE LAKE TABLE lake_test_optbad (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (bogus_opt 'x'); -- fail, AM rejects unknown option +ERROR: unrecognized parameter "bogus_opt" +DROP LAKE TABLE lake_test_opt; -- Catalog and volume are both required CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG VOLUME lake_test_vol; -- fail, no catalog ERROR: no foreign catalog specified @@ -190,15 +183,10 @@ SET iceberg_default_volume = 'lake_test_vol'; CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG; RESET iceberg_default_catalog; RESET iceberg_default_volume; --- A DISTRIBUTED clause is ignored with a warning -CREATE LAKE TABLE lake_test_t3 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); -WARNING: DISTRIBUTED clause has no effect for lake tables, using DISTRIBUTED RANDOMLY -SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_test_t3'::regclass; - policytype | distkey -------------+--------- - p | -(1 row) - +-- A DISTRIBUTED clause is rejected (lake tables are always distributed randomly) +CREATE LAKE TABLE lake_test_t3 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); -- fail +ERROR: DISTRIBUTED clause is not supported for lake tables +HINT: Lake tables are always distributed randomly; omit the DISTRIBUTED clause. -- The USING clause names the table format; only ICEBERG is supported (any case) CREATE LAKE TABLE lake_test_bad0 (a int) USING heap CATALOG lake_test_cat VOLUME lake_test_vol; -- fail, unsupported format ERROR: unsupported lake table format "HEAP" @@ -241,14 +229,12 @@ DETAIL: catalog lake_test_cat depends on server lake_test_srv volume lake_test_vol depends on server lake_test_srv table lake_test_t1 depends on volume lake_test_vol table lake_test_t2 depends on volume lake_test_vol -table lake_test_t3 depends on volume lake_test_vol table lake_test_t4 depends on volume lake_test_vol HINT: Use DROP ... CASCADE to drop the dependent objects too. DROP FOREIGN CATALOG lake_test_cat; -- fail, tables depend on it ERROR: cannot drop catalog lake_test_cat because other objects depend on it DETAIL: table lake_test_t1 depends on catalog lake_test_cat table lake_test_t2 depends on catalog lake_test_cat -table lake_test_t3 depends on catalog lake_test_cat table lake_test_t4 depends on catalog lake_test_cat HINT: Use DROP ... CASCADE to drop the dependent objects too. -- Dropping a lake table removes its pg_lake_table entry @@ -289,9 +275,7 @@ SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid -- DROP FOREIGN CATALOG ... CASCADE takes the remaining tables with it DROP FOREIGN CATALOG lake_test_cat CASCADE; -NOTICE: drop cascades to 2 other objects -DETAIL: drop cascades to table lake_test_t3 -drop cascades to table lake_test_t4 +NOTICE: drop cascades to table lake_test_t4 SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid WHERE c.relname LIKE 'lake\_test%'; count diff --git a/src/test/regress/sql/lake_table.sql b/src/test/regress/sql/lake_table.sql index 77e778161a6..b486e34dc83 100644 --- a/src/test/regress/sql/lake_table.sql +++ b/src/test/regress/sql/lake_table.sql @@ -55,14 +55,16 @@ SET iceberg_default_volume = 'no_such_volume'; -- fail -- Simulate a datalake provider with a heap-backed iceberg AM CREATE ACCESS METHOD iceberg TYPE TABLE HANDLER heap_tableam_handler; --- CREATE LAKE TABLE with explicit catalog and volume -CREATE LAKE TABLE lake_test_t1 (a int, b text) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (fileformat 'parquet'); -SELECT c.relname, lt.lttable_type, lt.ltoptions, fc.fcname, fv.fvname +-- CREATE LAKE TABLE with explicit catalog and volume. The format is the +-- table's access method (pg_class.relam) and its options are the relation's +-- reloptions; pg_lake_table records only the catalog/volume binding. +CREATE LAKE TABLE lake_test_t1 (a int, b text) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol; +SELECT c.relname, am.amname, c.reloptions, fc.fcname, fv.fvname FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid + JOIN pg_am am ON am.oid = c.relam JOIN pg_foreign_catalog fc ON fc.oid = lt.ltforeign_catalog JOIN pg_foreign_volume fv ON fv.oid = lt.ltforeign_volume; -SELECT a.amname FROM pg_am a JOIN pg_class c ON c.relam = a.oid WHERE c.relname = 'lake_test_t1'; -- Lake tables are always DISTRIBUTED RANDOMLY (policytype 'p', no distkey) SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_test_t1'::regclass; @@ -70,10 +72,12 @@ SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_te INSERT INTO lake_test_t1 VALUES (1, 'x'), (2, 'y'); SELECT count(*) FROM lake_test_t1; --- Lake tables get a TOAST table like plain tables, so wide values work -SELECT reltoastrelid <> 0 AS has_toast FROM pg_class WHERE relname = 'lake_test_t1'; -INSERT INTO lake_test_t1 VALUES (3, repeat('x', 500000)); -SELECT a, length(b) FROM lake_test_t1 WHERE a = 3; +-- OPTIONS become the relation's reloptions and are validated by the access +-- method: a value the AM accepts is stored, an unknown one is rejected. +CREATE LAKE TABLE lake_test_opt (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (fillfactor '70'); +SELECT reloptions FROM pg_class WHERE relname = 'lake_test_opt'; +CREATE LAKE TABLE lake_test_optbad (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (bogus_opt 'x'); -- fail, AM rejects unknown option +DROP LAKE TABLE lake_test_opt; -- Catalog and volume are both required CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG VOLUME lake_test_vol; -- fail, no catalog @@ -86,9 +90,8 @@ CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG; RESET iceberg_default_catalog; RESET iceberg_default_volume; --- A DISTRIBUTED clause is ignored with a warning -CREATE LAKE TABLE lake_test_t3 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); -SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_test_t3'::regclass; +-- A DISTRIBUTED clause is rejected (lake tables are always distributed randomly) +CREATE LAKE TABLE lake_test_t3 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); -- fail -- The USING clause names the table format; only ICEBERG is supported (any case) CREATE LAKE TABLE lake_test_bad0 (a int) USING heap CATALOG lake_test_cat VOLUME lake_test_vol; -- fail, unsupported format From a8b0627ab9d3bd0a3d298d89b9955e3c3702dd50 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Wed, 22 Jul 2026 15:48:23 +0800 Subject: [PATCH 22/29] parser: drop the DISTRIBUTED clause from the CREATE LAKE TABLE grammar Per PR review (andr-sokolov): lake tables are always distributed randomly, so a DISTRIBUTED clause has no place in the syntax. Remove OptDistributedBy from the CreateLakeTableStmt rules, so a DISTRIBUTED clause is now a plain syntax error rather than being parsed and then rejected; the relation is still forced to DISTRIBUTED RANDOMLY internally. --- src/backend/parser/gram.y | 14 ++------------ src/test/regress/expected/lake_table.out | 5 +++-- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 56434969f05..fdfb58d0d74 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -9184,7 +9184,6 @@ CreateLakeTableStmt: CREATE LAKE TABLE qualified_name '(' OptTableElementList ')' USING name OptForeignCatalog OptForeignVolume create_generic_options - OptDistributedBy { CreateLakeTableStmt *n = makeNode(CreateLakeTableStmt); char *table_type = pstrdup($9); @@ -9217,11 +9216,7 @@ CreateLakeTableStmt: n->table_type = table_type; n->foreign_catalog = $10 ? pstrdup($10) : NULL; n->foreign_volume = $11 ? pstrdup($11) : NULL; - if ($13 != NULL) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("DISTRIBUTED clause is not supported for lake tables"), - errhint("Lake tables are always distributed randomly; omit the DISTRIBUTED clause."))); + /* lake tables are always distributed randomly */ n->base.distributedBy = makeNode(DistributedBy); n->base.distributedBy->ptype = POLICYTYPE_PARTITIONED; n->base.distributedBy->keyCols = NIL; @@ -9231,7 +9226,6 @@ CreateLakeTableStmt: | CREATE LAKE TABLE IF_P NOT EXISTS qualified_name '(' OptTableElementList ')' USING name OptForeignCatalog OptForeignVolume create_generic_options - OptDistributedBy { CreateLakeTableStmt *n = makeNode(CreateLakeTableStmt); char *table_type = pstrdup($12); @@ -9264,11 +9258,7 @@ CreateLakeTableStmt: n->table_type = table_type; n->foreign_catalog = $13 ? pstrdup($13) : NULL; n->foreign_volume = $14 ? pstrdup($14) : NULL; - if ($16 != NULL) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("DISTRIBUTED clause is not supported for lake tables"), - errhint("Lake tables are always distributed randomly; omit the DISTRIBUTED clause."))); + /* lake tables are always distributed randomly */ n->base.distributedBy = makeNode(DistributedBy); n->base.distributedBy->ptype = POLICYTYPE_PARTITIONED; n->base.distributedBy->keyCols = NIL; diff --git a/src/test/regress/expected/lake_table.out b/src/test/regress/expected/lake_table.out index 2b446154e99..3fd873b3a4d 100644 --- a/src/test/regress/expected/lake_table.out +++ b/src/test/regress/expected/lake_table.out @@ -185,8 +185,9 @@ RESET iceberg_default_catalog; RESET iceberg_default_volume; -- A DISTRIBUTED clause is rejected (lake tables are always distributed randomly) CREATE LAKE TABLE lake_test_t3 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); -- fail -ERROR: DISTRIBUTED clause is not supported for lake tables -HINT: Lake tables are always distributed randomly; omit the DISTRIBUTED clause. +ERROR: syntax error at or near "DISTRIBUTED" +LINE 1: ...CEBERG CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTE... + ^ -- The USING clause names the table format; only ICEBERG is supported (any case) CREATE LAKE TABLE lake_test_bad0 (a int) USING heap CATALOG lake_test_cat VOLUME lake_test_vol; -- fail, unsupported format ERROR: unsupported lake table format "HEAP" From 445cd160956a324b24091a8e83a9f8e237d1ca5a Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Wed, 22 Jul 2026 16:22:24 +0800 Subject: [PATCH 23/29] laketablecmds: identify lake tables by pg_lake_table membership; validate format first Per PR review (andr-sokolov): - Rename RelationIsIcebergTable -> RelationIsLakeTable and identify a lake table by its pg_lake_table entry rather than by the iceberg access method: the check answers "is this a lake table", independent of any particular format's AM. - In ResolveLakeTableOptions, validate the USING format before checking access method existence, and resolve the AM by the format name via get_table_am_oid(stmt->table_type), so other formats can be supported later. - validate_table_type compares against ICEBERG_TABLE_AM_NAME rather than a literal. - gram.y: the USING clause names both the format and its like-named access method, so normalize it to lower case; a quoted "ICEBERG" resolves the same AM as an unquoted iceberg. The unsupported-format error now shows that name. --- src/backend/catalog/heap.c | 2 +- src/backend/commands/laketablecmds.c | 52 ++++++++++++++---------- src/backend/commands/tablecmds.c | 2 +- src/backend/parser/gram.y | 38 ++++++++--------- src/include/commands/laketablecmds.h | 2 +- src/test/regress/expected/lake_table.out | 2 +- 6 files changed, 51 insertions(+), 47 deletions(-) diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index b6171061700..3abd4701bc3 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -2324,7 +2324,7 @@ heap_drop_with_catalog(Oid relid) CheckTableForSerializableConflictIn(rel); /* If this is a lake table, remove its pg_lake_table entry */ - if (RelationIsIcebergTable(rel)) + if (RelationIsLakeTable(rel)) RemoveLakeTableEntry(relid); /* diff --git a/src/backend/commands/laketablecmds.c b/src/backend/commands/laketablecmds.c index ae260c3711d..f1ab3a1b813 100644 --- a/src/backend/commands/laketablecmds.c +++ b/src/backend/commands/laketablecmds.c @@ -195,23 +195,31 @@ GetIcebergTableAmOid(bool missing_ok) } /* - * RelationIsIcebergTable + * RelationIsLakeTable * - * True iff the relation uses the iceberg table access method. Resolved by - * access method name so the kernel does not depend on any particular - * extension's OID assignments. + * True iff the relation has a pg_lake_table entry, i.e. it was created by + * CREATE LAKE TABLE. Lake tables are told apart from ordinary relations by + * this catalog membership rather than by their access method. */ bool -RelationIsIcebergTable(Relation rel) +RelationIsLakeTable(Relation rel) { - Oid iceberg_amoid; - - if (!OidIsValid(rel->rd_rel->relam)) - return false; + Relation ltRel; + ScanKeyData skey; + SysScanDesc scan; + bool found; - iceberg_amoid = GetIcebergTableAmOid(true); + ltRel = table_open(LakeTableRelationId, AccessShareLock); + ScanKeyInit(&skey, + Anum_pg_lake_table_ltrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationGetRelid(rel))); + scan = systable_beginscan(ltRel, LakeTableRelidIndexId, true, NULL, 1, &skey); + found = HeapTupleIsValid(systable_getnext(scan)); + systable_endscan(scan); + table_close(ltRel, AccessShareLock); - return OidIsValid(iceberg_amoid) && rel->rd_rel->relam == iceberg_amoid; + return found; } /* @@ -225,7 +233,7 @@ validate_table_type(const char *table_type) (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("lake table format cannot be NULL"))); - if (strcmp(table_type, "ICEBERG") != 0) + if (strcmp(table_type, ICEBERG_TABLE_AM_NAME) != 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("unsupported lake table format \"%s\"", table_type), @@ -280,21 +288,23 @@ ResolveLakeTableOptions(CreateLakeTableStmt *stmt, const char *catalog_name; const char *volume_name; + /* Validate the table format named in the USING clause first, so an + * unsupported format is reported before anything else. */ + validate_table_type(stmt->table_type); + /* - * Lake tables are unusable without an extension providing the iceberg - * table access method; check it first so the install hint takes - * precedence over catalog/volume resolution errors. + * The format is implemented by a like-named table access method that a + * datalake extension provides; a lake table is unusable without it, so + * check it here (after the format) so the install hint takes precedence + * over catalog/volume resolution errors. */ - if (!OidIsValid(GetIcebergTableAmOid(true))) + if (!OidIsValid(get_table_am_oid(stmt->table_type, true))) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("table access method \"%s\" does not exist", - ICEBERG_TABLE_AM_NAME), + stmt->table_type), errhint("CREATE LAKE TABLE ... USING ICEBERG requires an extension that provides the \"%s\" table access method.", - ICEBERG_TABLE_AM_NAME))); - - /* Validate the table format named in the USING clause */ - validate_table_type(stmt->table_type); + stmt->table_type))); /* * Determine catalog name: use explicit value if provided, otherwise diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index a8d745cd798..7b56e1dd64c 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -19831,7 +19831,7 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) } /* Lake tables must remain DISTRIBUTED RANDOMLY */ - if (RelationIsIcebergTable(rel)) + if (RelationIsLakeTable(rel)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot change distribution policy of lake table \"%s\"", diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index fdfb58d0d74..d5b69d9752e 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -9186,19 +9186,16 @@ CreateLakeTableStmt: OptForeignCatalog OptForeignVolume create_generic_options { CreateLakeTableStmt *n = makeNode(CreateLakeTableStmt); - char *table_type = pstrdup($9); - char *access_method = pstrdup($9); + char *format = pstrdup($9); char *p; /* - * The USING format is case-insensitive: upper-case it for - * the stored table_type and lower-case it for the access - * method name, so a quoted "ICEBERG" resolves the same AM - * as an unquoted iceberg. + * The USING clause names both the lake table format and + * the access method that implements it; normalize to lower + * case so a quoted "ICEBERG" resolves the same AM as an + * unquoted iceberg. */ - for (p = table_type; *p; p++) - *p = pg_toupper((unsigned char) *p); - for (p = access_method; *p; p++) + for (p = format; *p; p++) *p = pg_tolower((unsigned char) *p); $4->relpersistence = RELPERSISTENCE_PERMANENT; @@ -9210,10 +9207,10 @@ CreateLakeTableStmt: n->base.options = $12; /* OPTIONS become reloptions, validated by the AM */ n->base.oncommit = ONCOMMIT_NOOP; n->base.tablespacename = NULL; - n->base.accessMethod = access_method; + n->base.accessMethod = format; n->base.if_not_exists = false; n->base.relKind = RELKIND_RELATION; - n->table_type = table_type; + n->table_type = format; n->foreign_catalog = $10 ? pstrdup($10) : NULL; n->foreign_volume = $11 ? pstrdup($11) : NULL; /* lake tables are always distributed randomly */ @@ -9228,19 +9225,16 @@ CreateLakeTableStmt: OptForeignCatalog OptForeignVolume create_generic_options { CreateLakeTableStmt *n = makeNode(CreateLakeTableStmt); - char *table_type = pstrdup($12); - char *access_method = pstrdup($12); + char *format = pstrdup($12); char *p; /* - * The USING format is case-insensitive: upper-case it for - * the stored table_type and lower-case it for the access - * method name, so a quoted "ICEBERG" resolves the same AM - * as an unquoted iceberg. + * The USING clause names both the lake table format and + * the access method that implements it; normalize to lower + * case so a quoted "ICEBERG" resolves the same AM as an + * unquoted iceberg. */ - for (p = table_type; *p; p++) - *p = pg_toupper((unsigned char) *p); - for (p = access_method; *p; p++) + for (p = format; *p; p++) *p = pg_tolower((unsigned char) *p); $7->relpersistence = RELPERSISTENCE_PERMANENT; @@ -9252,10 +9246,10 @@ CreateLakeTableStmt: n->base.options = $15; /* OPTIONS become reloptions, validated by the AM */ n->base.oncommit = ONCOMMIT_NOOP; n->base.tablespacename = NULL; - n->base.accessMethod = access_method; + n->base.accessMethod = format; n->base.if_not_exists = true; n->base.relKind = RELKIND_RELATION; - n->table_type = table_type; + n->table_type = format; n->foreign_catalog = $13 ? pstrdup($13) : NULL; n->foreign_volume = $14 ? pstrdup($14) : NULL; /* lake tables are always distributed randomly */ diff --git a/src/include/commands/laketablecmds.h b/src/include/commands/laketablecmds.h index 980d25784d8..4c14941aa25 100644 --- a/src/include/commands/laketablecmds.h +++ b/src/include/commands/laketablecmds.h @@ -53,7 +53,7 @@ extern const char *GetDefaultIcebergVolume(void); /* Lake table management */ extern Oid GetIcebergTableAmOid(bool missing_ok); -extern bool RelationIsIcebergTable(Relation rel); +extern bool RelationIsLakeTable(Relation rel); extern void ValidateLakeTableStmt(CreateLakeTableStmt *stmt); extern void CreateLakeTable(CreateLakeTableStmt *stmt, Oid relId); extern void RemoveLakeTableEntry(Oid relid); diff --git a/src/test/regress/expected/lake_table.out b/src/test/regress/expected/lake_table.out index 3fd873b3a4d..40e98e730a8 100644 --- a/src/test/regress/expected/lake_table.out +++ b/src/test/regress/expected/lake_table.out @@ -190,7 +190,7 @@ LINE 1: ...CEBERG CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTE... ^ -- The USING clause names the table format; only ICEBERG is supported (any case) CREATE LAKE TABLE lake_test_bad0 (a int) USING heap CATALOG lake_test_cat VOLUME lake_test_vol; -- fail, unsupported format -ERROR: unsupported lake table format "HEAP" +ERROR: unsupported lake table format "heap" HINT: The only supported format is ICEBERG (USING ICEBERG). CREATE LAKE TABLE lake_test_t4 (a int) USING "IceBerg" CATALOG lake_test_cat VOLUME lake_test_vol; -- quoted mixed-case format resolves the iceberg AM -- The iceberg AM is rejected for every path other than CREATE LAKE TABLE From 6924020c040b55517ca42fc6222d609ab4390c6f Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Wed, 22 Jul 2026 16:54:07 +0800 Subject: [PATCH 24/29] laketablecmds: use the format name in the missing access-method hint Per PR review (andr-sokolov): the hint hardcoded "ICEBERG"; use stmt->table_type so it stays correct once other lake table formats are supported. --- src/backend/commands/laketablecmds.c | 4 ++-- src/test/regress/expected/lake_table.out | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/commands/laketablecmds.c b/src/backend/commands/laketablecmds.c index f1ab3a1b813..afc92ab36f3 100644 --- a/src/backend/commands/laketablecmds.c +++ b/src/backend/commands/laketablecmds.c @@ -303,8 +303,8 @@ ResolveLakeTableOptions(CreateLakeTableStmt *stmt, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("table access method \"%s\" does not exist", stmt->table_type), - errhint("CREATE LAKE TABLE ... USING ICEBERG requires an extension that provides the \"%s\" table access method.", - stmt->table_type))); + errhint("CREATE LAKE TABLE ... USING \"%s\" requires an extension that provides the \"%s\" table access method.", + stmt->table_type, stmt->table_type))); /* * Determine catalog name: use explicit value if provided, otherwise diff --git a/src/test/regress/expected/lake_table.out b/src/test/regress/expected/lake_table.out index 40e98e730a8..78a457c15ab 100644 --- a/src/test/regress/expected/lake_table.out +++ b/src/test/regress/expected/lake_table.out @@ -118,7 +118,7 @@ SELECT count(DISTINCT gp_segment_id) > 1 AS on_all_segments -- Without a provider extension there is no iceberg table AM CREATE LAKE TABLE lake_test_t0 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol; -- fail with hint ERROR: table access method "iceberg" does not exist -HINT: CREATE LAKE TABLE ... USING ICEBERG requires an extension that provides the "iceberg" table access method. +HINT: CREATE LAKE TABLE ... USING "iceberg" requires an extension that provides the "iceberg" table access method. -- The default catalog/volume GUCs verify that the object exists SET iceberg_default_catalog = 'no_such_catalog'; -- fail ERROR: invalid value for parameter "iceberg_default_catalog": "no_such_catalog" From f4ac7dd4e5a9c9330c2048870a6fb18e46d4447b Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Thu, 23 Jul 2026 15:29:17 +0800 Subject: [PATCH 25/29] tests/commands: check pg_lake_table cleanup by saved OID; tidy the SET AM guard Per PR review (andr-sokolov): - The post-DROP checks joined pg_lake_table against pg_class, so once the relation was gone they returned 0 rows even if the pg_lake_table entry had leaked. Save the table's OID before the drop and probe pg_lake_table by ltrelid directly, so an orphaned row would actually be caught. - ATPrepSetAccessMethod: test OidIsValid(iceberg_amoid) once for the two iceberg checks, and splice ICEBERG_TABLE_AM_NAME into the message at compile time instead of passing the constant as a format argument. --- src/backend/commands/tablecmds.c | 25 +++++++++++++----------- src/test/regress/expected/lake_table.out | 15 +++++++------- src/test/regress/sql/lake_table.sql | 15 +++++++------- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 7b56e1dd64c..c59ff84a4ce 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -17124,17 +17124,20 @@ ATPrepSetAccessMethod(AlteredTableInfo *tab, Relation rel, const char *amname) * it with SET ACCESS METHOD. */ iceberg_amoid = GetIcebergTableAmOid(true); - if (OidIsValid(iceberg_amoid) && amoid == iceberg_amoid) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot change access method of table \"%s\" to \"%s\"", - RelationGetRelationName(rel), ICEBERG_TABLE_AM_NAME), - errhint("Use CREATE LAKE TABLE ... USING ICEBERG to create a lake table."))); - if (OidIsValid(iceberg_amoid) && rel->rd_rel->relam == iceberg_amoid) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot change access method of lake table \"%s\"", - RelationGetRelationName(rel)))); + if (OidIsValid(iceberg_amoid)) + { + if (amoid == iceberg_amoid) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot change access method of table \"%s\" to \"" ICEBERG_TABLE_AM_NAME "\"", + RelationGetRelationName(rel)), + errhint("Use CREATE LAKE TABLE ... USING ICEBERG to create a lake table."))); + if (rel->rd_rel->relam == iceberg_amoid) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot change access method of lake table \"%s\"", + RelationGetRelationName(rel)))); + } /* Save info for Phase 3 to do the real work */ tab->rewrite |= AT_REWRITE_ACCESS_METHOD; diff --git a/src/test/regress/expected/lake_table.out b/src/test/regress/expected/lake_table.out index 78a457c15ab..848789764c2 100644 --- a/src/test/regress/expected/lake_table.out +++ b/src/test/regress/expected/lake_table.out @@ -238,10 +238,11 @@ DETAIL: table lake_test_t1 depends on catalog lake_test_cat table lake_test_t2 depends on catalog lake_test_cat table lake_test_t4 depends on catalog lake_test_cat HINT: Use DROP ... CASCADE to drop the dependent objects too. --- Dropping a lake table removes its pg_lake_table entry +-- Dropping a lake table removes its pg_lake_table entry: remember the +-- table's OID so the check still finds an orphaned row after the drop +SELECT oid AS t1_oid FROM pg_class WHERE relname = 'lake_test_t1' \gset DROP LAKE TABLE lake_test_t1; -SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid - WHERE c.relname = 'lake_test_t1'; +SELECT count(*) FROM pg_lake_table WHERE ltrelid = :t1_oid; count ------- 0 @@ -266,19 +267,19 @@ SELECT count(*) FROM pg_lake_table WHERE ltrelid = 'lake_test_t2'::regclass; (1 row) -- the correct command still works +SELECT oid AS t2_oid FROM pg_class WHERE relname = 'lake_test_t2' \gset DROP LAKE TABLE lake_test_t2; -SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid - WHERE c.relname = 'lake_test_t2'; +SELECT count(*) FROM pg_lake_table WHERE ltrelid = :t2_oid; count ------- 0 (1 row) -- DROP FOREIGN CATALOG ... CASCADE takes the remaining tables with it +SELECT oid AS t4_oid FROM pg_class WHERE relname = 'lake_test_t4' \gset DROP FOREIGN CATALOG lake_test_cat CASCADE; NOTICE: drop cascades to table lake_test_t4 -SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid - WHERE c.relname LIKE 'lake\_test%'; +SELECT count(*) FROM pg_lake_table WHERE ltrelid = :t4_oid; count ------- 0 diff --git a/src/test/regress/sql/lake_table.sql b/src/test/regress/sql/lake_table.sql index b486e34dc83..deb5dba38bd 100644 --- a/src/test/regress/sql/lake_table.sql +++ b/src/test/regress/sql/lake_table.sql @@ -119,10 +119,11 @@ RESET ROLE; DROP SERVER lake_test_srv; -- fail, catalog and volume depend on it DROP FOREIGN CATALOG lake_test_cat; -- fail, tables depend on it --- Dropping a lake table removes its pg_lake_table entry +-- Dropping a lake table removes its pg_lake_table entry: remember the +-- table's OID so the check still finds an orphaned row after the drop +SELECT oid AS t1_oid FROM pg_class WHERE relname = 'lake_test_t1' \gset DROP LAKE TABLE lake_test_t1; -SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid - WHERE c.relname = 'lake_test_t1'; +SELECT count(*) FROM pg_lake_table WHERE ltrelid = :t1_oid; -- DROP LAKE TABLE rejects a non-lake table ... DROP LAKE TABLE lake_test_heap; -- fail, not a lake table @@ -132,14 +133,14 @@ DROP TABLE IF EXISTS lake_test_t2; -- IF EXISTS does not suppress wrong-type err -- the rejected drops must have left the table and its lake metadata intact SELECT count(*) FROM pg_lake_table WHERE ltrelid = 'lake_test_t2'::regclass; -- the correct command still works +SELECT oid AS t2_oid FROM pg_class WHERE relname = 'lake_test_t2' \gset DROP LAKE TABLE lake_test_t2; -SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid - WHERE c.relname = 'lake_test_t2'; +SELECT count(*) FROM pg_lake_table WHERE ltrelid = :t2_oid; -- DROP FOREIGN CATALOG ... CASCADE takes the remaining tables with it +SELECT oid AS t4_oid FROM pg_class WHERE relname = 'lake_test_t4' \gset DROP FOREIGN CATALOG lake_test_cat CASCADE; -SELECT count(*) FROM pg_lake_table lt JOIN pg_class c ON c.oid = lt.ltrelid - WHERE c.relname LIKE 'lake\_test%'; +SELECT count(*) FROM pg_lake_table WHERE ltrelid = :t4_oid; -- DROP variants DROP FOREIGN CATALOG lake_test_cat; -- fail, already gone From 5af075edc8b76a6d4804d116135ea1fe1dbf5f04 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Thu, 13 Aug 2026 17:40:46 +0800 Subject: [PATCH 26/29] Revert the kernel-side lake-table DDL scaffolding Review of this PR converged on doing lake tables as an extension instead of as kernel syntax: no new grammar, no new catalogs, no new node types, and nothing in src/backend that has to be maintained forever for one storage format. The replacement is a contrib module that reaches the same place through mechanisms PostgreSQL already has -- a table access method for "CREATE TABLE ... USING iceberg", and foreign servers plus user mappings for catalog endpoints and credentials. This reverts every kernel change made on this branch in one commit rather than rewriting the branch, so that the 64 inline review comments stay anchored to the code they were written about. It is also the cheapest way back: if the extension approach is later judged wrong, reverting this single commit restores the whole kernel scaffolding. The pre-revert tip is also kept as archive/iceberg-ddl-kernel-pr1842 on the author's fork. The resulting tree is identical to the merge base (9c3d48edf52), verified by comparing tree object ids rather than by reading the diff. The extension implementation follows in the next commits. --- .../src/test/regress/expected/oidjoins.out | 7 - .../test/regress/expected/sanity_check.out | 3 - doc/src/sgml/ref/allfiles.sgml | 6 - doc/src/sgml/ref/create_foreign_catalog.sgml | 144 ------ doc/src/sgml/ref/create_foreign_volume.sgml | 133 ----- doc/src/sgml/ref/create_lake_table.sgml | 184 ------- doc/src/sgml/ref/drop_foreign_catalog.sgml | 117 ----- doc/src/sgml/ref/drop_foreign_volume.sgml | 117 ----- doc/src/sgml/ref/drop_lake_table.sgml | 119 ----- doc/src/sgml/reference.sgml | 6 - src/backend/catalog/Makefile | 1 - src/backend/catalog/aclchk.c | 12 - src/backend/catalog/dependency.c | 12 - src/backend/catalog/heap.c | 5 - src/backend/catalog/objectaddress.c | 154 ------ src/backend/catalog/oid_dispatch.c | 36 -- src/backend/commands/Makefile | 1 - src/backend/commands/alter.c | 2 - src/backend/commands/dropcmds.c | 8 - src/backend/commands/event_trigger.c | 12 - src/backend/commands/foreigncmds.c | 271 ----------- src/backend/commands/laketablecmds.c | 458 ------------------ src/backend/commands/seclabel.c | 2 - src/backend/commands/tablecmds.c | 81 ---- src/backend/foreign/foreign.c | 91 ---- src/backend/nodes/copyfuncs.funcs.c | 67 --- src/backend/nodes/copyfuncs.switch.c | 9 - src/backend/nodes/equalfuncs.c | 47 -- src/backend/nodes/outfast.c | 32 -- src/backend/nodes/outfuncs.c | 14 - src/backend/nodes/outfuncs_common.c | 2 - src/backend/nodes/readfast.c | 50 -- src/backend/nodes/readfuncs_common.c | 2 - src/backend/parser/gram.y | 203 +------- src/backend/tcop/utility.c | 103 +--- src/backend/utils/cache/syscache.c | 26 - src/backend/utils/misc/guc_tables.c | 23 - src/bin/pg_dump/pg_dump.c | 13 - src/bin/psql/tab-complete.c | 90 +--- src/include/catalog/catversion.h | 2 +- src/include/catalog/dependency.h | 4 +- src/include/catalog/oid_dispatch.h | 4 - src/include/catalog/pg_foreign_catalog.h | 70 --- src/include/catalog/pg_foreign_volume.h | 69 --- src/include/catalog/pg_lake_table.h | 74 --- src/include/commands/defrem.h | 2 - src/include/commands/laketablecmds.h | 61 --- src/include/foreign/foreign.h | 12 - src/include/nodes/nodes.h | 3 - src/include/nodes/parsenodes.h | 30 -- src/include/parser/kwlist.h | 2 - src/include/tcop/cmdtaglist.h | 6 - src/include/utils/sync_guc_name.h | 2 - src/include/utils/syscache.h | 4 - src/test/regress/expected/lake_table.out | 304 ------------ src/test/regress/expected/oidjoins.out | 7 - src/test/regress/expected/sanity_check.out | 3 - src/test/regress/greenplum_schedule | 3 - src/test/regress/sql/lake_table.sql | 158 ------ .../singlenode_regress/expected/oidjoins.out | 7 - .../expected/sanity_check.out | 3 - 61 files changed, 8 insertions(+), 3485 deletions(-) delete mode 100644 doc/src/sgml/ref/create_foreign_catalog.sgml delete mode 100644 doc/src/sgml/ref/create_foreign_volume.sgml delete mode 100644 doc/src/sgml/ref/create_lake_table.sgml delete mode 100644 doc/src/sgml/ref/drop_foreign_catalog.sgml delete mode 100644 doc/src/sgml/ref/drop_foreign_volume.sgml delete mode 100644 doc/src/sgml/ref/drop_lake_table.sgml delete mode 100644 src/backend/commands/laketablecmds.c delete mode 100644 src/include/catalog/pg_foreign_catalog.h delete mode 100644 src/include/catalog/pg_foreign_volume.h delete mode 100644 src/include/catalog/pg_lake_table.h delete mode 100644 src/include/commands/laketablecmds.h delete mode 100644 src/test/regress/expected/lake_table.out delete mode 100644 src/test/regress/sql/lake_table.sql diff --git a/contrib/pax_storage/src/test/regress/expected/oidjoins.out b/contrib/pax_storage/src/test/regress/expected/oidjoins.out index 6dc91c68aec..19094e111dc 100644 --- a/contrib/pax_storage/src/test/regress/expected/oidjoins.out +++ b/contrib/pax_storage/src/test/regress/expected/oidjoins.out @@ -235,10 +235,6 @@ NOTICE: checking pg_foreign_data_wrapper {fdwhandler} => pg_proc {oid} NOTICE: checking pg_foreign_data_wrapper {fdwvalidator} => pg_proc {oid} NOTICE: checking pg_foreign_server {srvowner} => pg_authid {oid} NOTICE: checking pg_foreign_server {srvfdw} => pg_foreign_data_wrapper {oid} -NOTICE: checking pg_foreign_catalog {fcowner} => pg_authid {oid} -NOTICE: checking pg_foreign_catalog {fcserver} => pg_foreign_server {oid} -NOTICE: checking pg_foreign_volume {fvowner} => pg_authid {oid} -NOTICE: checking pg_foreign_volume {fvserver} => pg_foreign_server {oid} NOTICE: checking pg_user_mapping {umuser} => pg_authid {oid} NOTICE: checking pg_user_mapping {umserver} => pg_foreign_server {oid} NOTICE: checking pg_compression {compconstructor} => pg_proc {oid} @@ -251,9 +247,6 @@ NOTICE: checking pg_foreign_table {ftrelid} => pg_class {oid} NOTICE: checking pg_foreign_table {ftserver} => pg_foreign_server {oid} NOTICE: checking pg_foreign_table_seg {ftsrelid} => pg_class {oid} NOTICE: checking pg_foreign_table_seg {ftsserver} => pg_foreign_server {oid} -NOTICE: checking pg_lake_table {ltrelid} => pg_class {oid} -NOTICE: checking pg_lake_table {ltforeign_catalog} => pg_foreign_catalog {oid} -NOTICE: checking pg_lake_table {ltforeign_volume} => pg_foreign_volume {oid} NOTICE: checking pg_policy {polrelid} => pg_class {oid} NOTICE: checking pg_policy {polroles} => pg_authid {oid} NOTICE: checking pg_default_acl {defaclrole} => pg_authid {oid} diff --git a/contrib/pax_storage/src/test/regress/expected/sanity_check.out b/contrib/pax_storage/src/test/regress/expected/sanity_check.out index 2e36e3abd35..b61eee481bd 100644 --- a/contrib/pax_storage/src/test/regress/expected/sanity_check.out +++ b/contrib/pax_storage/src/test/regress/expected/sanity_check.out @@ -112,16 +112,13 @@ pg_enum|t pg_event_trigger|t pg_extension|t pg_extprotocol|t -pg_foreign_catalog|t pg_foreign_data_wrapper|t pg_foreign_server|t pg_foreign_table|t pg_foreign_table_seg|t -pg_foreign_volume|t pg_index|t pg_inherits|t pg_init_privs|t -pg_lake_table|t pg_language|t pg_largeobject|t pg_largeobject_metadata|t diff --git a/doc/src/sgml/ref/allfiles.sgml b/doc/src/sgml/ref/allfiles.sgml index 8819f0545b2..3d47bff7fef 100644 --- a/doc/src/sgml/ref/allfiles.sgml +++ b/doc/src/sgml/ref/allfiles.sgml @@ -69,14 +69,11 @@ Complete list of usable sgml source files in this directory. - - - @@ -121,14 +118,11 @@ Complete list of usable sgml source files in this directory. - - - diff --git a/doc/src/sgml/ref/create_foreign_catalog.sgml b/doc/src/sgml/ref/create_foreign_catalog.sgml deleted file mode 100644 index 771503a8bed..00000000000 --- a/doc/src/sgml/ref/create_foreign_catalog.sgml +++ /dev/null @@ -1,144 +0,0 @@ - - - - - CREATE FOREIGN CATALOG - - - - CREATE FOREIGN CATALOG - 7 - SQL - Language Statements - - - - CREATE FOREIGN CATALOG - define a new foreign catalog - - - - -CREATE FOREIGN CATALOG [ IF NOT EXISTS ] catalog_name - SERVER server_name - TYPE 'catalog_type' - [ OPTIONS ( option 'value' [, ...] ) ] - - - - - Description - - - CREATE FOREIGN CATALOG defines a new foreign catalog. - Foreign catalog names are global within a database. The user who creates - the catalog becomes its owner. - - - - Creating a foreign catalog requires USAGE privilege on - the referenced foreign server. - - - - The required catalog type is stored verbatim. Provider-specific validation - of the catalog type and options is outside the kernel, and generic options - currently have no kernel validator. - - - - - Parameters - - - - IF NOT EXISTS - - - Do not throw an error if a foreign catalog with the same name already - exists. A notice is issued in this case. - - - - - - catalog_name - - - The database-global name of the foreign catalog to be created. - - - - - - server_name - - - The name of an existing foreign server for the catalog. - - - - - - catalog_type - - - The required provider-specific catalog type. The value is stored - verbatim. - - - - - - option - - - The name of a provider-specific option for the catalog. - - - - - - value - - - The value of a provider-specific catalog option. - - - - - - - - Examples - - - Create a Hive catalog that uses the foreign server - hive_srv: - -CREATE FOREIGN CATALOG hive_cat SERVER hive_srv TYPE 'hive' OPTIONS (uri 'thrift://localhost:9083'); - - - - - Compatibility - - - CREATE FOREIGN CATALOG is an - Apache Cloudberry extension and is not defined in - the SQL standard. - - - - - See Also - - - - - - - - diff --git a/doc/src/sgml/ref/create_foreign_volume.sgml b/doc/src/sgml/ref/create_foreign_volume.sgml deleted file mode 100644 index 6bc3df3d28e..00000000000 --- a/doc/src/sgml/ref/create_foreign_volume.sgml +++ /dev/null @@ -1,133 +0,0 @@ - - - - - CREATE FOREIGN VOLUME - - - - CREATE FOREIGN VOLUME - 7 - SQL - Language Statements - - - - CREATE FOREIGN VOLUME - define a new foreign volume - - - - -CREATE FOREIGN VOLUME [ IF NOT EXISTS ] volume_name - SERVER server_name - [ OPTIONS ( option 'value' [, ...] ) ] - - - - - Description - - - CREATE FOREIGN VOLUME defines a new foreign volume. - Foreign volume names are global within a database. The user who creates - the volume becomes its owner. - - - - Creating a foreign volume requires USAGE privilege on - the referenced foreign server. - - - - Volume options are provider-specific. Provider-specific option validation - is outside the kernel, and generic options currently have no kernel - validator. - - - - - Parameters - - - - IF NOT EXISTS - - - Do not throw an error if a foreign volume with the same name already - exists. A notice is issued in this case. - - - - - - volume_name - - - The database-global name of the foreign volume to be created. - - - - - - server_name - - - The name of an existing foreign server for the volume. - - - - - - option - - - The name of a provider-specific option for the volume. - - - - - - value - - - The value of a provider-specific volume option. - - - - - - - - Examples - - - Create a volume for an object storage prefix using the foreign server - s3_srv: - -CREATE FOREIGN VOLUME s3_vol SERVER s3_srv OPTIONS (base_path 's3://bucket/prefix'); - - - - - Compatibility - - - CREATE FOREIGN VOLUME is an - Apache Cloudberry extension and is not defined in - the SQL standard. - - - - - See Also - - - - - - - - diff --git a/doc/src/sgml/ref/create_lake_table.sgml b/doc/src/sgml/ref/create_lake_table.sgml deleted file mode 100644 index 043c4d5e750..00000000000 --- a/doc/src/sgml/ref/create_lake_table.sgml +++ /dev/null @@ -1,184 +0,0 @@ - - - - - CREATE LAKE TABLE - - - - CREATE LAKE TABLE - 7 - SQL - Language Statements - - - - CREATE LAKE TABLE - define a new lake table - - - - -CREATE LAKE TABLE [ IF NOT EXISTS ] table_name ( [ - column_name data_type [, ... ] - ] ) - USING format - [ CATALOG catalog_name ] - [ VOLUME volume_name ] - [ OPTIONS ( option 'value' [, ...] ) ] - - - - - Description - - - CREATE LAKE TABLE defines a new lake table: a table whose - data lives in external object storage and is described by an open table - format. The parenthesized element list is required, but it can be empty. - - - - The USING clause names the table format. The only - supported format is ICEBERG, which requires an extension - that provides the iceberg table access method. - - - - The foreign catalog and volume can be specified by the - CATALOG and VOLUME clauses. If either - clause is omitted, the corresponding iceberg_default_catalog - or iceberg_default_volume configuration parameter is used. - - - - Lake tables are always distributed randomly, because their fragments are not - hash-distributed across segments. A DISTRIBUTED clause, - if given, is accepted with a warning and has no effect. - - - - - Parameters - - - - IF NOT EXISTS - - - Do not throw an error if a relation with the same name already exists. - A notice is issued in this case. - - - - - - table_name - - - The name, optionally schema-qualified, of the lake table to be created. - - - - - - column_name - - - The name of a column in the new table. - - - - - - data_type - - - The data type of a column in the new table. - - - - - - format - - - The lake table format. The only accepted value is - ICEBERG. - - - - - - catalog_name - - - The name of the foreign catalog to use. If omitted, the value of - iceberg_default_catalog is used. - - - - - - volume_name - - - The name of the foreign volume to use. If omitted, the value of - iceberg_default_volume is used. - - - - - - option - - - The name of an option for the lake table. - - - - - - value - - - The value of a lake table option. - - - - - - - - Examples - - - Create an Iceberg lake table using an explicit foreign catalog and volume: - -CREATE LAKE TABLE t (a int, b text) USING ICEBERG CATALOG hive_cat VOLUME s3_vol OPTIONS (fileformat 'parquet'); - - - - - Compatibility - - - CREATE LAKE TABLE is an - Apache Cloudberry extension and is not defined in - the SQL standard. - - - - - See Also - - - - - - - - - diff --git a/doc/src/sgml/ref/drop_foreign_catalog.sgml b/doc/src/sgml/ref/drop_foreign_catalog.sgml deleted file mode 100644 index 91c2a0d70cd..00000000000 --- a/doc/src/sgml/ref/drop_foreign_catalog.sgml +++ /dev/null @@ -1,117 +0,0 @@ - - - - - DROP FOREIGN CATALOG - - - - DROP FOREIGN CATALOG - 7 - SQL - Language Statements - - - - DROP FOREIGN CATALOG - remove a foreign catalog - - - - -DROP FOREIGN CATALOG [ IF EXISTS ] name [, ...] [ CASCADE | RESTRICT ] - - - - - Description - - - DROP FOREIGN CATALOG removes one or more foreign - catalogs. Foreign catalog names are global within a database. To execute - this command, the current user must own each catalog. - - - - Using CASCADE can also drop dependent lake tables. - - - - - Parameters - - - - IF EXISTS - - - Do not throw an error if a foreign catalog does not exist. A notice is - issued in this case. - - - - - - name - - - The database-global name of a foreign catalog to drop. - - - - - - CASCADE - - - Automatically drop objects that depend on the catalog, including - dependent lake tables, and in turn all objects that depend on those - objects (see ). - - - - - - RESTRICT - - - Refuse to drop the catalog if any objects depend on it. This is the - default. - - - - - - - - Examples - - - Drop the foreign catalog hive_cat: - -DROP FOREIGN CATALOG hive_cat; - - - - - Compatibility - - - DROP FOREIGN CATALOG is an - Apache Cloudberry extension and is not defined in - the SQL standard. - - - - - See Also - - - - - - - - diff --git a/doc/src/sgml/ref/drop_foreign_volume.sgml b/doc/src/sgml/ref/drop_foreign_volume.sgml deleted file mode 100644 index 7e8783cdeb9..00000000000 --- a/doc/src/sgml/ref/drop_foreign_volume.sgml +++ /dev/null @@ -1,117 +0,0 @@ - - - - - DROP FOREIGN VOLUME - - - - DROP FOREIGN VOLUME - 7 - SQL - Language Statements - - - - DROP FOREIGN VOLUME - remove a foreign volume - - - - -DROP FOREIGN VOLUME [ IF EXISTS ] name [, ...] [ CASCADE | RESTRICT ] - - - - - Description - - - DROP FOREIGN VOLUME removes one or more foreign volumes. - Foreign volume names are global within a database. To execute this - command, the current user must own each volume. - - - - Using CASCADE can also drop dependent lake tables. - - - - - Parameters - - - - IF EXISTS - - - Do not throw an error if a foreign volume does not exist. A notice is - issued in this case. - - - - - - name - - - The database-global name of a foreign volume to drop. - - - - - - CASCADE - - - Automatically drop objects that depend on the volume, including - dependent lake tables, and in turn all objects that depend on those - objects (see ). - - - - - - RESTRICT - - - Refuse to drop the volume if any objects depend on it. This is the - default. - - - - - - - - Examples - - - Drop the foreign volume s3_vol: - -DROP FOREIGN VOLUME s3_vol; - - - - - Compatibility - - - DROP FOREIGN VOLUME is an - Apache Cloudberry extension and is not defined in - the SQL standard. - - - - - See Also - - - - - - - - diff --git a/doc/src/sgml/ref/drop_lake_table.sgml b/doc/src/sgml/ref/drop_lake_table.sgml deleted file mode 100644 index 0847f7efbbb..00000000000 --- a/doc/src/sgml/ref/drop_lake_table.sgml +++ /dev/null @@ -1,119 +0,0 @@ - - - - - DROP LAKE TABLE - - - - DROP LAKE TABLE - 7 - SQL - Language Statements - - - - DROP LAKE TABLE - remove a lake table - - - - -DROP LAKE TABLE [ IF EXISTS ] name [, ...] [ CASCADE | RESTRICT ] - - - - - Description - - - DROP LAKE TABLE removes one or more lake tables. - Only the owner of a lake table can remove it. - - - - This command only drops lake tables. Plain - DROP TABLE rejects lake tables and is used for - ordinary tables. - - - - - Parameters - - - - IF EXISTS - - - Do not throw an error if the lake table does not exist. A notice is - issued in this case. - - - - - - name - - - The name, optionally schema-qualified, of a lake table to drop. - - - - - - CASCADE - - - Automatically drop objects that depend on the lake table, and in - turn all objects that depend on those objects (see ). - - - - - - RESTRICT - - - Refuse to drop the lake table if any objects depend on it. This is - the default. - - - - - - - - Examples - - - Drop the lake table t: - -DROP LAKE TABLE t; - - - - - Compatibility - - - DROP LAKE TABLE is an - Apache Cloudberry extension and is not defined in - the SQL standard. - - - - - See Also - - - - - - - - - diff --git a/doc/src/sgml/reference.sgml b/doc/src/sgml/reference.sgml index 3aaf0601793..a251363388e 100644 --- a/doc/src/sgml/reference.sgml +++ b/doc/src/sgml/reference.sgml @@ -96,14 +96,11 @@ &createDynamicTable; &createEventTrigger; &createExtension; - &createForeignCatalog; &createForeignDataWrapper; &createForeignTable; - &createForeignVolume; &createFunction; &createGroup; &createIndex; - &createLakeTable; &createLanguage; &createMaterializedView; &createOperator; @@ -147,14 +144,11 @@ &dropDynamicTable; &dropEventTrigger; &dropExtension; - &dropForeignCatalog; &dropForeignDataWrapper; &dropForeignTable; - &dropForeignVolume; &dropFunction; &dropGroup; &dropIndex; - &dropLakeTable; &dropLanguage; &dropMaterializedView; &dropOperator; diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 44598e25cff..5679abf152a 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -102,7 +102,6 @@ CATALOG_HEADERS := \ pg_subscription_rel.h gp_partition_template.h pg_task.h pg_task_run_history.h \ pg_profile.h pg_password_history.h pg_directory_table.h gp_storage_server.h \ gp_storage_user_mapping.h pg_tag.h pg_tag_description.h \ - pg_foreign_catalog.h pg_foreign_volume.h pg_lake_table.h \ gp_matview_aux.h \ gp_matview_tables.h diff --git a/src/backend/catalog/aclchk.c b/src/backend/catalog/aclchk.c index 573310a42e1..28e37f72ba1 100644 --- a/src/backend/catalog/aclchk.c +++ b/src/backend/catalog/aclchk.c @@ -3050,12 +3050,6 @@ aclcheck_error(AclResult aclerr, ObjectType objtype, case OBJECT_FOREIGN_SERVER: msg = gettext_noop("permission denied for foreign server %s"); break; - case OBJECT_FOREIGN_CATALOG: - msg = gettext_noop("permission denied for foreign catalog %s"); - break; - case OBJECT_FOREIGN_VOLUME: - msg = gettext_noop("permission denied for foreign volume %s"); - break; case OBJECT_FOREIGN_TABLE: msg = gettext_noop("permission denied for foreign table %s"); break; @@ -3204,12 +3198,6 @@ aclcheck_error(AclResult aclerr, ObjectType objtype, case OBJECT_FOREIGN_SERVER: msg = gettext_noop("must be owner of foreign server %s"); break; - case OBJECT_FOREIGN_CATALOG: - msg = gettext_noop("must be owner of foreign catalog %s"); - break; - case OBJECT_FOREIGN_VOLUME: - msg = gettext_noop("must be owner of foreign volume %s"); - break; case OBJECT_FOREIGN_TABLE: msg = gettext_noop("must be owner of foreign table %s"); break; diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index 2621f12ed19..f48824d9edc 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -42,10 +42,8 @@ #include "catalog/pg_directory_table.h" #include "catalog/pg_event_trigger.h" #include "catalog/pg_extension.h" -#include "catalog/pg_foreign_catalog.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" -#include "catalog/pg_foreign_volume.h" #include "catalog/pg_init_privs.h" #include "catalog/pg_language.h" #include "catalog/pg_largeobject.h" @@ -228,8 +226,6 @@ static const Oid object_classes[] = { ExtprotocolRelationId, /* OCLASS_EXTPROTOCOL */ GpMatviewAuxId, /* OCLASS_MATVIEW_AUX */ TaskRelationId, /* OCLASS_TASK */ - ForeignCatalogRelationId, /* OCLASS_FOREIGN_CATALOG */ - ForeignVolumeRelationId, /* OCLASS_FOREIGN_VOLUME */ }; /* @@ -1633,8 +1629,6 @@ doDeletion(const ObjectAddress *object, int flags) case OCLASS_TSTEMPLATE: case OCLASS_FDW: case OCLASS_FOREIGN_SERVER: - case OCLASS_FOREIGN_CATALOG: - case OCLASS_FOREIGN_VOLUME: case OCLASS_USER_MAPPING: case OCLASS_DEFACL: case OCLASS_EVENT_TRIGGER: @@ -3147,12 +3141,6 @@ getObjectClass(const ObjectAddress *object) case TagDescriptionRelationId: return OCLASS_TAG_DESCRIPTION; - case ForeignCatalogRelationId: - return OCLASS_FOREIGN_CATALOG; - - case ForeignVolumeRelationId: - return OCLASS_FOREIGN_VOLUME; - default: { struct CustomObjectClass *coc; diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 3abd4701bc3..0b62b174afa 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -72,7 +72,6 @@ #include "catalog/storage.h" #include "catalog/storage_directory_table.h" #include "catalog/storage_xlog.h" -#include "commands/laketablecmds.h" #include "commands/tablecmds.h" #include "commands/typecmds.h" #include "miscadmin.h" @@ -2323,10 +2322,6 @@ heap_drop_with_catalog(Oid relid) */ CheckTableForSerializableConflictIn(rel); - /* If this is a lake table, remove its pg_lake_table entry */ - if (RelationIsLakeTable(rel)) - RemoveLakeTableEntry(relid); - /* * Delete pg_foreign_table tuple first. */ diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index 3432197bfb7..db5294c89e9 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -40,10 +40,8 @@ #include "catalog/pg_event_trigger.h" #include "catalog/pg_extension.h" #include "catalog/pg_extprotocol.h" -#include "catalog/pg_foreign_catalog.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" -#include "catalog/pg_foreign_volume.h" #include "catalog/pg_language.h" #include "catalog/pg_largeobject.h" #include "catalog/pg_largeobject_metadata.h" @@ -307,34 +305,6 @@ static const ObjectPropertyType ObjectProperty[] = OBJECT_FOREIGN_SERVER, true }, - { - "foreign catalog", - ForeignCatalogRelationId, - ForeignCatalogOidIndexId, - FOREIGNCATALOGOID, - FOREIGNCATALOGNAME, - Anum_pg_foreign_catalog_oid, - Anum_pg_foreign_catalog_fcname, - InvalidAttrNumber, - Anum_pg_foreign_catalog_fcowner, - InvalidAttrNumber, - OBJECT_FOREIGN_CATALOG, - true - }, - { - "foreign volume", - ForeignVolumeRelationId, - ForeignVolumeOidIndexId, - FOREIGNVOLUMEOID, - FOREIGNVOLUMENAME, - Anum_pg_foreign_volume_oid, - Anum_pg_foreign_volume_fvname, - InvalidAttrNumber, - Anum_pg_foreign_volume_fvowner, - InvalidAttrNumber, - OBJECT_FOREIGN_VOLUME, - true - }, { "storage server", StorageServerRelationId, @@ -1025,14 +995,6 @@ static const struct object_type_map /* OCLASS_TAG */ { "tag", OBJECT_TAG - }, - /* OCLASS_FOREIGN_CATALOG */ - { - "catalog", OBJECT_FOREIGN_CATALOG - }, - /* OCLASS_FOREIGN_VOLUME */ - { - "volume", OBJECT_FOREIGN_VOLUME } }; @@ -1203,8 +1165,6 @@ get_object_address(ObjectType objtype, Node *object, case OBJECT_LANGUAGE: case OBJECT_FDW: case OBJECT_FOREIGN_SERVER: - case OBJECT_FOREIGN_CATALOG: - case OBJECT_FOREIGN_VOLUME: case OBJECT_EVENT_TRIGGER: case OBJECT_EXTPROTOCOL: case OBJECT_PARAMETER_ACL: @@ -1512,16 +1472,6 @@ get_object_address_unqualified(ObjectType objtype, address.objectId = get_foreign_server_oid(name, missing_ok); address.objectSubId = 0; break; - case OBJECT_FOREIGN_CATALOG: - address.classId = ForeignCatalogRelationId; - address.objectId = get_foreign_catalog_oid(name, missing_ok); - address.objectSubId = 0; - break; - case OBJECT_FOREIGN_VOLUME: - address.classId = ForeignVolumeRelationId; - address.objectId = get_foreign_volume_oid(name, missing_ok); - address.objectSubId = 0; - break; case OBJECT_EVENT_TRIGGER: address.classId = EventTriggerRelationId; address.objectId = get_event_trigger_oid(name, missing_ok); @@ -2556,8 +2506,6 @@ pg_get_object_address(PG_FUNCTION_ARGS) case OBJECT_EXTENSION: case OBJECT_FDW: case OBJECT_FOREIGN_SERVER: - case OBJECT_FOREIGN_CATALOG: - case OBJECT_FOREIGN_VOLUME: case OBJECT_STORAGE_SERVER: case OBJECT_LANGUAGE: case OBJECT_PARAMETER_ACL: @@ -2715,8 +2663,6 @@ check_object_ownership(Oid roleid, ObjectType objtype, ObjectAddress address, case OBJECT_EXTENSION: case OBJECT_FDW: case OBJECT_FOREIGN_SERVER: - case OBJECT_FOREIGN_CATALOG: - case OBJECT_FOREIGN_VOLUME: case OBJECT_LANGUAGE: case OBJECT_PUBLICATION: case OBJECT_SCHEMA: @@ -4009,50 +3955,6 @@ getObjectDescription(const ObjectAddress *object, bool missing_ok) break; } - case OCLASS_FOREIGN_CATALOG: - { - HeapTuple catTup; - Form_pg_foreign_catalog catForm; - - catTup = SearchSysCache1(FOREIGNCATALOGOID, - ObjectIdGetDatum(object->objectId)); - if (!HeapTupleIsValid(catTup)) - { - if (!missing_ok) - elog(ERROR, "cache lookup failed for foreign catalog %u", - object->objectId); - break; - } - - catForm = (Form_pg_foreign_catalog) GETSTRUCT(catTup); - appendStringInfo(&buffer, _("catalog %s"), - NameStr(catForm->fcname)); - ReleaseSysCache(catTup); - break; - } - - case OCLASS_FOREIGN_VOLUME: - { - HeapTuple volTup; - Form_pg_foreign_volume volForm; - - volTup = SearchSysCache1(FOREIGNVOLUMEOID, - ObjectIdGetDatum(object->objectId)); - if (!HeapTupleIsValid(volTup)) - { - if (!missing_ok) - elog(ERROR, "cache lookup failed for foreign volume %u", - object->objectId); - break; - } - - volForm = (Form_pg_foreign_volume) GETSTRUCT(volTup); - appendStringInfo(&buffer, _("volume %s"), - NameStr(volForm->fvname)); - ReleaseSysCache(volTup); - break; - } - case OCLASS_USER_MAPPING: { HeapTuple tup; @@ -5047,14 +4949,6 @@ getObjectTypeDescription(const ObjectAddress *object, bool missing_ok) appendStringInfoString(&buffer, "server"); break; - case OCLASS_FOREIGN_CATALOG: - appendStringInfoString(&buffer, "catalog"); - break; - - case OCLASS_FOREIGN_VOLUME: - appendStringInfoString(&buffer, "volume"); - break; - case OCLASS_USER_MAPPING: appendStringInfoString(&buffer, "user mapping"); break; @@ -6149,54 +6043,6 @@ getObjectIdentityParts(const ObjectAddress *object, break; } - case OCLASS_FOREIGN_CATALOG: - { - HeapTuple catTup; - Form_pg_foreign_catalog catForm; - - catTup = SearchSysCache1(FOREIGNCATALOGOID, - ObjectIdGetDatum(object->objectId)); - if (!HeapTupleIsValid(catTup)) - { - if (!missing_ok) - elog(ERROR, "cache lookup failed for foreign catalog %u", - object->objectId); - break; - } - - catForm = (Form_pg_foreign_catalog) GETSTRUCT(catTup); - appendStringInfoString(&buffer, - quote_identifier(NameStr(catForm->fcname))); - if (objname) - *objname = list_make1(pstrdup(NameStr(catForm->fcname))); - ReleaseSysCache(catTup); - break; - } - - case OCLASS_FOREIGN_VOLUME: - { - HeapTuple volTup; - Form_pg_foreign_volume volForm; - - volTup = SearchSysCache1(FOREIGNVOLUMEOID, - ObjectIdGetDatum(object->objectId)); - if (!HeapTupleIsValid(volTup)) - { - if (!missing_ok) - elog(ERROR, "cache lookup failed for foreign volume %u", - object->objectId); - break; - } - - volForm = (Form_pg_foreign_volume) GETSTRUCT(volTup); - appendStringInfoString(&buffer, - quote_identifier(NameStr(volForm->fvname))); - if (objname) - *objname = list_make1(pstrdup(NameStr(volForm->fvname))); - ReleaseSysCache(volTup); - break; - } - case OCLASS_STORAGE_SERVER: { StorageServer *srv; diff --git a/src/backend/catalog/oid_dispatch.c b/src/backend/catalog/oid_dispatch.c index 5271c65cc9f..888c66b6a73 100644 --- a/src/backend/catalog/oid_dispatch.c +++ b/src/backend/catalog/oid_dispatch.c @@ -98,10 +98,8 @@ #include "catalog/pg_extension.h" #include "catalog/pg_extprotocol.h" #include "catalog/pg_event_trigger.h" -#include "catalog/pg_foreign_catalog.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" -#include "catalog/pg_foreign_volume.h" #include "catalog/pg_language.h" #include "catalog/pg_largeobject_metadata.h" #include "catalog/pg_namespace.h" @@ -882,40 +880,6 @@ GetNewOidForForeignServer(Relation relation, Oid indexId, AttrNumber oidcolumn, } -Oid -GetNewOidForForeignCatalog(Relation relation, Oid indexId, AttrNumber oidcolumn, - char *catname) -{ - OidAssignment key; - - Assert(RelationGetRelid(relation) == ForeignCatalogRelationId); - Assert(indexId == ForeignCatalogOidIndexId); - Assert(oidcolumn == Anum_pg_foreign_catalog_oid); - - memset(&key, 0, sizeof(OidAssignment)); - key.type = T_OidAssignment; - key.objname = catname; - return GetNewOrPreassignedOid(relation, indexId, oidcolumn, &key); - -} - -Oid -GetNewOidForForeignVolume(Relation relation, Oid indexId, AttrNumber oidcolumn, - char *volumename) -{ - OidAssignment key; - - Assert(RelationGetRelid(relation) == ForeignVolumeRelationId); - Assert(indexId == ForeignVolumeOidIndexId); - Assert(oidcolumn == Anum_pg_foreign_volume_oid); - - memset(&key, 0, sizeof(OidAssignment)); - key.type = T_OidAssignment; - key.objname = volumename; - return GetNewOrPreassignedOid(relation, indexId, oidcolumn, &key); - -} - Oid GetNewOidForStorageServer(Relation relation, Oid indexId, AttrNumber oidcolumn, char *srvname) diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile index 1a7d2d51db1..3451b45d115 100644 --- a/src/backend/commands/Makefile +++ b/src/backend/commands/Makefile @@ -42,7 +42,6 @@ OBJS = \ foreigncmds.o \ functioncmds.o \ indexcmds.o \ - laketablecmds.o \ lockcmds.o \ matview.o \ opclasscmds.o \ diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index b125d7b560b..31c290530d7 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -774,8 +774,6 @@ AlterObjectNamespace_oid(Oid classId, Oid objid, Oid nspOid, case OCLASS_STORAGE_USER_MAPPING: case OCLASS_TAG: case OCLASS_TAG_DESCRIPTION: - case OCLASS_FOREIGN_CATALOG: - case OCLASS_FOREIGN_VOLUME: /* ignore object types that don't have schema-qualified names */ break; diff --git a/src/backend/commands/dropcmds.c b/src/backend/commands/dropcmds.c index 1a6a3e38ef4..9bed9866aac 100644 --- a/src/backend/commands/dropcmds.c +++ b/src/backend/commands/dropcmds.c @@ -456,14 +456,6 @@ does_not_exist_skipping(ObjectType objtype, Node *object) msg = gettext_noop("server \"%s\" does not exist, skipping"); name = strVal(object); break; - case OBJECT_FOREIGN_CATALOG: - msg = gettext_noop("foreign catalog \"%s\" does not exist, skipping"); - name = strVal(object); - break; - case OBJECT_FOREIGN_VOLUME: - msg = gettext_noop("foreign volume \"%s\" does not exist, skipping"); - name = strVal(object); - break; case OBJECT_STORAGE_SERVER: msg = gettext_noop("storage server \"%s\" does not exist, skipping"); name = strVal(object); diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index e609f9bb0f1..f46567a5b0c 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -965,8 +965,6 @@ EventTriggerSupportsObjectType(ObjectType obtype) case OBJECT_EXTENSION: case OBJECT_FDW: case OBJECT_FOREIGN_SERVER: - case OBJECT_FOREIGN_CATALOG: - case OBJECT_FOREIGN_VOLUME: case OBJECT_FOREIGN_TABLE: case OBJECT_FUNCTION: case OBJECT_INDEX: @@ -1069,8 +1067,6 @@ EventTriggerSupportsObjectClass(ObjectClass objclass) case OCLASS_TSCONFIG: case OCLASS_FDW: case OCLASS_FOREIGN_SERVER: - case OCLASS_FOREIGN_CATALOG: - case OCLASS_FOREIGN_VOLUME: case OCLASS_USER_MAPPING: case OCLASS_DEFACL: case OCLASS_EXTENSION: @@ -2067,10 +2063,6 @@ stringify_grant_objtype(ObjectType objtype) return "FOREIGN DATA WRAPPER"; case OBJECT_FOREIGN_SERVER: return "FOREIGN SERVER"; - case OBJECT_FOREIGN_CATALOG: - return "FOREIGN CATALOG"; - case OBJECT_FOREIGN_VOLUME: - return "FOREIGN VOLUME"; case OBJECT_STORAGE_SERVER: return "STORAGE SERVER"; case OBJECT_FUNCTION: @@ -2165,10 +2157,6 @@ stringify_adefprivs_objtype(ObjectType objtype) return "FOREIGN DATA WRAPPERS"; case OBJECT_FOREIGN_SERVER: return "FOREIGN SERVERS"; - case OBJECT_FOREIGN_CATALOG: - return "FOREIGN CATALOGS"; - case OBJECT_FOREIGN_VOLUME: - return "FOREIGN VOLUMES"; case OBJECT_STORAGE_SERVER: return "STORAGE SERVERS"; case OBJECT_FUNCTION: diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index 892871ef0cf..e260af42188 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -23,12 +23,10 @@ #include "catalog/indexing.h" #include "catalog/objectaccess.h" #include "catalog/oid_dispatch.h" -#include "catalog/pg_foreign_catalog.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" #include "catalog/pg_foreign_table_seg.h" -#include "catalog/pg_foreign_volume.h" #include "catalog/pg_proc.h" #include "catalog/pg_type.h" #include "catalog/pg_user_mapping.h" @@ -1028,275 +1026,6 @@ CreateForeignServer(CreateForeignServerStmt *stmt) } -/* - * Create a foreign catalog - */ -ObjectAddress -CreateForeignCatalog(CreateForeignCatalogStmt *stmt) -{ - Relation rel; - Datum catalogoptions; - Datum values[Natts_pg_foreign_catalog]; - bool nulls[Natts_pg_foreign_catalog]; - HeapTuple tuple; - Oid catalogId; - Oid ownerId; - AclResult aclresult; - ObjectAddress myself; - ObjectAddress referenced; - ForeignServer *server; - - rel = table_open(ForeignCatalogRelationId, RowExclusiveLock); - - /* For now the owner cannot be specified on create. Use effective user ID. */ - ownerId = GetUserId(); - - /* - * Check that there is no other foreign catalog by this name. Catalog - * names are global (like server names): every reference syntax (DROP - * CATALOG, the CATALOG clause of CREATE LAKE TABLE, GUCs) identifies - * a catalog by bare name, so the name alone must be unique. If there is - * one, do nothing if IF NOT EXISTS was specified. - */ - catalogId = get_foreign_catalog_oid(stmt->catalogname, true); - if (OidIsValid(catalogId)) - { - if (!stmt->if_not_exists) - ereport(ERROR, - (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("foreign catalog \"%s\" already exists", - stmt->catalogname))); - - /* - * If we are in an extension script, insist that the pre-existing - * object be a member of the extension, to avoid security risks. - */ - ObjectAddressSet(myself, ForeignCatalogRelationId, catalogId); - checkMembershipInCurrentExtension(&myself); - - /* OK to skip */ - ereport(NOTICE, - (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("foreign catalog \"%s\" already exists, skipping", - stmt->catalogname))); - table_close(rel, RowExclusiveLock); - return InvalidObjectAddress; - } - - /* - * Check that the server exists and that we have USAGE on it. - */ - server = GetForeignServerByName(stmt->servername, false); - - aclresult = object_aclcheck(ForeignServerRelationId, server->serverid, ownerId, ACL_USAGE); - if (aclresult != ACLCHECK_OK) - aclcheck_error(aclresult, OBJECT_FOREIGN_SERVER, server->servername); - - /* - * Insert tuple into pg_foreign_catalog. - */ - memset(values, 0, sizeof(values)); - memset(nulls, false, sizeof(nulls)); - - catalogId = GetNewOidForForeignCatalog(rel, ForeignCatalogOidIndexId, - Anum_pg_foreign_catalog_oid, - stmt->catalogname); - values[Anum_pg_foreign_catalog_oid - 1] = ObjectIdGetDatum(catalogId); - values[Anum_pg_foreign_catalog_fcname - 1] = - DirectFunctionCall1(namein, CStringGetDatum(stmt->catalogname)); - values[Anum_pg_foreign_catalog_fcowner - 1] = ObjectIdGetDatum(ownerId); - values[Anum_pg_foreign_catalog_fcserver - 1] = ObjectIdGetDatum(server->serverid); - - /* - * The catalog type is a required property (every catalog has a type such - * as 'hive', 'hdfs', ...). The grammar enforces the TYPE clause, so this - * is just a defensive check; the value is stored verbatim and validated by - * the datalake provider rather than the kernel. - */ - if (stmt->catalogtype == NULL || stmt->catalogtype[0] == '\0') - ereport(ERROR, - (errcode(ERRCODE_INVALID_OBJECT_DEFINITION), - errmsg("foreign catalog type cannot be empty"))); - values[Anum_pg_foreign_catalog_fctype - 1] = - CStringGetTextDatum(stmt->catalogtype); - - /* Add catalog options; there is no validator for them */ - catalogoptions = transformGenericOptions(ForeignCatalogRelationId, - PointerGetDatum(NULL), - stmt->options, - InvalidOid); - - if (PointerIsValid(DatumGetPointer(catalogoptions))) - values[Anum_pg_foreign_catalog_fcoptions - 1] = catalogoptions; - else - nulls[Anum_pg_foreign_catalog_fcoptions - 1] = true; - - tuple = heap_form_tuple(rel->rd_att, values, nulls); - - CatalogTupleInsert(rel, tuple); - - heap_freetuple(tuple); - - /* record dependencies */ - myself.classId = ForeignCatalogRelationId; - myself.objectId = catalogId; - myself.objectSubId = 0; - - referenced.classId = ForeignServerRelationId; - referenced.objectId = server->serverid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); - - recordDependencyOnOwner(ForeignCatalogRelationId, catalogId, ownerId); - - /* dependency on extension */ - recordDependencyOnCurrentExtension(&myself, false); - - /* Post creation hook for new foreign catalog */ - InvokeObjectPostCreateHook(ForeignCatalogRelationId, catalogId, 0); - - if (Gp_role == GP_ROLE_DISPATCH) - { - CdbDispatchUtilityStatement((Node *) stmt, - DF_WITH_SNAPSHOT | DF_CANCEL_ON_ERROR | DF_NEED_TWO_PHASE, - GetAssignedOidsForDispatch(), - NULL); - } - - table_close(rel, RowExclusiveLock); - - return myself; -} - - -/* - * Create a foreign volume - */ -ObjectAddress -CreateForeignVolume(CreateForeignVolumeStmt *stmt) -{ - Relation rel; - Datum volumeoptions; - Datum values[Natts_pg_foreign_volume]; - bool nulls[Natts_pg_foreign_volume]; - HeapTuple tuple; - Oid volumeId; - Oid ownerId; - AclResult aclresult; - ObjectAddress myself; - ObjectAddress referenced; - ForeignServer *server; - - rel = table_open(ForeignVolumeRelationId, RowExclusiveLock); - - /* For now the owner cannot be specified on create. Use effective user ID. */ - ownerId = GetUserId(); - - /* - * Check that there is no other foreign volume by this name. Volume - * names are global (like server names): every reference syntax (DROP - * VOLUME, the VOLUME clause of CREATE LAKE TABLE, GUCs) identifies - * a volume by bare name, so the name alone must be unique. If there is - * one, do nothing if IF NOT EXISTS was specified. - */ - volumeId = get_foreign_volume_oid(stmt->volumename, true); - if (OidIsValid(volumeId)) - { - if (!stmt->if_not_exists) - ereport(ERROR, - (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("foreign volume \"%s\" already exists", - stmt->volumename))); - - /* - * If we are in an extension script, insist that the pre-existing - * object be a member of the extension, to avoid security risks. - */ - ObjectAddressSet(myself, ForeignVolumeRelationId, volumeId); - checkMembershipInCurrentExtension(&myself); - - /* OK to skip */ - ereport(NOTICE, - (errcode(ERRCODE_DUPLICATE_OBJECT), - errmsg("foreign volume \"%s\" already exists, skipping", - stmt->volumename))); - table_close(rel, RowExclusiveLock); - return InvalidObjectAddress; - } - - /* - * Check that the server exists and that we have USAGE on it. - */ - server = GetForeignServerByName(stmt->servername, false); - - aclresult = object_aclcheck(ForeignServerRelationId, server->serverid, ownerId, ACL_USAGE); - if (aclresult != ACLCHECK_OK) - aclcheck_error(aclresult, OBJECT_FOREIGN_SERVER, server->servername); - - /* - * Insert tuple into pg_foreign_volume. - */ - memset(values, 0, sizeof(values)); - memset(nulls, false, sizeof(nulls)); - - volumeId = GetNewOidForForeignVolume(rel, ForeignVolumeOidIndexId, - Anum_pg_foreign_volume_oid, - stmt->volumename); - values[Anum_pg_foreign_volume_oid - 1] = ObjectIdGetDatum(volumeId); - values[Anum_pg_foreign_volume_fvname - 1] = - DirectFunctionCall1(namein, CStringGetDatum(stmt->volumename)); - values[Anum_pg_foreign_volume_fvowner - 1] = ObjectIdGetDatum(ownerId); - values[Anum_pg_foreign_volume_fvserver - 1] = ObjectIdGetDatum(server->serverid); - - /* Add volume options; there is no validator for them */ - volumeoptions = transformGenericOptions(ForeignVolumeRelationId, - PointerGetDatum(NULL), - stmt->options, - InvalidOid); - - if (PointerIsValid(DatumGetPointer(volumeoptions))) - values[Anum_pg_foreign_volume_fvoptions - 1] = volumeoptions; - else - nulls[Anum_pg_foreign_volume_fvoptions - 1] = true; - - tuple = heap_form_tuple(rel->rd_att, values, nulls); - - CatalogTupleInsert(rel, tuple); - - heap_freetuple(tuple); - - /* record dependencies */ - myself.classId = ForeignVolumeRelationId; - myself.objectId = volumeId; - myself.objectSubId = 0; - - referenced.classId = ForeignServerRelationId; - referenced.objectId = server->serverid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); - - recordDependencyOnOwner(ForeignVolumeRelationId, volumeId, ownerId); - - /* dependency on extension */ - recordDependencyOnCurrentExtension(&myself, false); - - /* Post creation hook for new foreign volume */ - InvokeObjectPostCreateHook(ForeignVolumeRelationId, volumeId, 0); - - if (Gp_role == GP_ROLE_DISPATCH) - { - CdbDispatchUtilityStatement((Node *) stmt, - DF_WITH_SNAPSHOT | DF_CANCEL_ON_ERROR | DF_NEED_TWO_PHASE, - GetAssignedOidsForDispatch(), - NULL); - } - - table_close(rel, RowExclusiveLock); - - return myself; -} - - /* * Alter foreign server */ diff --git a/src/backend/commands/laketablecmds.c b/src/backend/commands/laketablecmds.c deleted file mode 100644 index afc92ab36f3..00000000000 --- a/src/backend/commands/laketablecmds.c +++ /dev/null @@ -1,458 +0,0 @@ -/*------------------------------------------------------------------------- - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * laketablecmds.c - * lake table creation/manipulation commands - * - * IDENTIFICATION - * src/backend/commands/laketablecmds.c - * - *------------------------------------------------------------------------- - */ -#include "postgres.h" - -#include "access/genam.h" -#include "access/htup_details.h" -#include "access/reloptions.h" -#include "access/table.h" -#include "access/xact.h" -#include "catalog/catalog.h" -#include "catalog/dependency.h" -#include "catalog/indexing.h" -#include "catalog/objectaccess.h" -#include "catalog/pg_foreign_catalog.h" -#include "catalog/pg_foreign_volume.h" -#include "catalog/pg_lake_table.h" -#include "commands/defrem.h" -#include "commands/laketablecmds.h" -#include "foreign/foreign.h" -#include "miscadmin.h" -#include "utils/builtins.h" -#include "utils/fmgroids.h" -#include "utils/rel.h" - -/* GUC variables for default Iceberg catalog and volume */ -char *iceberg_default_catalog = NULL; -char *iceberg_default_volume = NULL; - -/* - * check_iceberg_default_catalog: validate new iceberg_default_catalog GUC value - */ -bool -check_iceberg_default_catalog(char **newval, void **extra, GucSource source) -{ - /* - * If we aren't inside a transaction, or connected to a database, we - * cannot do the catalog accesses necessary to verify the name. Must - * accept the value on faith. - */ - if (IsTransactionState() && MyDatabaseId != InvalidOid) - { - if (**newval != '\0') - { - Oid catalog_oid = get_foreign_catalog_oid(*newval, true); - - if (!OidIsValid(catalog_oid)) - { - /* - * When source == PGC_S_TEST, don't throw a hard error for a - * nonexistent catalog, only a NOTICE. See comments in guc.h. - */ - if (source == PGC_S_TEST) - { - ereport(NOTICE, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("foreign catalog \"%s\" does not exist", - *newval))); - } - else - { - GUC_check_errdetail("Foreign catalog \"%s\" does not exist.", - *newval); - return false; - } - } - } - } - - return true; -} - -/* - * check_iceberg_default_volume: validate new iceberg_default_volume GUC value - */ -bool -check_iceberg_default_volume(char **newval, void **extra, GucSource source) -{ - /* - * If we aren't inside a transaction, or connected to a database, we - * cannot do the catalog accesses necessary to verify the name. Must - * accept the value on faith. - */ - if (IsTransactionState() && MyDatabaseId != InvalidOid) - { - if (**newval != '\0') - { - Oid volume_oid = get_foreign_volume_oid(*newval, true); - - if (!OidIsValid(volume_oid)) - { - /* - * When source == PGC_S_TEST, don't throw a hard error for a - * nonexistent volume, only a NOTICE. See comments in guc.h. - */ - if (source == PGC_S_TEST) - { - ereport(NOTICE, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("foreign volume \"%s\" does not exist", - *newval))); - } - else - { - GUC_check_errdetail("Foreign volume \"%s\" does not exist.", - *newval); - return false; - } - } - } - } - - return true; -} - -/* - * GetDefaultIcebergCatalog -- get the name of the current default Iceberg catalog - * - * Returns NULL if no default catalog is set. - * This function hides the iceberg_default_catalog GUC variable. - */ -const char * -GetDefaultIcebergCatalog(void) -{ - if (iceberg_default_catalog == NULL || iceberg_default_catalog[0] == '\0') - return NULL; - - /* - * Verify that the catalog still exists. We don't cache this because - * the catalog could be dropped after the GUC was set. - */ - if (!OidIsValid(get_foreign_catalog_oid(iceberg_default_catalog, true))) - return NULL; - - return iceberg_default_catalog; -} - -/* - * GetDefaultIcebergVolume -- get the name of the current default Iceberg volume - * - * Returns NULL if no default volume is set. - * This function hides the iceberg_default_volume GUC variable. - */ -const char * -GetDefaultIcebergVolume(void) -{ - if (iceberg_default_volume == NULL || iceberg_default_volume[0] == '\0') - return NULL; - - /* - * Verify that the volume still exists. We don't cache this because - * the volume could be dropped after the GUC was set. - */ - if (!OidIsValid(get_foreign_volume_oid(iceberg_default_volume, true))) - return NULL; - - return iceberg_default_volume; -} - -/* - * GetIcebergTableAmOid - * - * Look up the OID of the iceberg table access method, which is provided by - * a datalake extension rather than the kernel. Returns InvalidOid if the - * access method is not installed and missing_ok is true. - */ -Oid -GetIcebergTableAmOid(bool missing_ok) -{ - return get_table_am_oid(ICEBERG_TABLE_AM_NAME, missing_ok); -} - -/* - * RelationIsLakeTable - * - * True iff the relation has a pg_lake_table entry, i.e. it was created by - * CREATE LAKE TABLE. Lake tables are told apart from ordinary relations by - * this catalog membership rather than by their access method. - */ -bool -RelationIsLakeTable(Relation rel) -{ - Relation ltRel; - ScanKeyData skey; - SysScanDesc scan; - bool found; - - ltRel = table_open(LakeTableRelationId, AccessShareLock); - ScanKeyInit(&skey, - Anum_pg_lake_table_ltrelid, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(RelationGetRelid(rel))); - scan = systable_beginscan(ltRel, LakeTableRelidIndexId, true, NULL, 1, &skey); - found = HeapTupleIsValid(systable_getnext(scan)); - systable_endscan(scan); - table_close(ltRel, AccessShareLock); - - return found; -} - -/* - * Validate table type - */ -static void -validate_table_type(const char *table_type) -{ - if (!table_type) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("lake table format cannot be NULL"))); - - if (strcmp(table_type, ICEBERG_TABLE_AM_NAME) != 0) - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("unsupported lake table format \"%s\"", table_type), - errhint("The only supported format is ICEBERG (USING ICEBERG)."))); -} - -/* - * Validate foreign catalog exists - */ -static Oid -validate_foreign_catalog(const char *catalog_name) -{ - if (!catalog_name || catalog_name[0] == '\0') - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("no foreign catalog specified"), - errhint("Specify CATALOG in CREATE LAKE TABLE or set iceberg_default_catalog."))); - - return get_foreign_catalog_oid(catalog_name, false); -} - -/* - * Validate foreign volume exists - */ -static Oid -validate_foreign_volume(const char *volume_name) -{ - if (!volume_name || volume_name[0] == '\0') - ereport(ERROR, - (errcode(ERRCODE_INVALID_PARAMETER_VALUE), - errmsg("no foreign volume specified"), - errhint("Specify VOLUME in CREATE LAKE TABLE or set iceberg_default_volume."))); - - return get_foreign_volume_oid(volume_name, false); -} - -/* - * ResolveLakeTableOptions - * - * Resolve and validate the table type, catalog and volume of a - * CreateLakeTableStmt, returning the catalog/volume OIDs. - * - * Also exposed (via ValidateLakeTableStmt) so ProcessUtilitySlow can run - * the validation on the QD before DefineRelation: DefineRelation dispatches - * the statement to the QEs, so a validation failure raised only inside - * CreateLakeTable() would surface as a confusing QE-annotated error. - */ -static void -ResolveLakeTableOptions(CreateLakeTableStmt *stmt, - Oid *catalog_oid_out, Oid *volume_oid_out) -{ - const char *catalog_name; - const char *volume_name; - - /* Validate the table format named in the USING clause first, so an - * unsupported format is reported before anything else. */ - validate_table_type(stmt->table_type); - - /* - * The format is implemented by a like-named table access method that a - * datalake extension provides; a lake table is unusable without it, so - * check it here (after the format) so the install hint takes precedence - * over catalog/volume resolution errors. - */ - if (!OidIsValid(get_table_am_oid(stmt->table_type, true))) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("table access method \"%s\" does not exist", - stmt->table_type), - errhint("CREATE LAKE TABLE ... USING \"%s\" requires an extension that provides the \"%s\" table access method.", - stmt->table_type, stmt->table_type))); - - /* - * Determine catalog name: use explicit value if provided, otherwise - * fall back to the iceberg_default_catalog GUC. When the GUC is set - * but its catalog has been dropped, say so instead of the generic - * "no foreign catalog specified". - */ - catalog_name = stmt->foreign_catalog; - if (catalog_name == NULL || catalog_name[0] == '\0') - { - catalog_name = GetDefaultIcebergCatalog(); - if (catalog_name == NULL && - iceberg_default_catalog != NULL && iceberg_default_catalog[0] != '\0') - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("default iceberg catalog \"%s\" does not exist", - iceberg_default_catalog), - errhint("Set iceberg_default_catalog to an existing foreign catalog."))); - } - - /* - * Determine volume name: use explicit value if provided, otherwise - * fall back to the iceberg_default_volume GUC. - */ - volume_name = stmt->foreign_volume; - if (volume_name == NULL || volume_name[0] == '\0') - { - volume_name = GetDefaultIcebergVolume(); - if (volume_name == NULL && - iceberg_default_volume != NULL && iceberg_default_volume[0] != '\0') - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("default iceberg volume \"%s\" does not exist", - iceberg_default_volume), - errhint("Set iceberg_default_volume to an existing foreign volume."))); - } - - *catalog_oid_out = validate_foreign_catalog(catalog_name); - - /* - * A volume is required for every lake table, even when the catalog - * vends the table's physical location: the QEs still read and write - * the data files through the volume's storage endpoint and - * credentials. Without this check the missing volume only surfaces - * later, deep in the access method's create path. - */ - *volume_oid_out = validate_foreign_volume(volume_name); -} - -/* - * ValidateLakeTableStmt - * - * QD-side pre-DefineRelation validation wrapper; see ResolveLakeTableOptions. - */ -void -ValidateLakeTableStmt(CreateLakeTableStmt *stmt) -{ - Oid catalog_oid; - Oid volume_oid; - - ResolveLakeTableOptions(stmt, &catalog_oid, &volume_oid); -} - -/* - * CreateLakeTable - * - * Create a lake table entry in pg_lake_table after the base table has been - * created. - */ -void -CreateLakeTable(CreateLakeTableStmt *stmt, Oid relId) -{ - Relation lake_rel; - Datum values[Natts_pg_lake_table]; - bool nulls[Natts_pg_lake_table]; - HeapTuple tuple; - Oid catalog_oid; - Oid volume_oid; - ObjectAddress myself; - ObjectAddress referenced; - - ResolveLakeTableOptions(stmt, &catalog_oid, &volume_oid); - - /* - * Make the just-created base relation (from DefineRelation) visible to - * this command before we record dependencies on it. - */ - CommandCounterIncrement(); - - lake_rel = table_open(LakeTableRelationId, RowExclusiveLock); - - /* - * Insert tuple into pg_lake_table. - */ - memset(values, 0, sizeof(values)); - memset(nulls, false, sizeof(nulls)); - - values[Anum_pg_lake_table_ltrelid - 1] = ObjectIdGetDatum(relId); - values[Anum_pg_lake_table_ltforeign_catalog - 1] = ObjectIdGetDatum(catalog_oid); - values[Anum_pg_lake_table_ltforeign_volume - 1] = ObjectIdGetDatum(volume_oid); - - tuple = heap_form_tuple(lake_rel->rd_att, values, nulls); - - CatalogTupleInsert(lake_rel, tuple); - - /* Record dependencies on the foreign catalog and volume */ - myself.classId = RelationRelationId; - myself.objectId = relId; - myself.objectSubId = 0; - - referenced.classId = ForeignCatalogRelationId; - referenced.objectId = catalog_oid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); - - referenced.classId = ForeignVolumeRelationId; - referenced.objectId = volume_oid; - referenced.objectSubId = 0; - recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); - - heap_freetuple(tuple); - table_close(lake_rel, RowExclusiveLock); - - CommandCounterIncrement(); - InvokeObjectPostCreateHook(LakeTableRelationId, relId, 0); -} - -/* - * RemoveLakeTableEntry - * - * Remove the pg_lake_table entry for the given relation. - */ -void -RemoveLakeTableEntry(Oid relid) -{ - Relation ltRel; - HeapTuple tup; - ScanKeyData skey; - SysScanDesc scan; - - ltRel = table_open(LakeTableRelationId, RowExclusiveLock); - ScanKeyInit(&skey, - Anum_pg_lake_table_ltrelid, - BTEqualStrategyNumber, F_OIDEQ, - ObjectIdGetDatum(relid)); - scan = systable_beginscan(ltRel, LakeTableRelidIndexId, true, NULL, 1, &skey); - while (HeapTupleIsValid(tup = systable_getnext(scan))) - CatalogTupleDelete(ltRel, &tup->t_self); - systable_endscan(scan); - table_close(ltRel, RowExclusiveLock); -} diff --git a/src/backend/commands/seclabel.c b/src/backend/commands/seclabel.c index cd64f90f1d7..d4018ac0348 100644 --- a/src/backend/commands/seclabel.c +++ b/src/backend/commands/seclabel.c @@ -76,8 +76,6 @@ SecLabelSupportsObjectType(ObjectType objtype) case OBJECT_EXTENSION: case OBJECT_FDW: case OBJECT_FOREIGN_SERVER: - case OBJECT_FOREIGN_CATALOG: - case OBJECT_FOREIGN_VOLUME: case OBJECT_INDEX: case OBJECT_OPCLASS: case OBJECT_OPERATOR: diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index c59ff84a4ce..b8b69336b89 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -69,7 +69,6 @@ #include "commands/comment.h" #include "commands/createas.h" #include "commands/defrem.h" -#include "commands/laketablecmds.h" #include "commands/matview.h" #include "commands/event_trigger.h" #include "commands/policy.h" @@ -256,7 +255,6 @@ struct DropRelationCallbackState { /* These fields are set by RemoveRelations: */ char expected_relkind; - bool iceberg_only; /* DROP LAKE TABLE: require iceberg AM */ LOCKMODE heap_lockmode; /* These fields are state to track which subsidiary locks are held: */ Oid heapOid; @@ -940,24 +938,6 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, amHandlerOid = get_table_am_handler_oid(accessMethod, false); } - /* - * Lake tables must be created through CREATE LAKE TABLE, which also - * creates the pg_lake_table catalog entries the iceberg access method - * relies on. A relation created with the iceberg AM through any other - * path (CREATE TABLE ... USING iceberg, CTAS, matview, - * default_table_access_method, partition child) would be unusable and - * undroppable, so reject it up front. CreateLakeTableStmt embeds - * CreateStmt as its first member, so nodeTag() distinguishes the paths. - */ - if (OidIsValid(accessMethodId) && - accessMethodId == GetIcebergTableAmOid(true) && - nodeTag(stmt) != T_CreateLakeTableStmt) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot create table \"%s\" with access method \"%s\"", - stmt->relation->relname, ICEBERG_TABLE_AM_NAME), - errhint("Use CREATE LAKE TABLE ... USING ICEBERG instead."))); - /* * GPDB: for partitioned tables, inherit reloptions from the parent. * Note this is applicable only if the parent has the same AM as the child. @@ -1999,7 +1979,6 @@ RemoveRelations(DropStmt *drop) /* Look up the appropriate relation using namespace search. */ state.expected_relkind = relkind; - state.iceberg_only = drop->isiceberg; state.heap_lockmode = drop->concurrent ? ShareUpdateExclusiveLock : AccessExclusiveLock; /* We must initialize these fields to show that no locks are held: */ @@ -2221,36 +2200,6 @@ RangeVarCallbackForDropRelation(const RangeVar *rel, Oid relOid, Oid oldRelOid, DropErrorMsgWrongType(rel->relname, classform->relkind, state->expected_relkind); - /* - * Iceberg (lake) tables share RELKIND_RELATION with ordinary tables and are - * told apart only by their access method. DROP LAKE TABLE must target one; - * plain DROP TABLE must NOT (mirrors the foreign-table rule) -- direct the user - * to the matching command in each case. - */ - if (state->expected_relkind == RELKIND_RELATION) - { - Oid iceberg_amoid = GetIcebergTableAmOid(true); - bool is_iceberg = classform->relkind == RELKIND_RELATION && - OidIsValid(iceberg_amoid) && - classform->relam == iceberg_amoid; - - if (state->iceberg_only) - { - if (!is_iceberg) - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("\"%s\" is not a lake table", rel->relname), - errhint("Use DROP TABLE to remove a table."))); - } - else if (is_iceberg) - { - ereport(ERROR, - (errcode(ERRCODE_WRONG_OBJECT_TYPE), - errmsg("\"%s\" is not a table", rel->relname), - errhint("Use DROP LAKE TABLE to remove a lake table."))); - } - } - /* Allow DROP to either table owner or schema owner */ if (!object_ownercheck(RelationRelationId, relOid, GetUserId()) && !object_ownercheck(NamespaceRelationId, classform->relnamespace, GetUserId())) @@ -17110,7 +17059,6 @@ static void ATPrepSetAccessMethod(AlteredTableInfo *tab, Relation rel, const char *amname) { Oid amoid; - Oid iceberg_amoid; /* Check that the table access method exists */ amoid = get_table_am_oid(amname, false); @@ -17118,27 +17066,6 @@ ATPrepSetAccessMethod(AlteredTableInfo *tab, Relation rel, const char *amname) if (rel->rd_rel->relam == amoid) return; - /* - * The iceberg AM relies on catalog entries that only the CREATE/DROP - * LAKE TABLE paths manage, so a table cannot be converted to or from - * it with SET ACCESS METHOD. - */ - iceberg_amoid = GetIcebergTableAmOid(true); - if (OidIsValid(iceberg_amoid)) - { - if (amoid == iceberg_amoid) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot change access method of table \"%s\" to \"" ICEBERG_TABLE_AM_NAME "\"", - RelationGetRelationName(rel)), - errhint("Use CREATE LAKE TABLE ... USING ICEBERG to create a lake table."))); - if (rel->rd_rel->relam == iceberg_amoid) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot change access method of lake table \"%s\"", - RelationGetRelationName(rel)))); - } - /* Save info for Phase 3 to do the real work */ tab->rewrite |= AT_REWRITE_ACCESS_METHOD; tab->newAccessMethod = amoid; @@ -19832,14 +19759,6 @@ ATExecSetDistributedBy(Relation rel, Node *node, AlterTableCmd *cmd) (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("SET DISTRIBUTED REPLICATED is not supported for external table"))); } - - /* Lake tables must remain DISTRIBUTED RANDOMLY */ - if (RelationIsLakeTable(rel)) - ereport(ERROR, - (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), - errmsg("cannot change distribution policy of lake table \"%s\"", - RelationGetRelationName(rel)), - errhint("Lake tables must use DISTRIBUTED RANDOMLY because data is stored on object storage."))); } if (Gp_role == GP_ROLE_DISPATCH) diff --git a/src/backend/foreign/foreign.c b/src/backend/foreign/foreign.c index aff82399f59..0fadf562ee4 100644 --- a/src/backend/foreign/foreign.c +++ b/src/backend/foreign/foreign.c @@ -15,12 +15,10 @@ #include "access/htup_details.h" #include "access/reloptions.h" #include "access/table.h" -#include "catalog/pg_foreign_catalog.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" #include "catalog/pg_foreign_table_seg.h" -#include "catalog/pg_foreign_volume.h" #include "catalog/pg_user_mapping.h" #include "cdb/cdbgang.h" #include "cdb/cdbutil.h" @@ -984,95 +982,6 @@ get_foreign_server_oid(const char *servername, bool missing_ok) return oid; } -/* - * get_foreign_catalog_oid - given a foreign catalog name, look up the OID - * - * If missing_ok is false, throw an error if name not found. If true, just - * return InvalidOid. - */ -Oid -get_foreign_catalog_oid(const char *catalogname, bool missing_ok) -{ - Oid oid; - - oid = GetSysCacheOid1(FOREIGNCATALOGNAME, - Anum_pg_foreign_catalog_oid, - CStringGetDatum(catalogname)); - if (!OidIsValid(oid) && !missing_ok) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("foreign catalog \"%s\" does not exist", - catalogname))); - - return oid; -} - -/* - * get_foreign_volume_oid - given a foreign volume name, look up the OID - * - * If missing_ok is false, throw an error if name not found. If true, just - * return InvalidOid. - */ -Oid -get_foreign_volume_oid(const char *volumename, bool missing_ok) -{ - Oid oid; - - oid = GetSysCacheOid1(FOREIGNVOLUMENAME, - Anum_pg_foreign_volume_oid, - CStringGetDatum(volumename)); - if (!OidIsValid(oid) && !missing_ok) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("foreign volume \"%s\" does not exist", - volumename))); - - return oid; -} - -/* - * GetForeignVolumeByName - look up a foreign volume by name - */ -ForeignVolume * -GetForeignVolumeByName(const char *volumename, bool missing_ok) -{ - HeapTuple tp; - Form_pg_foreign_volume fvform; - ForeignVolume *volume; - Datum datum; - bool isnull; - - tp = SearchSysCache1(FOREIGNVOLUMENAME, - PointerGetDatum(volumename)); - if (!HeapTupleIsValid(tp)) - { - if (!missing_ok) - ereport(ERROR, - (errcode(ERRCODE_UNDEFINED_OBJECT), - errmsg("foreign volume \"%s\" does not exist", - volumename))); - return NULL; - } - - fvform = (Form_pg_foreign_volume) GETSTRUCT(tp); - - volume = (ForeignVolume *) palloc(sizeof(ForeignVolume)); - volume->volumeid = fvform->oid; - volume->serverid = fvform->fvserver; - volume->volumename = pstrdup(NameStr(fvform->fvname)); - - /* Extract the volume options */ - datum = SysCacheGetAttr(FOREIGNVOLUMENAME, - tp, - Anum_pg_foreign_volume_fvoptions, - &isnull); - volume->options = (isnull) ? NIL : untransformRelOptions(datum); - - ReleaseSysCache(tp); - - return volume; -} - /* * Get a copy of an existing local path for a given join relation. * diff --git a/src/backend/nodes/copyfuncs.funcs.c b/src/backend/nodes/copyfuncs.funcs.c index dcb93a84b3d..6f312d17574 100644 --- a/src/backend/nodes/copyfuncs.funcs.c +++ b/src/backend/nodes/copyfuncs.funcs.c @@ -2748,33 +2748,6 @@ _copyCreateForeignServerStmt(const CreateForeignServerStmt *from) return newnode; } -static CreateForeignCatalogStmt * -_copyCreateForeignCatalogStmt(const CreateForeignCatalogStmt *from) -{ - CreateForeignCatalogStmt *newnode = makeNode(CreateForeignCatalogStmt); - - COPY_STRING_FIELD(catalogname); - COPY_STRING_FIELD(servername); - COPY_STRING_FIELD(catalogtype); - COPY_SCALAR_FIELD(if_not_exists); - COPY_NODE_FIELD(options); - - return newnode; -} - -static CreateForeignVolumeStmt * -_copyCreateForeignVolumeStmt(const CreateForeignVolumeStmt *from) -{ - CreateForeignVolumeStmt *newnode = makeNode(CreateForeignVolumeStmt); - - COPY_STRING_FIELD(volumename); - COPY_STRING_FIELD(servername); - COPY_SCALAR_FIELD(if_not_exists); - COPY_NODE_FIELD(options); - - return newnode; -} - static AlterForeignServerStmt * _copyAlterForeignServerStmt(const AlterForeignServerStmt *from) { @@ -3385,44 +3358,6 @@ _copyCreateDirectoryTableStmt(const CreateDirectoryTableStmt *from) return newnode; } -static CreateLakeTableStmt * -_copyCreateLakeTableStmt(const CreateLakeTableStmt *from) -{ - CreateLakeTableStmt *newnode = makeNode(CreateLakeTableStmt); - - COPY_NODE_FIELD(base.relation); - COPY_NODE_FIELD(base.tableElts); - COPY_NODE_FIELD(base.inhRelations); - COPY_NODE_FIELD(base.partbound); - COPY_NODE_FIELD(base.partspec); - COPY_NODE_FIELD(base.ofTypename); - COPY_NODE_FIELD(base.constraints); - COPY_NODE_FIELD(base.options); - COPY_SCALAR_FIELD(base.oncommit); - COPY_STRING_FIELD(base.tablespacename); - COPY_STRING_FIELD(base.accessMethod); - COPY_SCALAR_FIELD(base.if_not_exists); - COPY_SCALAR_FIELD(base.gp_style_alter_part); - COPY_NODE_FIELD(base.distributedBy); - COPY_NODE_FIELD(base.partitionBy); - COPY_SCALAR_FIELD(base.relKind); - COPY_SCALAR_FIELD(base.ownerid); - COPY_SCALAR_FIELD(base.buildAoBlkdir); - COPY_NODE_FIELD(base.attr_encodings); - COPY_SCALAR_FIELD(base.isCtas); - COPY_NODE_FIELD(base.intoQuery); - COPY_NODE_FIELD(base.intoPolicy); - COPY_NODE_FIELD(base.part_idx_oids); - COPY_NODE_FIELD(base.part_idx_names); - COPY_NODE_FIELD(base.tags); - COPY_SCALAR_FIELD(base.origin); - COPY_STRING_FIELD(table_type); - COPY_STRING_FIELD(foreign_catalog); - COPY_STRING_FIELD(foreign_volume); - - return newnode; -} - static AlterDirectoryTableStmt * _copyAlterDirectoryTableStmt(const AlterDirectoryTableStmt *from) { @@ -3446,7 +3381,6 @@ _copyDropStmt(const DropStmt *from) COPY_SCALAR_FIELD(missing_ok); COPY_SCALAR_FIELD(concurrent); COPY_SCALAR_FIELD(isdynamic); - COPY_SCALAR_FIELD(isiceberg); return newnode; } @@ -3462,7 +3396,6 @@ _copyDropDirectoryTableStmt(const DropDirectoryTableStmt *from) COPY_SCALAR_FIELD(base.missing_ok); COPY_SCALAR_FIELD(base.concurrent); COPY_SCALAR_FIELD(base.isdynamic); - COPY_SCALAR_FIELD(base.isiceberg); COPY_SCALAR_FIELD(with_content); return newnode; diff --git a/src/backend/nodes/copyfuncs.switch.c b/src/backend/nodes/copyfuncs.switch.c index 585e90be3c1..69dcef19150 100644 --- a/src/backend/nodes/copyfuncs.switch.c +++ b/src/backend/nodes/copyfuncs.switch.c @@ -567,12 +567,6 @@ case T_CreateForeignServerStmt: retval = _copyCreateForeignServerStmt(from); break; - case T_CreateForeignCatalogStmt: - retval = _copyCreateForeignCatalogStmt(from); - break; - case T_CreateForeignVolumeStmt: - retval = _copyCreateForeignVolumeStmt(from); - break; case T_AlterForeignServerStmt: retval = _copyAlterForeignServerStmt(from); break; @@ -705,9 +699,6 @@ case T_CreateDirectoryTableStmt: retval = _copyCreateDirectoryTableStmt(from); break; - case T_CreateLakeTableStmt: - retval = _copyCreateLakeTableStmt(from); - break; case T_AlterDirectoryTableStmt: retval = _copyAlterDirectoryTableStmt(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index b56e94873a2..eb77ea9169f 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -1493,7 +1493,6 @@ _equalDropStmt(const DropStmt *a, const DropStmt *b) COMPARE_SCALAR_FIELD(missing_ok); COMPARE_SCALAR_FIELD(concurrent); COMPARE_SCALAR_FIELD(isdynamic); - COMPARE_SCALAR_FIELD(isiceberg); return true; } @@ -2164,29 +2163,6 @@ _equalCreateForeignServerStmt(const CreateForeignServerStmt *a, const CreateFore return true; } -static bool -_equalCreateForeignCatalogStmt(const CreateForeignCatalogStmt *a, const CreateForeignCatalogStmt *b) -{ - COMPARE_STRING_FIELD(catalogname); - COMPARE_STRING_FIELD(servername); - COMPARE_STRING_FIELD(catalogtype); - COMPARE_SCALAR_FIELD(if_not_exists); - COMPARE_NODE_FIELD(options); - - return true; -} - -static bool -_equalCreateForeignVolumeStmt(const CreateForeignVolumeStmt *a, const CreateForeignVolumeStmt *b) -{ - COMPARE_STRING_FIELD(volumename); - COMPARE_STRING_FIELD(servername); - COMPARE_SCALAR_FIELD(if_not_exists); - COMPARE_NODE_FIELD(options); - - return true; -} - static bool _equalAddForeignSegStmt(const AddForeignSegStmt *a, const AddForeignSegStmt *b) { @@ -3510,19 +3486,6 @@ _equalCreateDirectoryTableStmt(const CreateDirectoryTableStmt *a, const CreateDi return true; } -static bool -_equalCreateLakeTableStmt(const CreateLakeTableStmt *a, const CreateLakeTableStmt *b) -{ - if (!_equalCreateStmt(&a->base, &b->base)) - return false; - - COMPARE_STRING_FIELD(table_type); - COMPARE_STRING_FIELD(foreign_catalog); - COMPARE_STRING_FIELD(foreign_volume); - - return true; -} - static bool _equalAlterDirectoryTableStmt(const AlterDirectoryTableStmt *a, const AlterDirectoryTableStmt *b) { @@ -4284,12 +4247,6 @@ equal(const void *a, const void *b) case T_CreateForeignServerStmt: retval = _equalCreateForeignServerStmt(a, b); break; - case T_CreateForeignCatalogStmt: - retval = _equalCreateForeignCatalogStmt(a, b); - break; - case T_CreateForeignVolumeStmt: - retval = _equalCreateForeignVolumeStmt(a, b); - break; case T_AddForeignSegStmt: retval = _equalAddForeignSegStmt(a, b); break; @@ -4645,10 +4602,6 @@ equal(const void *a, const void *b) retval = _equalCreateDirectoryTableStmt(a, b); break; - case T_CreateLakeTableStmt: - retval = _equalCreateLakeTableStmt(a, b); - break; - case T_AlterDirectoryTableStmt: retval = _equalAlterDirectoryTableStmt(a, b); break; diff --git a/src/backend/nodes/outfast.c b/src/backend/nodes/outfast.c index 90d72d34559..f31bfa87045 100644 --- a/src/backend/nodes/outfast.c +++ b/src/backend/nodes/outfast.c @@ -673,29 +673,6 @@ _outCreateForeignServerStmt(StringInfo str, CreateForeignServerStmt *node) WRITE_NODE_FIELD(options); } -static void -_outCreateForeignCatalogStmt(StringInfo str, CreateForeignCatalogStmt *node) -{ - WRITE_NODE_TYPE("CREATEFOREIGNCATALOGSTMT"); - - WRITE_STRING_FIELD(catalogname); - WRITE_STRING_FIELD(servername); - WRITE_STRING_FIELD(catalogtype); - WRITE_BOOL_FIELD(if_not_exists); - WRITE_NODE_FIELD(options); -} - -static void -_outCreateForeignVolumeStmt(StringInfo str, CreateForeignVolumeStmt *node) -{ - WRITE_NODE_TYPE("CREATEFOREIGNVOLUMESTMT"); - - WRITE_STRING_FIELD(volumename); - WRITE_STRING_FIELD(servername); - WRITE_BOOL_FIELD(if_not_exists); - WRITE_NODE_FIELD(options); -} - static void _outAddForeignSegstmt(StringInfo str, AddForeignSegStmt *node) { @@ -1841,12 +1818,6 @@ _outNode(StringInfo str, void *obj) case T_CreateForeignServerStmt: _outCreateForeignServerStmt(str, obj); break; - case T_CreateForeignCatalogStmt: - _outCreateForeignCatalogStmt(str, obj); - break; - case T_CreateForeignVolumeStmt: - _outCreateForeignVolumeStmt(str, obj); - break; case T_AddForeignSegStmt: _outAddForeignSegstmt(str, obj); break; @@ -1969,9 +1940,6 @@ _outNode(StringInfo str, void *obj) case T_CreateDirectoryTableStmt: _outCreateDirectoryTableStmt(str, obj); break; - case T_CreateLakeTableStmt: - _outCreateLakeTableStmt(str, obj); - break; case T_AlterDirectoryTableStmt: _outAlterDirectoryTableStmt(str, obj); break; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index cea71a8aa01..c48ded5a813 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -4270,17 +4270,6 @@ _outCreateDirectoryTableStmt(StringInfo str, const CreateDirectoryTableStmt *nod WRITE_STRING_FIELD(location); } -static void -_outCreateLakeTableStmt(StringInfo str, const CreateLakeTableStmt *node) -{ - WRITE_NODE_TYPE("CREATELAKETABLESTMT"); - - _outCreateStmtInfo(str, (const CreateStmt *) node); - WRITE_STRING_FIELD(table_type); - WRITE_STRING_FIELD(foreign_catalog); - WRITE_STRING_FIELD(foreign_volume); -} - static void _outAlterDirectoryTableStmt(StringInfo str, const AlterDirectoryTableStmt *node) { @@ -5623,9 +5612,6 @@ outNode(StringInfo str, const void *obj) case T_CreateDirectoryTableStmt: _outCreateDirectoryTableStmt(str, obj); break; - case T_CreateLakeTableStmt: - _outCreateLakeTableStmt(str, obj); - break; case T_AlterDirectoryTableStmt: _outAlterDirectoryTableStmt(str, obj); break; diff --git a/src/backend/nodes/outfuncs_common.c b/src/backend/nodes/outfuncs_common.c index e7d6abd13fe..c518e38db0d 100644 --- a/src/backend/nodes/outfuncs_common.c +++ b/src/backend/nodes/outfuncs_common.c @@ -668,7 +668,6 @@ _outDropStmt(StringInfo str, const DropStmt *node) WRITE_BOOL_FIELD(missing_ok); WRITE_BOOL_FIELD(concurrent); WRITE_BOOL_FIELD(isdynamic); - WRITE_BOOL_FIELD(isiceberg); } static void @@ -1810,7 +1809,6 @@ _outDropStmtInfo(StringInfo str, const DropStmt *node) WRITE_BOOL_FIELD(missing_ok); WRITE_BOOL_FIELD(concurrent); WRITE_BOOL_FIELD(isdynamic); - WRITE_BOOL_FIELD(isiceberg); } static void diff --git a/src/backend/nodes/readfast.c b/src/backend/nodes/readfast.c index 031c9d062f4..ff3cb5eaddf 100644 --- a/src/backend/nodes/readfast.c +++ b/src/backend/nodes/readfast.c @@ -1598,33 +1598,6 @@ _readCreateForeignServerStmt(void) READ_DONE(); } -static CreateForeignCatalogStmt * -_readCreateForeignCatalogStmt(void) -{ - READ_LOCALS(CreateForeignCatalogStmt); - - READ_STRING_FIELD(catalogname); - READ_STRING_FIELD(servername); - READ_STRING_FIELD(catalogtype); - READ_BOOL_FIELD(if_not_exists); - READ_NODE_FIELD(options); - - READ_DONE(); -} - -static CreateForeignVolumeStmt * -_readCreateForeignVolumeStmt(void) -{ - READ_LOCALS(CreateForeignVolumeStmt); - - READ_STRING_FIELD(volumename); - READ_STRING_FIELD(servername); - READ_BOOL_FIELD(if_not_exists); - READ_NODE_FIELD(options); - - READ_DONE(); -} - static AddForeignSegStmt * _readAddForeignSegStmt(void) { @@ -1937,20 +1910,6 @@ _readCreateDirectoryTableStmt(void) READ_DONE(); } -static CreateLakeTableStmt * -_readCreateLakeTableStmt(void) -{ - READ_LOCALS(CreateLakeTableStmt); - - _readCreateStmt_common(&local_node->base); - - READ_STRING_FIELD(table_type); - READ_STRING_FIELD(foreign_catalog); - READ_STRING_FIELD(foreign_volume); - - READ_DONE(); -} - static AlterDirectoryTableStmt * _readAlterDirectoryTableStmt(void) { @@ -2922,12 +2881,6 @@ readNodeBinary(void) case T_CreateForeignServerStmt: return_value = _readCreateForeignServerStmt(); break; - case T_CreateForeignCatalogStmt: - return_value = _readCreateForeignCatalogStmt(); - break; - case T_CreateForeignVolumeStmt: - return_value = _readCreateForeignVolumeStmt(); - break; case T_AddForeignSegStmt: return_value = _readAddForeignSegStmt(); break; @@ -3038,9 +2991,6 @@ readNodeBinary(void) case T_CreateDirectoryTableStmt: return_value = _readCreateDirectoryTableStmt(); break; - case T_CreateLakeTableStmt: - return_value = _readCreateLakeTableStmt(); - break; case T_AlterDirectoryTableStmt: return_value = _readAlterDirectoryTableStmt(); break; diff --git a/src/backend/nodes/readfuncs_common.c b/src/backend/nodes/readfuncs_common.c index 882a38628be..f895e4a4468 100644 --- a/src/backend/nodes/readfuncs_common.c +++ b/src/backend/nodes/readfuncs_common.c @@ -738,7 +738,6 @@ _readDropStmt_common(DropStmt *local_node) READ_BOOL_FIELD(missing_ok); READ_BOOL_FIELD(concurrent); READ_BOOL_FIELD(isdynamic); - READ_BOOL_FIELD(isiceberg); /* Force 'missing_ok' in QEs */ #ifdef COMPILING_BINARY_FUNCS @@ -1149,7 +1148,6 @@ _readDropStmt(void) READ_BOOL_FIELD(missing_ok); READ_BOOL_FIELD(concurrent); READ_BOOL_FIELD(isdynamic); - READ_BOOL_FIELD(isiceberg); /* Force 'missing_ok' in QEs */ #ifdef COMPILING_BINARY_FUNCS diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index d5b69d9752e..bc657554219 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -313,7 +313,6 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ CreateSchemaStmt CreateSeqStmt CreateStmt CreateStatsStmt CreateStorageServerStmt CreateStorageUserMappingStmt CreateTableSpaceStmt CreateFdwStmt CreateForeignServerStmt CreateForeignTableStmt CreateDirectoryTableStmt - CreateLakeTableStmt CreateForeignCatalogStmt CreateForeignVolumeStmt CreateAssertionStmt CreateTransformStmt CreateTrigStmt CreateEventTrigStmt CreateUserStmt CreateUserMappingStmt CreateRoleStmt CreatePolicyStmt CreatedbStmt CreateWarehouseStmt DeclareCursorStmt DefineStmt DeleteStmt DiscardStmt DoStmt @@ -411,7 +410,6 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ %type OptProfileElem %type opt_type -%type OptForeignCatalog OptForeignVolume %type foreign_server_version opt_foreign_server_version %type opt_in_database @@ -841,7 +839,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ KEY KEYS - LABEL LAKE LANGUAGE LARGE_P LAST_P LATERAL_P + LABEL LANGUAGE LARGE_P LAST_P LATERAL_P LEADING LEAKPROOF LEAST LEFT LEVEL LIKE LIMIT LISTEN LOAD LOCAL LOCALTIME LOCALTIMESTAMP LOCATION LOCK_P LOCKED LOCUS LOGGED @@ -885,7 +883,7 @@ static void check_expressions_in_partition_key(PartitionSpec *spec, core_yyscan_ UNLISTEN UNLOGGED UNTIL UPDATE USER USING VACUUM VALID VALIDATE VALIDATOR VALUE_P VALUES VARCHAR VARIADIC VARYING - VERBOSE VERSION_P VIEW VIEWS VOLATILE VOLUME + VERBOSE VERSION_P VIEW VIEWS VOLATILE WHEN WHERE WHITESPACE_P WINDOW WITH WITHIN WITHOUT WORK WRAPPER WRITE @@ -1537,9 +1535,6 @@ stmt: | CreateConversionStmt | CreateDomainStmt | CreateDirectoryTableStmt - | CreateLakeTableStmt - | CreateForeignCatalogStmt - | CreateForeignVolumeStmt | CreateExtensionStmt | CreateExternalStmt | CreateFdwStmt @@ -9098,169 +9093,6 @@ CreateDirectoryTableStmt: } ; -/***************************************************************************** - * - * QUERY: - * CREATE FOREIGN CATALOG name SERVER server_name OPTIONS (...) - * - *****************************************************************************/ - -CreateForeignCatalogStmt: - CREATE FOREIGN CATALOG_P name SERVER name TYPE_P Sconst create_generic_options - { - CreateForeignCatalogStmt *n = makeNode(CreateForeignCatalogStmt); - n->catalogname = $4; - n->servername = $6; - n->catalogtype = $8; - n->options = $9; - n->if_not_exists = false; - $$ = (Node *) n; - } - | CREATE FOREIGN CATALOG_P IF_P NOT EXISTS name SERVER name TYPE_P Sconst create_generic_options - { - CreateForeignCatalogStmt *n = makeNode(CreateForeignCatalogStmt); - n->catalogname = $7; - n->servername = $9; - n->catalogtype = $11; - n->options = $12; - n->if_not_exists = true; - $$ = (Node *) n; - } - ; - -/***************************************************************************** - * - * QUERY: - * CREATE FOREIGN VOLUME name SERVER server_name OPTIONS (...) - * - *****************************************************************************/ - -CreateForeignVolumeStmt: - CREATE FOREIGN VOLUME name SERVER name create_generic_options - { - CreateForeignVolumeStmt *n = makeNode(CreateForeignVolumeStmt); - n->volumename = $4; - n->servername = $6; - n->options = $7; - n->if_not_exists = false; - $$ = (Node *) n; - } - | CREATE FOREIGN VOLUME IF_P NOT EXISTS name SERVER name create_generic_options - { - CreateForeignVolumeStmt *n = makeNode(CreateForeignVolumeStmt); - n->volumename = $7; - n->servername = $9; - n->options = $10; - n->if_not_exists = true; - $$ = (Node *) n; - } - ; - -OptForeignCatalog: - CATALOG_P name { $$ = $2; } - | /*EMPTY*/ { $$ = NULL; } - ; - -OptForeignVolume: - VOLUME name { $$ = $2; } - | /*EMPTY*/ { $$ = NULL; } - ; - -/***************************************************************************** - * - * QUERY: - * CREATE LAKE TABLE relname (columns) USING format - * [CATALOG cat] [VOLUME vol] OPTIONS (...) - * - * A lake table stores its data on external object storage; fragments are - * not hash-distributed across segments, so the distribution policy is - * forced to RANDOM to keep UPDATE/DELETE correct. The USING clause names - * the table format (currently only ICEBERG); the format also determines the - * table access method the relation is created with. - * - *****************************************************************************/ - -CreateLakeTableStmt: - CREATE LAKE TABLE qualified_name '(' OptTableElementList ')' - USING name - OptForeignCatalog OptForeignVolume create_generic_options - { - CreateLakeTableStmt *n = makeNode(CreateLakeTableStmt); - char *format = pstrdup($9); - char *p; - - /* - * The USING clause names both the lake table format and - * the access method that implements it; normalize to lower - * case so a quoted "ICEBERG" resolves the same AM as an - * unquoted iceberg. - */ - for (p = format; *p; p++) - *p = pg_tolower((unsigned char) *p); - - $4->relpersistence = RELPERSISTENCE_PERMANENT; - n->base.relation = $4; - n->base.tableElts = $6; - n->base.inhRelations = NIL; - n->base.ofTypename = NULL; - n->base.constraints = NIL; - n->base.options = $12; /* OPTIONS become reloptions, validated by the AM */ - n->base.oncommit = ONCOMMIT_NOOP; - n->base.tablespacename = NULL; - n->base.accessMethod = format; - n->base.if_not_exists = false; - n->base.relKind = RELKIND_RELATION; - n->table_type = format; - n->foreign_catalog = $10 ? pstrdup($10) : NULL; - n->foreign_volume = $11 ? pstrdup($11) : NULL; - /* lake tables are always distributed randomly */ - n->base.distributedBy = makeNode(DistributedBy); - n->base.distributedBy->ptype = POLICYTYPE_PARTITIONED; - n->base.distributedBy->keyCols = NIL; - n->base.distributedBy->numsegments = -1; - $$ = (Node *) n; - } - | CREATE LAKE TABLE IF_P NOT EXISTS qualified_name '(' OptTableElementList ')' - USING name - OptForeignCatalog OptForeignVolume create_generic_options - { - CreateLakeTableStmt *n = makeNode(CreateLakeTableStmt); - char *format = pstrdup($12); - char *p; - - /* - * The USING clause names both the lake table format and - * the access method that implements it; normalize to lower - * case so a quoted "ICEBERG" resolves the same AM as an - * unquoted iceberg. - */ - for (p = format; *p; p++) - *p = pg_tolower((unsigned char) *p); - - $7->relpersistence = RELPERSISTENCE_PERMANENT; - n->base.relation = $7; - n->base.tableElts = $9; - n->base.inhRelations = NIL; - n->base.ofTypename = NULL; - n->base.constraints = NIL; - n->base.options = $15; /* OPTIONS become reloptions, validated by the AM */ - n->base.oncommit = ONCOMMIT_NOOP; - n->base.tablespacename = NULL; - n->base.accessMethod = format; - n->base.if_not_exists = true; - n->base.relKind = RELKIND_RELATION; - n->table_type = format; - n->foreign_catalog = $13 ? pstrdup($13) : NULL; - n->foreign_volume = $14 ? pstrdup($14) : NULL; - /* lake tables are always distributed randomly */ - n->base.distributedBy = makeNode(DistributedBy); - n->base.distributedBy->ptype = POLICYTYPE_PARTITIONED; - n->base.distributedBy->keyCols = NIL; - n->base.distributedBy->numsegments = -1; - $$ = (Node *) n; - } - ; - /***************************************************************************** * * QUERY: @@ -10550,31 +10382,6 @@ DropStmt: DROP object_type_any_name IF_P EXISTS any_name_list opt_drop_behavior n->isdynamic = true; $$ = (Node *)n; } -/* DROP LAKE TABLE */ - | DROP LAKE TABLE IF_P EXISTS any_name_list opt_drop_behavior - { - DropStmt *n = makeNode(DropStmt); - n->removeType = OBJECT_TABLE; - n->missing_ok = true; - n->objects = $6; - n->behavior = $7; - n->concurrent = false; - n->isdynamic = false; - n->isiceberg = true; - $$ = (Node *)n; - } - | DROP LAKE TABLE any_name_list opt_drop_behavior - { - DropStmt *n = makeNode(DropStmt); - n->removeType = OBJECT_TABLE; - n->missing_ok = false; - n->objects = $4; - n->behavior = $5; - n->concurrent = false; - n->isdynamic = false; - n->isiceberg = true; - $$ = (Node *)n; - } ; /* object types taking any_name/any_name_list */ @@ -10622,8 +10429,6 @@ drop_type_name: | PUBLICATION { $$ = OBJECT_PUBLICATION; } | SCHEMA { $$ = OBJECT_SCHEMA; } | SERVER { $$ = OBJECT_FOREIGN_SERVER; } - | FOREIGN CATALOG_P { $$ = OBJECT_FOREIGN_CATALOG; } - | FOREIGN VOLUME { $$ = OBJECT_FOREIGN_VOLUME; } | PROTOCOL { $$ = OBJECT_EXTPROTOCOL; } ; @@ -21400,7 +21205,6 @@ unreserved_keyword: | KEY | KEYS | LABEL - | LAKE | LANGUAGE | LARGE_P | LAST_P @@ -21611,7 +21415,6 @@ unreserved_keyword: | VIEW | VIEWS | VOLATILE - | VOLUME | WAREHOUSE | WAREHOUSE_SIZE | WEB /* gp */ @@ -22403,7 +22206,6 @@ bare_label_keyword: | KEY | KEYS | LABEL - | LAKE | LANGUAGE | LARGE_P | LAST_P @@ -22663,7 +22465,6 @@ bare_label_keyword: | VIEW | VIEWS | VOLATILE - | VOLUME | WAREHOUSE | WAREHOUSE_SIZE | WEB diff --git a/src/backend/tcop/utility.c b/src/backend/tcop/utility.c index 530078e0799..021c69ad031 100644 --- a/src/backend/tcop/utility.c +++ b/src/backend/tcop/utility.c @@ -38,7 +38,6 @@ #include "commands/dbcommands.h" #include "commands/defrem.h" #include "commands/dirtablecmds.h" -#include "commands/laketablecmds.h" #include "commands/discard.h" #include "commands/event_trigger.h" #include "commands/explain.h" @@ -211,8 +210,6 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree) case T_CreateExtensionStmt: case T_CreateFdwStmt: case T_CreateForeignServerStmt: - case T_CreateForeignCatalogStmt: - case T_CreateForeignVolumeStmt: case T_CreateForeignTableStmt: case T_AddForeignSegStmt: case T_CreateFunctionStmt: @@ -265,7 +262,6 @@ ClassifyUtilityCommandAsReadOnly(Node *parsetree) case T_AlterResourceGroupStmt: case T_AlterTagStmt: case T_CreateDirectoryTableStmt: - case T_CreateLakeTableStmt: case T_AlterDirectoryTableStmt: case T_DropDirectoryTableStmt: case T_CreateProfileStmt: @@ -1452,7 +1448,6 @@ ProcessUtilitySlow(ParseState *pstate, case T_CreateStmt: case T_CreateForeignTableStmt: case T_CreateDirectoryTableStmt: - case T_CreateLakeTableStmt: { List *stmts; RangeVar *table_rv = NULL; @@ -1660,70 +1655,6 @@ ProcessUtilitySlow(ParseState *pstate, secondaryObject, stmt); } - else if (IsA(stmt, CreateLakeTableStmt)) - { - CreateLakeTableStmt *cstmt = (CreateLakeTableStmt *) stmt; - Datum toast_options; - static char *validnsps[] = HEAP_RELOPT_NAMESPACES; - - /* Remember transformed RangeVar for LIKE */ - table_rv = cstmt->base.relation; - - /* - * Validate catalog/volume resolution up front: - * the statement is dispatched to the QEs, so a - * failure raised only later inside - * CreateLakeTable() would surface as a confusing - * QE-annotated error. - */ - if (Gp_role == GP_ROLE_DISPATCH) - ValidateLakeTableStmt(cstmt); - - /* - * Create the table itself. Dispatch manually - * below (like the plain CreateStmt path above) - * so that the TOAST table exists before the - * statement is sent and its OID is included in - * the dispatched OID list. - */ - address = DefineRelation(&cstmt->base, - RELKIND_RELATION, - InvalidOid, NULL, - queryString, - false, - true, - NULL); - /* Create the lake table metadata entry */ - CreateLakeTable(cstmt, address.objectId); - EventTriggerCollectSimpleCommand(address, - secondaryObject, - stmt); - - /* - * Lake tables are backed by a real table access - * method, so let NewRelationCreateToastTable - * decide whether a secondary relation is needed, - * just like plain CREATE TABLE. - */ - CommandCounterIncrement(); - - toast_options = transformRelOptions((Datum) 0, - cstmt->base.options, - "toast", - validnsps, - true, - false); - NewRelationCreateToastTable(address.objectId, - toast_options); - - if (Gp_role == GP_ROLE_DISPATCH && ENABLE_DISPATCH()) - CdbDispatchUtilityStatement((Node *) stmt, - DF_CANCEL_ON_ERROR | - DF_NEED_TWO_PHASE | - DF_WITH_SNAPSHOT, - GetAssignedOidsForDispatch(), - NULL); - } else if (IsA(stmt, TableLikeClause)) { /* @@ -2252,14 +2183,6 @@ ProcessUtilitySlow(ParseState *pstate, address = CreateForeignServer((CreateForeignServerStmt *) parsetree); break; - case T_CreateForeignCatalogStmt: - address = CreateForeignCatalog((CreateForeignCatalogStmt *) parsetree); - break; - - case T_CreateForeignVolumeStmt: - address = CreateForeignVolume((CreateForeignVolumeStmt *) parsetree); - break; - case T_AlterForeignServerStmt: address = AlterForeignServer((AlterForeignServerStmt *) parsetree); break; @@ -3332,14 +3255,6 @@ CreateCommandTag(Node *parsetree) tag = CMDTAG_CREATE_SERVER; break; - case T_CreateForeignCatalogStmt: - tag = CMDTAG_CREATE_FOREIGN_CATALOG; - break; - - case T_CreateForeignVolumeStmt: - tag = CMDTAG_CREATE_FOREIGN_VOLUME; - break; - case T_AlterForeignServerStmt: tag = CMDTAG_ALTER_SERVER; break; @@ -3396,10 +3311,6 @@ CreateCommandTag(Node *parsetree) tag = CMDTAG_CREATE_DIRECTORY_TABLE; break; - case T_CreateLakeTableStmt: - tag = CMDTAG_CREATE_LAKE_TABLE; - break; - case T_AlterDirectoryTableStmt: tag = CMDTAG_ALTER_DIRECTORY_TABLE; break; @@ -3412,10 +3323,7 @@ CreateCommandTag(Node *parsetree) switch (((DropStmt *) parsetree)->removeType) { case OBJECT_TABLE: - if (((DropStmt *) parsetree)->isiceberg) - tag = CMDTAG_DROP_LAKE_TABLE; - else - tag = CMDTAG_DROP_TABLE; + tag = CMDTAG_DROP_TABLE; break; case OBJECT_SEQUENCE: tag = CMDTAG_DROP_SEQUENCE; @@ -3513,12 +3421,6 @@ CreateCommandTag(Node *parsetree) case OBJECT_FOREIGN_SERVER: tag = CMDTAG_DROP_SERVER; break; - case OBJECT_FOREIGN_CATALOG: - tag = CMDTAG_DROP_FOREIGN_CATALOG; - break; - case OBJECT_FOREIGN_VOLUME: - tag = CMDTAG_DROP_FOREIGN_VOLUME; - break; case OBJECT_STORAGE_SERVER: tag = CMDTAG_DROP_STORAGE_SERVER; break; @@ -4297,8 +4199,6 @@ GetCommandLogLevel(Node *parsetree) case T_CreateFdwStmt: case T_AlterFdwStmt: case T_CreateForeignServerStmt: - case T_CreateForeignCatalogStmt: - case T_CreateForeignVolumeStmt: case T_AlterForeignServerStmt: case T_CreateStorageServerStmt: case T_AlterStorageServerStmt: @@ -4311,7 +4211,6 @@ GetCommandLogLevel(Node *parsetree) case T_DropStorageUserMappingStmt: case T_ImportForeignSchemaStmt: case T_CreateDirectoryTableStmt: - case T_CreateLakeTableStmt: case T_AlterDirectoryTableStmt: case T_DropDirectoryTableStmt: lev = LOGSTMT_DDL; diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c index 556c7536205..a8901a957eb 100644 --- a/src/backend/utils/cache/syscache.c +++ b/src/backend/utils/cache/syscache.c @@ -44,11 +44,9 @@ #include "catalog/pg_enum.h" #include "catalog/pg_event_trigger.h" #include "catalog/pg_extension.h" -#include "catalog/pg_foreign_catalog.h" #include "catalog/pg_foreign_data_wrapper.h" #include "catalog/pg_foreign_server.h" #include "catalog/pg_foreign_table.h" -#include "catalog/pg_foreign_volume.h" #include "catalog/pg_language.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" @@ -365,18 +363,6 @@ static const struct cachedesc cacheinfo[] = { KEY(Anum_pg_foreign_data_wrapper_oid), 2 }, - [FOREIGNCATALOGNAME] = { - ForeignCatalogRelationId, - ForeignCatalogNameIndexId, - KEY(Anum_pg_foreign_catalog_fcname), - 2 - }, - [FOREIGNCATALOGOID] = { - ForeignCatalogRelationId, - ForeignCatalogOidIndexId, - KEY(Anum_pg_foreign_catalog_oid), - 2 - }, [FOREIGNSERVERNAME] = { ForeignServerRelationId, ForeignServerNameIndexId, @@ -407,18 +393,6 @@ static const struct cachedesc cacheinfo[] = { KEY(Anum_pg_foreign_table_ftrelid), 4 }, - [FOREIGNVOLUMENAME] = { - ForeignVolumeRelationId, - ForeignVolumeNameIndexId, - KEY(Anum_pg_foreign_volume_fvname), - 2 - }, - [FOREIGNVOLUMEOID] = { - ForeignVolumeRelationId, - ForeignVolumeOidIndexId, - KEY(Anum_pg_foreign_volume_oid), - 2 - }, [GPPOLICYID] = { GpPolicyRelationId, GpPolicyLocalOidIndexId, diff --git a/src/backend/utils/misc/guc_tables.c b/src/backend/utils/misc/guc_tables.c index c4498a09fc4..1ff92994527 100644 --- a/src/backend/utils/misc/guc_tables.c +++ b/src/backend/utils/misc/guc_tables.c @@ -39,7 +39,6 @@ #include "catalog/storage_directory_table.h" #include "commands/async.h" #include "commands/tablespace.h" -#include "commands/laketablecmds.h" #include "commands/trigger.h" #include "commands/user.h" #include "commands/vacuum.h" @@ -4120,28 +4119,6 @@ struct config_string ConfigureNamesString[] = check_temp_tablespaces, assign_temp_tablespaces, NULL }, - { - {"iceberg_default_catalog", PGC_USERSET, CLIENT_CONN_STATEMENT, - gettext_noop("Sets the default foreign catalog to create Iceberg tables in."), - gettext_noop("An empty string means no default catalog."), - GUC_IS_NAME - }, - &iceberg_default_catalog, - "", - check_iceberg_default_catalog, NULL, NULL - }, - - { - {"iceberg_default_volume", PGC_USERSET, CLIENT_CONN_STATEMENT, - gettext_noop("Sets the default foreign volume to create Iceberg tables in."), - gettext_noop("An empty string means no default volume."), - GUC_IS_NAME - }, - &iceberg_default_volume, - "", - check_iceberg_default_volume, NULL, NULL - }, - { {"createrole_self_grant", PGC_USERSET, CLIENT_CONN_STATEMENT, gettext_noop("Sets whether a CREATEROLE user automatically grants " diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index d7857a6abad..7604417e262 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -2134,19 +2134,6 @@ selectDumpableTable(TableInfo *tbinfo, Archive *fout) pg_log_warning("unsupport am pax yet, current relation \"%s\" will be ignore", tbinfo->dobj.name); } - - /* - * Lake tables cannot be reproduced by pg_dump: their data lives in - * external object storage managed through the iceberg access method's - * foreign catalog and volume. Skip them. - */ - if (tbinfo->amname && strcmp(tbinfo->amname, "iceberg") == 0) - { - tbinfo->dobj.dump = DUMP_COMPONENT_NONE; - - pg_log_warning("lake table \"%s\" is not supported by pg_dump and will be ignored", - tbinfo->dobj.name); - } } /* diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 65bc6727a41..81a81578c0b 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -668,35 +668,11 @@ static const SchemaQuery Query_for_list_of_foreign_tables = { .result = "c.relname", }; -/* - * Exclude lake tables; Query_for_list_of_iceberg_tables serves DROP LAKE - * TABLE. With no iceberg AM, the NOT EXISTS subquery finds no match, so - * ordinary tables remain listed. - */ static const SchemaQuery Query_for_list_of_tables = { .catname = "pg_catalog.pg_class c", .selcondition = "c.relkind IN (" CppAsString2(RELKIND_RELATION) ", " - CppAsString2(RELKIND_PARTITIONED_TABLE) ") AND " - "NOT EXISTS (SELECT 1 FROM pg_catalog.pg_am a " - "WHERE a.oid = c.relam AND a.amname = 'iceberg' AND a.amtype = 't')", - .viscondition = "pg_catalog.pg_table_is_visible(c.oid)", - .namespace = "c.relnamespace", - .result = "c.relname", -}; - -/* - * A lake table is an ordinary relation whose access method is the - * "iceberg" AM -- the same rule DROP LAKE TABLE validates against. When no - * provider installed that AM the scalar subquery yields NULL and the list is - * empty, which is correct (no relation can be a lake table). - */ -static const SchemaQuery Query_for_list_of_iceberg_tables = { - .catname = "pg_catalog.pg_class c", - .selcondition = - "c.relkind IN (" CppAsString2(RELKIND_RELATION) ") AND " - "c.relam = (SELECT oid FROM pg_catalog.pg_am " - "WHERE amname = 'iceberg' AND amtype = 't')", + CppAsString2(RELKIND_PARTITIONED_TABLE) ")", .viscondition = "pg_catalog.pg_table_is_visible(c.oid)", .namespace = "c.relnamespace", .result = "c.relname", @@ -1101,16 +1077,6 @@ static const SchemaQuery Query_for_trigger_of_table = { " FROM pg_catalog.pg_foreign_server "\ " WHERE srvname LIKE '%s'" -#define Query_for_list_of_foreign_catalogs \ -" SELECT fcname "\ -" FROM pg_catalog.pg_foreign_catalog "\ -" WHERE fcname LIKE '%s'" - -#define Query_for_list_of_foreign_volumes \ -" SELECT fvname "\ -" FROM pg_catalog.pg_foreign_volume "\ -" WHERE fvname LIKE '%s'" - #define Query_for_list_of_user_mappings \ " SELECT usename "\ " FROM pg_catalog.pg_user_mappings "\ @@ -1308,15 +1274,12 @@ static const pgsql_thing_t words_after_create[] = { {"EVENT TRIGGER", NULL, NULL, NULL}, {"EXTENSION", Query_for_list_of_extensions}, {"EXTERNAL TABLE", NULL, NULL, NULL}, - {"FOREIGN CATALOG", Query_for_list_of_foreign_catalogs, NULL, NULL, NULL, THING_NO_ALTER}, {"FOREIGN DATA WRAPPER", NULL, NULL, NULL}, {"FOREIGN TABLE", NULL, NULL, NULL}, - {"FOREIGN VOLUME", Query_for_list_of_foreign_volumes, NULL, NULL, NULL, THING_NO_ALTER}, {"FUNCTION", NULL, NULL, Query_for_list_of_functions}, {"GROUP", Query_for_list_of_roles}, {"INCREMENTAL MATERIALIZED VIEW", NULL, NULL, &Query_for_list_of_matviews, NULL, THING_NO_DROP | THING_NO_ALTER}, {"INDEX", NULL, NULL, &Query_for_list_of_indexes}, - {"LAKE TABLE", NULL, NULL, &Query_for_list_of_iceberg_tables, NULL, THING_NO_ALTER}, {"LANGUAGE", Query_for_list_of_languages}, {"LARGE OBJECT", NULL, NULL, NULL, NULL, THING_NO_CREATE | THING_NO_DROP}, {"MATERIALIZED VIEW", NULL, NULL, &Query_for_list_of_matviews}, @@ -3079,32 +3042,7 @@ psql_completion(const char *text, int start, int end) /* CREATE FOREIGN */ else if (Matches("CREATE", "FOREIGN")) - COMPLETE_WITH("CATALOG", "DATA WRAPPER", "TABLE", "VOLUME"); - - /* CREATE FOREIGN CATALOG */ - else if (Matches("CREATE", "FOREIGN", "CATALOG", MatchAny)) - COMPLETE_WITH("SERVER"); - else if (Matches("CREATE", "FOREIGN", "CATALOG", MatchAny, "SERVER")) - COMPLETE_WITH_QUERY(Query_for_list_of_servers); - else if (Matches("CREATE", "FOREIGN", "CATALOG", MatchAny, "SERVER", MatchAny)) - COMPLETE_WITH("OPTIONS"); - - /* CREATE FOREIGN VOLUME */ - else if (Matches("CREATE", "FOREIGN", "VOLUME", MatchAny)) - COMPLETE_WITH("SERVER"); - else if (Matches("CREATE", "FOREIGN", "VOLUME", MatchAny, "SERVER")) - COMPLETE_WITH_QUERY(Query_for_list_of_servers); - else if (Matches("CREATE", "FOREIGN", "VOLUME", MatchAny, "SERVER", MatchAny)) - COMPLETE_WITH("OPTIONS"); - - /* CREATE LAKE TABLE */ - else if (Matches("CREATE", "LAKE")) - COMPLETE_WITH("TABLE"); - /* CREATE LAKE TABLE ... USING */ - else if (HeadMatches("CREATE", "LAKE", "TABLE") && TailMatches("USING")) - COMPLETE_WITH("ICEBERG"); - else if (HeadMatches("CREATE", "LAKE", "TABLE") && TailMatches("USING", "ICEBERG")) - COMPLETE_WITH("CATALOG", "VOLUME", "OPTIONS"); + COMPLETE_WITH("DATA WRAPPER", "TABLE"); /* CREATE FOREIGN DATA WRAPPER */ else if (Matches("CREATE", "FOREIGN", "DATA", "WRAPPER", MatchAny)) @@ -3871,8 +3809,6 @@ psql_completion(const char *text, int start, int end) Matches("DROP", "EVENT", "TRIGGER", MatchAny) || Matches("DROP", "FOREIGN", "DATA", "WRAPPER", MatchAny) || Matches("DROP", "FOREIGN", "TABLE", MatchAny) || - Matches("DROP", "FOREIGN", "CATALOG|VOLUME", MatchAny) || - Matches("DROP", "LAKE", "TABLE", MatchAny) || Matches("DROP", "DIRECTORY", "TABLE", MatchAny) || Matches("DROP", "TEXT", "SEARCH", "CONFIGURATION|DICTIONARY|PARSER|TEMPLATE", MatchAny)) COMPLETE_WITH("CASCADE", "RESTRICT"); @@ -3883,15 +3819,7 @@ psql_completion(const char *text, int start, int end) else if (Matches("DROP", "AGGREGATE|FUNCTION|PROCEDURE|ROUTINE", MatchAny, "(")) COMPLETE_WITH_FUNCTION_ARG(prev2_wd); else if (Matches("DROP", "FOREIGN")) - COMPLETE_WITH("CATALOG", "DATA WRAPPER", "TABLE", "VOLUME"); - else if (Matches("DROP", "FOREIGN", "CATALOG")) - COMPLETE_WITH_QUERY(Query_for_list_of_foreign_catalogs); - else if (Matches("DROP", "FOREIGN", "VOLUME")) - COMPLETE_WITH_QUERY(Query_for_list_of_foreign_volumes); - else if (Matches("DROP", "LAKE")) - COMPLETE_WITH("TABLE"); - else if (Matches("DROP", "LAKE", "TABLE")) - COMPLETE_WITH_SCHEMA_QUERY(Query_for_list_of_iceberg_tables); + COMPLETE_WITH("DATA WRAPPER", "TABLE"); else if (Matches("DROP", "DATABASE", MatchAny)) COMPLETE_WITH("WITH ("); else if (HeadMatches("DROP", "DATABASE") && (ends_with(prev_wd, '('))) @@ -4112,18 +4040,6 @@ psql_completion(const char *text, int start, int end) else if (TailMatches("FOREIGN", "SERVER")) COMPLETE_WITH_QUERY(Query_for_list_of_servers); -/* CATALOG, e.g. the CATALOG clause of CREATE LAKE TABLE */ - else if (TailMatches("CATALOG") && - !TailMatches("CREATE", MatchAny, MatchAny) && - !TailMatches("FOREIGN", MatchAny)) - COMPLETE_WITH_QUERY(Query_for_list_of_foreign_catalogs); - -/* VOLUME, e.g. the VOLUME clause of CREATE LAKE TABLE */ - else if (TailMatches("VOLUME") && - !TailMatches("CREATE", MatchAny, MatchAny) && - !TailMatches("FOREIGN", MatchAny)) - COMPLETE_WITH_QUERY(Query_for_list_of_foreign_volumes); - /* STORAGE SERVER */ else if (TailMatches("ALTER", "STORAGE", "SERVER")) COMPLETE_WITH_QUERY(Query_for_list_of_storage_servers); diff --git a/src/include/catalog/catversion.h b/src/include/catalog/catversion.h index 79fea91acf3..851e58debc3 100644 --- a/src/include/catalog/catversion.h +++ b/src/include/catalog/catversion.h @@ -60,6 +60,6 @@ */ /* 3yyymmddN */ -#define CATALOG_VERSION_NO 302607221 +#define CATALOG_VERSION_NO 302606111 #endif diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index ac2ccb596b1..6a7ae2abea9 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -151,11 +151,9 @@ typedef enum ObjectClass OCLASS_EXTPROTOCOL, /* pg_extprotocol */ OCLASS_MATVIEW_AUX, /* gp_matview_aux */ OCLASS_TASK, /* pg_task */ - OCLASS_FOREIGN_CATALOG, /* pg_foreign_catalog */ - OCLASS_FOREIGN_VOLUME, /* pg_foreign_volume */ } ObjectClass; -#define LAST_OCLASS OCLASS_FOREIGN_VOLUME +#define LAST_OCLASS OCLASS_TASK /* flag bits for performDeletion/performMultipleDeletions: */ #define PERFORM_DELETION_INTERNAL 0x0001 /* internal action */ diff --git a/src/include/catalog/oid_dispatch.h b/src/include/catalog/oid_dispatch.h index 9c0574faf38..6ccd20ac4bd 100644 --- a/src/include/catalog/oid_dispatch.h +++ b/src/include/catalog/oid_dispatch.h @@ -65,10 +65,6 @@ extern Oid GetNewOidForForeignDataWrapper(Relation relation, Oid indexId, AttrNu char *fdwname); extern Oid GetNewOidForForeignServer(Relation relation, Oid indexId, AttrNumber oidcolumn, char *srvname); -extern Oid GetNewOidForForeignCatalog(Relation relation, Oid indexId, AttrNumber oidcolumn, - char *catname); -extern Oid GetNewOidForForeignVolume(Relation relation, Oid indexId, AttrNumber oidcolumn, - char *volumename); extern Oid GetNewOidForStorageServer(Relation relation, Oid indexId, AttrNumber oidcolumn, char *srvname); extern Oid GetNewOidForLanguage(Relation relation, Oid indexId, AttrNumber oidcolumn, diff --git a/src/include/catalog/pg_foreign_catalog.h b/src/include/catalog/pg_foreign_catalog.h deleted file mode 100644 index 547f01c6a9c..00000000000 --- a/src/include/catalog/pg_foreign_catalog.h +++ /dev/null @@ -1,70 +0,0 @@ -/*------------------------------------------------------------------------- - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * pg_foreign_catalog.h - * definition of the "foreign catalog" system catalog (pg_foreign_catalog) - * - * src/include/catalog/pg_foreign_catalog.h - * - * NOTES - * The Catalog.pm module reads this file and derives schema - * information. - * - *------------------------------------------------------------------------- - */ -#ifndef PG_FOREIGN_CATALOG_H -#define PG_FOREIGN_CATALOG_H - -#include "catalog/genbki.h" -#include "catalog/pg_foreign_catalog_d.h" - -/* ---------------- - * pg_foreign_catalog definition. cpp turns this into - * typedef struct FormData_pg_foreign_catalog - * ---------------- - */ -CATALOG(pg_foreign_catalog,8549,ForeignCatalogRelationId) -{ - Oid oid; /* oid */ - - NameData fcname; /* foreign catalog name */ - - Oid fcowner BKI_LOOKUP(pg_authid); /* owner of the foreign catalog */ - - Oid fcserver BKI_LOOKUP(pg_foreign_server); /* foreign server this catalog belongs to */ - -#ifdef CATALOG_VARLEN /* variable-length fields start here */ - text fctype BKI_FORCE_NOT_NULL; /* catalog type, e.g. 'hive' */ - text fcoptions[1]; /* foreign catalog options */ -#endif -} FormData_pg_foreign_catalog; - -/* ---------------- - * Form_pg_foreign_catalog corresponds to a pointer to a tuple with - * the format of pg_foreign_catalog relation. - * ---------------- - */ -typedef FormData_pg_foreign_catalog *Form_pg_foreign_catalog; - -DECLARE_TOAST(pg_foreign_catalog, 8550, 8551); - -DECLARE_UNIQUE_INDEX_PKEY(pg_foreign_catalog_oid_index, 8552, ForeignCatalogOidIndexId, on pg_foreign_catalog using btree(oid oid_ops)); -DECLARE_UNIQUE_INDEX(pg_foreign_catalog_name_index, 8553, ForeignCatalogNameIndexId, on pg_foreign_catalog using btree(fcname name_ops)); - -#endif /* PG_FOREIGN_CATALOG_H */ diff --git a/src/include/catalog/pg_foreign_volume.h b/src/include/catalog/pg_foreign_volume.h deleted file mode 100644 index d3e377f1939..00000000000 --- a/src/include/catalog/pg_foreign_volume.h +++ /dev/null @@ -1,69 +0,0 @@ -/*------------------------------------------------------------------------- - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * pg_foreign_volume.h - * definition of the "foreign volume" system catalog (pg_foreign_volume) - * - * src/include/catalog/pg_foreign_volume.h - * - * NOTES - * The Catalog.pm module reads this file and derives schema - * information. - * - *------------------------------------------------------------------------- - */ -#ifndef PG_FOREIGN_VOLUME_H -#define PG_FOREIGN_VOLUME_H - -#include "catalog/genbki.h" -#include "catalog/pg_foreign_volume_d.h" - -/* ---------------- - * pg_foreign_volume definition. cpp turns this into - * typedef struct FormData_pg_foreign_volume - * ---------------- - */ -CATALOG(pg_foreign_volume,8554,ForeignVolumeRelationId) -{ - Oid oid; /* oid */ - - NameData fvname; /* foreign volume name */ - - Oid fvowner BKI_LOOKUP(pg_authid); /* owner of the foreign volume */ - - Oid fvserver BKI_LOOKUP(pg_foreign_server); /* foreign server this volume belongs to */ - -#ifdef CATALOG_VARLEN /* variable-length fields start here */ - text fvoptions[1]; /* foreign volume options */ -#endif -} FormData_pg_foreign_volume; - -/* ---------------- - * Form_pg_foreign_volume corresponds to a pointer to a tuple with - * the format of pg_foreign_volume relation. - * ---------------- - */ -typedef FormData_pg_foreign_volume *Form_pg_foreign_volume; - -DECLARE_TOAST(pg_foreign_volume, 8555, 8556); - -DECLARE_UNIQUE_INDEX_PKEY(pg_foreign_volume_oid_index, 8557, ForeignVolumeOidIndexId, on pg_foreign_volume using btree(oid oid_ops)); -DECLARE_UNIQUE_INDEX(pg_foreign_volume_name_index, 8558, ForeignVolumeNameIndexId, on pg_foreign_volume using btree(fvname name_ops)); - -#endif /* PG_FOREIGN_VOLUME_H */ diff --git a/src/include/catalog/pg_lake_table.h b/src/include/catalog/pg_lake_table.h deleted file mode 100644 index e4d4ff48699..00000000000 --- a/src/include/catalog/pg_lake_table.h +++ /dev/null @@ -1,74 +0,0 @@ -/*------------------------------------------------------------------------- - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * pg_lake_table.h - * definition of the "lake table" system catalog (pg_lake_table) - * - * src/include/catalog/pg_lake_table.h - * - * NOTES - * The Catalog.pm module reads this file and derives schema - * information. - * - *------------------------------------------------------------------------- - */ -#ifndef PG_LAKE_TABLE_H -#define PG_LAKE_TABLE_H - -#include "catalog/genbki.h" -#include "catalog/pg_lake_table_d.h" -#include "nodes/pg_list.h" - -/* ---------------- - * pg_lake_table definition. cpp turns this into - * typedef struct FormData_pg_lake_table - * ---------------- - */ -CATALOG(pg_lake_table,9901,LakeTableRelationId) -{ - Oid ltrelid BKI_LOOKUP(pg_class); /* OID of the lake table relation */ - Oid ltforeign_catalog BKI_LOOKUP_OPT(pg_foreign_catalog); /* OID of foreign catalog */ - Oid ltforeign_volume BKI_LOOKUP_OPT(pg_foreign_volume); /* OID of foreign volume */ -} FormData_pg_lake_table; - -/* ---------------- - * Form_pg_lake_table corresponds to a pointer to a tuple with - * the format of pg_lake_table relation. - * - * A lake table's format is its access method (pg_class.relam) and its - * options are the relation's reloptions (pg_class.reloptions), validated by - * the access method; pg_lake_table only records the catalog/volume binding. - * ---------------- - */ -typedef FormData_pg_lake_table *Form_pg_lake_table; - -DECLARE_UNIQUE_INDEX_PKEY(pg_lake_table_relid_index, 9902, LakeTableRelidIndexId, on pg_lake_table using btree(ltrelid oid_ops)); - -/* ---------------- - * Lake table structure for caching - * ---------------- - */ -typedef struct LakeTable -{ - Oid relid; /* OID of the lake table relation */ - char *foreign_catalog; /* foreign catalog name */ - char *foreign_volume; /* foreign volume name */ -} LakeTable; - -#endif /* PG_LAKE_TABLE_H */ diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index 72f3562a033..7af15a37f52 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -133,8 +133,6 @@ extern ObjectAddress CreateForeignDataWrapper(ParseState *pstate, CreateFdwStmt extern ObjectAddress AlterForeignDataWrapper(ParseState *pstate, AlterFdwStmt *stmt); extern ObjectAddress CreateForeignServer(CreateForeignServerStmt *stmt); extern ObjectAddress AlterForeignServer(AlterForeignServerStmt *stmt); -extern ObjectAddress CreateForeignCatalog(CreateForeignCatalogStmt *stmt); -extern ObjectAddress CreateForeignVolume(CreateForeignVolumeStmt *stmt); extern ObjectAddress CreateStorageServer(CreateStorageServerStmt *stmt); extern ObjectAddress AlterStorageServer(AlterStorageServerStmt *stmt); extern Oid RemoveStorageServer(DropStorageServerStmt *stmt); diff --git a/src/include/commands/laketablecmds.h b/src/include/commands/laketablecmds.h deleted file mode 100644 index 4c14941aa25..00000000000 --- a/src/include/commands/laketablecmds.h +++ /dev/null @@ -1,61 +0,0 @@ -/*------------------------------------------------------------------------- - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - * laketablecmds.h - * prototypes for laketablecmds.c. - * - * src/include/commands/laketablecmds.h - * - *------------------------------------------------------------------------- - */ -#ifndef LAKETABLECMDS_H -#define LAKETABLECMDS_H - -#include "catalog/pg_lake_table.h" -#include "nodes/parsenodes.h" -#include "utils/guc.h" -#include "utils/rel.h" - -/* - * Name of the table access method lake tables are created with. The - * kernel only provides the DDL scaffolding; the access method itself is - * provided by a datalake extension. - */ -#define ICEBERG_TABLE_AM_NAME "iceberg" - -/* GUC variables */ -extern char *iceberg_default_catalog; -extern char *iceberg_default_volume; - -/* GUC check hooks */ -extern bool check_iceberg_default_catalog(char **newval, void **extra, GucSource source); -extern bool check_iceberg_default_volume(char **newval, void **extra, GucSource source); - -/* Functions to get default values */ -extern const char *GetDefaultIcebergCatalog(void); -extern const char *GetDefaultIcebergVolume(void); - -/* Lake table management */ -extern Oid GetIcebergTableAmOid(bool missing_ok); -extern bool RelationIsLakeTable(Relation rel); -extern void ValidateLakeTableStmt(CreateLakeTableStmt *stmt); -extern void CreateLakeTable(CreateLakeTableStmt *stmt, Oid relId); -extern void RemoveLakeTableEntry(Oid relid); - -#endif /* LAKETABLECMDS_H */ diff --git a/src/include/foreign/foreign.h b/src/include/foreign/foreign.h index 417ac80d5fb..f95f0d331e7 100644 --- a/src/include/foreign/foreign.h +++ b/src/include/foreign/foreign.h @@ -62,14 +62,6 @@ typedef struct ForeignTable int32 num_segments; /* the number of segments of the foreign table */ } ForeignTable; -typedef struct ForeignVolume -{ - Oid volumeid; /* volume Oid */ - Oid serverid; /* server Oid */ - char *volumename; /* name of the volume */ - List *options; /* fvoptions as DefElem list */ -} ForeignVolume; - /* Flags for GetForeignServerExtended */ #define FSV_MISSING_OK 0x01 @@ -92,15 +84,11 @@ extern ForeignDataWrapper *GetForeignDataWrapperByName(const char *fdwname, bool missing_ok); extern ForeignTable *GetForeignTable(Oid relid); extern bool rel_is_external_table(Oid relid); -extern ForeignVolume *GetForeignVolumeByName(const char *volumename, - bool missing_ok); extern List *GetForeignColumnOptions(Oid relid, AttrNumber attnum); extern Oid get_foreign_data_wrapper_oid(const char *fdwname, bool missing_ok); extern Oid get_foreign_server_oid(const char *servername, bool missing_ok); -extern Oid get_foreign_catalog_oid(const char *catalogname, bool missing_ok); -extern Oid get_foreign_volume_oid(const char *volumename, bool missing_ok); extern Oid GetForeignServerSegByRelid(Oid tableOid); extern List *GetForeignServerSegsByRelId(Oid relid); diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 9efc253e44b..bd2c1bcf58c 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -526,8 +526,6 @@ typedef enum NodeTag T_AlterFdwStmt, T_CreateForeignServerStmt, T_AlterForeignServerStmt, - T_CreateForeignCatalogStmt, - T_CreateForeignVolumeStmt, T_CreateStorageServerStmt, T_AlterStorageServerStmt, T_DropStorageServerStmt, @@ -574,7 +572,6 @@ typedef enum NodeTag T_CreateDirectoryTableStmt, T_AlterDirectoryTableStmt, T_DropDirectoryTableStmt, - T_CreateLakeTableStmt, T_CreateFileSpaceStmt, T_FileSpaceEntry, T_DropFileSpaceStmt, diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index d43898765b0..b79846b3d6d 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -2260,8 +2260,6 @@ typedef enum ObjectType OBJECT_EXTENSION, OBJECT_FDW, OBJECT_FOREIGN_SERVER, - OBJECT_FOREIGN_CATALOG, - OBJECT_FOREIGN_VOLUME, OBJECT_STORAGE_SERVER, OBJECT_FOREIGN_TABLE, OBJECT_FUNCTION, @@ -3273,25 +3271,6 @@ typedef struct CreateForeignServerStmt List *options; /* generic options to server */ } CreateForeignServerStmt; -typedef struct CreateForeignCatalogStmt -{ - NodeTag type; - char *catalogname; /* foreign catalog name */ - char *servername; /* server name */ - char *catalogtype; /* catalog type, e.g. 'hive' */ - bool if_not_exists; /* just do nothing if it already exists? */ - List *options; /* generic options to catalog */ -} CreateForeignCatalogStmt; - -typedef struct CreateForeignVolumeStmt -{ - NodeTag type; - char *volumename; /* foreign volume name */ - char *servername; /* server name */ - bool if_not_exists; /* just do nothing if it already exists? */ - List *options; /* generic options to volume */ -} CreateForeignVolumeStmt; - typedef struct AlterForeignServerStmt { NodeTag type; @@ -3799,14 +3778,6 @@ typedef struct CreateDirectoryTableStmt char *location; /* dtlocation for pg_directory_table */ } CreateDirectoryTableStmt; -typedef struct CreateLakeTableStmt -{ - CreateStmt base; /* base table creation info */ - char *table_type; /* lake table format, e.g. "ICEBERG" (validation only) */ - char *foreign_catalog; /* foreign catalog name, or NULL */ - char *foreign_volume; /* foreign volume name, or NULL */ -} CreateLakeTableStmt; - typedef struct AlterDirectoryTableStmt { NodeTag type; @@ -3832,7 +3803,6 @@ typedef struct DropStmt bool missing_ok; /* skip error if object is missing? */ bool concurrent; /* drop index concurrently? */ bool isdynamic; /* drop a dynamic table? */ - bool isiceberg; /* drop an iceberg (lake) table? */ } DropStmt; /* ---------------------- diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 60315094de6..24b6936bd46 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -272,7 +272,6 @@ PG_KEYWORD("json_objectagg", JSON_OBJECTAGG, COL_NAME_KEYWORD, BARE_LABEL) PG_KEYWORD("key", KEY, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("keys", KEYS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("label", LABEL, UNRESERVED_KEYWORD, BARE_LABEL) -PG_KEYWORD("lake", LAKE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("language", LANGUAGE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("large", LARGE_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("last", LAST_P, UNRESERVED_KEYWORD, BARE_LABEL) @@ -549,7 +548,6 @@ PG_KEYWORD("version", VERSION_P, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("view", VIEW, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("views", VIEWS, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("volatile", VOLATILE, UNRESERVED_KEYWORD, BARE_LABEL) -PG_KEYWORD("volume", VOLUME, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("warehouse", WAREHOUSE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("warehouse_size", WAREHOUSE_SIZE, UNRESERVED_KEYWORD, BARE_LABEL) PG_KEYWORD("web", WEB, UNRESERVED_KEYWORD, BARE_LABEL) diff --git a/src/include/tcop/cmdtaglist.h b/src/include/tcop/cmdtaglist.h index 66684f8d6f9..21db8a4d19d 100644 --- a/src/include/tcop/cmdtaglist.h +++ b/src/include/tcop/cmdtaglist.h @@ -106,13 +106,10 @@ PG_CMDTAG(CMDTAG_CREATE_DYNAMIC_TABLE, "CREATE DYNAMIC TABLE", true, false, fals PG_CMDTAG(CMDTAG_CREATE_EVENT_TRIGGER, "CREATE EVENT TRIGGER", false, false, false) PG_CMDTAG(CMDTAG_CREATE_EXTENSION, "CREATE EXTENSION", true, false, false) PG_CMDTAG(CMDTAG_CREATE_EXTERNAL, "CREATE EXTERNAL TABLE", true, false, false) -PG_CMDTAG(CMDTAG_CREATE_FOREIGN_CATALOG, "CREATE FOREIGN CATALOG", true, false, false) PG_CMDTAG(CMDTAG_CREATE_FOREIGN_DATA_WRAPPER, "CREATE FOREIGN DATA WRAPPER", true, false, false) PG_CMDTAG(CMDTAG_CREATE_FOREIGN_TABLE, "CREATE FOREIGN TABLE", true, false, false) -PG_CMDTAG(CMDTAG_CREATE_FOREIGN_VOLUME, "CREATE FOREIGN VOLUME", true, false, false) PG_CMDTAG(CMDTAG_CREATE_FUNCTION, "CREATE FUNCTION", true, false, false) PG_CMDTAG(CMDTAG_CREATE_INDEX, "CREATE INDEX", true, false, false) -PG_CMDTAG(CMDTAG_CREATE_LAKE_TABLE, "CREATE LAKE TABLE", true, false, false) PG_CMDTAG(CMDTAG_CREATE_LANGUAGE, "CREATE LANGUAGE", true, false, false) PG_CMDTAG(CMDTAG_CREATE_MATERIALIZED_VIEW, "CREATE MATERIALIZED VIEW", true, false, false) PG_CMDTAG(CMDTAG_CREATE_OPERATOR, "CREATE OPERATOR", true, false, false) @@ -179,13 +176,10 @@ PG_CMDTAG(CMDTAG_DROP_DOMAIN, "DROP DOMAIN", true, false, false) PG_CMDTAG(CMDTAG_DROP_DYNAMIC_TABLE, "DROP DYNAMIC TABLE", true, false, false) PG_CMDTAG(CMDTAG_DROP_EVENT_TRIGGER, "DROP EVENT TRIGGER", false, false, false) PG_CMDTAG(CMDTAG_DROP_EXTENSION, "DROP EXTENSION", true, false, false) -PG_CMDTAG(CMDTAG_DROP_FOREIGN_CATALOG, "DROP FOREIGN CATALOG", true, false, false) PG_CMDTAG(CMDTAG_DROP_FOREIGN_DATA_WRAPPER, "DROP FOREIGN DATA WRAPPER", true, false, false) PG_CMDTAG(CMDTAG_DROP_FOREIGN_TABLE, "DROP FOREIGN TABLE", true, false, false) -PG_CMDTAG(CMDTAG_DROP_FOREIGN_VOLUME, "DROP FOREIGN VOLUME", true, false, false) PG_CMDTAG(CMDTAG_DROP_FUNCTION, "DROP FUNCTION", true, false, false) PG_CMDTAG(CMDTAG_DROP_INDEX, "DROP INDEX", true, false, false) -PG_CMDTAG(CMDTAG_DROP_LAKE_TABLE, "DROP LAKE TABLE", true, false, false) PG_CMDTAG(CMDTAG_DROP_LANGUAGE, "DROP LANGUAGE", true, false, false) PG_CMDTAG(CMDTAG_DROP_MATERIALIZED_VIEW, "DROP MATERIALIZED VIEW", true, false, false) PG_CMDTAG(CMDTAG_DROP_OPERATOR, "DROP OPERATOR", true, false, false) diff --git a/src/include/utils/sync_guc_name.h b/src/include/utils/sync_guc_name.h index 6c829ace83a..dfb6e946bac 100644 --- a/src/include/utils/sync_guc_name.h +++ b/src/include/utils/sync_guc_name.h @@ -122,8 +122,6 @@ "gp_workfile_limit_per_query", "gp_write_shared_snapshot", "hash_mem_multiplier", - "iceberg_default_catalog", - "iceberg_default_volume", "ignore_system_indexes", "ignore_checksum_failure", "IntervalStyle", diff --git a/src/include/utils/syscache.h b/src/include/utils/syscache.h index 5ed42d82b15..e790dfe2af5 100644 --- a/src/include/utils/syscache.h +++ b/src/include/utils/syscache.h @@ -60,8 +60,6 @@ enum SysCacheIdentifier EVENTTRIGGEROID, EXTPROTOCOLOID, EXTPROTOCOLNAME, - FOREIGNCATALOGNAME, - FOREIGNCATALOGOID, FOREIGNDATAWRAPPERNAME, FOREIGNDATAWRAPPEROID, FOREIGNSERVERNAME, @@ -69,8 +67,6 @@ enum SysCacheIdentifier STORAGESERVERNAME, STORAGESERVEROID, FOREIGNTABLEREL, - FOREIGNVOLUMENAME, - FOREIGNVOLUMEOID, GPPOLICYID, AORELID, INDEXRELID, diff --git a/src/test/regress/expected/lake_table.out b/src/test/regress/expected/lake_table.out deleted file mode 100644 index 848789764c2..00000000000 --- a/src/test/regress/expected/lake_table.out +++ /dev/null @@ -1,304 +0,0 @@ --- --- Test lake table DDL: FOREIGN CATALOG, FOREIGN VOLUME, LAKE TABLE --- --- Display the lake table catalogs -\d+ pg_foreign_catalog - Table "pg_catalog.pg_foreign_catalog" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ------------+--------+-----------+----------+---------+----------+--------------+------------- - oid | oid | | not null | | plain | | - fcname | name | | not null | | plain | | - fcowner | oid | | not null | | plain | | - fcserver | oid | | not null | | plain | | - fctype | text | C | not null | | extended | | - fcoptions | text[] | C | | | extended | | -Indexes: - "pg_foreign_catalog_oid_index" PRIMARY KEY, btree (oid) - "pg_foreign_catalog_name_index" UNIQUE CONSTRAINT, btree (fcname) - -\d+ pg_foreign_volume - Table "pg_catalog.pg_foreign_volume" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ------------+--------+-----------+----------+---------+----------+--------------+------------- - oid | oid | | not null | | plain | | - fvname | name | | not null | | plain | | - fvowner | oid | | not null | | plain | | - fvserver | oid | | not null | | plain | | - fvoptions | text[] | C | | | extended | | -Indexes: - "pg_foreign_volume_oid_index" PRIMARY KEY, btree (oid) - "pg_foreign_volume_name_index" UNIQUE CONSTRAINT, btree (fvname) - -\d+ pg_lake_table - Table "pg_catalog.pg_lake_table" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description --------------------+------+-----------+----------+---------+---------+--------------+------------- - ltrelid | oid | | not null | | plain | | - ltforeign_catalog | oid | | not null | | plain | | - ltforeign_volume | oid | | not null | | plain | | -Indexes: - "pg_lake_table_relid_index" PRIMARY KEY, btree (ltrelid) - --- Setup: foreign servers for the catalogs and volumes to hang off -CREATE FOREIGN DATA WRAPPER lake_test_fdw; -CREATE SERVER lake_test_srv FOREIGN DATA WRAPPER lake_test_fdw; -CREATE SERVER lake_test_srv2 FOREIGN DATA WRAPPER lake_test_fdw; --- CREATE FOREIGN CATALOG: TYPE is a required first-class property -CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv TYPE 'hive' OPTIONS (uri 'thrift://localhost:9083'); -CREATE FOREIGN CATALOG lake_test_notype SERVER lake_test_srv; -- fail, TYPE is required -ERROR: syntax error at or near ";" -LINE 1: ...REATE FOREIGN CATALOG lake_test_notype SERVER lake_test_srv; - ^ -CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv TYPE 'hive'; -- fail, duplicate -ERROR: foreign catalog "lake_test_cat" already exists -CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv TYPE 'hive'; -- skip with notice -NOTICE: foreign catalog "lake_test_cat" already exists, skipping --- catalog names are global: the same name on another server is still a duplicate -CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv2 TYPE 'hive'; -- fail, duplicate -ERROR: foreign catalog "lake_test_cat" already exists -CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv2 TYPE 'hive'; -- skip with notice -NOTICE: foreign catalog "lake_test_cat" already exists, skipping -CREATE FOREIGN CATALOG lake_test_bad SERVER no_such_server TYPE 'hive'; -- fail, no server -ERROR: server "no_such_server" does not exist -SELECT fcname, fctype, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%'; - fcname | fctype | fcoptions ----------------+--------+------------------------------- - lake_test_cat | hive | {uri=thrift://localhost:9083} -(1 row) - --- CREATE FOREIGN VOLUME -CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv OPTIONS (base_path 's3://bucket/prefix'); -CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv; -- fail, duplicate -ERROR: foreign volume "lake_test_vol" already exists -CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv; -- skip with notice -NOTICE: foreign volume "lake_test_vol" already exists, skipping --- volume names are global: the same name on another server is still a duplicate -CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv2; -- fail, duplicate -ERROR: foreign volume "lake_test_vol" already exists -CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv2; -- skip with notice -NOTICE: foreign volume "lake_test_vol" already exists, skipping -CREATE FOREIGN VOLUME lake_test_bad SERVER no_such_server; -- fail, no server -ERROR: server "no_such_server" does not exist -SELECT fvname, fvoptions FROM pg_foreign_volume WHERE fvname LIKE 'lake\_test%'; - fvname | fvoptions ----------------+-------------------------------- - lake_test_vol | {base_path=s3://bucket/prefix} -(1 row) - --- Object descriptions -SELECT pg_catalog.pg_describe_object('pg_foreign_catalog'::regclass, oid, 0) - FROM pg_foreign_catalog WHERE fcname = 'lake_test_cat'; - pg_describe_object ------------------------ - catalog lake_test_cat -(1 row) - -SELECT pg_catalog.pg_describe_object('pg_foreign_volume'::regclass, oid, 0) - FROM pg_foreign_volume WHERE fvname = 'lake_test_vol'; - pg_describe_object ----------------------- - volume lake_test_vol -(1 row) - --- Catalog and volume rows are dispatched to all segments -SELECT count(DISTINCT gp_segment_id) > 1 AS on_all_segments - FROM gp_dist_random('pg_foreign_catalog') WHERE fcname = 'lake_test_cat'; - on_all_segments ------------------ - t -(1 row) - -SELECT count(DISTINCT gp_segment_id) > 1 AS on_all_segments - FROM gp_dist_random('pg_foreign_volume') WHERE fvname = 'lake_test_vol'; - on_all_segments ------------------ - t -(1 row) - --- Without a provider extension there is no iceberg table AM -CREATE LAKE TABLE lake_test_t0 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol; -- fail with hint -ERROR: table access method "iceberg" does not exist -HINT: CREATE LAKE TABLE ... USING "iceberg" requires an extension that provides the "iceberg" table access method. --- The default catalog/volume GUCs verify that the object exists -SET iceberg_default_catalog = 'no_such_catalog'; -- fail -ERROR: invalid value for parameter "iceberg_default_catalog": "no_such_catalog" -DETAIL: Foreign catalog "no_such_catalog" does not exist. -SET iceberg_default_volume = 'no_such_volume'; -- fail -ERROR: invalid value for parameter "iceberg_default_volume": "no_such_volume" -DETAIL: Foreign volume "no_such_volume" does not exist. --- Simulate a datalake provider with a heap-backed iceberg AM -CREATE ACCESS METHOD iceberg TYPE TABLE HANDLER heap_tableam_handler; --- CREATE LAKE TABLE with explicit catalog and volume. The format is the --- table's access method (pg_class.relam) and its options are the relation's --- reloptions; pg_lake_table records only the catalog/volume binding. -CREATE LAKE TABLE lake_test_t1 (a int, b text) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol; -SELECT c.relname, am.amname, c.reloptions, fc.fcname, fv.fvname - FROM pg_lake_table lt - JOIN pg_class c ON c.oid = lt.ltrelid - JOIN pg_am am ON am.oid = c.relam - JOIN pg_foreign_catalog fc ON fc.oid = lt.ltforeign_catalog - JOIN pg_foreign_volume fv ON fv.oid = lt.ltforeign_volume; - relname | amname | reloptions | fcname | fvname ---------------+---------+------------+---------------+--------------- - lake_test_t1 | iceberg | | lake_test_cat | lake_test_vol -(1 row) - --- Lake tables are always DISTRIBUTED RANDOMLY (policytype 'p', no distkey) -SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_test_t1'::regclass; - policytype | distkey -------------+--------- - p | -(1 row) - --- The (heap-backed) table is usable -INSERT INTO lake_test_t1 VALUES (1, 'x'), (2, 'y'); -SELECT count(*) FROM lake_test_t1; - count -------- - 2 -(1 row) - --- OPTIONS become the relation's reloptions and are validated by the access --- method: a value the AM accepts is stored, an unknown one is rejected. -CREATE LAKE TABLE lake_test_opt (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (fillfactor '70'); -SELECT reloptions FROM pg_class WHERE relname = 'lake_test_opt'; - reloptions ------------------ - {fillfactor=70} -(1 row) - -CREATE LAKE TABLE lake_test_optbad (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (bogus_opt 'x'); -- fail, AM rejects unknown option -ERROR: unrecognized parameter "bogus_opt" -DROP LAKE TABLE lake_test_opt; --- Catalog and volume are both required -CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG VOLUME lake_test_vol; -- fail, no catalog -ERROR: no foreign catalog specified -HINT: Specify CATALOG in CREATE LAKE TABLE or set iceberg_default_catalog. -CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG CATALOG lake_test_cat; -- fail, no volume -ERROR: no foreign volume specified -HINT: Specify VOLUME in CREATE LAKE TABLE or set iceberg_default_volume. --- ... unless the GUCs provide defaults -SET iceberg_default_catalog = 'lake_test_cat'; -SET iceberg_default_volume = 'lake_test_vol'; -CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG; -RESET iceberg_default_catalog; -RESET iceberg_default_volume; --- A DISTRIBUTED clause is rejected (lake tables are always distributed randomly) -CREATE LAKE TABLE lake_test_t3 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); -- fail -ERROR: syntax error at or near "DISTRIBUTED" -LINE 1: ...CEBERG CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTE... - ^ --- The USING clause names the table format; only ICEBERG is supported (any case) -CREATE LAKE TABLE lake_test_bad0 (a int) USING heap CATALOG lake_test_cat VOLUME lake_test_vol; -- fail, unsupported format -ERROR: unsupported lake table format "heap" -HINT: The only supported format is ICEBERG (USING ICEBERG). -CREATE LAKE TABLE lake_test_t4 (a int) USING "IceBerg" CATALOG lake_test_cat VOLUME lake_test_vol; -- quoted mixed-case format resolves the iceberg AM --- The iceberg AM is rejected for every path other than CREATE LAKE TABLE -CREATE TABLE lake_test_bad1 (a int) USING iceberg DISTRIBUTED RANDOMLY; -- fail -ERROR: cannot create table "lake_test_bad1" with access method "iceberg" -HINT: Use CREATE LAKE TABLE ... USING ICEBERG instead. -CREATE TABLE lake_test_bad2 USING iceberg AS SELECT 1 AS a DISTRIBUTED RANDOMLY; -- fail -ERROR: cannot create table "lake_test_bad2" with access method "iceberg" -HINT: Use CREATE LAKE TABLE ... USING ICEBERG instead. -SET default_table_access_method = iceberg; -CREATE TABLE lake_test_bad3 (a int) DISTRIBUTED RANDOMLY; -- fail -ERROR: cannot create table "lake_test_bad3" with access method "iceberg" -HINT: Use CREATE LAKE TABLE ... USING ICEBERG instead. -RESET default_table_access_method; -CREATE TABLE lake_test_heap (a int) DISTRIBUTED RANDOMLY; -ALTER TABLE lake_test_heap SET ACCESS METHOD iceberg; -- fail -ERROR: cannot change access method of table "lake_test_heap" to "iceberg" -HINT: Use CREATE LAKE TABLE ... USING ICEBERG to create a lake table. -ALTER TABLE lake_test_t1 SET ACCESS METHOD heap; -- fail -ERROR: cannot change access method of lake table "lake_test_t1" -ALTER TABLE lake_test_t1 SET DISTRIBUTED BY (a); -- fail -ERROR: cannot change distribution policy of lake table "lake_test_t1" -HINT: Lake tables must use DISTRIBUTED RANDOMLY because data is stored on object storage. --- Only the owner can drop a catalog or volume -CREATE ROLE regress_lake_user; -NOTICE: resource queue required -- using default resource queue "pg_default" -SET ROLE regress_lake_user; -DROP FOREIGN CATALOG lake_test_cat; -- fail, not owner -ERROR: must be owner of foreign catalog lake_test_cat -DROP FOREIGN VOLUME lake_test_vol; -- fail, not owner -ERROR: must be owner of foreign volume lake_test_vol -RESET ROLE; --- Dependencies: the server holds the catalog/volume, which hold the tables -DROP SERVER lake_test_srv; -- fail, catalog and volume depend on it -ERROR: cannot drop server lake_test_srv because other objects depend on it -DETAIL: catalog lake_test_cat depends on server lake_test_srv -volume lake_test_vol depends on server lake_test_srv -table lake_test_t1 depends on volume lake_test_vol -table lake_test_t2 depends on volume lake_test_vol -table lake_test_t4 depends on volume lake_test_vol -HINT: Use DROP ... CASCADE to drop the dependent objects too. -DROP FOREIGN CATALOG lake_test_cat; -- fail, tables depend on it -ERROR: cannot drop catalog lake_test_cat because other objects depend on it -DETAIL: table lake_test_t1 depends on catalog lake_test_cat -table lake_test_t2 depends on catalog lake_test_cat -table lake_test_t4 depends on catalog lake_test_cat -HINT: Use DROP ... CASCADE to drop the dependent objects too. --- Dropping a lake table removes its pg_lake_table entry: remember the --- table's OID so the check still finds an orphaned row after the drop -SELECT oid AS t1_oid FROM pg_class WHERE relname = 'lake_test_t1' \gset -DROP LAKE TABLE lake_test_t1; -SELECT count(*) FROM pg_lake_table WHERE ltrelid = :t1_oid; - count -------- - 0 -(1 row) - --- DROP LAKE TABLE rejects a non-lake table ... -DROP LAKE TABLE lake_test_heap; -- fail, not a lake table -ERROR: "lake_test_heap" is not a lake table -HINT: Use DROP TABLE to remove a table. --- plain DROP TABLE must reject a lake table (mirrors foreign-table behavior) -DROP TABLE lake_test_t2; -ERROR: "lake_test_t2" is not a table -HINT: Use DROP LAKE TABLE to remove a lake table. -DROP TABLE IF EXISTS lake_test_t2; -- IF EXISTS does not suppress wrong-type errors -ERROR: "lake_test_t2" is not a table -HINT: Use DROP LAKE TABLE to remove a lake table. --- the rejected drops must have left the table and its lake metadata intact -SELECT count(*) FROM pg_lake_table WHERE ltrelid = 'lake_test_t2'::regclass; - count -------- - 1 -(1 row) - --- the correct command still works -SELECT oid AS t2_oid FROM pg_class WHERE relname = 'lake_test_t2' \gset -DROP LAKE TABLE lake_test_t2; -SELECT count(*) FROM pg_lake_table WHERE ltrelid = :t2_oid; - count -------- - 0 -(1 row) - --- DROP FOREIGN CATALOG ... CASCADE takes the remaining tables with it -SELECT oid AS t4_oid FROM pg_class WHERE relname = 'lake_test_t4' \gset -DROP FOREIGN CATALOG lake_test_cat CASCADE; -NOTICE: drop cascades to table lake_test_t4 -SELECT count(*) FROM pg_lake_table WHERE ltrelid = :t4_oid; - count -------- - 0 -(1 row) - --- DROP variants -DROP FOREIGN CATALOG lake_test_cat; -- fail, already gone -ERROR: foreign catalog "lake_test_cat" does not exist -DROP FOREIGN CATALOG IF EXISTS lake_test_cat; -- skip with notice -NOTICE: foreign catalog "lake_test_cat" does not exist, skipping -DROP FOREIGN VOLUME lake_test_vol; -DROP FOREIGN VOLUME lake_test_vol; -- fail, already gone -ERROR: foreign volume "lake_test_vol" does not exist -DROP FOREIGN VOLUME IF EXISTS lake_test_vol; -- skip with notice -NOTICE: foreign volume "lake_test_vol" does not exist, skipping --- Cleanup -DROP TABLE lake_test_heap; -DROP ROLE regress_lake_user; -DROP SERVER lake_test_srv; -DROP SERVER lake_test_srv2; -DROP FOREIGN DATA WRAPPER lake_test_fdw; -DROP ACCESS METHOD iceberg; diff --git a/src/test/regress/expected/oidjoins.out b/src/test/regress/expected/oidjoins.out index 6dc91c68aec..19094e111dc 100644 --- a/src/test/regress/expected/oidjoins.out +++ b/src/test/regress/expected/oidjoins.out @@ -235,10 +235,6 @@ NOTICE: checking pg_foreign_data_wrapper {fdwhandler} => pg_proc {oid} NOTICE: checking pg_foreign_data_wrapper {fdwvalidator} => pg_proc {oid} NOTICE: checking pg_foreign_server {srvowner} => pg_authid {oid} NOTICE: checking pg_foreign_server {srvfdw} => pg_foreign_data_wrapper {oid} -NOTICE: checking pg_foreign_catalog {fcowner} => pg_authid {oid} -NOTICE: checking pg_foreign_catalog {fcserver} => pg_foreign_server {oid} -NOTICE: checking pg_foreign_volume {fvowner} => pg_authid {oid} -NOTICE: checking pg_foreign_volume {fvserver} => pg_foreign_server {oid} NOTICE: checking pg_user_mapping {umuser} => pg_authid {oid} NOTICE: checking pg_user_mapping {umserver} => pg_foreign_server {oid} NOTICE: checking pg_compression {compconstructor} => pg_proc {oid} @@ -251,9 +247,6 @@ NOTICE: checking pg_foreign_table {ftrelid} => pg_class {oid} NOTICE: checking pg_foreign_table {ftserver} => pg_foreign_server {oid} NOTICE: checking pg_foreign_table_seg {ftsrelid} => pg_class {oid} NOTICE: checking pg_foreign_table_seg {ftsserver} => pg_foreign_server {oid} -NOTICE: checking pg_lake_table {ltrelid} => pg_class {oid} -NOTICE: checking pg_lake_table {ltforeign_catalog} => pg_foreign_catalog {oid} -NOTICE: checking pg_lake_table {ltforeign_volume} => pg_foreign_volume {oid} NOTICE: checking pg_policy {polrelid} => pg_class {oid} NOTICE: checking pg_policy {polroles} => pg_authid {oid} NOTICE: checking pg_default_acl {defaclrole} => pg_authid {oid} diff --git a/src/test/regress/expected/sanity_check.out b/src/test/regress/expected/sanity_check.out index f5d55362906..4625d100bf0 100644 --- a/src/test/regress/expected/sanity_check.out +++ b/src/test/regress/expected/sanity_check.out @@ -124,16 +124,13 @@ pg_enum|t pg_event_trigger|t pg_extension|t pg_extprotocol|t -pg_foreign_catalog|t pg_foreign_data_wrapper|t pg_foreign_server|t pg_foreign_table|t pg_foreign_table_seg|t -pg_foreign_volume|t pg_index|t pg_inherits|t pg_init_privs|t -pg_lake_table|t pg_language|t pg_largeobject|t pg_largeobject_metadata|t diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index 44a916cc218..84e8766844b 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -358,9 +358,6 @@ test: am_encoding # tests of directory table test: directory_table -# tests of lake table DDL (foreign catalog, foreign volume, iceberg table) -test: lake_table - # test if motion sockets are created with the gp_segment_configuration.address test: motion_socket diff --git a/src/test/regress/sql/lake_table.sql b/src/test/regress/sql/lake_table.sql deleted file mode 100644 index deb5dba38bd..00000000000 --- a/src/test/regress/sql/lake_table.sql +++ /dev/null @@ -1,158 +0,0 @@ --- --- Test lake table DDL: FOREIGN CATALOG, FOREIGN VOLUME, LAKE TABLE --- - --- Display the lake table catalogs -\d+ pg_foreign_catalog -\d+ pg_foreign_volume -\d+ pg_lake_table - --- Setup: foreign servers for the catalogs and volumes to hang off -CREATE FOREIGN DATA WRAPPER lake_test_fdw; -CREATE SERVER lake_test_srv FOREIGN DATA WRAPPER lake_test_fdw; -CREATE SERVER lake_test_srv2 FOREIGN DATA WRAPPER lake_test_fdw; - --- CREATE FOREIGN CATALOG: TYPE is a required first-class property -CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv TYPE 'hive' OPTIONS (uri 'thrift://localhost:9083'); -CREATE FOREIGN CATALOG lake_test_notype SERVER lake_test_srv; -- fail, TYPE is required -CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv TYPE 'hive'; -- fail, duplicate -CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv TYPE 'hive'; -- skip with notice --- catalog names are global: the same name on another server is still a duplicate -CREATE FOREIGN CATALOG lake_test_cat SERVER lake_test_srv2 TYPE 'hive'; -- fail, duplicate -CREATE FOREIGN CATALOG IF NOT EXISTS lake_test_cat SERVER lake_test_srv2 TYPE 'hive'; -- skip with notice -CREATE FOREIGN CATALOG lake_test_bad SERVER no_such_server TYPE 'hive'; -- fail, no server -SELECT fcname, fctype, fcoptions FROM pg_foreign_catalog WHERE fcname LIKE 'lake\_test%'; - --- CREATE FOREIGN VOLUME -CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv OPTIONS (base_path 's3://bucket/prefix'); -CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv; -- fail, duplicate -CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv; -- skip with notice --- volume names are global: the same name on another server is still a duplicate -CREATE FOREIGN VOLUME lake_test_vol SERVER lake_test_srv2; -- fail, duplicate -CREATE FOREIGN VOLUME IF NOT EXISTS lake_test_vol SERVER lake_test_srv2; -- skip with notice -CREATE FOREIGN VOLUME lake_test_bad SERVER no_such_server; -- fail, no server -SELECT fvname, fvoptions FROM pg_foreign_volume WHERE fvname LIKE 'lake\_test%'; - --- Object descriptions -SELECT pg_catalog.pg_describe_object('pg_foreign_catalog'::regclass, oid, 0) - FROM pg_foreign_catalog WHERE fcname = 'lake_test_cat'; -SELECT pg_catalog.pg_describe_object('pg_foreign_volume'::regclass, oid, 0) - FROM pg_foreign_volume WHERE fvname = 'lake_test_vol'; - --- Catalog and volume rows are dispatched to all segments -SELECT count(DISTINCT gp_segment_id) > 1 AS on_all_segments - FROM gp_dist_random('pg_foreign_catalog') WHERE fcname = 'lake_test_cat'; -SELECT count(DISTINCT gp_segment_id) > 1 AS on_all_segments - FROM gp_dist_random('pg_foreign_volume') WHERE fvname = 'lake_test_vol'; - --- Without a provider extension there is no iceberg table AM -CREATE LAKE TABLE lake_test_t0 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol; -- fail with hint - --- The default catalog/volume GUCs verify that the object exists -SET iceberg_default_catalog = 'no_such_catalog'; -- fail -SET iceberg_default_volume = 'no_such_volume'; -- fail - --- Simulate a datalake provider with a heap-backed iceberg AM -CREATE ACCESS METHOD iceberg TYPE TABLE HANDLER heap_tableam_handler; - --- CREATE LAKE TABLE with explicit catalog and volume. The format is the --- table's access method (pg_class.relam) and its options are the relation's --- reloptions; pg_lake_table records only the catalog/volume binding. -CREATE LAKE TABLE lake_test_t1 (a int, b text) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol; -SELECT c.relname, am.amname, c.reloptions, fc.fcname, fv.fvname - FROM pg_lake_table lt - JOIN pg_class c ON c.oid = lt.ltrelid - JOIN pg_am am ON am.oid = c.relam - JOIN pg_foreign_catalog fc ON fc.oid = lt.ltforeign_catalog - JOIN pg_foreign_volume fv ON fv.oid = lt.ltforeign_volume; --- Lake tables are always DISTRIBUTED RANDOMLY (policytype 'p', no distkey) -SELECT policytype, distkey FROM gp_distribution_policy WHERE localoid = 'lake_test_t1'::regclass; - --- The (heap-backed) table is usable -INSERT INTO lake_test_t1 VALUES (1, 'x'), (2, 'y'); -SELECT count(*) FROM lake_test_t1; - --- OPTIONS become the relation's reloptions and are validated by the access --- method: a value the AM accepts is stored, an unknown one is rejected. -CREATE LAKE TABLE lake_test_opt (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (fillfactor '70'); -SELECT reloptions FROM pg_class WHERE relname = 'lake_test_opt'; -CREATE LAKE TABLE lake_test_optbad (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol OPTIONS (bogus_opt 'x'); -- fail, AM rejects unknown option -DROP LAKE TABLE lake_test_opt; - --- Catalog and volume are both required -CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG VOLUME lake_test_vol; -- fail, no catalog -CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG CATALOG lake_test_cat; -- fail, no volume - --- ... unless the GUCs provide defaults -SET iceberg_default_catalog = 'lake_test_cat'; -SET iceberg_default_volume = 'lake_test_vol'; -CREATE LAKE TABLE lake_test_t2 (a int) USING ICEBERG; -RESET iceberg_default_catalog; -RESET iceberg_default_volume; - --- A DISTRIBUTED clause is rejected (lake tables are always distributed randomly) -CREATE LAKE TABLE lake_test_t3 (a int) USING ICEBERG CATALOG lake_test_cat VOLUME lake_test_vol DISTRIBUTED BY (a); -- fail - --- The USING clause names the table format; only ICEBERG is supported (any case) -CREATE LAKE TABLE lake_test_bad0 (a int) USING heap CATALOG lake_test_cat VOLUME lake_test_vol; -- fail, unsupported format -CREATE LAKE TABLE lake_test_t4 (a int) USING "IceBerg" CATALOG lake_test_cat VOLUME lake_test_vol; -- quoted mixed-case format resolves the iceberg AM - --- The iceberg AM is rejected for every path other than CREATE LAKE TABLE -CREATE TABLE lake_test_bad1 (a int) USING iceberg DISTRIBUTED RANDOMLY; -- fail -CREATE TABLE lake_test_bad2 USING iceberg AS SELECT 1 AS a DISTRIBUTED RANDOMLY; -- fail -SET default_table_access_method = iceberg; -CREATE TABLE lake_test_bad3 (a int) DISTRIBUTED RANDOMLY; -- fail -RESET default_table_access_method; -CREATE TABLE lake_test_heap (a int) DISTRIBUTED RANDOMLY; -ALTER TABLE lake_test_heap SET ACCESS METHOD iceberg; -- fail -ALTER TABLE lake_test_t1 SET ACCESS METHOD heap; -- fail -ALTER TABLE lake_test_t1 SET DISTRIBUTED BY (a); -- fail - --- Only the owner can drop a catalog or volume -CREATE ROLE regress_lake_user; -SET ROLE regress_lake_user; -DROP FOREIGN CATALOG lake_test_cat; -- fail, not owner -DROP FOREIGN VOLUME lake_test_vol; -- fail, not owner -RESET ROLE; - --- Dependencies: the server holds the catalog/volume, which hold the tables -DROP SERVER lake_test_srv; -- fail, catalog and volume depend on it -DROP FOREIGN CATALOG lake_test_cat; -- fail, tables depend on it - --- Dropping a lake table removes its pg_lake_table entry: remember the --- table's OID so the check still finds an orphaned row after the drop -SELECT oid AS t1_oid FROM pg_class WHERE relname = 'lake_test_t1' \gset -DROP LAKE TABLE lake_test_t1; -SELECT count(*) FROM pg_lake_table WHERE ltrelid = :t1_oid; - --- DROP LAKE TABLE rejects a non-lake table ... -DROP LAKE TABLE lake_test_heap; -- fail, not a lake table --- plain DROP TABLE must reject a lake table (mirrors foreign-table behavior) -DROP TABLE lake_test_t2; -DROP TABLE IF EXISTS lake_test_t2; -- IF EXISTS does not suppress wrong-type errors --- the rejected drops must have left the table and its lake metadata intact -SELECT count(*) FROM pg_lake_table WHERE ltrelid = 'lake_test_t2'::regclass; --- the correct command still works -SELECT oid AS t2_oid FROM pg_class WHERE relname = 'lake_test_t2' \gset -DROP LAKE TABLE lake_test_t2; -SELECT count(*) FROM pg_lake_table WHERE ltrelid = :t2_oid; - --- DROP FOREIGN CATALOG ... CASCADE takes the remaining tables with it -SELECT oid AS t4_oid FROM pg_class WHERE relname = 'lake_test_t4' \gset -DROP FOREIGN CATALOG lake_test_cat CASCADE; -SELECT count(*) FROM pg_lake_table WHERE ltrelid = :t4_oid; - --- DROP variants -DROP FOREIGN CATALOG lake_test_cat; -- fail, already gone -DROP FOREIGN CATALOG IF EXISTS lake_test_cat; -- skip with notice -DROP FOREIGN VOLUME lake_test_vol; -DROP FOREIGN VOLUME lake_test_vol; -- fail, already gone -DROP FOREIGN VOLUME IF EXISTS lake_test_vol; -- skip with notice - --- Cleanup -DROP TABLE lake_test_heap; -DROP ROLE regress_lake_user; -DROP SERVER lake_test_srv; -DROP SERVER lake_test_srv2; -DROP FOREIGN DATA WRAPPER lake_test_fdw; -DROP ACCESS METHOD iceberg; diff --git a/src/test/singlenode_regress/expected/oidjoins.out b/src/test/singlenode_regress/expected/oidjoins.out index b6bb1a499ce..b1dca18dc97 100644 --- a/src/test/singlenode_regress/expected/oidjoins.out +++ b/src/test/singlenode_regress/expected/oidjoins.out @@ -235,10 +235,6 @@ NOTICE: checking pg_foreign_data_wrapper {fdwhandler} => pg_proc {oid} NOTICE: checking pg_foreign_data_wrapper {fdwvalidator} => pg_proc {oid} NOTICE: checking pg_foreign_server {srvowner} => pg_authid {oid} NOTICE: checking pg_foreign_server {srvfdw} => pg_foreign_data_wrapper {oid} -NOTICE: checking pg_foreign_catalog {fcowner} => pg_authid {oid} -NOTICE: checking pg_foreign_catalog {fcserver} => pg_foreign_server {oid} -NOTICE: checking pg_foreign_volume {fvowner} => pg_authid {oid} -NOTICE: checking pg_foreign_volume {fvserver} => pg_foreign_server {oid} NOTICE: checking pg_user_mapping {umuser} => pg_authid {oid} NOTICE: checking pg_user_mapping {umserver} => pg_foreign_server {oid} NOTICE: checking pg_compression {compconstructor} => pg_proc {oid} @@ -251,9 +247,6 @@ NOTICE: checking pg_foreign_table {ftrelid} => pg_class {oid} NOTICE: checking pg_foreign_table {ftserver} => pg_foreign_server {oid} NOTICE: checking pg_foreign_table_seg {ftsrelid} => pg_class {oid} NOTICE: checking pg_foreign_table_seg {ftsserver} => pg_foreign_server {oid} -NOTICE: checking pg_lake_table {ltrelid} => pg_class {oid} -NOTICE: checking pg_lake_table {ltforeign_catalog} => pg_foreign_catalog {oid} -NOTICE: checking pg_lake_table {ltforeign_volume} => pg_foreign_volume {oid} NOTICE: checking pg_policy {polrelid} => pg_class {oid} NOTICE: checking pg_policy {polroles} => pg_authid {oid} NOTICE: checking pg_default_acl {defaclrole} => pg_authid {oid} diff --git a/src/test/singlenode_regress/expected/sanity_check.out b/src/test/singlenode_regress/expected/sanity_check.out index 844f094fa57..a5a122ac32d 100644 --- a/src/test/singlenode_regress/expected/sanity_check.out +++ b/src/test/singlenode_regress/expected/sanity_check.out @@ -135,16 +135,13 @@ pg_enum|t pg_event_trigger|t pg_extension|t pg_extprotocol|t -pg_foreign_catalog|t pg_foreign_data_wrapper|t pg_foreign_server|t pg_foreign_table|t pg_foreign_table_seg|t -pg_foreign_volume|t pg_index|t pg_inherits|t pg_init_privs|t -pg_lake_table|t pg_language|t pg_largeobject|t pg_largeobject_metadata|t From b671a35f3ea0a391ea0397e7110079c45b6c0846 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Fri, 31 Jul 2026 09:38:01 +0800 Subject: [PATCH 27/29] datalake_fdw: Apache Iceberg lake tables as an extension (skeleton) Add contrib/datalake_fdw, a skeleton for Iceberg lake-table support that needs no kernel changes: a lake table is an ordinary CREATE TABLE ... USING iceberg, and it names a catalog server and a volume server in its reloptions, both created through foreign-data wrappers this extension registers. Mapping a table to a pair of foreign servers is what keeps the kernel out of it. Server options, ownership, privileges and dump/restore already exist for foreign servers, reloptions already reach every segment with pg_class, and recording the two servers in pg_depend makes DROP SERVER refuse to strand a table -- none of which needs new catalogs or grammar. Where that mapping is kept is one function's business. Every operation reads it through pg_iceberg_get_table_info(), whose signature and result types match the existing datalake_fdw implementation this work is the upstream half of -- that one keeps the same mapping in a system catalog of its own, which an extension cannot add. Holding the interface still means the layers above it are the same code on both sides, and that the storage can be reconsidered later without touching a caller. The DDL path is complete against a stub metadata engine, so CREATE TABLE and DROP TABLE work end to end with no catalog service, object store, Arrow or JVM in the picture. Everything that would touch data reports a clean "iceberg: is not supported yet". The interfaces the later work plugs into ship whole so they can be reviewed before there is an implementation behind them: the IcebergMetaEngine vtable with a capability bitmap the registry validates and dispatches through, the FormatReader/FormatWriter instance interfaces, and the storage facade over open/read/write/list. Details worth a reviewer's attention: * Table metadata always goes through one engine, the Java agent, and nothing selects between implementations -- no option, no setting. The vtable stays because the implementation is expected to change; that is a property of the build, never of a table or a session, so an existing table can never be reinterpreted by a configuration change. * The table access method fills every callback GetTableAmRoutine() asserts. ANALYZE succeeds as a zero-sample no-op through relation_acquire_sample_rows, which keeps it off the scan path that reports not-supported, and VACUUM is a no-op, so database-wide maintenance never dies on a lake table. * The object-access hook records the server dependencies on the coordinator and on every segment, while only GP_ROLE_DISPATCH calls the metadata engine, so each node can protect its own catalog and the remote side sees one call. Utility-mode DDL is refused rather than creating local state without dispatch. * VACUUM FULL is refused in the utility hook, not in the access method: relation rewriting creates a transient relation first, which reaches OAT_POST_CREATE and has the engine create a table remotely before the rewrite reports its error, leaving an orphan behind. * Credentials are refused in server options and belong in user mappings, which stay optional so ambient object-store credentials remain usable. Binding resolution never reads them, so DDL and DROP work with none configured. * Volume URIs are parsed once, in the options layer, into a versioned DatalakeLocation; backends receive only that canonical form. * Option names are macros in per-wrapper option modules, next to the typed struct each one parses into and the per-catalog-type parse function that fills it, so that support for a further catalog or storage protocol is an addition rather than a rewrite. Option lookup itself is one shared set of accessors. The keys users write are Apache Iceberg's -- uri, warehouse, and rest for a catalog reached over the REST protocol -- because the specification defines one protocol that several implementations answer, and an SQL surface tied to one of them would make every other one need a second spelling. polaris is accepted as an alias of rest, since that is what the existing implementation calls it. The macro names, struct names and field names stay that implementation's, so the divergence is one string per key rather than a different shape. * A DlErrCode says which kind of failure occurred and nothing else, which is not enough to diagnose one -- a remote catalog's message, its own error class, and a stack from wherever it threw have to arrive somewhere. Implementations record that alongside the code they return, and the entry points facing PostgreSQL turn both into one report: the message as DETAIL, a stack only for a session that asked for log-level detail. Recording allocates nothing and raises nothing, so a cleanup path crossing back from C++ can use it. The SQLSTATE follows the code rather than being internal_error throughout, which also keeps a source location out of user-visible output. * C++ translation units reach the server headers through common/dl_pg_api.h, which applies extern "C" -- without it the module builds and then fails to dlopen on a mangled errmsg. The C/C++ boundary macros follow the PAX pattern, including deferring ereport() until after the catch handler is left, since longjmp() out of a handler is undefined. Exported symbols are limited to the PG entry points listed in exports.txt, ELF and Mach-O each getting the right linker mechanism, so a future static Arrow cannot leak into other extensions. * A schema-level dump round-trips. pg_dump writes DISTRIBUTED RANDOMLY and ALTER TABLE ... OWNER TO for a table like this, so both are accepted -- a guard that refuses what this module's own dump emits refuses to restore it. Neither can desynchronise anything: the distribution clause asks for the policy that would have been injected anyway, and ownership is local catalog state. Every other ALTER form, and a distribution clause naming columns, stay refused. Dumping the *contents* of a lake table still fails, because scanning does; a full pg_dump of a database containing one therefore does not work yet, and what a dump of externally owned table data should even mean is the open question behind that. Test material lives under test/automation, one directory per category, with the module's Makefile pointing pg_regress at the category that needs no external service; make installcheck from the module and make test from the harness run the same cases. Testing against a real catalog or object store cannot be done by comparing against a recorded transcript, so the harness is what those categories will be added to, and it already reports a category whose services are absent as skipped rather than passed. The suite covers the DDL path including per-segment catalog state, the rejection matrices and the privilege model; installcheck is green on a three-segment cluster and does not depend on the order the cases run in. Per-segment assertions compare against gp_segment_configuration rather than naming segments, so they hold on a cluster of any size, and each guard has a case showing what it does *not* refuse -- renaming a schema that holds no lake table, for instance -- because a guard wider than its problem passes its own tests just as well. CI runs the suite as its own matrix entry, ic-datalake-fdw, whose demo cluster is created with shared_preload_libraries='datalake_fdw' -- the module installs process-wide hooks, so _PG_init refuses to load any other way, and a generic cluster could not run these cases at all. That is the same mechanism two existing entries already use. ("make check" would need the temp-config this module also ships; it exists in-tree only, since PGXS refuses the target.) The error channel has no coverage yet: no statement can make the stub engine fail, so the first implementation that can fail is what brings a case for it. --- .github/workflows/build-cloudberry.yml | 4 + contrib/Makefile | 1 + contrib/datalake_fdw/.gitignore | 8 + contrib/datalake_fdw/Makefile | 109 ++ contrib/datalake_fdw/datalake_fdw--1.0.sql | 40 + contrib/datalake_fdw/datalake_fdw.conf | 25 + contrib/datalake_fdw/datalake_fdw.control | 23 + contrib/datalake_fdw/exports.txt | 32 + .../src/am_iceberg/pg_iceberg_am_handler.c | 509 +++++++++ .../src/am_iceberg/pg_iceberg_ddl.c | 302 ++++++ .../src/am_iceberg/pg_iceberg_ddl.h | 42 + .../src/am_iceberg/pg_iceberg_extensible.c | 964 ++++++++++++++++++ .../src/am_iceberg/pg_iceberg_guc.c | 67 ++ .../src/am_iceberg/pg_iceberg_guc.h | 37 + .../src/am_iceberg/pg_iceberg_options.c | 616 +++++++++++ .../src/am_iceberg/pg_iceberg_options.h | 155 +++ .../src/am_iceberg/pg_iceberg_reject.c | 44 + .../src/am_iceberg/pg_iceberg_reject.h | 36 + .../src/common/backend_registry.cpp | 122 +++ .../src/common/backend_registry.h | 100 ++ .../src/common/datalake_location.h | 50 + contrib/datalake_fdw/src/common/dl_err.c | 218 ++++ contrib/datalake_fdw/src/common/dl_err.h | 120 +++ contrib/datalake_fdw/src/common/dl_kv.h | 44 + .../datalake_fdw/src/common/dl_option_util.c | 65 ++ .../datalake_fdw/src/common/dl_option_util.h | 63 ++ contrib/datalake_fdw/src/common/dl_pg_api.h | 45 + contrib/datalake_fdw/src/common/dl_wrappers.h | 193 ++++ .../src/common/file_system_wrapper.cpp | 232 +++++ .../src/common/file_system_wrapper.h | 113 ++ .../datalake_fdw/src/common/parser_option.c | 94 ++ .../datalake_fdw/src/common/parser_option.h | 54 + .../src/common/s3_file_system.cpp | 261 +++++ contrib/datalake_fdw/src/format/format.h | 117 +++ .../datalake_fdw/src/format/format_registry.c | 48 + .../iceberg_catalog_fdw/iceberg_catalog_fdw.c | 226 ++++ .../iceberg_catalog_option.c | 185 ++++ .../iceberg_catalog_option.h | 171 ++++ .../iceberg_volume_fdw/iceberg_volume_fdw.c | 157 +++ .../iceberg_volume_option.c | 94 ++ .../iceberg_volume_option.h | 146 +++ .../src/meta/engine_stub/stub_engine.c | 102 ++ .../src/meta/engine_stub/stub_engine.h | 36 + .../src/meta/iceberg_meta_engine.h | 149 +++ .../datalake_fdw/src/meta/meta_engine_init.c | 49 + .../datalake_fdw/src/meta/meta_engine_init.h | 34 + .../src/meta/meta_engine_registry.c | 245 +++++ contrib/datalake_fdw/test/automation/Makefile | 52 + .../datalake_fdw/test/automation/README.md | 80 ++ .../test/automation/config/test_config.env | 39 + .../scripts/setup/check_services.sh | 60 ++ .../scripts/test/run_smoke_tests.sh | 120 +++ .../scripts/utils/common_functions.sh | 100 ++ .../iceberg_am/expected/iceberg_am_acl.out | 75 ++ .../iceberg_am/expected/iceberg_am_ddl.out | 227 +++++ .../iceberg_am/expected/iceberg_am_reject.out | 457 +++++++++ .../smoke/iceberg_am/sql/iceberg_am_acl.sql | 78 ++ .../smoke/iceberg_am/sql/iceberg_am_ddl.sql | 162 +++ .../iceberg_am/sql/iceberg_am_reject.sql | 379 +++++++ 59 files changed, 8376 insertions(+) create mode 100644 contrib/datalake_fdw/.gitignore create mode 100644 contrib/datalake_fdw/Makefile create mode 100644 contrib/datalake_fdw/datalake_fdw--1.0.sql create mode 100644 contrib/datalake_fdw/datalake_fdw.conf create mode 100644 contrib/datalake_fdw/datalake_fdw.control create mode 100644 contrib/datalake_fdw/exports.txt create mode 100644 contrib/datalake_fdw/src/am_iceberg/pg_iceberg_am_handler.c create mode 100644 contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.c create mode 100644 contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.h create mode 100644 contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c create mode 100644 contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c create mode 100644 contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h create mode 100644 contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c create mode 100644 contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h create mode 100644 contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.c create mode 100644 contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.h create mode 100644 contrib/datalake_fdw/src/common/backend_registry.cpp create mode 100644 contrib/datalake_fdw/src/common/backend_registry.h create mode 100644 contrib/datalake_fdw/src/common/datalake_location.h create mode 100644 contrib/datalake_fdw/src/common/dl_err.c create mode 100644 contrib/datalake_fdw/src/common/dl_err.h create mode 100644 contrib/datalake_fdw/src/common/dl_kv.h create mode 100644 contrib/datalake_fdw/src/common/dl_option_util.c create mode 100644 contrib/datalake_fdw/src/common/dl_option_util.h create mode 100644 contrib/datalake_fdw/src/common/dl_pg_api.h create mode 100644 contrib/datalake_fdw/src/common/dl_wrappers.h create mode 100644 contrib/datalake_fdw/src/common/file_system_wrapper.cpp create mode 100644 contrib/datalake_fdw/src/common/file_system_wrapper.h create mode 100644 contrib/datalake_fdw/src/common/parser_option.c create mode 100644 contrib/datalake_fdw/src/common/parser_option.h create mode 100644 contrib/datalake_fdw/src/common/s3_file_system.cpp create mode 100644 contrib/datalake_fdw/src/format/format.h create mode 100644 contrib/datalake_fdw/src/format/format_registry.c create mode 100644 contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_fdw.c create mode 100644 contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.c create mode 100644 contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.h create mode 100644 contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c create mode 100644 contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.c create mode 100644 contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h create mode 100644 contrib/datalake_fdw/src/meta/engine_stub/stub_engine.c create mode 100644 contrib/datalake_fdw/src/meta/engine_stub/stub_engine.h create mode 100644 contrib/datalake_fdw/src/meta/iceberg_meta_engine.h create mode 100644 contrib/datalake_fdw/src/meta/meta_engine_init.c create mode 100644 contrib/datalake_fdw/src/meta/meta_engine_init.h create mode 100644 contrib/datalake_fdw/src/meta/meta_engine_registry.c create mode 100644 contrib/datalake_fdw/test/automation/Makefile create mode 100644 contrib/datalake_fdw/test/automation/README.md create mode 100644 contrib/datalake_fdw/test/automation/config/test_config.env create mode 100755 contrib/datalake_fdw/test/automation/scripts/setup/check_services.sh create mode 100755 contrib/datalake_fdw/test/automation/scripts/test/run_smoke_tests.sh create mode 100644 contrib/datalake_fdw/test/automation/scripts/utils/common_functions.sh create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_acl.out create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_ddl.out create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_acl.sql create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_ddl.sql create mode 100644 contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_reject.sql diff --git a/.github/workflows/build-cloudberry.yml b/.github/workflows/build-cloudberry.yml index 6289785bb14..bceeaeb3e02 100644 --- a/.github/workflows/build-cloudberry.yml +++ b/.github/workflows/build-cloudberry.yml @@ -317,6 +317,10 @@ jobs: "contrib/pg_buffercache:installcheck", "contrib/sslinfo:installcheck"] }, + {"test":"ic-datalake-fdw", + "make_configs":["contrib/datalake_fdw:installcheck"], + "shared_preload_libraries":"datalake_fdw" + }, {"test":"ic-gpcontrib", "make_configs":["gpcontrib/orafce:installcheck", "gpcontrib/zstd:installcheck", diff --git a/contrib/Makefile b/contrib/Makefile index 3a2591e0366..c2a1d396bb8 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -25,6 +25,7 @@ SUBDIRS = \ btree_gin \ btree_gist \ citext \ + datalake_fdw \ dblink \ dict_int \ dict_xsyn \ diff --git a/contrib/datalake_fdw/.gitignore b/contrib/datalake_fdw/.gitignore new file mode 100644 index 00000000000..e769026571a --- /dev/null +++ b/contrib/datalake_fdw/.gitignore @@ -0,0 +1,8 @@ +# Generated subdirectories +/log/ +/results/ +/tmp_check/ + +# Generated from exports.txt at build time +/exports.map +/exports_darwin.list diff --git a/contrib/datalake_fdw/Makefile b/contrib/datalake_fdw/Makefile new file mode 100644 index 00000000000..bd1d0179513 --- /dev/null +++ b/contrib/datalake_fdw/Makefile @@ -0,0 +1,109 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# contrib/datalake_fdw/Makefile + +MODULE_big = datalake_fdw +EXTENSION = datalake_fdw +DATA = datalake_fdw--1.0.sql + +OBJS = \ + src/am_iceberg/pg_iceberg_am_handler.o \ + src/am_iceberg/pg_iceberg_extensible.o \ + src/am_iceberg/pg_iceberg_ddl.o \ + src/am_iceberg/pg_iceberg_options.o \ + src/am_iceberg/pg_iceberg_guc.o \ + src/am_iceberg/pg_iceberg_reject.o \ + src/iceberg_catalog_fdw/iceberg_catalog_fdw.o \ + src/iceberg_catalog_fdw/iceberg_catalog_option.o \ + src/iceberg_volume_fdw/iceberg_volume_fdw.o \ + src/iceberg_volume_fdw/iceberg_volume_option.o \ + src/meta/meta_engine_registry.o \ + src/meta/meta_engine_init.o \ + src/meta/engine_stub/stub_engine.o \ + src/format/format_registry.o \ + src/common/dl_err.o \ + src/common/dl_option_util.o \ + src/common/parser_option.o \ + src/common/file_system_wrapper.o \ + src/common/s3_file_system.o \ + src/common/backend_registry.o + +# Use the documented PGXS knobs: pgxs.mk appends these AFTER the flags configure +# chose, so optimization/warning settings survive. A pre-include +# "override CFLAGS +=" would give CFLAGS override origin and silently discard +# Makefile.global's own "CFLAGS = @CFLAGS@" assignment. +PG_CFLAGS = -fvisibility=hidden +PG_CXXFLAGS = -fvisibility=hidden -fvisibility-inlines-hidden -std=c++17 +PG_CPPFLAGS = -I$(srcdir)/src + +# The regression cases live with the rest of the test material rather than in a +# second place of their own; pg_regress is pointed at them. REGRESS_OPTS is +# expanded after the --inputdir that Makefile.global supplies, so this wins. +# +# _PG_init refuses to run outside shared_preload_libraries, so any server used +# to test this module has to be started with it. For an in-tree "make check" +# that is what the temp-config supplies. For "make installcheck" -- which is +# what CI runs, against a cluster created with the library already preloaded -- +# pg_regress ignores it. Note that "make check" exists in-tree only; under PGXS +# pgxs.mk refuses the target outright. +REGRESS = iceberg_am_ddl iceberg_am_reject iceberg_am_acl +REGRESS_OPTS = --temp-config=$(srcdir)/datalake_fdw.conf \ + --inputdir=$(srcdir)/test/automation/sqlrepo/smoke/iceberg_am + +EXTRA_CLEAN = exports_darwin.list exports.map + +# Keep the aggregate target as make's default goal. +all: + +ifdef USE_PGXS +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) +else +subdir = contrib/datalake_fdw +top_builddir = ../.. +include $(top_builddir)/src/Makefile.global +include $(top_srcdir)/contrib/contrib-global.mk +endif + +# Everything below needs variables that Makefile.global defines (PORTNAME), and +# SHLIB_LINK additions still apply because the link recipe expands it when it +# runs. + +# Shared libraries are linked with $(CC) (see src/Makefile.shlib COMPILER), so a +# module containing C++ translation units must pull in the C++ runtime itself. +SHLIB_LINK += -lstdc++ + +# Arrow and other C++ dependencies land in this module later; the export list is +# the single place that decides what stays visible, so the mechanism ships now. +ifeq ($(PORTNAME), darwin) +EXPORT_LIST = exports_darwin.list +SHLIB_LINK += -Wl,-exported_symbols_list,exports_darwin.list + +exports_darwin.list: exports.txt + sed -e '/^[[:space:]]*#/d' -e '/^[[:space:]]*$$/d' -e 's/^/_/' $< > $@ +else +EXPORT_LIST = exports.map +SHLIB_LINK += -Wl,--version-script=exports.map -Wl,--exclude-libs,ALL + +exports.map: exports.txt + { echo '{ global:'; sed -e '/^[[:space:]]*#/d' -e '/^[[:space:]]*$$/d' -e 's/$$/;/' $<; echo 'local: *; };'; } > $@ +endif + +all: $(EXPORT_LIST) +$(shlib): $(EXPORT_LIST) diff --git a/contrib/datalake_fdw/datalake_fdw--1.0.sql b/contrib/datalake_fdw/datalake_fdw--1.0.sql new file mode 100644 index 00000000000..f8e87df5477 --- /dev/null +++ b/contrib/datalake_fdw/datalake_fdw--1.0.sql @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * contrib/datalake_fdw/datalake_fdw--1.0.sql + */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION datalake_fdw" to load this file. \quit + +CREATE FUNCTION iceberg_am_handler(internal) +RETURNS table_am_handler AS 'MODULE_PATHNAME' LANGUAGE C; + +CREATE ACCESS METHOD iceberg TYPE TABLE HANDLER iceberg_am_handler; + +CREATE FUNCTION iceberg_catalog_fdw_validator(text[], oid) +RETURNS void AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +CREATE FOREIGN DATA WRAPPER iceberg_catalog_fdw + VALIDATOR iceberg_catalog_fdw_validator; + +CREATE FUNCTION iceberg_volume_fdw_validator(text[], oid) +RETURNS void AS 'MODULE_PATHNAME' LANGUAGE C STRICT; + +CREATE FOREIGN DATA WRAPPER iceberg_volume_fdw + VALIDATOR iceberg_volume_fdw_validator; diff --git a/contrib/datalake_fdw/datalake_fdw.conf b/contrib/datalake_fdw/datalake_fdw.conf new file mode 100644 index 00000000000..7e1c7c5a785 --- /dev/null +++ b/contrib/datalake_fdw/datalake_fdw.conf @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# contrib/datalake_fdw/datalake_fdw.conf +# +# Configuration for the temporary server that "make check" starts. The module +# installs process-wide hooks, so _PG_init refuses to run outside +# shared_preload_libraries; without this the first statement that reaches the +# access method would fail to load the library instead of testing it. + +shared_preload_libraries = 'datalake_fdw' diff --git a/contrib/datalake_fdw/datalake_fdw.control b/contrib/datalake_fdw/datalake_fdw.control new file mode 100644 index 00000000000..263990ba8b1 --- /dev/null +++ b/contrib/datalake_fdw/datalake_fdw.control @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# contrib/datalake_fdw/datalake_fdw.control + +comment = 'Apache Iceberg lake tables for Cloudberry (demo skeleton)' +default_version = '1.0' +module_pathname = '$libdir/datalake_fdw' +relocatable = false diff --git a/contrib/datalake_fdw/exports.txt b/contrib/datalake_fdw/exports.txt new file mode 100644 index 00000000000..0db251366df --- /dev/null +++ b/contrib/datalake_fdw/exports.txt @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# contrib/datalake_fdw/exports.txt +# +# Single source of truth for exported symbols; the linker script for each +# platform is generated from it at build time. Adding a line here is an API +# decision, so it is one a reviewer has to see. Comment lines are stripped +# when the script is generated. + +_PG_init +Pg_magic_func +pg_finfo_iceberg_am_handler +iceberg_am_handler +pg_finfo_iceberg_catalog_fdw_validator +iceberg_catalog_fdw_validator +pg_finfo_iceberg_volume_fdw_validator +iceberg_volume_fdw_validator diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_am_handler.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_am_handler.c new file mode 100644 index 00000000000..4a610fc3238 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_am_handler.c @@ -0,0 +1,509 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_am_handler.c + * Table access method callbacks for Iceberg tables. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_am_handler.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/multixact.h" +#include "access/tableam.h" +#include "am_iceberg/pg_iceberg_options.h" +#include "am_iceberg/pg_iceberg_reject.h" +#include "fmgr.h" + +PG_FUNCTION_INFO_V1(iceberg_am_handler); + +static const TupleTableSlotOps * +pg_iceberg_slot_callbacks(Relation rel pg_attribute_unused()) +{ + return &TTSOpsVirtual; +} + +static TableScanDesc +pg_iceberg_scan_begin(Relation rel pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + int nkeys pg_attribute_unused(), + struct ScanKeyData *key pg_attribute_unused(), + ParallelTableScanDesc pscan pg_attribute_unused(), + uint32 flags pg_attribute_unused()) +{ + pg_iceberg_not_supported("SELECT"); +} + +static void +pg_iceberg_scan_end(TableScanDesc scan pg_attribute_unused()) +{ + pg_iceberg_not_supported("SELECT"); +} + +static void +pg_iceberg_scan_rescan(TableScanDesc scan pg_attribute_unused(), + struct ScanKeyData *key pg_attribute_unused(), + bool set_params pg_attribute_unused(), + bool allow_strat pg_attribute_unused(), + bool allow_sync pg_attribute_unused(), + bool allow_pagemode pg_attribute_unused()) +{ + pg_iceberg_not_supported("SELECT"); +} + +static bool +pg_iceberg_scan_getnextslot(TableScanDesc scan pg_attribute_unused(), + ScanDirection direction pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused()) +{ + pg_iceberg_not_supported("SELECT"); +} + +static Size +pg_iceberg_parallelscan_estimate(Relation rel pg_attribute_unused()) +{ + pg_iceberg_not_supported("parallel scan"); +} + +static Size +pg_iceberg_parallelscan_initialize(Relation rel pg_attribute_unused(), + ParallelTableScanDesc pscan pg_attribute_unused()) +{ + pg_iceberg_not_supported("parallel scan"); +} + +static void +pg_iceberg_parallelscan_reinitialize(Relation rel pg_attribute_unused(), + ParallelTableScanDesc pscan pg_attribute_unused()) +{ + pg_iceberg_not_supported("parallel scan"); +} + +static struct IndexFetchTableData * +pg_iceberg_index_fetch_begin(Relation rel pg_attribute_unused()) +{ + pg_iceberg_not_supported("index access"); +} + +static void +pg_iceberg_index_fetch_reset(struct IndexFetchTableData *data pg_attribute_unused()) +{ + pg_iceberg_not_supported("index access"); +} + +static void +pg_iceberg_index_fetch_end(struct IndexFetchTableData *data pg_attribute_unused()) +{ + pg_iceberg_not_supported("index access"); +} + +static bool +pg_iceberg_index_fetch_tuple(struct IndexFetchTableData *scan pg_attribute_unused(), + ItemPointer tid pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + bool *call_again pg_attribute_unused(), + bool *all_dead pg_attribute_unused()) +{ + pg_iceberg_not_supported("index access"); +} + +static bool +pg_iceberg_tuple_fetch_row_version(Relation rel pg_attribute_unused(), + ItemPointer tid pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused()) +{ + pg_iceberg_not_supported("tuple fetch by TID"); +} + +static bool +pg_iceberg_tuple_tid_valid(TableScanDesc scan pg_attribute_unused(), + ItemPointer tid pg_attribute_unused()) +{ + pg_iceberg_not_supported("tuple fetch by TID"); +} + +static void +pg_iceberg_tuple_get_latest_tid(TableScanDesc scan pg_attribute_unused(), + ItemPointer tid pg_attribute_unused()) +{ + pg_iceberg_not_supported("tuple fetch by TID"); +} + +static bool +pg_iceberg_tuple_satisfies_snapshot(Relation rel pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused()) +{ + pg_iceberg_not_supported("tuple fetch by TID"); +} + +static TransactionId +pg_iceberg_index_delete_tuples(Relation rel pg_attribute_unused(), + TM_IndexDeleteOp *delstate pg_attribute_unused()) +{ + pg_iceberg_not_supported("index maintenance"); +} + +static void +pg_iceberg_tuple_insert(Relation rel pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + CommandId cid pg_attribute_unused(), + int options pg_attribute_unused(), + struct BulkInsertStateData *bistate pg_attribute_unused()) +{ + pg_iceberg_not_supported("INSERT"); +} + +static void +pg_iceberg_tuple_insert_speculative(Relation rel pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + CommandId cid pg_attribute_unused(), + int options pg_attribute_unused(), + struct BulkInsertStateData *bistate pg_attribute_unused(), + uint32 specToken pg_attribute_unused()) +{ + pg_iceberg_not_supported("INSERT ... ON CONFLICT"); +} + +static void +pg_iceberg_tuple_complete_speculative(Relation rel pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + uint32 specToken pg_attribute_unused(), + bool succeeded pg_attribute_unused()) +{ + pg_iceberg_not_supported("INSERT ... ON CONFLICT"); +} + +static void +pg_iceberg_multi_insert(Relation rel pg_attribute_unused(), + TupleTableSlot **slots pg_attribute_unused(), + int nslots pg_attribute_unused(), + CommandId cid pg_attribute_unused(), + int options pg_attribute_unused(), + struct BulkInsertStateData *bistate pg_attribute_unused()) +{ + pg_iceberg_not_supported("INSERT"); +} + +static TM_Result +pg_iceberg_tuple_delete(Relation rel pg_attribute_unused(), + ItemPointer tid pg_attribute_unused(), + CommandId cid pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + Snapshot crosscheck pg_attribute_unused(), + bool wait pg_attribute_unused(), + TM_FailureData *tmfd pg_attribute_unused(), + bool changingPart pg_attribute_unused()) +{ + pg_iceberg_not_supported("DELETE"); +} + +static TM_Result +pg_iceberg_tuple_update(Relation rel pg_attribute_unused(), + ItemPointer otid pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + CommandId cid pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + Snapshot crosscheck pg_attribute_unused(), + bool wait pg_attribute_unused(), + TM_FailureData *tmfd pg_attribute_unused(), + LockTupleMode *lockmode pg_attribute_unused(), + TU_UpdateIndexes *update_indexes pg_attribute_unused()) +{ + pg_iceberg_not_supported("UPDATE"); +} + +static TM_Result +pg_iceberg_tuple_lock(Relation rel pg_attribute_unused(), + ItemPointer tid pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused(), + CommandId cid pg_attribute_unused(), + LockTupleMode mode pg_attribute_unused(), + LockWaitPolicy wait_policy pg_attribute_unused(), + uint8 flags pg_attribute_unused(), + TM_FailureData *tmfd pg_attribute_unused()) +{ + pg_iceberg_not_supported("row locking (SELECT ... FOR UPDATE)"); +} + +static void +pg_iceberg_relation_set_new_filelocator(Relation rel pg_attribute_unused(), + const RelFileLocator *newrlocator pg_attribute_unused(), + char persistence pg_attribute_unused(), + TransactionId *freezeXid, + MultiXactId *minmulti) +{ + /* + * Iceberg data lives in external object storage, so creating local smgr + * storage here would be both unnecessary and misleading. + */ + *freezeXid = InvalidTransactionId; + *minmulti = InvalidMultiXactId; + return; +} + +static void +pg_iceberg_relation_nontransactional_truncate(Relation rel pg_attribute_unused()) +{ + pg_iceberg_not_supported("TRUNCATE"); +} + +static void +pg_iceberg_relation_copy_data(Relation rel pg_attribute_unused(), + const RelFileLocator *newrlocator pg_attribute_unused()) +{ + pg_iceberg_not_supported("ALTER TABLE ... SET TABLESPACE"); +} + +static void +pg_iceberg_relation_copy_for_cluster(Relation OldTable pg_attribute_unused(), + Relation NewTable pg_attribute_unused(), + Relation OldIndex pg_attribute_unused(), + bool use_sort pg_attribute_unused(), + TransactionId OldestXmin pg_attribute_unused(), + TransactionId *xid_cutoff pg_attribute_unused(), + MultiXactId *multi_cutoff pg_attribute_unused(), + double *num_tuples pg_attribute_unused(), + double *tups_vacuumed pg_attribute_unused(), + double *tups_recently_dead pg_attribute_unused()) +{ + pg_iceberg_not_supported("CLUSTER / VACUUM FULL"); +} + +static void +pg_iceberg_relation_vacuum(Relation rel pg_attribute_unused(), + struct VacuumParams *params pg_attribute_unused(), + BufferAccessStrategy bstrategy pg_attribute_unused()) +{ + /* + * A database-wide VACUUM or autovacuum must not fail merely because it + * encounters an Iceberg table. There is no local storage to vacuum. + */ + return; +} + +static bool +pg_iceberg_scan_analyze_next_block(TableScanDesc scan pg_attribute_unused(), + BlockNumber blockno pg_attribute_unused(), + BufferAccessStrategy bstrategy pg_attribute_unused()) +{ + /* Defensive fallback; relation_acquire_sample_rows bypasses this path. */ + return false; +} + +static bool +pg_iceberg_scan_analyze_next_tuple(TableScanDesc scan pg_attribute_unused(), + TransactionId OldestXmin pg_attribute_unused(), + double *liverows pg_attribute_unused(), + double *deadrows pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused()) +{ + /* Defensive fallback; relation_acquire_sample_rows bypasses this path. */ + return false; +} + +static int +pg_iceberg_relation_acquire_sample_rows(Relation onerel pg_attribute_unused(), + int elevel pg_attribute_unused(), + HeapTuple *rows pg_attribute_unused(), + int targrows pg_attribute_unused(), + double *totalrows, + double *totaldeadrows) +{ + /* + * analyze.c uses this callback directly when present and therefore never + * starts a table_beginscan_analyze() scan. This lets ANALYZE succeed as a + * zero-sample no-op while ordinary scans remain unsupported. + */ + *totalrows = 0; + *totaldeadrows = 0; + return 0; +} + +static double +pg_iceberg_index_build_range_scan(Relation table_rel pg_attribute_unused(), + Relation index_rel pg_attribute_unused(), + struct IndexInfo *index_info pg_attribute_unused(), + bool allow_sync pg_attribute_unused(), + bool anyvisible pg_attribute_unused(), + bool progress pg_attribute_unused(), + BlockNumber start_blockno pg_attribute_unused(), + BlockNumber numblocks pg_attribute_unused(), + IndexBuildCallback callback pg_attribute_unused(), + void *callback_state pg_attribute_unused(), + TableScanDesc scan pg_attribute_unused()) +{ + pg_iceberg_not_supported("CREATE INDEX"); +} + +static void +pg_iceberg_index_validate_scan(Relation table_rel pg_attribute_unused(), + Relation index_rel pg_attribute_unused(), + struct IndexInfo *index_info pg_attribute_unused(), + Snapshot snapshot pg_attribute_unused(), + struct ValidateIndexState *state pg_attribute_unused()) +{ + pg_iceberg_not_supported("CREATE INDEX"); +} + +static uint64 +pg_iceberg_relation_size(Relation rel pg_attribute_unused(), + ForkNumber forkNumber pg_attribute_unused()) +{ + return 0; +} + +static BlockSequence * +pg_iceberg_relation_get_block_sequences(Relation rel pg_attribute_unused(), + int *numSequences) +{ + *numSequences = 0; + return palloc0(sizeof(BlockSequence)); +} + +static void +pg_iceberg_relation_get_block_sequence(Relation rel pg_attribute_unused(), + BlockNumber blkNum pg_attribute_unused(), + BlockSequence *sequence pg_attribute_unused()) +{ + pg_iceberg_not_supported("block sequence access"); +} + +static bool +pg_iceberg_relation_needs_toast_table(Relation rel pg_attribute_unused()) +{ + return false; +} + +static void +pg_iceberg_relation_estimate_size(Relation rel pg_attribute_unused(), + int32 *attr_widths pg_attribute_unused(), + BlockNumber *pages, + double *tuples, + double *allvisfrac) +{ + *pages = 0; + *tuples = 0; + *allvisfrac = 0; +} + +static bool +pg_iceberg_scan_sample_next_block(TableScanDesc scan pg_attribute_unused(), + struct SampleScanState *scanstate pg_attribute_unused()) +{ + pg_iceberg_not_supported("TABLESAMPLE"); +} + +static bool +pg_iceberg_scan_sample_next_tuple(TableScanDesc scan pg_attribute_unused(), + struct SampleScanState *scanstate pg_attribute_unused(), + TupleTableSlot *slot pg_attribute_unused()) +{ + pg_iceberg_not_supported("TABLESAMPLE"); +} + +/* + * Optional callbacks below remain NULL. Iceberg currently has no local tuple + * scanning, index, bulk-insert, TOAST, bitmap-scan, DML-state, file-swap, or + * column-encoding implementation. + */ +static const TableAmRoutine pg_iceberg_methods = { + .type = T_TableAmRoutine, + + .slot_callbacks = pg_iceberg_slot_callbacks, + + .scan_begin = pg_iceberg_scan_begin, + .scan_begin_extractcolumns = NULL, + .scan_begin_extractcolumns_bm = NULL, + .scan_end = pg_iceberg_scan_end, + .scan_rescan = pg_iceberg_scan_rescan, + .scan_getnextslot = pg_iceberg_scan_getnextslot, + .scan_set_tidrange = NULL, + .scan_getnextslot_tidrange = NULL, + .scan_flags = NULL, + + .parallelscan_estimate = pg_iceberg_parallelscan_estimate, + .parallelscan_initialize = pg_iceberg_parallelscan_initialize, + .parallelscan_reinitialize = pg_iceberg_parallelscan_reinitialize, + + .index_fetch_begin = pg_iceberg_index_fetch_begin, + .index_fetch_reset = pg_iceberg_index_fetch_reset, + .index_fetch_end = pg_iceberg_index_fetch_end, + .index_fetch_tuple = pg_iceberg_index_fetch_tuple, + .index_unique_check = NULL, + + .tuple_fetch_row_version = pg_iceberg_tuple_fetch_row_version, + .tuple_tid_valid = pg_iceberg_tuple_tid_valid, + .tuple_get_latest_tid = pg_iceberg_tuple_get_latest_tid, + .tuple_satisfies_snapshot = pg_iceberg_tuple_satisfies_snapshot, + .index_delete_tuples = pg_iceberg_index_delete_tuples, + + .tuple_insert = pg_iceberg_tuple_insert, + .tuple_insert_speculative = pg_iceberg_tuple_insert_speculative, + .tuple_complete_speculative = pg_iceberg_tuple_complete_speculative, + .multi_insert = pg_iceberg_multi_insert, + .tuple_delete = pg_iceberg_tuple_delete, + .tuple_update = pg_iceberg_tuple_update, + .tuple_lock = pg_iceberg_tuple_lock, + .finish_bulk_insert = NULL, + + .relation_set_new_filelocator = pg_iceberg_relation_set_new_filelocator, + .relation_nontransactional_truncate = pg_iceberg_relation_nontransactional_truncate, + .relation_copy_data = pg_iceberg_relation_copy_data, + .relation_copy_for_cluster = pg_iceberg_relation_copy_for_cluster, + .relation_vacuum = pg_iceberg_relation_vacuum, + .scan_analyze_next_block = pg_iceberg_scan_analyze_next_block, + .scan_analyze_next_tuple = pg_iceberg_scan_analyze_next_tuple, + .relation_acquire_sample_rows = pg_iceberg_relation_acquire_sample_rows, + .index_build_range_scan = pg_iceberg_index_build_range_scan, + .index_validate_scan = pg_iceberg_index_validate_scan, + + .relation_size = pg_iceberg_relation_size, + .relation_get_block_sequences = pg_iceberg_relation_get_block_sequences, + .relation_get_block_sequence = pg_iceberg_relation_get_block_sequence, + .relation_needs_toast_table = pg_iceberg_relation_needs_toast_table, + .relation_toast_am = NULL, + .relation_fetch_toast_slice = NULL, + + .relation_estimate_size = pg_iceberg_relation_estimate_size, + + .scan_bitmap_next_block = NULL, + .scan_bitmap_next_tuple = NULL, + .scan_sample_next_block = pg_iceberg_scan_sample_next_block, + .scan_sample_next_tuple = pg_iceberg_scan_sample_next_tuple, + + .dml_init = NULL, + .dml_fini = NULL, + .amoptions = pg_iceberg_amoptions, + .swap_relation_files = NULL, + .validate_column_encoding_clauses = NULL, + .transform_column_encoding_clauses = NULL, +}; + +Datum +iceberg_am_handler(PG_FUNCTION_ARGS) +{ + PG_RETURN_POINTER(&pg_iceberg_methods); +} diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.c new file mode 100644 index 00000000000..ee70aa7fa83 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.c @@ -0,0 +1,302 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_ddl.c + * Object-access integration for the Iceberg table lifecycle. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/htup_details.h" +#include "access/relation.h" +#include "am_iceberg/pg_iceberg_ddl.h" +#include "am_iceberg/pg_iceberg_options.h" +#include "am_iceberg/pg_iceberg_reject.h" +#include "catalog/pg_class.h" +#include "cdb/cdbvars.h" +#include "commands/defrem.h" +#include "meta/iceberg_meta_engine.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/rel.h" +#include "utils/syscache.h" + +object_access_hook_type pg_iceberg_prev_object_access_hook; + +static Oid get_rel_relam(Oid relid); +static MetaCtx table_info_meta_ctx(const IcebergTableInfo *info); +static void iceberg_post_create(Oid objectId); +static void iceberg_drop(Oid objectId); + +/* + * This tree has no lsyscache get_rel_relam() helper, so provide the same + * missing-ok syscache lookup locally. + */ +static Oid +get_rel_relam(Oid relid) +{ + HeapTuple tuple; + Oid relam = InvalidOid; + + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (HeapTupleIsValid(tuple)) + { + relam = ((Form_pg_class) GETSTRUCT(tuple))->relam; + ReleaseSysCache(tuple); + } + + return relam; +} + +static MetaCtx +table_info_meta_ctx(const IcebergTableInfo *info) +{ + MetaCtx ctx = { + .catalog_name = info->catalog_name, + .namespace_name = info->opts->namespace, + .table_name = info->opts->table, + .catalog_props = info->catalog_props, + .n_catalog_props = info->n_catalog_props, + .credential_props = NULL, + .n_credential_props = 0 + }; + + return ctx; +} + +static void +iceberg_post_create(Oid objectId) +{ + Relation rel; + Oid amoid; + IcebergTableInfo *info; + + amoid = pg_iceberg_am_oid(); + if (!OidIsValid(amoid)) + return; + + rel = relation_open(objectId, AccessShareLock); + if (rel->rd_rel->relam != amoid) + { + relation_close(rel, AccessShareLock); + return; + } + + info = pg_iceberg_get_table_info_rel(rel); + InsertLakeTableEntry(objectId, info); + + if (Gp_role == GP_ROLE_DISPATCH) + { + const IcebergMetaEngine *engine = get_meta_engine(); + MetaCtx ctx = table_info_meta_ctx(info); + MetaTableDef def = {.schema_json = ""}; + MetaTable *table_metadata = NULL; + DlErrCode rc; + + /* Always cross the metadata-engine boundary through its wrapper. */ + rc = meta_engine_create_table(engine, &ctx, &def, &table_metadata); + relation_close(rel, AccessShareLock); + + /* ctx borrows from info, so release only once the call has returned. */ + pg_iceberg_free_table_info(info); + + if (rc != DL_OK) + dl_error_report(ERROR, rc, "create_table"); + return; + } + + relation_close(rel, AccessShareLock); + pg_iceberg_free_table_info(info); +} + +static void +iceberg_drop(Oid objectId) +{ + Relation rel; + Oid amoid; + IcebergTableInfo *info; + const IcebergMetaEngine *engine; + MetaCtx ctx; + DlErrCode rc; + + amoid = pg_iceberg_am_oid(); + if (!OidIsValid(amoid) || get_rel_relam(objectId) != amoid) + return; + + /* + * The dispatcher makes the single remote call; everyone else only drops + * local catalog rows and the dependencies recorded at creation. + * + * A utility-mode backend deliberately falls in the second group. It talks + * to one node, so letting it delete the remote table would remove metadata + * the other nodes still reference. The utility guard refuses the + * statements that reach a lake table directly; an indirect cascade that + * slips past it drops the local rows only, which is recoverable, unlike a + * remote catalog entry deleted on one node's say-so. + */ + if (Gp_role != GP_ROLE_DISPATCH) + return; + + /* + * Defensive: no path the regression suite covers -- DROP TABLE, DROP SERVER + * CASCADE, DROP SCHEMA CASCADE -- reaches this hook with the relation no + * longer openable, so the open below has always succeeded. It stays a try + * rather than an open because a hook that raised here would make the object + * undroppable, and there is nothing to reconstruct the mapping from anyway. + */ + rel = try_relation_open(objectId, AccessShareLock, false); + if (rel == NULL) + return; + + /* + * Resolving the mapping can fail -- a server option that no longer parses, + * a wrapper renamed out from under the table -- and DROP is exactly the + * statement that must still work in that state. Report what could not be + * cleaned up remotely and let the local drop proceed, rather than leaving + * the user with a table that cannot be dropped at all. + */ + info = NULL; + PG_TRY(); + { + info = pg_iceberg_get_table_info_rel(rel); + } + PG_CATCH(); + { + MemoryContext ctxt = MemoryContextSwitchTo(TopTransactionContext); + ErrorData *edata = CopyErrorData(); + + MemoryContextSwitchTo(ctxt); + + /* + * Only a mapping that no longer describes anything usable may be + * downgraded here. A cancelled query or an out-of-memory failure has + * nothing to do with the mapping, and turning one of those into a + * warning would drop the table while pretending the statement + * succeeded. + */ + if (edata->sqlerrcode != ERRCODE_INVALID_TABLE_DEFINITION && + edata->sqlerrcode != ERRCODE_INVALID_PARAMETER_VALUE && + edata->sqlerrcode != ERRCODE_UNDEFINED_OBJECT) + { + FreeErrorData(edata); + relation_close(rel, AccessShareLock); + PG_RE_THROW(); + } + + FlushErrorState(); + relation_close(rel, AccessShareLock); + ereport(WARNING, + (errmsg("iceberg: dropping \"%s\" without notifying the metadata engine", + get_rel_name(objectId)), + errdetail("%s", edata->message))); + FreeErrorData(edata); + return; + } + PG_END_TRY(); + + engine = get_meta_engine(); + ctx = table_info_meta_ctx(info); + + /* + * Dropping the table drops this database's reference to it. Whether the + * lake data goes too is the table's own decision, recorded in its options + * when it was created; the default is to leave it, so that dropping a + * reference cannot destroy data another reader still expects to find. + * + * A table's identity is its (catalog, namespace, name) triple, so there is + * nothing to fence this call against: a remote table under that name is by + * definition the table being dropped. + */ + rc = meta_engine_drop_table(engine, &ctx, info->opts->purge_on_drop); + relation_close(rel, AccessShareLock); + + /* ctx borrows from info, so release only once the call has returned. */ + pg_iceberg_free_table_info(info); + + /* + * Do not strand the local table when the remote drop fails. The catalog + * DROP continues and the failure stays visible; reconciling what the + * remote catalog still holds is the metadata agent's job, not something + * this skeleton can record durably. + */ + if (rc != DL_OK) + dl_error_report(WARNING, rc, "drop_table"); +} + +void +pg_iceberg_object_access(ObjectAccessType access, + Oid classId, + Oid objectId, + int subId, + void *arg) +{ + if (pg_iceberg_prev_object_access_hook) + (*pg_iceberg_prev_object_access_hook) (access, classId, objectId, + subId, arg); + + if (classId != RelationRelationId || subId != 0) + return; + + switch (access) + { + case OAT_POST_CREATE: + { + ObjectAccessPostCreate *created = (ObjectAccessPostCreate *) arg; + + /* + * Relations the system builds for itself -- the transient heap + * of a rewrite above all -- must never reach the metadata + * engine. They carry a generated name, they live only until + * the rewrite swaps them in, and creating them remotely leaves + * an orphan the moment the local work rolls back. The utility + * hook refuses the statements that rewrite a lake table; this + * is the backstop for whatever it does not see. + */ + if (created != NULL && created->is_internal) + return; + + iceberg_post_create(objectId); + } + break; + case OAT_DROP: + iceberg_drop(objectId); + break; + case OAT_TRUNCATE: + + /* + * TRUNCATE of a table created in an earlier transaction never + * reaches the access method's nontransactional path: it goes + * through relation_set_new_filelocator, which succeeds because a + * lake table has no local storage to reset. Without this event + * the statement would report success while every Iceberg data file + * stayed exactly where it was. + */ + if (OidIsValid(pg_iceberg_am_oid()) && + get_rel_relam(objectId) == pg_iceberg_am_oid()) + pg_iceberg_not_supported("TRUNCATE"); + break; + default: + break; + } +} diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.h b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.h new file mode 100644 index 00000000000..c6cd354a732 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.h @@ -0,0 +1,42 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_ddl.h + * Object-access integration for the Iceberg table lifecycle. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_ddl.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_ICEBERG_DDL_H +#define PG_ICEBERG_DDL_H + +#include "catalog/objectaccess.h" + +extern object_access_hook_type pg_iceberg_prev_object_access_hook; + +extern void pg_iceberg_object_access(ObjectAccessType access, + Oid classId, + Oid objectId, + int subId, + void *arg); + +#endif /* PG_ICEBERG_DDL_H */ diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c new file mode 100644 index 00000000000..f675a1ef24d --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c @@ -0,0 +1,964 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_extensible.c + * Extension initialization and the Iceberg utility guard. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_extensible.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/genam.h" +#include "access/htup_details.h" +#include "access/table.h" +#include "access/tableam.h" +#include "am_iceberg/pg_iceberg_ddl.h" +#include "am_iceberg/pg_iceberg_guc.h" +#include "am_iceberg/pg_iceberg_options.h" +#include "am_iceberg/pg_iceberg_reject.h" +#include "catalog/namespace.h" +#include "catalog/pg_class.h" +#include "catalog/pg_depend.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_namespace.h" +#include "cdb/cdbvars.h" +#include "commands/defrem.h" +#include "common/backend_registry.h" +#include "fmgr.h" +#include "foreign/foreign.h" +#include "meta/iceberg_meta_engine.h" +#include "meta/meta_engine_init.h" +#include "miscadmin.h" +#include "nodes/makefuncs.h" +#include "nodes/parsenodes.h" +#include "storage/lmgr.h" +#include "tcop/utility.h" +#include "utils/fmgroids.h" +#include "utils/syscache.h" + +PG_MODULE_MAGIC; + +static ProcessUtility_hook_type prev_ProcessUtility_hook; + +static bool iceberg_is_effective_am(const char *accessMethod); +static Oid get_rel_relam(Oid relid); +static bool relid_is_iceberg(Oid relid); +static bool server_referenced_by_iceberg(Oid srvid); +static bool rangevar_is_iceberg(RangeVar *relation); +static const char *string_object_name(Node *object); +static Oid lock_object_by_name(Oid classid, + Oid (*lookup) (const char *name, bool missing_ok), + const char *name, LOCKMODE lockmode); +static bool locked_server_referenced_by_iceberg(const char *servername); +static bool utility_drop_targets_iceberg(DropStmt *stmt); +static bool alter_table_targets_iceberg_am(AlterTableStmt *stmt); +static bool alter_table_is_owner_only(AlterTableStmt *stmt); +static bool database_has_iceberg_table(void); +static bool schema_has_iceberg_table(const char *schemaname); +static bool server_belongs_to_module(const char *servername); +static bool is_module_fdw_name(const char *fdwname); +static const char *find_reloption(List *options, const char *name); +static void lock_create_servers(Oid catalog_srvid, Oid volume_srvid, + LOCKMODE lockmode); +static void unlock_create_servers(Oid catalog_srvid, Oid volume_srvid, + LOCKMODE lockmode); +static void validate_create_binding(const char *catalog_name, + const char *volume_name); +static void prepare_iceberg_create(CreateStmt *stmt); +static void reject_utility_mode_ddl(const char *subject) pg_attribute_noreturn(); +static void reject_targeted_operation(const char *operation); +static void pg_iceberg_ProcessUtility(PlannedStmt *pstmt, + const char *queryString, + bool readOnlyTree, + ProcessUtilityContext context, + ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, + QueryCompletion *qc); + +/* + * Resolve an omitted access method exactly as core CREATE TABLE does. Keep + * this as the single source of truth for every relation-creating node handled + * by the hook. + */ +static bool +iceberg_is_effective_am(const char *accessMethod) +{ + if (accessMethod != NULL) + return strcmp(accessMethod, "iceberg") == 0; + return default_table_access_method != NULL && + strcmp(default_table_access_method, "iceberg") == 0; +} + +/* + * This tree has no lsyscache get_rel_relam() helper, so provide the same + * missing-ok syscache lookup locally. + */ +static Oid +get_rel_relam(Oid relid) +{ + HeapTuple tuple; + Oid relam = InvalidOid; + + tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); + if (HeapTupleIsValid(tuple)) + { + relam = ((Form_pg_class) GETSTRUCT(tuple))->relam; + ReleaseSysCache(tuple); + } + + return relam; +} + +static bool +relid_is_iceberg(Oid relid) +{ + Oid iceberg_am_oid; + + /* + * Look the OID up every time instead of caching it: DROP EXTENSION + * followed by CREATE EXTENSION hands out a new OID, and a cached one would + * make these guards silently stop matching. The lookup is syscache-backed, + * and it must be missing-ok because the predicate is consulted for + * arbitrary relations before the extension exists. + */ + iceberg_am_oid = get_table_am_oid("iceberg", true); + + return OidIsValid(iceberg_am_oid) && OidIsValid(relid) && + get_rel_relam(relid) == iceberg_am_oid; +} + +static bool +server_referenced_by_iceberg(Oid srvid) +{ + Relation depend_rel; + ScanKeyData keys[2]; + SysScanDesc scan; + HeapTuple tuple; + bool referenced = false; + + if (!OidIsValid(srvid)) + return false; + + depend_rel = table_open(DependRelationId, AccessShareLock); + + ScanKeyInit(&keys[0], + Anum_pg_depend_refclassid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(ForeignServerRelationId)); + ScanKeyInit(&keys[1], + Anum_pg_depend_refobjid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(srvid)); + + scan = systable_beginscan(depend_rel, DependReferenceIndexId, true, + NULL, lengthof(keys), keys); + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + Form_pg_depend dependency = (Form_pg_depend) GETSTRUCT(tuple); + + if (dependency->classid == RelationRelationId && + relid_is_iceberg(dependency->objid)) + { + referenced = true; + break; + } + } + + systable_endscan(scan); + table_close(depend_rel, AccessShareLock); + + return referenced; +} + +static bool +rangevar_is_iceberg(RangeVar *relation) +{ + Oid relid; + + if (relation == NULL) + return false; + + relid = RangeVarGetRelid(relation, AccessShareLock, true); + return OidIsValid(relid) && relid_is_iceberg(relid); +} + +static const char * +string_object_name(Node *object) +{ + if (object != NULL && IsA(object, String)) + return strVal(object); + return NULL; +} + +/* + * Resolve a name to an object OID and lock that object exclusively, so that a + * guard can decide about it without racing the statement it is guarding. + * + * The re-resolution is what makes this correct rather than merely locked. + * Acquiring the lock can mean waiting, and the transactions waited for are free + * to rename this object away and give its name to a different one. A guard + * that skipped the recheck would then hold a lock on an object the statement no + * longer names, and would scan it instead of the object about to be changed. + * This is the lookup-lock-recheck loop PostgreSQL applies to relations for the + * same reason. + * + * Returns InvalidOid when the name resolves to nothing, holding no lock. + */ +static Oid +lock_object_by_name(Oid classid, + Oid (*lookup) (const char *name, bool missing_ok), + const char *name, LOCKMODE lockmode) +{ + for (;;) + { + Oid objectid = lookup(name, true); + + if (!OidIsValid(objectid)) + return InvalidOid; + + LockDatabaseObject(classid, objectid, 0, lockmode); + + if (lookup(name, true) == objectid) + return objectid; + + UnlockDatabaseObject(classid, objectid, 0, lockmode); + } +} + +/* + * The exclusive counterpart of the share lock CREATE takes on the servers it + * binds to. ALTER statements name one server, so ascending-OID order is + * trivial; preserve that ordering if a future statement form locks more than + * one. Lock before scanning so the dependency decision cannot race CREATE. + */ +static bool +locked_server_referenced_by_iceberg(const char *servername) +{ + Oid srvid; + + if (servername == NULL) + return false; + + srvid = lock_object_by_name(ForeignServerRelationId, + get_foreign_server_oid, servername, + AccessExclusiveLock); + if (!OidIsValid(srvid)) + return false; + + return server_referenced_by_iceberg(srvid); +} + +static bool +utility_drop_targets_iceberg(DropStmt *stmt) +{ + ListCell *lc; + + if (stmt->removeType == OBJECT_TABLE) + { + foreach(lc, stmt->objects) + { + RangeVar *relation = + makeRangeVarFromNameList((List *) lfirst(lc)); + + if (rangevar_is_iceberg(relation)) + return true; + } + } + else if (stmt->removeType == OBJECT_FOREIGN_SERVER) + { + foreach(lc, stmt->objects) + { + Node *object = (Node *) lfirst(lc); + + if (locked_server_referenced_by_iceberg( + string_object_name(object))) + return true; + } + } + + return false; +} + +/* + * Does this statement convert its target into a lake table? + */ +static bool +alter_table_targets_iceberg_am(AlterTableStmt *stmt) +{ + ListCell *lc; + + foreach(lc, stmt->cmds) + { + AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lc); + + if (cmd->subtype == AT_SetAccessMethod && + iceberg_is_effective_am(cmd->name)) + return true; + } + + return false; +} + +/* + * Does any lake table exist in this database? + * + * Used by the statements that name no relation at all, where there is nothing + * to match against and the only safe answer is to refuse if such a table could + * be reached. + */ +static bool +database_has_iceberg_table(void) +{ + Relation class_rel; + ScanKeyData key; + SysScanDesc scan; + HeapTuple tuple; + Oid iceberg_am_oid; + bool found = false; + + iceberg_am_oid = get_table_am_oid("iceberg", true); + if (!OidIsValid(iceberg_am_oid)) + return false; + + class_rel = table_open(RelationRelationId, AccessShareLock); + ScanKeyInit(&key, + Anum_pg_class_relam, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(iceberg_am_oid)); + scan = systable_beginscan(class_rel, InvalidOid, false, NULL, 1, &key); + if (HeapTupleIsValid(tuple = systable_getnext(scan))) + found = true; + systable_endscan(scan); + table_close(class_rel, AccessShareLock); + + return found; +} + +/* + * Does the named schema contain a lake table? + * + * A lake table's namespace is not just where it lives locally: it is the + * namespace this module reports to the metadata engine. Renaming the schema + * would therefore silently repoint the table at a different external namespace, + * leaving whatever it named before behind. The check is scoped to schemas that + * actually contain one, so renaming any other schema stays unaffected. + */ +static bool +schema_has_iceberg_table(const char *schemaname) +{ + Relation class_rel; + ScanKeyData key[2]; + SysScanDesc scan; + HeapTuple tuple; + Oid iceberg_am_oid; + Oid namespace_oid; + bool found = false; + + if (schemaname == NULL) + return false; + + iceberg_am_oid = get_table_am_oid("iceberg", true); + if (!OidIsValid(iceberg_am_oid)) + return false; + + /* + * The exclusive counterpart of the share lock CREATE takes on its target + * namespace, same as the server guard: without it a concurrent CREATE could + * add a lake table to this schema after the scan below and before the + * rename runs, and that table would then live in the renamed schema while + * the metadata engine had already been told the old name. + */ + namespace_oid = lock_object_by_name(NamespaceRelationId, + get_namespace_oid, schemaname, + AccessExclusiveLock); + if (!OidIsValid(namespace_oid)) + return false; + + class_rel = table_open(RelationRelationId, AccessShareLock); + ScanKeyInit(&key[0], + Anum_pg_class_relam, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(iceberg_am_oid)); + ScanKeyInit(&key[1], + Anum_pg_class_relnamespace, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(namespace_oid)); + scan = systable_beginscan(class_rel, InvalidOid, false, NULL, 2, key); + if (HeapTupleIsValid(tuple = systable_getnext(scan))) + found = true; + systable_endscan(scan); + table_close(class_rel, AccessShareLock); + + return found; +} + +/* + * Does this ALTER TABLE do nothing but change the owner? + * + * Every other form is refused while the access method is unfinished, but this + * one has to go through: pg_dump writes ALTER TABLE ... OWNER TO for every + * table it dumps, so refusing it means refusing to restore a dump this module + * produced. Ownership is local catalog state -- it cannot reach the external + * table or change what the mapping resolves to -- so letting it through costs + * nothing that the refusal was protecting. + */ +static bool +alter_table_is_owner_only(AlterTableStmt *stmt) +{ + ListCell *lc; + + if (stmt->cmds == NIL) + return false; + + foreach(lc, stmt->cmds) + { + AlterTableCmd *cmd = (AlterTableCmd *) lfirst(lc); + + if (cmd->subtype != AT_ChangeOwner) + return false; + } + + return true; +} + +/* + * Is this a server belonging to one of this module's wrappers? + * + * Distinct from "referenced by a lake table": a server can be mutated before + * any table names it, and that is exactly the window in which the coordinator + * and a segment can be left holding different options for the same server name. + */ +static bool +server_belongs_to_module(const char *servername) +{ + ForeignServer *server; + Oid catalog_fdw; + Oid volume_fdw; + + if (servername == NULL) + return false; + + server = GetForeignServerByName(servername, true); + if (server == NULL) + return false; + + catalog_fdw = pg_iceberg_catalog_fdw_oid(true); + volume_fdw = pg_iceberg_volume_fdw_oid(true); + + return (OidIsValid(catalog_fdw) && server->fdwid == catalog_fdw) || + (OidIsValid(volume_fdw) && server->fdwid == volume_fdw); +} + +/* + * Is this one of the two wrappers this extension registers? + * + * Mappings name their servers, and a server is resolved back to its wrapper by + * the wrapper's name. Renaming one therefore breaks every mapping lookup at + * once -- including on the DROP path, which then cannot tell the metadata engine + * anything. Refused whether or not a table exists yet, because the breakage is + * in the lookup rather than in any particular table. + */ +static bool +is_module_fdw_name(const char *fdwname) +{ + return fdwname != NULL && + (strcmp(fdwname, "iceberg_catalog_fdw") == 0 || + strcmp(fdwname, "iceberg_volume_fdw") == 0); +} + +static const char * +find_reloption(List *options, const char *name) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, name) == 0) + return defGetString(def); + } + + return NULL; +} + +static void +lock_create_servers(Oid catalog_srvid, Oid volume_srvid, LOCKMODE lockmode) +{ + /* + * Ascending OID order, so that two CREATEs naming the same pair in opposite + * order cannot deadlock. ALTER-side server guards take the + * AccessExclusiveLock counterpart before scanning pg_depend. + */ + if (catalog_srvid < volume_srvid) + { + LockDatabaseObject(ForeignServerRelationId, catalog_srvid, 0, lockmode); + LockDatabaseObject(ForeignServerRelationId, volume_srvid, 0, lockmode); + } + else if (volume_srvid < catalog_srvid) + { + LockDatabaseObject(ForeignServerRelationId, volume_srvid, 0, lockmode); + LockDatabaseObject(ForeignServerRelationId, catalog_srvid, 0, lockmode); + } + else + LockDatabaseObject(ForeignServerRelationId, catalog_srvid, 0, lockmode); +} + +static void +unlock_create_servers(Oid catalog_srvid, Oid volume_srvid, LOCKMODE lockmode) +{ + UnlockDatabaseObject(ForeignServerRelationId, catalog_srvid, 0, lockmode); + if (volume_srvid != catalog_srvid) + UnlockDatabaseObject(ForeignServerRelationId, volume_srvid, 0, lockmode); +} + +static void +validate_create_binding(const char *catalog_name, const char *volume_name) +{ + ForeignServer *catalog_server; + ForeignServer *volume_server; + + /* + * Resolve, lock, and only then read what was locked -- the same + * lookup-lock-recheck the guards use, for the same reason. Two things go + * wrong without it: the name can come to denote a different server while + * this backend waits for the lock, so the checks below would describe a + * server the statement no longer names; and a concurrent ALTER SERVER that + * commits during that wait leaves any copy fetched beforehand stale, so the + * definitive read has to happen afterwards. + */ + for (;;) + { + Oid catalog_srvid = get_foreign_server_oid(catalog_name, false); + Oid volume_srvid = get_foreign_server_oid(volume_name, false); + + lock_create_servers(catalog_srvid, volume_srvid, AccessShareLock); + + if (get_foreign_server_oid(catalog_name, true) == catalog_srvid && + get_foreign_server_oid(volume_name, true) == volume_srvid) + { + catalog_server = GetForeignServer(catalog_srvid); + volume_server = GetForeignServer(volume_srvid); + break; + } + + unlock_create_servers(catalog_srvid, volume_srvid, AccessShareLock); + } + + if (catalog_server->fdwid != pg_iceberg_catalog_fdw_oid(false)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("server \"%s\" is not an iceberg catalog server", + catalog_name))); + if (volume_server->fdwid != pg_iceberg_volume_fdw_oid(false)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("server \"%s\" is not an iceberg volume server", + volume_name))); + + pg_iceberg_check_server_usage(catalog_server->serverid); + pg_iceberg_check_server_usage(volume_server->serverid); +} + +static void +prepare_iceberg_create(CreateStmt *stmt) +{ + DistributedBy *distributed_by; + const char *catalog_name; + const char *volume_name; + + if (stmt->accessMethod == NULL) + stmt->accessMethod = pstrdup("iceberg"); + + /* + * A lake table is distributed randomly: rows live outside PostgreSQL, so no + * local key can describe where they are. That policy is injected below. + * + * A QE receives the policy the QD injected and transformed, including the + * resolved segment count, so it accepts exactly that shape. + * + * On the dispatcher, an explicit clause is accepted only when it asks for + * what would have been injected anyway. Refusing every clause looks + * stricter but is wrong in one case that matters: pg_dump writes + * DISTRIBUTED RANDOMLY into the CREATE TABLE it emits, so refusing it means + * refusing to restore a dump this module produced. A clause naming columns + * still cannot be honoured and is refused with the syntax the user wrote. + */ + if (stmt->distributedBy != NULL) + { + if (stmt->distributedBy->ptype != POLICYTYPE_PARTITIONED) + pg_iceberg_not_supported( + stmt->distributedBy->ptype == POLICYTYPE_REPLICATED ? + "DISTRIBUTED REPLICATED" : "this distribution policy"); + if (stmt->distributedBy->keyCols != NIL) + pg_iceberg_not_supported("DISTRIBUTED BY"); + } + else if (Gp_role == GP_ROLE_EXECUTE) + pg_iceberg_not_supported("DISTRIBUTED BY"); + if (stmt->partspec != NULL || stmt->partbound != NULL) + pg_iceberg_not_supported("partitioned tables"); + if (stmt->inhRelations != NIL) + pg_iceberg_not_supported("INHERITS"); + if (stmt->ofTypename != NULL) + pg_iceberg_not_supported("typed tables (OF type)"); + if (stmt->relation->relpersistence == RELPERSISTENCE_TEMP) + pg_iceberg_not_supported("TEMP tables"); + if (stmt->relation->relpersistence == RELPERSISTENCE_UNLOGGED) + pg_iceberg_not_supported("UNLOGGED tables"); + if (stmt->oncommit != ONCOMMIT_NOOP) + pg_iceberg_not_supported("ON COMMIT"); + if (stmt->tablespacename != NULL) + pg_iceberg_not_supported("TABLESPACE"); + + if (Gp_role != GP_ROLE_EXECUTE) + { + distributed_by = makeNode(DistributedBy); + distributed_by->ptype = POLICYTYPE_PARTITIONED; + distributed_by->numsegments = -1; + distributed_by->keyCols = NIL; + stmt->distributedBy = distributed_by; + } + + catalog_name = find_reloption(stmt->options, "catalog"); + if (catalog_name == NULL) + { + if (iceberg_default_catalog == NULL || + iceberg_default_catalog[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("no catalog specified"), + errhint("Specify WITH (catalog = '...') or SET iceberg.default_catalog."))); + + stmt->options = lappend(stmt->options, + makeDefElem("catalog", + (Node *) makeString( + pstrdup(iceberg_default_catalog)), + -1)); + catalog_name = iceberg_default_catalog; + } + + volume_name = find_reloption(stmt->options, "volume"); + if (volume_name == NULL) + { + if (iceberg_default_volume == NULL || + iceberg_default_volume[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("no volume specified"), + errhint("Specify WITH (volume = '...') or SET iceberg.default_volume."))); + + stmt->options = lappend(stmt->options, + makeDefElem("volume", + (Node *) makeString( + pstrdup(iceberg_default_volume)), + -1)); + volume_name = iceberg_default_volume; + } + + /* + * This makes the QD fail before dispatch; QEs repeat the checks against + * their local catalog copies. + */ + validate_create_binding(catalog_name, volume_name); +} + +static void +reject_utility_mode_ddl(const char *subject) +{ + /* + * A utility-mode backend would create, alter, or drop only local catalog + * state, without dispatch. That would break the invariant that every node + * agrees about a lake table's mapping and about the servers it names. + */ + pg_iceberg_not_supported(psprintf("utility-mode DDL on %s", subject)); +} + +static void +reject_targeted_operation(const char *operation) +{ + if (Gp_role == GP_ROLE_UTILITY) + reject_utility_mode_ddl("iceberg tables"); + pg_iceberg_not_supported(operation); +} + +static void +pg_iceberg_ProcessUtility(PlannedStmt *pstmt, + const char *queryString, + bool readOnlyTree, + ProcessUtilityContext context, + ParamListInfo params, + QueryEnvironment *queryEnv, + DestReceiver *dest, + QueryCompletion *qc) +{ + Node *parsetree = pstmt->utilityStmt; + + switch (nodeTag(parsetree)) + { + case T_CreateStmt: + { + CreateStmt *stmt = (CreateStmt *) parsetree; + + if (iceberg_is_effective_am(stmt->accessMethod)) + { + if (Gp_role == GP_ROLE_UTILITY) + reject_utility_mode_ddl("iceberg tables"); + + /* + * Every successful Iceberg CREATE is mutated. Preserve a + * protected parse tree by replacing the PlannedStmt and + * mutating only its copy. + */ + if (readOnlyTree) + { + pstmt = copyObject(pstmt); + readOnlyTree = false; + parsetree = pstmt->utilityStmt; + stmt = (CreateStmt *) parsetree; + } + prepare_iceberg_create(stmt); + } + } + break; + + case T_CreateTableAsStmt: + { + CreateTableAsStmt *stmt = (CreateTableAsStmt *) parsetree; + + if (stmt->into != NULL && + iceberg_is_effective_am(stmt->into->accessMethod)) + reject_targeted_operation( + "CREATE TABLE AS / CREATE MATERIALIZED VIEW"); + } + break; + + case T_AlterTableStmt: + { + AlterTableStmt *stmt = (AlterTableStmt *) parsetree; + + if (rangevar_is_iceberg(stmt->relation)) + { + /* + * Utility mode is refused for every form, including the one + * accepted below: core does not dispatch a utility-mode + * ALTER TABLE, so an owner change made there would land on + * the connected node alone and leave the catalogs + * disagreeing about who owns the table. + */ + if (Gp_role == GP_ROLE_UTILITY) + reject_utility_mode_ddl("iceberg tables"); + + if (!alter_table_is_owner_only(stmt)) + reject_targeted_operation( + "ALTER TABLE on iceberg tables"); + } + + /* + * Converting some other table INTO a lake table has to be + * refused here as well, and the guard above does not see it: + * the relation is still a heap when the statement arrives. + * Left alone, the rewrite would reach the metadata engine + * through the transient relation it builds -- under a + * generated name, with none of the checks CREATE TABLE makes, + * including whether the user may use the servers at all. + */ + if (alter_table_targets_iceberg_am(stmt)) + reject_targeted_operation( + "ALTER TABLE ... SET ACCESS METHOD iceberg"); + } + break; + + case T_RenameStmt: + { + RenameStmt *stmt = (RenameStmt *) parsetree; + + if ((stmt->renameType == OBJECT_TABLE || + stmt->renameType == OBJECT_COLUMN) && + rangevar_is_iceberg(stmt->relation)) + reject_targeted_operation("RENAME on iceberg tables"); + if (stmt->renameType == OBJECT_FOREIGN_SERVER && + locked_server_referenced_by_iceberg( + string_object_name(stmt->object))) + reject_targeted_operation( + "RENAME on servers referenced by iceberg tables"); + if (stmt->renameType == OBJECT_SCHEMA && + schema_has_iceberg_table(stmt->subname)) + reject_targeted_operation( + "RENAME on schemas containing iceberg tables"); + if (stmt->renameType == OBJECT_FDW && + is_module_fdw_name(string_object_name(stmt->object))) + reject_targeted_operation( + "RENAME on the iceberg foreign-data wrappers"); + } + break; + + case T_AlterObjectSchemaStmt: + { + AlterObjectSchemaStmt *stmt = + (AlterObjectSchemaStmt *) parsetree; + + if (stmt->objectType == OBJECT_TABLE && + rangevar_is_iceberg(stmt->relation)) + reject_targeted_operation( + "SET SCHEMA on iceberg tables"); + } + break; + + case T_AlterOwnerStmt: + { + AlterOwnerStmt *stmt = (AlterOwnerStmt *) parsetree; + + if (stmt->objectType == OBJECT_TABLE && + rangevar_is_iceberg(stmt->relation)) + reject_targeted_operation( + "ALTER OWNER on iceberg tables"); + if (stmt->objectType == OBJECT_FOREIGN_SERVER && + locked_server_referenced_by_iceberg( + string_object_name(stmt->object))) + reject_targeted_operation( + "ALTER OWNER on servers referenced by iceberg tables"); + } + break; + + case T_AlterForeignServerStmt: + { + AlterForeignServerStmt *stmt = + (AlterForeignServerStmt *) parsetree; + + /* + * Utility mode first, and for any server of ours rather than + * only for one a table already names. Core does not dispatch a + * utility-mode ALTER SERVER, so the options would change on the + * connected node alone; a table created afterwards would then + * resolve the same server name to different options depending on + * which node resolved it, and nothing downstream would notice. + * The window is before any dependency exists, which is precisely + * what the reference check below cannot see. + */ + if (Gp_role == GP_ROLE_UTILITY && + server_belongs_to_module(stmt->servername)) + reject_utility_mode_ddl("iceberg servers"); + + /* Both VERSION and OPTIONS forms use this parse node. */ + if (locked_server_referenced_by_iceberg(stmt->servername)) + reject_targeted_operation( + "ALTER SERVER on servers referenced by iceberg tables"); + } + break; + + case T_VacuumStmt: + /* + * Plain VACUUM and ANALYZE are no-ops for iceberg tables and stay + * allowed, but VACUUM FULL must be refused here rather than in the + * table AM: rewriting a relation first creates a transient one, + * which reaches OAT_POST_CREATE and makes the metadata engine + * create a table in the remote catalog. The subsequent + * relation_copy_for_cluster error rolls back the local catalog, + * yet the remote side would keep an orphan behind. + */ + { + VacuumStmt *stmt = (VacuumStmt *) parsetree; + ListCell *lc; + bool is_full = false; + + foreach(lc, stmt->options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, "full") == 0) + is_full = defGetBoolean(def); + } + + if (is_full) + { + /* + * A database-wide VACUUM FULL names no relation, so there + * is nothing to match: refuse it outright while any lake + * table exists, rather than let it reach one and rewrite + * it. + */ + if (stmt->rels == NIL) + { + if (database_has_iceberg_table()) + reject_targeted_operation( + "VACUUM FULL while iceberg tables exist"); + } + else + { + foreach(lc, stmt->rels) + { + VacuumRelation *vrel = (VacuumRelation *) lfirst(lc); + + if (rangevar_is_iceberg(vrel->relation)) + reject_targeted_operation( + "VACUUM FULL on iceberg tables"); + } + } + } + } + break; + + case T_DropStmt: + /* + * Plain DROP TABLE is supported outside utility mode; OAT_DROP + * performs the engine call. DROP SERVER protection otherwise + * comes from the dependencies recorded on every node. + */ + if (Gp_role == GP_ROLE_UTILITY && + utility_drop_targets_iceberg((DropStmt *) parsetree)) + reject_utility_mode_ddl("iceberg tables"); + break; + + default: + break; + } + + if (prev_ProcessUtility_hook) + (*prev_ProcessUtility_hook) (pstmt, queryString, readOnlyTree, + context, params, queryEnv, dest, qc); + else + standard_ProcessUtility(pstmt, queryString, readOnlyTree, + context, params, queryEnv, dest, qc); +} + +void +_PG_init(void) +{ + if (!process_shared_preload_libraries_in_progress) + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("datalake_fdw must be loaded via shared_preload_libraries"), + errhint("Add \"datalake_fdw\" to shared_preload_libraries and restart the server."))); + + pg_iceberg_define_gucs(); + pg_iceberg_register_reloptions(); + DatalakeRegisterMetaEngines(); + datalake_register_storage_backends(); + + prev_ProcessUtility_hook = ProcessUtility_hook; + ProcessUtility_hook = pg_iceberg_ProcessUtility; + + pg_iceberg_prev_object_access_hook = object_access_hook; + object_access_hook = pg_iceberg_object_access; +} diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c new file mode 100644 index 00000000000..71dc5c53ef1 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c @@ -0,0 +1,67 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_guc.c + * Configuration variables for Iceberg table creation. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "am_iceberg/pg_iceberg_guc.h" +#include "utils/guc.h" + +char *iceberg_default_catalog; +char *iceberg_default_volume; + +void +pg_iceberg_define_gucs(void) +{ + /* + * Do not install check hooks for these names. The servers they name are + * not necessarily present at assignment time, and assignment happens at + * different moments on the coordinator and on the segments. CREATE TABLE + * validates the values against the catalog the executing backend sees. + */ + DefineCustomStringVariable("iceberg.default_catalog", + "Default catalog server for new Iceberg tables.", + NULL, + &iceberg_default_catalog, + "", + PGC_USERSET, + 0, + NULL, + NULL, + NULL); + + DefineCustomStringVariable("iceberg.default_volume", + "Default volume server for new Iceberg tables.", + NULL, + &iceberg_default_volume, + "", + PGC_USERSET, + 0, + NULL, + NULL, + NULL); +} diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h new file mode 100644 index 00000000000..8cd8d0a4400 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h @@ -0,0 +1,37 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_guc.h + * Configuration variables for Iceberg table creation. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_guc.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_ICEBERG_GUC_H +#define PG_ICEBERG_GUC_H + +extern char *iceberg_default_catalog; +extern char *iceberg_default_volume; + +extern void pg_iceberg_define_gucs(void); + +#endif /* PG_ICEBERG_GUC_H */ diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c new file mode 100644 index 00000000000..6475546223c --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c @@ -0,0 +1,616 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_options.c + * How a lake table names its catalog and volume, and how that is read back. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/relation.h" +#include "access/reloptions.h" +#include "am_iceberg/pg_iceberg_options.h" +#include "catalog/dependency.h" +#include "catalog/objectaddress.h" +#include "catalog/pg_class.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_user_mapping.h" +#include "commands/defrem.h" +#include "foreign/foreign.h" +#include "iceberg_catalog_fdw/iceberg_catalog_option.h" +#include "iceberg_volume_fdw/iceberg_volume_option.h" +#include "miscadmin.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/syscache.h" + +typedef struct IcebergRelOptions +{ + int32 vl_len_; + int catalog_off; + int volume_off; + int fileformat_off; + bool purge_on_drop; +} IcebergRelOptions; + +static relopt_kind iceberg_relopt_kind; + +static char *iceberg_relopt_string(IcebergRelOptions *opts, int off); +static MetaKv *defelems_to_kvs(List *options, int *n_props); +static DlErrCode invalid_location(char **errdetail, char *detail); +static bool s3_bucket_alnum(char ch); +static bool s3_bucket_char(char ch); + +/* + * Register a private reloption kind for iceberg table access-method options. + */ +void +pg_iceberg_register_reloptions(void) +{ + if (iceberg_relopt_kind != 0) + return; + + iceberg_relopt_kind = add_reloption_kind(); + + add_string_reloption(iceberg_relopt_kind, "catalog", + "iceberg catalog foreign server name", "", NULL, + AccessExclusiveLock); + add_string_reloption(iceberg_relopt_kind, "volume", + "iceberg volume foreign server name", "", NULL, + AccessExclusiveLock); + add_string_reloption(iceberg_relopt_kind, "fileformat", + "iceberg data file format", "parquet", NULL, + AccessExclusiveLock); + + /* + * Dropping the table means dropping this database's reference to it; the + * data belongs to the lake and stays there. Deleting it as well has to be + * asked for, and the answer belongs to the table rather than to a session: + * a setting could make the same DROP destroy data or not depending on who + * typed it, and a reloption travels with the table into a dump. + */ + add_bool_reloption(iceberg_relopt_kind, "purge_on_drop", + "delete the lake data when the table is dropped", + false, AccessExclusiveLock); +} + +bytea * +pg_iceberg_amoptions(Datum reloptions, char relkind, bool validate) +{ + static const relopt_parse_elt tab[] = { + {"catalog", RELOPT_TYPE_STRING, + offsetof(IcebergRelOptions, catalog_off)}, + {"volume", RELOPT_TYPE_STRING, + offsetof(IcebergRelOptions, volume_off)}, + {"fileformat", RELOPT_TYPE_STRING, + offsetof(IcebergRelOptions, fileformat_off)}, + {"purge_on_drop", RELOPT_TYPE_BOOL, + offsetof(IcebergRelOptions, purge_on_drop)} + }; + + Assert(iceberg_relopt_kind != 0); + + /* + * The option set does not depend on relkind; the same mapping applies to + * every relation kind that can carry this access method. + */ + + /* + * Whether the named servers exist is deliberately not checked here: + * amoptions runs in relcache and utility contexts where such lookups are + * unsafe or premature. The use points check instead. + */ + return (bytea *) build_reloptions(reloptions, validate, + iceberg_relopt_kind, + sizeof(IcebergRelOptions), + tab, lengthof(tab)); +} + +/* + * rd_options belongs to the relcache. Every returned string is copied into + * the caller's current memory context; callers must never retain a pointer + * into rd_options itself. + */ +static char * +iceberg_relopt_string(IcebergRelOptions *opts, int off) +{ + if (opts == NULL || off == 0) + return NULL; + + return pstrdup(((char *) opts) + off); +} + +static MetaKv * +defelems_to_kvs(List *options, int *n_props) +{ + MetaKv *props; + ListCell *lc; + int i = 0; + int count = list_length(options); + + *n_props = count; + if (count == 0) + return NULL; + + props = palloc0(sizeof(MetaKv) * count); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + props[i].key = pstrdup(def->defname); + props[i].value = pstrdup(defGetString(def)); + i++; + } + + return props; +} + +Oid +pg_iceberg_catalog_fdw_oid(bool missing_ok) +{ + ForeignDataWrapper *fdw; + + fdw = GetForeignDataWrapperByName("iceberg_catalog_fdw", missing_ok); + return fdw == NULL ? InvalidOid : fdw->fdwid; +} + +Oid +pg_iceberg_volume_fdw_oid(bool missing_ok) +{ + ForeignDataWrapper *fdw; + + fdw = GetForeignDataWrapperByName("iceberg_volume_fdw", missing_ok); + return fdw == NULL ? InvalidOid : fdw->fdwid; +} + +/* + * OID of this extension's access method, or InvalidOid while it does not + * exist. + * + * Missing-ok because the AM is absent while CREATE EXTENSION is still + * installing it. The result is deliberately not cached: DROP EXTENSION + * followed by CREATE EXTENSION produces a new OID, and a stale cached one + * would make callers silently treat lake tables as ordinary relations. The + * lookup is syscache-backed. + */ +Oid +pg_iceberg_am_oid(void) +{ + return get_table_am_oid("iceberg", true); +} + +IcebergTableInfo * +pg_iceberg_get_table_info_rel(Relation rel) +{ + IcebergRelOptions *opts; + IcebergTableInfo *info; + IcebergCatalogOptions *catalog_options; + IcebergVolumeOptions *volume_options; + ForeignServer *catalog_server; + ForeignServer *volume_server; + char *catalog_server_name; + char *volume_server_name; + char *parse_detail = NULL; + DlErrCode parse_result; + Oid amoid; + + /* + * rd_options is only an IcebergRelOptions when this access method put it + * there. Every other access method has its own layout -- a heap's + * StdRdOptions would present fillfactor and toast_tuple_target where the + * string offsets are read below, and pstrdup() would then run off the + * allocation. The callers in this module check the access method before + * getting here, but this function is reachable from anywhere, so it cannot + * rely on that. + */ + amoid = pg_iceberg_am_oid(); + if (!OidIsValid(amoid) || rel->rd_rel->relam != amoid) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not an iceberg table", + RelationGetRelationName(rel)))); + + opts = (IcebergRelOptions *) rel->rd_options; + + catalog_server_name = iceberg_relopt_string(opts, + opts == NULL ? 0 : opts->catalog_off); + volume_server_name = iceberg_relopt_string(opts, + opts == NULL ? 0 : opts->volume_off); + + /* + * fileformat is a valid option and is persisted, but nothing consumes it + * until the format layer exists, so the result does not carry it. + */ + + if (catalog_server_name == NULL || catalog_server_name[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("iceberg table \"%s\" has no catalog binding", + RelationGetRelationName(rel)), + errhint("Specify WITH (catalog = '...', volume = '...'), or set " + "iceberg.default_catalog and iceberg.default_volume " + "before CREATE TABLE."))); + + if (volume_server_name == NULL || volume_server_name[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("iceberg table \"%s\" has no volume binding", + RelationGetRelationName(rel)), + errhint("Specify WITH (catalog = '...', volume = '...'), or set " + "iceberg.default_catalog and iceberg.default_volume " + "before CREATE TABLE."))); + + catalog_server = GetForeignServerByName(catalog_server_name, false); + if (catalog_server->fdwid != pg_iceberg_catalog_fdw_oid(false)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("server \"%s\" is not an iceberg catalog server", + catalog_server_name))); + + volume_server = GetForeignServerByName(volume_server_name, false); + if (volume_server->fdwid != pg_iceberg_volume_fdw_oid(false)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_TABLE_DEFINITION), + errmsg("server \"%s\" is not an iceberg volume server", + volume_server_name))); + + catalog_options = get_iceberg_catalog_options(catalog_server); + volume_options = get_iceberg_volume_options(volume_server); + + info = palloc0(sizeof(IcebergTableInfo)); + info->catalog_name = pstrdup(catalog_options->foreign_catalog.catalog_name); + info->catalog_server_name = catalog_server_name; + info->volume_name = pstrdup(volume_server->servername); + info->volume_server_name = volume_server_name; + + info->opts = palloc0(sizeof(IcebergTableOptions)); + + /* + * Copied rather than aliased to info->catalog_name: the two fields are + * released independently. + */ + info->opts->catalog = pstrdup(info->catalog_name); + info->opts->namespace = get_namespace_name(RelationGetNamespace(rel)); + info->opts->table = pstrdup(RelationGetRelationName(rel)); + info->opts->location = NULL; + info->opts->purge_on_drop = opts != NULL && opts->purge_on_drop; + + info->catalog_srvid = catalog_server->serverid; + info->volume_srvid = volume_server->serverid; + info->catalog_props = defelems_to_kvs(catalog_server->options, + &info->n_catalog_props); + + parse_result = pg_iceberg_parse_location(volume_options->foreign_volume.base_path, + volume_options->volume_server.endpoint, + volume_options->volume_server.region, + &info->volume_location, + &parse_detail); + if (parse_result != DL_OK) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg volume server \"%s\" has an invalid %s", + volume_server_name, + DATALAKE_ICEBERG_VOLUME_BASE_PATH), + errdetail("%s", parse_detail))); + + /* + * Credential resolution intentionally does not happen here. Stub and + * DDL-only paths must be able to describe a table with zero credentials. + */ + return info; +} + +/* + * Same result from a relation OID. + * + * Not for use from an object access hook: while a relation is being dropped its + * relcache entry may already be gone, and a hook has to decide what to do about + * that rather than error out. Those callers open the relation themselves and + * use pg_iceberg_get_table_info_rel(). + */ +IcebergTableInfo * +pg_iceberg_get_table_info(Oid relid) +{ + Relation rel; + IcebergTableInfo *info; + + rel = try_relation_open(relid, AccessShareLock, false); + if (rel == NULL) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_OBJECT), + errmsg("lake table entry not found for relation %u", relid))); + + info = pg_iceberg_get_table_info_rel(rel); + relation_close(rel, AccessShareLock); + + return info; +} + +void +pg_iceberg_free_table_info(IcebergTableInfo *info) +{ + int i; + + if (info == NULL) + return; + + if (info->catalog_name) + pfree(info->catalog_name); + if (info->catalog_server_name) + pfree(info->catalog_server_name); + if (info->volume_name) + pfree(info->volume_name); + if (info->volume_server_name) + pfree(info->volume_server_name); + + if (info->opts) + { + if (info->opts->catalog) + pfree(info->opts->catalog); + if (info->opts->namespace) + pfree(info->opts->namespace); + if (info->opts->table) + pfree(info->opts->table); + if (info->opts->location) + pfree(info->opts->location); + pfree(info->opts); + } + + /* + * The parsed location and the property array are allocated by this module + * too; a destructor that released only the names would let a statement + * resolving many tables accumulate the rest until its context is reset. + */ + if (info->volume_location.scheme) + pfree(info->volume_location.scheme); + if (info->volume_location.authority) + pfree(info->volume_location.authority); + if (info->volume_location.path_prefix) + pfree(info->volume_location.path_prefix); + if (info->volume_location.endpoint) + pfree(info->volume_location.endpoint); + if (info->volume_location.region) + pfree(info->volume_location.region); + + for (i = 0; i < info->n_catalog_props; i++) + { + if (info->catalog_props[i].key) + pfree(info->catalog_props[i].key); + if (info->catalog_props[i].value) + pfree(info->catalog_props[i].value); + } + if (info->catalog_props) + pfree(info->catalog_props); + + pfree(info); +} + +/* + * Make the mapping enforceable by recording what the table depends on. + * + * The mapping itself needs no insertion: it is part of the relation, written by + * the CREATE TABLE that produced it. What has to be added is the pair of + * dependency rows that stop either server from being dropped out from under the + * table, and that carry it along when one is dropped with CASCADE. Recorded on + * the dispatcher and on every segment, so that DROP SERVER is refused locally on + * whichever node first evaluates it. + * + * There is deliberately no RemoveLakeTableEntry() counterpart: both the + * reloptions and these rows belong to the relation, so they are removed by the + * same delete that removes it. + */ +void +InsertLakeTableEntry(Oid relid, const IcebergTableInfo *info) +{ + ObjectAddress table; + ObjectAddress server; + + ObjectAddressSet(table, RelationRelationId, relid); + + ObjectAddressSet(server, ForeignServerRelationId, info->catalog_srvid); + recordDependencyOn(&table, &server, DEPENDENCY_NORMAL); + + ObjectAddressSet(server, ForeignServerRelationId, info->volume_srvid); + recordDependencyOn(&table, &server, DEPENDENCY_NORMAL); +} + +MetaKv * +pg_iceberg_resolve_credentials(Oid serverid, Oid auth_userid, int *n_props) +{ + HeapTuple tuple; + Datum options_datum; + bool isnull; + List *options; + MetaKv *props; + + Assert(n_props != NULL); + *n_props = 0; + + /* + * GetUserMapping() cannot be used here because it ereports when neither a + * user-specific nor PUBLIC mapping exists. User mappings are optional in + * v1, so perform the same two syscache probes with missing-ok semantics. + */ + tuple = SearchSysCache2(USERMAPPINGUSERSERVER, + ObjectIdGetDatum(auth_userid), + ObjectIdGetDatum(serverid)); + if (!HeapTupleIsValid(tuple) && OidIsValid(auth_userid)) + tuple = SearchSysCache2(USERMAPPINGUSERSERVER, + ObjectIdGetDatum(InvalidOid), + ObjectIdGetDatum(serverid)); + + if (!HeapTupleIsValid(tuple)) + return NULL; + + options_datum = SysCacheGetAttr(USERMAPPINGUSERSERVER, tuple, + Anum_pg_user_mapping_umoptions, + &isnull); + if (isnull) + { + ReleaseSysCache(tuple); + return NULL; + } + + options = untransformRelOptions(options_datum); + props = defelems_to_kvs(options, n_props); + ReleaseSysCache(tuple); + + return props; +} + +void +pg_iceberg_check_server_usage(Oid serverid) +{ + AclResult aclresult; + + aclresult = object_aclcheck(ForeignServerRelationId, serverid, + GetUserId(), ACL_USAGE); + if (aclresult == ACLCHECK_NO_PRIV) + aclcheck_error(aclresult, OBJECT_FOREIGN_SERVER, + GetForeignServer(serverid)->servername); +} + +static DlErrCode +invalid_location(char **errdetail, char *detail) +{ + if (errdetail != NULL) + *errdetail = detail; + else + pfree(detail); + + return DL_ERR_INVALID_OPTION; +} + +static bool +s3_bucket_alnum(char ch) +{ + return (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9'); +} + +static bool +s3_bucket_char(char ch) +{ + return s3_bucket_alnum(ch) || ch == '.' || ch == '-'; +} + +DlErrCode +pg_iceberg_parse_location(const char *uri, const char *endpoint, + const char *region, DatalakeLocation *out, + char **errdetail) +{ + const char *scheme_end; + const char *authority_start; + const char *path_start; + Size scheme_len; + Size authority_len; + Size path_len; + bool is_s3; + Size i; + + Assert(out != NULL); + memset(out, 0, sizeof(*out)); + if (errdetail != NULL) + *errdetail = NULL; + + if (uri == NULL) + return invalid_location(errdetail, + pstrdup("location URI is null")); + + scheme_end = strstr(uri, "://"); + if (scheme_end == NULL) + return invalid_location(errdetail, + psprintf("location URI \"%s\" is missing \"://\"", + uri)); + + scheme_len = scheme_end - uri; + is_s3 = scheme_len == strlen(DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_S3) && + strncmp(uri, DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_S3, scheme_len) == 0; + if (!is_s3 && + !(scheme_len == strlen(DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_HDFS) && + strncmp(uri, DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_HDFS, scheme_len) == 0)) + return invalid_location(errdetail, + psprintf("location URI \"%s\" has unsupported scheme; expected %s or %s", + uri, + DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_S3, + DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_HDFS)); + + if (strchr(uri, '?') != NULL) + return invalid_location(errdetail, + psprintf("location URI \"%s\" must not contain a query", + uri)); + if (strchr(uri, '#') != NULL) + return invalid_location(errdetail, + psprintf("location URI \"%s\" must not contain a fragment", + uri)); + + authority_start = scheme_end + 3; + path_start = strchr(authority_start, '/'); + authority_len = path_start == NULL ? + strlen(authority_start) : (Size) (path_start - authority_start); + + if (authority_len == 0) + return invalid_location(errdetail, + psprintf("location URI \"%s\" has an empty authority", + uri)); + if (memchr(authority_start, '@', authority_len) != NULL) + return invalid_location(errdetail, + psprintf("location URI \"%s\" must not contain userinfo", + uri)); + + if (is_s3) + { + if (authority_len < 3 || authority_len > 63) + return invalid_location(errdetail, + psprintf("s3 bucket in location URI \"%s\" must be 3 to 63 characters", + uri)); + if (!s3_bucket_alnum(authority_start[0]) || + !s3_bucket_alnum(authority_start[authority_len - 1])) + return invalid_location(errdetail, + psprintf("s3 bucket in location URI \"%s\" must start and end with a lowercase letter or digit", + uri)); + for (i = 0; i < authority_len; i++) + { + if (!s3_bucket_char(authority_start[i])) + return invalid_location(errdetail, + psprintf("s3 bucket in location URI \"%s\" contains an invalid character", + uri)); + } + } + + path_len = path_start == NULL ? 0 : strlen(path_start); + while (path_len > 0 && path_start[path_len - 1] == '/') + path_len--; + + out->schema_version = DATALAKE_LOCATION_SCHEMA_VERSION; + out->scheme = pnstrdup(uri, scheme_len); + out->authority = pnstrdup(authority_start, authority_len); + out->path_prefix = path_len == 0 ? + pstrdup("") : pnstrdup(path_start, path_len); + out->endpoint = endpoint == NULL ? NULL : pstrdup(endpoint); + out->region = region == NULL ? NULL : pstrdup(region); + + return DL_OK; +} diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h new file mode 100644 index 00000000000..d081ffacdb0 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h @@ -0,0 +1,155 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_options.h + * How a lake table names its catalog and volume, and how that is read back. + * + * A relation using this access method holds no data locally: its metadata lives + * in an external Iceberg catalog and its files live on external storage. The + * mapping from the relation to the two foreign servers that describe those + * places is therefore part of the table definition, and every operation on the + * table starts by reading it. + * + * pg_iceberg_get_table_info() is that read. It is the only place that knows + * where the mapping is stored -- here, the relation's own reloptions -- so the + * storage can be reconsidered later without touching a single caller. + * + * Its signature and result types are those of the reference implementation: the + * existing implementation of this feature that this work derives from and is + * meant to replace, which keeps the same mapping in a system catalog of its own + * that an extension cannot add. The difference stops inside this function, and + * code written against either one compiles against the other. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_options.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_ICEBERG_OPTIONS_H +#define PG_ICEBERG_OPTIONS_H + +#ifdef __cplusplus +/* + * IcebergTableOptions.namespace carries the reference implementation's field + * name, which is a keyword in C++. Keeping the name is what lets that code + * move here unchanged; the cost is that this header is C-only. A C++ layer + * that needs the mapping should be given an accessor rather than this struct. + */ +#error "pg_iceberg_options.h is C-only; see the comment above this #error" +#endif + +#include "postgres.h" + +#include "common/datalake_location.h" +#include "common/dl_err.h" +#include "meta/iceberg_meta_engine.h" +#include "utils/relcache.h" + +/* + * Reference implementation: IcebergTableOptions. + * + * Deferred, names kept for the port: autovacuum_enabled, compression, + * compression_level -- all of them write-path options. + */ +typedef struct IcebergTableOptions +{ + char *catalog; /* catalog name within the external catalog */ + char *namespace; /* namespace within the external catalog */ + char *table; /* table name within the external catalog */ + char *location; /* optional location override, NULL when the + * volume's own base path applies */ + + /* + * Whether DROP TABLE should delete the lake data as well. No counterpart + * in the reference implementation, which deletes it unconditionally; here + * the default is not to, so that dropping a reference cannot destroy data + * another reader still expects to find. + */ + bool purge_on_drop; +} IcebergTableOptions; + +/* + * Reference implementation: IcebergTableInfo. + * + * The reference implementation distinguishes a catalog object from the server + * hosting it, and likewise for volumes. An extension cannot add catalog or + * volume objects to the system catalogs, so here a server names exactly one of + * each and catalog_name/volume_name fall back to the server name. Keeping all + * four fields means a caller that reads either one still gets a usable answer. + * + * Fields below the marker have no counterpart in the reference implementation. + * They stay at the end so that the shared prefix keeps its layout. + */ +typedef struct IcebergTableInfo +{ + char *catalog_name; + char *catalog_server_name; + char *volume_name; + char *volume_server_name; + IcebergTableOptions *opts; + + /* --- extension-only, keep last --- */ + Oid catalog_srvid; /* for USAGE checks and dependency records */ + Oid volume_srvid; + DatalakeLocation volume_location; /* parsed once from the volume server */ + MetaKv *catalog_props; /* catalog server options, non-secret only */ + int n_catalog_props; +} IcebergTableInfo; + +extern void pg_iceberg_register_reloptions(void); +extern bytea *pg_iceberg_amoptions(Datum reloptions, char relkind, + bool validate); + +/* + * Reference implementation: pg_iceberg_get_table_info(). Errors out when relid + * is not a lake table with a resolvable mapping. + * + * The _rel variant is what this tree calls: reading reloptions needs the + * relation anyway, so a caller holding one should not pay for a second open. + * Callers reached from object access hooks must use it -- see the note in + * pg_iceberg_options.c. + */ +extern IcebergTableInfo *pg_iceberg_get_table_info(Oid relid); +extern IcebergTableInfo *pg_iceberg_get_table_info_rel(Relation rel); +extern void pg_iceberg_free_table_info(IcebergTableInfo *info); + +/* + * Record the dependency rows that make the mapping durable. Named after the + * reference implementation's catalog-side equivalent so that the two lifecycles + * read alike. There is no removal counterpart: both the reloptions and these + * rows belong to the relation, so the delete that removes it removes them too. + */ +extern void InsertLakeTableEntry(Oid relid, const IcebergTableInfo *info); + +/* OID of this extension's access method, InvalidOid when it does not exist. */ +extern Oid pg_iceberg_am_oid(void); + +extern MetaKv *pg_iceberg_resolve_credentials(Oid serverid, Oid auth_userid, + int *n_props); +extern void pg_iceberg_check_server_usage(Oid serverid); +extern DlErrCode pg_iceberg_parse_location(const char *uri, + const char *endpoint, + const char *region, + DatalakeLocation *out, + char **errdetail); +extern Oid pg_iceberg_catalog_fdw_oid(bool missing_ok); +extern Oid pg_iceberg_volume_fdw_oid(bool missing_ok); + +#endif /* PG_ICEBERG_OPTIONS_H */ diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.c b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.c new file mode 100644 index 00000000000..a90be0114d6 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.c @@ -0,0 +1,44 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_reject.c + * Common rejection path for unsupported Iceberg operations. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "am_iceberg/pg_iceberg_reject.h" + +void +pg_iceberg_not_supported(const char *operation) +{ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("iceberg: %s is not supported yet", operation))); +} + +/* ------------------------------------------------------------------------ + * Utility-statement reject matrix (placeholder for the hooks task). + * ------------------------------------------------------------------------ + */ diff --git a/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.h b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.h new file mode 100644 index 00000000000..2db11592e87 --- /dev/null +++ b/contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.h @@ -0,0 +1,36 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * pg_iceberg_reject.h + * Common rejection path for unsupported Iceberg operations. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/am_iceberg/pg_iceberg_reject.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PG_ICEBERG_REJECT_H +#define PG_ICEBERG_REJECT_H + +#include "c.h" + +extern void pg_iceberg_not_supported(const char *operation) pg_attribute_noreturn(); + +#endif /* PG_ICEBERG_REJECT_H */ diff --git a/contrib/datalake_fdw/src/common/backend_registry.cpp b/contrib/datalake_fdw/src/common/backend_registry.cpp new file mode 100644 index 00000000000..78fb04a5e33 --- /dev/null +++ b/contrib/datalake_fdw/src/common/backend_registry.cpp @@ -0,0 +1,122 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * backend_registry.cpp + * Registry of the storage backends, one per protocol. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/backend_registry.cpp + * + *------------------------------------------------------------------------- + */ + +#include "common/dl_pg_api.h" + +#include + +#include "common/backend_registry.h" +#include "common/dl_wrappers.h" + +typedef struct DatalakeStorageBackend +{ + const char *scheme; + const struct DatalakeStorageOps *ops; +} DatalakeStorageBackend; + +/* Room for s3 and hdfs, plus space to grow without revisiting this. */ +static DatalakeStorageBackend storage_backends[4]; +static int nstorage_backends; + +extern DlErrCode datalake_register_s3_backend(void); + +static bool +storage_ops_are_complete(const struct DatalakeStorageOps *ops) +{ + /* + * A partially filled table would turn into a null call at the first + * operation the backend forgot, so refuse it at registration instead. + */ + return ops != NULL && + ops->fs_open != NULL && + ops->fs_close != NULL && + ops->fs_list != NULL && + ops->file_open != NULL && + ops->file_read != NULL && + ops->file_write != NULL && + ops->file_close != NULL && + ops->file_abort != NULL; +} + +DlErrCode +datalake_register_storage_backend(const char *scheme, + const struct DatalakeStorageOps *ops) +{ + int i; + + if (scheme == NULL || scheme[0] == '\0' || !storage_ops_are_complete(ops)) + return DL_ERR_INVALID_OPTION; + + for (i = 0; i < nstorage_backends; i++) + { + if (strcmp(storage_backends[i].scheme, scheme) == 0) + return DL_ERR_ALREADY_EXISTS; + } + + if (nstorage_backends >= (int) lengthof(storage_backends)) + return DL_ERR_INTERNAL; + + storage_backends[nstorage_backends].scheme = scheme; + storage_backends[nstorage_backends].ops = ops; + nstorage_backends++; + + return DL_OK; +} + +const struct DatalakeStorageOps * +datalake_lookup_storage_backend(const char *scheme) +{ + int i; + + if (scheme == NULL) + return NULL; + + for (i = 0; i < nstorage_backends; i++) + { + if (strcmp(storage_backends[i].scheme, scheme) == 0) + return storage_backends[i].ops; + } + + return NULL; +} + +extern "C" void +datalake_register_storage_backends(void) +{ + DL_TRY + { + DlErrCode rc = datalake_register_s3_backend(); + + /* Registering twice is harmless; anything else is a coding error. */ + if (rc != DL_OK && rc != DL_ERR_ALREADY_EXISTS) + ereport(ERROR, + (errmsg("datalake_fdw: could not register the s3 storage backend: %s", + dl_err_message(rc)))); + } + DL_CATCH_END(); +} diff --git a/contrib/datalake_fdw/src/common/backend_registry.h b/contrib/datalake_fdw/src/common/backend_registry.h new file mode 100644 index 00000000000..5432502ed42 --- /dev/null +++ b/contrib/datalake_fdw/src/common/backend_registry.h @@ -0,0 +1,100 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * backend_registry.h + * Registry of the storage backends, one per protocol. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/backend_registry.h + * + *------------------------------------------------------------------------- + */ + +#ifndef BACKEND_REGISTRY_H +#define BACKEND_REGISTRY_H + +#include + +#include "common/file_system_wrapper.h" + +#ifdef __cplusplus + +/* + * One storage protocol's implementation of the facade in + * common/file_system_wrapper.h. The operations mirror it one for one, so a + * backend is written against the same contract its callers see. + */ +struct DatalakeStorageOps +{ + DlErrCode (*fs_open) (const DatalakeLocation *location, + const DlKeyValue *credentials, int ncredentials, + DatalakeFileSystem *fs_out); + void (*fs_close) (DatalakeFileSystem fs); /* releases fs */ + DlErrCode (*fs_list) (DatalakeFileSystem fs, const char *prefix, + char ***names_out, int *nnames_out); + DlErrCode (*file_open) (DatalakeFileSystem fs, const char *path, + DatalakeFileMode mode, DatalakeFile *file_out); + DlErrCode (*file_read) (DatalakeFile file, void *buffer, int64_t length, + int64_t *nread); + DlErrCode (*file_write) (DatalakeFile file, const void *buffer, + int64_t length); + DlErrCode (*file_close) (DatalakeFile file); /* releases file */ + void (*file_abort) (DatalakeFile file); /* releases file */ +}; + +/* + * Every handle a backend hands out starts with this field, which is how the + * facade finds its way back to the right operations. A handle lives until a + * cleanup entry point consumes it; there is no closed-but-alive state, because + * keeping one would mean either leaking every handle or letting a backend free + * memory the facade still reads. + */ +struct DatalakeFileSystemData +{ + const struct DatalakeStorageOps *ops; +}; + +struct DatalakeFileData +{ + const struct DatalakeStorageOps *ops; +}; + +extern DlErrCode datalake_register_storage_backend(const char *scheme, + const struct DatalakeStorageOps *ops); +extern const struct DatalakeStorageOps *datalake_lookup_storage_backend(const char *scheme); + +#endif /* __cplusplus */ + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* + * Registration is an explicit call rather than a static initializer: the order + * static initializers run in a shared module is not something to depend on, + * and _PG_init is where this is meant to happen. + */ +extern void datalake_register_storage_backends(void); + +#ifdef __cplusplus +} +#endif + +#endif /* BACKEND_REGISTRY_H */ diff --git a/contrib/datalake_fdw/src/common/datalake_location.h b/contrib/datalake_fdw/src/common/datalake_location.h new file mode 100644 index 00000000000..28bede6f71f --- /dev/null +++ b/contrib/datalake_fdw/src/common/datalake_location.h @@ -0,0 +1,50 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * datalake_location.h + * The canonical form of a lake table storage location. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/datalake_location.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DATALAKE_LOCATION_H +#define DATALAKE_LOCATION_H + +#include + +/* Canonical, versioned location form. URIs are parsed ONCE (options layer); + * every backend receives only this struct and must never re-parse URIs. */ +typedef struct DatalakeLocation { + uint32_t schema_version; /* = 1 */ + char *scheme; /* v1 whitelist: "s3" | "hdfs" */ + char *authority; /* s3: bucket (validated); hdfs: namenode[:port] */ + char *path_prefix; /* normalized: always starts with '/', never ends with '/' + * (a bare "/" normalizes to "") */ + char *endpoint; /* optional, may be NULL */ + char *region; /* optional, may be NULL */ +} DatalakeLocation; +#define DATALAKE_LOCATION_SCHEMA_VERSION 1 + +/* Join paths as full = path_prefix + "/" + relative; relative never starts + * with '/'. */ + +#endif /* DATALAKE_LOCATION_H */ diff --git a/contrib/datalake_fdw/src/common/dl_err.c b/contrib/datalake_fdw/src/common/dl_err.c new file mode 100644 index 00000000000..4780d9d44a0 --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_err.c @@ -0,0 +1,218 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_err.c + * Error codes shared by the layers below the access method, and the + * channel that carries what the code alone cannot say. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_err.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "common/dl_err.h" +#include "utils/guc.h" + +/* + * One record per backend. A backend runs one statement at a time, and nothing + * here outlives the report it feeds, so there is no reason to key this by + * anything finer. + */ +static DlErrorDetail dl_error_detail; + +static void dl_error_copy_field(char *dest, Size dest_size, const char *src); +static int dl_error_sqlstate(DlErrCode code); + +/* + * SQLSTATE for a failure below the access method. + * + * Only a defect in this module is an internal error. A remote catalog that + * refuses a name, or storage that will not answer, is a condition a client can + * act on and deserves a code that says so -- and reporting everything as + * ERRCODE_INTERNAL_ERROR has a second cost here: this server appends the + * raising source location to internal errors, which puts a file and line number + * into user-visible output and into every expected-output file. + */ +static int +dl_error_sqlstate(DlErrCode code) +{ + switch (code) + { + case DL_OK: + case DL_ERR_INTERNAL: + return ERRCODE_INTERNAL_ERROR; + case DL_ERR_NOT_SUPPORTED: + return ERRCODE_FEATURE_NOT_SUPPORTED; + case DL_ERR_INVALID_OPTION: + return ERRCODE_INVALID_PARAMETER_VALUE; + case DL_ERR_NOT_FOUND: + return ERRCODE_UNDEFINED_OBJECT; + case DL_ERR_ALREADY_EXISTS: + return ERRCODE_DUPLICATE_TABLE; + case DL_ERR_IO: + return ERRCODE_IO_ERROR; + } + + return ERRCODE_INTERNAL_ERROR; +} + +static void +dl_error_copy_field(char *dest, Size dest_size, const char *src) +{ + if (src == NULL) + { + dest[0] = '\0'; + return; + } + + /* Truncates rather than failing; see the header for why. */ + strlcpy(dest, src, dest_size); +} + +void +dl_error_reset(void) +{ + dl_error_detail.code = DL_OK; + dl_error_detail.remote_code = 0; + dl_error_detail.operation[0] = '\0'; + dl_error_detail.type[0] = '\0'; + dl_error_detail.message[0] = '\0'; + dl_error_detail.stack[0] = '\0'; +} + +void +dl_error_set(DlErrCode code, const char *operation, const char *type, + const char *message) +{ + dl_error_detail.code = code; + dl_error_detail.remote_code = 0; + dl_error_copy_field(dl_error_detail.operation, + sizeof(dl_error_detail.operation), operation); + dl_error_copy_field(dl_error_detail.type, + sizeof(dl_error_detail.type), type); + dl_error_copy_field(dl_error_detail.message, + sizeof(dl_error_detail.message), message); + dl_error_detail.stack[0] = '\0'; +} + +void +dl_error_set_remote_code(int remote_code) +{ + dl_error_detail.remote_code = remote_code; +} + +void +dl_error_set_stack(const char *stack) +{ + dl_error_copy_field(dl_error_detail.stack, + sizeof(dl_error_detail.stack), stack); +} + +const DlErrorDetail * +dl_error_get(void) +{ + return &dl_error_detail; +} + +const char * +dl_err_message(DlErrCode code) +{ + switch (code) + { + case DL_OK: + return "success"; + case DL_ERR_NOT_SUPPORTED: + return "operation not supported"; + case DL_ERR_INVALID_OPTION: + return "invalid option"; + case DL_ERR_NOT_FOUND: + return "not found"; + case DL_ERR_ALREADY_EXISTS: + return "already exists"; + case DL_ERR_IO: + return "I/O error"; + case DL_ERR_INTERNAL: + return "internal error"; + } + + return "unknown error"; +} + +void +dl_error_report(int elevel, DlErrCode code, const char *prefix) +{ + const DlErrorDetail *detail = dl_error_get(); + StringInfoData detail_buf; + bool has_detail; + + /* + * Detail recorded against a different code belongs to some other failure -- + * an implementation that reported this one without recording anything, for + * instance. Reporting it here would attribute the wrong cause. + */ + has_detail = (detail->code == code && + (detail->message[0] != '\0' || + detail->type[0] != '\0' || + detail->remote_code != 0)); + + if (!has_detail) + { + ereport(elevel, + (errcode(dl_error_sqlstate(code)), + errmsg("iceberg: %s failed: %s", prefix, + dl_err_message(code)))); + return; + } + + initStringInfo(&detail_buf); + + if (detail->operation[0] != '\0') + appendStringInfo(&detail_buf, "%s: ", detail->operation); + + if (detail->message[0] != '\0') + appendStringInfoString(&detail_buf, detail->message); + else + appendStringInfoString(&detail_buf, dl_err_message(code)); + + if (detail->type[0] != '\0') + appendStringInfo(&detail_buf, " (%s", detail->type); + if (detail->remote_code != 0) + appendStringInfo(&detail_buf, "%s%d", + detail->type[0] != '\0' ? ", code " : " (code ", + detail->remote_code); + if (detail->type[0] != '\0' || detail->remote_code != 0) + appendStringInfoChar(&detail_buf, ')'); + + /* + * A stack describes the implementation, not the statement, so it is offered + * only to a session that asked to see log-level detail. + */ + if (detail->stack[0] != '\0' && client_min_messages <= LOG) + appendStringInfo(&detail_buf, "\nStack:\n%s", detail->stack); + + ereport(elevel, + (errcode(dl_error_sqlstate(code)), + errmsg("iceberg: %s failed", prefix), + errdetail("%s", detail_buf.data))); + + pfree(detail_buf.data); +} diff --git a/contrib/datalake_fdw/src/common/dl_err.h b/contrib/datalake_fdw/src/common/dl_err.h new file mode 100644 index 00000000000..67949644c95 --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_err.h @@ -0,0 +1,120 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_err.h + * Error codes shared by the layers below the access method, and the + * channel that carries what the code alone cannot say. + * + * A code says which kind of failure occurred. It cannot say which table the + * remote catalog rejected, what the storage service answered, or where a remote + * implementation threw -- and those are the only things that make such a failure + * diagnosable. Every layer below the access method therefore reports a code and + * additionally records the detail here; the entry points that face PostgreSQL + * turn both into one ereport. + * + * The detail is recorded into fixed-size storage on purpose. Recording happens + * on paths that must not allocate and must not raise -- a cleanup callback + * crossing back from C++ is one of them -- so a setter that could palloc, and + * therefore could fail, is not usable there. Long values are truncated, which + * is the right trade against losing the report entirely. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_err.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_ERR_H +#define DL_ERR_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum DlErrCode { + DL_OK = 0, + DL_ERR_NOT_SUPPORTED, + DL_ERR_INVALID_OPTION, + DL_ERR_NOT_FOUND, + DL_ERR_ALREADY_EXISTS, + DL_ERR_IO, + DL_ERR_INTERNAL, +} DlErrCode; + +#define DL_ERR_FIELD_LEN 128 +#define DL_ERR_MSG_LEN 1024 +#define DL_ERR_STACK_LEN 4096 + +/* + * What an implementation below the access method has to say about its last + * failure. The field set follows what a remote metadata engine reports, so + * that connecting one is a matter of filling this in rather than changing it. + */ +typedef struct DlErrorDetail +{ + DlErrCode code; + int remote_code; /* implementation's own numeric code, 0 + * when it has none */ + char operation[DL_ERR_FIELD_LEN]; /* which call failed */ + char type[DL_ERR_FIELD_LEN]; /* implementation's error class */ + char message[DL_ERR_MSG_LEN]; + char stack[DL_ERR_STACK_LEN]; +} DlErrorDetail; + +/* + * Discard any recorded detail. Called by the dispatch wrappers before entering + * an implementation, so that a report can never describe an earlier failure. + */ +extern void dl_error_reset(void); + +/* + * Record the detail for a failure that is being reported as `code`. Allocates + * nothing and raises nothing, so it is callable from a cleanup path. NULL is + * accepted for any string and leaves that field empty. + */ +extern void dl_error_set(DlErrCode code, const char *operation, + const char *type, const char *message); + +/* Record the implementation's own numeric code, when it reports one. */ +extern void dl_error_set_remote_code(int remote_code); + +/* Record a stack from the failing implementation. */ +extern void dl_error_set_stack(const char *stack); + +/* The recorded detail, never NULL; code is DL_OK when nothing was recorded. */ +extern const DlErrorDetail *dl_error_get(void); + +/* Short, code-only wording, for when there is nothing recorded to add. */ +extern const char *dl_err_message(DlErrCode code); + +/* + * Report a failure to PostgreSQL: `prefix` names what was being attempted, the + * recorded message becomes the detail, and a recorded stack is included only + * when the session asked for log-level detail -- a stack is for whoever is + * debugging the implementation, not for whoever ran the statement. + * + * Detail recorded against a different code is ignored rather than misattributed. + */ +extern void dl_error_report(int elevel, DlErrCode code, const char *prefix); + +#ifdef __cplusplus +} +#endif + +#endif /* DL_ERR_H */ diff --git a/contrib/datalake_fdw/src/common/dl_kv.h b/contrib/datalake_fdw/src/common/dl_kv.h new file mode 100644 index 00000000000..f5db0db31d8 --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_kv.h @@ -0,0 +1,44 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_kv.h + * A configuration pair as options and mappings deliver it. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_kv.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_KV_H +#define DL_KV_H + +/* + * A configuration pair as it arrives from a foreign server, a user mapping or + * a table option. It lives in common/ because both the storage layer and the + * metadata layer consume such pairs, and neither should have to include the + * other's headers to name the type. + */ +typedef struct DlKeyValue +{ + char *key; + char *value; +} DlKeyValue; + +#endif /* DL_KV_H */ diff --git a/contrib/datalake_fdw/src/common/dl_option_util.c b/contrib/datalake_fdw/src/common/dl_option_util.c new file mode 100644 index 00000000000..c2cfbdaef54 --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_option_util.c @@ -0,0 +1,65 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_option_util.c + * Option policy shared by the catalog and volume option validators. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_option_util.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "common/dl_option_util.h" + +bool +dl_is_credential_option(const char *name) +{ + static const char *const credential_options[] = { + DL_OPTION_KEY_USERNAME, + DL_OPTION_KEY_KRB_CLIENT_KEYTAB, + DL_OPTION_KEY_CLIENT_ID, + DL_OPTION_KEY_CLIENT_SECRET, + DL_OPTION_KEY_ACCESS_KEY_ID, + DL_OPTION_KEY_SECRET_ACCESS_KEY, + DL_OPTION_KEY_SESSION_TOKEN, + + /* + * Not options this module accepts anywhere, but names users reach for + * out of habit. Listing them turns "unrecognized option" into the hint + * that says where credentials actually go. + */ + "user", + "password", + "token", + "access_key", + "secret_key" + }; + int i; + + for (i = 0; i < lengthof(credential_options); i++) + { + if (strcmp(name, credential_options[i]) == 0) + return true; + } + + return false; +} diff --git a/contrib/datalake_fdw/src/common/dl_option_util.h b/contrib/datalake_fdw/src/common/dl_option_util.h new file mode 100644 index 00000000000..2ac33fadf77 --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_option_util.h @@ -0,0 +1,63 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_option_util.h + * Option policy shared by the catalog and volume option validators. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_option_util.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_OPTION_UTIL_H +#define DL_OPTION_UTIL_H + +#include "postgres.h" + +/* + * The option keys that identify or authenticate a principal. + * + * They are defined here, below the option modules that name them, because + * dl_is_credential_option() and those modules have to agree: a key the modules + * accept but this list does not know is a credential the server would store in + * the clear. One definition each is what makes disagreement impossible; the + * per-module macros below are spelled the way the reference implementation + * spells them and resolve to these. + */ +#define DL_OPTION_KEY_USERNAME "username" +#define DL_OPTION_KEY_KRB_CLIENT_KEYTAB "krb_client_keytab" +#define DL_OPTION_KEY_CLIENT_ID "client_id" +#define DL_OPTION_KEY_CLIENT_SECRET "client_secret" +#define DL_OPTION_KEY_ACCESS_KEY_ID "access_key_id" +#define DL_OPTION_KEY_SECRET_ACCESS_KEY "secret_access_key" +#define DL_OPTION_KEY_SESSION_TOKEN "session_token" + +/* + * True for an option that identifies or authenticates a principal. Such an + * option belongs to a user mapping, never to a server, so that one server can + * be shared by roles with different credentials and so that the value is not + * readable through pg_foreign_server by every role holding USAGE. + * + * Both option validators consult this, which is why it lives here rather than + * being spelled out twice. + */ +extern bool dl_is_credential_option(const char *name); + +#endif /* DL_OPTION_UTIL_H */ diff --git a/contrib/datalake_fdw/src/common/dl_pg_api.h b/contrib/datalake_fdw/src/common/dl_pg_api.h new file mode 100644 index 00000000000..6584db45fff --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_pg_api.h @@ -0,0 +1,45 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_pg_api.h + * The PostgreSQL headers, safe to include from C++. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_pg_api.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_PG_API_H +#define DL_PG_API_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "postgres.h" + +#include "access/xact.h" +#include "utils/elog.h" + +#ifdef __cplusplus +} +#endif + +#endif /* DL_PG_API_H */ diff --git a/contrib/datalake_fdw/src/common/dl_wrappers.h b/contrib/datalake_fdw/src/common/dl_wrappers.h new file mode 100644 index 00000000000..737d97fae7d --- /dev/null +++ b/contrib/datalake_fdw/src/common/dl_wrappers.h @@ -0,0 +1,193 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * dl_wrappers.h + * The boundaries between C++ code and the PostgreSQL runtime. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/dl_wrappers.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_WRAPPERS_H +#define DL_WRAPPERS_H + +#include + +#include "common/dl_err.h" +#include "common/dl_pg_api.h" + +/* + * Exception-boundary classes: + * + * 1. PG-called C entry (extern "C" handler/UDF/hook): + * DL_TRY { } DL_CATCH_END(); converts C++ exceptions to ereport(ERROR, + * ...) at the boundary; no C++ exception may escape into PG stack frames. + * 2. Status-returning DlErrCode ABI (vtable implementations): functions must + * be noexcept; DL_ABI_GUARD_BEGIN / DL_ABI_GUARD_END(errvar) converts every + * exception to DL_ERR_INTERNAL. + * 3. Void cleanup ABI (close, abort, iterator close): functions must be + * noexcept and idempotent, and must never ereport. The cleanup guard logs + * a best-effort WARNING only outside error cleanup and otherwise swallows. + * 4. C++ calling PG APIs: DL_WRAP_START; ... DL_WRAP_END; converts a PG longjmp + * to DlPgError so the C++ caller can handle it without crossing the ABI. + */ + +#ifdef __cplusplus + +#include + +class DlPgError : public std::exception { +public: + const char *what() const noexcept override + { + return "PostgreSQL error"; + } +}; + +/* Save and restore PG's longjmp targets around a C++ call site. */ +class DlPgExceptionStack { +public: + DlPgExceptionStack(void **exception_stack, void **error_context_stack) + : exception_stack_(exception_stack), + error_context_stack_(error_context_stack), + saved_exception_stack_(*exception_stack), + saved_error_context_stack_(*error_context_stack) + { + } + + ~DlPgExceptionStack() + { + *exception_stack_ = saved_exception_stack_; + *error_context_stack_ = saved_error_context_stack_; + } + + void SetLocalJmp(void *local_jump) + { + *exception_stack_ = local_jump; + } + +private: + void **exception_stack_; + void **error_context_stack_; + void *saved_exception_stack_; + void *saved_error_context_stack_; +}; + +static inline bool +dl_can_log_cleanup_warning(void) +{ + return !in_error_recursion_trouble() && !IsAbortInProgress() && + !IsAbortedTransactionBlockState(); +} + +/* + * Class 1: a C entry point called by PostgreSQL. + * + * ereport(ERROR) unwinds with longjmp(), and longjmp()ing out of a C++ catch + * handler leaves the in-flight exception alive, which is undefined behavior. + * So the handler only records what happened -- the message is copied into a + * local buffer because the exception object dies with the handler -- and the + * ereport() happens after the try/catch statement has been left, the same way + * PAX defers it to CBDB_END_TRY(). + */ +#define DL_ERROR_MSG_MAX 512 + +#define DL_TRY \ + do { \ + bool dl_pending_error_ = false; \ + char dl_error_msg_[DL_ERROR_MSG_MAX]; \ +\ + dl_error_msg_[0] = '\0'; \ + try + +#define DL_CATCH_END() \ + catch (const std::exception &e) \ + { \ + dl_pending_error_ = true; \ + strlcpy(dl_error_msg_, e.what(), sizeof(dl_error_msg_)); \ + } \ + catch (...) \ + { \ + dl_pending_error_ = true; \ + strlcpy(dl_error_msg_, "unknown C++ exception", \ + sizeof(dl_error_msg_)); \ + } \ + if (dl_pending_error_) \ + ereport(ERROR, \ + (errcode(ERRCODE_INTERNAL_ERROR), \ + errmsg("datalake_fdw: %s", dl_error_msg_))); \ + } while (0) + +#define DL_ABI_GUARD_BEGIN \ + try \ + { + +#define DL_ABI_GUARD_END(errvar) \ + } \ + catch (...) \ + { \ + (errvar) = DL_ERR_INTERNAL; \ + } + +#define DL_CLEANUP_GUARD_BEGIN \ + do { \ + bool dl_cleanup_failed_ = false; \ +\ + try \ + { + +/* + * Class 3 cleanup guards must never ereport(), so the report is a WARNING at + * most -- and it is emitted after the handler has been left, because even + * elog() can escalate into an ERROR while the error subsystem is in trouble. + */ +#define DL_CLEANUP_GUARD_END \ + } \ + catch (...) \ + { \ + dl_cleanup_failed_ = true; \ + } \ + if (dl_cleanup_failed_ && dl_can_log_cleanup_warning()) \ + elog(WARNING, "datalake_fdw: C++ exception during cleanup"); \ + } while (0) + +/* Modeled on the PAX CBDB_WRAP_START/END saved-exception-stack pattern. */ +#define DL_WRAP_START \ + sigjmp_buf dl_local_sigjmp_buf; \ + { \ + DlPgExceptionStack dl_exception_stack( \ + reinterpret_cast(&PG_exception_stack), \ + reinterpret_cast(&error_context_stack)); \ + if (sigsetjmp(dl_local_sigjmp_buf, 0) == 0) \ + { \ + dl_exception_stack.SetLocalJmp(&dl_local_sigjmp_buf) + +#define DL_WRAP_END \ + } \ + else \ + { \ + throw DlPgError(); \ + } \ + } + +#endif /* __cplusplus */ + +#endif /* DL_WRAPPERS_H */ diff --git a/contrib/datalake_fdw/src/common/file_system_wrapper.cpp b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp new file mode 100644 index 00000000000..2b576fe4344 --- /dev/null +++ b/contrib/datalake_fdw/src/common/file_system_wrapper.cpp @@ -0,0 +1,232 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * file_system_wrapper.cpp + * Storage facade dispatching to the registered backend. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/file_system_wrapper.cpp + * + *------------------------------------------------------------------------- + */ + +#include "common/dl_pg_api.h" + +#include "common/backend_registry.h" +#include "common/dl_wrappers.h" +#include "common/file_system_wrapper.h" + +/* + * Dispatch only: each call finds the backend registered for the location's + * scheme and hands the work over. Nothing here is reachable from SQL in this + * skeleton, so what the regression suite asserts is the behaviour of the + * layers above. + */ + +extern "C" DlErrCode +datalake_fs_open(const DatalakeLocation *location, + const DlKeyValue *credentials, int ncredentials, + DatalakeFileSystem *fs_out) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + const struct DatalakeStorageOps *ops; + + if (fs_out == NULL || location == NULL || location->scheme == NULL || + ncredentials < 0) + rc = DL_ERR_INVALID_OPTION; + else + { + *fs_out = NULL; + ops = datalake_lookup_storage_backend(location->scheme); + + if (ops == NULL) + rc = DL_ERR_NOT_SUPPORTED; + else + { + rc = ops->fs_open(location, credentials, ncredentials, fs_out); + + if (rc == DL_OK && *fs_out == NULL) + rc = DL_ERR_INTERNAL; + else if (rc == DL_OK) + (*fs_out)->ops = ops; + } + } + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +extern "C" void +datalake_fs_close(DatalakeFileSystem *fs) +{ + DL_CLEANUP_GUARD_BEGIN + { + /* + * Clear the caller's handle before releasing it, so that a repeated + * close -- the normal shape of resource-owner cleanup after an error + * that already closed things -- finds nothing to do instead of + * reaching a backend that has freed itself. + */ + if (fs != NULL && *fs != NULL) + { + DatalakeFileSystem doomed = *fs; + + *fs = NULL; + doomed->ops->fs_close(doomed); + } + } + DL_CLEANUP_GUARD_END; +} + +extern "C" DlErrCode +datalake_fs_list(DatalakeFileSystem fs, const char *prefix, + char ***names_out, int *nnames_out) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (fs == NULL || prefix == NULL || names_out == NULL || + nnames_out == NULL) + rc = DL_ERR_INVALID_OPTION; + else + { + *names_out = NULL; + *nnames_out = 0; + rc = fs->ops->fs_list(fs, prefix, names_out, nnames_out); + } + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +extern "C" DlErrCode +datalake_file_open(DatalakeFileSystem fs, const char *path, + DatalakeFileMode mode, DatalakeFile *file_out) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (file_out == NULL || fs == NULL || path == NULL) + rc = DL_ERR_INVALID_OPTION; + else if (mode != DATALAKE_FILE_READ && mode != DATALAKE_FILE_WRITE) + rc = DL_ERR_INVALID_OPTION; + else + { + *file_out = NULL; + rc = fs->ops->file_open(fs, path, mode, file_out); + + if (rc == DL_OK && *file_out == NULL) + rc = DL_ERR_INTERNAL; + else if (rc == DL_OK) + (*file_out)->ops = fs->ops; + } + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +extern "C" DlErrCode +datalake_file_read(DatalakeFile file, void *buffer, int64_t length, + int64_t *nread) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (file == NULL || nread == NULL || length < 0) + rc = DL_ERR_INVALID_OPTION; + else + { + *nread = 0; + rc = file->ops->file_read(file, buffer, length, nread); + } + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +extern "C" DlErrCode +datalake_file_write(DatalakeFile file, const void *buffer, int64_t length) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (file == NULL || length < 0) + rc = DL_ERR_INVALID_OPTION; + else + rc = file->ops->file_write(file, buffer, length); + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +extern "C" DlErrCode +datalake_file_close(DatalakeFile *file) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (file == NULL || *file == NULL) + rc = DL_ERR_INVALID_OPTION; + else + { + DatalakeFile doomed = *file; + + /* + * The handle is consumed even when the close reports an error: + * the backend has released it either way, and there is nothing + * left to retry the close against. + */ + *file = NULL; + rc = doomed->ops->file_close(doomed); + } + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +extern "C" void +datalake_file_abort(DatalakeFile *file) +{ + DL_CLEANUP_GUARD_BEGIN + { + /* Cleared first, so a repeated abort finds nothing to do. */ + if (file != NULL && *file != NULL) + { + DatalakeFile doomed = *file; + + *file = NULL; + doomed->ops->file_abort(doomed); + } + } + DL_CLEANUP_GUARD_END; +} diff --git a/contrib/datalake_fdw/src/common/file_system_wrapper.h b/contrib/datalake_fdw/src/common/file_system_wrapper.h new file mode 100644 index 00000000000..37b6c037d2c --- /dev/null +++ b/contrib/datalake_fdw/src/common/file_system_wrapper.h @@ -0,0 +1,113 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * file_system_wrapper.h + * Storage facade over one protocol: open, read, write, list. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/file_system_wrapper.h + * + *------------------------------------------------------------------------- + */ + +#ifndef FILE_SYSTEM_WRAPPER_H +#define FILE_SYSTEM_WRAPPER_H + +#include + +#include "common/datalake_location.h" +#include "common/dl_err.h" +#include "common/dl_kv.h" + +/* + * A file system reached over one storage protocol, and an open file in it. + * Both are opaque: callers hold a handle and pass it back, exactly as they do + * for a File or a BufFile, so a backend can keep whatever state it needs + * without any of it becoming part of this interface. + * + * This is deliberately a facade over open/read/write/close/list and not a + * storage framework. Its only consumer is the format layer, and keeping the + * surface this narrow is what lets the implementation be replaced -- by an + * Arrow filesystem, say -- without the layers above noticing. + */ +typedef struct DatalakeFileSystemData *DatalakeFileSystem; +typedef struct DatalakeFileData *DatalakeFile; + +typedef enum DatalakeFileMode +{ + DATALAKE_FILE_READ, + DATALAKE_FILE_WRITE +} DatalakeFileMode; + +#ifdef __cplusplus +extern "C" +{ +#endif + +/* + * The location names the protocol and the bucket or namenode; credentials are + * resolved separately and may be empty, in which case the backend falls back + * to whatever ambient credentials it finds. + */ +extern DlErrCode datalake_fs_open(const DatalakeLocation *location, + const DlKeyValue *credentials, + int ncredentials, + DatalakeFileSystem *fs_out); + +/* + * Cleanup entry point: releases the file system and clears the caller's + * handle, so a repeated call has nothing left to act on. It never raises, + * because it runs on the resource-owner path during transaction abort. + * Passing a handle by value could not clear it, and the second call would + * then reach a backend that had already freed itself. + */ +extern void datalake_fs_close(DatalakeFileSystem *fs); + +extern DlErrCode datalake_fs_list(DatalakeFileSystem fs, const char *prefix, + char ***names_out, int *nnames_out); + +extern DlErrCode datalake_file_open(DatalakeFileSystem fs, const char *path, + DatalakeFileMode mode, + DatalakeFile *file_out); + +extern DlErrCode datalake_file_read(DatalakeFile file, void *buffer, + int64_t length, int64_t *nread); + +extern DlErrCode datalake_file_write(DatalakeFile file, const void *buffer, + int64_t length); + +/* + * Finishes the file and clears the caller's handle. Errors worth reporting + * surface here, and the handle is consumed whether or not one does: there is + * nothing left to retry against. + */ +extern DlErrCode datalake_file_close(DatalakeFile *file); + +/* + * Cleanup entry point for the failure path: discards the file and clears the + * caller's handle. Never raises; anything worth reporting comes out of + * datalake_file_close() instead. + */ +extern void datalake_file_abort(DatalakeFile *file); + +#ifdef __cplusplus +} +#endif + +#endif /* FILE_SYSTEM_WRAPPER_H */ diff --git a/contrib/datalake_fdw/src/common/parser_option.c b/contrib/datalake_fdw/src/common/parser_option.c new file mode 100644 index 00000000000..9e4a4ab0469 --- /dev/null +++ b/contrib/datalake_fdw/src/common/parser_option.c @@ -0,0 +1,94 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * parser_option.c + * Typed accessors over a DefElem option list. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/parser_option.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "commands/defrem.h" +#include "common/parser_option.h" +#include "utils/builtins.h" + +/* + * Return the value of the named option, or NULL when it is absent. + * + * The comparison is case-sensitive, matching how the server stores and + * de-duplicates option names. Matching case-insensitively here would let + * "type" and a quoted "TYPE" both be stored -- the generic duplicate check + * would not see them as the same option -- and then silently return whichever + * came first, which for a credential is the wrong one to pick at random. + */ +char * +get_string_option(List *options, const char *option_name) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, option_name) == 0) + return defGetString(def); + } + + return NULL; +} + +/* + * Boolean accessor that also reports whether the option was written at all. + * + * Callers that forward options to the metadata engine need that distinction: + * an unset boolean may fall back to site configuration, while an explicit + * false has to override it. + * + * Unlike the reference implementation, an unparsable value raises an error + * instead of silently yielding the default -- a typo in a boolean server + * option should not read as "you asked for the default". Stored values are + * already validated by the option validators, so this only fires on input + * paths that have not been through them. + */ +bool +get_bool_option_ex(List *options, const char *option_name, + bool default_value, bool *isset) +{ + char *value = get_string_option(options, option_name); + bool parsed_value; + + Assert(isset != NULL); + *isset = false; + + if (value == NULL) + return default_value; + + if (!parse_bool(value, &parsed_value)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid boolean value \"%s\" for option \"%s\"", + value, option_name))); + + *isset = true; + return parsed_value; +} diff --git a/contrib/datalake_fdw/src/common/parser_option.h b/contrib/datalake_fdw/src/common/parser_option.h new file mode 100644 index 00000000000..10a8181e3e0 --- /dev/null +++ b/contrib/datalake_fdw/src/common/parser_option.h @@ -0,0 +1,54 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * parser_option.h + * Typed accessors over a DefElem option list. + * + * Every layer that reads SERVER, USER MAPPING or table options goes through + * these accessors rather than walking the list itself, so that lookup and + * absent-versus-empty are decided in one place. + * + * The reference implementation -- the existing implementation of this feature + * that this work derives from and is meant to replace -- also carries integer + * and defaulting-boolean accessors (getIntOption, getBoolOption). They are + * omitted here rather than shipped unused; add them under those names, as thin + * wrappers over the accessors below, together with the first option that needs + * them. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/parser_option.h + * + *------------------------------------------------------------------------- + */ + +#ifndef PARSER_OPTION_H +#define PARSER_OPTION_H + +#include "postgres.h" + +#include "nodes/pg_list.h" + +/* Reference implementation: getStringOption() */ +extern char *get_string_option(List *options, const char *option_name); + +/* Reference implementation: getBoolOptionEx() */ +extern bool get_bool_option_ex(List *options, const char *option_name, + bool default_value, bool *isset); + +#endif /* PARSER_OPTION_H */ diff --git a/contrib/datalake_fdw/src/common/s3_file_system.cpp b/contrib/datalake_fdw/src/common/s3_file_system.cpp new file mode 100644 index 00000000000..d255200528b --- /dev/null +++ b/contrib/datalake_fdw/src/common/s3_file_system.cpp @@ -0,0 +1,261 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * s3_file_system.cpp + * The S3 storage backend. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/common/s3_file_system.cpp + * + *------------------------------------------------------------------------- + */ + +#include "common/dl_pg_api.h" + +#include "common/backend_registry.h" +#include "common/dl_wrappers.h" + +#include + +/* + * The S3 backend, without an S3 client yet: the shape a backend takes is what + * this file establishes, so that the change adding a real client replaces + * method bodies rather than the structure around them. Every entry point + * reports that the operation is not supported. + */ +class S3FileSystem +{ +public: + DlErrCode + Initialize(const DatalakeLocation *location, const DlKeyValue *credentials, + int ncredentials) + { + (void) location; + (void) credentials; + (void) ncredentials; + + return DL_ERR_NOT_SUPPORTED; + } + + DlErrCode + OpenFile(const char *path, DatalakeFileMode mode, DatalakeFile *file_out) + { + (void) path; + (void) mode; + + if (file_out != NULL) + *file_out = NULL; + + return DL_ERR_NOT_SUPPORTED; + } + + DlErrCode + List(const char *prefix, char ***names_out, int *nnames_out) + { + (void) prefix; + + if (names_out != NULL) + *names_out = NULL; + if (nnames_out != NULL) + *nnames_out = 0; + + return DL_ERR_NOT_SUPPORTED; + } +}; + +/* + * A handle the facade can hold. + * + * Deriving from the C struct rather than embedding it as a first member is what + * makes recovering the handle defined behaviour: a derived-to-base pointer + * conversion and a static_cast back are guaranteed for any class, while the + * first-member trick is only guaranteed for standard-layout types -- which this + * is not, because of the unique_ptr. The C side still sees a plain + * DatalakeFileSystemData, since that is what the base subobject is. + */ +struct S3FileSystemHandle : public DatalakeFileSystemData +{ + std::unique_ptr impl; +}; + +static DlErrCode +s3_fs_open(const DatalakeLocation *location, const DlKeyValue *credentials, + int ncredentials, DatalakeFileSystem *fs_out) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + if (fs_out == NULL) + rc = DL_ERR_INVALID_OPTION; + else + { + /* + * Owned by unique_ptr until the handle is published, so that an + * exception from the second allocation or from Initialize() -- + * which the guard below turns into an error code -- cannot leave + * the first allocation behind. + */ + std::unique_ptr handle(new S3FileSystemHandle()); + + *fs_out = NULL; + handle->impl.reset(new S3FileSystem()); + rc = handle->impl->Initialize(location, credentials, ncredentials); + + if (rc == DL_OK) + *fs_out = handle.release(); + } + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +static void +s3_fs_close(DatalakeFileSystem fs) +{ + DL_CLEANUP_GUARD_BEGIN + { + /* The facade has already cleared its caller's handle. */ + delete static_cast(fs); + } + DL_CLEANUP_GUARD_END; +} + +static DlErrCode +s3_fs_list(DatalakeFileSystem fs, const char *prefix, char ***names_out, + int *nnames_out) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + S3FileSystemHandle *handle = static_cast(fs); + + if (handle == NULL) + rc = DL_ERR_INVALID_OPTION; + else + rc = handle->impl->List(prefix, names_out, nnames_out); + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +static DlErrCode +s3_file_open(DatalakeFileSystem fs, const char *path, DatalakeFileMode mode, + DatalakeFile *file_out) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + S3FileSystemHandle *handle = static_cast(fs); + + if (handle == NULL) + rc = DL_ERR_INVALID_OPTION; + else + rc = handle->impl->OpenFile(path, mode, file_out); + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +static DlErrCode +s3_file_read(DatalakeFile file, void *buffer, int64_t length, int64_t *nread) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + (void) file; + (void) buffer; + (void) length; + + if (nread != NULL) + *nread = 0; + + rc = DL_ERR_NOT_SUPPORTED; + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +static DlErrCode +s3_file_write(DatalakeFile file, const void *buffer, int64_t length) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + (void) file; + (void) buffer; + (void) length; + + rc = DL_ERR_NOT_SUPPORTED; + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +static DlErrCode +s3_file_close(DatalakeFile file) +{ + DlErrCode rc = DL_ERR_INTERNAL; + + DL_ABI_GUARD_BEGIN + { + (void) file; + + rc = DL_ERR_NOT_SUPPORTED; + } + DL_ABI_GUARD_END(rc); + + return rc; +} + +static void +s3_file_abort(DatalakeFile file) +{ + DL_CLEANUP_GUARD_BEGIN + { + (void) file; + } + DL_CLEANUP_GUARD_END; +} + +static const struct DatalakeStorageOps s3_storage_ops = { + s3_fs_open, + s3_fs_close, + s3_fs_list, + s3_file_open, + s3_file_read, + s3_file_write, + s3_file_close, + s3_file_abort +}; + +DlErrCode +datalake_register_s3_backend(void) +{ + return datalake_register_storage_backend("s3", &s3_storage_ops); +} diff --git a/contrib/datalake_fdw/src/format/format.h b/contrib/datalake_fdw/src/format/format.h new file mode 100644 index 00000000000..003101ae550 --- /dev/null +++ b/contrib/datalake_fdw/src/format/format.h @@ -0,0 +1,117 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * format.h + * Reader and writer interfaces for lake table data files. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/format.h + * + *------------------------------------------------------------------------- + */ + +#ifndef DL_FORMAT_H +#define DL_FORMAT_H + +#include +#include + +#include "common/dl_err.h" + +/* Arrow C data interface: stable public ABI. */ +#ifndef ARROW_C_DATA_INTERFACE +#define ARROW_C_DATA_INTERFACE + +struct ArrowSchema { + const char *format; + const char *name; + const char *metadata; + int64_t flags; + int64_t n_children; + struct ArrowSchema **children; + struct ArrowSchema *dictionary; + void (*release)(struct ArrowSchema *); + void *private_data; +}; + +struct ArrowArray { + int64_t length; + int64_t null_count; + int64_t offset; + int64_t n_buffers; + int64_t n_children; + const void **buffers; + struct ArrowArray **children; + struct ArrowArray *dictionary; + void (*release)(struct ArrowArray *); + void *private_data; +}; +#endif /* ARROW_C_DATA_INTERFACE */ + +typedef struct Fragment Fragment; /* opaque in skeleton */ +typedef struct ProjectionSet ProjectionSet; +typedef struct RowGroupFilterSet RowGroupFilterSet; +typedef struct WriterOptions WriterOptions; +typedef struct FileMeta FileMeta; +typedef struct DeleteFileSet DeleteFileSet; + +/* Readers/writers are INSTANCES (ops + impl); configuration travels with the instance. + * No global slots or trampolines, ever. */ +typedef struct FormatReader FormatReader; +typedef struct FormatReaderOps { + /* Each batch yields ArrowArray+ArrowSchema; last column is a hidden int64 file-row + * ordinal (for MoR positional deletes). */ + DlErrCode (*next_batch)(FormatReader *, struct ArrowArray *out, + struct ArrowSchema *schema, bool *eof); + void (*close)(FormatReader *); /* void cleanup ABI: noexcept, idempotent, never ereport */ +} FormatReaderOps; +struct FormatReader { const FormatReaderOps *ops; void *impl; }; + +typedef struct FormatWriter FormatWriter; +typedef struct FormatWriterOps { + DlErrCode (*write_batch)(FormatWriter *, struct ArrowArray *batch); /* success == consumed */ + /* Rolling support: actual bytes encoded into the sink so far. Valid to query after a + * successful write_batch; on failure returns an error code and *out is invalid. + * The write.c orchestration layer rolls files (finish -> new open_writer) when this + * reaches the soft target; overshoot of at most one batch is allowed. */ + DlErrCode (*bytes_written)(FormatWriter *, int64_t *out); + DlErrCode (*finish)(FormatWriter *, FileMeta **meta); /* reportable close-time errors + * surface ONLY here */ + void (*abort)(FormatWriter *); /* void cleanup ABI: noexcept, idempotent, never ereport */ +} FormatWriterOps; +struct FormatWriter { const FormatWriterOps *ops; void *impl; }; + +typedef struct FormatRoutine { + uint32_t abi_version, struct_size; /* same prefix-compat semantics as meta engine */ + const char *name; /* "parquet" */ + DlErrCode (*open_reader)(const Fragment *, const ProjectionSet *, + const RowGroupFilterSet *, FormatReader **out); + DlErrCode (*open_writer)(const char *path, /* TupleDesc */ void *tupdesc, + const WriterOptions *, FormatWriter **out); +} FormatRoutine; + +extern const FormatRoutine *GetFormatRoutine(const char *format); + +/* MoR positional-delete decorator: consumes the inner instance, returns a new instance. + * close(outer) exactly-once: releases itself then close(inner); idempotent; on open + * failure the wrapper owns releasing inner. */ +extern DlErrCode WrapPositionDeleteFilter(FormatReader *inner, const DeleteFileSet *, + FormatReader **out); + +#endif /* DL_FORMAT_H */ diff --git a/contrib/datalake_fdw/src/format/format_registry.c b/contrib/datalake_fdw/src/format/format_registry.c new file mode 100644 index 00000000000..83370a51f64 --- /dev/null +++ b/contrib/datalake_fdw/src/format/format_registry.c @@ -0,0 +1,48 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * format_registry.c + * Lookup of the reader and writer for a data file format. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/format/format_registry.c + * + *------------------------------------------------------------------------- + */ + +#include + +#include "format/format.h" + +/* No formats in the skeleton; parquet lands in PR-3/4. Callers must treat + * NULL as not-supported. */ +const FormatRoutine * +GetFormatRoutine(const char *format) +{ + return NULL; +} + +DlErrCode +WrapPositionDeleteFilter(FormatReader *inner, const DeleteFileSet *delete_files, + FormatReader **out) +{ + if (out != NULL) + *out = NULL; + return DL_ERR_NOT_SUPPORTED; +} diff --git a/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_fdw.c b/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_fdw.c new file mode 100644 index 00000000000..ba408c6ab0c --- /dev/null +++ b/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_fdw.c @@ -0,0 +1,226 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_catalog_fdw.c + * Option validator for Iceberg catalog foreign servers. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_fdw.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/reloptions.h" +#include "am_iceberg/pg_iceberg_reject.h" +#include "catalog/pg_foreign_data_wrapper.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_user_mapping.h" +#include "commands/defrem.h" +#include "common/dl_option_util.h" +#include "common/parser_option.h" +#include "fmgr.h" +#include "iceberg_catalog_fdw/iceberg_catalog_option.h" + +PG_FUNCTION_INFO_V1(iceberg_catalog_fdw_validator); + +static bool is_catalog_server_option(const char *name); +static bool is_catalog_user_mapping_option(const char *name); +static void check_catalog_server_type(const char *server_type); + +/* + * The server options this wrapper accepts. A storage protocol is never among + * them: where the data files live is decided by the volume server. + */ +static bool +is_catalog_server_option(const char *name) +{ + return strcmp(name, DATALAKE_ICEBERG_CATALOG_SERVER_TYPE) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_URL) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_NAME) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_WAREHOUSE_LOCATION_PREFIX) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM) == 0; +} + +/* + * The user mapping options this wrapper accepts: the union over catalog types, + * because a mapping is validated without reference to the server it belongs to. + * parse_iceberg_catalog_user_mapping_options() is what narrows them by type. + */ +static bool +is_catalog_user_mapping_option(const char *name) +{ + return strcmp(name, DATALAKE_ICEBERG_CATALOG_USERNAME) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_AUTH_METHOD) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_KRB_SERVICE_PRINCIPAL) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_KRB_CLIENT_PRINCIPAL) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_KRB_CLIENT_KEYTAB) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_CLIENT_ID) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_CLIENT_SECRET) == 0 || + strcmp(name, DATALAKE_ICEBERG_CATALOG_SCOPE) == 0; +} + +static void +check_catalog_server_type(const char *server_type) +{ + if (pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HIVE) == 0 || + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST) == 0 || + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_POLARIS) == 0 || + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_BUILTIN) == 0) + return; + + /* + * Names the vocabulary defines but this module cannot serve yet. Refusing + * them is what keeps a server from being created against a catalog no + * statement could subsequently use. + */ + if (pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HADOOP) == 0 || + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_S3) == 0) + pg_iceberg_not_supported(psprintf("catalog type \"%s\"", server_type)); + + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid iceberg catalog type \"%s\"", server_type), + errhint("Allowed types are \"%s\", \"%s\" and \"%s\"; \"%s\" is accepted as an alias of \"%s\".", + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HIVE, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_BUILTIN, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_POLARIS, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST))); +} + +Datum +iceberg_catalog_fdw_validator(PG_FUNCTION_ARGS) +{ + List *options = untransformRelOptions(PG_GETARG_DATUM(0)); + Oid catalog = PG_GETARG_OID(1); + ListCell *lc; + const char *server_type; + const char *url; + const char *realm; + + /* + * CREATE FOREIGN DATA WRAPPER invokes its validator with an empty array. + * Permit that bootstrap call, but this FDW has no wrapper-level options. + */ + if (catalog == ForeignDataWrapperRelationId && options == NIL) + PG_RETURN_VOID(); + + if (catalog != ForeignServerRelationId && + catalog != UserMappingRelationId) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("iceberg_catalog_fdw has no options in this context"))); + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + const char *name = def->defname; + + if (catalog == ForeignServerRelationId) + { + if (dl_is_credential_option(name)) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("credential option \"%s\" is not allowed on an iceberg catalog server", + name), + errhint("credentials belong in CREATE USER MAPPING ... OPTIONS (...)"))); + + if (!is_catalog_server_option(name)) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid iceberg catalog server option \"%s\"", + name), + errhint("Allowed options are \"%s\", \"%s\", \"%s\", \"%s\" and \"%s\".", + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE, + DATALAKE_ICEBERG_CATALOG_URL, + DATALAKE_ICEBERG_CATALOG_NAME, + DATALAKE_ICEBERG_CATALOG_WAREHOUSE_LOCATION_PREFIX, + DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM))); + + /* Reject an empty value here rather than at first use. */ + if (defGetString(def)[0] == '\0') + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg catalog server option \"%s\" cannot be empty", + name))); + } + else if (!is_catalog_user_mapping_option(name)) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid iceberg catalog user mapping option \"%s\"", + name))); + } + + if (catalog == UserMappingRelationId) + PG_RETURN_VOID(); + + /* + * Cross-option rules run once the whole list has been seen, so that they do + * not depend on the order the options were written in. + */ + server_type = get_string_option(options, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE); + url = get_string_option(options, DATALAKE_ICEBERG_CATALOG_URL); + realm = get_string_option(options, + DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM); + + if (server_type == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg catalog server option \"%s\" is required", + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE))); + + check_catalog_server_type(server_type); + + if (pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_BUILTIN) == 0) + { + if (url != NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg catalog type \"%s\" forbids server option \"%s\"", + server_type, DATALAKE_ICEBERG_CATALOG_URL))); + } + else if (url == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg catalog type \"%s\" requires server option \"%s\"", + server_type, DATALAKE_ICEBERG_CATALOG_URL))); + + if (realm != NULL && + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST) != 0 && + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_POLARIS) != 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg catalog server option \"%s\" applies only to catalog type \"%s\"", + DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST))); + + PG_RETURN_VOID(); +} diff --git a/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.c b/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.c new file mode 100644 index 00000000000..9587132ecba --- /dev/null +++ b/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.c @@ -0,0 +1,185 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_catalog_option.c + * Option vocabulary and parsed forms for Iceberg catalog servers. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "common/parser_option.h" +#include "iceberg_catalog_fdw/iceberg_catalog_option.h" +#include "utils/builtins.h" + +static void parse_hive_catalog_server_options(IcebergCatalogServerOptions *options, + List *server_options); +static void parse_rest_catalog_server_options(IcebergCatalogServerOptions *options, + List *server_options); +static void parse_hive_user_mapping_options(HiveUserMappingOptions *options, + List *user_options); +static void parse_polaris_user_mapping_options(PolarisUserMappingOptions *options, + List *user_options); + +/* + * Reference implementation: parseHiveCatalogServerOptions(). + * + * That version also accepts "hive_metastore_uri" as a second spelling of the + * same option, for servers created before the key was renamed. This extension + * has never been released, so there is nothing to be compatible with and only + * one spelling is accepted. + */ +static void +parse_hive_catalog_server_options(IcebergCatalogServerOptions *options, + List *server_options) +{ + options->hive_metastore_uri = + get_string_option(server_options, DATALAKE_ICEBERG_CATALOG_URL); +} + +/* Reference implementation: parsePolarisCatalogServerOptions() */ +static void +parse_rest_catalog_server_options(IcebergCatalogServerOptions *options, + List *server_options) +{ + options->polaris_server_url = + get_string_option(server_options, DATALAKE_ICEBERG_CATALOG_URL); + + /* + * Sent as the realm header on every request. Optional: the metadata engine + * applies its own default when unset. + */ + options->polaris_server_realm = + get_string_option(server_options, + DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM); +} + +void +parse_iceberg_catalog_server_options(IcebergCatalogServerOptions *options, + List *server_options) +{ + options->server_type = + get_string_option(server_options, DATALAKE_ICEBERG_CATALOG_SERVER_TYPE); + + if (options->server_type == NULL) + return; + + if (pg_strcasecmp(options->server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HIVE) == 0) + parse_hive_catalog_server_options(options, server_options); + else if (pg_strcasecmp(options->server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST) == 0 || + pg_strcasecmp(options->server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_POLARIS) == 0) + parse_rest_catalog_server_options(options, server_options); +} + +/* Reference implementation: parseHiveUserMappingOptions() */ +static void +parse_hive_user_mapping_options(HiveUserMappingOptions *options, + List *user_options) +{ + options->username = + get_string_option(user_options, DATALAKE_ICEBERG_CATALOG_USERNAME); + options->auth_method = + get_string_option(user_options, DATALAKE_ICEBERG_CATALOG_AUTH_METHOD); + options->krb_service_principal = + get_string_option(user_options, + DATALAKE_ICEBERG_CATALOG_KRB_SERVICE_PRINCIPAL); + options->krb_client_principal = + get_string_option(user_options, + DATALAKE_ICEBERG_CATALOG_KRB_CLIENT_PRINCIPAL); + options->krb_client_keytab = + get_string_option(user_options, + DATALAKE_ICEBERG_CATALOG_KRB_CLIENT_KEYTAB); +} + +/* Reference implementation: parsePolarisUserMappingOptions() */ +static void +parse_polaris_user_mapping_options(PolarisUserMappingOptions *options, + List *user_options) +{ + options->client_id = + get_string_option(user_options, DATALAKE_ICEBERG_CATALOG_CLIENT_ID); + options->client_secret = + get_string_option(user_options, DATALAKE_ICEBERG_CATALOG_CLIENT_SECRET); + options->scope = + get_string_option(user_options, DATALAKE_ICEBERG_CATALOG_SCOPE); +} + +void +parse_iceberg_catalog_user_mapping_options(IcebergCatalogUserMappingOptions *options, + List *user_options, + const char *server_type) +{ + if (server_type == NULL) + return; + + if (pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HIVE) == 0) + parse_hive_user_mapping_options(&options->hive, user_options); + else if (pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST) == 0 || + pg_strcasecmp(server_type, + DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_POLARIS) == 0) + parse_polaris_user_mapping_options(&options->polaris, user_options); +} + +/* + * Reference implementation: parseIcebergForeignCatalogOptions(). + * + * server_name supplies the default catalog name, because a catalog server here + * names exactly one Iceberg catalog. + */ +void +parse_iceberg_foreign_catalog_options(IcebergForeignCatalogOptions *options, + List *catalog_options, + const char *server_name) +{ + options->catalog_name = + get_string_option(catalog_options, DATALAKE_ICEBERG_CATALOG_NAME); + if (options->catalog_name == NULL) + options->catalog_name = pstrdup(server_name); + + options->warehouse_location_prefix = + get_string_option(catalog_options, + DATALAKE_ICEBERG_CATALOG_WAREHOUSE_LOCATION_PREFIX); +} + +IcebergCatalogOptions * +get_iceberg_catalog_options(ForeignServer *server) +{ + IcebergCatalogOptions *options; + + Assert(server != NULL); + + options = (IcebergCatalogOptions *) palloc0(sizeof(IcebergCatalogOptions)); + + parse_iceberg_catalog_server_options(&options->catalog_server, + server->options); + parse_iceberg_foreign_catalog_options(&options->foreign_catalog, + server->options, + server->servername); + + return options; +} diff --git a/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.h b/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.h new file mode 100644 index 00000000000..df6e3b6b9a1 --- /dev/null +++ b/contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.h @@ -0,0 +1,171 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_catalog_option.h + * Option vocabulary and parsed forms for Iceberg catalog servers. + * + * "The reference implementation", here and in the other option modules, means + * the existing implementation of this feature that this work derives from and + * is meant to replace. Its option key macros, struct names and field names are + * reproduced exactly, so that a parser for a further catalog type can move + * between the two as an addition rather than a rewrite. Where the two models + * genuinely differ, the difference is called out on the field. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/iceberg_catalog_fdw/iceberg_catalog_option.h + * + *------------------------------------------------------------------------- + */ + +#ifndef ICEBERG_CATALOG_OPTION_H +#define ICEBERG_CATALOG_OPTION_H + +#include "postgres.h" + +#include "common/dl_option_util.h" +#include "foreign/foreign.h" +#include "nodes/pg_list.h" + +/* Catalog server options */ +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE "type" +#define DATALAKE_ICEBERG_CATALOG_URL "uri" +#define DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM "polaris_server_realm" + +/* + * Recognized catalog server types. + * + * The names are Apache Iceberg's, not this module's: a catalog reached over the + * REST protocol is "rest", because the specification defines one protocol that + * several implementations answer. "polaris" is accepted as an alias for it -- + * Polaris is one such implementation, and it is the spelling the reference + * implementation uses, so servers written for that one keep working. + * + * A storage protocol is never a catalog type -- where the data files live is + * volume business -- but the reference implementation defines these two names, + * so they are kept here to stay one vocabulary; the validator refuses them + * until an implementation exists. + */ +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HIVE "hive" +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_REST "rest" +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_POLARIS "polaris" +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_BUILTIN "builtin" +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_HADOOP "hadoop" +#define DATALAKE_ICEBERG_CATALOG_SERVER_TYPE_S3 "s3" + +/* Catalog user mapping options */ +#define DATALAKE_ICEBERG_CATALOG_USERNAME DL_OPTION_KEY_USERNAME +#define DATALAKE_ICEBERG_CATALOG_AUTH_METHOD "auth_method" +#define DATALAKE_ICEBERG_CATALOG_KRB_SERVICE_PRINCIPAL "krb_service_principal" +#define DATALAKE_ICEBERG_CATALOG_KRB_CLIENT_PRINCIPAL "krb_client_principal" +#define DATALAKE_ICEBERG_CATALOG_KRB_CLIENT_KEYTAB DL_OPTION_KEY_KRB_CLIENT_KEYTAB +#define DATALAKE_ICEBERG_CATALOG_CLIENT_ID DL_OPTION_KEY_CLIENT_ID +#define DATALAKE_ICEBERG_CATALOG_CLIENT_SECRET DL_OPTION_KEY_CLIENT_SECRET +#define DATALAKE_ICEBERG_CATALOG_SCOPE "scope" + +/* Catalog identity options */ +#define DATALAKE_ICEBERG_CATALOG_NAME "catalog_name" +#define DATALAKE_ICEBERG_CATALOG_WAREHOUSE_LOCATION_PREFIX "warehouse" + +typedef struct IcebergCatalogServerOptions +{ + char *server_type; /* DATALAKE_ICEBERG_CATALOG_SERVER_TYPE */ + char *hive_metastore_uri; /* DATALAKE_ICEBERG_CATALOG_URL, hive */ + char *polaris_server_url; /* DATALAKE_ICEBERG_CATALOG_URL, rest */ + char *polaris_server_realm; /* DATALAKE_ICEBERG_CATALOG_POLARIS_SERVER_REALM */ + + /* + * The reference implementation also carries server_name, naming a section + * of a site configuration file. There is no such file here: an option that + * would be accepted and then ignored is worse than one that is refused, so + * it is left out until site configuration exists. + */ +} IcebergCatalogServerOptions; + +typedef struct HiveUserMappingOptions +{ + char *username; + char *auth_method; + char *krb_service_principal; + char *krb_client_principal; + char *krb_client_keytab; +} HiveUserMappingOptions; + +typedef struct PolarisUserMappingOptions +{ + char *client_id; + char *client_secret; + char *scope; +} PolarisUserMappingOptions; + +typedef struct IcebergCatalogUserMappingOptions +{ + HiveUserMappingOptions hive; + PolarisUserMappingOptions polaris; +} IcebergCatalogUserMappingOptions; + +typedef struct IcebergForeignCatalogOptions +{ + /* + * The reference implementation reads these from a foreign catalog object + * that a server can hold several of. This extension cannot add a catalog + * of its own to the system catalogs, so a catalog server names exactly one + * Iceberg catalog and both values come from that server's options; + * catalog_name defaults to the server name when unset. + */ + char *catalog_name; /* DATALAKE_ICEBERG_CATALOG_NAME */ + char *warehouse_location_prefix; /* DATALAKE_ICEBERG_CATALOG_WAREHOUSE_LOCATION_PREFIX */ + + /* + * Deferred, with the reference implementation's names kept for the port: + * enable_metadata_cache / metadata_cache_ttl / auto_refresh_metadata / + * total_segment / split_size / filter_string. + */ +} IcebergForeignCatalogOptions; + +typedef struct IcebergCatalogOptions +{ + IcebergCatalogServerOptions catalog_server; + IcebergCatalogUserMappingOptions catalog_user; + IcebergForeignCatalogOptions foreign_catalog; +} IcebergCatalogOptions; + +/* Reference implementation: parseIcebergCatalogServerOptions() */ +extern void parse_iceberg_catalog_server_options(IcebergCatalogServerOptions *options, + List *server_options); + +/* Reference implementation: parseIcebergCatalogUserMappingOptions() */ +extern void parse_iceberg_catalog_user_mapping_options(IcebergCatalogUserMappingOptions *options, + List *user_options, + const char *server_type); + +/* Reference implementation: parseIcebergForeignCatalogOptions() */ +extern void parse_iceberg_foreign_catalog_options(IcebergForeignCatalogOptions *options, + List *catalog_options, + const char *server_name); + +/* + * Reference implementation: getIcebergCatalogOptions(), which additionally + * fills catalog_user from the invoking role's user mapping. Credentials are + * resolved separately and lazily here, because a DDL path must be able to + * describe a table without reading anyone's secrets; catalog_user is left + * zeroed by this function. + */ +extern IcebergCatalogOptions *get_iceberg_catalog_options(ForeignServer *server); + +#endif /* ICEBERG_CATALOG_OPTION_H */ diff --git a/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c new file mode 100644 index 00000000000..cff9f80f333 --- /dev/null +++ b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c @@ -0,0 +1,157 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_volume_fdw.c + * Option validator for Iceberg volume foreign servers. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_fdw.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "access/reloptions.h" +#include "am_iceberg/pg_iceberg_options.h" +#include "catalog/pg_foreign_data_wrapper.h" +#include "catalog/pg_foreign_server.h" +#include "catalog/pg_user_mapping.h" +#include "commands/defrem.h" +#include "common/dl_option_util.h" +#include "fmgr.h" +#include "iceberg_volume_fdw/iceberg_volume_option.h" + +PG_FUNCTION_INFO_V1(iceberg_volume_fdw_validator); + +static bool is_volume_server_option(const char *name); +static bool is_volume_user_mapping_option(const char *name); + +static bool +is_volume_server_option(const char *name) +{ + return strcmp(name, DATALAKE_ICEBERG_VOLUME_BASE_PATH) == 0 || + strcmp(name, DATALAKE_ICEBERG_VOLUME_ENDPOINT) == 0 || + strcmp(name, DATALAKE_ICEBERG_VOLUME_REGION) == 0 || + strcmp(name, DATALAKE_ICEBERG_VOLUME_PATH_STYLE_ACCESS) == 0; +} + +/* + * Every credential here is optional, so that ambient storage credentials -- an + * instance profile, a ticket cache -- remain a valid deployment choice. + */ +static bool +is_volume_user_mapping_option(const char *name) +{ + return strcmp(name, DATALAKE_ICEBERG_VOLUME_USERNAME) == 0 || + strcmp(name, DATALAKE_ICEBERG_VOLUME_AWS_ACCESS_KEY_ID) == 0 || + strcmp(name, DATALAKE_ICEBERG_VOLUME_AWS_SECRET_ACCESS_KEY) == 0 || + strcmp(name, DATALAKE_ICEBERG_VOLUME_AWS_SESSION_TOKEN) == 0; +} + +Datum +iceberg_volume_fdw_validator(PG_FUNCTION_ARGS) +{ + List *options = untransformRelOptions(PG_GETARG_DATUM(0)); + Oid catalog = PG_GETARG_OID(1); + ListCell *lc; + IcebergVolumeServerOptions server_options; + IcebergForeignVolumeOptions volume_options; + DatalakeLocation location; + char *parse_detail = NULL; + DlErrCode parse_result; + + /* + * CREATE FOREIGN DATA WRAPPER invokes its validator with an empty array. + * Permit that bootstrap call, but this FDW has no wrapper-level options. + */ + if (catalog == ForeignDataWrapperRelationId && options == NIL) + PG_RETURN_VOID(); + + if (catalog != ForeignServerRelationId && + catalog != UserMappingRelationId) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("iceberg_volume_fdw has no options in this context"))); + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + const char *name = def->defname; + + if (catalog == ForeignServerRelationId) + { + if (dl_is_credential_option(name)) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("credential option \"%s\" is not allowed on an iceberg volume server", + name), + errhint("credentials belong in CREATE USER MAPPING ... OPTIONS (...)"))); + + if (!is_volume_server_option(name)) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid iceberg volume server option \"%s\"", + name), + errhint("Allowed options are \"%s\", \"%s\", \"%s\" and \"%s\".", + DATALAKE_ICEBERG_VOLUME_BASE_PATH, + DATALAKE_ICEBERG_VOLUME_ENDPOINT, + DATALAKE_ICEBERG_VOLUME_REGION, + DATALAKE_ICEBERG_VOLUME_PATH_STYLE_ACCESS))); + } + else if (!is_volume_user_mapping_option(name)) + ereport(ERROR, + (errcode(ERRCODE_FDW_INVALID_OPTION_NAME), + errmsg("invalid iceberg volume user mapping option \"%s\"", + name))); + } + + if (catalog == UserMappingRelationId) + PG_RETURN_VOID(); + + /* + * Parse with the same functions the use points parse with, so a server this + * validator accepted cannot fail to parse afterwards. path_style_access is + * checked as a side effect: the accessor refuses a non-boolean value. + */ + memset(&server_options, 0, sizeof(server_options)); + memset(&volume_options, 0, sizeof(volume_options)); + parse_iceberg_volume_server_options(&server_options, options); + parse_iceberg_foreign_volume_options(&volume_options, options); + + if (volume_options.base_path == NULL) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("iceberg volume server option \"%s\" is required", + DATALAKE_ICEBERG_VOLUME_BASE_PATH))); + + parse_result = pg_iceberg_parse_location(volume_options.base_path, + server_options.endpoint, + server_options.region, + &location, &parse_detail); + if (parse_result != DL_OK) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid iceberg volume %s \"%s\"", + DATALAKE_ICEBERG_VOLUME_BASE_PATH, + volume_options.base_path), + errdetail("%s", parse_detail))); + + PG_RETURN_VOID(); +} diff --git a/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.c b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.c new file mode 100644 index 00000000000..b304f27620e --- /dev/null +++ b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.c @@ -0,0 +1,94 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_volume_option.c + * Option vocabulary and parsed forms for Iceberg volume servers. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "common/parser_option.h" +#include "iceberg_volume_fdw/iceberg_volume_option.h" + +void +parse_iceberg_volume_server_options(IcebergVolumeServerOptions *options, + List *server_options) +{ + options->endpoint = + get_string_option(server_options, DATALAKE_ICEBERG_VOLUME_ENDPOINT); + options->region = + get_string_option(server_options, DATALAKE_ICEBERG_VOLUME_REGION); + + /* + * Absence has to stay distinguishable from an explicit false: the metadata + * engine merges what a server states over its own defaults per key, so an + * unset boolean must not arrive as one the user chose. + */ + options->path_style_access = + get_bool_option_ex(server_options, + DATALAKE_ICEBERG_VOLUME_PATH_STYLE_ACCESS, + false, &options->path_style_access_set); +} + +void +parse_iceberg_volume_user_mapping_options(IcebergVolumeUserMappingOptions *options, + List *user_options) +{ + options->username = + get_string_option(user_options, DATALAKE_ICEBERG_VOLUME_USERNAME); + options->aws_access_key_id = + get_string_option(user_options, + DATALAKE_ICEBERG_VOLUME_AWS_ACCESS_KEY_ID); + options->aws_secret_access_key = + get_string_option(user_options, + DATALAKE_ICEBERG_VOLUME_AWS_SECRET_ACCESS_KEY); + options->aws_session_token = + get_string_option(user_options, + DATALAKE_ICEBERG_VOLUME_AWS_SESSION_TOKEN); +} + +void +parse_iceberg_foreign_volume_options(IcebergForeignVolumeOptions *options, + List *volume_options) +{ + options->base_path = + get_string_option(volume_options, DATALAKE_ICEBERG_VOLUME_BASE_PATH); +} + +IcebergVolumeOptions * +get_iceberg_volume_options(ForeignServer *server) +{ + IcebergVolumeOptions *options; + + Assert(server != NULL); + + options = (IcebergVolumeOptions *) palloc0(sizeof(IcebergVolumeOptions)); + + parse_iceberg_volume_server_options(&options->volume_server, + server->options); + parse_iceberg_foreign_volume_options(&options->foreign_volume, + server->options); + + return options; +} diff --git a/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h new file mode 100644 index 00000000000..2b4ab16b46c --- /dev/null +++ b/contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h @@ -0,0 +1,146 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_volume_option.h + * Option vocabulary and parsed forms for Iceberg volume servers. + * + * As on the catalog side, the key macros, struct names and field names are + * reproduced from the reference implementation -- the existing implementation + * of this feature that this work derives from and is meant to replace -- so + * that support for a further storage protocol moves across as an addition. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/iceberg_volume_fdw/iceberg_volume_option.h + * + *------------------------------------------------------------------------- + */ + +#ifndef ICEBERG_VOLUME_OPTION_H +#define ICEBERG_VOLUME_OPTION_H + +#include "postgres.h" + +#include "common/dl_option_util.h" +#include "foreign/foreign.h" +#include "nodes/pg_list.h" + +/* Volume server options */ +#define DATALAKE_ICEBERG_VOLUME_ENDPOINT "endpoint" +#define DATALAKE_ICEBERG_VOLUME_REGION "region" +#define DATALAKE_ICEBERG_VOLUME_PATH_STYLE_ACCESS "path_style_access" + +/* Volume user mapping options */ +#define DATALAKE_ICEBERG_VOLUME_USERNAME DL_OPTION_KEY_USERNAME +#define DATALAKE_ICEBERG_VOLUME_AWS_ACCESS_KEY_ID DL_OPTION_KEY_ACCESS_KEY_ID +#define DATALAKE_ICEBERG_VOLUME_AWS_SECRET_ACCESS_KEY DL_OPTION_KEY_SECRET_ACCESS_KEY +#define DATALAKE_ICEBERG_VOLUME_AWS_SESSION_TOKEN DL_OPTION_KEY_SESSION_TOKEN + +/* Volume location option */ +#define DATALAKE_ICEBERG_VOLUME_BASE_PATH "base_path" + +/* + * Storage protocols. Unlike the reference implementation, no server option + * names the protocol: base_path carries a URI, so its scheme already says which + * protocol this volume speaks, and a separate option could only disagree with + * it. The names are kept because the parsed location is compared against them. + */ +#define DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_S3 "s3" +#define DATALAKE_ICEBERG_VOLUME_SERVER_TYPE_HDFS "hdfs" + +typedef struct IcebergVolumeServerOptions +{ + char *endpoint; /* DATALAKE_ICEBERG_VOLUME_ENDPOINT */ + char *region; /* DATALAKE_ICEBERG_VOLUME_REGION */ + bool path_style_access; /* DATALAKE_ICEBERG_VOLUME_PATH_STYLE_ACCESS */ + bool path_style_access_set; /* user actually wrote path_style_access */ + + /* + * Deferred, with the reference implementation's names kept for the port. + * server_type and bucket_name are absent by design instead: both are + * derived from base_path, and DatalakeLocation is the parsed form. + * + * AWS: role_arn / external_id / user_arn / current_kms_key / + * allowed_kms_keys / sts_endpoint / sts_unavailable / endpoint_internal + * Azure: tenant_id / multi_tenant_app_name / consent_url / hierarchical + * HDFS: hdfs_namenodes / hdfs_port / hdfs_auth_method / krb_principal / + * krb_principal_keytab / krb_service_principal / + * hadoop_rpc_protection / data_transfer_protocol / is_ha_supported / + * dfs_nameservices / dfs_ha_namenodes / dfs_namenode_rpc_address / + * dfs_client_failover_proxy_provider / + * dfs_client_use_datanode_hostname + */ +} IcebergVolumeServerOptions; + +typedef struct IcebergVolumeUserMappingOptions +{ + char *username; /* DATALAKE_ICEBERG_VOLUME_USERNAME */ + char *aws_access_key_id; /* DATALAKE_ICEBERG_VOLUME_AWS_ACCESS_KEY_ID */ + char *aws_secret_access_key; /* DATALAKE_ICEBERG_VOLUME_AWS_SECRET_ACCESS_KEY */ + + /* + * Temporary credentials are three values, not two, and are what AWS + * recommends over long-lived keys; without this field the mapping could + * only express the long-lived form. + */ + char *aws_session_token; /* DATALAKE_ICEBERG_VOLUME_AWS_SESSION_TOKEN */ +} IcebergVolumeUserMappingOptions; + +typedef struct IcebergForeignVolumeOptions +{ + /* + * The reference implementation reads these from a foreign volume object; as + * with the catalog side, a volume server here names exactly one volume and + * the value comes from that server's options. + * + * Deferred, names kept: enable_caching / allow_writes / fileIOConfig / + * table_identifier. + */ + char *base_path; /* DATALAKE_ICEBERG_VOLUME_BASE_PATH */ +} IcebergForeignVolumeOptions; + +typedef struct IcebergVolumeOptions +{ + IcebergVolumeServerOptions volume_server; + IcebergVolumeUserMappingOptions volume_user; + IcebergForeignVolumeOptions foreign_volume; +} IcebergVolumeOptions; + +/* Reference implementation: parseIcebergVolumeServerOptions() */ +extern void parse_iceberg_volume_server_options(IcebergVolumeServerOptions *options, + List *server_options); + +/* Reference implementation: parseIcebergVolumeUserMappingOptions() */ +extern void parse_iceberg_volume_user_mapping_options(IcebergVolumeUserMappingOptions *options, + List *user_options); + +/* Reference implementation: parseIcebergForeignVolumeOptions() */ +extern void parse_iceberg_foreign_volume_options(IcebergForeignVolumeOptions *options, + List *volume_options); + +/* + * Reference implementation: getIcebergVolumeOptions(). volume_user is left + * zeroed for the same reason as on the catalog side. + * + * The reference implementation's buildVolumeBasePath() has no counterpart: + * base_path is parsed once into a DatalakeLocation, and every layer below + * receives that instead of re-parsing a URI. + */ +extern IcebergVolumeOptions *get_iceberg_volume_options(ForeignServer *server); + +#endif /* ICEBERG_VOLUME_OPTION_H */ diff --git a/contrib/datalake_fdw/src/meta/engine_stub/stub_engine.c b/contrib/datalake_fdw/src/meta/engine_stub/stub_engine.c new file mode 100644 index 00000000000..dfd84c5e54e --- /dev/null +++ b/contrib/datalake_fdw/src/meta/engine_stub/stub_engine.c @@ -0,0 +1,102 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * stub_engine.c + * A metadata engine that reports what it would have done. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/meta/engine_stub/stub_engine.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "meta/engine_stub/stub_engine.h" +#include "meta/iceberg_meta_engine.h" + +static DlErrCode +stub_create_table(const MetaCtx *ctx, const MetaTableDef *def, MetaTable **out) +{ + + ereport(NOTICE, + (errmsg("stub engine: created iceberg table \"%s.%s\" in catalog \"%s\"", + ctx->namespace_name, ctx->table_name, ctx->catalog_name))); + *out = NULL; + return DL_OK; +} + +static DlErrCode +stub_drop_table(const MetaCtx *ctx, bool purge) +{ + /* + * Report whether the data would have gone with it, so that a regression can + * see which of the two things a DROP asked for. + */ + ereport(NOTICE, + (errmsg("stub engine: dropped iceberg table \"%s.%s\" from catalog \"%s\"%s", + ctx->namespace_name, ctx->table_name, ctx->catalog_name, + purge ? ", purging data" : ", keeping data"))); + return DL_OK; +} + +static DlErrCode +stub_table_exists(const MetaCtx *ctx, bool *exists) +{ + + /* The stub has no remote catalog. */ + *exists = false; + return DL_OK; +} + +/* + * Nothing to load in the skeleton; the method remains a member of the lifecycle + * family, so it exists and refuses. + * + * It refuses the way a real engine has to: the code says what kind of failure it + * is, and everything specific to this failure -- which table, which operation, + * what the implementation calls it -- is recorded for the reporting layer. A + * remote engine records the message and stack it received here instead. + */ +static DlErrCode +stub_load_table(const MetaCtx *ctx, MetaTable **out) +{ + dl_error_set(DL_ERR_NOT_SUPPORTED, "load_table", "StubEngine", + psprintf("the stub engine holds no metadata for \"%s.%s\"", + ctx->namespace_name, ctx->table_name)); + return DL_ERR_NOT_SUPPORTED; +} + +static const IcebergMetaEngine stub_engine = +{ + .abi_version = DL_META_ENGINE_ABI_VERSION, + .struct_size = sizeof(IcebergMetaEngine), + .capabilities = DL_CAP_TABLE_LIFECYCLE, + .name = "stub", + .load_table = stub_load_table, + .create_table = stub_create_table, + .drop_table = stub_drop_table, + .table_exists = stub_table_exists +}; + +DlErrCode +RegisterStubMetaEngine(void) +{ + return RegisterMetaEngine(&stub_engine); +} diff --git a/contrib/datalake_fdw/src/meta/engine_stub/stub_engine.h b/contrib/datalake_fdw/src/meta/engine_stub/stub_engine.h new file mode 100644 index 00000000000..df87bb58d2e --- /dev/null +++ b/contrib/datalake_fdw/src/meta/engine_stub/stub_engine.h @@ -0,0 +1,36 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * stub_engine.h + * A metadata engine that reports what it would have done. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/meta/engine_stub/stub_engine.h + * + *------------------------------------------------------------------------- + */ + +#ifndef STUB_ENGINE_H +#define STUB_ENGINE_H + +#include "common/dl_err.h" + +extern DlErrCode RegisterStubMetaEngine(void); + +#endif /* STUB_ENGINE_H */ diff --git a/contrib/datalake_fdw/src/meta/iceberg_meta_engine.h b/contrib/datalake_fdw/src/meta/iceberg_meta_engine.h new file mode 100644 index 00000000000..ef80835eb27 --- /dev/null +++ b/contrib/datalake_fdw/src/meta/iceberg_meta_engine.h @@ -0,0 +1,149 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * iceberg_meta_engine.h + * The metadata engine interface and its central dispatch. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/meta/iceberg_meta_engine.h + * + *------------------------------------------------------------------------- + */ + +#ifndef ICEBERG_META_ENGINE_H +#define ICEBERG_META_ENGINE_H + +/* + * This is the header an engine implementation includes, and the next one is + * expected to be C++. Everything below therefore has to keep C linkage: a C++ + * translation unit that saw these as C++ declarations would emit mangled + * references and fail to link against the C registry. The server headers come + * in through dl_pg_api.h for the same reason. + */ +#include "common/dl_pg_api.h" + +#include "common/dl_err.h" +#include "common/dl_kv.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Server, mapping and table options all arrive as plain pairs. */ +typedef DlKeyValue MetaKv; + +typedef struct MetaCtx { /* identity + mapping, no credentials in skeleton */ + const char *catalog_name; /* catalog name within the external catalog */ + const char *namespace_name; /* PG schema name */ + const char *table_name; + const MetaKv *catalog_props; int n_catalog_props; + const MetaKv *credential_props; int n_credential_props; /* empty in stub paths */ +} MetaCtx; + +typedef struct MetaTableDef { const char *schema_json; } MetaTableDef; +typedef struct MetaTable { char *metadata_location; char *table_uuid; } MetaTable; +/* Opaque in skeleton: */ +typedef struct MetaStatistics MetaStatistics; +typedef struct MetaAppendRequest MetaAppendRequest; +typedef struct MetaCommitAppendRequest MetaCommitAppendRequest; +typedef struct MetaUpdateRequest MetaUpdateRequest; +typedef struct MetaCommitUpdateRequest MetaCommitUpdateRequest; +typedef struct MetaStageResult MetaStageResult; +typedef struct MetaCommitResult MetaCommitResult; +typedef struct MetaFilterExpr MetaFilterExpr; +typedef struct MetaFragmentIter MetaFragmentIter; +typedef struct MetaFragmentBatch MetaFragmentBatch; +typedef struct MetaFileGroupIter MetaFileGroupIter; +typedef struct MetaFileGroup MetaFileGroup; +typedef struct MetaFileGroupList MetaFileGroupList; +typedef struct MetaAlterTableRequest MetaAlterTableRequest; +typedef struct MetaTruncateRequest MetaTruncateRequest; + +#define DL_META_ENGINE_ABI_VERSION 1 + +typedef struct IcebergMetaEngine { + uint32_t abi_version; /* must equal DL_META_ENGINE_ABI_VERSION */ + uint32_t struct_size; /* PREFIX-compat: registry only touches fields covered by + * struct_size; validation bound is the minimal v1 prefix, + * NOT sizeof(current struct). Tail may only grow. */ + uint64_t capabilities; /* DL_CAP_* method-family bitmap, see below */ + const char *name; /* "agent" / "builtin" / "stub" */ + + DlErrCode (*load_table)(const MetaCtx *, MetaTable **); + DlErrCode (*create_table)(const MetaCtx *, const MetaTableDef *, MetaTable **); + DlErrCode (*drop_table)(const MetaCtx *, bool purge); + DlErrCode (*table_exists)(const MetaCtx *, bool *); + DlErrCode (*get_statistics)(const MetaCtx *, int64_t snapshot, MetaStatistics **); + + /* Iceberg single-table OCC: stage (append/update) then commit_*; NOT a cross-table + * distributed atomic commit. */ + DlErrCode (*append)(const MetaCtx *, const MetaAppendRequest *, MetaStageResult *); + DlErrCode (*commit_append)(const MetaCtx *, const MetaCommitAppendRequest *, MetaCommitResult *); + DlErrCode (*update)(const MetaCtx *, const MetaUpdateRequest *, MetaStageResult *); + DlErrCode (*commit_update)(const MetaCtx *, const MetaCommitUpdateRequest *, MetaCommitResult *); + + DlErrCode (*get_fragment)(const MetaCtx *, const char *metadata_location, + const MetaFilterExpr *, uint32_t batch_hint, MetaFragmentIter **); + DlErrCode (*plan_file_groups)(const MetaCtx *, const char *plan_options_json, MetaFileGroupIter **); + DlErrCode (*commit_file_groups)(const MetaCtx *, const MetaFileGroupList *, const char *, MetaCommitResult *); + DlErrCode (*alter_table)(const MetaCtx *, const MetaAlterTableRequest *, MetaCommitResult *); + DlErrCode (*truncate_table)(const MetaCtx *, const MetaTruncateRequest *, MetaCommitResult *); + + /* iterator close callbacks are "void cleanup ABI": noexcept, idempotent, never ereport */ + DlErrCode (*fragment_iter_next_batch)(MetaFragmentIter *, MetaFragmentBatch **); + void (*fragment_iter_close)(MetaFragmentIter *); + DlErrCode (*file_group_iter_next)(MetaFileGroupIter *, MetaFileGroup **); + void (*file_group_iter_close)(MetaFileGroupIter *); +} IcebergMetaEngine; + +#define DL_CAP_TABLE_LIFECYCLE (UINT64CONST(1) << 0) /* load_table, create_table, drop_table, table_exists */ +#define DL_CAP_STATISTICS (UINT64CONST(1) << 1) /* get_statistics */ +#define DL_CAP_APPEND (UINT64CONST(1) << 2) /* append, commit_append */ +#define DL_CAP_UPDATE (UINT64CONST(1) << 3) /* update, commit_update */ +#define DL_CAP_GET_FRAGMENT (UINT64CONST(1) << 4) /* get_fragment, fragment_iter_next_batch, fragment_iter_close */ +#define DL_CAP_REWRITE (UINT64CONST(1) << 5) /* plan_file_groups, commit_file_groups, file_group_iter_next, file_group_iter_close */ +#define DL_CAP_ALTER (UINT64CONST(1) << 6) /* alter_table */ +#define DL_CAP_TRUNCATE (UINT64CONST(1) << 7) /* truncate_table */ + +extern DlErrCode RegisterMetaEngine(const IcebergMetaEngine *engine); + +/* + * Returns the engine every lake table goes through. The engine is not + * selectable: nothing in a table's definition, and no configuration setting, + * picks between implementations, so a table can never be reinterpreted by a + * later change. The vtable indirection remains because the implementation + * behind it is expected to change -- the Java agent today, an in-process C++ + * one once it exists -- not because a deployment gets to choose. + */ +extern const IcebergMetaEngine *get_meta_engine(void); + +extern DlErrCode meta_engine_create_table(const IcebergMetaEngine *, const MetaCtx *, + const MetaTableDef *, MetaTable **); +extern DlErrCode meta_engine_drop_table(const IcebergMetaEngine *, const MetaCtx *, + bool purge); +extern DlErrCode meta_engine_table_exists(const IcebergMetaEngine *, const MetaCtx *, bool *); +extern DlErrCode meta_engine_load_table(const IcebergMetaEngine *, const MetaCtx *, MetaTable **); + +/* Remaining method families follow the same central-dispatch pattern in later PRs. */ + +#ifdef __cplusplus +} +#endif + +#endif /* ICEBERG_META_ENGINE_H */ diff --git a/contrib/datalake_fdw/src/meta/meta_engine_init.c b/contrib/datalake_fdw/src/meta/meta_engine_init.c new file mode 100644 index 00000000000..689b3d2c8d1 --- /dev/null +++ b/contrib/datalake_fdw/src/meta/meta_engine_init.c @@ -0,0 +1,49 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * meta_engine_init.c + * Registration of the metadata engine this build provides. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/meta/meta_engine_init.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "common/dl_err.h" +#include "meta/engine_stub/stub_engine.h" +#include "meta/meta_engine_init.h" + +/* + * Called from _PG_init (extensible.c, later task); agent and builtin engines + * join in later PRs. SQL-visible NOTICE behavior is covered by skel-regress; + * this skeleton has no separate C test harness. + */ +void +DatalakeRegisterMetaEngines(void) +{ + DlErrCode rc; + + rc = RegisterStubMetaEngine(); + if (rc != DL_OK) + elog(ERROR, "datalake_fdw: failed to register stub meta engine: %s", + dl_err_message(rc)); +} diff --git a/contrib/datalake_fdw/src/meta/meta_engine_init.h b/contrib/datalake_fdw/src/meta/meta_engine_init.h new file mode 100644 index 00000000000..c20bc5fd6ba --- /dev/null +++ b/contrib/datalake_fdw/src/meta/meta_engine_init.h @@ -0,0 +1,34 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * meta_engine_init.h + * Registration of the metadata engine this build provides. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/meta/meta_engine_init.h + * + *------------------------------------------------------------------------- + */ + +#ifndef META_ENGINE_INIT_H +#define META_ENGINE_INIT_H + +extern void DatalakeRegisterMetaEngines(void); + +#endif /* META_ENGINE_INIT_H */ diff --git a/contrib/datalake_fdw/src/meta/meta_engine_registry.c b/contrib/datalake_fdw/src/meta/meta_engine_registry.c new file mode 100644 index 00000000000..d4bb021c123 --- /dev/null +++ b/contrib/datalake_fdw/src/meta/meta_engine_registry.c @@ -0,0 +1,245 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * meta_engine_registry.c + * Metadata engine registry, capability checks and dispatch. + * + * IDENTIFICATION + * contrib/datalake_fdw/src/meta/meta_engine_registry.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include +#include + +#include "meta/iceberg_meta_engine.h" + +/* Registered engine pointers are borrowed; each engine must have static lifetime. */ +static const IcebergMetaEngine *engines[8]; +static int n_engines; + +/* + * v1 minimal prefix = through file_group_iter_close; the registry must never + * touch fields beyond engine->struct_size. Do NOT compare against + * sizeof(IcebergMetaEngine) -- that breaks prefix compatibility once the struct + * grows. + */ +#define DL_META_ENGINE_V1_MIN_SIZE \ + (offsetof(IcebergMetaEngine, file_group_iter_close) + \ + sizeof(((IcebergMetaEngine *) 0)->file_group_iter_close)) + +typedef struct DlCapabilityFamily +{ + uint64 bit; + const char *family; + size_t offsets[5]; + int n; +} DlCapabilityFamily; + +static const DlCapabilityFamily capability_families[] = +{ + {DL_CAP_TABLE_LIFECYCLE, "table lifecycle", + {offsetof(IcebergMetaEngine, load_table), + offsetof(IcebergMetaEngine, create_table), + offsetof(IcebergMetaEngine, drop_table), + offsetof(IcebergMetaEngine, table_exists)}, 4}, + {DL_CAP_STATISTICS, "statistics", + {offsetof(IcebergMetaEngine, get_statistics)}, 1}, + {DL_CAP_APPEND, "append", + {offsetof(IcebergMetaEngine, append), + offsetof(IcebergMetaEngine, commit_append)}, 2}, + {DL_CAP_UPDATE, "update", + {offsetof(IcebergMetaEngine, update), + offsetof(IcebergMetaEngine, commit_update)}, 2}, + {DL_CAP_GET_FRAGMENT, "get fragment", + {offsetof(IcebergMetaEngine, get_fragment), + offsetof(IcebergMetaEngine, fragment_iter_next_batch), + offsetof(IcebergMetaEngine, fragment_iter_close)}, 3}, + {DL_CAP_REWRITE, "rewrite", + {offsetof(IcebergMetaEngine, plan_file_groups), + offsetof(IcebergMetaEngine, commit_file_groups), + offsetof(IcebergMetaEngine, file_group_iter_next), + offsetof(IcebergMetaEngine, file_group_iter_close)}, 4}, + {DL_CAP_ALTER, "alter", + {offsetof(IcebergMetaEngine, alter_table)}, 1}, + {DL_CAP_TRUNCATE, "truncate", + {offsetof(IcebergMetaEngine, truncate_table)}, 1} +}; + +/* + * Is the method at this offset present, and set? + * + * A method that falls beyond the engine's struct_size is not part of the + * object at all: reading it would run past what the engine allocated. Such a + * method counts as absent, which is exactly what prefix compatibility means -- + * an engine built against an older header stays loadable, and every capability + * whose family reaches into the missing tail must be left unset. + */ +static bool +meta_engine_method_is_nonnull(const IcebergMetaEngine *engine, size_t offset) +{ + void (*method)(void); + + if (offset + sizeof(method) > engine->struct_size) + return false; + + memcpy(&method, (const char *) engine + offset, sizeof(method)); + return method != NULL; +} + +DlErrCode +RegisterMetaEngine(const IcebergMetaEngine *engine) +{ + int i; + int j; + + if (engine == NULL) + return DL_ERR_INVALID_OPTION; + if (engine->abi_version != DL_META_ENGINE_ABI_VERSION) + return DL_ERR_INVALID_OPTION; + if (engine->struct_size < DL_META_ENGINE_V1_MIN_SIZE) + return DL_ERR_INVALID_OPTION; + if (engine->name == NULL) + return DL_ERR_INVALID_OPTION; + + for (i = 0; i < n_engines; i++) + { + if (strcmp(engines[i]->name, engine->name) == 0) + return DL_ERR_ALREADY_EXISTS; + } + if (n_engines >= lengthof(engines)) + return DL_ERR_INTERNAL; + + for (i = 0; i < lengthof(capability_families); i++) + { + const DlCapabilityFamily *family = &capability_families[i]; + bool capability_set = + (engine->capabilities & family->bit) != 0; + + for (j = 0; j < family->n; j++) + { + bool method_is_nonnull = + meta_engine_method_is_nonnull(engine, family->offsets[j]); + + if (capability_set != method_is_nonnull) + { + elog(WARNING, + "datalake_fdw: meta engine \"%s\" has an invalid %s capability family", + engine->name, family->family); + return DL_ERR_INVALID_OPTION; + } + } + } + + /* + * One engine per build, by design: nothing selects between implementations + * at run time, so a second registration would silently decide which one + * every table goes through, depending on registration order. + */ + if (n_engines > 0) + return DL_ERR_ALREADY_EXISTS; + + engines[n_engines++] = engine; + return DL_OK; +} + +const IcebergMetaEngine * +get_meta_engine(void) +{ + /* + * Exactly one engine is registered, by DatalakeRegisterMetaEngines(); the + * array exists so that adding a second implementation later is a matter of + * changing what gets registered, not of reworking the call sites. + */ + if (n_engines != 1) + elog(ERROR, + "datalake_fdw: expected exactly one metadata engine, found %d", + n_engines); + + return engines[0]; +} + +/* + * What every dispatch wrapper does before reaching the engine: discard detail + * recorded by an earlier call, so that a later report can only describe this + * one, and refuse a method the engine does not advertise. + * + * The reset happens before the capability check on purpose. A refusal produced + * here has no detail of its own, and leaving an earlier one in place would let + * it be reported as the cause. + */ +static DlErrCode +meta_engine_enter(const IcebergMetaEngine *engine, uint64 capability) +{ + dl_error_reset(); + + if (engine == NULL) + return DL_ERR_INTERNAL; + if ((engine->capabilities & capability) == 0) + return DL_ERR_NOT_SUPPORTED; + + return DL_OK; +} + +DlErrCode +meta_engine_create_table(const IcebergMetaEngine *engine, const MetaCtx *ctx, + const MetaTableDef *def, MetaTable **out) +{ + DlErrCode rc = meta_engine_enter(engine, DL_CAP_TABLE_LIFECYCLE); + + if (rc != DL_OK) + return rc; + return engine->create_table(ctx, def, out); +} + +DlErrCode +meta_engine_drop_table(const IcebergMetaEngine *engine, const MetaCtx *ctx, + bool purge) +{ + DlErrCode rc = meta_engine_enter(engine, DL_CAP_TABLE_LIFECYCLE); + + if (rc != DL_OK) + return rc; + return engine->drop_table(ctx, purge); +} + +DlErrCode +meta_engine_table_exists(const IcebergMetaEngine *engine, const MetaCtx *ctx, + bool *exists) +{ + DlErrCode rc = meta_engine_enter(engine, DL_CAP_TABLE_LIFECYCLE); + + if (rc != DL_OK) + return rc; + return engine->table_exists(ctx, exists); +} + +DlErrCode +meta_engine_load_table(const IcebergMetaEngine *engine, const MetaCtx *ctx, + MetaTable **out) +{ + DlErrCode rc = meta_engine_enter(engine, DL_CAP_TABLE_LIFECYCLE); + + if (rc != DL_OK) + return rc; + return engine->load_table(ctx, out); +} diff --git a/contrib/datalake_fdw/test/automation/Makefile b/contrib/datalake_fdw/test/automation/Makefile new file mode 100644 index 00000000000..fcedd800c0b --- /dev/null +++ b/contrib/datalake_fdw/test/automation/Makefile @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# contrib/datalake_fdw/test/automation/Makefile + +.PHONY: all test smoke-test check-services list-categories help clean + +all: test + +help: + @echo 'datalake_fdw automation tests' + @echo + @echo ' make test run the smoke categories (default)' + @echo ' make smoke-test same' + @echo ' make check-services report which external services answer' + @echo ' make list-categories list categories and what each one needs' + @echo ' make clean remove run output' + @echo + @echo ' CATEGORIES="..." run only these categories' + @echo + @echo 'Service addresses come from config/test_config.env, which reads them' + @echo 'from the environment first. A category whose services are absent is' + @echo 'skipped and reported as skipped.' + +test: smoke-test + +smoke-test: + @bash scripts/test/run_smoke_tests.sh + +check-services: + @bash scripts/setup/check_services.sh + +list-categories: + @printf '%-16s %s\n' 'CATEGORY' 'REQUIRES' + @printf '%-16s %s\n' 'iceberg_am' '(nothing external)' + +clean: + rm -rf sqlrepo/smoke/*/results diff --git a/contrib/datalake_fdw/test/automation/README.md b/contrib/datalake_fdw/test/automation/README.md new file mode 100644 index 00000000000..f6586ed0e5c --- /dev/null +++ b/contrib/datalake_fdw/test/automation/README.md @@ -0,0 +1,80 @@ + + +# datalake_fdw automation tests + +Lake tables are only half local. Once the metadata engine is connected, the +behaviour worth testing is what happens against a real Hive metastore, real +object storage and a real HDFS -- which comparison against a recorded transcript +cannot express, because the interesting cases are the ones where an external +service is slow, absent, or disagrees. This directory is where those tests go, +and it exists now so that they are added here rather than somewhere new. + +Everything currently here needs no external service. + +## Running + +```sh +make test # run the smoke categories +make check-services # what answers right now +make list-categories # categories, and what each one needs +``` + +Service addresses come from `config/test_config.env`, which takes them from the +environment first, so a run against an existing deployment needs no edit: + +```sh +DL_HMS_HOST=metastore.example DL_S3_ENDPOINT=http://minio:9000 make test +``` + +A category whose services are absent is **skipped and reported as skipped**, so +a developer without a metastore still gets a useful run and nobody reads a skip +as a pass. + +## Layout + +``` +config/ service addresses and switches +scripts/setup/ service probes +scripts/test/ category runners +scripts/utils/ shared shell helpers +sqlrepo/smoke/ one directory per category + iceberg_am/ DDL, refusals and privileges -- no external service +``` + +`sqlrepo/smoke/iceberg_am` holds the cases pg_regress runs; the module's +`Makefile` points `--inputdir` here, so `make installcheck` from the module +directory and `make test` from this one run the same cases. They live here +rather than in a `sql/` directory of their own so that there is one place to look +for test material. + +## What arrives with the metadata engine + +Named here so the shape is known before the code lands, rather than reserved as +empty directories: + +- `docker/` -- compose definitions bringing up a metastore, MinIO and a + single-node HDFS, so a category can run anywhere +- `prepare/` -- fixture loading per service: create the warehouse, the bucket, + the namespaces +- `lib/` -- SQL and shell fragments shared by categories +- `tools/` -- data generators for the volume and scale categories +- `sqlrepo/smoke/iceberg_hive`, `iceberg_s3`, `iceberg_hdfs` -- the read and + write paths against each service +- `sqlrepo/feature`, `sqlrepo/negative` -- everything past the smoke level diff --git a/contrib/datalake_fdw/test/automation/config/test_config.env b/contrib/datalake_fdw/test/automation/config/test_config.env new file mode 100644 index 00000000000..1389becc489 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/config/test_config.env @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Where the external services live. Overridden from the environment, so a run +# against an existing deployment needs no edit here. +# +# Nothing in the smoke suite needs any of these yet: the categories that do +# arrive with the metadata engine. They are named now because the runner +# already decides what to skip from them. + +# Hive metastore, for catalog type "hive". +DL_HMS_HOST="${DL_HMS_HOST:-localhost}" +DL_HMS_PORT="${DL_HMS_PORT:-9083}" + +# S3-compatible object storage, for volumes with an s3:// base path. +DL_S3_ENDPOINT="${DL_S3_ENDPOINT:-http://localhost:9000}" +DL_S3_BUCKET="${DL_S3_BUCKET:-datalake-test}" +DL_S3_REGION="${DL_S3_REGION:-us-east-1}" + +# HDFS namenode, for volumes with an hdfs:// base path. +DL_HDFS_HOST="${DL_HDFS_HOST:-localhost}" +DL_HDFS_PORT="${DL_HDFS_PORT:-8020}" + +# How long a service probe waits before calling the service absent. +DL_PROBE_TIMEOUT="${DL_PROBE_TIMEOUT:-3}" diff --git a/contrib/datalake_fdw/test/automation/scripts/setup/check_services.sh b/contrib/datalake_fdw/test/automation/scripts/setup/check_services.sh new file mode 100755 index 00000000000..dc300d9dc45 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/scripts/setup/check_services.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Report which external services are reachable. +# +# Informational on purpose: it exits 0 whether or not anything answered, because +# its output decides which categories the runner skips, and a missing service is +# a reason to skip a category rather than to fail a run. + +set -u + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../utils/common_functions.sh +. "$script_dir/../utils/common_functions.sh" + +dl_load_config + +report() +{ + local name="$1" where="$2" state="$3" + + printf '%-16s %-32s %s\n' "$name" "$where" "$state" +} + +printf '%-16s %-32s %s\n' 'SERVICE' 'ADDRESS' 'STATE' + +if dl_tcp_is_open "$DL_HMS_HOST" "$DL_HMS_PORT" "$DL_PROBE_TIMEOUT"; then + report 'hive-metastore' "$DL_HMS_HOST:$DL_HMS_PORT" 'available' +else + report 'hive-metastore' "$DL_HMS_HOST:$DL_HMS_PORT" 'absent' +fi + +if dl_http_is_open "$DL_S3_ENDPOINT" "$DL_PROBE_TIMEOUT"; then + report 's3' "$DL_S3_ENDPOINT" 'available' +else + report 's3' "$DL_S3_ENDPOINT" 'absent' +fi + +if dl_tcp_is_open "$DL_HDFS_HOST" "$DL_HDFS_PORT" "$DL_PROBE_TIMEOUT"; then + report 'hdfs' "$DL_HDFS_HOST:$DL_HDFS_PORT" 'available' +else + report 'hdfs' "$DL_HDFS_HOST:$DL_HDFS_PORT" 'absent' +fi + +exit 0 diff --git a/contrib/datalake_fdw/test/automation/scripts/test/run_smoke_tests.sh b/contrib/datalake_fdw/test/automation/scripts/test/run_smoke_tests.sh new file mode 100755 index 00000000000..ce707a1fed3 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/scripts/test/run_smoke_tests.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Run the smoke categories under sqlrepo/smoke. +# +# A category names its required services in REQUIRED_SERVICES below. One with +# none is always run; one whose services are absent is skipped and reported as +# skipped, so a developer without a Hive metastore still gets a useful run and +# nobody mistakes a skip for a pass. + +set -u -o pipefail + +script_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../utils/common_functions.sh +. "$script_dir/../utils/common_functions.sh" + +automation_dir="$(dl_automation_dir)" +module_dir="$(cd -- "$automation_dir/../.." && pwd)" + +dl_load_config + +# category:services -- an empty service list means "no external dependency" +CATEGORY_SERVICES=" +iceberg_am: +" + +categories="${CATEGORIES:-iceberg_am}" + +services_for() +{ + local category="$1" line + + while read -r line; do + [ -n "$line" ] || continue + case "$line" in + "$category":*) printf '%s' "${line#*:}"; return 0 ;; + esac + done <<-EOF + $CATEGORY_SERVICES + EOF + + printf 'unknown' +} + +service_is_available() +{ + case "$1" in + hive-metastore) + dl_tcp_is_open "$DL_HMS_HOST" "$DL_HMS_PORT" "$DL_PROBE_TIMEOUT" ;; + s3) + dl_http_is_open "$DL_S3_ENDPOINT" "$DL_PROBE_TIMEOUT" ;; + hdfs) + dl_tcp_is_open "$DL_HDFS_HOST" "$DL_HDFS_PORT" "$DL_PROBE_TIMEOUT" ;; + *) + return 1 ;; + esac +} + +run_iceberg_am() +{ + # These cases are expected-output cases, so pg_regress runs them; the module + # Makefile already points it at sqlrepo/smoke/iceberg_am. + make -C "$module_dir" USE_PGXS=1 installcheck +} + +failed=0 +skipped=0 +ran=0 + +for category in $categories; do + required="$(services_for "$category")" + + if [ "$required" = 'unknown' ]; then + dl_warn "unknown category \"$category\"" + failed=$((failed + 1)) + continue + fi + + missing='' + for service in $required; do + service_is_available "$service" || missing="$missing $service" + done + + if [ -n "$missing" ]; then + dl_info "SKIP $category (absent:$missing)" + skipped=$((skipped + 1)) + continue + fi + + dl_info "RUN $category" + case "$category" in + iceberg_am) run_iceberg_am ;; + *) dl_warn "category \"$category\" has no runner"; false ;; + esac + + if [ $? -eq 0 ]; then + ran=$((ran + 1)) + else + dl_warn "FAIL $category" + failed=$((failed + 1)) + fi +done + +dl_info "passed=$ran skipped=$skipped failed=$failed" +[ "$failed" -eq 0 ] diff --git a/contrib/datalake_fdw/test/automation/scripts/utils/common_functions.sh b/contrib/datalake_fdw/test/automation/scripts/utils/common_functions.sh new file mode 100644 index 00000000000..3a9d92298e1 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/scripts/utils/common_functions.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Shared helpers. Sourced, never executed. + +# Absolute path of the automation directory, whichever directory the caller +# started from. +dl_automation_dir() +{ + local here + here="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" + cd -- "$here/../.." && pwd +} + +dl_load_config() +{ + local automation_dir + automation_dir="$(dl_automation_dir)" + + # shellcheck source=../../config/test_config.env + . "$automation_dir/config/test_config.env" +} + +dl_info() +{ + printf '[automation] %s\n' "$*" +} + +dl_warn() +{ + printf '[automation] %s\n' "$*" >&2 +} + +dl_die() +{ + dl_warn "$*" + exit 1 +} + +# Name of a working timeout command, or empty when there is none. macOS ships +# neither; coreutils installs it as gtimeout. +dl_timeout_command() +{ + local candidate + + for candidate in timeout gtimeout; do + if command -v "$candidate" >/dev/null 2>&1; then + printf '%s' "$candidate" + return 0 + fi + done + + return 1 +} + +# True when something is listening on host:port. Uses bash's own /dev/tcp so +# that a probe needs no tool that might not be installed. +# +# Host and port are passed as arguments rather than interpolated into the +# program text: they come from the environment, and a shell metacharacter in one +# would otherwise be executed. Being unable to probe is reported as a harness +# failure, not as "service absent" -- silently skipping a category because a +# tool is missing is how a suite stops testing anything without saying so. +dl_tcp_is_open() +{ + local host="$1" port="$2" seconds="${3:-3}" timeout_cmd + + if ! timeout_cmd="$(dl_timeout_command)"; then + dl_die "no timeout command found (install coreutils for gtimeout)" + fi + + "$timeout_cmd" "$seconds" bash -c \ + 'exec 3<>/dev/tcp/"$1"/"$2"' _ "$host" "$port" 2>/dev/null +} + +# True when an HTTP endpoint answers at all; any status counts, because a probe +# asks whether the service is there, not whether a request would succeed. +dl_http_is_open() +{ + local url="$1" timeout="${2:-3}" + + command -v curl >/dev/null 2>&1 || return 1 + curl --silent --show-error --output /dev/null \ + --max-time "$timeout" "$url" 2>/dev/null +} diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_acl.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_acl.out new file mode 100644 index 00000000000..c03534b1ca3 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_acl.out @@ -0,0 +1,75 @@ +-- Foreign-server USAGE is required; user mappings remain optional. +SET client_min_messages = warning; +RESET ROLE; +DROP SCHEMA IF EXISTS dlskel_s CASCADE; +DROP SERVER IF EXISTS dlskel_acl_cat CASCADE; +DROP SERVER IF EXISTS dlskel_acl_vol CASCADE; +DROP ROLE IF EXISTS dlskel_user; +RESET client_min_messages; +-- Quiet, so that the output does not depend on whether another test in the +-- same database created the extension first. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +RESET client_min_messages; +CREATE SERVER dlskel_acl_cat + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_acl_vol + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/acl', + endpoint 'http://fake:9000'); +-- Which NOTICE CREATE ROLE emits depends on gp_resource_manager, and neither +-- is what this case is about. +SET client_min_messages = warning; +CREATE ROLE dlskel_user LOGIN; +RESET client_min_messages; +CREATE SCHEMA dlskel_s; +GRANT CREATE, USAGE ON SCHEMA dlskel_s TO dlskel_user; +SET ROLE dlskel_user; +CREATE TABLE dlskel_s.t (a int) + USING iceberg + WITH (catalog = 'dlskel_acl_cat', volume = 'dlskel_acl_vol'); +ERROR: permission denied for foreign server dlskel_acl_cat +RESET ROLE; +GRANT USAGE ON FOREIGN SERVER dlskel_acl_cat TO dlskel_user; +SET ROLE dlskel_user; +CREATE TABLE dlskel_s.t (a int) + USING iceberg + WITH (catalog = 'dlskel_acl_cat', volume = 'dlskel_acl_vol'); +ERROR: permission denied for foreign server dlskel_acl_vol +RESET ROLE; +GRANT USAGE ON FOREIGN SERVER dlskel_acl_vol TO dlskel_user; +SET ROLE dlskel_user; +CREATE TABLE dlskel_s.t (a int) + USING iceberg + WITH (catalog = 'dlskel_acl_cat', volume = 'dlskel_acl_vol'); +NOTICE: stub engine: created iceberg table "dlskel_s.t" in catalog "dlskel_acl_cat" +CREATE USER MAPPING FOR dlskel_user + SERVER dlskel_acl_cat + OPTIONS (username 'u', auth_method 'simple'); +-- AWS temporary credentials are three values; the mapping has to be able to +-- hold all of them. +CREATE USER MAPPING FOR dlskel_user + SERVER dlskel_acl_vol + OPTIONS (access_key_id 'k', secret_access_key 's', session_token 't'); +-- A server-side key is not a user mapping key. +ALTER USER MAPPING FOR dlskel_user + SERVER dlskel_acl_cat + OPTIONS (ADD warehouse 'x'); +ERROR: invalid iceberg catalog user mapping option "warehouse" +RESET ROLE; +DROP SCHEMA dlskel_s CASCADE; +NOTICE: drop cascades to table dlskel_s.t +NOTICE: stub engine: dropped iceberg table "dlskel_s.t" from catalog "dlskel_acl_cat", keeping data +DROP USER MAPPING FOR dlskel_user SERVER dlskel_acl_cat; +DROP USER MAPPING FOR dlskel_user SERVER dlskel_acl_vol; +DROP SERVER dlskel_acl_cat; +DROP SERVER dlskel_acl_vol; +DROP ROLE dlskel_user; +SET client_min_messages = warning; +RESET ROLE; +DROP SCHEMA IF EXISTS dlskel_s CASCADE; +DROP SERVER IF EXISTS dlskel_acl_cat CASCADE; +DROP SERVER IF EXISTS dlskel_acl_vol CASCADE; +DROP ROLE IF EXISTS dlskel_user; +RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_ddl.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_ddl.out new file mode 100644 index 00000000000..eb61b4a08b0 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_ddl.out @@ -0,0 +1,227 @@ +-- Happy-path DDL, binding persistence, distributed catalog state, and drops. +SET client_min_messages = warning; +DROP TABLE IF EXISTS dlskel_t, dlskel_t2, dlskel_t3, dlskel_t_dump, + dlskel_t_keep, dlskel_t_purge CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat_rest CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +RESET client_min_messages; +-- Quiet, so that the output does not depend on whether another test in the +-- same database created the extension first. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +RESET client_min_messages; +CREATE SERVER dlskel_cat + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_cat_rest + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'rest', uri 'https://fake:443'); +DROP SERVER dlskel_cat_rest; +CREATE SERVER dlskel_vol + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/prefix', + endpoint 'http://fake:9000'); +CREATE TABLE dlskel_t (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +NOTICE: stub engine: created iceberg table "public.dlskel_t" in catalog "dlskel_cat" +SELECT relname, amname, reloptions +FROM pg_class c +JOIN pg_am a ON a.oid = c.relam +WHERE relname = 'dlskel_t'; + relname | amname | reloptions +----------+---------+---------------------------------------- + dlskel_t | iceberg | {catalog=dlskel_cat,volume=dlskel_vol} +(1 row) + +SELECT policytype, distkey +FROM gp_distribution_policy +WHERE localoid = 'dlskel_t'::regclass; + policytype | distkey +------------+--------- + p | +(1 row) + +-- Exactly one pg_class row on each primary segment. Counting rows in total +-- would accept one segment missing its row as long as another had two, which is +-- the very divergence this is here to catch; so compare the set of segments that +-- have exactly one row against the set of primaries. +SELECT count(*) = 0 AS every_segment_has_exactly_one +FROM (SELECT content FROM gp_segment_configuration + WHERE content >= 0 AND role = 'p' + EXCEPT + SELECT gp_segment_id FROM gp_dist_random('pg_class') + WHERE relname = 'dlskel_t' + GROUP BY gp_segment_id HAVING count(*) = 1) missing_or_duplicated; + every_segment_has_exactly_one +------------------------------- + t +(1 row) + +SELECT oid AS dlskel_cat_oid +FROM pg_foreign_server +WHERE srvname = 'dlskel_cat' +\gset +SELECT oid AS dlskel_vol_oid +FROM pg_foreign_server +WHERE srvname = 'dlskel_vol' +\gset +SELECT 'dlskel_t'::regclass::oid AS dlskel_t_oid +\gset +SELECT b.binding, + NOT EXISTS (SELECT content FROM gp_segment_configuration + WHERE content >= 0 AND role = 'p' + EXCEPT + SELECT gp_segment_id FROM gp_dist_random('pg_depend') + WHERE classid = 'pg_class'::regclass + AND objid = :dlskel_t_oid + AND refclassid = 'pg_foreign_server'::regclass + AND refobjid = b.refobjid + GROUP BY gp_segment_id HAVING count(*) = 1) + AS every_segment_has_exactly_one +FROM (VALUES ('catalog', :dlskel_cat_oid::oid), + ('volume', :dlskel_vol_oid::oid)) AS b(binding, refobjid) +ORDER BY b.binding; + binding | every_segment_has_exactly_one +---------+------------------------------- + catalog | t + volume | t +(2 rows) + +ANALYZE dlskel_t; +VACUUM dlskel_t; +SELECT reltuples IN (-1, 0) AS no_local_stats +FROM pg_class +WHERE oid = 'dlskel_t'::regclass; + no_local_stats +---------------- + t +(1 row) + +-- pg_dump writes DISTRIBUTED RANDOMLY into the CREATE TABLE it emits, so this +-- is the statement a restore replays; refusing it would mean refusing to +-- restore a dump this module produced. It has to yield the same policy as the +-- clause the module injects on its own. +CREATE TABLE dlskel_t_dump (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + DISTRIBUTED RANDOMLY; +NOTICE: stub engine: created iceberg table "public.dlskel_t_dump" in catalog "dlskel_cat" +SELECT policytype, distkey +FROM gp_distribution_policy +WHERE localoid = 'dlskel_t_dump'::regclass; + policytype | distkey +------------+--------- + p | +(1 row) + +DROP TABLE dlskel_t_dump; +NOTICE: stub engine: dropped iceberg table "public.dlskel_t_dump" from catalog "dlskel_cat", keeping data +-- Dropping the table drops this database's reference to it; the lake data stays +-- unless the table said otherwise. The default and the explicit form both have +-- to be observable, which is why the stub reports which one it was asked for. +CREATE TABLE dlskel_t_keep (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +NOTICE: stub engine: created iceberg table "public.dlskel_t_keep" in catalog "dlskel_cat" +DROP TABLE dlskel_t_keep; +NOTICE: stub engine: dropped iceberg table "public.dlskel_t_keep" from catalog "dlskel_cat", keeping data +CREATE TABLE dlskel_t_purge (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', + purge_on_drop = true); +NOTICE: stub engine: created iceberg table "public.dlskel_t_purge" in catalog "dlskel_cat" +SELECT reloptions FROM pg_class WHERE relname = 'dlskel_t_purge'; + reloptions +----------------------------------------------------------- + {catalog=dlskel_cat,volume=dlskel_vol,purge_on_drop=true} +(1 row) + +DROP TABLE dlskel_t_purge; +NOTICE: stub engine: dropped iceberg table "public.dlskel_t_purge" from catalog "dlskel_cat", purging data +SET iceberg.default_catalog = 'dlskel_cat'; +SET iceberg.default_volume = 'dlskel_vol'; +CREATE TABLE dlskel_t2 (a int) USING iceberg; +NOTICE: stub engine: created iceberg table "public.dlskel_t2" in catalog "dlskel_cat" +SELECT relname, amname, reloptions +FROM pg_class c +JOIN pg_am a ON a.oid = c.relam +WHERE relname = 'dlskel_t2'; + relname | amname | reloptions +-----------+---------+---------------------------------------- + dlskel_t2 | iceberg | {catalog=dlskel_cat,volume=dlskel_vol} +(1 row) + +RESET iceberg.default_catalog; +RESET iceberg.default_volume; +SET default_table_access_method = iceberg; +SET iceberg.default_catalog = 'dlskel_cat'; +SET iceberg.default_volume = 'dlskel_vol'; +CREATE TABLE dlskel_t3 (a int); +NOTICE: stub engine: created iceberg table "public.dlskel_t3" in catalog "dlskel_cat" +\set HIDE_TABLEAM off +\d+ dlskel_t3 + Table "public.dlskel_t3" + Column | Type | Collation | Nullable | Default | Storage | Stats target | Description +--------+---------+-----------+----------+---------+---------+--------------+------------- + a | integer | | | | plain | | +Distributed randomly +Access method: iceberg +Options: catalog=dlskel_cat, volume=dlskel_vol + +RESET default_table_access_method; +RESET iceberg.default_catalog; +RESET iceberg.default_volume; +DROP SERVER dlskel_cat; +ERROR: cannot drop server dlskel_cat because other objects depend on it +DETAIL: table dlskel_t depends on server dlskel_cat +table dlskel_t2 depends on server dlskel_cat +table dlskel_t3 depends on server dlskel_cat +HINT: Use DROP ... CASCADE to drop the dependent objects too. +DROP SERVER dlskel_vol; +ERROR: cannot drop server dlskel_vol because other objects depend on it +DETAIL: table dlskel_t depends on server dlskel_vol +table dlskel_t2 depends on server dlskel_vol +table dlskel_t3 depends on server dlskel_vol +HINT: Use DROP ... CASCADE to drop the dependent objects too. +DROP TABLE dlskel_t; +NOTICE: stub engine: dropped iceberg table "public.dlskel_t" from catalog "dlskel_cat", keeping data +SELECT b.binding, + NOT EXISTS (SELECT 1 FROM gp_dist_random('pg_depend') + WHERE classid = 'pg_class'::regclass + AND objid = :dlskel_t_oid + AND refclassid = 'pg_foreign_server'::regclass + AND refobjid = b.refobjid) + AS gone_from_every_segment +FROM (VALUES ('catalog', :dlskel_cat_oid::oid), + ('volume', :dlskel_vol_oid::oid)) AS b(binding, refobjid) +ORDER BY b.binding; + binding | gone_from_every_segment +---------+------------------------- + catalog | t + volume | t +(2 rows) + +DROP SERVER dlskel_cat CASCADE; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table dlskel_t2 +drop cascades to table dlskel_t3 +NOTICE: stub engine: dropped iceberg table "public.dlskel_t3" from catalog "dlskel_cat", keeping data +NOTICE: stub engine: dropped iceberg table "public.dlskel_t2" from catalog "dlskel_cat", keeping data +DROP SERVER dlskel_vol CASCADE; +SELECT gp_segment_id, relname +FROM gp_dist_random('pg_class') +WHERE relname LIKE 'dlskel\_%' ESCAPE '\' +ORDER BY 1, 2; + gp_segment_id | relname +---------------+--------- +(0 rows) + +SET client_min_messages = warning; +DROP TABLE IF EXISTS dlskel_t, dlskel_t2, dlskel_t3, dlskel_t_dump, + dlskel_t_keep, dlskel_t_purge CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat_rest CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out new file mode 100644 index 00000000000..db5fde07cf4 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/expected/iceberg_am_reject.out @@ -0,0 +1,457 @@ +-- Unsupported data paths, CREATE/ALTER guards, and binding validators. +-- Errors raised on a segment carry its address and pid, which vary per run. +-- start_matchsubs +-- m/ \(seg[0-9]+[^)]* pid=[0-9]+\)/ +-- s/ \(seg[0-9]+[^)]* pid=[0-9]+\)// +-- end_matchsubs +SET client_min_messages = warning; +DROP MATERIALIZED VIEW IF EXISTS dlskel_mv CASCADE; +DROP TABLE IF EXISTS + dlskel_r, dlskel_r2, dlskel_bad, dlskel_bad_dist, + dlskel_bad_part, dlskel_bad_inherits, dlskel_bad_typed, + dlskel_bad_temp, dlskel_bad_unlogged, dlskel_bad_oncommit, + dlskel_bad_tablespace, dlskel_bad_missing_server, + dlskel_bad_wrong_catalog, dlskel_bad_no_catalog, + dlskel_bad_reloption, dlskel_bad_engine, dlskel_r_new, dlskel_heap, + dlskel_bad_repl, dlskel_bad_purge + CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat2 CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +DROP SERVER IF EXISTS dlskel_free CASCADE; +DROP SERVER IF EXISTS dlskel_free2 CASCADE; +DROP SERVER IF EXISTS dlskel_bad_server CASCADE; +DROP SERVER IF EXISTS dlskel_bad_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_user CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_keytab CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_clientid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_keyid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_case CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_type CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_hadoop CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_builtin CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_nourl CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_realm CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_empty CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_bool CASCADE; +DROP SERVER IF EXISTS dlskel_bad_nopath CASCADE; +DROP SERVER IF EXISTS dlskel_bad_scheme CASCADE; +DROP SERVER IF EXISTS dlskel_bad_authority CASCADE; +DROP SERVER IF EXISTS dlskel_bad_query CASCADE; +DROP SERVER IF EXISTS dlskel_bad_userinfo CASCADE; +DROP SERVER IF EXISTS dlskel_bad_bucket CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch2 CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain2 CASCADE; +DROP TYPE IF EXISTS dlskel_type CASCADE; +DROP ROLE IF EXISTS dlskel_role; +RESET client_min_messages; +-- Quiet, so that the output does not depend on whether another test in the +-- same database created the extension first. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +RESET client_min_messages; +CREATE SERVER dlskel_cat + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_vol + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/reject', + endpoint 'http://fake:9000'); +CREATE SERVER dlskel_free + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://free:9083'); +-- Which NOTICE CREATE ROLE emits depends on gp_resource_manager, and neither +-- is what this case is about. +SET client_min_messages = warning; +CREATE ROLE dlskel_role; +RESET client_min_messages; +CREATE TYPE dlskel_type AS (a int); +CREATE TABLE dlskel_r (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +NOTICE: stub engine: created iceberg table "public.dlskel_r" in catalog "dlskel_cat" +SELECT * FROM dlskel_r; +ERROR: iceberg: SELECT is not supported yet (seg0 slice1 172.17.0.2:40000 pid=92001) +INSERT INTO dlskel_r VALUES (1, 'x'); +ERROR: iceberg: INSERT is not supported yet (seg1 172.17.0.2:40001 pid=92002) +UPDATE dlskel_r SET a = 1; +ERROR: iceberg: SELECT is not supported yet (seg0 172.17.0.2:40000 pid=92001) +DELETE FROM dlskel_r; +ERROR: iceberg: SELECT is not supported yet (seg0 172.17.0.2:40000 pid=92001) +COPY dlskel_r FROM stdin; +ERROR: iceberg: INSERT is not supported yet +CONTEXT: COPY dlskel_r, line 1 +COPY dlskel_r TO stdout; +ERROR: iceberg: SELECT is not supported yet +CREATE INDEX ON dlskel_r (a); +ERROR: iceberg: CREATE INDEX is not supported yet +SELECT * FROM dlskel_r TABLESAMPLE BERNOULLI (10); +ERROR: iceberg: SELECT is not supported yet (seg0 slice1 172.17.0.2:40000 pid=92001) +-- A table from an earlier transaction takes the new-filelocator path rather +-- than the access method's truncate callback, so this is the case that would +-- silently report success if only the callback rejected it. +TRUNCATE dlskel_r; +ERROR: iceberg: TRUNCATE is not supported yet +-- Same for a table created in this transaction, which does reach the callback. +BEGIN; +CREATE TABLE dlskel_r_new (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +NOTICE: stub engine: created iceberg table "public.dlskel_r_new" in catalog "dlskel_cat" +TRUNCATE dlskel_r_new; +ERROR: iceberg: TRUNCATE is not supported yet +ROLLBACK; +-- A multi-table TRUNCATE must refuse before truncating the heap beside it, so +-- the row below has to survive the attempt. +CREATE TABLE dlskel_heap (a int) DISTRIBUTED BY (a); +INSERT INTO dlskel_heap VALUES (1); +TRUNCATE dlskel_heap, dlskel_r; +ERROR: iceberg: TRUNCATE is not supported yet +SELECT count(*) AS heap_rows_kept FROM dlskel_heap; + heap_rows_kept +---------------- + 1 +(1 row) + +VACUUM FULL dlskel_r; +ERROR: iceberg: VACUUM FULL on iceberg tables is not supported yet +SELECT * FROM dlskel_r FOR UPDATE; +ERROR: iceberg: SELECT is not supported yet (seg0 slice1 172.17.0.2:40000 pid=92001) +CREATE TABLE dlskel_bad_dist (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + DISTRIBUTED BY (a); +ERROR: iceberg: DISTRIBUTED BY is not supported yet +CREATE TABLE dlskel_bad_part (a int) + PARTITION BY RANGE (a) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: partitioned tables is not supported yet +CREATE TABLE dlskel_bad_inherits (b text) + INHERITS (dlskel_r) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: INHERITS is not supported yet +CREATE TABLE dlskel_bad_typed OF dlskel_type + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: typed tables (OF type) is not supported yet +CREATE TEMP TABLE dlskel_bad_temp (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: TEMP tables is not supported yet +CREATE UNLOGGED TABLE dlskel_bad_unlogged (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +ERROR: iceberg: UNLOGGED tables is not supported yet +BEGIN; +CREATE TABLE dlskel_bad_oncommit (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + ON COMMIT DROP; +ERROR: iceberg: ON COMMIT is not supported yet +ROLLBACK; +CREATE TABLE dlskel_bad_tablespace (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + TABLESPACE pg_default; +ERROR: iceberg: TABLESPACE is not supported yet +CREATE TABLE dlskel_bad USING iceberg AS SELECT 1; +ERROR: iceberg: CREATE TABLE AS / CREATE MATERIALIZED VIEW is not supported yet +CREATE MATERIALIZED VIEW dlskel_mv USING iceberg AS SELECT 1; +ERROR: iceberg: CREATE TABLE AS / CREATE MATERIALIZED VIEW is not supported yet +-- Converting a heap into a lake table has to be refused too: the relation is +-- still a heap when the statement arrives, so the guard above does not see it. +ALTER TABLE dlskel_heap SET ACCESS METHOD iceberg; +ERROR: iceberg: ALTER TABLE ... SET ACCESS METHOD iceberg is not supported yet +SET iceberg.default_catalog = 'dlskel_cat'; +SET iceberg.default_volume = 'dlskel_vol'; +ALTER TABLE dlskel_heap SET ACCESS METHOD iceberg; +ERROR: iceberg: ALTER TABLE ... SET ACCESS METHOD iceberg is not supported yet +RESET iceberg.default_catalog; +RESET iceberg.default_volume; +-- A column-bearing clause still cannot be honoured, and neither can a +-- replicated policy. +CREATE TABLE dlskel_bad_repl (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + DISTRIBUTED REPLICATED; +ERROR: iceberg: DISTRIBUTED REPLICATED is not supported yet +-- Renaming a schema would repoint its lake tables at a different external +-- namespace, so a schema holding one is refused -- and, just as importantly, a +-- schema holding none is not: the guard has to be no wider than the problem. +CREATE SCHEMA dlskel_sch; +CREATE TABLE dlskel_sch.t (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +NOTICE: stub engine: created iceberg table "dlskel_sch.t" in catalog "dlskel_cat" +BEGIN; +ALTER SCHEMA dlskel_sch RENAME TO dlskel_sch2; +ERROR: iceberg: RENAME on schemas containing iceberg tables is not supported yet +ROLLBACK; +CREATE SCHEMA dlskel_plain; +ALTER SCHEMA dlskel_plain RENAME TO dlskel_plain2; +DROP SCHEMA dlskel_plain2; +-- Renaming a wrapper breaks every mapping lookup at once, including the one +-- DROP needs, so it is refused whether or not a table exists yet. +-- Each attempt is rolled back: if the guard ever regresses, an accepted rename +-- would leave the extension's own wrapper under a different name, which nothing +-- in the cleanup below can undo and which breaks every later run. +BEGIN; +ALTER FOREIGN DATA WRAPPER iceberg_catalog_fdw RENAME TO dlskel_other_fdw; +ERROR: iceberg: RENAME on the iceberg foreign-data wrappers is not supported yet +ROLLBACK; +BEGIN; +ALTER FOREIGN DATA WRAPPER iceberg_volume_fdw RENAME TO dlskel_other_fdw; +ERROR: iceberg: RENAME on the iceberg foreign-data wrappers is not supported yet +ROLLBACK; +ALTER TABLE dlskel_r ADD COLUMN c int; +ERROR: iceberg: ALTER TABLE on iceberg tables is not supported yet +ALTER TABLE dlskel_r SET (fillfactor = 90); +ERROR: iceberg: ALTER TABLE on iceberg tables is not supported yet +ALTER TABLE dlskel_r SET ACCESS METHOD heap; +ERROR: iceberg: ALTER TABLE on iceberg tables is not supported yet +ALTER TABLE dlskel_r SET DISTRIBUTED BY (a); +ERROR: iceberg: ALTER TABLE on iceberg tables is not supported yet +ALTER TABLE dlskel_r RENAME TO dlskel_r2; +ERROR: iceberg: RENAME on iceberg tables is not supported yet +ALTER TABLE dlskel_r RENAME COLUMN a TO aa; +ERROR: iceberg: RENAME on iceberg tables is not supported yet +ALTER TABLE dlskel_r SET SCHEMA public; +ERROR: iceberg: SET SCHEMA on iceberg tables is not supported yet +-- OWNER TO is the one ALTER TABLE form that goes through: pg_dump writes it for +-- every table, and ownership cannot reach the external table. Put it back +-- afterwards so the rest of the file still owns what it created. +ALTER TABLE dlskel_r OWNER TO dlskel_role; +SELECT relname, pg_get_userbyid(relowner) AS owner +FROM pg_class WHERE relname = 'dlskel_r'; + relname | owner +----------+------------- + dlskel_r | dlskel_role +(1 row) + +ALTER TABLE dlskel_r OWNER TO CURRENT_USER; +ALTER SERVER dlskel_cat + OPTIONS (SET uri 'thrift://other:9083'); +ERROR: iceberg: ALTER SERVER on servers referenced by iceberg tables is not supported yet +ALTER SERVER dlskel_cat VERSION '2'; +ERROR: iceberg: ALTER SERVER on servers referenced by iceberg tables is not supported yet +ALTER SERVER dlskel_cat RENAME TO dlskel_cat2; +ERROR: iceberg: RENAME on servers referenced by iceberg tables is not supported yet +ALTER SERVER dlskel_cat OWNER TO dlskel_role; +ERROR: iceberg: ALTER OWNER on servers referenced by iceberg tables is not supported yet +ALTER SERVER dlskel_free + OPTIONS (SET uri 'thrift://other:9083'); +ALTER SERVER dlskel_free VERSION '2'; +ALTER SERVER dlskel_free OWNER TO dlskel_role; +ALTER SERVER dlskel_free RENAME TO dlskel_free2; +RESET iceberg.default_catalog; +RESET iceberg.default_volume; +CREATE TABLE dlskel_bad_missing_server (a int) + USING iceberg + WITH (catalog = 'dlskel_missing', volume = 'dlskel_vol'); +ERROR: server "dlskel_missing" does not exist +CREATE TABLE dlskel_bad_wrong_catalog (a int) + USING iceberg + WITH (catalog = 'dlskel_vol', volume = 'dlskel_vol'); +ERROR: server "dlskel_vol" is not an iceberg catalog server +CREATE TABLE dlskel_bad_no_catalog (a int) + USING iceberg + WITH (volume = 'dlskel_vol'); +ERROR: no catalog specified +HINT: Specify WITH (catalog = '...') or SET iceberg.default_catalog. +CREATE TABLE dlskel_bad_purge (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', + purge_on_drop = 'perhaps'); +ERROR: invalid value for boolean option "purge_on_drop": perhaps +CREATE TABLE dlskel_bad_reloption (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', nonsense = 'x'); +ERROR: unrecognized parameter "nonsense" +CREATE SERVER dlskel_bad_server + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', nonsense 'x'); +ERROR: invalid iceberg catalog server option "nonsense" +HINT: Allowed options are "type", "uri", "catalog_name", "warehouse" and "polaris_server_realm". +CREATE SERVER dlskel_bad_secret + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (secret_key 'x'); +ERROR: credential option "secret_key" is not allowed on an iceberg volume server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +-- The catalog wrapper has its own allowlist, and its own credential keys. +CREATE SERVER dlskel_bad_cat_secret + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', password 'x'); +ERROR: credential option "password" is not allowed on an iceberg catalog server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +CREATE SERVER dlskel_bad_cat_token + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'polaris', uri 'https://fake:443', client_secret 'x'); +ERROR: credential option "client_secret" is not allowed on an iceberg catalog server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +-- Every key the credential list mirrors from an option module, so that a key +-- renamed in one place and not the other fails here instead of silently +-- ceasing to be caught. +CREATE SERVER dlskel_bad_cat_user + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', username 'u'); +ERROR: credential option "username" is not allowed on an iceberg catalog server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +CREATE SERVER dlskel_bad_cat_keytab + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', krb_client_keytab '/k'); +ERROR: credential option "krb_client_keytab" is not allowed on an iceberg catalog server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +CREATE SERVER dlskel_bad_cat_clientid + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'rest', uri 'https://fake:443', client_id 'i'); +ERROR: credential option "client_id" is not allowed on an iceberg catalog server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +CREATE SERVER dlskel_bad_vol_token + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', session_token 't'); +ERROR: credential option "session_token" is not allowed on an iceberg volume server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +CREATE SERVER dlskel_bad_vol_keyid + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', access_key_id 'k'); +ERROR: credential option "access_key_id" is not allowed on an iceberg volume server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +CREATE SERVER dlskel_bad_vol_secret + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', secret_access_key 's'); +ERROR: credential option "secret_access_key" is not allowed on an iceberg volume server +HINT: credentials belong in CREATE USER MAPPING ... OPTIONS (...) +-- Option names are matched exactly, as the server itself matches them; a quoted +-- variant is a different, unknown option rather than a second spelling. +CREATE SERVER dlskel_bad_cat_case + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS ("TYPE" 'hive', uri 'thrift://fake:9083'); +ERROR: invalid iceberg catalog server option "TYPE" +HINT: Allowed options are "type", "uri", "catalog_name", "warehouse" and "polaris_server_realm". +-- A catalog type outside the vocabulary, and one that is in it but has no +-- implementation behind it yet. +CREATE SERVER dlskel_bad_cat_type + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'nosuchcatalog', uri 'thrift://fake:9083'); +ERROR: invalid iceberg catalog type "nosuchcatalog" +HINT: Allowed types are "hive", "rest" and "builtin"; "polaris" is accepted as an alias of "rest". +CREATE SERVER dlskel_bad_cat_hadoop + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hadoop', uri 'hdfs://fake:8020'); +ERROR: iceberg: catalog type "hadoop" is not supported yet +-- builtin needs no url; supplying one means the two disagree. +CREATE SERVER dlskel_bad_cat_builtin + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'builtin', uri 'thrift://fake:9083'); +ERROR: iceberg catalog type "builtin" forbids server option "uri" +-- hive does need one. +CREATE SERVER dlskel_bad_cat_nourl + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive'); +ERROR: iceberg catalog type "hive" requires server option "uri" +-- The realm applies to one catalog type only. +CREATE SERVER dlskel_bad_cat_realm + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', + polaris_server_realm 'OTHER'); +ERROR: iceberg catalog server option "polaris_server_realm" applies only to catalog type "rest" +-- An empty value is refused where it is written, not where it is first read. +CREATE SERVER dlskel_bad_cat_empty + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri ''); +ERROR: iceberg catalog server option "uri" cannot be empty +-- Not a boolean. +CREATE SERVER dlskel_bad_vol_bool + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', path_style_access 'perhaps'); +ERROR: invalid boolean value "perhaps" for option "path_style_access" +CREATE SERVER dlskel_bad_nopath + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (endpoint 'http://fake:9000'); +ERROR: iceberg volume server option "base_path" is required +CREATE SERVER dlskel_bad_scheme + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 'ftp://x/y'); +ERROR: invalid iceberg volume base_path "ftp://x/y" +DETAIL: location URI "ftp://x/y" has unsupported scheme; expected s3 or hdfs +CREATE SERVER dlskel_bad_authority + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://'); +ERROR: invalid iceberg volume base_path "s3://" +DETAIL: location URI "s3://" has an empty authority +CREATE SERVER dlskel_bad_query + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://b/p?versionId=3'); +ERROR: invalid iceberg volume base_path "s3://b/p?versionId=3" +DETAIL: location URI "s3://b/p?versionId=3" must not contain a query +CREATE SERVER dlskel_bad_userinfo + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://user@b/p'); +ERROR: invalid iceberg volume base_path "s3://user@b/p" +DETAIL: location URI "s3://user@b/p" must not contain userinfo +CREATE SERVER dlskel_bad_bucket + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://UPPER_case/p'); +ERROR: invalid iceberg volume base_path "s3://UPPER_case/p" +DETAIL: s3 bucket in location URI "s3://UPPER_case/p" must start and end with a lowercase letter or digit +-- The metadata engine is not selectable, so naming one is an unknown option. +CREATE TABLE dlskel_bad_engine (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', + engine = 'agent'); +ERROR: unrecognized parameter "engine" +SET client_min_messages = warning; +DROP MATERIALIZED VIEW IF EXISTS dlskel_mv CASCADE; +DROP TABLE IF EXISTS + dlskel_r, dlskel_r2, dlskel_bad, dlskel_bad_dist, + dlskel_bad_part, dlskel_bad_inherits, dlskel_bad_typed, + dlskel_bad_temp, dlskel_bad_unlogged, dlskel_bad_oncommit, + dlskel_bad_tablespace, dlskel_bad_missing_server, + dlskel_bad_wrong_catalog, dlskel_bad_no_catalog, + dlskel_bad_reloption, dlskel_bad_engine, dlskel_r_new, dlskel_heap, + dlskel_bad_repl, dlskel_bad_purge + CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat2 CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +DROP SERVER IF EXISTS dlskel_free CASCADE; +DROP SERVER IF EXISTS dlskel_free2 CASCADE; +DROP SERVER IF EXISTS dlskel_bad_server CASCADE; +DROP SERVER IF EXISTS dlskel_bad_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_user CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_keytab CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_clientid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_keyid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_case CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_type CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_hadoop CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_builtin CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_nourl CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_realm CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_empty CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_bool CASCADE; +DROP SERVER IF EXISTS dlskel_bad_nopath CASCADE; +DROP SERVER IF EXISTS dlskel_bad_scheme CASCADE; +DROP SERVER IF EXISTS dlskel_bad_authority CASCADE; +DROP SERVER IF EXISTS dlskel_bad_query CASCADE; +DROP SERVER IF EXISTS dlskel_bad_userinfo CASCADE; +DROP SERVER IF EXISTS dlskel_bad_bucket CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch2 CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain2 CASCADE; +DROP TYPE IF EXISTS dlskel_type CASCADE; +DROP ROLE IF EXISTS dlskel_role; +RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_acl.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_acl.sql new file mode 100644 index 00000000000..40d2020a0ce --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_acl.sql @@ -0,0 +1,78 @@ +-- Foreign-server USAGE is required; user mappings remain optional. + +SET client_min_messages = warning; +RESET ROLE; +DROP SCHEMA IF EXISTS dlskel_s CASCADE; +DROP SERVER IF EXISTS dlskel_acl_cat CASCADE; +DROP SERVER IF EXISTS dlskel_acl_vol CASCADE; +DROP ROLE IF EXISTS dlskel_user; +RESET client_min_messages; + +-- Quiet, so that the output does not depend on whether another test in the +-- same database created the extension first. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +RESET client_min_messages; + +CREATE SERVER dlskel_acl_cat + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_acl_vol + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/acl', + endpoint 'http://fake:9000'); +-- Which NOTICE CREATE ROLE emits depends on gp_resource_manager, and neither +-- is what this case is about. +SET client_min_messages = warning; +CREATE ROLE dlskel_user LOGIN; +RESET client_min_messages; +CREATE SCHEMA dlskel_s; +GRANT CREATE, USAGE ON SCHEMA dlskel_s TO dlskel_user; + +SET ROLE dlskel_user; +CREATE TABLE dlskel_s.t (a int) + USING iceberg + WITH (catalog = 'dlskel_acl_cat', volume = 'dlskel_acl_vol'); + +RESET ROLE; +GRANT USAGE ON FOREIGN SERVER dlskel_acl_cat TO dlskel_user; +SET ROLE dlskel_user; +CREATE TABLE dlskel_s.t (a int) + USING iceberg + WITH (catalog = 'dlskel_acl_cat', volume = 'dlskel_acl_vol'); + +RESET ROLE; +GRANT USAGE ON FOREIGN SERVER dlskel_acl_vol TO dlskel_user; +SET ROLE dlskel_user; +CREATE TABLE dlskel_s.t (a int) + USING iceberg + WITH (catalog = 'dlskel_acl_cat', volume = 'dlskel_acl_vol'); + +CREATE USER MAPPING FOR dlskel_user + SERVER dlskel_acl_cat + OPTIONS (username 'u', auth_method 'simple'); +-- AWS temporary credentials are three values; the mapping has to be able to +-- hold all of them. +CREATE USER MAPPING FOR dlskel_user + SERVER dlskel_acl_vol + OPTIONS (access_key_id 'k', secret_access_key 's', session_token 't'); +-- A server-side key is not a user mapping key. +ALTER USER MAPPING FOR dlskel_user + SERVER dlskel_acl_cat + OPTIONS (ADD warehouse 'x'); + +RESET ROLE; +DROP SCHEMA dlskel_s CASCADE; +DROP USER MAPPING FOR dlskel_user SERVER dlskel_acl_cat; +DROP USER MAPPING FOR dlskel_user SERVER dlskel_acl_vol; +DROP SERVER dlskel_acl_cat; +DROP SERVER dlskel_acl_vol; +DROP ROLE dlskel_user; + +SET client_min_messages = warning; +RESET ROLE; +DROP SCHEMA IF EXISTS dlskel_s CASCADE; +DROP SERVER IF EXISTS dlskel_acl_cat CASCADE; +DROP SERVER IF EXISTS dlskel_acl_vol CASCADE; +DROP ROLE IF EXISTS dlskel_user; +RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_ddl.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_ddl.sql new file mode 100644 index 00000000000..365fe7e0b00 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_ddl.sql @@ -0,0 +1,162 @@ +-- Happy-path DDL, binding persistence, distributed catalog state, and drops. + +SET client_min_messages = warning; +DROP TABLE IF EXISTS dlskel_t, dlskel_t2, dlskel_t3, dlskel_t_dump, + dlskel_t_keep, dlskel_t_purge CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat_rest CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +RESET client_min_messages; + +-- Quiet, so that the output does not depend on whether another test in the +-- same database created the extension first. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +RESET client_min_messages; + +CREATE SERVER dlskel_cat + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_cat_rest + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'rest', uri 'https://fake:443'); +DROP SERVER dlskel_cat_rest; +CREATE SERVER dlskel_vol + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/prefix', + endpoint 'http://fake:9000'); + +CREATE TABLE dlskel_t (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); + +SELECT relname, amname, reloptions +FROM pg_class c +JOIN pg_am a ON a.oid = c.relam +WHERE relname = 'dlskel_t'; + +SELECT policytype, distkey +FROM gp_distribution_policy +WHERE localoid = 'dlskel_t'::regclass; + +-- Exactly one pg_class row on each primary segment. Counting rows in total +-- would accept one segment missing its row as long as another had two, which is +-- the very divergence this is here to catch; so compare the set of segments that +-- have exactly one row against the set of primaries. +SELECT count(*) = 0 AS every_segment_has_exactly_one +FROM (SELECT content FROM gp_segment_configuration + WHERE content >= 0 AND role = 'p' + EXCEPT + SELECT gp_segment_id FROM gp_dist_random('pg_class') + WHERE relname = 'dlskel_t' + GROUP BY gp_segment_id HAVING count(*) = 1) missing_or_duplicated; + +SELECT oid AS dlskel_cat_oid +FROM pg_foreign_server +WHERE srvname = 'dlskel_cat' +\gset +SELECT oid AS dlskel_vol_oid +FROM pg_foreign_server +WHERE srvname = 'dlskel_vol' +\gset +SELECT 'dlskel_t'::regclass::oid AS dlskel_t_oid +\gset + +SELECT b.binding, + NOT EXISTS (SELECT content FROM gp_segment_configuration + WHERE content >= 0 AND role = 'p' + EXCEPT + SELECT gp_segment_id FROM gp_dist_random('pg_depend') + WHERE classid = 'pg_class'::regclass + AND objid = :dlskel_t_oid + AND refclassid = 'pg_foreign_server'::regclass + AND refobjid = b.refobjid + GROUP BY gp_segment_id HAVING count(*) = 1) + AS every_segment_has_exactly_one +FROM (VALUES ('catalog', :dlskel_cat_oid::oid), + ('volume', :dlskel_vol_oid::oid)) AS b(binding, refobjid) +ORDER BY b.binding; + +ANALYZE dlskel_t; +VACUUM dlskel_t; +SELECT reltuples IN (-1, 0) AS no_local_stats +FROM pg_class +WHERE oid = 'dlskel_t'::regclass; + +-- pg_dump writes DISTRIBUTED RANDOMLY into the CREATE TABLE it emits, so this +-- is the statement a restore replays; refusing it would mean refusing to +-- restore a dump this module produced. It has to yield the same policy as the +-- clause the module injects on its own. +CREATE TABLE dlskel_t_dump (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + DISTRIBUTED RANDOMLY; +SELECT policytype, distkey +FROM gp_distribution_policy +WHERE localoid = 'dlskel_t_dump'::regclass; +DROP TABLE dlskel_t_dump; + +-- Dropping the table drops this database's reference to it; the lake data stays +-- unless the table said otherwise. The default and the explicit form both have +-- to be observable, which is why the stub reports which one it was asked for. +CREATE TABLE dlskel_t_keep (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +DROP TABLE dlskel_t_keep; +CREATE TABLE dlskel_t_purge (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', + purge_on_drop = true); +SELECT reloptions FROM pg_class WHERE relname = 'dlskel_t_purge'; +DROP TABLE dlskel_t_purge; + +SET iceberg.default_catalog = 'dlskel_cat'; +SET iceberg.default_volume = 'dlskel_vol'; +CREATE TABLE dlskel_t2 (a int) USING iceberg; +SELECT relname, amname, reloptions +FROM pg_class c +JOIN pg_am a ON a.oid = c.relam +WHERE relname = 'dlskel_t2'; +RESET iceberg.default_catalog; +RESET iceberg.default_volume; + +SET default_table_access_method = iceberg; +SET iceberg.default_catalog = 'dlskel_cat'; +SET iceberg.default_volume = 'dlskel_vol'; +CREATE TABLE dlskel_t3 (a int); +\set HIDE_TABLEAM off +\d+ dlskel_t3 +RESET default_table_access_method; +RESET iceberg.default_catalog; +RESET iceberg.default_volume; + +DROP SERVER dlskel_cat; +DROP SERVER dlskel_vol; + +DROP TABLE dlskel_t; +SELECT b.binding, + NOT EXISTS (SELECT 1 FROM gp_dist_random('pg_depend') + WHERE classid = 'pg_class'::regclass + AND objid = :dlskel_t_oid + AND refclassid = 'pg_foreign_server'::regclass + AND refobjid = b.refobjid) + AS gone_from_every_segment +FROM (VALUES ('catalog', :dlskel_cat_oid::oid), + ('volume', :dlskel_vol_oid::oid)) AS b(binding, refobjid) +ORDER BY b.binding; + +DROP SERVER dlskel_cat CASCADE; +DROP SERVER dlskel_vol CASCADE; + +SELECT gp_segment_id, relname +FROM gp_dist_random('pg_class') +WHERE relname LIKE 'dlskel\_%' ESCAPE '\' +ORDER BY 1, 2; + +SET client_min_messages = warning; +DROP TABLE IF EXISTS dlskel_t, dlskel_t2, dlskel_t3, dlskel_t_dump, + dlskel_t_keep, dlskel_t_purge CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat_rest CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +RESET client_min_messages; diff --git a/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_reject.sql b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_reject.sql new file mode 100644 index 00000000000..6f376f61e49 --- /dev/null +++ b/contrib/datalake_fdw/test/automation/sqlrepo/smoke/iceberg_am/sql/iceberg_am_reject.sql @@ -0,0 +1,379 @@ +-- Unsupported data paths, CREATE/ALTER guards, and binding validators. +-- Errors raised on a segment carry its address and pid, which vary per run. +-- start_matchsubs +-- m/ \(seg[0-9]+[^)]* pid=[0-9]+\)/ +-- s/ \(seg[0-9]+[^)]* pid=[0-9]+\)// +-- end_matchsubs + +SET client_min_messages = warning; +DROP MATERIALIZED VIEW IF EXISTS dlskel_mv CASCADE; +DROP TABLE IF EXISTS + dlskel_r, dlskel_r2, dlskel_bad, dlskel_bad_dist, + dlskel_bad_part, dlskel_bad_inherits, dlskel_bad_typed, + dlskel_bad_temp, dlskel_bad_unlogged, dlskel_bad_oncommit, + dlskel_bad_tablespace, dlskel_bad_missing_server, + dlskel_bad_wrong_catalog, dlskel_bad_no_catalog, + dlskel_bad_reloption, dlskel_bad_engine, dlskel_r_new, dlskel_heap, + dlskel_bad_repl, dlskel_bad_purge + CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat2 CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +DROP SERVER IF EXISTS dlskel_free CASCADE; +DROP SERVER IF EXISTS dlskel_free2 CASCADE; +DROP SERVER IF EXISTS dlskel_bad_server CASCADE; +DROP SERVER IF EXISTS dlskel_bad_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_user CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_keytab CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_clientid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_keyid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_case CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_type CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_hadoop CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_builtin CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_nourl CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_realm CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_empty CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_bool CASCADE; +DROP SERVER IF EXISTS dlskel_bad_nopath CASCADE; +DROP SERVER IF EXISTS dlskel_bad_scheme CASCADE; +DROP SERVER IF EXISTS dlskel_bad_authority CASCADE; +DROP SERVER IF EXISTS dlskel_bad_query CASCADE; +DROP SERVER IF EXISTS dlskel_bad_userinfo CASCADE; +DROP SERVER IF EXISTS dlskel_bad_bucket CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch2 CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain2 CASCADE; +DROP TYPE IF EXISTS dlskel_type CASCADE; +DROP ROLE IF EXISTS dlskel_role; +RESET client_min_messages; + +-- Quiet, so that the output does not depend on whether another test in the +-- same database created the extension first. +SET client_min_messages = warning; +CREATE EXTENSION IF NOT EXISTS datalake_fdw; +RESET client_min_messages; + +CREATE SERVER dlskel_cat + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_vol + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/reject', + endpoint 'http://fake:9000'); +CREATE SERVER dlskel_free + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://free:9083'); +-- Which NOTICE CREATE ROLE emits depends on gp_resource_manager, and neither +-- is what this case is about. +SET client_min_messages = warning; +CREATE ROLE dlskel_role; +RESET client_min_messages; +CREATE TYPE dlskel_type AS (a int); + +CREATE TABLE dlskel_r (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); + +SELECT * FROM dlskel_r; +INSERT INTO dlskel_r VALUES (1, 'x'); +UPDATE dlskel_r SET a = 1; +DELETE FROM dlskel_r; +COPY dlskel_r FROM stdin; +1 x +\. +COPY dlskel_r TO stdout; +CREATE INDEX ON dlskel_r (a); +SELECT * FROM dlskel_r TABLESAMPLE BERNOULLI (10); + +-- A table from an earlier transaction takes the new-filelocator path rather +-- than the access method's truncate callback, so this is the case that would +-- silently report success if only the callback rejected it. +TRUNCATE dlskel_r; + +-- Same for a table created in this transaction, which does reach the callback. +BEGIN; +CREATE TABLE dlskel_r_new (a int, b text) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +TRUNCATE dlskel_r_new; +ROLLBACK; + +-- A multi-table TRUNCATE must refuse before truncating the heap beside it, so +-- the row below has to survive the attempt. +CREATE TABLE dlskel_heap (a int) DISTRIBUTED BY (a); +INSERT INTO dlskel_heap VALUES (1); +TRUNCATE dlskel_heap, dlskel_r; +SELECT count(*) AS heap_rows_kept FROM dlskel_heap; + +VACUUM FULL dlskel_r; +SELECT * FROM dlskel_r FOR UPDATE; + +CREATE TABLE dlskel_bad_dist (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + DISTRIBUTED BY (a); +CREATE TABLE dlskel_bad_part (a int) + PARTITION BY RANGE (a) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +CREATE TABLE dlskel_bad_inherits (b text) + INHERITS (dlskel_r) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +CREATE TABLE dlskel_bad_typed OF dlskel_type + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +CREATE TEMP TABLE dlskel_bad_temp (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +CREATE UNLOGGED TABLE dlskel_bad_unlogged (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +BEGIN; +CREATE TABLE dlskel_bad_oncommit (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + ON COMMIT DROP; +ROLLBACK; +CREATE TABLE dlskel_bad_tablespace (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + TABLESPACE pg_default; + +CREATE TABLE dlskel_bad USING iceberg AS SELECT 1; +CREATE MATERIALIZED VIEW dlskel_mv USING iceberg AS SELECT 1; + +-- Converting a heap into a lake table has to be refused too: the relation is +-- still a heap when the statement arrives, so the guard above does not see it. +ALTER TABLE dlskel_heap SET ACCESS METHOD iceberg; +SET iceberg.default_catalog = 'dlskel_cat'; +SET iceberg.default_volume = 'dlskel_vol'; +ALTER TABLE dlskel_heap SET ACCESS METHOD iceberg; +RESET iceberg.default_catalog; +RESET iceberg.default_volume; + +-- A column-bearing clause still cannot be honoured, and neither can a +-- replicated policy. +CREATE TABLE dlskel_bad_repl (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol') + DISTRIBUTED REPLICATED; + +-- Renaming a schema would repoint its lake tables at a different external +-- namespace, so a schema holding one is refused -- and, just as importantly, a +-- schema holding none is not: the guard has to be no wider than the problem. +CREATE SCHEMA dlskel_sch; +CREATE TABLE dlskel_sch.t (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol'); +BEGIN; +ALTER SCHEMA dlskel_sch RENAME TO dlskel_sch2; +ROLLBACK; +CREATE SCHEMA dlskel_plain; +ALTER SCHEMA dlskel_plain RENAME TO dlskel_plain2; +DROP SCHEMA dlskel_plain2; + +-- Renaming a wrapper breaks every mapping lookup at once, including the one +-- DROP needs, so it is refused whether or not a table exists yet. +-- Each attempt is rolled back: if the guard ever regresses, an accepted rename +-- would leave the extension's own wrapper under a different name, which nothing +-- in the cleanup below can undo and which breaks every later run. +BEGIN; +ALTER FOREIGN DATA WRAPPER iceberg_catalog_fdw RENAME TO dlskel_other_fdw; +ROLLBACK; +BEGIN; +ALTER FOREIGN DATA WRAPPER iceberg_volume_fdw RENAME TO dlskel_other_fdw; +ROLLBACK; + +ALTER TABLE dlskel_r ADD COLUMN c int; +ALTER TABLE dlskel_r SET (fillfactor = 90); +ALTER TABLE dlskel_r SET ACCESS METHOD heap; +ALTER TABLE dlskel_r SET DISTRIBUTED BY (a); +ALTER TABLE dlskel_r RENAME TO dlskel_r2; +ALTER TABLE dlskel_r RENAME COLUMN a TO aa; +ALTER TABLE dlskel_r SET SCHEMA public; +-- OWNER TO is the one ALTER TABLE form that goes through: pg_dump writes it for +-- every table, and ownership cannot reach the external table. Put it back +-- afterwards so the rest of the file still owns what it created. +ALTER TABLE dlskel_r OWNER TO dlskel_role; +SELECT relname, pg_get_userbyid(relowner) AS owner +FROM pg_class WHERE relname = 'dlskel_r'; +ALTER TABLE dlskel_r OWNER TO CURRENT_USER; + +ALTER SERVER dlskel_cat + OPTIONS (SET uri 'thrift://other:9083'); +ALTER SERVER dlskel_cat VERSION '2'; +ALTER SERVER dlskel_cat RENAME TO dlskel_cat2; +ALTER SERVER dlskel_cat OWNER TO dlskel_role; + +ALTER SERVER dlskel_free + OPTIONS (SET uri 'thrift://other:9083'); +ALTER SERVER dlskel_free VERSION '2'; +ALTER SERVER dlskel_free OWNER TO dlskel_role; +ALTER SERVER dlskel_free RENAME TO dlskel_free2; + +RESET iceberg.default_catalog; +RESET iceberg.default_volume; +CREATE TABLE dlskel_bad_missing_server (a int) + USING iceberg + WITH (catalog = 'dlskel_missing', volume = 'dlskel_vol'); +CREATE TABLE dlskel_bad_wrong_catalog (a int) + USING iceberg + WITH (catalog = 'dlskel_vol', volume = 'dlskel_vol'); +CREATE TABLE dlskel_bad_no_catalog (a int) + USING iceberg + WITH (volume = 'dlskel_vol'); +CREATE TABLE dlskel_bad_purge (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', + purge_on_drop = 'perhaps'); +CREATE TABLE dlskel_bad_reloption (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', nonsense = 'x'); + +CREATE SERVER dlskel_bad_server + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', nonsense 'x'); +CREATE SERVER dlskel_bad_secret + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (secret_key 'x'); +-- The catalog wrapper has its own allowlist, and its own credential keys. +CREATE SERVER dlskel_bad_cat_secret + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', password 'x'); +CREATE SERVER dlskel_bad_cat_token + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'polaris', uri 'https://fake:443', client_secret 'x'); +-- Every key the credential list mirrors from an option module, so that a key +-- renamed in one place and not the other fails here instead of silently +-- ceasing to be caught. +CREATE SERVER dlskel_bad_cat_user + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', username 'u'); +CREATE SERVER dlskel_bad_cat_keytab + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', krb_client_keytab '/k'); +CREATE SERVER dlskel_bad_cat_clientid + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'rest', uri 'https://fake:443', client_id 'i'); +CREATE SERVER dlskel_bad_vol_token + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', session_token 't'); +CREATE SERVER dlskel_bad_vol_keyid + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', access_key_id 'k'); +CREATE SERVER dlskel_bad_vol_secret + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', secret_access_key 's'); +-- Option names are matched exactly, as the server itself matches them; a quoted +-- variant is a different, unknown option rather than a second spelling. +CREATE SERVER dlskel_bad_cat_case + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS ("TYPE" 'hive', uri 'thrift://fake:9083'); +-- A catalog type outside the vocabulary, and one that is in it but has no +-- implementation behind it yet. +CREATE SERVER dlskel_bad_cat_type + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'nosuchcatalog', uri 'thrift://fake:9083'); +CREATE SERVER dlskel_bad_cat_hadoop + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hadoop', uri 'hdfs://fake:8020'); +-- builtin needs no url; supplying one means the two disagree. +CREATE SERVER dlskel_bad_cat_builtin + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'builtin', uri 'thrift://fake:9083'); +-- hive does need one. +CREATE SERVER dlskel_bad_cat_nourl + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive'); +-- The realm applies to one catalog type only. +CREATE SERVER dlskel_bad_cat_realm + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri 'thrift://fake:9083', + polaris_server_realm 'OTHER'); +-- An empty value is refused where it is written, not where it is first read. +CREATE SERVER dlskel_bad_cat_empty + FOREIGN DATA WRAPPER iceberg_catalog_fdw + OPTIONS (type 'hive', uri ''); +-- Not a boolean. +CREATE SERVER dlskel_bad_vol_bool + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://dlskel-bucket/p', path_style_access 'perhaps'); +CREATE SERVER dlskel_bad_nopath + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (endpoint 'http://fake:9000'); +CREATE SERVER dlskel_bad_scheme + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 'ftp://x/y'); +CREATE SERVER dlskel_bad_authority + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://'); +CREATE SERVER dlskel_bad_query + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://b/p?versionId=3'); +CREATE SERVER dlskel_bad_userinfo + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://user@b/p'); +CREATE SERVER dlskel_bad_bucket + FOREIGN DATA WRAPPER iceberg_volume_fdw + OPTIONS (base_path 's3://UPPER_case/p'); + +-- The metadata engine is not selectable, so naming one is an unknown option. +CREATE TABLE dlskel_bad_engine (a int) + USING iceberg + WITH (catalog = 'dlskel_cat', volume = 'dlskel_vol', + engine = 'agent'); + +SET client_min_messages = warning; +DROP MATERIALIZED VIEW IF EXISTS dlskel_mv CASCADE; +DROP TABLE IF EXISTS + dlskel_r, dlskel_r2, dlskel_bad, dlskel_bad_dist, + dlskel_bad_part, dlskel_bad_inherits, dlskel_bad_typed, + dlskel_bad_temp, dlskel_bad_unlogged, dlskel_bad_oncommit, + dlskel_bad_tablespace, dlskel_bad_missing_server, + dlskel_bad_wrong_catalog, dlskel_bad_no_catalog, + dlskel_bad_reloption, dlskel_bad_engine, dlskel_r_new, dlskel_heap, + dlskel_bad_repl, dlskel_bad_purge + CASCADE; +DROP SERVER IF EXISTS dlskel_cat CASCADE; +DROP SERVER IF EXISTS dlskel_cat2 CASCADE; +DROP SERVER IF EXISTS dlskel_vol CASCADE; +DROP SERVER IF EXISTS dlskel_free CASCADE; +DROP SERVER IF EXISTS dlskel_free2 CASCADE; +DROP SERVER IF EXISTS dlskel_bad_server CASCADE; +DROP SERVER IF EXISTS dlskel_bad_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_user CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_keytab CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_clientid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_keyid CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_secret CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_token CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_case CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_type CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_hadoop CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_builtin CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_nourl CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_realm CASCADE; +DROP SERVER IF EXISTS dlskel_bad_cat_empty CASCADE; +DROP SERVER IF EXISTS dlskel_bad_vol_bool CASCADE; +DROP SERVER IF EXISTS dlskel_bad_nopath CASCADE; +DROP SERVER IF EXISTS dlskel_bad_scheme CASCADE; +DROP SERVER IF EXISTS dlskel_bad_authority CASCADE; +DROP SERVER IF EXISTS dlskel_bad_query CASCADE; +DROP SERVER IF EXISTS dlskel_bad_userinfo CASCADE; +DROP SERVER IF EXISTS dlskel_bad_bucket CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch CASCADE; +DROP SCHEMA IF EXISTS dlskel_sch2 CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain CASCADE; +DROP SCHEMA IF EXISTS dlskel_plain2 CASCADE; +DROP TYPE IF EXISTS dlskel_type CASCADE; +DROP ROLE IF EXISTS dlskel_role; +RESET client_min_messages; From c624a99ab4e2e52e62ed207aabd706d7b4575495 Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Wed, 5 Aug 2026 11:32:13 +0800 Subject: [PATCH 28/29] interconnect: restart in fast mode so the test does not race crash recovery The test sets shared_preload_libraries and restarts with "gpstop -raiq". An immediate shutdown skips the shutdown checkpoint, so the control file is left in a state other than DB_SHUTDOWNED and the next startup performs crash recovery: xlogrecovery.c sets InRecovery, xlog.c calls PerformWalRecovery(), which signals PMSIGNAL_RECOVERY_STARTED, and the postmaster moves to PM_RECOVERY. In that state canAcceptConnections() answers CAC_NOTCONSISTENT, reported as "the database system is not accepting connections" with detail "Hot standby mode is disabled". gpstart makes exactly such a connection right after pg_ctl returns, to read the segment configuration, so gpstop -r exits CRITICAL and the restart is reported as failed. The damage does not stop there. psql gives up at the \c that follows, so every statement in the file is skipped and the test fails as a whole; the cleanup at the end of the file never runs; and gpstart never got past starting the coordinator in admin mode, so the cluster is left with no segments up. Suites that run after this one in the same job then lose their Gather Motion nodes and fail as well. Shut down fast instead. A fast shutdown writes the shutdown checkpoint, the control file says DB_SHUTDOWNED, no recovery runs, PM_RECOVERY is never entered, and CAC_NOTCONSISTENT cannot be returned -- the failure becomes unreachable rather than merely less likely. Fast is also what the rest of the tree already uses: gpstop -raf/-arf appear in dozens of places, and this file was the only user of -raiq. Measured on a three-segment demo cluster, dirtying 1.5M coordinator rows before each restart so that recovery is slow enough to lose the race reliably: -raiq failed 2/2 with the message above, -rafq passed 3/3 with all three segments still up afterwards. pg_controldata confirms the mechanism at the other end -- "in production" after an immediate shutdown, "shut down" after a fast one. The test still passes under pg_regress with the change. --- contrib/interconnect/sql/interconnect.sql | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/contrib/interconnect/sql/interconnect.sql b/contrib/interconnect/sql/interconnect.sql index 4e6555b6b82..32bbef92295 100644 --- a/contrib/interconnect/sql/interconnect.sql +++ b/contrib/interconnect/sql/interconnect.sql @@ -1,6 +1,12 @@ -- start_ignore \! gpconfig -c shared_preload_libraries -v "interconnect" -\! gpstop -raiq +-- Restart in fast mode, not immediate: an immediate shutdown skips the +-- shutdown checkpoint, so the next startup runs crash recovery, and while the +-- postmaster is in PM_RECOVERY it rejects the connection gpstart makes to read +-- the segment configuration ("the database system is not accepting +-- connections"). gpstop -r then fails, psql gives up at the \c below, and the +-- whole file is skipped with only the coordinator left running. +\! gpstop -rafq \c DROP TABLE IF EXISTS test_ic_data; CREATE EXTENSION IF NOT EXISTS interconnect; @@ -83,5 +89,5 @@ DROP EXTENSION interconnect; -- start_ignore \! gpconfig -r shared_preload_libraries -\! gpstop -raiq +\! gpstop -rafq -- end_ignore From 6f40dd0f7dbf45739ca86812ab87681e054328ca Mon Sep 17 00:00:00 2001 From: MisterRaindrop <278811821@qq.com> Date: Thu, 13 Aug 2026 18:11:02 +0800 Subject: [PATCH 29/29] contrib: build datalake_fdw only with --enable-datalake-fdw Review asked for the module to be off by default and enabled explicitly, the way PAX is, rather than being added to the unconditional contrib SUBDIRS list. The mechanism is the same one PAX and ic-udp2 use: a PGAC_ARG_BOOL defaulting to no, substituted into Makefile.global, and a conditional in contrib/Makefile that puts the directory in ALWAYS_SUBDIRS when disabled so the clean targets still reach it. The option carries no AC_DEFINE. PAX and ic-udp2 define one because C code tests it; nothing here does, and adding an unread macro would mean touching src/include/pg_config.h.in for no reader. --enable-pxf and --enable-orafce already use this shorter form. configure is regenerated by hand rather than wholesale. The committed configure and configure.ac are currently out of sync in both directions -- the PAX liburing block has changes in configure.ac that were never regenerated, and the Darwin python shared-library lookup exists in configure but not in configure.ac -- so a full autoconf run would have swept eight unrelated hunks into this commit. Only the four hunks belonging to this option were applied. --enable-datalake-fdw is added to the CI configure line, without which the ic-datalake-fdw job would install nothing and test nothing. Verified by configuring both ways on a real tree: with the option, "checking whether to build with datalake_fdw support ... yes", Makefile.global gets enable_datalake_fdw = yes, and contrib puts datalake_fdw in SUBDIRS; without it, no / no / and the directory appears in ALWAYS_SUBDIRS instead. --- configure | 38 +++++++++++++++++++ configure.ac | 13 +++++++ contrib/Makefile | 7 +++- .../scripts/configure-cloudberry.sh | 1 + src/Makefile.global.in | 1 + 5 files changed, 59 insertions(+), 1 deletion(-) diff --git a/configure b/configure index d2ffd6b00bd..ecd8933e38e 100755 --- a/configure +++ b/configure @@ -752,6 +752,7 @@ ICU_CFLAGS with_icu enable_thread_safety INCLUDES +enable_datalake_fdw enable_pax CMAKE ZSTD_LIBS @@ -917,6 +918,7 @@ enable_ic_udp2 enable_ic_proxy enable_preload_ic_module enable_pax +enable_datalake_fdw enable_thread_safety with_icu with_tcl @@ -1642,6 +1644,7 @@ Optional Features: --disable-preload-ic-module disable preload interconnect module --enable-pax enable PAX support + --enable-datalake-fdw build with Apache Iceberg lake table support --disable-thread-safety disable thread-safety in client libraries --enable-openssl-redirect enable redirect openssl interface to internal @@ -10004,6 +10007,41 @@ $as_echo "checking whether to build with PAX support ... no" >&6; } fi +# +# datalake_fdw support +# +# Off by default like the other storage integrations. The module builds with +# no external dependency today, but the read and write paths bring an +# object-store SDK and Arrow with them, so it is opt-in from the start rather +# than becoming opt-in later. +# + + +# Check whether --enable-datalake-fdw was given. +if test "${enable_datalake_fdw+set}" = set; then : + enableval=$enable_datalake_fdw; + case $enableval in + yes) + : + ;; + no) + : + ;; + *) + as_fn_error $? "no argument expected for --enable-datalake-fdw option" "$LINENO" 5 + ;; + esac + +else + enable_datalake_fdw=no + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: checking whether to build with datalake_fdw support ... $enable_datalake_fdw" >&5 +$as_echo "checking whether to build with datalake_fdw support ... $enable_datalake_fdw" >&6; } + + # # Include directories # diff --git a/configure.ac b/configure.ac index dd259c4f07f..6294cbecef4 100644 --- a/configure.ac +++ b/configure.ac @@ -1072,6 +1072,19 @@ else fi AC_SUBST(enable_pax) +# +# datalake_fdw support +# +# Off by default like the other storage integrations. The module builds with +# no external dependency today, but the read and write paths bring an +# object-store SDK and Arrow with them, so it is opt-in from the start rather +# than becoming opt-in later. +# +PGAC_ARG_BOOL(enable, datalake-fdw, no, + [build with Apache Iceberg lake table support]) +AC_MSG_RESULT([checking whether to build with datalake_fdw support ... $enable_datalake_fdw]) +AC_SUBST(enable_datalake_fdw) + # # Include directories # diff --git a/contrib/Makefile b/contrib/Makefile index c2a1d396bb8..94e992b345d 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -25,7 +25,6 @@ SUBDIRS = \ btree_gin \ btree_gist \ citext \ - datalake_fdw \ dblink \ dict_int \ dict_xsyn \ @@ -111,6 +110,12 @@ else ALWAYS_SUBDIRS += pax_storage endif +ifeq ($(enable_datalake_fdw),yes) +SUBDIRS += datalake_fdw +else +ALWAYS_SUBDIRS += datalake_fdw +endif + ifeq ($(enable_ic_udp2),yes) SUBDIRS += udp2 else diff --git a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh index eda9586c28f..ea34bbb4581 100755 --- a/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh +++ b/devops/build/automation/cloudberry/scripts/configure-cloudberry.sh @@ -153,6 +153,7 @@ fi log_section "Configure" execute_cmd ./configure --prefix=${BUILD_DESTINATION} \ --disable-external-fts \ + --enable-datalake-fdw \ --enable-gpcloud \ --enable-ic-proxy \ --enable-mapreduce \ diff --git a/src/Makefile.global.in b/src/Makefile.global.in index 74ba8d0c370..f00f0be7091 100644 --- a/src/Makefile.global.in +++ b/src/Makefile.global.in @@ -231,6 +231,7 @@ enable_orafce = @enable_orafce@ enable_mapreduce = @enable_mapreduce@ enable_shared_postgres_backend = @enable_shared_postgres_backend@ enable_link_postgres_with_shared = @enable_link_postgres_with_shared@ +enable_datalake_fdw = @enable_datalake_fdw@ enable_gpcloud = @enable_gpcloud@ enable_ic_proxy = @enable_ic_proxy@ enable_ic_udp2 = @enable_ic_udp2@