Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.GenericArray;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.factories.Factory;
import org.apache.paimon.factories.FactoryException;
import org.apache.paimon.factories.FactoryUtil;
import org.apache.paimon.fs.FileStatus;
Expand Down Expand Up @@ -165,9 +166,23 @@ public IcebergCommitCallback(FileStoreTable table, String commitUser) {
FactoryUtil.discoverFactory(
IcebergCommitCallback.class.getClassLoader(),
IcebergMetadataCommitterFactory.class,
storageType.toString());
} catch (FactoryException ignore) {
storageType.committerFactoryIdentifier());
} catch (FactoryException e) {
metadataCommitterFactory = null;
// storage types without a committer have no factory by design, so a miss is expected
if (storageType.requiresMetadataCommitter()) {
LOG.warn(
"No IcebergMetadataCommitterFactory for '{}={}' found on the classpath, so "
+ "table {} will not be synced to the external catalog (commits and "
+ "metadata files are unaffected). Check that the module providing it "
+ "is deployed and that its META-INF/services/{} entry survived "
+ "shading. Cause: {}",
IcebergOptions.METADATA_ICEBERG_STORAGE.key(),
storageType,
table.fullName(),
Factory.class.getName(),
e.getMessage());
}
}
this.metadataCommitter =
metadataCommitterFactory == null ? null : metadataCommitterFactory.create(table);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -198,27 +198,46 @@ public int previousVersionsMax() {

/** Where to store Iceberg metadata. */
public enum StorageType implements DescribedEnum {
DISABLED("disabled", "Disable Iceberg compatibility support."),
TABLE_LOCATION("table-location", "Store Iceberg metadata in each table's directory."),
DISABLED("disabled", "Disable Iceberg compatibility support.", false),
TABLE_LOCATION(
"table-location", "Store Iceberg metadata in each table's directory.", false),
HADOOP_CATALOG(
"hadoop-catalog",
"Store Iceberg metadata in a separate directory. "
+ "This directory can be specified as the warehouse directory of an Iceberg Hadoop catalog."),
+ "This directory can be specified as the warehouse directory of an Iceberg Hadoop catalog.",
false),
HIVE_CATALOG(
"hive-catalog",
"Not only store Iceberg metadata like hadoop-catalog, "
+ "but also create Iceberg external table in Hive."),
+ "but also create Iceberg external table in Hive.",
true),
REST_CATALOG(
"rest-catalog",
"Store Iceberg metadata in a REST catalog. "
+ "This allows integration with Iceberg REST catalog services.");
+ "This allows integration with Iceberg REST catalog services.",
true);

private final String value;
private final String description;
private final boolean requiresMetadataCommitter;

StorageType(String value, String description) {
StorageType(String value, String description, boolean requiresMetadataCommitter) {
this.value = value;
this.description = description;
this.requiresMetadataCommitter = requiresMetadataCommitter;
}

/** Whether this storage type syncs metadata to an external catalog via a committer. */
public boolean requiresMetadataCommitter() {
return requiresMetadataCommitter;
}

/**
* Identifier this storage type's {@code IcebergMetadataCommitterFactory} is registered
* under.
*/
public String committerFactoryIdentifier() {
return value;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public class IcebergHiveMetadataCommitterFactory implements IcebergMetadataCommi

@Override
public String identifier() {
return IcebergOptions.StorageType.HIVE_CATALOG.toString();
return IcebergOptions.StorageType.HIVE_CATALOG.committerFactoryIdentifier();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
public class IcebergRESTMetadataCommitterFactory implements IcebergMetadataCommitterFactory {
@Override
public String identifier() {
return IcebergOptions.StorageType.REST_CATALOG.toString();
return IcebergOptions.StorageType.REST_CATALOG.committerFactoryIdentifier();
}

@Override
Expand Down
Loading