From c729808264f8a5e2de43eb1500c64780689a71de Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Wed, 18 Nov 2026 18:20:31 +0000 Subject: [PATCH 1/5] feat(pdk): add trigger CDC API contracts --- .../entity/triggercdc/TriggerCdcBatch.java | 158 ++++++ .../triggercdc/TriggerCdcCapabilities.java | 411 +++++++++++++++ .../triggercdc/TriggerCdcCaptureImage.java | 24 + .../triggercdc/TriggerCdcCleanupResult.java | 187 +++++++ .../entity/triggercdc/TriggerCdcConfig.java | 262 ++++++++++ .../triggercdc/TriggerCdcConstants.java | 69 +++ .../entity/triggercdc/TriggerCdcDdlMode.java | 35 ++ .../triggercdc/TriggerCdcErrorPolicy.java | 24 + .../entity/triggercdc/TriggerCdcEvent.java | 281 ++++++++++ .../triggercdc/TriggerCdcInspectResult.java | 207 ++++++++ .../triggercdc/TriggerCdcInstallMode.java | 20 + .../triggercdc/TriggerCdcInstallPlan.java | 177 +++++++ .../triggercdc/TriggerCdcInstallResult.java | 482 ++++++++++++++++++ .../entity/triggercdc/TriggerCdcObject.java | 132 +++++ .../triggercdc/TriggerCdcObjectState.java | 111 ++++ .../triggercdc/TriggerCdcObjectType.java | 25 + .../entity/triggercdc/TriggerCdcOffset.java | 139 +++++ .../triggercdc/TriggerCdcOperation.java | 32 ++ .../triggercdc/TriggerCdcStatement.java | 112 ++++ .../triggercdc/TriggerCdcStatementType.java | 15 + .../entity/triggercdc/TriggerCdcStatus.java | 43 ++ .../entity/triggercdc/TriggerCdcTable.java | 156 ++++++ .../triggercdc/TriggerCdcTableResult.java | 229 +++++++++ .../apis/functions/ConnectorFunctions.java | 88 ++++ .../tapdata/pdk/apis/functions/PDKMethod.java | 10 +- .../TriggerCdcAcknowledgeFunction.java | 36 ++ .../TriggerCdcCapabilitiesFunction.java | 26 + .../triggercdc/TriggerCdcCleanupFunction.java | 49 ++ .../triggercdc/TriggerCdcEnsureFunction.java | 58 +++ .../triggercdc/TriggerCdcInspectFunction.java | 30 ++ .../triggercdc/TriggerCdcPlanFunction.java | 30 ++ .../triggercdc/TriggerCdcReadFunction.java | 68 +++ .../triggercdc/TriggerCdcReleaseFunction.java | 34 ++ 33 files changed, 3759 insertions(+), 1 deletion(-) create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcBatch.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCapabilities.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCaptureImage.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConfig.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConstants.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcDdlMode.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcErrorPolicy.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcEvent.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInspectResult.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallMode.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallPlan.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObject.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectState.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectType.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcOffset.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcOperation.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatement.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatementType.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatus.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTable.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTableResult.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcAcknowledgeFunction.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcCapabilitiesFunction.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcCleanupFunction.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcEnsureFunction.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcInspectFunction.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcPlanFunction.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReadFunction.java create mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReleaseFunction.java diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcBatch.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcBatch.java new file mode 100644 index 00000000..020c609c --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcBatch.java @@ -0,0 +1,158 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * A batch of change records returned by + * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcReadFunction}. + * + *

Scope: one batch holds the records of exactly one capture, the one selected by the + * {@code captureId} argument of the read call. {@code captureId} and {@code generation} echo that + * selector and the generation actually read, so a batch can never be acked against another + * capture. A task that captures several tables receives one batch per capture and does its own + * merging; this contract does not define a combined batch.

+ * + *

The batch carries a {@code batchToken}. The framework must call + * {@code TriggerCdcAcknowledgeFunction} with that token only after the downstream sink and the + * recoverable checkpoint are both durable. Returning a batch does not mean the events are + * consumed.

+ * + *

Delivery is at least once: a crash between "sink written" and "batch acknowledged" replays + * the batch, which is expected and must be tolerated by the target side. End to end exactly once + * depends on the target write and dedup abilities, never on this contract alone.

+ */ +public class TriggerCdcBatch implements Serializable { + private static final long serialVersionUID = 1L; + + /** + * Opaque token referring to the delivered set of change records of this capture. Connectors + * should persist the referenced ids (a token can also be a row range marker) instead of + * forcing the engine to carry a huge id list inside the task checkpoint. A token is scoped to + * its capture and is only meaningful in {@code acknowledge} for the same {@code captureId}. + */ + protected String batchToken; + /** + * Capture these records belong to; must equal the {@code captureId} the read call selected. + */ + protected String captureId; + /** + * Generation of the capture as read. The framework persists it on the first call, so that a + * later read with a different generation can be rejected instead of silently resumed. + */ + protected Long generation; + protected List events; + /** True when more unacknowledged records of this capture are waiting right now. */ + protected boolean hasMore; + /** + * Total number of records of this capture still unacknowledged for this consumer, when cheap + * to compute. + */ + protected Long pendingCount; + /** Capture time of the oldest unacknowledged record, for lag monitoring. */ + protected Long oldestPendingCapturedAt; + + public static TriggerCdcBatch create() { + return new TriggerCdcBatch(); + } + + public TriggerCdcBatch event(TriggerCdcEvent event) { + if (events == null) { + events = new ArrayList<>(); + } + events.add(event); + return this; + } + + /** True when this batch carries no change record. */ + public boolean isEmpty() { + return events == null || events.isEmpty(); + } + + public String getBatchToken() { + return batchToken; + } + + public void setBatchToken(String batchToken) { + this.batchToken = batchToken; + } + + public TriggerCdcBatch batchToken(String batchToken) { + this.batchToken = batchToken; + return this; + } + + public String getCaptureId() { + return captureId; + } + + public void setCaptureId(String captureId) { + this.captureId = captureId; + } + + public TriggerCdcBatch captureId(String captureId) { + this.captureId = captureId; + return this; + } + + public Long getGeneration() { + return generation; + } + + public void setGeneration(Long generation) { + this.generation = generation; + } + + public TriggerCdcBatch generation(Long generation) { + this.generation = generation; + return this; + } + + public List getEvents() { + return events; + } + + public void setEvents(List events) { + this.events = events; + } + + public boolean isHasMore() { + return hasMore; + } + + public void setHasMore(boolean hasMore) { + this.hasMore = hasMore; + } + + public TriggerCdcBatch hasMore(boolean hasMore) { + this.hasMore = hasMore; + return this; + } + + public Long getPendingCount() { + return pendingCount; + } + + public void setPendingCount(Long pendingCount) { + this.pendingCount = pendingCount; + } + + public TriggerCdcBatch pendingCount(Long pendingCount) { + this.pendingCount = pendingCount; + return this; + } + + public Long getOldestPendingCapturedAt() { + return oldestPendingCapturedAt; + } + + public void setOldestPendingCapturedAt(Long oldestPendingCapturedAt) { + this.oldestPendingCapturedAt = oldestPendingCapturedAt; + } + + public TriggerCdcBatch oldestPendingCapturedAt(Long oldestPendingCapturedAt) { + this.oldestPendingCapturedAt = oldestPendingCapturedAt; + return this; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCapabilities.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCapabilities.java new file mode 100644 index 00000000..2ca12f80 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCapabilities.java @@ -0,0 +1,411 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * What a specific connector can really do with trigger based CDC. + * + *

This is the contract that keeps the framework honest: "the database has triggers" never + * implies "the connector supports full CDC". Every optional feature must be declared here and + * the framework must not use a feature that is not declared.

+ * + *

Declarations are per database and version: a connector must report the version it + * validated against, because trigger behaviour and DDL visibility differ between releases.

+ */ +public class TriggerCdcCapabilities implements Serializable { + private static final long serialVersionUID = 1L; + + /** Database type this declaration belongs to. */ + protected String databaseType; + /** Database version the declaration was verified against. */ + protected String databaseVersion; + /** Connector build that produced this declaration. */ + protected String connectorVersion; + + /** + * Operations observable through the trigger. {@link TriggerCdcOperation#TRUNCATE} appears here + * if and only if {@link #isTruncateCaptureSupported()} is {@code true}. + */ + protected Set dmlOperations; + /** Capture images the connector can materialise. */ + protected Set captureImages; + /** DDL levels the connector honours. */ + protected Set ddlModes; + /** Error policies the connector implements. STRICT must always be present. */ + protected Set errorPolicies; + + /** + * The transaction id is stored with each change record. Useful for dedup and for grouping + * events of the same source transaction. + */ + protected boolean transactionIdSupported; + /** A per transaction sequence is stored, so the order inside one transaction is provable. */ + protected boolean transactionSequenceSupported; + /** + * The connector can prove a capture boundary across installation, so no change committed + * while the trigger is being created is lost. + * + *

Without it, the framework must not offer an automatic full-then-incremental handover.

+ */ + protected boolean captureBoundarySupported; + /** + * The database exposes a verified commit ordered cursor. Informational only. + * + *

It records what a future, versioned contract could exploit to replace per batch + * acknowledgement with a high watermark. It must not be used to skip acknowledgement + * under the current contract: the position is moved by + * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcAcknowledgeFunction} + * alone, and a connector that advances its own cursor claiming this flag is betting on a + * guarantee the framework has not verified.

+ */ + protected boolean commitOrderedCursorSupported; + /** Ordering of changes of the same business key is provable and preserved. */ + protected boolean perKeyOrderingSupported; + /** Commit order across different tables of one source transaction is provable. */ + protected boolean crossTableOrderingSupported; + /** A whole source transaction can be replayed atomically on the target. */ + protected boolean atomicReplaySupported; + /** TRUNCATE can be observed as its own event; must match {@link TriggerCdcOperation#TRUNCATE}. */ + protected boolean truncateCaptureSupported; + /** The connector can detect and build a safe starting point for newly created tables. */ + protected boolean autoDiscoverNewTableSupported; + /** Several consumers may read the same journal without stealing each other's events. */ + protected boolean multiConsumerSupported; + /** The journal survives a connector restart without losing events. */ + protected boolean durableJournal; + /** The trigger can be installed and removed without taking a long exclusive table lock. */ + protected boolean onlineInstallSupported; + + /** Largest journal payload a single change may produce, in bytes. 0 means unknown. */ + protected long maxPayloadBytes; + + /** + * Column types the connector refuses to capture, for example LOB or spatial types it cannot + * round trip. Refusing at pre-check time is required; silently truncating is not. + */ + protected List unsupportedTypes; + + /** Connector specific notes, for example the minimum supported database version. */ + protected Map extra; + + public static TriggerCdcCapabilities create() { + return new TriggerCdcCapabilities(); + } + + public TriggerCdcCapabilities dmlOperation(TriggerCdcOperation operation) { + if (dmlOperations == null) { + dmlOperations = new LinkedHashSet<>(); + } + dmlOperations.add(operation); + return this; + } + + public TriggerCdcCapabilities captureImage(TriggerCdcCaptureImage captureImage) { + if (captureImages == null) { + captureImages = new LinkedHashSet<>(); + } + captureImages.add(captureImage); + return this; + } + + public TriggerCdcCapabilities ddlMode(TriggerCdcDdlMode ddlMode) { + if (ddlModes == null) { + ddlModes = new LinkedHashSet<>(); + } + ddlModes.add(ddlMode); + return this; + } + + public TriggerCdcCapabilities errorPolicy(TriggerCdcErrorPolicy errorPolicy) { + if (errorPolicies == null) { + errorPolicies = new LinkedHashSet<>(); + } + errorPolicies.add(errorPolicy); + return this; + } + + public TriggerCdcCapabilities unsupportedType(String unsupportedType) { + if (unsupportedTypes == null) { + unsupportedTypes = new ArrayList<>(); + } + unsupportedTypes.add(unsupportedType); + return this; + } + + public boolean supports(TriggerCdcOperation operation) { + return dmlOperations != null && dmlOperations.contains(operation); + } + + public boolean supports(TriggerCdcCaptureImage captureImage) { + return captureImages != null && captureImages.contains(captureImage); + } + + public boolean supports(TriggerCdcDdlMode ddlMode) { + return ddlModes != null && ddlModes.contains(ddlMode); + } + + public boolean supports(TriggerCdcErrorPolicy errorPolicy) { + return errorPolicies != null && errorPolicies.contains(errorPolicy); + } + + public String getDatabaseType() { + return databaseType; + } + + public void setDatabaseType(String databaseType) { + this.databaseType = databaseType; + } + + public TriggerCdcCapabilities databaseType(String databaseType) { + this.databaseType = databaseType; + return this; + } + + public String getDatabaseVersion() { + return databaseVersion; + } + + public void setDatabaseVersion(String databaseVersion) { + this.databaseVersion = databaseVersion; + } + + public TriggerCdcCapabilities databaseVersion(String databaseVersion) { + this.databaseVersion = databaseVersion; + return this; + } + + public String getConnectorVersion() { + return connectorVersion; + } + + public void setConnectorVersion(String connectorVersion) { + this.connectorVersion = connectorVersion; + } + + public TriggerCdcCapabilities connectorVersion(String connectorVersion) { + this.connectorVersion = connectorVersion; + return this; + } + + public Set getDmlOperations() { + return dmlOperations; + } + + public void setDmlOperations(Set dmlOperations) { + this.dmlOperations = dmlOperations; + } + + public Set getCaptureImages() { + return captureImages; + } + + public void setCaptureImages(Set captureImages) { + this.captureImages = captureImages; + } + + public Set getDdlModes() { + return ddlModes; + } + + public void setDdlModes(Set ddlModes) { + this.ddlModes = ddlModes; + } + + public Set getErrorPolicies() { + return errorPolicies; + } + + public void setErrorPolicies(Set errorPolicies) { + this.errorPolicies = errorPolicies; + } + + public boolean isTransactionIdSupported() { + return transactionIdSupported; + } + + public void setTransactionIdSupported(boolean transactionIdSupported) { + this.transactionIdSupported = transactionIdSupported; + } + + public TriggerCdcCapabilities transactionIdSupported(boolean transactionIdSupported) { + this.transactionIdSupported = transactionIdSupported; + return this; + } + + public boolean isTransactionSequenceSupported() { + return transactionSequenceSupported; + } + + public void setTransactionSequenceSupported(boolean transactionSequenceSupported) { + this.transactionSequenceSupported = transactionSequenceSupported; + } + + public TriggerCdcCapabilities transactionSequenceSupported(boolean transactionSequenceSupported) { + this.transactionSequenceSupported = transactionSequenceSupported; + return this; + } + + public boolean isCaptureBoundarySupported() { + return captureBoundarySupported; + } + + public void setCaptureBoundarySupported(boolean captureBoundarySupported) { + this.captureBoundarySupported = captureBoundarySupported; + } + + public TriggerCdcCapabilities captureBoundarySupported(boolean captureBoundarySupported) { + this.captureBoundarySupported = captureBoundarySupported; + return this; + } + + public boolean isCommitOrderedCursorSupported() { + return commitOrderedCursorSupported; + } + + public void setCommitOrderedCursorSupported(boolean commitOrderedCursorSupported) { + this.commitOrderedCursorSupported = commitOrderedCursorSupported; + } + + public TriggerCdcCapabilities commitOrderedCursorSupported(boolean commitOrderedCursorSupported) { + this.commitOrderedCursorSupported = commitOrderedCursorSupported; + return this; + } + + public boolean isPerKeyOrderingSupported() { + return perKeyOrderingSupported; + } + + public void setPerKeyOrderingSupported(boolean perKeyOrderingSupported) { + this.perKeyOrderingSupported = perKeyOrderingSupported; + } + + public TriggerCdcCapabilities perKeyOrderingSupported(boolean perKeyOrderingSupported) { + this.perKeyOrderingSupported = perKeyOrderingSupported; + return this; + } + + public boolean isCrossTableOrderingSupported() { + return crossTableOrderingSupported; + } + + public void setCrossTableOrderingSupported(boolean crossTableOrderingSupported) { + this.crossTableOrderingSupported = crossTableOrderingSupported; + } + + public TriggerCdcCapabilities crossTableOrderingSupported(boolean crossTableOrderingSupported) { + this.crossTableOrderingSupported = crossTableOrderingSupported; + return this; + } + + public boolean isAtomicReplaySupported() { + return atomicReplaySupported; + } + + public void setAtomicReplaySupported(boolean atomicReplaySupported) { + this.atomicReplaySupported = atomicReplaySupported; + } + + public TriggerCdcCapabilities atomicReplaySupported(boolean atomicReplaySupported) { + this.atomicReplaySupported = atomicReplaySupported; + return this; + } + + public boolean isTruncateCaptureSupported() { + return truncateCaptureSupported; + } + + public void setTruncateCaptureSupported(boolean truncateCaptureSupported) { + this.truncateCaptureSupported = truncateCaptureSupported; + } + + public TriggerCdcCapabilities truncateCaptureSupported(boolean truncateCaptureSupported) { + this.truncateCaptureSupported = truncateCaptureSupported; + return this; + } + + public boolean isAutoDiscoverNewTableSupported() { + return autoDiscoverNewTableSupported; + } + + public void setAutoDiscoverNewTableSupported(boolean autoDiscoverNewTableSupported) { + this.autoDiscoverNewTableSupported = autoDiscoverNewTableSupported; + } + + public TriggerCdcCapabilities autoDiscoverNewTableSupported(boolean autoDiscoverNewTableSupported) { + this.autoDiscoverNewTableSupported = autoDiscoverNewTableSupported; + return this; + } + + public boolean isMultiConsumerSupported() { + return multiConsumerSupported; + } + + public void setMultiConsumerSupported(boolean multiConsumerSupported) { + this.multiConsumerSupported = multiConsumerSupported; + } + + public TriggerCdcCapabilities multiConsumerSupported(boolean multiConsumerSupported) { + this.multiConsumerSupported = multiConsumerSupported; + return this; + } + + public boolean isDurableJournal() { + return durableJournal; + } + + public void setDurableJournal(boolean durableJournal) { + this.durableJournal = durableJournal; + } + + public TriggerCdcCapabilities durableJournal(boolean durableJournal) { + this.durableJournal = durableJournal; + return this; + } + + public boolean isOnlineInstallSupported() { + return onlineInstallSupported; + } + + public void setOnlineInstallSupported(boolean onlineInstallSupported) { + this.onlineInstallSupported = onlineInstallSupported; + } + + public TriggerCdcCapabilities onlineInstallSupported(boolean onlineInstallSupported) { + this.onlineInstallSupported = onlineInstallSupported; + return this; + } + + public long getMaxPayloadBytes() { + return maxPayloadBytes; + } + + public void setMaxPayloadBytes(long maxPayloadBytes) { + this.maxPayloadBytes = maxPayloadBytes; + } + + public TriggerCdcCapabilities maxPayloadBytes(long maxPayloadBytes) { + this.maxPayloadBytes = maxPayloadBytes; + return this; + } + + public List getUnsupportedTypes() { + return unsupportedTypes; + } + + public void setUnsupportedTypes(List unsupportedTypes) { + this.unsupportedTypes = unsupportedTypes; + } + + public Map getExtra() { + return extra; + } + + public void setExtra(Map extra) { + this.extra = extra; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCaptureImage.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCaptureImage.java new file mode 100644 index 00000000..692dde99 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCaptureImage.java @@ -0,0 +1,24 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +/** + * How much of the row image is mirrored into the CDC journal. + */ +public enum TriggerCdcCaptureImage { + /** + * Default. INSERT records the full after image, UPDATE records the full before and after + * images, DELETE records the full before image. + * + *

Full images are needed to support filter condition changes, primary key changes, + * deletes and lookups on the target side. They cost more storage, but they never force a + * later re-read of the source table, which would observe already updated data.

+ */ + FULL, + /** + * Only key columns plus changed columns are mirrored. Lower storage cost, but the target + * side can no longer rebuild the previous / next full row state. + * + *

Only usable when a connector declares it in + * {@link TriggerCdcCapabilities#getCaptureImages()}. Never assume it as a default.

+ */ + MINIMAL +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java new file mode 100644 index 00000000..a7891a18 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java @@ -0,0 +1,187 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * Outcome of + * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcCleanupFunction}. + * + *

One rule decides what this object may report: a change record may be removed only after + * every live consumer acknowledged it. Retention is not an exception to that rule by default — + * an unacknowledged record is reported through {@link #getRemainingCount()} instead of being + * deleted.

+ * + *

The single exception is opted into explicitly through + * {@link TriggerCdcConfig#getForceCleanUnacknowledged()} (default {@code false}). When it is + * enabled and the connector reclaims records that were not acknowledged, it must (a) count them + * in {@link #getDeletedUnacknowledgedCount()}, (b) list every affected consumer in + * {@link #getIntegrityBrokenConsumers()}, and (c) persist that marking together with the delete. + * Reclaiming unacknowledged records without reporting the affected consumers is silent data loss + * and a contract violation, not a tuning choice.

+ * + *

Because (a) and (b) are plain data, the framework can check the contract instead of trusting + * prose: call {@link #validate(TriggerCdcConfig)} on every returned result and reject a cleanup + * that reports inconsistencies. A connector that removes rows and leaves + * {@link #getDeletedUnacknowledgedCount()} at {@code 0} is claiming it removed nothing + * unacknowledged, and is answerable for that claim.

+ */ +public class TriggerCdcCleanupResult implements Serializable { + private static final long serialVersionUID = 1L; + + /** Records removed by this call. */ + protected long deletedCount; + /** + * Records kept because at least one live consumer has not acknowledged them. A non zero value + * is not an error: it is the signal the framework alarms on and the input for capacity + * planning. + */ + protected long remainingCount; + /** + * How many of {@link #getDeletedCount()} were not acknowledged by every live consumer — the + * only records that can damage a consumer. Must be {@code 0} outside the force clean opt-in. + * + *

This field exists so the contract is checkable rather than aspirational: without it a + * connector could delete unacknowledged rows, report a total {@code deletedCount}, and leave + * {@link #getIntegrityBrokenConsumers()} empty with nothing able to tell.

+ */ + protected long deletedUnacknowledgedCount; + /** + * Consumers whose data is incomplete because this call removed records they had not + * acknowledged. Must be empty when {@link #getDeletedUnacknowledgedCount()} is {@code 0}, and + * must list every affected consumer when it is greater than {@code 0}: a forced clean that + * deletes unacknowledged rows but reports no consumer is rejected by the framework. + */ + protected List integrityBrokenConsumers; + protected List warnings; + + public static TriggerCdcCleanupResult create() { + return new TriggerCdcCleanupResult(); + } + + public TriggerCdcCleanupResult warning(String warning) { + if (warnings == null) { + warnings = new ArrayList<>(); + } + warnings.add(warning); + return this; + } + + /** + * Reports a consumer whose stream now has a hole. Only valid together with a non zero + * {@link #getDeletedUnacknowledgedCount()}: a hole can only come from a removed record. + */ + public TriggerCdcCleanupResult integrityBrokenConsumer(String consumerId) { + if (integrityBrokenConsumers == null) { + integrityBrokenConsumers = new ArrayList<>(); + } + integrityBrokenConsumers.add(consumerId); + return this; + } + + /** True when this call removed records that a consumer had not acknowledged. */ + public boolean hasIntegrityBrokenConsumers() { + return integrityBrokenConsumers != null && !integrityBrokenConsumers.isEmpty(); + } + + /** + * Checks this result against the rules in this class and returns one message per violation; + * an empty list means the result is a valid cleanup outcome. + * + *

The framework must call this after every {@code cleanup} invocation and treat a non empty + * list the same as {@link TriggerCdcErrorPolicy#STRICT} would treat a failed write: the delete + * is not trusted and the affected tasks are marked integrity broken. The check cannot verify + * what the database actually did, only that the report is internally consistent — a connector + * that both deletes unacknowledged records and hides them is out of contract, and no result + * object can catch that.

+ * + * @param config the configuration the cleanup ran with, or {@code null} when unknown; with a + * {@code null} config only the report's internal consistency is checked + */ + public List validate(TriggerCdcConfig config) { + List violations = new ArrayList<>(); + if (deletedCount < 0) { + violations.add("deletedCount must not be negative: " + deletedCount); + } + if (remainingCount < 0) { + violations.add("remainingCount must not be negative: " + remainingCount); + } + if (deletedUnacknowledgedCount < 0) { + violations.add("deletedUnacknowledgedCount must not be negative: " + deletedUnacknowledgedCount); + } + if (deletedUnacknowledgedCount > deletedCount) { + violations.add("deletedUnacknowledgedCount (" + deletedUnacknowledgedCount + + ") exceeds deletedCount (" + deletedCount + ")"); + } + if (deletedUnacknowledgedCount > 0) { + if (config != null && !config.forceCleanUnacknowledgedEnabled()) { + violations.add("removed " + deletedUnacknowledgedCount + + " unacknowledged record(s) while forceCleanUnacknowledged is false"); + } + if (!hasIntegrityBrokenConsumers()) { + violations.add("removed " + deletedUnacknowledgedCount + + " unacknowledged record(s) but reported no integrity broken consumer"); + } + } else if (hasIntegrityBrokenConsumers()) { + violations.add("reported integrity broken consumer(s) " + integrityBrokenConsumers + + " but deletedUnacknowledgedCount is 0"); + } + return violations; + } + + public long getDeletedCount() { + return deletedCount; + } + + public void setDeletedCount(long deletedCount) { + this.deletedCount = deletedCount; + } + + public TriggerCdcCleanupResult deletedCount(long deletedCount) { + this.deletedCount = deletedCount; + return this; + } + + public long getRemainingCount() { + return remainingCount; + } + + public void setRemainingCount(long remainingCount) { + this.remainingCount = remainingCount; + } + + public TriggerCdcCleanupResult remainingCount(long remainingCount) { + this.remainingCount = remainingCount; + return this; + } + + public long getDeletedUnacknowledgedCount() { + return deletedUnacknowledgedCount; + } + + public void setDeletedUnacknowledgedCount(long deletedUnacknowledgedCount) { + this.deletedUnacknowledgedCount = deletedUnacknowledgedCount; + } + + public TriggerCdcCleanupResult deletedUnacknowledgedCount(long deletedUnacknowledgedCount) { + this.deletedUnacknowledgedCount = deletedUnacknowledgedCount; + return this; + } + + public List getIntegrityBrokenConsumers() { + return integrityBrokenConsumers; + } + + public void setIntegrityBrokenConsumers(List integrityBrokenConsumers) { + this.integrityBrokenConsumers = integrityBrokenConsumers; + } + + public List getWarnings() { + return warnings; + } + + public void setWarnings(List warnings) { + this.warnings = warnings; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConfig.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConfig.java new file mode 100644 index 00000000..2e0f0fb9 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConfig.java @@ -0,0 +1,262 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; + +/** + * Connection level Trigger CDC configuration: where the journal lives and how it behaves. + * + *

Ownership rule: the storage location ({@code catalog} / {@code schema}) is defined once + * per connection, while table subscriptions and reader tuning belong to the task. Two tasks + * capturing the same source table must share one set of capture objects, so the storage + * location must not be derived from the task.

+ */ +public class TriggerCdcConfig implements Serializable { + private static final long serialVersionUID = 1L; + + /** + * Catalog of the CDC storage. Null when the database has no catalog concept; the + * connector then ignores it. Never guess a default, because writing the journal into the + * wrong catalog is a data leak between databases. + */ + protected String catalog; + /** Schema (or database / keyspace) that holds the CDC objects. */ + protected String schema; + + protected TriggerCdcInstallMode installMode; + /** How often the framework polls the journal. */ + protected Long pollIntervalMs; + /** How many change records a single read returns at most. */ + protected Integer batchSize; + protected TriggerCdcCaptureImage captureImage; + protected TriggerCdcDdlMode ddlMode; + protected TriggerCdcErrorPolicy errorPolicy; + /** + * How long acknowledged change records are kept before cleanup. Retention must never beat + * the acknowledgement rule: unacknowledged events are never deleted because of TTL alone. + */ + protected Long retentionSeconds; + /** + * Explicit opt-in that lets cleanup reclaim journal space from records not every live + * consumer has acknowledged. Defaults to {@code false} and must stay opt-in: deleting such a + * record destroys the data of a consumer that never saw it, so it must never happen as a + * silent side effect of a TTL. + * + *

When enabled, records older than {@link #retentionSeconds} may be removed even though + * some live consumer still needs them, but the connector then has to (a) count them in + * {@link TriggerCdcCleanupResult#getDeletedUnacknowledgedCount()}, (b) list every affected + * consumer in {@link TriggerCdcCleanupResult#getIntegrityBrokenConsumers()} and (c) persist + * that marking no later than the delete, so the framework can degrade or repair those tasks + * instead of discovering the gap on its own. The framework enforces all three through + * {@link TriggerCdcCleanupResult#validate(TriggerCdcConfig)}.

+ */ + protected Boolean forceCleanUnacknowledged; + /** + * Whether {@code TRUNCATE} is captured as its own event. Not every engine can observe it, and it + * must never be expanded into row level DELETE events. + * + *

Must agree with the capability declaration: this option may only be honoured when the + * connector reports {@link TriggerCdcCapabilities#isTruncateCaptureSupported()}, and a capture + * reporting {@link TriggerCdcOperation#TRUNCATE} events has to be declared capable.

+ */ + protected Boolean captureTruncate; + + public static TriggerCdcConfig create() { + return new TriggerCdcConfig(); + } + + /** + * Fills every null field that has a documented default and returns {@code this}. + * + *

{@link #getCatalog()} is deliberately left untouched: it has no default, because the + * database either has no catalog concept or the catalog was chosen explicitly, and guessing one + * would write the journal into the wrong database.

+ * + *

Callers should use this instead of relying on {@code null} meaning "default", so the + * effective configuration can be logged and stored verbatim. After this call every field except + * {@code catalog} is non null.

+ */ + public TriggerCdcConfig withDefaults() { + if (schema == null) { + schema = TriggerCdcConstants.DEFAULT_STORAGE_SCHEMA; + } + if (installMode == null) { + installMode = TriggerCdcInstallMode.AUTO; + } + if (pollIntervalMs == null) { + pollIntervalMs = TriggerCdcConstants.DEFAULT_POLL_INTERVAL_MS; + } + if (batchSize == null) { + batchSize = TriggerCdcConstants.DEFAULT_BATCH_SIZE; + } + if (captureImage == null) { + captureImage = TriggerCdcCaptureImage.FULL; + } + if (ddlMode == null) { + ddlMode = TriggerCdcDdlMode.DETECT_ONLY; + } + if (errorPolicy == null) { + errorPolicy = TriggerCdcErrorPolicy.STRICT; + } + if (retentionSeconds == null) { + retentionSeconds = TriggerCdcConstants.DEFAULT_RETENTION_SECONDS; + } + if (forceCleanUnacknowledged == null) { + forceCleanUnacknowledged = Boolean.FALSE; + } + if (captureTruncate == null) { + captureTruncate = Boolean.FALSE; + } + return this; + } + + public String getCatalog() { + return catalog; + } + + public void setCatalog(String catalog) { + this.catalog = catalog; + } + + public TriggerCdcConfig catalog(String catalog) { + this.catalog = catalog; + return this; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public TriggerCdcConfig schema(String schema) { + this.schema = schema; + return this; + } + + public TriggerCdcInstallMode getInstallMode() { + return installMode; + } + + public void setInstallMode(TriggerCdcInstallMode installMode) { + this.installMode = installMode; + } + + public TriggerCdcConfig installMode(TriggerCdcInstallMode installMode) { + this.installMode = installMode; + return this; + } + + public Long getPollIntervalMs() { + return pollIntervalMs; + } + + public void setPollIntervalMs(Long pollIntervalMs) { + this.pollIntervalMs = pollIntervalMs; + } + + public TriggerCdcConfig pollIntervalMs(Long pollIntervalMs) { + this.pollIntervalMs = pollIntervalMs; + return this; + } + + public Integer getBatchSize() { + return batchSize; + } + + public void setBatchSize(Integer batchSize) { + this.batchSize = batchSize; + } + + public TriggerCdcConfig batchSize(Integer batchSize) { + this.batchSize = batchSize; + return this; + } + + public TriggerCdcCaptureImage getCaptureImage() { + return captureImage; + } + + public void setCaptureImage(TriggerCdcCaptureImage captureImage) { + this.captureImage = captureImage; + } + + public TriggerCdcConfig captureImage(TriggerCdcCaptureImage captureImage) { + this.captureImage = captureImage; + return this; + } + + public TriggerCdcDdlMode getDdlMode() { + return ddlMode; + } + + public void setDdlMode(TriggerCdcDdlMode ddlMode) { + this.ddlMode = ddlMode; + } + + public TriggerCdcConfig ddlMode(TriggerCdcDdlMode ddlMode) { + this.ddlMode = ddlMode; + return this; + } + + public TriggerCdcErrorPolicy getErrorPolicy() { + return errorPolicy; + } + + public void setErrorPolicy(TriggerCdcErrorPolicy errorPolicy) { + this.errorPolicy = errorPolicy; + } + + public TriggerCdcConfig errorPolicy(TriggerCdcErrorPolicy errorPolicy) { + this.errorPolicy = errorPolicy; + return this; + } + + public Long getRetentionSeconds() { + return retentionSeconds; + } + + public void setRetentionSeconds(Long retentionSeconds) { + this.retentionSeconds = retentionSeconds; + } + + public TriggerCdcConfig retentionSeconds(Long retentionSeconds) { + this.retentionSeconds = retentionSeconds; + return this; + } + + public Boolean getForceCleanUnacknowledged() { + return forceCleanUnacknowledged; + } + + public void setForceCleanUnacknowledged(Boolean forceCleanUnacknowledged) { + this.forceCleanUnacknowledged = forceCleanUnacknowledged; + } + + public TriggerCdcConfig forceCleanUnacknowledged(Boolean forceCleanUnacknowledged) { + this.forceCleanUnacknowledged = forceCleanUnacknowledged; + return this; + } + + /** + * Null safe reading of {@link #getForceCleanUnacknowledged()}: an unset value means + * "not opted in", never "the connector may decide". + */ + public boolean forceCleanUnacknowledgedEnabled() { + return Boolean.TRUE.equals(forceCleanUnacknowledged); + } + + public Boolean getCaptureTruncate() { + return captureTruncate; + } + + public void setCaptureTruncate(Boolean captureTruncate) { + this.captureTruncate = captureTruncate; + } + + public TriggerCdcConfig captureTruncate(Boolean captureTruncate) { + this.captureTruncate = captureTruncate; + return this; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConstants.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConstants.java new file mode 100644 index 00000000..663c68e3 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConstants.java @@ -0,0 +1,69 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +/** + * Names and defaults shared by the framework and every Trigger CDC connector. + * + *

These values are the protocol, not a database specific detail: the framework uses them to + * talk about the CDC storage layout, the connector uses them when it materialises the objects. + * A connector may only deviate from the names when the database cannot represent them, and it + * must then report the real names back through {@link TriggerCdcObject#getName()}.

+ */ +public final class TriggerCdcConstants { + private TriggerCdcConstants() { + } + + /** + * Capability id the framework exposes when a connector registered the Trigger CDC + * functions. Manager / UI can use it as a single flag to show the mode at all. + */ + public static final String CAPABILITY_TRIGGER_CDC = "trigger_cdc"; + + /** + * Capability ids are derived from the {@link io.tapdata.pdk.apis.functions.ConnectorFunctions} + * field names. They are listed here so callers do not hard code strings. + */ + public static final String CAPABILITY_TRIGGER_CDC_CAPABILITIES = "trigger_cdc_capabilities_function"; + public static final String CAPABILITY_TRIGGER_CDC_PLAN = "trigger_cdc_plan_function"; + public static final String CAPABILITY_TRIGGER_CDC_ENSURE = "trigger_cdc_ensure_function"; + public static final String CAPABILITY_TRIGGER_CDC_READ = "trigger_cdc_read_function"; + public static final String CAPABILITY_TRIGGER_CDC_ACKNOWLEDGE = "trigger_cdc_acknowledge_function"; + public static final String CAPABILITY_TRIGGER_CDC_INSPECT = "trigger_cdc_inspect_function"; + public static final String CAPABILITY_TRIGGER_CDC_CLEANUP = "trigger_cdc_cleanup_function"; + public static final String CAPABILITY_TRIGGER_CDC_RELEASE = "trigger_cdc_release_function"; + + /** Default CDC storage namespace when the user did not pick one. */ + public static final String DEFAULT_STORAGE_SCHEMA = "cdc"; + + /** Registry of installed captures. */ + public static final String OBJECT_CAPTURE_REGISTRY = "tap_capture_registry"; + /** Immutable table structure versions. */ + public static final String OBJECT_SCHEMA_HISTORY = "tap_schema_history"; + /** Per source table journal, suffix is appended to this prefix. */ + public static final String OBJECT_CHANGE_TABLE_PREFIX = "tap_changes_"; + /** Per consumer acknowledgement of delivered events. */ + public static final String OBJECT_CONSUMER_ACK = "tap_consumer_ack"; + /** Optional capture error bucket. */ + public static final String OBJECT_CAPTURE_ERROR = "tap_capture_errors"; + + /** Default polling interval of the journal reader. To be tuned by load testing. */ + public static final long DEFAULT_POLL_INTERVAL_MS = 500L; + /** Default batch size of the journal reader. To be tuned by load testing. */ + public static final int DEFAULT_BATCH_SIZE = 1000; + /** + * Default journal retention, matching a 7 day operational window. + * + *

Retention applies to acknowledged records only. It is not a licence to delete a record a + * live consumer still needs: that requires the explicit + * {@link TriggerCdcConfig#getForceCleanUnacknowledged()} opt-in.

+ */ + public static final long DEFAULT_RETENTION_SECONDS = 7L * 24L * 60L * 60L; + + /** + * Version of the journal payload encoding. Written next to every change record so that a + * later reader can decode records written by an older connector build. + */ + public static final int PAYLOAD_VERSION = 1; + + /** Database name prefix of the generated trigger, before the source table name. */ + public static final String TRIGGER_NAME_PREFIX = "tap_trg_"; +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcDdlMode.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcDdlMode.java new file mode 100644 index 00000000..b4f623a1 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcDdlMode.java @@ -0,0 +1,35 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +/** + * How structural changes of the captured source tables are handled. + * + *

"This database supports triggers" does not imply "this connector supports DDL capture". + * Each connector declares which levels it can honour in + * {@link TriggerCdcCapabilities#getDdlModes()}.

+ */ +public enum TriggerCdcDdlMode { + /** + * DDL is captured by a native database facility (for example a PostgreSQL event trigger). + * The adapter must declare exactly which DDL statements are observable. + */ + NATIVE_DDL, + /** + * DDL goes through a coordinated workflow: register a new schema version, rebuild the + * trigger, then switch consumption from the old to the new schema version behind a table + * level barrier. + * + *

DML arriving inside the coordination window is buffered; exceeding the buffer means + * changes may have been dropped, so the capture must be marked + * {@link TriggerCdcStatus#REPAIR_REQUIRED} instead of pretending the switch was lossless.

+ */ + MANAGED_DDL, + /** + * Default for the first release. The framework periodically compares the live table + * structure and the installed trigger set against the registered schema version, and + * alarms on drift. + * + *

Detection cannot reconstruct the DDL history between two checks, so it never claims + * lossless DDL replication.

+ */ + DETECT_ONLY +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcErrorPolicy.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcErrorPolicy.java new file mode 100644 index 00000000..48840e05 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcErrorPolicy.java @@ -0,0 +1,24 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +/** + * What happens when the trigger fails to append the change record into the CDC journal. + * + *

"Never block the business" and "never lose a change" cannot both be guaranteed by a + * plain trigger, because the trigger body normally runs inside the business transaction. + * This enum makes the trade off explicit instead of hiding it.

+ */ +public enum TriggerCdcErrorPolicy { + /** + * Default, and the only policy a connector must implement. If the CDC journal write + * fails, the business statement fails as well, so the capture stays complete. + */ + STRICT, + /** + * The business statement is allowed to continue, so changes may be lost. A task using this + * policy must be marked {@link TriggerCdcStatus#REPAIR_REQUIRED} — not merely + * {@link TriggerCdcStatus#DEGRADED}, because a loss is not a state that operator attention + * alone can fix — and requires a re-check or a full resync. Only exposed when the connector + * declares it in {@link TriggerCdcCapabilities#getErrorPolicies()}. + */ + BEST_EFFORT +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcEvent.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcEvent.java new file mode 100644 index 00000000..c3c0afe0 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcEvent.java @@ -0,0 +1,281 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; +import java.util.Map; + +/** + * One decoded change record read from the CDC journal. + * + *

The connector decodes the journal payload (which it wrote itself inside the trigger) into + * plain Java maps, so the engine can map it onto standard Tap DML / DDL events without knowing + * the database payload format.

+ * + *

Decoding rules that matter for correctness:

+ *
    + *
  • Decimal and big integer values must keep their precision and scale.
  • + *
  • Binary, timestamp and time zone values need an explicit, documented encoding.
  • + *
  • NULL, missing column and "not captured" must stay distinguishable.
  • + *
  • Values are the ones observed at trigger time; never re-read the source row later, + * because it may already have been updated again.
  • + *
+ */ +public class TriggerCdcEvent implements Serializable { + private static final long serialVersionUID = 1L; + + /** + * Unique id of the change record. Used for acknowledgement and de-duplication. + * + *

It does not imply commit order, see {@link TriggerCdcOffset}.

+ */ + protected String eventId; + /** + * Capture this record belongs to. Inside a read batch it always equals the {@code captureId} + * the caller selected, so an event can never be mistakenly acked against another capture. + */ + protected String captureId; + /** Generation of the capture when the record was written. */ + protected Long generation; + /** Stable id of the source table. */ + protected String sourceTableId; + protected String sourceTableName; + protected TriggerCdcOperation operation; + /** Schema version the payload was encoded with. */ + protected Long schemaVersion; + /** Primary key before the change, null for INSERT. */ + protected Map keyBefore; + /** Primary key after the change, null for DELETE. Differs from keyBefore on key updates. */ + protected Map keyAfter; + /** Full row before the change, when the capture image provides it. */ + protected Map before; + /** Full row after the change, when the capture image provides it. */ + protected Map after; + /** Source transaction id, only when {@code transactionIdSupported}. */ + protected String transactionId; + /** Position inside the source transaction, only when {@code transactionSequenceSupported}. */ + protected Long transactionSeq; + /** Capture time in epoch millis, for observation only, not a safety cursor. */ + protected Long capturedAt; + /** Raw or normalized DDL text, only for {@link TriggerCdcOperation#DDL} events. */ + protected String ddlPayload; + /** Version of the payload encoding. */ + protected Integer payloadVersion; + + public static TriggerCdcEvent create() { + return new TriggerCdcEvent(); + } + + public String getEventId() { + return eventId; + } + + public void setEventId(String eventId) { + this.eventId = eventId; + } + + public TriggerCdcEvent eventId(String eventId) { + this.eventId = eventId; + return this; + } + + public String getCaptureId() { + return captureId; + } + + public void setCaptureId(String captureId) { + this.captureId = captureId; + } + + public TriggerCdcEvent captureId(String captureId) { + this.captureId = captureId; + return this; + } + + public Long getGeneration() { + return generation; + } + + public void setGeneration(Long generation) { + this.generation = generation; + } + + public TriggerCdcEvent generation(Long generation) { + this.generation = generation; + return this; + } + + public String getSourceTableId() { + return sourceTableId; + } + + public void setSourceTableId(String sourceTableId) { + this.sourceTableId = sourceTableId; + } + + public TriggerCdcEvent sourceTableId(String sourceTableId) { + this.sourceTableId = sourceTableId; + return this; + } + + public String getSourceTableName() { + return sourceTableName; + } + + public void setSourceTableName(String sourceTableName) { + this.sourceTableName = sourceTableName; + } + + public TriggerCdcEvent sourceTableName(String sourceTableName) { + this.sourceTableName = sourceTableName; + return this; + } + + public TriggerCdcOperation getOperation() { + return operation; + } + + public void setOperation(TriggerCdcOperation operation) { + this.operation = operation; + } + + public TriggerCdcEvent operation(TriggerCdcOperation operation) { + this.operation = operation; + return this; + } + + public Long getSchemaVersion() { + return schemaVersion; + } + + public void setSchemaVersion(Long schemaVersion) { + this.schemaVersion = schemaVersion; + } + + public TriggerCdcEvent schemaVersion(Long schemaVersion) { + this.schemaVersion = schemaVersion; + return this; + } + + public Map getKeyBefore() { + return keyBefore; + } + + public void setKeyBefore(Map keyBefore) { + this.keyBefore = keyBefore; + } + + public TriggerCdcEvent keyBefore(Map keyBefore) { + this.keyBefore = keyBefore; + return this; + } + + public Map getKeyAfter() { + return keyAfter; + } + + public void setKeyAfter(Map keyAfter) { + this.keyAfter = keyAfter; + } + + public TriggerCdcEvent keyAfter(Map keyAfter) { + this.keyAfter = keyAfter; + return this; + } + + public Map getBefore() { + return before; + } + + public void setBefore(Map before) { + this.before = before; + } + + public TriggerCdcEvent before(Map before) { + this.before = before; + return this; + } + + public Map getAfter() { + return after; + } + + public void setAfter(Map after) { + this.after = after; + } + + public TriggerCdcEvent after(Map after) { + this.after = after; + return this; + } + + public String getTransactionId() { + return transactionId; + } + + public void setTransactionId(String transactionId) { + this.transactionId = transactionId; + } + + public TriggerCdcEvent transactionId(String transactionId) { + this.transactionId = transactionId; + return this; + } + + public Long getTransactionSeq() { + return transactionSeq; + } + + public void setTransactionSeq(Long transactionSeq) { + this.transactionSeq = transactionSeq; + } + + public TriggerCdcEvent transactionSeq(Long transactionSeq) { + this.transactionSeq = transactionSeq; + return this; + } + + public Long getCapturedAt() { + return capturedAt; + } + + public void setCapturedAt(Long capturedAt) { + this.capturedAt = capturedAt; + } + + public TriggerCdcEvent capturedAt(Long capturedAt) { + this.capturedAt = capturedAt; + return this; + } + + public String getDdlPayload() { + return ddlPayload; + } + + public void setDdlPayload(String ddlPayload) { + this.ddlPayload = ddlPayload; + } + + public TriggerCdcEvent ddlPayload(String ddlPayload) { + this.ddlPayload = ddlPayload; + return this; + } + + public Integer getPayloadVersion() { + return payloadVersion; + } + + public void setPayloadVersion(Integer payloadVersion) { + this.payloadVersion = payloadVersion; + } + + public TriggerCdcEvent payloadVersion(Integer payloadVersion) { + this.payloadVersion = payloadVersion; + return this; + } + + @Override + public String toString() { + return "TriggerCdcEvent{eventId=" + eventId + ", captureId=" + captureId + ", generation=" + generation + + ", sourceTableName=" + sourceTableName + ", operation=" + operation + + ", schemaVersion=" + schemaVersion + ", capturedAt=" + capturedAt + "}"; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInspectResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInspectResult.java new file mode 100644 index 00000000..c51ab1de --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInspectResult.java @@ -0,0 +1,207 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * Health report of one capture, produced by + * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcInspectFunction}. + * + *

The report drives the DEGRADED / REPAIR_REQUIRED states. It is also the only honest way to + * report a structure drift: metadata polling can detect that a table changed, but it cannot + * reconstruct the DDL history, so the framework must degrade instead of guessing.

+ */ +public class TriggerCdcInspectResult implements Serializable { + private static final long serialVersionUID = 1L; + + protected TriggerCdcStatus status; + protected String captureId; + protected Long generation; + protected String sourceTableId; + protected String sourceTableName; + protected Long schemaVersion; + /** State of the trigger, the journal table and the registry entries. */ + protected List objects; + /** True when the live table structure no longer matches the registered schema version. */ + protected boolean schemaDrift; + /** Columns added or dropped since the registered version, when drift was detected. */ + protected List driftedColumns; + /** + * True when a change may have been lost, for example a disabled trigger window, a + * BEST_EFFORT write failure, an overflowed managed DDL buffer, or a forced retention clean + * that removed records a consumer had not acknowledged (see + * {@link TriggerCdcCleanupResult#getIntegrityBrokenConsumers()}). + */ + protected boolean dataIntegrityBroken; + protected String integrityMessage; + /** Number of change records not yet acknowledged by any consumer. */ + protected Long unackedCount; + /** Capture time of the oldest unacknowledged record, for lag monitoring. */ + protected Long oldestUnackedCapturedAt; + /** Bytes the journal occupies, when the database can report it cheaply. */ + protected Long journalBytes; + protected List warnings; + protected List errors; + + public static TriggerCdcInspectResult create() { + return new TriggerCdcInspectResult(); + } + + public TriggerCdcInspectResult object(TriggerCdcObjectState objectState) { + if (objects == null) { + objects = new ArrayList<>(); + } + objects.add(objectState); + return this; + } + + public TriggerCdcInspectResult warning(String warning) { + if (warnings == null) { + warnings = new ArrayList<>(); + } + warnings.add(warning); + return this; + } + + public TriggerCdcInspectResult error(String error) { + if (errors == null) { + errors = new ArrayList<>(); + } + errors.add(error); + return this; + } + + public TriggerCdcStatus getStatus() { + return status; + } + + public void setStatus(TriggerCdcStatus status) { + this.status = status; + } + + public TriggerCdcInspectResult status(TriggerCdcStatus status) { + this.status = status; + return this; + } + + public String getCaptureId() { + return captureId; + } + + public void setCaptureId(String captureId) { + this.captureId = captureId; + } + + public Long getGeneration() { + return generation; + } + + public void setGeneration(Long generation) { + this.generation = generation; + } + + public String getSourceTableId() { + return sourceTableId; + } + + public void setSourceTableId(String sourceTableId) { + this.sourceTableId = sourceTableId; + } + + public String getSourceTableName() { + return sourceTableName; + } + + public void setSourceTableName(String sourceTableName) { + this.sourceTableName = sourceTableName; + } + + public Long getSchemaVersion() { + return schemaVersion; + } + + public void setSchemaVersion(Long schemaVersion) { + this.schemaVersion = schemaVersion; + } + + public List getObjects() { + return objects; + } + + public void setObjects(List objects) { + this.objects = objects; + } + + public boolean isSchemaDrift() { + return schemaDrift; + } + + public void setSchemaDrift(boolean schemaDrift) { + this.schemaDrift = schemaDrift; + } + + public List getDriftedColumns() { + return driftedColumns; + } + + public void setDriftedColumns(List driftedColumns) { + this.driftedColumns = driftedColumns; + } + + public boolean isDataIntegrityBroken() { + return dataIntegrityBroken; + } + + public void setDataIntegrityBroken(boolean dataIntegrityBroken) { + this.dataIntegrityBroken = dataIntegrityBroken; + } + + public String getIntegrityMessage() { + return integrityMessage; + } + + public void setIntegrityMessage(String integrityMessage) { + this.integrityMessage = integrityMessage; + } + + public Long getUnackedCount() { + return unackedCount; + } + + public void setUnackedCount(Long unackedCount) { + this.unackedCount = unackedCount; + } + + public Long getOldestUnackedCapturedAt() { + return oldestUnackedCapturedAt; + } + + public void setOldestUnackedCapturedAt(Long oldestUnackedCapturedAt) { + this.oldestUnackedCapturedAt = oldestUnackedCapturedAt; + } + + public Long getJournalBytes() { + return journalBytes; + } + + public void setJournalBytes(Long journalBytes) { + this.journalBytes = journalBytes; + } + + public List getWarnings() { + return warnings; + } + + public void setWarnings(List warnings) { + this.warnings = warnings; + } + + public List getErrors() { + return errors; + } + + public void setErrors(List errors) { + this.errors = errors; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallMode.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallMode.java new file mode 100644 index 00000000..113a4f29 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallMode.java @@ -0,0 +1,20 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +/** + * Who executes the generated CDC objects and triggers. + */ +public enum TriggerCdcInstallMode { + /** + * Default. The framework generates the statements and executes them itself, then verifies + * the result. Installation must be idempotent and must never overwrite an object Tapdata + * does not own. + */ + AUTO, + /** + * The framework only generates the statements through + * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcPlanFunction}; a DBA + * installs them and Tapdata verifies the result afterwards. Both paths must produce exactly + * the same object set and the same version. + */ + MANUAL +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallPlan.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallPlan.java new file mode 100644 index 00000000..2821b4c4 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallPlan.java @@ -0,0 +1,177 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * Read only result of {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcPlanFunction}. + * + *

A plan must be side effect free: nothing is created, so the UI can show the cost, the + * privileges and the blocking problems before the operator commits.

+ */ +public class TriggerCdcInstallPlan implements Serializable { + private static final long serialVersionUID = 1L; + + protected String catalog; + protected String schema; + /** Objects that will exist after installation. */ + protected List objects; + /** + * Ordered statements to execute. A plan without statements can never be installed — in + * {@code AUTO} nothing would be created, in {@code MANUAL} the DBA would have nothing to run — + * so an empty list always blocks the plan and must be reported through + * {@link #isBlocked()} with a reason in {@link #getBlockedReason()}. + */ + protected List statements; + /** Tables the connector accepts into this capture. */ + protected List supportedTables; + /** + * Tables the connector refuses, with the reason inside + * {@link TriggerCdcTableResult#getMessage()}. A blocked table fails the plan instead of + * being silently skipped, because a silently skipped table is a silent data gap. + */ + protected List blockedTables; + /** Non blocking remarks: cost expectation, high update rate warnings, existing triggers. */ + protected List warnings; + /** True when installation must not proceed. */ + protected boolean blocked; + /** Why the plan is blocked, when {@link #blocked} is true. */ + protected String blockedReason; + + public static TriggerCdcInstallPlan create() { + return new TriggerCdcInstallPlan(); + } + + public TriggerCdcInstallPlan object(TriggerCdcObject object) { + if (objects == null) { + objects = new ArrayList<>(); + } + objects.add(object); + return this; + } + + public TriggerCdcInstallPlan statement(TriggerCdcStatement statement) { + if (statements == null) { + statements = new ArrayList<>(); + } + statements.add(statement); + return this; + } + + public TriggerCdcInstallPlan warning(String warning) { + if (warnings == null) { + warnings = new ArrayList<>(); + } + warnings.add(warning); + return this; + } + + public TriggerCdcInstallPlan supportedTable(TriggerCdcTable table) { + if (supportedTables == null) { + supportedTables = new ArrayList<>(); + } + supportedTables.add(table); + return this; + } + + public TriggerCdcInstallPlan blockedTable(TriggerCdcTableResult result) { + if (blockedTables == null) { + blockedTables = new ArrayList<>(); + } + blockedTables.add(result); + return this; + } + + public String getCatalog() { + return catalog; + } + + public void setCatalog(String catalog) { + this.catalog = catalog; + } + + public TriggerCdcInstallPlan catalog(String catalog) { + this.catalog = catalog; + return this; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public TriggerCdcInstallPlan schema(String schema) { + this.schema = schema; + return this; + } + + public List getObjects() { + return objects; + } + + public void setObjects(List objects) { + this.objects = objects; + } + + public List getStatements() { + return statements; + } + + public void setStatements(List statements) { + this.statements = statements; + } + + public List getSupportedTables() { + return supportedTables; + } + + public void setSupportedTables(List supportedTables) { + this.supportedTables = supportedTables; + } + + public List getBlockedTables() { + return blockedTables; + } + + public void setBlockedTables(List blockedTables) { + this.blockedTables = blockedTables; + } + + public List getWarnings() { + return warnings; + } + + public void setWarnings(List warnings) { + this.warnings = warnings; + } + + public boolean isBlocked() { + return blocked; + } + + public void setBlocked(boolean blocked) { + this.blocked = blocked; + } + + public TriggerCdcInstallPlan blocked(boolean blocked) { + this.blocked = blocked; + return this; + } + + public String getBlockedReason() { + return blockedReason; + } + + public void setBlockedReason(String blockedReason) { + this.blockedReason = blockedReason; + } + + public TriggerCdcInstallPlan blockedReason(String blockedReason) { + this.blockedReason = blockedReason; + return this; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java new file mode 100644 index 00000000..a39027f2 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java @@ -0,0 +1,482 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Outcome of {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcEnsureFunction}. + * + *

{@code ensure} must be idempotent: running it twice yields the same capture ids and the + * same generation. Only a real reinstall increments the generation.

+ * + *

Fail closed: {@link #getCaptureIds()} is the framework's authoritative read list, so the + * result is checked as data rather than trusted as prose. {@link #validate()} returns one message + * per inconsistency (a success without a capture id, two tables sharing a capture id, a requested + * table with no outcome at all, an outcome for a table nobody asked for, no successful table at + * all) and {@link #getCaptureIds()} throws instead of filtering such entries out. Call + * {@link #isValid()} or {@link #isSuccess()} before using the result.

+ * + *

Completeness needs both sides: a report can only be checked against the request it answers. + * The result therefore carries the requested tables (see {@link #requestedTables(Collection)}), and + * the framework may instead pass the request it actually made to + * {@link #validate(Collection)} / {@link #getCaptureIds(Collection)}. Without a request the capture + * list cannot be proven to cover every table, so an unknown request is itself a violation: the + * silent failure mode this guards against is a connector that reports only the tables it managed to + * install, leaving the rest neither in the capture list nor in the failure list.

+ */ +public class TriggerCdcInstallResult implements Serializable { + private static final long serialVersionUID = 1L; + + protected TriggerCdcStatus status; + protected String catalog; + protected String schema; + /** Per table outcome, in the order the tables were requested. */ + protected List tableResults; + /** + * The tables {@code ensure} was asked for. Kept so {@link #validate()} can prove that every + * requested table has exactly one outcome; without it a report is only self consistent, not + * complete. + */ + protected List requestedTables; + /** Non blocking remarks raised during installation. */ + protected List warnings; + + public static TriggerCdcInstallResult create() { + return new TriggerCdcInstallResult(); + } + + public TriggerCdcInstallResult tableResult(TriggerCdcTableResult tableResult) { + if (tableResults == null) { + tableResults = new ArrayList<>(); + } + tableResults.add(tableResult); + return this; + } + + public TriggerCdcInstallResult warning(String warning) { + if (warnings == null) { + warnings = new ArrayList<>(); + } + warnings.add(warning); + return this; + } + + /** + * Records one table that {@code ensure} was asked to capture. Pair it with the outcome reported + * in {@link #tableResult(TriggerCdcTableResult)}: completeness is checked across the two lists, + * so a requested table without an outcome is rejected instead of being read as "nothing to do". + */ + public TriggerCdcInstallResult requestedTable(TriggerCdcTable table) { + if (requestedTables == null) { + requestedTables = new ArrayList<>(); + } + requestedTables.add(table); + return this; + } + + /** + * Records the full request. {@code ensure} receives this list as an argument, so calling this (or + * letting the framework pass the same list to {@link #validate(Collection)}) is what makes the + * result checkable: one outcome per requested table, no extras, no duplicates. + */ + public TriggerCdcInstallResult requestedTables(Collection tables) { + if (tables == null) { + this.requestedTables = null; + return this; + } + this.requestedTables = new ArrayList<>(tables); + return this; + } + + /** True only when every requested table was installed or reused successfully and the result is + * structurally valid (see {@link #validate()}). */ + public boolean isSuccess() { + if (!isValid()) { + return false; + } + for (TriggerCdcTableResult tableResult : tableResults) { + if (!tableResult.isSuccessReported()) { + return false; + } + } + return true; + } + + /** + * True when this result is structurally usable: every table has an outcome, and every entry + * that claims success carries a non blank, unique capture id. + * + *

Equivalent to {@code validate().isEmpty()}. A partial install (some tables reported as + * {@code failure}) is still a valid result - those tables simply contribute no capture - but a + * result where no table succeeded is a violation, because its capture list would be + * empty and a framework that iterates it would look healthy while reading nothing.

+ */ + public boolean isValid() { + return validate().isEmpty(); + } + + /** + * Checks this result against the rules in this class, using the request recorded through + * {@link #requestedTables(Collection)}, and returns one message per violation; an empty list + * means the result is usable. + * + *

Equivalent to {@code validate(getRequestedTables())}.

+ */ + public List validate() { + return validate(requestedTables); + } + + /** + * Checks this result against the tables the caller actually asked {@code ensure} for, and + * returns one message per violation; an empty list means the result is usable. + * + *

Why the request is required: a report that only lists the tables the connector managed to + * install is self consistent and still wrong. If {@code ensure} is called for A and B and only + * A comes back, every entry looks fine, {@link #getCaptureIds()} would return A alone, and B + * would be captured by nothing - no capture to read, and no failure entry to retry. Only the + * request can tell the difference, so this method requires exactly one outcome per requested + * table (a {@code failure} outcome counts, because it is retryable) and rejects the three ways + * a set can fail to line up: a requested table with no outcome, more than one outcome for the + * same table, and an outcome for a table that was never requested.

+ * + *

The remaining rules are about readability of what is reported: a success must carry a non + * blank capture id, no two tables may share a capture id, and at least one table must have been + * installed, otherwise the capture list would be empty and a framework iterating it would look + * like a healthy no-op install. Both used to be filtered away silently; now they are reported + * here and rejected instead.

+ * + *

The framework must call this (or {@link #isSuccess()}) before trusting the result; the + * check verifies the report against the request, not what the database really did.

+ * + * @param requestedTables the tables handed to {@code ensure}; never {@code null} and never empty + */ + public List validate(Collection requestedTables) { + List violations = new ArrayList<>(); + Map requested = indexRequested(requestedTables, violations); + if (tableResults == null || tableResults.isEmpty()) { + violations.add("ensure reported no per table result; at least one table outcome is " + + "required, otherwise the capture list is unknowable"); + return violations; + } + Map captureOwners = new LinkedHashMap<>(); + Map outcomeOwners = new LinkedHashMap<>(); + List readable = new ArrayList<>(); + for (int i = 0; i < tableResults.size(); i++) { + TriggerCdcTableResult tableResult = tableResults.get(i); + String owner = describe(tableResult, i); + if (tableResult == null) { + violations.add("table result at index " + i + " is null; every requested table needs " + + "an explicit outcome"); + continue; + } + String key = tableKey(tableResult.getSourceTableId(), tableResult.getTableName()); + if (key == null) { + violations.add(owner + " carries neither sourceTableId nor tableName, so its outcome " + + "can be matched to no requested table and no capture can be attributed"); + } else { + String previousOutcome = outcomeOwners.put(key, owner); + if (previousOutcome != null) { + violations.add(owner + " and " + previousOutcome + " are the same table reported " + + "twice; two outcomes for one table make its authoritative capture " + + "unknowable"); + } + if (!requested.isEmpty() && !requested.containsKey(key)) { + violations.add(owner + " was not requested; an outcome for a table outside the " + + "request would add a capture nobody asked the framework to own"); + } + } + if (!tableResult.isSuccessReported()) { + continue; + } + if (!tableResult.hasCaptureId()) { + violations.add(owner + " is reported successful but has no captureId: its journal " + + "cannot be addressed, so it would be installed and never read"); + continue; + } + String captureId = tableResult.getCaptureId().trim(); + String previousOwner = captureOwners.put(captureId, owner); + if (previousOwner != null) { + violations.add("captureId " + captureId + " is claimed by both " + previousOwner + + " and " + owner + "; one capture id must address exactly one table"); + } + readable.add(owner); + } + for (Map.Entry entry : requested.entrySet()) { + if (!outcomeOwners.containsKey(entry.getKey())) { + violations.add(entry.getValue() + " has no outcome in the result: a requested table " + + "that is neither reported successful nor reported failed is silently " + + "uninstalled, and nothing in the capture list or in getFailedTables() " + + "catches it"); + } + } + if (readable.isEmpty()) { + violations.add("no table was installed successfully, so there is no capture to read; " + + "see getFailedTables() for the reasons"); + } + return violations; + } + + /** + * Indexes the request by table identity and reports the ways the request itself is unusable. + * + *

An unknown request (null or empty) is a violation, not a free pass: with no request there + * is nothing to prove the report complete against, and "cannot check" must not read as "fine". + * The same applies to a requested table with no stable identity - no outcome could ever be + * matched to it.

+ */ + private static Map indexRequested(Collection requestedTables, + List violations) { + Map requested = new LinkedHashMap<>(); + if (requestedTables == null || requestedTables.isEmpty()) { + violations.add("the set of requested tables is unknown, so the report cannot be checked " + + "for completeness: a requested table that is simply absent from the result " + + "would look identical to one that was never asked for. Record the request with " + + "requestedTables(...), or pass it to validate(requestedTables)"); + return requested; + } + int index = 0; + for (TriggerCdcTable table : requestedTables) { + String owner = describeRequested(table, index); + if (table == null) { + violations.add(owner + " is null; a request must name the tables it asks for"); + index++; + continue; + } + String key = tableKey(table.getSourceTableId(), table.getTableName()); + if (key == null) { + violations.add(owner + " has neither sourceTableId nor tableName, so no outcome can " + + "be matched to it"); + index++; + continue; + } + String previous = requested.put(key, owner); + if (previous != null) { + violations.add(owner + " and " + previous + " are the same table; the request lists " + + "it twice, so one capture would be reported for two requested tables"); + } + index++; + } + return requested; + } + + /** + * Identity of a table for matching an outcome to the request: the stable id when the connector + * echoes it, the physical name as a fallback. The two spaces are prefixed differently so an id + * that happens to look like a table name cannot be mistaken for one. + */ + private static String tableKey(String sourceTableId, String tableName) { + if (sourceTableId != null && !sourceTableId.trim().isEmpty()) { + return "id:" + sourceTableId.trim(); + } + if (tableName != null && !tableName.trim().isEmpty()) { + return "name:" + tableName.trim(); + } + return null; + } + + private static String describeRequested(TriggerCdcTable table, int index) { + if (table == null) { + return "requested table at index " + index; + } + StringBuilder text = new StringBuilder("requested table ") + .append(emptyToIndex(table.getTableName(), index)); + if (table.getSourceTableId() != null && !table.getSourceTableId().isEmpty()) { + text.append(" (sourceTableId ").append(table.getSourceTableId()).append(")"); + } + return text.toString(); + } + + private static String emptyToIndex(String tableName, int index) { + return tableName == null || tableName.isEmpty() ? "#" + index : tableName; + } + + private static String describe(TriggerCdcTableResult tableResult, int index) { + if (tableResult == null) { + return "table result at index " + index; + } + StringBuilder text = new StringBuilder("table ") + .append(emptyToIndex(tableResult.getTableName(), index)); + if (tableResult.getSourceTableId() != null && !tableResult.getSourceTableId().isEmpty()) { + text.append(" (sourceTableId ").append(tableResult.getSourceTableId()).append(")"); + } + return text.toString(); + } + + /** + * Requested tables that have no outcome at all in this result. + * + *

Non empty means {@code ensure} skipped part of the request: those tables are not in the + * capture list (so they will not be read) and not in {@link #getFailedTables()} (so nobody will + * retry them). That is why {@link #validate()} treats it as a violation rather than a warning.

+ */ + public List getMissingTables() { + List missing = new ArrayList<>(); + if (requestedTables == null || requestedTables.isEmpty()) { + return missing; + } + Map reported = reportedTables(); + for (TriggerCdcTable table : requestedTables) { + String key = table == null ? null : tableKey(table.getSourceTableId(), table.getTableName()); + if (key != null && !reported.containsKey(key)) { + missing.add(table); + } + } + return missing; + } + + private Map reportedTables() { + Map reported = new LinkedHashMap<>(); + if (tableResults == null) { + return reported; + } + for (int i = 0; i < tableResults.size(); i++) { + TriggerCdcTableResult tableResult = tableResults.get(i); + if (tableResult == null) { + continue; + } + String key = tableKey(tableResult.getSourceTableId(), tableResult.getTableName()); + if (key != null) { + reported.put(key, describe(tableResult, i)); + } + } + return reported; + } + + /** + * Tables that are not usable: outright failures, plus entries that claim success but carry no + * capture id (those are installed but unreadable, so they are reported here to be fixed rather + * than dropped from {@link #getCaptureIds()}). + * + *

Requested tables with no outcome at all cannot appear here - there is no entry to return - + * so they are exposed by {@link #getMissingTables()} and reported by {@link #validate()}.

+ */ + public List getFailedTables() { + List failed = new ArrayList<>(); + if (tableResults != null) { + for (TriggerCdcTableResult tableResult : tableResults) { + if (tableResult != null && !tableResult.isSuccess()) { + failed.add(tableResult); + } + } + } + return failed; + } + + /** + * Capture ids of the successfully installed tables, in request order. + * + *

This is the authoritative capture list a multi table task iterates: {@code read} and + * {@code acknowledge} work on one capture at a time (one source table is one capture), so the + * framework keeps one {@code TriggerCdcOffset} per entry of this list and per consumer. Failed + * tables contribute no capture id, which is why a partial install must be read as "these + * captures exist", not "the requested set exists".

+ * + *

Fail closed: nothing is skipped and nothing is deduplicated. A successful table without a + * capture id, two tables sharing one capture id, an outcome for a table that was not requested, + * or a requested table with no outcome at all makes the whole list untrustworthy - returning the + * rest would silently leave a table unread - so this method throws instead of filtering. An + * empty list is never returned either: when every table failed there is nothing to read, and a + * silently empty capture list would look like "installed, nothing to do". Both conditions are + * reported by {@link #validate()}.

+ * + *

Completeness is checked against the request recorded on this result. The framework normally + * also has the request it just made, so it should call + * {@link #getCaptureIds(Collection)} with that list: it is the authoritative answer to "what did + * we ask for", and it cannot be forged by an incomplete report.

+ * + * @throws IllegalStateException when {@link #validate()} reports a violation + */ + public List getCaptureIds() { + return getCaptureIds(requestedTables); + } + + /** + * Same as {@link #getCaptureIds()}, but checks completeness against the request the framework + * actually made rather than the one recorded on the result. + * + * @param requestedTables the tables handed to {@code ensure}; never {@code null} and never empty + * @throws IllegalStateException when {@link #validate(Collection)} reports a violation + */ + public List getCaptureIds(Collection requestedTables) { + List violations = validate(requestedTables); + if (!violations.isEmpty()) { + throw new IllegalStateException("install result rejected, its capture list is not " + + "trustworthy: " + String.join("; ", violations)); + } + List captureIds = new ArrayList<>(tableResults.size()); + for (TriggerCdcTableResult tableResult : tableResults) { + if (tableResult.isSuccessReported()) { + captureIds.add(tableResult.getCaptureId().trim()); + } + } + return captureIds; + } + + public TriggerCdcStatus getStatus() { + return status; + } + + public void setStatus(TriggerCdcStatus status) { + this.status = status; + } + + public TriggerCdcInstallResult status(TriggerCdcStatus status) { + this.status = status; + return this; + } + + public String getCatalog() { + return catalog; + } + + public void setCatalog(String catalog) { + this.catalog = catalog; + } + + public TriggerCdcInstallResult catalog(String catalog) { + this.catalog = catalog; + return this; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public TriggerCdcInstallResult schema(String schema) { + this.schema = schema; + return this; + } + + public List getTableResults() { + return tableResults; + } + + public void setTableResults(List tableResults) { + this.tableResults = tableResults; + } + + /** Tables {@code ensure} was asked for; see {@link #requestedTables(Collection)}. */ + public List getRequestedTables() { + return requestedTables; + } + + public void setRequestedTables(List requestedTables) { + this.requestedTables = requestedTables; + } + + public List getWarnings() { + return warnings; + } + + public void setWarnings(List warnings) { + this.warnings = warnings; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObject.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObject.java new file mode 100644 index 00000000..dfe497ee --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObject.java @@ -0,0 +1,132 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; + +/** + * A database object that a capture owns or touches. + * + *

{@code ownedByTapdata} is the guard rail for uninstall: the framework may only drop + * objects it created itself, and must never drop an entire customer schema.

+ */ +public class TriggerCdcObject implements Serializable { + private static final long serialVersionUID = 1L; + + protected TriggerCdcObjectType type; + protected String catalog; + protected String schema; + /** Physical object name, as the connector will really create it. */ + protected String name; + /** + * True when this object is created and owned by Tapdata, so releasing may drop it. + * + *

Defaults to {@code false} on purpose: the guard rail has to fail closed. A connector that + * forgets to set the flag leaves an existing customer object untouchable instead of + * accidentally licensing a {@code DROP} on it.

+ */ + protected boolean ownedByTapdata; + /** True when the object already exists and will be reused instead of created. */ + protected boolean alreadyExists; + /** Human readable note, for example why a name differs from the suggested one. */ + protected String comment; + + public static TriggerCdcObject create() { + return new TriggerCdcObject(); + } + + public TriggerCdcObjectType getType() { + return type; + } + + public void setType(TriggerCdcObjectType type) { + this.type = type; + } + + public TriggerCdcObject type(TriggerCdcObjectType type) { + this.type = type; + return this; + } + + public String getCatalog() { + return catalog; + } + + public void setCatalog(String catalog) { + this.catalog = catalog; + } + + public TriggerCdcObject catalog(String catalog) { + this.catalog = catalog; + return this; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public TriggerCdcObject schema(String schema) { + this.schema = schema; + return this; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public TriggerCdcObject name(String name) { + this.name = name; + return this; + } + + public boolean isOwnedByTapdata() { + return ownedByTapdata; + } + + public void setOwnedByTapdata(boolean ownedByTapdata) { + this.ownedByTapdata = ownedByTapdata; + } + + public TriggerCdcObject ownedByTapdata(boolean ownedByTapdata) { + this.ownedByTapdata = ownedByTapdata; + return this; + } + + public boolean isAlreadyExists() { + return alreadyExists; + } + + public void setAlreadyExists(boolean alreadyExists) { + this.alreadyExists = alreadyExists; + } + + public TriggerCdcObject alreadyExists(boolean alreadyExists) { + this.alreadyExists = alreadyExists; + return this; + } + + public String getComment() { + return comment; + } + + public void setComment(String comment) { + this.comment = comment; + } + + public TriggerCdcObject comment(String comment) { + this.comment = comment; + return this; + } + + @Override + public String toString() { + return "TriggerCdcObject{type=" + type + ", schema=" + schema + ", name=" + name + + ", ownedByTapdata=" + ownedByTapdata + ", alreadyExists=" + alreadyExists + "}"; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectState.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectState.java new file mode 100644 index 00000000..6cff5919 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectState.java @@ -0,0 +1,111 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; + +/** + * Live state of one object a capture depends on, as observed by + * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcInspectFunction}. + */ +public class TriggerCdcObjectState implements Serializable { + private static final long serialVersionUID = 1L; + + protected TriggerCdcObjectType type; + protected String schema; + protected String name; + /** False when the object is missing, which breaks the capture. */ + protected boolean present; + /** + * For triggers: false when the trigger exists but is disabled, so changes silently stop + * being journaled. + */ + protected boolean enabled; + protected String message; + + public static TriggerCdcObjectState create() { + return new TriggerCdcObjectState(); + } + + public TriggerCdcObjectType getType() { + return type; + } + + public void setType(TriggerCdcObjectType type) { + this.type = type; + } + + public TriggerCdcObjectState type(TriggerCdcObjectType type) { + this.type = type; + return this; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public TriggerCdcObjectState schema(String schema) { + this.schema = schema; + return this; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public TriggerCdcObjectState name(String name) { + this.name = name; + return this; + } + + public boolean isPresent() { + return present; + } + + public void setPresent(boolean present) { + this.present = present; + } + + public TriggerCdcObjectState present(boolean present) { + this.present = present; + return this; + } + + public boolean isEnabled() { + return enabled; + } + + public void setEnabled(boolean enabled) { + this.enabled = enabled; + } + + public TriggerCdcObjectState enabled(boolean enabled) { + this.enabled = enabled; + return this; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public TriggerCdcObjectState message(String message) { + this.message = message; + return this; + } + + @Override + public String toString() { + return "TriggerCdcObjectState{type=" + type + ", name=" + name + ", present=" + present + + ", enabled=" + enabled + ", message=" + message + "}"; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectType.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectType.java new file mode 100644 index 00000000..2f601927 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectType.java @@ -0,0 +1,25 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +/** + * The kinds of database objects a capture needs. + * + *

Object names are suggestions owned by the framework + * (see {@link TriggerCdcConstants}); the connector maps them onto real database + * semantics (catalog / schema / database / keyspace...).

+ */ +public enum TriggerCdcObjectType { + /** The namespace (schema / database) that holds the CDC objects. */ + STORAGE_NAMESPACE, + /** Registry of installed captures: source table identity, trigger names, generation, state. */ + CAPTURE_REGISTRY, + /** Immutable history of registered table structure versions. */ + SCHEMA_HISTORY, + /** Per source table journal of captured changes. */ + CHANGE_TABLE, + /** Per consumer persistence of acknowledged events. */ + CONSUMER_ACK, + /** Optional bucket for capture errors the database can isolate safely. */ + CAPTURE_ERROR, + /** The trigger (or trigger set) on the source table itself. */ + TRIGGER +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcOffset.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcOffset.java new file mode 100644 index 00000000..8439a045 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcOffset.java @@ -0,0 +1,139 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; + +/** + * Reader position of one consumer on one capture. + * + *

Scope: the framework stores one offset per {@code (consumerId, captureId)} pair, never one + * offset per task. One source table is one capture and owns its own journal, so a task that + * captures several tables has several independent positions, and each + * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcReadFunction read} call is + * given the offset of the capture it reads. There is no cross capture offset: comparing two + * captures' tokens or reusing one capture's offset for another is meaningless and must be + * rejected.

+ * + *

Important: {@code lastEventId} is informational only. A database sequence gives unique + * values, not commit order, so {@code event_id > lastEventId} is not a safe cursor:

+ * + *
+ * tx A allocates event_id=100, not yet committed
+ * tx B allocates event_id=101, commits first
+ * reader sees 101, advances the cursor to 101
+ * tx A commits 100 -> 100 is now permanently skipped
+ * 
+ * + *

The safe cursor is {@code batchToken}: the connector returns it with a delivered batch and + * the framework only advances it through + * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcAcknowledgeFunction} after + * the downstream sink is durable. A connector must therefore treat an unacknowledged batch as + * still pending on every read, even when a larger event id is already visible.

+ */ +public class TriggerCdcOffset implements Serializable { + private static final long serialVersionUID = 1L; + + /** + * Capture this offset belongs to; also the key under which it is persisted. It must equal the + * {@code captureId} passed to a read call - a mismatch means the checkpoint of another capture + * was restored and must be rejected instead of resumed. + */ + protected String captureId; + /** + * Generation of the capture. A connector must reject an offset whose generation does not + * match the installed capture, instead of resuming from a stale position. + */ + protected Long generation; + /** + * Opaque token of the last acknowledged batch. A token that was delivered but not yet + * acknowledged is never stored here: it lives only in the returned + * {@link TriggerCdcBatch}, so the connector re-delivers that batch until + * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcAcknowledgeFunction} + * reports the downstream write as durable. + */ + protected String batchToken; + /** Informational only. Never use it as a safe cursor, see the class javadoc. */ + protected String lastEventId; + /** Informational only, for lag monitoring. */ + protected Long lastCapturedAt; + + public static TriggerCdcOffset create() { + return new TriggerCdcOffset(); + } + + /** True when nothing has been acknowledged yet, so the connector starts from the beginning. */ + public boolean isFresh() { + return batchToken == null || batchToken.isEmpty(); + } + + public String getCaptureId() { + return captureId; + } + + public void setCaptureId(String captureId) { + this.captureId = captureId; + } + + public TriggerCdcOffset captureId(String captureId) { + this.captureId = captureId; + return this; + } + + public Long getGeneration() { + return generation; + } + + public void setGeneration(Long generation) { + this.generation = generation; + } + + public TriggerCdcOffset generation(Long generation) { + this.generation = generation; + return this; + } + + public String getBatchToken() { + return batchToken; + } + + public void setBatchToken(String batchToken) { + this.batchToken = batchToken; + } + + public TriggerCdcOffset batchToken(String batchToken) { + this.batchToken = batchToken; + return this; + } + + public String getLastEventId() { + return lastEventId; + } + + public void setLastEventId(String lastEventId) { + this.lastEventId = lastEventId; + } + + public TriggerCdcOffset lastEventId(String lastEventId) { + this.lastEventId = lastEventId; + return this; + } + + public Long getLastCapturedAt() { + return lastCapturedAt; + } + + public void setLastCapturedAt(Long lastCapturedAt) { + this.lastCapturedAt = lastCapturedAt; + } + + public TriggerCdcOffset lastCapturedAt(Long lastCapturedAt) { + this.lastCapturedAt = lastCapturedAt; + return this; + } + + @Override + public String toString() { + return "TriggerCdcOffset{captureId=" + captureId + ", generation=" + generation + + ", batchToken=" + batchToken + ", lastEventId=" + lastEventId + + ", lastCapturedAt=" + lastCapturedAt + "}"; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcOperation.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcOperation.java new file mode 100644 index 00000000..7e762f12 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcOperation.java @@ -0,0 +1,32 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +/** + * The kind of change a trigger based CDC journal entry represents. + * + *

A connector must only report the operations that it declared in + * {@link TriggerCdcCapabilities#getDmlOperations()}. Reporting an operation the database + * trigger cannot really observe would silently lose data.

+ */ +public enum TriggerCdcOperation { + INSERT, + UPDATE, + DELETE, + /** + * Structural change captured by a native DDL trigger / event trigger, or by the managed + * DDL workflow. See {@link TriggerCdcDdlMode}. + */ + DDL, + /** + * Table level {@code TRUNCATE}, observed as its own event. + * + *

This is not a row level operation and must never be expanded into a stream of DELETE + * events, because the number of affected rows is unknown at capture time. An event of this kind + * carries the table identity only: {@code before} / {@code after} / key images are null, and + * the sink has to apply a truncate (or a documented equivalent such as a partition drop).

+ * + *

Duplicated authority is forbidden: a connector reports this operation if and only if + * {@link TriggerCdcCapabilities#isTruncateCaptureSupported()} is {@code true}. Reporting one + * without the other is out of contract.

+ */ + TRUNCATE +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatement.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatement.java new file mode 100644 index 00000000..f575a62f --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatement.java @@ -0,0 +1,112 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +/** + * One DDL statement of a {@link TriggerCdcInstallPlan}. + * + *

The plan is also the MANUAL install artefact: a DBA executes exactly these statements. + * Therefore the statements must be complete, ordered, and identical to what + * {@code ensure} would run.

+ */ +public class TriggerCdcStatement implements Serializable { + private static final long serialVersionUID = 1L; + + protected TriggerCdcStatementType type; + protected String sql; + /** What the statement is for, shown in the UI and in the generated install script. */ + protected String description; + /** True when re-running the statement is harmless. */ + protected boolean idempotent; + /** True when the statement locks a customer table, so the UI can warn about it. */ + protected boolean mayLockSourceTable; + /** Privileges a DBA needs for this statement, for example "CREATE TRIGGER". */ + protected List requiredPrivileges; + + public static TriggerCdcStatement create() { + return new TriggerCdcStatement(); + } + + public TriggerCdcStatement privilege(String privilege) { + if (requiredPrivileges == null) { + requiredPrivileges = new ArrayList<>(); + } + requiredPrivileges.add(privilege); + return this; + } + + public TriggerCdcStatementType getType() { + return type; + } + + public void setType(TriggerCdcStatementType type) { + this.type = type; + } + + public TriggerCdcStatement type(TriggerCdcStatementType type) { + this.type = type; + return this; + } + + public String getSql() { + return sql; + } + + public void setSql(String sql) { + this.sql = sql; + } + + public TriggerCdcStatement sql(String sql) { + this.sql = sql; + return this; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public TriggerCdcStatement description(String description) { + this.description = description; + return this; + } + + public boolean isIdempotent() { + return idempotent; + } + + public void setIdempotent(boolean idempotent) { + this.idempotent = idempotent; + } + + public TriggerCdcStatement idempotent(boolean idempotent) { + this.idempotent = idempotent; + return this; + } + + public boolean isMayLockSourceTable() { + return mayLockSourceTable; + } + + public void setMayLockSourceTable(boolean mayLockSourceTable) { + this.mayLockSourceTable = mayLockSourceTable; + } + + public TriggerCdcStatement mayLockSourceTable(boolean mayLockSourceTable) { + this.mayLockSourceTable = mayLockSourceTable; + return this; + } + + public List getRequiredPrivileges() { + return requiredPrivileges; + } + + public void setRequiredPrivileges(List requiredPrivileges) { + this.requiredPrivileges = requiredPrivileges; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatementType.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatementType.java new file mode 100644 index 00000000..7ad992fd --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatementType.java @@ -0,0 +1,15 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +/** + * Kind of DDL statement inside a {@link TriggerCdcInstallPlan}. + */ +public enum TriggerCdcStatementType { + CREATE_NAMESPACE, + CREATE_TABLE, + CREATE_INDEX, + CREATE_TRIGGER, + DROP_TRIGGER, + DROP_TABLE, + DROP_NAMESPACE, + OTHER +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatus.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatus.java new file mode 100644 index 00000000..ae1f18bb --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatus.java @@ -0,0 +1,43 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +/** + * Lifecycle state of one capture (one source table, one generation). + * + *
+ * PLANNED -> INSTALLING -> CAPTURING -> SNAPSHOTTING -> STREAMING
+ *                                |                              |
+ *                                +---------> DEGRADED / REPAIR_REQUIRED
+ * 
+ */ +public enum TriggerCdcStatus { + /** Plan generated, nothing created yet. */ + PLANNED, + /** Objects / triggers are being created. Failure is retryable and reported per object. */ + INSTALLING, + /** Triggers are live and journaling changes, no consumer is reading yet. */ + CAPTURING, + /** A full snapshot is running while the journal keeps accumulating changes. */ + SNAPSHOTTING, + /** Incremental consumption is running normally. */ + STREAMING, + /** + * Capture still works and no change is known to be lost, but a declared guarantee is not + * met: for example a structure drift that still needs an operator decision, journal growth + * beyond the expected volume, or an unregistered consumer holding retention. Requires + * operator attention. + */ + DEGRADED, + /** + * A change is known or suspected to be lost, so the target can no longer be trusted: a + * {@link TriggerCdcErrorPolicy#BEST_EFFORT} journal write failure, a window in which the + * trigger was disabled, an overflowed managed DDL buffer, or a forced clean that removed + * records a consumer had not acknowledged (see + * {@link TriggerCdcCleanupResult#getIntegrityBrokenConsumers()}). A repair or a full resync + * is required. + */ + REPAIR_REQUIRED, + /** Installation or a later verification failed. */ + FAILED, + /** Objects released, no longer capturing. */ + RELEASED +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTable.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTable.java new file mode 100644 index 00000000..a324c9e3 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTable.java @@ -0,0 +1,156 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; + +/** + * One captured source table. + * + *

{@code sourceTableId} is the stable identity (a Tapdata table id), while + * {@code tableName} may be renamed. The journal keys on the id, never on the name, so a + * rename cannot silently orphan already captured events.

+ * + *

{@code captureId} and {@code generation} are assigned by the connector during + * {@code ensure} and echoed by the framework afterwards. A reinstalled capture gets a new + * generation so that old offsets are rejected instead of silently resumed.

+ * + *

Granularity: one source table is one capture. Each capture owns its own journal, and + * {@code read} / {@code acknowledge} are called once per capture, so a table list is also the + * list of independent read positions a task has to keep.

+ */ +public class TriggerCdcTable implements Serializable { + private static final long serialVersionUID = 1L; + + protected String catalog; + protected String schema; + /** Physical table name in the source database. */ + protected String tableName; + /** Stable Tapdata table id. Never derived from the name at read time. */ + protected String sourceTableId; + /** + * Assigned by the connector; null in an install request. Ignored on input, echoed on output. + * It identifies one per table capture, which is also the unit of {@code read} and + * {@code acknowledge}. + */ + protected String captureId; + /** Assigned by the connector; null in an install request. */ + protected Long generation; + /** Physical journal table that holds the changes of this source table. */ + protected String changeTableName; + /** Optional trigger name, assigned by the connector. */ + protected String triggerName; + + public static TriggerCdcTable create() { + return new TriggerCdcTable(); + } + + public String getCatalog() { + return catalog; + } + + public void setCatalog(String catalog) { + this.catalog = catalog; + } + + public TriggerCdcTable catalog(String catalog) { + this.catalog = catalog; + return this; + } + + public String getSchema() { + return schema; + } + + public void setSchema(String schema) { + this.schema = schema; + } + + public TriggerCdcTable schema(String schema) { + this.schema = schema; + return this; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public TriggerCdcTable tableName(String tableName) { + this.tableName = tableName; + return this; + } + + public String getSourceTableId() { + return sourceTableId; + } + + public void setSourceTableId(String sourceTableId) { + this.sourceTableId = sourceTableId; + } + + public TriggerCdcTable sourceTableId(String sourceTableId) { + this.sourceTableId = sourceTableId; + return this; + } + + public String getCaptureId() { + return captureId; + } + + public void setCaptureId(String captureId) { + this.captureId = captureId; + } + + public TriggerCdcTable captureId(String captureId) { + this.captureId = captureId; + return this; + } + + public Long getGeneration() { + return generation; + } + + public void setGeneration(Long generation) { + this.generation = generation; + } + + public TriggerCdcTable generation(Long generation) { + this.generation = generation; + return this; + } + + public String getChangeTableName() { + return changeTableName; + } + + public void setChangeTableName(String changeTableName) { + this.changeTableName = changeTableName; + } + + public TriggerCdcTable changeTableName(String changeTableName) { + this.changeTableName = changeTableName; + return this; + } + + public String getTriggerName() { + return triggerName; + } + + public void setTriggerName(String triggerName) { + this.triggerName = triggerName; + } + + public TriggerCdcTable triggerName(String triggerName) { + this.triggerName = triggerName; + return this; + } + + @Override + public String toString() { + return "TriggerCdcTable{catalog=" + catalog + ", schema=" + schema + ", tableName=" + tableName + + ", sourceTableId=" + sourceTableId + ", captureId=" + captureId + ", generation=" + generation + + ", changeTableName=" + changeTableName + ", triggerName=" + triggerName + "}"; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTableResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTableResult.java new file mode 100644 index 00000000..32960d69 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTableResult.java @@ -0,0 +1,229 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import java.io.Serializable; + +/** + * Per table outcome of a plan / install / release operation. + * + *

Installation is reported per object, not as one boolean, because a partial install must be + * retried idempotently and the operator needs to know which table is missing its trigger.

+ * + *

Fail closed: a successful install entry is only a success when it carries the + * capture identity it created or reused. {@code ensure} assigns + * {@link #getCaptureId()} per source table and that id is the only thing that can select the + * journal later, so a success without an id would be a table installed but never read — an + * invisible data gap. {@link #isSuccess()} therefore returns {@code false} for such an entry and + * the enclosing {@link TriggerCdcInstallResult} reports it as a failure to fix, instead of + * quietly dropping it from the read list.

+ */ +public class TriggerCdcTableResult implements Serializable { + private static final long serialVersionUID = 1L; + + protected String sourceTableId; + protected String tableName; + protected boolean success; + /** Stable error code the framework can translate; null on success. */ + protected String code; + /** Human readable reason; for a blocked table this says why it was refused. */ + protected String message; + protected String captureId; + protected Long generation; + protected String changeTableName; + protected String triggerName; + + public static TriggerCdcTableResult create() { + return new TriggerCdcTableResult(); + } + + /** + * Reports a table as installed or reused. + * + *

Fail closed by construction: the table must carry a non blank + * {@link TriggerCdcTable#getCaptureId()}, otherwise there is no way to address its journal and + * the "success" would silently mean "installed but unreadable". Use + * {@link #failure(TriggerCdcTable, String, String)} for anything that did not become a usable + * capture.

+ * + *

This factory is for {@code ensure} outcomes only. A read only plan reports accepted tables + * through {@link TriggerCdcInstallPlan#supportedTable(TriggerCdcTable)} and refusals through + * {@link #failure(TriggerCdcTable, String, String)}; plan time tables have no capture id yet, + * so calling this there would rightly fail.

+ * + * @throws IllegalArgumentException when {@code table} is null or has no capture id + */ + public static TriggerCdcTableResult success(TriggerCdcTable table) { + if (table == null) { + throw new IllegalArgumentException("table must not be null: a success needs the capture it installed"); + } + if (table.getCaptureId() == null || table.getCaptureId().trim().isEmpty()) { + throw new IllegalArgumentException("table " + table.getTableName() + + " is reported successful but has no captureId; the journal would be unreachable"); + } + TriggerCdcTableResult result = new TriggerCdcTableResult(); + result.success = true; + result.sourceTableId = table.getSourceTableId(); + result.tableName = table.getTableName(); + result.captureId = table.getCaptureId(); + result.generation = table.getGeneration(); + result.changeTableName = table.getChangeTableName(); + result.triggerName = table.getTriggerName(); + return result; + } + + public static TriggerCdcTableResult failure(TriggerCdcTable table, String code, String message) { + TriggerCdcTableResult result = new TriggerCdcTableResult(); + result.success = false; + result.code = code; + result.message = message; + if (table != null) { + result.sourceTableId = table.getSourceTableId(); + result.tableName = table.getTableName(); + } + return result; + } + + public String getSourceTableId() { + return sourceTableId; + } + + public void setSourceTableId(String sourceTableId) { + this.sourceTableId = sourceTableId; + } + + public TriggerCdcTableResult sourceTableId(String sourceTableId) { + this.sourceTableId = sourceTableId; + return this; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + + public TriggerCdcTableResult tableName(String tableName) { + this.tableName = tableName; + return this; + } + + /** + * True only when this entry reports a success and carries a usable capture id. + * + *

The raw flag is deliberately not enough: {@link #success(boolean)} can be set before + * {@link #captureId(String)} in a builder chain, and a success that never receives an id must + * be treated as a failure here rather than passed on as "installed".

+ */ + public boolean isSuccess() { + return success && hasCaptureId(); + } + + /** True when {@link #getCaptureId()} is present and not blank. */ + public boolean hasCaptureId() { + return captureId != null && !captureId.trim().isEmpty(); + } + + /** The reported success flag, before the capture identity requirement is applied. */ + public boolean isSuccessReported() { + return success; + } + + public void setSuccess(boolean success) { + this.success = success; + } + + /** + * Sets the raw success flag. Pair {@code true} with {@link #captureId(String)}: a success + * without a capture id is reported as a failure by {@link #isSuccess()}. + */ + public TriggerCdcTableResult success(boolean success) { + this.success = success; + return this; + } + + public String getCode() { + return code; + } + + public void setCode(String code) { + this.code = code; + } + + public TriggerCdcTableResult code(String code) { + this.code = code; + return this; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public TriggerCdcTableResult message(String message) { + this.message = message; + return this; + } + + public String getCaptureId() { + return captureId; + } + + public void setCaptureId(String captureId) { + this.captureId = captureId; + } + + public TriggerCdcTableResult captureId(String captureId) { + this.captureId = captureId; + return this; + } + + public Long getGeneration() { + return generation; + } + + public void setGeneration(Long generation) { + this.generation = generation; + } + + public TriggerCdcTableResult generation(Long generation) { + this.generation = generation; + return this; + } + + public String getChangeTableName() { + return changeTableName; + } + + public void setChangeTableName(String changeTableName) { + this.changeTableName = changeTableName; + } + + public TriggerCdcTableResult changeTableName(String changeTableName) { + this.changeTableName = changeTableName; + return this; + } + + public String getTriggerName() { + return triggerName; + } + + public void setTriggerName(String triggerName) { + this.triggerName = triggerName; + } + + public TriggerCdcTableResult triggerName(String triggerName) { + this.triggerName = triggerName; + return this; + } + + @Override + public String toString() { + return "TriggerCdcTableResult{tableName=" + tableName + ", sourceTableId=" + sourceTableId + + ", success=" + success + ", captureId=" + captureId + ", code=" + code + + ", message=" + message + "}"; + } +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/ConnectorFunctions.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/ConnectorFunctions.java index cb0694a6..a8c5bc64 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/ConnectorFunctions.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/ConnectorFunctions.java @@ -10,6 +10,7 @@ import io.tapdata.pdk.apis.functions.connector.TapFunction; import io.tapdata.pdk.apis.functions.connector.common.QueryHashByAdvanceFilterFunction; import io.tapdata.pdk.apis.functions.connector.common.ReleaseExternalFunction; +import io.tapdata.pdk.apis.functions.connector.triggercdc.*; import io.tapdata.pdk.apis.functions.connector.source.*; import io.tapdata.pdk.apis.functions.connector.target.*; import io.tapdata.pdk.apis.partition.FieldMinMaxValue; @@ -87,6 +88,15 @@ public class ConnectorFunctions extends ConnectionFunctions protected FlushOffsetFunction flushOffsetFunction; protected ProcessControlFunction processControlFunction; + protected TriggerCdcCapabilitiesFunction triggerCdcCapabilitiesFunction; + protected TriggerCdcPlanFunction triggerCdcPlanFunction; + protected TriggerCdcEnsureFunction triggerCdcEnsureFunction; + protected TriggerCdcReadFunction triggerCdcReadFunction; + protected TriggerCdcAcknowledgeFunction triggerCdcAcknowledgeFunction; + protected TriggerCdcInspectFunction triggerCdcInspectFunction; + protected TriggerCdcCleanupFunction triggerCdcCleanupFunction; + protected TriggerCdcReleaseFunction triggerCdcReleaseFunction; + public ConnectorFunctions supportFlushOffsetFunction(FlushOffsetFunction function) { flushOffsetFunction = function; return this; @@ -97,6 +107,46 @@ public ConnectorFunctions supportProcessControlFunction(ProcessControlFunction f return this; } + public ConnectorFunctions supportTriggerCdcCapabilitiesFunction(TriggerCdcCapabilitiesFunction function) { + triggerCdcCapabilitiesFunction = function; + return this; + } + + public ConnectorFunctions supportTriggerCdcPlanFunction(TriggerCdcPlanFunction function) { + triggerCdcPlanFunction = function; + return this; + } + + public ConnectorFunctions supportTriggerCdcEnsureFunction(TriggerCdcEnsureFunction function) { + triggerCdcEnsureFunction = function; + return this; + } + + public ConnectorFunctions supportTriggerCdcReadFunction(TriggerCdcReadFunction function) { + triggerCdcReadFunction = function; + return this; + } + + public ConnectorFunctions supportTriggerCdcAcknowledgeFunction(TriggerCdcAcknowledgeFunction function) { + triggerCdcAcknowledgeFunction = function; + return this; + } + + public ConnectorFunctions supportTriggerCdcInspectFunction(TriggerCdcInspectFunction function) { + triggerCdcInspectFunction = function; + return this; + } + + public ConnectorFunctions supportTriggerCdcCleanupFunction(TriggerCdcCleanupFunction function) { + triggerCdcCleanupFunction = function; + return this; + } + + public ConnectorFunctions supportTriggerCdcReleaseFunction(TriggerCdcReleaseFunction function) { + triggerCdcReleaseFunction = function; + return this; + } + public ConnectorFunctions supportTransactionBeginFunction(TransactionBeginFunction function) { transactionBeginFunction = function; return this; @@ -220,6 +270,12 @@ public List getCapabilities() { } } catch (Throwable ignored) {} } + if (triggerCdcCapabilitiesFunction != null && triggerCdcPlanFunction != null + && triggerCdcEnsureFunction != null && triggerCdcReadFunction != null + && triggerCdcAcknowledgeFunction != null) { + fieldArray.add(Capability.create(io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConstants.CAPABILITY_TRIGGER_CDC) + .type(Capability.TYPE_OTHER)); + } return fieldArray; } @@ -491,6 +547,38 @@ public ControlFunction getControlFunction() { return controlFunction; } + public TriggerCdcCapabilitiesFunction getTriggerCdcCapabilitiesFunction() { + return triggerCdcCapabilitiesFunction; + } + + public TriggerCdcPlanFunction getTriggerCdcPlanFunction() { + return triggerCdcPlanFunction; + } + + public TriggerCdcEnsureFunction getTriggerCdcEnsureFunction() { + return triggerCdcEnsureFunction; + } + + public TriggerCdcReadFunction getTriggerCdcReadFunction() { + return triggerCdcReadFunction; + } + + public TriggerCdcAcknowledgeFunction getTriggerCdcAcknowledgeFunction() { + return triggerCdcAcknowledgeFunction; + } + + public TriggerCdcInspectFunction getTriggerCdcInspectFunction() { + return triggerCdcInspectFunction; + } + + public TriggerCdcCleanupFunction getTriggerCdcCleanupFunction() { + return triggerCdcCleanupFunction; + } + + public TriggerCdcReleaseFunction getTriggerCdcReleaseFunction() { + return triggerCdcReleaseFunction; + } + public CreateIndexFunction getCreateIndexFunction() { return createIndexFunction; } diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/PDKMethod.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/PDKMethod.java index 54c66127..75264870 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/PDKMethod.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/PDKMethod.java @@ -67,7 +67,15 @@ public enum PDKMethod { DROP_PARTITION_TABLE_FUNCTION, GET_STREAM_OFFSET, EXPORT_EVENT_SQL, - EXECUTE_COMMAND + EXECUTE_COMMAND, + TRIGGER_CDC_CAPABILITIES, + TRIGGER_CDC_PLAN, + TRIGGER_CDC_ENSURE, + TRIGGER_CDC_READ, + TRIGGER_CDC_ACKNOWLEDGE, + TRIGGER_CDC_INSPECT, + TRIGGER_CDC_CLEANUP, + TRIGGER_CDC_RELEASE ; PDKMethod() { diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcAcknowledgeFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcAcknowledgeFunction.java new file mode 100644 index 00000000..035066d1 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcAcknowledgeFunction.java @@ -0,0 +1,36 @@ +package io.tapdata.pdk.apis.functions.connector.triggercdc; + +import io.tapdata.pdk.apis.context.TapConnectorContext; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConfig; +import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; + +/** + * Marks a delivered batch of one capture as safely processed by one consumer. + * + *

The framework calls this only after the downstream write and the recoverable checkpoint are + * both durable. Connectors must treat the acknowledgement as the single source of truth for the + * consumer position on that capture, and must keep the referenced records visible until every + * live consumer has acknowledged them.

+ * + *

Acknowledgement is per capture, mirroring {@link TriggerCdcReadFunction}: a task that + * captures several tables acknowledges each capture separately, so one capture can never appear + * consumed because a sibling capture advanced. A token belongs to exactly one capture and may + * only be acknowledged against it.

+ * + *

Acknowledgement is the reason the mode can be called reliable: a crash between "written" and + * "acknowledged" replays the batch (at least once), and the target side must be idempotent.

+ */ +public interface TriggerCdcAcknowledgeFunction extends TapConnectorFunction { + /** + * @param connectorContext the connection / node context + * @param config connection level trigger CDC configuration + * @param consumerId stable id of the subscribing task / channel + * @param captureId capture the token belongs to, the same value that was passed to + * {@link TriggerCdcReadFunction}. A token that does not belong to this + * capture must be rejected instead of acknowledged. + * @param batchToken token returned by {@link TriggerCdcReadFunction} for the batch of + * this capture + */ + void acknowledge(TapConnectorContext connectorContext, TriggerCdcConfig config, String consumerId, + String captureId, String batchToken) throws Throwable; +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcCapabilitiesFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcCapabilitiesFunction.java new file mode 100644 index 00000000..6bacff59 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcCapabilitiesFunction.java @@ -0,0 +1,26 @@ +package io.tapdata.pdk.apis.functions.connector.triggercdc; + +import io.tapdata.pdk.apis.context.TapConnectorContext; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcCapabilities; +import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; + +/** + * Declares what trigger based CDC this connector really supports. + * + *

Registering this function is what enables the {@code triggerCdc} increment mode for a + * connector. A connector without it must not be offered the mode, and the framework must not + * call any other Trigger CDC function on it.

+ * + *

Declarations are a contract, not a hint: the framework relies on them to decide whether a + * capture boundary exists, whether DDL may be replayed, and whether a full-then-incremental + * handover is safe. Overstating a capability causes silent data loss.

+ */ +public interface TriggerCdcCapabilitiesFunction extends TapConnectorFunction { + /** + * @param connectorContext the connection / node context + * @return the declaration for the connected database and its version; never null + * @throws Throwable when the database version cannot be determined, so the mode stays + * unavailable instead of being guessed + */ + TriggerCdcCapabilities capabilities(TapConnectorContext connectorContext) throws Throwable; +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcCleanupFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcCleanupFunction.java new file mode 100644 index 00000000..6b56f1e1 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcCleanupFunction.java @@ -0,0 +1,49 @@ +package io.tapdata.pdk.apis.functions.connector.triggercdc; + +import io.tapdata.pdk.apis.context.TapConnectorContext; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcCleanupResult; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConfig; +import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; + +/** + * Reclaims journal space for records that every live consumer has acknowledged. + * + *

Rules:

+ *
    + *
  • Delete acknowledged records only. Retention does not override acknowledgement: an + * unacknowledged record is reported through + * {@link TriggerCdcCleanupResult#getRemainingCount()} and kept, so the framework can + * alarm. The only exception is the explicit opt-in + * {@link TriggerCdcConfig#getForceCleanUnacknowledged()}, which defaults to {@code false}.
  • + *
  • Always fill in {@link TriggerCdcCleanupResult#getDeletedUnacknowledgedCount()} — {@code 0} + * when only acknowledged records were removed. This is what makes the acknowledgement rule + * enforceable: the framework calls {@link TriggerCdcCleanupResult#validate(TriggerCdcConfig)} + * on the returned object and rejects a report that deletes unacknowledged records without + * the opt-in or without naming the affected consumers.
  • + *
  • When the opt-in is enabled and unacknowledged records are reclaimed, the connector must + * report every affected consumer through + * {@link TriggerCdcCleanupResult#getIntegrityBrokenConsumers()} and persist that marking + * no later than the delete itself — in the same transaction when the engine supports it, + * otherwise by marking first and deleting second, so no window exists in which the rows are + * gone and the consumers look healthy. A forced clean that reports no affected consumer is a + * contract violation, because the framework would silently keep running a consumer with a + * hole in its stream.
  • + *
  • Keep the journal bounded per batch; do not run one unbounded DELETE that holds locks on + * the customer database.
  • + *
  • Records referenced by a registered consumer that is merely paused must survive until that + * consumer is explicitly unregistered — unless the force clean opt-in above applies, in + * which case the consumer has to be reported as integrity broken.
  • + *
+ */ +public interface TriggerCdcCleanupFunction extends TapConnectorFunction { + /** + * @param connectorContext the connection / node context + * @param config connection level trigger CDC configuration: retention window plus the + * force clean opt-in that decides whether unacknowledged records may + * ever be reclaimed + * @return what was removed and what was kept, never null; when unacknowledged records were + * removed, {@link TriggerCdcCleanupResult#getDeletedUnacknowledgedCount()} is greater than + * {@code 0} and the affected consumers are non empty + */ + TriggerCdcCleanupResult cleanup(TapConnectorContext connectorContext, TriggerCdcConfig config) throws Throwable; +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcEnsureFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcEnsureFunction.java new file mode 100644 index 00000000..8ec0633f --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcEnsureFunction.java @@ -0,0 +1,58 @@ +package io.tapdata.pdk.apis.functions.connector.triggercdc; + +import io.tapdata.pdk.apis.context.TapConnectorContext; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConfig; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcInstallResult; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcTable; +import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; + +import java.util.List; + +/** + * Creates the CDC objects and triggers, then verifies them. Idempotent. + * + *

Requirements:

+ *
    + *
  • Running it twice returns the same capture ids and the same generation. Only a real + * reinstall increments the generation, so a stale offset can be rejected instead of + * silently resumed.
  • + *
  • Never overwrite an object Tapdata does not own. Report a conflict instead.
  • + *
  • Creating or replacing a trigger may lock the source table: use a short lock timeout, + * fail fast and stay retryable instead of waiting on customer traffic.
  • + *
  • Success must include the verification that the trigger is really enabled and that the + * installed capture set covers everything the caller asked for.
  • + *
  • Each successful table result must carry a non blank {@code captureId} that is unique + * within the result: that id is the only address of the journal. A success without an id, or + * two tables sharing an id, is rejected as an invalid result (see + * {@code TriggerCdcInstallResult#validate()}) instead of shrinking the capture list — + * dropping either would mean an installed table that is never read.
  • + *
  • Every requested table needs its own outcome, success or failure: a table that is + * simply absent from the result is neither readable nor retryable, so it is the worst + * outcome of all. Report it through {@code TriggerCdcInstallResult#requestedTables(...)} + * (or let the framework pass {@code tables} to {@code validate(tables)}) so the check can + * tell an omitted table from an unrequested one. Three shapes are rejected: a requested + * table with no outcome, one table with several outcomes, and an outcome for a table that + * was never requested.
  • + *
+ * + *

Whether installation forms a valid capture boundary is declared through + * {@code TriggerCdcCapabilities.captureBoundarySupported}. "CREATE TRIGGER returned" alone is + * never proof that no concurrent transaction was missed.

+ */ +public interface TriggerCdcEnsureFunction extends TapConnectorFunction { + /** + * @param connectorContext the connection / node context + * @param config connection level trigger CDC configuration + * @param tables source tables to capture; capture ids and generations are filled in + * by the connector. One table yields one capture, and that capture is + * the unit of {@link TriggerCdcReadFunction} and + * {@link TriggerCdcAcknowledgeFunction} + * @return per table outcome, never null, with exactly one outcome per requested table (a + * failure is a valid outcome; silence is not) and the request retained through + * {@code TriggerCdcInstallResult#requestedTables(Collection)}. The framework passes + * {@code tables} to {@code validate(tables)} / {@code getCaptureIds(tables)}, so a + * result that does not account for every requested table is rejected rather than read + * as a smaller capture set + */ + TriggerCdcInstallResult ensure(TapConnectorContext connectorContext, TriggerCdcConfig config, List tables) throws Throwable; +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcInspectFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcInspectFunction.java new file mode 100644 index 00000000..56ebc2fb --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcInspectFunction.java @@ -0,0 +1,30 @@ +package io.tapdata.pdk.apis.functions.connector.triggercdc; + +import io.tapdata.pdk.apis.context.TapConnectorContext; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConfig; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcInspectResult; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcTable; +import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; + +import java.util.List; + +/** + * Health and integrity inspection of installed captures. + * + *

Used for periodic monitoring and before resuming a task. It must detect at least: a missing + * or disabled trigger, a missing journal table, a structure drift against the registered schema + * version, backlog size and lag, and journal growth.

+ * + *

Drift detection is not DDL capture. Once drift is detected the capture must be marked + * degraded, because the DDL history between two inspections cannot be reconstructed and the + * changes of the missed window cannot be replayed.

+ */ +public interface TriggerCdcInspectFunction extends TapConnectorFunction { + /** + * @param connectorContext the connection / node context + * @param config connection level trigger CDC configuration + * @param tables captures to inspect; empty means all captures known to the registry + * @return one report per capture, never null + */ + List inspect(TapConnectorContext connectorContext, TriggerCdcConfig config, List tables) throws Throwable; +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcPlanFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcPlanFunction.java new file mode 100644 index 00000000..d6d03add --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcPlanFunction.java @@ -0,0 +1,30 @@ +package io.tapdata.pdk.apis.functions.connector.triggercdc; + +import io.tapdata.pdk.apis.context.TapConnectorContext; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConfig; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcInstallPlan; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcTable; +import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; + +import java.util.List; + +/** + * Builds the installation plan without touching the database. + * + *

Must be side effect free so the UI can show objects, statements, required privileges, + * warnings and blocking problems before anything is created. The same statements feed the + * {@code MANUAL} install path used by a DBA, so they must match what {@code ensure} would run.

+ * + *

A plan must report problems instead of skipping them: a table whose types cannot be + * captured, or a table without a primary key, belongs in + * {@link TriggerCdcInstallPlan#getBlockedTables()} so the operator sees the gap.

+ */ +public interface TriggerCdcPlanFunction extends TapConnectorFunction { + /** + * @param connectorContext the connection / node context + * @param config connection level trigger CDC configuration; already defaulted + * @param tables source tables the task wants to capture + * @return the plan, never null + */ + TriggerCdcInstallPlan plan(TapConnectorContext connectorContext, TriggerCdcConfig config, List tables) throws Throwable; +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReadFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReadFunction.java new file mode 100644 index 00000000..0604b348 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReadFunction.java @@ -0,0 +1,68 @@ +package io.tapdata.pdk.apis.functions.connector.triggercdc; + +import io.tapdata.pdk.apis.context.TapConnectorContext; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcBatch; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConfig; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcOffset; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcTable; +import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; + +/** + * Reads committed, not yet acknowledged change records for one consumer on one capture. + * + *

Scope of a single call: exactly one capture. {@link TriggerCdcEnsureFunction} installs one + * capture per source table and assigns its {@link TriggerCdcTable#getCaptureId()}, and every + * capture owns its own journal table. A task that captures several tables therefore owns several + * captures and must call {@code read} once per capture, each with its own offset. One call never + * spans two captures: a journal table, a batch token and an offset all belong to exactly one + * capture, and mixing them would make "acknowledged" ambiguous. Merging and ordering the batches + * of different captures is the framework's job, not this function's.

+ * + *

This is deliberately not a {@code streamRead}: reliable consumption must not depend on the + * progress of a callback. The read returns a batch plus a token, and only + * {@link TriggerCdcAcknowledgeFunction} moves the position.

+ * + *

Paging rules that prevent silent loss:

+ *
    + *
  • Never treat {@code event_id > last_event_id} as the cursor. A sequence gives unique + * values, not commit order, so a late committing transaction with a smaller id would be + * skipped forever. Each call must re-check the whole unacknowledged set of this capture; + * ids may be used for paging inside a round, not to permanently exclude smaller ids.
  • + *
  • Only committed rows may be returned. Records of a rolled back transaction must never + * become visible.
  • + *
  • A {@code captureId} that is null or unknown must be rejected, not guessed: a fresh offset + * carries no identity, so the selector is the only thing that can pick the journal.
  • + *
  • A generation that differs from the installed capture must be rejected, never reused. The + * returned batch always echoes the generation actually read, so the framework can persist + * it after the first call.
  • + *
  • When the batch is empty, return an empty batch rather than null.
  • + *
+ */ +public interface TriggerCdcReadFunction extends TapConnectorFunction { + /** + * @param connectorContext the connection / node context + * @param config connection level trigger CDC configuration + * @param consumerId stable id of the subscribing task / channel + * @param captureId capture to read, taken from the {@code captureId} of a successful + * {@link TriggerCdcEnsureFunction} result (see + * {@code TriggerCdcInstallResult#getCaptureIds()}). It is the routing + * authority: only the journal of this capture may be read, and the + * returned batch must echo it. One capture per call; a multi table + * task iterates its captures and merges the batches itself. + * @param offset last acknowledged position of this capture for this consumer; + * the framework keeps one offset per {@code (consumerId, captureId)}. + * Its {@link TriggerCdcOffset#getBatchToken()} is the last + * acknowledged batch token, never a merely delivered one. + * {@code null} or a fresh offset means start from the beginning of this + * capture's journal. A non-null + * {@link TriggerCdcOffset#getCaptureId()} must equal {@code captureId}; + * a mismatch must be rejected, because it means a checkpoint of + * another capture was restored here. + * @param batchSize maximum number of records to return in this call; it wins over + * {@link TriggerCdcConfig#getBatchSize()}, which the framework only uses + * as the default it passes in here + * @return the batch of this capture, never null + */ + TriggerCdcBatch read(TapConnectorContext connectorContext, TriggerCdcConfig config, String consumerId, + String captureId, TriggerCdcOffset offset, int batchSize) throws Throwable; +} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReleaseFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReleaseFunction.java new file mode 100644 index 00000000..7dc0ab5b --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReleaseFunction.java @@ -0,0 +1,34 @@ +package io.tapdata.pdk.apis.functions.connector.triggercdc; + +import io.tapdata.pdk.apis.context.TapConnectorContext; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConfig; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcTable; +import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; + +import java.util.List; + +/** + * Releases a consumer, and optionally the capture objects themselves. + * + *

Two separate actions on purpose:

+ *
    + *
  • Unregistering a consumer stops its claim on journal records, so cleanup may remove them. + * A consumer holds one position per capture, so unregistering must drop all of them + * at once; a leftover position would keep a journal alive forever. This must be an explicit + * operation, never an implicit side effect of a crash or a timeout, otherwise a temporarily + * paused task silently loses its data.
  • + *
  • Dropping objects happens only when the framework owns them and no other consumer + * references the capture. Never drop an entire customer schema.
  • + *
+ */ +public interface TriggerCdcReleaseFunction extends TapConnectorFunction { + /** + * @param connectorContext the connection / node context + * @param config connection level trigger CDC configuration + * @param consumerId consumer to unregister; null skips the unregistration step + * @param tables captures to consider; ignored when {@code dropObjects} is false + * @param dropObjects true to also remove the triggers and journal objects Tapdata owns + */ + void release(TapConnectorContext connectorContext, TriggerCdcConfig config, String consumerId, + List tables, boolean dropObjects) throws Throwable; +} From 85384dd11dd67dbd3f6424f72a2d6c4158d56c34 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Thu, 19 Nov 2026 01:44:39 +0000 Subject: [PATCH 2/5] refactor(pdk): narrow trigger CDC API and bump release versions --- docs/trigger-cdc.md | 57 ++++ error-code-root/contract-tests/pom.xml | 6 +- error-code-root/error-code-bom/pom.xml | 2 +- error-code-root/error-code-core/pom.xml | 2 +- error-code-root/error-code-parent/pom.xml | 2 +- error-code-root/error-code-scanner/pom.xml | 2 +- error-code-root/pdk-error-code/pom.xml | 2 +- error-code-root/pom.xml | 6 +- plugin-kit/connector-archetypes/pom.xml | 2 +- plugin-kit/pom.xml | 20 +- plugin-kit/tapdata-api/pom.xml | 2 +- plugin-kit/tapdata-common/pom.xml | 2 +- .../async-tools-module/pom.xml | 2 +- .../tapdata-modules/class-supervisor/pom.xml | 2 +- .../tapdata-modules/modules-api/pom.xml | 2 +- .../mongodb-storage-module/pom.xml | 2 +- plugin-kit/tapdata-modules/pom.xml | 2 +- .../script-engines-module/pom.xml | 2 +- .../service-skeleton-module/pom.xml | 2 +- .../tapdata-storage-module/pom.xml | 2 +- .../websocket-client-module/pom.xml | 2 +- .../websocket-server-module/pom.xml | 2 +- plugin-kit/tapdata-pdk-api/pom.xml | 11 +- .../entity/triggercdc/TriggerCdcBatch.java | 158 ---------- .../triggercdc/TriggerCdcCleanupResult.java | 7 + .../entity/triggercdc/TriggerCdcConfig.java | 19 +- .../triggercdc/TriggerCdcConstants.java | 7 +- .../entity/triggercdc/TriggerCdcEvent.java | 281 ------------------ .../triggercdc/TriggerCdcInspectResult.java | 207 ------------- .../triggercdc/TriggerCdcInstallMode.java | 20 -- .../triggercdc/TriggerCdcInstallPlan.java | 177 ----------- .../triggercdc/TriggerCdcInstallResult.java | 21 ++ .../entity/triggercdc/TriggerCdcObject.java | 132 -------- .../triggercdc/TriggerCdcObjectState.java | 111 ------- .../triggercdc/TriggerCdcObjectType.java | 25 -- .../entity/triggercdc/TriggerCdcOffset.java | 62 ++-- .../triggercdc/TriggerCdcStatement.java | 112 ------- .../triggercdc/TriggerCdcStatementType.java | 15 - .../entity/triggercdc/TriggerCdcStatus.java | 8 +- .../entity/triggercdc/TriggerCdcTable.java | 2 +- .../triggercdc/TriggerCdcTableResult.java | 18 +- .../apis/functions/ConnectorFunctions.java | 55 +--- .../tapdata/pdk/apis/functions/PDKMethod.java | 7 +- .../connector/source/StreamReadFunction.java | 4 + .../TriggerCdcAcknowledgeFunction.java | 38 ++- .../TriggerCdcCapabilitiesFunction.java | 26 -- .../triggercdc/TriggerCdcEnsureFunction.java | 68 ++--- .../triggercdc/TriggerCdcInspectFunction.java | 30 -- .../triggercdc/TriggerCdcPlanFunction.java | 30 -- .../triggercdc/TriggerCdcReadFunction.java | 68 ----- .../triggercdc/TriggerCdcReleaseFunction.java | 34 --- .../triggercdc/TriggerCdcContractTest.java | 169 +++++++++++ plugin-kit/tapdata-pdk-runner/pom.xml | 4 +- plugin-kit/tapdata-proxy/pom.xml | 2 +- plugin-kit/tapdata-tests/pom.xml | 2 +- pom.xml | 2 +- 56 files changed, 390 insertions(+), 1667 deletions(-) create mode 100644 docs/trigger-cdc.md delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcBatch.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcEvent.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInspectResult.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallMode.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallPlan.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObject.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectState.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectType.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatement.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatementType.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcCapabilitiesFunction.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcInspectFunction.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcPlanFunction.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReadFunction.java delete mode 100644 plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReleaseFunction.java create mode 100644 plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java diff --git a/docs/trigger-cdc.md b/docs/trigger-cdc.md new file mode 100644 index 00000000..b2fdad2c --- /dev/null +++ b/docs/trigger-cdc.md @@ -0,0 +1,57 @@ +# Trigger CDC PDK contract + +Trigger CDC adds three connector functions: `ensure`, `acknowledge`, and `cleanup`. +Reading uses the existing `StreamReadFunction` and ordinary `TapEvent` types. +`ConnectorFunctions` advertises `trigger_cdc` only when all four are registered. +Database/version-specific guarantees come back in `TriggerCdcInstallResult.capabilities`. +Registration advertises implementation support; it does not establish database privileges, +DDL support or a safe snapshot boundary. Those must be verified by ensure. + +## Lifecycle and checkpoint integration + +1. Call ensure with the effective storage config, a stable consumer id and the original + requested table list. It installs/verifies owned objects and registers the durable + subscription before a snapshot or read can start. Repeat on resume. Keep subscriptions + across stop/crash and do not expire them by TTL. +2. Check `validate(originalTables)` and require every table to succeed before starting the + task. Persist every capture id and generation. A partial result describes retryable + failures, not permission to silently omit those tables. Reject full-to-incremental + handover if the connector cannot prove a capture boundary. +3. Invoke streamRead on the same node context. The connector uses the consumer/config/capture + mapping retained by ensure and polls committed journal records. The offset is a complete + `Map` keyed by capture id, even for a single table. Validate keys, + identities, generations and token ownership before resume. Emit ordinary TapEvents and a + fresh map/value snapshot on each callback. Stop on schema drift, disabled/missing triggers + or a subscription marked integrity broken. +4. Each token describes the cumulative delivered set for its capture, including earlier + unacknowledged batches. An out-of-order committed event must still be delivered: sequence + allocation order is not commit order. Track row membership rather than an event-id high + watermark. In-memory delivery tracking can avoid continuous duplicates while a checkpoint + is pending; loss of that tracking after restart safely replays unacknowledged events. +5. Only after every destination's writes and the corresponding recoverable checkpoint are + durable, pass each checkpoint entry to acknowledge. Tokens are bound to consumer, capture + and generation. Duplicate or older acknowledgements are idempotent and cannot regress + progress. A crash before acknowledgement may cause replay; targets need idempotent writes. +6. Run bounded cleanup. Delete only rows acknowledged by every registered consumer and old + enough under retention. A paused or deleted task still holds its subscription in this + first version. Forced deletion requires explicit opt-in, usable affected consumer ids, + and integrity markings persisted before or atomically with the delete. Validate each result. + +The current engine callback in `HazelcastSourcePdkDataNode.generateStreamReadConsumer` +enqueues events and updates an in-memory offset. Returning from it is not a durable checkpoint +notification. The engine integration must connect step 5 to durable progress covering every +sink; until then, it must not acknowledge on callback return or enable journal reclamation +based on delivery. This PR provides the PDK contracts, not that engine implementation. + +## Scope + +There are no separate capability-query, SQL-plan, read, periodic-inspect or release APIs. +Ensure returns verified capabilities and performs startup checks; the stream reader performs +ongoing integrity checks. SQL preview/manual installation, automatic subscription retirement +and automatic trigger/table removal are deferred. Operators must explicitly retire abandoned +subscriptions outside this API; automatic expiry would lose paused consumers' data. + +Tapdata API artifacts use `2.0.12-SNAPSHOT`; the parent, runner, modules and error-code +artifacts use `2.12-SNAPSHOT`. The companion changes in the engine/manager and connectors +repositories must be applied with this release. Third-party libraries and Scala's binary +version are independent of this release. diff --git a/error-code-root/contract-tests/pom.xml b/error-code-root/contract-tests/pom.xml index 1dae07d1..709a510f 100644 --- a/error-code-root/contract-tests/pom.xml +++ b/error-code-root/contract-tests/pom.xml @@ -12,9 +12,9 @@ - io.tapdataerror-code-scanner2.11-SNAPSHOTtest - io.tapdatapdk-error-code2.11-SNAPSHOTtest - io.tapdatatapdata-api2.0.11-SNAPSHOTtest + io.tapdataerror-code-scanner2.12-SNAPSHOTtest + io.tapdatapdk-error-code2.12-SNAPSHOTtest + io.tapdatatapdata-api2.0.12-SNAPSHOTtest org.junit.jupiterjunit-jupiter5.8.1test diff --git a/error-code-root/error-code-bom/pom.xml b/error-code-root/error-code-bom/pom.xml index a5be5a4c..65859b18 100644 --- a/error-code-root/error-code-bom/pom.xml +++ b/error-code-root/error-code-bom/pom.xml @@ -6,7 +6,7 @@ io.tapdata error-code-root - 2.11-SNAPSHOT + 2.12-SNAPSHOT error-code-bom diff --git a/error-code-root/error-code-core/pom.xml b/error-code-root/error-code-core/pom.xml index d4717169..9562d5a1 100644 --- a/error-code-root/error-code-core/pom.xml +++ b/error-code-root/error-code-core/pom.xml @@ -6,7 +6,7 @@ io.tapdata error-code-parent - 2.11-SNAPSHOT + 2.12-SNAPSHOT ../error-code-parent/pom.xml diff --git a/error-code-root/error-code-parent/pom.xml b/error-code-root/error-code-parent/pom.xml index 4d649a73..667e4d31 100644 --- a/error-code-root/error-code-parent/pom.xml +++ b/error-code-root/error-code-parent/pom.xml @@ -6,7 +6,7 @@ io.tapdata error-code-root - 2.11-SNAPSHOT + 2.12-SNAPSHOT error-code-parent diff --git a/error-code-root/error-code-scanner/pom.xml b/error-code-root/error-code-scanner/pom.xml index d28d4608..ffb066dd 100644 --- a/error-code-root/error-code-scanner/pom.xml +++ b/error-code-root/error-code-scanner/pom.xml @@ -6,7 +6,7 @@ io.tapdata error-code-parent - 2.11-SNAPSHOT + 2.12-SNAPSHOT ../error-code-parent/pom.xml diff --git a/error-code-root/pdk-error-code/pom.xml b/error-code-root/pdk-error-code/pom.xml index c2fe16ef..fdfe7b9d 100644 --- a/error-code-root/pdk-error-code/pom.xml +++ b/error-code-root/pdk-error-code/pom.xml @@ -6,7 +6,7 @@ io.tapdata error-code-parent - 2.11-SNAPSHOT + 2.12-SNAPSHOT ../error-code-parent/pom.xml diff --git a/error-code-root/pom.xml b/error-code-root/pom.xml index dfb2e4ad..dad25690 100644 --- a/error-code-root/pom.xml +++ b/error-code-root/pom.xml @@ -6,10 +6,10 @@ io.tapdata tapdata-common-lib - 2.11-SNAPSHOT + 2.12-SNAPSHOT error-code-root - 2.11-SNAPSHOT + 2.12-SNAPSHOT Error Code Root pom @@ -22,7 +22,7 @@ - 2.11-SNAPSHOT + 2.12-SNAPSHOT diff --git a/plugin-kit/connector-archetypes/pom.xml b/plugin-kit/connector-archetypes/pom.xml index 3e37e16f..3ebebdab 100644 --- a/plugin-kit/connector-archetypes/pom.xml +++ b/plugin-kit/connector-archetypes/pom.xml @@ -5,7 +5,7 @@ plugin-kit io.tapdata - 2.11-SNAPSHOT + 2.12-SNAPSHOT 4.0.0 diff --git a/plugin-kit/pom.xml b/plugin-kit/pom.xml index 21aaa14b..0eab88d4 100644 --- a/plugin-kit/pom.xml +++ b/plugin-kit/pom.xml @@ -5,9 +5,9 @@ io.tapdata tapdata-common-lib - 2.11-SNAPSHOT + 2.12-SNAPSHOT - 2.11-SNAPSHOT + 2.12-SNAPSHOT plugin-kit plugin-kit pom @@ -25,19 +25,19 @@ ${project.artifactId}-v${project.version} ${project.artifactId}-v${project.version} - 2.11-SNAPSHOT - 2.0.11-SNAPSHOT - 2.0.11-SNAPSHOT + 2.12-SNAPSHOT + 2.0.12-SNAPSHOT + 2.0.12-SNAPSHOT 1.0-SNAPSHOT 5.8.1 1.8.1 4.11.0 4.11.0 - 2.11-SNAPSHOT - 2.11-SNAPSHOT - 2.11-SNAPSHOT - 2.11-SNAPSHOT - 2.11-SNAPSHOT + 2.12-SNAPSHOT + 2.12-SNAPSHOT + 2.12-SNAPSHOT + 2.12-SNAPSHOT + 2.12-SNAPSHOT 2.18.0 3.12.0 1.2.83 diff --git a/plugin-kit/tapdata-api/pom.xml b/plugin-kit/tapdata-api/pom.xml index 5a4bb3ec..9b21e5ad 100644 --- a/plugin-kit/tapdata-api/pom.xml +++ b/plugin-kit/tapdata-api/pom.xml @@ -5,7 +5,7 @@ tapdata-api io.tapdata tapdata-api - 2.0.11-SNAPSHOT + 2.0.12-SNAPSHOT 3.8.1 diff --git a/plugin-kit/tapdata-common/pom.xml b/plugin-kit/tapdata-common/pom.xml index 69612bbb..d1d8c60d 100644 --- a/plugin-kit/tapdata-common/pom.xml +++ b/plugin-kit/tapdata-common/pom.xml @@ -5,7 +5,7 @@ plugin-kit io.tapdata - 2.11-SNAPSHOT + 2.12-SNAPSHOT 4.0.0 diff --git a/plugin-kit/tapdata-modules/async-tools-module/pom.xml b/plugin-kit/tapdata-modules/async-tools-module/pom.xml index 090c67a1..b5d2ba91 100644 --- a/plugin-kit/tapdata-modules/async-tools-module/pom.xml +++ b/plugin-kit/tapdata-modules/async-tools-module/pom.xml @@ -5,7 +5,7 @@ io.tapdata tapdata-modules - 2.11-SNAPSHOT + 2.12-SNAPSHOT async-tools-module async-tool-module diff --git a/plugin-kit/tapdata-modules/class-supervisor/pom.xml b/plugin-kit/tapdata-modules/class-supervisor/pom.xml index 4d111757..6ff4e474 100644 --- a/plugin-kit/tapdata-modules/class-supervisor/pom.xml +++ b/plugin-kit/tapdata-modules/class-supervisor/pom.xml @@ -6,7 +6,7 @@ io.tapdata tapdata-modules - 2.11-SNAPSHOT + 2.12-SNAPSHOT class-supervisor diff --git a/plugin-kit/tapdata-modules/modules-api/pom.xml b/plugin-kit/tapdata-modules/modules-api/pom.xml index 1ab86a09..e4b82095 100644 --- a/plugin-kit/tapdata-modules/modules-api/pom.xml +++ b/plugin-kit/tapdata-modules/modules-api/pom.xml @@ -5,7 +5,7 @@ io.tapdata tapdata-modules - 2.11-SNAPSHOT + 2.12-SNAPSHOT modules-api modules-api diff --git a/plugin-kit/tapdata-modules/mongodb-storage-module/pom.xml b/plugin-kit/tapdata-modules/mongodb-storage-module/pom.xml index 2c57e0a1..b5f60e6f 100644 --- a/plugin-kit/tapdata-modules/mongodb-storage-module/pom.xml +++ b/plugin-kit/tapdata-modules/mongodb-storage-module/pom.xml @@ -5,7 +5,7 @@ io.tapdata tapdata-modules - 2.11-SNAPSHOT + 2.12-SNAPSHOT mongodb-storage-module mongodb-storage-module diff --git a/plugin-kit/tapdata-modules/pom.xml b/plugin-kit/tapdata-modules/pom.xml index 90195d5c..06656e06 100644 --- a/plugin-kit/tapdata-modules/pom.xml +++ b/plugin-kit/tapdata-modules/pom.xml @@ -4,7 +4,7 @@ io.tapdata plugin-kit - 2.11-SNAPSHOT + 2.12-SNAPSHOT 4.0.0 tapdata-modules diff --git a/plugin-kit/tapdata-modules/script-engines-module/pom.xml b/plugin-kit/tapdata-modules/script-engines-module/pom.xml index 9cc0c5ac..61cbc805 100644 --- a/plugin-kit/tapdata-modules/script-engines-module/pom.xml +++ b/plugin-kit/tapdata-modules/script-engines-module/pom.xml @@ -5,7 +5,7 @@ io.tapdata tapdata-modules - 2.11-SNAPSHOT + 2.12-SNAPSHOT script-engine-module script-engine-module diff --git a/plugin-kit/tapdata-modules/service-skeleton-module/pom.xml b/plugin-kit/tapdata-modules/service-skeleton-module/pom.xml index b9de115b..b85e1aec 100644 --- a/plugin-kit/tapdata-modules/service-skeleton-module/pom.xml +++ b/plugin-kit/tapdata-modules/service-skeleton-module/pom.xml @@ -5,7 +5,7 @@ io.tapdata tapdata-modules - 2.11-SNAPSHOT + 2.12-SNAPSHOT service-skeleton-module service-skeleton-module diff --git a/plugin-kit/tapdata-modules/tapdata-storage-module/pom.xml b/plugin-kit/tapdata-modules/tapdata-storage-module/pom.xml index 0c4eb92a..a897ed10 100644 --- a/plugin-kit/tapdata-modules/tapdata-storage-module/pom.xml +++ b/plugin-kit/tapdata-modules/tapdata-storage-module/pom.xml @@ -5,7 +5,7 @@ io.tapdata tapdata-modules - 2.11-SNAPSHOT + 2.12-SNAPSHOT tapdata-storage-module tapdata-storage-module diff --git a/plugin-kit/tapdata-modules/websocket-client-module/pom.xml b/plugin-kit/tapdata-modules/websocket-client-module/pom.xml index c4de3bcc..641e9b56 100644 --- a/plugin-kit/tapdata-modules/websocket-client-module/pom.xml +++ b/plugin-kit/tapdata-modules/websocket-client-module/pom.xml @@ -5,7 +5,7 @@ io.tapdata tapdata-modules - 2.11-SNAPSHOT + 2.12-SNAPSHOT websocket-client-module websocket-client-module diff --git a/plugin-kit/tapdata-modules/websocket-server-module/pom.xml b/plugin-kit/tapdata-modules/websocket-server-module/pom.xml index 8c4c1241..fccfa6a1 100644 --- a/plugin-kit/tapdata-modules/websocket-server-module/pom.xml +++ b/plugin-kit/tapdata-modules/websocket-server-module/pom.xml @@ -5,7 +5,7 @@ io.tapdata tapdata-modules - 2.11-SNAPSHOT + 2.12-SNAPSHOT websocket-server-module websocket-server-module diff --git a/plugin-kit/tapdata-pdk-api/pom.xml b/plugin-kit/tapdata-pdk-api/pom.xml index 5a573311..9d3370f2 100644 --- a/plugin-kit/tapdata-pdk-api/pom.xml +++ b/plugin-kit/tapdata-pdk-api/pom.xml @@ -4,12 +4,12 @@ io.tapdata plugin-kit - 2.11-SNAPSHOT + 2.12-SNAPSHOT 4.0.0 tapdata-pdk-api tapdata-pdk-api - 2.0.11-SNAPSHOT + 2.0.12-SNAPSHOT @@ -20,7 +20,7 @@ io.tapdata tapdata-api - 2.0.11-SNAPSHOT + 2.0.12-SNAPSHOT io.tapdata @@ -39,6 +39,11 @@ + + org.apache.maven.plugins + maven-surefire-plugin + 3.2.5 + org.apache.felix maven-bundle-plugin diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcBatch.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcBatch.java deleted file mode 100644 index 020c609c..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcBatch.java +++ /dev/null @@ -1,158 +0,0 @@ -package io.tapdata.pdk.apis.entity.triggercdc; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -/** - * A batch of change records returned by - * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcReadFunction}. - * - *

Scope: one batch holds the records of exactly one capture, the one selected by the - * {@code captureId} argument of the read call. {@code captureId} and {@code generation} echo that - * selector and the generation actually read, so a batch can never be acked against another - * capture. A task that captures several tables receives one batch per capture and does its own - * merging; this contract does not define a combined batch.

- * - *

The batch carries a {@code batchToken}. The framework must call - * {@code TriggerCdcAcknowledgeFunction} with that token only after the downstream sink and the - * recoverable checkpoint are both durable. Returning a batch does not mean the events are - * consumed.

- * - *

Delivery is at least once: a crash between "sink written" and "batch acknowledged" replays - * the batch, which is expected and must be tolerated by the target side. End to end exactly once - * depends on the target write and dedup abilities, never on this contract alone.

- */ -public class TriggerCdcBatch implements Serializable { - private static final long serialVersionUID = 1L; - - /** - * Opaque token referring to the delivered set of change records of this capture. Connectors - * should persist the referenced ids (a token can also be a row range marker) instead of - * forcing the engine to carry a huge id list inside the task checkpoint. A token is scoped to - * its capture and is only meaningful in {@code acknowledge} for the same {@code captureId}. - */ - protected String batchToken; - /** - * Capture these records belong to; must equal the {@code captureId} the read call selected. - */ - protected String captureId; - /** - * Generation of the capture as read. The framework persists it on the first call, so that a - * later read with a different generation can be rejected instead of silently resumed. - */ - protected Long generation; - protected List events; - /** True when more unacknowledged records of this capture are waiting right now. */ - protected boolean hasMore; - /** - * Total number of records of this capture still unacknowledged for this consumer, when cheap - * to compute. - */ - protected Long pendingCount; - /** Capture time of the oldest unacknowledged record, for lag monitoring. */ - protected Long oldestPendingCapturedAt; - - public static TriggerCdcBatch create() { - return new TriggerCdcBatch(); - } - - public TriggerCdcBatch event(TriggerCdcEvent event) { - if (events == null) { - events = new ArrayList<>(); - } - events.add(event); - return this; - } - - /** True when this batch carries no change record. */ - public boolean isEmpty() { - return events == null || events.isEmpty(); - } - - public String getBatchToken() { - return batchToken; - } - - public void setBatchToken(String batchToken) { - this.batchToken = batchToken; - } - - public TriggerCdcBatch batchToken(String batchToken) { - this.batchToken = batchToken; - return this; - } - - public String getCaptureId() { - return captureId; - } - - public void setCaptureId(String captureId) { - this.captureId = captureId; - } - - public TriggerCdcBatch captureId(String captureId) { - this.captureId = captureId; - return this; - } - - public Long getGeneration() { - return generation; - } - - public void setGeneration(Long generation) { - this.generation = generation; - } - - public TriggerCdcBatch generation(Long generation) { - this.generation = generation; - return this; - } - - public List getEvents() { - return events; - } - - public void setEvents(List events) { - this.events = events; - } - - public boolean isHasMore() { - return hasMore; - } - - public void setHasMore(boolean hasMore) { - this.hasMore = hasMore; - } - - public TriggerCdcBatch hasMore(boolean hasMore) { - this.hasMore = hasMore; - return this; - } - - public Long getPendingCount() { - return pendingCount; - } - - public void setPendingCount(Long pendingCount) { - this.pendingCount = pendingCount; - } - - public TriggerCdcBatch pendingCount(Long pendingCount) { - this.pendingCount = pendingCount; - return this; - } - - public Long getOldestPendingCapturedAt() { - return oldestPendingCapturedAt; - } - - public void setOldestPendingCapturedAt(Long oldestPendingCapturedAt) { - this.oldestPendingCapturedAt = oldestPendingCapturedAt; - } - - public TriggerCdcBatch oldestPendingCapturedAt(Long oldestPendingCapturedAt) { - this.oldestPendingCapturedAt = oldestPendingCapturedAt; - return this; - } -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java index a7891a18..16ac9879 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java @@ -119,6 +119,13 @@ public List validate(TriggerCdcConfig config) { violations.add("removed " + deletedUnacknowledgedCount + " unacknowledged record(s) while forceCleanUnacknowledged is false"); } + if (hasIntegrityBrokenConsumers()) { + for (String consumerId : integrityBrokenConsumers) { + if (consumerId == null || consumerId.trim().isEmpty()) { + violations.add("integrity broken consumer id must not be null or blank"); + } + } + } if (!hasIntegrityBrokenConsumers()) { violations.add("removed " + deletedUnacknowledgedCount + " unacknowledged record(s) but reported no integrity broken consumer"); diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConfig.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConfig.java index 2e0f0fb9..89901ac6 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConfig.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConfig.java @@ -22,8 +22,7 @@ public class TriggerCdcConfig implements Serializable { /** Schema (or database / keyspace) that holds the CDC objects. */ protected String schema; - protected TriggerCdcInstallMode installMode; - /** How often the framework polls the journal. */ + /** How often the connector stream reader polls the journal. */ protected Long pollIntervalMs; /** How many change records a single read returns at most. */ protected Integer batchSize; @@ -79,9 +78,6 @@ public TriggerCdcConfig withDefaults() { if (schema == null) { schema = TriggerCdcConstants.DEFAULT_STORAGE_SCHEMA; } - if (installMode == null) { - installMode = TriggerCdcInstallMode.AUTO; - } if (pollIntervalMs == null) { pollIntervalMs = TriggerCdcConstants.DEFAULT_POLL_INTERVAL_MS; } @@ -135,19 +131,6 @@ public TriggerCdcConfig schema(String schema) { return this; } - public TriggerCdcInstallMode getInstallMode() { - return installMode; - } - - public void setInstallMode(TriggerCdcInstallMode installMode) { - this.installMode = installMode; - } - - public TriggerCdcConfig installMode(TriggerCdcInstallMode installMode) { - this.installMode = installMode; - return this; - } - public Long getPollIntervalMs() { return pollIntervalMs; } diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConstants.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConstants.java index 663c68e3..caab03db 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConstants.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcConstants.java @@ -6,7 +6,7 @@ *

These values are the protocol, not a database specific detail: the framework uses them to * talk about the CDC storage layout, the connector uses them when it materialises the objects. * A connector may only deviate from the names when the database cannot represent them, and it - * must then report the real names back through {@link TriggerCdcObject#getName()}.

+ * must then report the real names back through {@link TriggerCdcTable#getChangeTableName()}.

*/ public final class TriggerCdcConstants { private TriggerCdcConstants() { @@ -22,14 +22,9 @@ private TriggerCdcConstants() { * Capability ids are derived from the {@link io.tapdata.pdk.apis.functions.ConnectorFunctions} * field names. They are listed here so callers do not hard code strings. */ - public static final String CAPABILITY_TRIGGER_CDC_CAPABILITIES = "trigger_cdc_capabilities_function"; - public static final String CAPABILITY_TRIGGER_CDC_PLAN = "trigger_cdc_plan_function"; public static final String CAPABILITY_TRIGGER_CDC_ENSURE = "trigger_cdc_ensure_function"; - public static final String CAPABILITY_TRIGGER_CDC_READ = "trigger_cdc_read_function"; public static final String CAPABILITY_TRIGGER_CDC_ACKNOWLEDGE = "trigger_cdc_acknowledge_function"; - public static final String CAPABILITY_TRIGGER_CDC_INSPECT = "trigger_cdc_inspect_function"; public static final String CAPABILITY_TRIGGER_CDC_CLEANUP = "trigger_cdc_cleanup_function"; - public static final String CAPABILITY_TRIGGER_CDC_RELEASE = "trigger_cdc_release_function"; /** Default CDC storage namespace when the user did not pick one. */ public static final String DEFAULT_STORAGE_SCHEMA = "cdc"; diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcEvent.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcEvent.java deleted file mode 100644 index c3c0afe0..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcEvent.java +++ /dev/null @@ -1,281 +0,0 @@ -package io.tapdata.pdk.apis.entity.triggercdc; - -import java.io.Serializable; -import java.util.Map; - -/** - * One decoded change record read from the CDC journal. - * - *

The connector decodes the journal payload (which it wrote itself inside the trigger) into - * plain Java maps, so the engine can map it onto standard Tap DML / DDL events without knowing - * the database payload format.

- * - *

Decoding rules that matter for correctness:

- *
    - *
  • Decimal and big integer values must keep their precision and scale.
  • - *
  • Binary, timestamp and time zone values need an explicit, documented encoding.
  • - *
  • NULL, missing column and "not captured" must stay distinguishable.
  • - *
  • Values are the ones observed at trigger time; never re-read the source row later, - * because it may already have been updated again.
  • - *
- */ -public class TriggerCdcEvent implements Serializable { - private static final long serialVersionUID = 1L; - - /** - * Unique id of the change record. Used for acknowledgement and de-duplication. - * - *

It does not imply commit order, see {@link TriggerCdcOffset}.

- */ - protected String eventId; - /** - * Capture this record belongs to. Inside a read batch it always equals the {@code captureId} - * the caller selected, so an event can never be mistakenly acked against another capture. - */ - protected String captureId; - /** Generation of the capture when the record was written. */ - protected Long generation; - /** Stable id of the source table. */ - protected String sourceTableId; - protected String sourceTableName; - protected TriggerCdcOperation operation; - /** Schema version the payload was encoded with. */ - protected Long schemaVersion; - /** Primary key before the change, null for INSERT. */ - protected Map keyBefore; - /** Primary key after the change, null for DELETE. Differs from keyBefore on key updates. */ - protected Map keyAfter; - /** Full row before the change, when the capture image provides it. */ - protected Map before; - /** Full row after the change, when the capture image provides it. */ - protected Map after; - /** Source transaction id, only when {@code transactionIdSupported}. */ - protected String transactionId; - /** Position inside the source transaction, only when {@code transactionSequenceSupported}. */ - protected Long transactionSeq; - /** Capture time in epoch millis, for observation only, not a safety cursor. */ - protected Long capturedAt; - /** Raw or normalized DDL text, only for {@link TriggerCdcOperation#DDL} events. */ - protected String ddlPayload; - /** Version of the payload encoding. */ - protected Integer payloadVersion; - - public static TriggerCdcEvent create() { - return new TriggerCdcEvent(); - } - - public String getEventId() { - return eventId; - } - - public void setEventId(String eventId) { - this.eventId = eventId; - } - - public TriggerCdcEvent eventId(String eventId) { - this.eventId = eventId; - return this; - } - - public String getCaptureId() { - return captureId; - } - - public void setCaptureId(String captureId) { - this.captureId = captureId; - } - - public TriggerCdcEvent captureId(String captureId) { - this.captureId = captureId; - return this; - } - - public Long getGeneration() { - return generation; - } - - public void setGeneration(Long generation) { - this.generation = generation; - } - - public TriggerCdcEvent generation(Long generation) { - this.generation = generation; - return this; - } - - public String getSourceTableId() { - return sourceTableId; - } - - public void setSourceTableId(String sourceTableId) { - this.sourceTableId = sourceTableId; - } - - public TriggerCdcEvent sourceTableId(String sourceTableId) { - this.sourceTableId = sourceTableId; - return this; - } - - public String getSourceTableName() { - return sourceTableName; - } - - public void setSourceTableName(String sourceTableName) { - this.sourceTableName = sourceTableName; - } - - public TriggerCdcEvent sourceTableName(String sourceTableName) { - this.sourceTableName = sourceTableName; - return this; - } - - public TriggerCdcOperation getOperation() { - return operation; - } - - public void setOperation(TriggerCdcOperation operation) { - this.operation = operation; - } - - public TriggerCdcEvent operation(TriggerCdcOperation operation) { - this.operation = operation; - return this; - } - - public Long getSchemaVersion() { - return schemaVersion; - } - - public void setSchemaVersion(Long schemaVersion) { - this.schemaVersion = schemaVersion; - } - - public TriggerCdcEvent schemaVersion(Long schemaVersion) { - this.schemaVersion = schemaVersion; - return this; - } - - public Map getKeyBefore() { - return keyBefore; - } - - public void setKeyBefore(Map keyBefore) { - this.keyBefore = keyBefore; - } - - public TriggerCdcEvent keyBefore(Map keyBefore) { - this.keyBefore = keyBefore; - return this; - } - - public Map getKeyAfter() { - return keyAfter; - } - - public void setKeyAfter(Map keyAfter) { - this.keyAfter = keyAfter; - } - - public TriggerCdcEvent keyAfter(Map keyAfter) { - this.keyAfter = keyAfter; - return this; - } - - public Map getBefore() { - return before; - } - - public void setBefore(Map before) { - this.before = before; - } - - public TriggerCdcEvent before(Map before) { - this.before = before; - return this; - } - - public Map getAfter() { - return after; - } - - public void setAfter(Map after) { - this.after = after; - } - - public TriggerCdcEvent after(Map after) { - this.after = after; - return this; - } - - public String getTransactionId() { - return transactionId; - } - - public void setTransactionId(String transactionId) { - this.transactionId = transactionId; - } - - public TriggerCdcEvent transactionId(String transactionId) { - this.transactionId = transactionId; - return this; - } - - public Long getTransactionSeq() { - return transactionSeq; - } - - public void setTransactionSeq(Long transactionSeq) { - this.transactionSeq = transactionSeq; - } - - public TriggerCdcEvent transactionSeq(Long transactionSeq) { - this.transactionSeq = transactionSeq; - return this; - } - - public Long getCapturedAt() { - return capturedAt; - } - - public void setCapturedAt(Long capturedAt) { - this.capturedAt = capturedAt; - } - - public TriggerCdcEvent capturedAt(Long capturedAt) { - this.capturedAt = capturedAt; - return this; - } - - public String getDdlPayload() { - return ddlPayload; - } - - public void setDdlPayload(String ddlPayload) { - this.ddlPayload = ddlPayload; - } - - public TriggerCdcEvent ddlPayload(String ddlPayload) { - this.ddlPayload = ddlPayload; - return this; - } - - public Integer getPayloadVersion() { - return payloadVersion; - } - - public void setPayloadVersion(Integer payloadVersion) { - this.payloadVersion = payloadVersion; - } - - public TriggerCdcEvent payloadVersion(Integer payloadVersion) { - this.payloadVersion = payloadVersion; - return this; - } - - @Override - public String toString() { - return "TriggerCdcEvent{eventId=" + eventId + ", captureId=" + captureId + ", generation=" + generation - + ", sourceTableName=" + sourceTableName + ", operation=" + operation - + ", schemaVersion=" + schemaVersion + ", capturedAt=" + capturedAt + "}"; - } -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInspectResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInspectResult.java deleted file mode 100644 index c51ab1de..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInspectResult.java +++ /dev/null @@ -1,207 +0,0 @@ -package io.tapdata.pdk.apis.entity.triggercdc; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -/** - * Health report of one capture, produced by - * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcInspectFunction}. - * - *

The report drives the DEGRADED / REPAIR_REQUIRED states. It is also the only honest way to - * report a structure drift: metadata polling can detect that a table changed, but it cannot - * reconstruct the DDL history, so the framework must degrade instead of guessing.

- */ -public class TriggerCdcInspectResult implements Serializable { - private static final long serialVersionUID = 1L; - - protected TriggerCdcStatus status; - protected String captureId; - protected Long generation; - protected String sourceTableId; - protected String sourceTableName; - protected Long schemaVersion; - /** State of the trigger, the journal table and the registry entries. */ - protected List objects; - /** True when the live table structure no longer matches the registered schema version. */ - protected boolean schemaDrift; - /** Columns added or dropped since the registered version, when drift was detected. */ - protected List driftedColumns; - /** - * True when a change may have been lost, for example a disabled trigger window, a - * BEST_EFFORT write failure, an overflowed managed DDL buffer, or a forced retention clean - * that removed records a consumer had not acknowledged (see - * {@link TriggerCdcCleanupResult#getIntegrityBrokenConsumers()}). - */ - protected boolean dataIntegrityBroken; - protected String integrityMessage; - /** Number of change records not yet acknowledged by any consumer. */ - protected Long unackedCount; - /** Capture time of the oldest unacknowledged record, for lag monitoring. */ - protected Long oldestUnackedCapturedAt; - /** Bytes the journal occupies, when the database can report it cheaply. */ - protected Long journalBytes; - protected List warnings; - protected List errors; - - public static TriggerCdcInspectResult create() { - return new TriggerCdcInspectResult(); - } - - public TriggerCdcInspectResult object(TriggerCdcObjectState objectState) { - if (objects == null) { - objects = new ArrayList<>(); - } - objects.add(objectState); - return this; - } - - public TriggerCdcInspectResult warning(String warning) { - if (warnings == null) { - warnings = new ArrayList<>(); - } - warnings.add(warning); - return this; - } - - public TriggerCdcInspectResult error(String error) { - if (errors == null) { - errors = new ArrayList<>(); - } - errors.add(error); - return this; - } - - public TriggerCdcStatus getStatus() { - return status; - } - - public void setStatus(TriggerCdcStatus status) { - this.status = status; - } - - public TriggerCdcInspectResult status(TriggerCdcStatus status) { - this.status = status; - return this; - } - - public String getCaptureId() { - return captureId; - } - - public void setCaptureId(String captureId) { - this.captureId = captureId; - } - - public Long getGeneration() { - return generation; - } - - public void setGeneration(Long generation) { - this.generation = generation; - } - - public String getSourceTableId() { - return sourceTableId; - } - - public void setSourceTableId(String sourceTableId) { - this.sourceTableId = sourceTableId; - } - - public String getSourceTableName() { - return sourceTableName; - } - - public void setSourceTableName(String sourceTableName) { - this.sourceTableName = sourceTableName; - } - - public Long getSchemaVersion() { - return schemaVersion; - } - - public void setSchemaVersion(Long schemaVersion) { - this.schemaVersion = schemaVersion; - } - - public List getObjects() { - return objects; - } - - public void setObjects(List objects) { - this.objects = objects; - } - - public boolean isSchemaDrift() { - return schemaDrift; - } - - public void setSchemaDrift(boolean schemaDrift) { - this.schemaDrift = schemaDrift; - } - - public List getDriftedColumns() { - return driftedColumns; - } - - public void setDriftedColumns(List driftedColumns) { - this.driftedColumns = driftedColumns; - } - - public boolean isDataIntegrityBroken() { - return dataIntegrityBroken; - } - - public void setDataIntegrityBroken(boolean dataIntegrityBroken) { - this.dataIntegrityBroken = dataIntegrityBroken; - } - - public String getIntegrityMessage() { - return integrityMessage; - } - - public void setIntegrityMessage(String integrityMessage) { - this.integrityMessage = integrityMessage; - } - - public Long getUnackedCount() { - return unackedCount; - } - - public void setUnackedCount(Long unackedCount) { - this.unackedCount = unackedCount; - } - - public Long getOldestUnackedCapturedAt() { - return oldestUnackedCapturedAt; - } - - public void setOldestUnackedCapturedAt(Long oldestUnackedCapturedAt) { - this.oldestUnackedCapturedAt = oldestUnackedCapturedAt; - } - - public Long getJournalBytes() { - return journalBytes; - } - - public void setJournalBytes(Long journalBytes) { - this.journalBytes = journalBytes; - } - - public List getWarnings() { - return warnings; - } - - public void setWarnings(List warnings) { - this.warnings = warnings; - } - - public List getErrors() { - return errors; - } - - public void setErrors(List errors) { - this.errors = errors; - } -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallMode.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallMode.java deleted file mode 100644 index 113a4f29..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallMode.java +++ /dev/null @@ -1,20 +0,0 @@ -package io.tapdata.pdk.apis.entity.triggercdc; - -/** - * Who executes the generated CDC objects and triggers. - */ -public enum TriggerCdcInstallMode { - /** - * Default. The framework generates the statements and executes them itself, then verifies - * the result. Installation must be idempotent and must never overwrite an object Tapdata - * does not own. - */ - AUTO, - /** - * The framework only generates the statements through - * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcPlanFunction}; a DBA - * installs them and Tapdata verifies the result afterwards. Both paths must produce exactly - * the same object set and the same version. - */ - MANUAL -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallPlan.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallPlan.java deleted file mode 100644 index 2821b4c4..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallPlan.java +++ /dev/null @@ -1,177 +0,0 @@ -package io.tapdata.pdk.apis.entity.triggercdc; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -/** - * Read only result of {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcPlanFunction}. - * - *

A plan must be side effect free: nothing is created, so the UI can show the cost, the - * privileges and the blocking problems before the operator commits.

- */ -public class TriggerCdcInstallPlan implements Serializable { - private static final long serialVersionUID = 1L; - - protected String catalog; - protected String schema; - /** Objects that will exist after installation. */ - protected List objects; - /** - * Ordered statements to execute. A plan without statements can never be installed — in - * {@code AUTO} nothing would be created, in {@code MANUAL} the DBA would have nothing to run — - * so an empty list always blocks the plan and must be reported through - * {@link #isBlocked()} with a reason in {@link #getBlockedReason()}. - */ - protected List statements; - /** Tables the connector accepts into this capture. */ - protected List supportedTables; - /** - * Tables the connector refuses, with the reason inside - * {@link TriggerCdcTableResult#getMessage()}. A blocked table fails the plan instead of - * being silently skipped, because a silently skipped table is a silent data gap. - */ - protected List blockedTables; - /** Non blocking remarks: cost expectation, high update rate warnings, existing triggers. */ - protected List warnings; - /** True when installation must not proceed. */ - protected boolean blocked; - /** Why the plan is blocked, when {@link #blocked} is true. */ - protected String blockedReason; - - public static TriggerCdcInstallPlan create() { - return new TriggerCdcInstallPlan(); - } - - public TriggerCdcInstallPlan object(TriggerCdcObject object) { - if (objects == null) { - objects = new ArrayList<>(); - } - objects.add(object); - return this; - } - - public TriggerCdcInstallPlan statement(TriggerCdcStatement statement) { - if (statements == null) { - statements = new ArrayList<>(); - } - statements.add(statement); - return this; - } - - public TriggerCdcInstallPlan warning(String warning) { - if (warnings == null) { - warnings = new ArrayList<>(); - } - warnings.add(warning); - return this; - } - - public TriggerCdcInstallPlan supportedTable(TriggerCdcTable table) { - if (supportedTables == null) { - supportedTables = new ArrayList<>(); - } - supportedTables.add(table); - return this; - } - - public TriggerCdcInstallPlan blockedTable(TriggerCdcTableResult result) { - if (blockedTables == null) { - blockedTables = new ArrayList<>(); - } - blockedTables.add(result); - return this; - } - - public String getCatalog() { - return catalog; - } - - public void setCatalog(String catalog) { - this.catalog = catalog; - } - - public TriggerCdcInstallPlan catalog(String catalog) { - this.catalog = catalog; - return this; - } - - public String getSchema() { - return schema; - } - - public void setSchema(String schema) { - this.schema = schema; - } - - public TriggerCdcInstallPlan schema(String schema) { - this.schema = schema; - return this; - } - - public List getObjects() { - return objects; - } - - public void setObjects(List objects) { - this.objects = objects; - } - - public List getStatements() { - return statements; - } - - public void setStatements(List statements) { - this.statements = statements; - } - - public List getSupportedTables() { - return supportedTables; - } - - public void setSupportedTables(List supportedTables) { - this.supportedTables = supportedTables; - } - - public List getBlockedTables() { - return blockedTables; - } - - public void setBlockedTables(List blockedTables) { - this.blockedTables = blockedTables; - } - - public List getWarnings() { - return warnings; - } - - public void setWarnings(List warnings) { - this.warnings = warnings; - } - - public boolean isBlocked() { - return blocked; - } - - public void setBlocked(boolean blocked) { - this.blocked = blocked; - } - - public TriggerCdcInstallPlan blocked(boolean blocked) { - this.blocked = blocked; - return this; - } - - public String getBlockedReason() { - return blockedReason; - } - - public void setBlockedReason(String blockedReason) { - this.blockedReason = blockedReason; - } - - public TriggerCdcInstallPlan blockedReason(String blockedReason) { - this.blockedReason = blockedReason; - return this; - } -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java index a39027f2..066b4700 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java @@ -31,6 +31,8 @@ public class TriggerCdcInstallResult implements Serializable { private static final long serialVersionUID = 1L; + /** Database/version-specific guarantees verified during ensure; required even on failure. */ + protected TriggerCdcCapabilities capabilities; protected TriggerCdcStatus status; protected String catalog; protected String schema; @@ -45,6 +47,19 @@ public class TriggerCdcInstallResult implements Serializable { /** Non blocking remarks raised during installation. */ protected List warnings; + public TriggerCdcCapabilities getCapabilities() { + return capabilities; + } + + public void setCapabilities(TriggerCdcCapabilities capabilities) { + this.capabilities = capabilities; + } + + public TriggerCdcInstallResult capabilities(TriggerCdcCapabilities capabilities) { + this.capabilities = capabilities; + return this; + } + public static TriggerCdcInstallResult create() { return new TriggerCdcInstallResult(); } @@ -156,6 +171,9 @@ public List validate() { */ public List validate(Collection requestedTables) { List violations = new ArrayList<>(); + if (capabilities == null) { + violations.add("ensure must report the verified database capabilities"); + } Map requested = indexRequested(requestedTables, violations); if (tableResults == null || tableResults.isEmpty()) { violations.add("ensure reported no per table result; at least one table outcome is " @@ -197,6 +215,9 @@ public List validate(Collection requestedTables) { + "cannot be addressed, so it would be installed and never read"); continue; } + if (tableResult.getGeneration() == null) { + violations.add(owner + " is reported successful but has no generation"); + } String captureId = tableResult.getCaptureId().trim(); String previousOwner = captureOwners.put(captureId, owner); if (previousOwner != null) { diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObject.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObject.java deleted file mode 100644 index dfe497ee..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObject.java +++ /dev/null @@ -1,132 +0,0 @@ -package io.tapdata.pdk.apis.entity.triggercdc; - -import java.io.Serializable; - -/** - * A database object that a capture owns or touches. - * - *

{@code ownedByTapdata} is the guard rail for uninstall: the framework may only drop - * objects it created itself, and must never drop an entire customer schema.

- */ -public class TriggerCdcObject implements Serializable { - private static final long serialVersionUID = 1L; - - protected TriggerCdcObjectType type; - protected String catalog; - protected String schema; - /** Physical object name, as the connector will really create it. */ - protected String name; - /** - * True when this object is created and owned by Tapdata, so releasing may drop it. - * - *

Defaults to {@code false} on purpose: the guard rail has to fail closed. A connector that - * forgets to set the flag leaves an existing customer object untouchable instead of - * accidentally licensing a {@code DROP} on it.

- */ - protected boolean ownedByTapdata; - /** True when the object already exists and will be reused instead of created. */ - protected boolean alreadyExists; - /** Human readable note, for example why a name differs from the suggested one. */ - protected String comment; - - public static TriggerCdcObject create() { - return new TriggerCdcObject(); - } - - public TriggerCdcObjectType getType() { - return type; - } - - public void setType(TriggerCdcObjectType type) { - this.type = type; - } - - public TriggerCdcObject type(TriggerCdcObjectType type) { - this.type = type; - return this; - } - - public String getCatalog() { - return catalog; - } - - public void setCatalog(String catalog) { - this.catalog = catalog; - } - - public TriggerCdcObject catalog(String catalog) { - this.catalog = catalog; - return this; - } - - public String getSchema() { - return schema; - } - - public void setSchema(String schema) { - this.schema = schema; - } - - public TriggerCdcObject schema(String schema) { - this.schema = schema; - return this; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public TriggerCdcObject name(String name) { - this.name = name; - return this; - } - - public boolean isOwnedByTapdata() { - return ownedByTapdata; - } - - public void setOwnedByTapdata(boolean ownedByTapdata) { - this.ownedByTapdata = ownedByTapdata; - } - - public TriggerCdcObject ownedByTapdata(boolean ownedByTapdata) { - this.ownedByTapdata = ownedByTapdata; - return this; - } - - public boolean isAlreadyExists() { - return alreadyExists; - } - - public void setAlreadyExists(boolean alreadyExists) { - this.alreadyExists = alreadyExists; - } - - public TriggerCdcObject alreadyExists(boolean alreadyExists) { - this.alreadyExists = alreadyExists; - return this; - } - - public String getComment() { - return comment; - } - - public void setComment(String comment) { - this.comment = comment; - } - - public TriggerCdcObject comment(String comment) { - this.comment = comment; - return this; - } - - @Override - public String toString() { - return "TriggerCdcObject{type=" + type + ", schema=" + schema + ", name=" + name - + ", ownedByTapdata=" + ownedByTapdata + ", alreadyExists=" + alreadyExists + "}"; - } -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectState.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectState.java deleted file mode 100644 index 6cff5919..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectState.java +++ /dev/null @@ -1,111 +0,0 @@ -package io.tapdata.pdk.apis.entity.triggercdc; - -import java.io.Serializable; - -/** - * Live state of one object a capture depends on, as observed by - * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcInspectFunction}. - */ -public class TriggerCdcObjectState implements Serializable { - private static final long serialVersionUID = 1L; - - protected TriggerCdcObjectType type; - protected String schema; - protected String name; - /** False when the object is missing, which breaks the capture. */ - protected boolean present; - /** - * For triggers: false when the trigger exists but is disabled, so changes silently stop - * being journaled. - */ - protected boolean enabled; - protected String message; - - public static TriggerCdcObjectState create() { - return new TriggerCdcObjectState(); - } - - public TriggerCdcObjectType getType() { - return type; - } - - public void setType(TriggerCdcObjectType type) { - this.type = type; - } - - public TriggerCdcObjectState type(TriggerCdcObjectType type) { - this.type = type; - return this; - } - - public String getSchema() { - return schema; - } - - public void setSchema(String schema) { - this.schema = schema; - } - - public TriggerCdcObjectState schema(String schema) { - this.schema = schema; - return this; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public TriggerCdcObjectState name(String name) { - this.name = name; - return this; - } - - public boolean isPresent() { - return present; - } - - public void setPresent(boolean present) { - this.present = present; - } - - public TriggerCdcObjectState present(boolean present) { - this.present = present; - return this; - } - - public boolean isEnabled() { - return enabled; - } - - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public TriggerCdcObjectState enabled(boolean enabled) { - this.enabled = enabled; - return this; - } - - public String getMessage() { - return message; - } - - public void setMessage(String message) { - this.message = message; - } - - public TriggerCdcObjectState message(String message) { - this.message = message; - return this; - } - - @Override - public String toString() { - return "TriggerCdcObjectState{type=" + type + ", name=" + name + ", present=" + present - + ", enabled=" + enabled + ", message=" + message + "}"; - } -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectType.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectType.java deleted file mode 100644 index 2f601927..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcObjectType.java +++ /dev/null @@ -1,25 +0,0 @@ -package io.tapdata.pdk.apis.entity.triggercdc; - -/** - * The kinds of database objects a capture needs. - * - *

Object names are suggestions owned by the framework - * (see {@link TriggerCdcConstants}); the connector maps them onto real database - * semantics (catalog / schema / database / keyspace...).

- */ -public enum TriggerCdcObjectType { - /** The namespace (schema / database) that holds the CDC objects. */ - STORAGE_NAMESPACE, - /** Registry of installed captures: source table identity, trigger names, generation, state. */ - CAPTURE_REGISTRY, - /** Immutable history of registered table structure versions. */ - SCHEMA_HISTORY, - /** Per source table journal of captured changes. */ - CHANGE_TABLE, - /** Per consumer persistence of acknowledged events. */ - CONSUMER_ACK, - /** Optional bucket for capture errors the database can isolate safely. */ - CAPTURE_ERROR, - /** The trigger (or trigger set) on the source table itself. */ - TRIGGER -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcOffset.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcOffset.java index 8439a045..5bda540d 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcOffset.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcOffset.java @@ -3,53 +3,39 @@ import java.io.Serializable; /** - * Reader position of one consumer on one capture. + * Delivered position of one consumer on one capture, carried through the existing stream API. * - *

Scope: the framework stores one offset per {@code (consumerId, captureId)} pair, never one - * offset per task. One source table is one capture and owns its own journal, so a task that - * captures several tables has several independent positions, and each - * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcReadFunction read} call is - * given the offset of the capture it reads. There is no cross capture offset: comparing two - * captures' tokens or reusing one capture's offset for another is meaningless and must be - * rejected.

+ *

StreamReadFunction receives/emits a {@code Map} keyed by capture id, + * even for a one-table task. The map contains the latest delivered token for every subscribed + * capture. Each callback emits a fresh snapshot (map and values) that must not subsequently be + * mutated. Events use the existing TapEvent types; no parallel event/batch protocol is needed.

* - *

Important: {@code lastEventId} is informational only. A database sequence gives unique - * values, not commit order, so {@code event_id > lastEventId} is not a safe cursor:

+ *

A token identifies the cumulative set of events delivered on that capture, including previous + * unacknowledged batches. It is bound to the consumer, capture and generation. The engine persists + * the whole map as its normal checkpoint and acknowledges its entries only after all downstream + * writes and that checkpoint are durable. Delivery alone never permits journal deletion. On + * restart, unacknowledged records may replay; targets must tolerate at-least-once delivery.

* - *
- * tx A allocates event_id=100, not yet committed
- * tx B allocates event_id=101, commits first
- * reader sees 101, advances the cursor to 101
- * tx A commits 100 -> 100 is now permanently skipped
- * 
+ *

Before reading, validate the map against the captures registered by ensure: reject unknown + * keys, a key/captureId mismatch, missing identities or generations, stale generations, and + * foreign tokens. A null start offset starts from the registered subscription's beginning; + * a non-null checkpoint must cover the exact registered capture set, including fresh entries. + * Do not infer journal routing from a token alone.

* - *

The safe cursor is {@code batchToken}: the connector returns it with a delivered batch and - * the framework only advances it through - * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcAcknowledgeFunction} after - * the downstream sink is durable. A connector must therefore treat an unacknowledged batch as - * still pending on every read, even when a larger event id is already visible.

+ *

Sequence allocation is not commit order. For example, transaction A allocates 100, B allocates + * and commits 101, then A commits 100. Never use {@code event_id > lastEventId} as a permanent + * cursor. Recheck the committed unacknowledged set, tracking delivered rows during a running + * session without deleting them. Acknowledge membership in the token's set, never a numeric high + * watermark. Check capture integrity while polling and fail on drift or disabled triggers.

*/ public class TriggerCdcOffset implements Serializable { private static final long serialVersionUID = 1L; - /** - * Capture this offset belongs to; also the key under which it is persisted. It must equal the - * {@code captureId} passed to a read call - a mismatch means the checkpoint of another capture - * was restored and must be rejected instead of resumed. - */ + /** Capture identity; must equal the key under which this offset is checkpointed. */ protected String captureId; - /** - * Generation of the capture. A connector must reject an offset whose generation does not - * match the installed capture, instead of resuming from a stale position. - */ + /** Installed generation; a mismatch must fail instead of resuming a stale checkpoint. */ protected Long generation; - /** - * Opaque token of the last acknowledged batch. A token that was delivered but not yet - * acknowledged is never stored here: it lives only in the returned - * {@link TriggerCdcBatch}, so the connector re-delivers that batch until - * {@link io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcAcknowledgeFunction} - * reports the downstream write as durable. - */ + /** Cumulative delivered-set token; only a durable checkpoint authorizes acknowledgement. */ protected String batchToken; /** Informational only. Never use it as a safe cursor, see the class javadoc. */ protected String lastEventId; @@ -60,7 +46,7 @@ public static TriggerCdcOffset create() { return new TriggerCdcOffset(); } - /** True when nothing has been acknowledged yet, so the connector starts from the beginning. */ + /** True when no batch has been delivered for this position yet. */ public boolean isFresh() { return batchToken == null || batchToken.isEmpty(); } diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatement.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatement.java deleted file mode 100644 index f575a62f..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatement.java +++ /dev/null @@ -1,112 +0,0 @@ -package io.tapdata.pdk.apis.entity.triggercdc; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -/** - * One DDL statement of a {@link TriggerCdcInstallPlan}. - * - *

The plan is also the MANUAL install artefact: a DBA executes exactly these statements. - * Therefore the statements must be complete, ordered, and identical to what - * {@code ensure} would run.

- */ -public class TriggerCdcStatement implements Serializable { - private static final long serialVersionUID = 1L; - - protected TriggerCdcStatementType type; - protected String sql; - /** What the statement is for, shown in the UI and in the generated install script. */ - protected String description; - /** True when re-running the statement is harmless. */ - protected boolean idempotent; - /** True when the statement locks a customer table, so the UI can warn about it. */ - protected boolean mayLockSourceTable; - /** Privileges a DBA needs for this statement, for example "CREATE TRIGGER". */ - protected List requiredPrivileges; - - public static TriggerCdcStatement create() { - return new TriggerCdcStatement(); - } - - public TriggerCdcStatement privilege(String privilege) { - if (requiredPrivileges == null) { - requiredPrivileges = new ArrayList<>(); - } - requiredPrivileges.add(privilege); - return this; - } - - public TriggerCdcStatementType getType() { - return type; - } - - public void setType(TriggerCdcStatementType type) { - this.type = type; - } - - public TriggerCdcStatement type(TriggerCdcStatementType type) { - this.type = type; - return this; - } - - public String getSql() { - return sql; - } - - public void setSql(String sql) { - this.sql = sql; - } - - public TriggerCdcStatement sql(String sql) { - this.sql = sql; - return this; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public TriggerCdcStatement description(String description) { - this.description = description; - return this; - } - - public boolean isIdempotent() { - return idempotent; - } - - public void setIdempotent(boolean idempotent) { - this.idempotent = idempotent; - } - - public TriggerCdcStatement idempotent(boolean idempotent) { - this.idempotent = idempotent; - return this; - } - - public boolean isMayLockSourceTable() { - return mayLockSourceTable; - } - - public void setMayLockSourceTable(boolean mayLockSourceTable) { - this.mayLockSourceTable = mayLockSourceTable; - } - - public TriggerCdcStatement mayLockSourceTable(boolean mayLockSourceTable) { - this.mayLockSourceTable = mayLockSourceTable; - return this; - } - - public List getRequiredPrivileges() { - return requiredPrivileges; - } - - public void setRequiredPrivileges(List requiredPrivileges) { - this.requiredPrivileges = requiredPrivileges; - } -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatementType.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatementType.java deleted file mode 100644 index 7ad992fd..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatementType.java +++ /dev/null @@ -1,15 +0,0 @@ -package io.tapdata.pdk.apis.entity.triggercdc; - -/** - * Kind of DDL statement inside a {@link TriggerCdcInstallPlan}. - */ -public enum TriggerCdcStatementType { - CREATE_NAMESPACE, - CREATE_TABLE, - CREATE_INDEX, - CREATE_TRIGGER, - DROP_TRIGGER, - DROP_TABLE, - DROP_NAMESPACE, - OTHER -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatus.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatus.java index ae1f18bb..53a5d470 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatus.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcStatus.java @@ -4,14 +4,12 @@ * Lifecycle state of one capture (one source table, one generation). * *
- * PLANNED -> INSTALLING -> CAPTURING -> SNAPSHOTTING -> STREAMING
+ * INSTALLING -> CAPTURING -> SNAPSHOTTING -> STREAMING
  *                                |                              |
  *                                +---------> DEGRADED / REPAIR_REQUIRED
  * 
*/ public enum TriggerCdcStatus { - /** Plan generated, nothing created yet. */ - PLANNED, /** Objects / triggers are being created. Failure is retryable and reported per object. */ INSTALLING, /** Triggers are live and journaling changes, no consumer is reading yet. */ @@ -37,7 +35,5 @@ public enum TriggerCdcStatus { */ REPAIR_REQUIRED, /** Installation or a later verification failed. */ - FAILED, - /** Objects released, no longer capturing. */ - RELEASED + FAILED } diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTable.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTable.java index a324c9e3..b309a40d 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTable.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTable.java @@ -14,7 +14,7 @@ * generation so that old offsets are rejected instead of silently resumed.

* *

Granularity: one source table is one capture. Each capture owns its own journal, and - * {@code read} / {@code acknowledge} are called once per capture, so a table list is also the + * stream checkpoints and acknowledgements keep independent entries per capture, so a table list is the * list of independent read positions a task has to keep.

*/ public class TriggerCdcTable implements Serializable { diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTableResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTableResult.java index 32960d69..56d3a3b4 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTableResult.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcTableResult.java @@ -3,7 +3,7 @@ import java.io.Serializable; /** - * Per table outcome of a plan / install / release operation. + * Per table outcome of an installation or verification. * *

Installation is reported per object, not as one boolean, because a partial install must be * retried idempotently and the operator needs to know which table is missing its trigger.

@@ -44,12 +44,7 @@ public static TriggerCdcTableResult create() { * {@link #failure(TriggerCdcTable, String, String)} for anything that did not become a usable * capture.

* - *

This factory is for {@code ensure} outcomes only. A read only plan reports accepted tables - * through {@link TriggerCdcInstallPlan#supportedTable(TriggerCdcTable)} and refusals through - * {@link #failure(TriggerCdcTable, String, String)}; plan time tables have no capture id yet, - * so calling this there would rightly fail.

- * - * @throws IllegalArgumentException when {@code table} is null or has no capture id + * @throws IllegalArgumentException when the table, capture id or generation is missing */ public static TriggerCdcTableResult success(TriggerCdcTable table) { if (table == null) { @@ -59,6 +54,9 @@ public static TriggerCdcTableResult success(TriggerCdcTable table) { throw new IllegalArgumentException("table " + table.getTableName() + " is reported successful but has no captureId; the journal would be unreachable"); } + if (table.getGeneration() == null) { + throw new IllegalArgumentException("successful capture must include generation"); + } TriggerCdcTableResult result = new TriggerCdcTableResult(); result.success = true; result.sourceTableId = table.getSourceTableId(); @@ -109,14 +107,14 @@ public TriggerCdcTableResult tableName(String tableName) { } /** - * True only when this entry reports a success and carries a usable capture id. + * True only when this entry reports a success and carries a usable capture id and generation. * *

The raw flag is deliberately not enough: {@link #success(boolean)} can be set before * {@link #captureId(String)} in a builder chain, and a success that never receives an id must * be treated as a failure here rather than passed on as "installed".

*/ public boolean isSuccess() { - return success && hasCaptureId(); + return success && hasCaptureId() && generation != null; } /** True when {@link #getCaptureId()} is present and not blank. */ @@ -135,7 +133,7 @@ public void setSuccess(boolean success) { /** * Sets the raw success flag. Pair {@code true} with {@link #captureId(String)}: a success - * without a capture id is reported as a failure by {@link #isSuccess()}. + * without a capture id or generation is reported as a failure by {@link #isSuccess()}. */ public TriggerCdcTableResult success(boolean success) { this.success = success; diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/ConnectorFunctions.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/ConnectorFunctions.java index a8c5bc64..3aed8e0f 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/ConnectorFunctions.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/ConnectorFunctions.java @@ -88,14 +88,9 @@ public class ConnectorFunctions extends ConnectionFunctions protected FlushOffsetFunction flushOffsetFunction; protected ProcessControlFunction processControlFunction; - protected TriggerCdcCapabilitiesFunction triggerCdcCapabilitiesFunction; - protected TriggerCdcPlanFunction triggerCdcPlanFunction; protected TriggerCdcEnsureFunction triggerCdcEnsureFunction; - protected TriggerCdcReadFunction triggerCdcReadFunction; protected TriggerCdcAcknowledgeFunction triggerCdcAcknowledgeFunction; - protected TriggerCdcInspectFunction triggerCdcInspectFunction; protected TriggerCdcCleanupFunction triggerCdcCleanupFunction; - protected TriggerCdcReleaseFunction triggerCdcReleaseFunction; public ConnectorFunctions supportFlushOffsetFunction(FlushOffsetFunction function) { flushOffsetFunction = function; @@ -107,46 +102,21 @@ public ConnectorFunctions supportProcessControlFunction(ProcessControlFunction f return this; } - public ConnectorFunctions supportTriggerCdcCapabilitiesFunction(TriggerCdcCapabilitiesFunction function) { - triggerCdcCapabilitiesFunction = function; - return this; - } - - public ConnectorFunctions supportTriggerCdcPlanFunction(TriggerCdcPlanFunction function) { - triggerCdcPlanFunction = function; - return this; - } - public ConnectorFunctions supportTriggerCdcEnsureFunction(TriggerCdcEnsureFunction function) { triggerCdcEnsureFunction = function; return this; } - public ConnectorFunctions supportTriggerCdcReadFunction(TriggerCdcReadFunction function) { - triggerCdcReadFunction = function; - return this; - } - public ConnectorFunctions supportTriggerCdcAcknowledgeFunction(TriggerCdcAcknowledgeFunction function) { triggerCdcAcknowledgeFunction = function; return this; } - public ConnectorFunctions supportTriggerCdcInspectFunction(TriggerCdcInspectFunction function) { - triggerCdcInspectFunction = function; - return this; - } - public ConnectorFunctions supportTriggerCdcCleanupFunction(TriggerCdcCleanupFunction function) { triggerCdcCleanupFunction = function; return this; } - public ConnectorFunctions supportTriggerCdcReleaseFunction(TriggerCdcReleaseFunction function) { - triggerCdcReleaseFunction = function; - return this; - } - public ConnectorFunctions supportTransactionBeginFunction(TransactionBeginFunction function) { transactionBeginFunction = function; return this; @@ -270,9 +240,8 @@ public List getCapabilities() { } } catch (Throwable ignored) {} } - if (triggerCdcCapabilitiesFunction != null && triggerCdcPlanFunction != null - && triggerCdcEnsureFunction != null && triggerCdcReadFunction != null - && triggerCdcAcknowledgeFunction != null) { + if (triggerCdcEnsureFunction != null && streamReadFunction != null + && triggerCdcAcknowledgeFunction != null && triggerCdcCleanupFunction != null) { fieldArray.add(Capability.create(io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConstants.CAPABILITY_TRIGGER_CDC) .type(Capability.TYPE_OTHER)); } @@ -547,38 +516,18 @@ public ControlFunction getControlFunction() { return controlFunction; } - public TriggerCdcCapabilitiesFunction getTriggerCdcCapabilitiesFunction() { - return triggerCdcCapabilitiesFunction; - } - - public TriggerCdcPlanFunction getTriggerCdcPlanFunction() { - return triggerCdcPlanFunction; - } - public TriggerCdcEnsureFunction getTriggerCdcEnsureFunction() { return triggerCdcEnsureFunction; } - public TriggerCdcReadFunction getTriggerCdcReadFunction() { - return triggerCdcReadFunction; - } - public TriggerCdcAcknowledgeFunction getTriggerCdcAcknowledgeFunction() { return triggerCdcAcknowledgeFunction; } - public TriggerCdcInspectFunction getTriggerCdcInspectFunction() { - return triggerCdcInspectFunction; - } - public TriggerCdcCleanupFunction getTriggerCdcCleanupFunction() { return triggerCdcCleanupFunction; } - public TriggerCdcReleaseFunction getTriggerCdcReleaseFunction() { - return triggerCdcReleaseFunction; - } - public CreateIndexFunction getCreateIndexFunction() { return createIndexFunction; } diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/PDKMethod.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/PDKMethod.java index 75264870..ea3a1914 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/PDKMethod.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/PDKMethod.java @@ -68,14 +68,9 @@ public enum PDKMethod { GET_STREAM_OFFSET, EXPORT_EVENT_SQL, EXECUTE_COMMAND, - TRIGGER_CDC_CAPABILITIES, - TRIGGER_CDC_PLAN, TRIGGER_CDC_ENSURE, - TRIGGER_CDC_READ, TRIGGER_CDC_ACKNOWLEDGE, - TRIGGER_CDC_INSPECT, - TRIGGER_CDC_CLEANUP, - TRIGGER_CDC_RELEASE + TRIGGER_CDC_CLEANUP ; PDKMethod() { diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/source/StreamReadFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/source/StreamReadFunction.java index 6be6abd8..5e7a69d8 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/source/StreamReadFunction.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/source/StreamReadFunction.java @@ -8,6 +8,10 @@ public interface StreamReadFunction extends TapConnectorFunction { /** + * For Trigger CDC, ensure first registers the node's consumer and captures. The offset is a + * complete map keyed by capture id as specified by + * {@link io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcOffset}. Emit ordinary TapEvents and + * a fresh offset snapshot; callback return does not authorize acknowledgement or cleanup. * * @param nodeContext the node context in a DAG * @param offsetState if null, means start from very beginning, otherwise is the start point for batch reading. diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcAcknowledgeFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcAcknowledgeFunction.java index 035066d1..4a8d9e14 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcAcknowledgeFunction.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcAcknowledgeFunction.java @@ -2,35 +2,31 @@ import io.tapdata.pdk.apis.context.TapConnectorContext; import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConfig; +import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcOffset; import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; /** - * Marks a delivered batch of one capture as safely processed by one consumer. + * Confirms a recoverable checkpoint of one consumer on one capture. Idempotent. * - *

The framework calls this only after the downstream write and the recoverable checkpoint are - * both durable. Connectors must treat the acknowledgement as the single source of truth for the - * consumer position on that capture, and must keep the referenced records visible until every - * live consumer has acknowledged them.

+ *

The engine calls this only after every downstream destination has durably written the + * covered events and the corresponding checkpoint is durably recoverable. Returning from + * StreamReadConsumer.accept means delivery to the engine, not durable acknowledgement. Until + * that durable engine integration exists, cleanup must not treat delivered events as consumed.

* - *

Acknowledgement is per capture, mirroring {@link TriggerCdcReadFunction}: a task that - * captures several tables acknowledges each capture separately, so one capture can never appear - * consumed because a sibling capture advanced. A token belongs to exactly one capture and may - * only be acknowledged against it.

- * - *

Acknowledgement is the reason the mode can be called reliable: a crash between "written" and - * "acknowledged" replays the batch (at least once), and the target side must be idempotent.

+ *

Reject null/blank consumer ids, missing capture id/generation/token, unknown subscriptions, + * and tokens not issued to that consumer, capture and generation. A token covers the cumulative + * delivered set of its capture, including earlier unacknowledged batches. Acknowledge exactly + * that set, never every row with a smaller event id: allocation order is not commit order. A + * delayed/repeated acknowledgement must not regress progress. Persist acknowledgement before + * returning; a crash between checkpoint persistence and this call safely causes replay.

*/ public interface TriggerCdcAcknowledgeFunction extends TapConnectorFunction { /** - * @param connectorContext the connection / node context - * @param config connection level trigger CDC configuration - * @param consumerId stable id of the subscribing task / channel - * @param captureId capture the token belongs to, the same value that was passed to - * {@link TriggerCdcReadFunction}. A token that does not belong to this - * capture must be rejected instead of acknowledged. - * @param batchToken token returned by {@link TriggerCdcReadFunction} for the batch of - * this capture + * @param connectorContext the node context + * @param config effective connection-level trigger CDC configuration + * @param consumerId stable subscription identity passed to ensure + * @param offset one capture's offset from a durable checkpoint, never a delivery-only offset */ void acknowledge(TapConnectorContext connectorContext, TriggerCdcConfig config, String consumerId, - String captureId, String batchToken) throws Throwable; + TriggerCdcOffset offset) throws Throwable; } diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcCapabilitiesFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcCapabilitiesFunction.java deleted file mode 100644 index 6bacff59..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcCapabilitiesFunction.java +++ /dev/null @@ -1,26 +0,0 @@ -package io.tapdata.pdk.apis.functions.connector.triggercdc; - -import io.tapdata.pdk.apis.context.TapConnectorContext; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcCapabilities; -import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; - -/** - * Declares what trigger based CDC this connector really supports. - * - *

Registering this function is what enables the {@code triggerCdc} increment mode for a - * connector. A connector without it must not be offered the mode, and the framework must not - * call any other Trigger CDC function on it.

- * - *

Declarations are a contract, not a hint: the framework relies on them to decide whether a - * capture boundary exists, whether DDL may be replayed, and whether a full-then-incremental - * handover is safe. Overstating a capability causes silent data loss.

- */ -public interface TriggerCdcCapabilitiesFunction extends TapConnectorFunction { - /** - * @param connectorContext the connection / node context - * @return the declaration for the connected database and its version; never null - * @throws Throwable when the database version cannot be determined, so the mode stays - * unavailable instead of being guessed - */ - TriggerCdcCapabilities capabilities(TapConnectorContext connectorContext) throws Throwable; -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcEnsureFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcEnsureFunction.java index 8ec0633f..71ae68cd 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcEnsureFunction.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcEnsureFunction.java @@ -9,50 +9,38 @@ import java.util.List; /** - * Creates the CDC objects and triggers, then verifies them. Idempotent. + * Installs and verifies capture objects and registers a durable consumer before snapshot/read. * - *

Requirements:

- *
    - *
  • Running it twice returns the same capture ids and the same generation. Only a real - * reinstall increments the generation, so a stale offset can be rejected instead of - * silently resumed.
  • - *
  • Never overwrite an object Tapdata does not own. Report a conflict instead.
  • - *
  • Creating or replacing a trigger may lock the source table: use a short lock timeout, - * fail fast and stay retryable instead of waiting on customer traffic.
  • - *
  • Success must include the verification that the trigger is really enabled and that the - * installed capture set covers everything the caller asked for.
  • - *
  • Each successful table result must carry a non blank {@code captureId} that is unique - * within the result: that id is the only address of the journal. A success without an id, or - * two tables sharing an id, is rejected as an invalid result (see - * {@code TriggerCdcInstallResult#validate()}) instead of shrinking the capture list — - * dropping either would mean an installed table that is never read.
  • - *
  • Every requested table needs its own outcome, success or failure: a table that is - * simply absent from the result is neither readable nor retryable, so it is the worst - * outcome of all. Report it through {@code TriggerCdcInstallResult#requestedTables(...)} - * (or let the framework pass {@code tables} to {@code validate(tables)}) so the check can - * tell an omitted table from an unrequested one. Three shapes are rejected: a requested - * table with no outcome, one table with several outcomes, and an outcome for a table that - * was never requested.
  • - *
+ *

Idempotent for the same consumer and tables: reuse capture ids and generations. Increment + * the generation only on a real reinstall and reject old checkpoints afterwards. Never overwrite + * objects Tapdata does not own. Bound installation lock waits and report each table's outcome, + * including failures. Each success needs a unique nonblank capture id and a generation.

* - *

Whether installation forms a valid capture boundary is declared through - * {@code TriggerCdcCapabilities.captureBoundarySupported}. "CREATE TRIGGER returned" alone is - * never proof that no concurrent transaction was missed.

+ *

Database/version-specific capabilities are returned in the install result rather than + * obtained through a separate function. Validate requested options and unsupported types before + * changing objects. A capture boundary must be verified, not inferred from CREATE TRIGGER + * returning. Without a verified boundary, do not offer automatic full-to-incremental handover.

+ * + *

Call again before resume to verify triggers, journal and schema versions. The registered + * {@link io.tapdata.pdk.apis.functions.connector.source.StreamReadFunction} polls those captures, + * emits ordinary TapEvents, and must stop on missing/disabled triggers, schema drift or a broken + * consumer. There is no separate read, plan, inspect or release function in this version. + * Subscriptions survive stop, crash and task deletion until explicitly retired outside this API; + * they must never expire based on time alone.

+ * + *

The connector retains this consumer/config/capture mapping in its node-scoped runtime for + * streamRead; do not put task-specific state in a shared connection. The engine must validate the + * result against its original table list and persist the capture identities before starting reads. + * A partial installation must not silently shrink the requested task.

*/ public interface TriggerCdcEnsureFunction extends TapConnectorFunction { /** - * @param connectorContext the connection / node context - * @param config connection level trigger CDC configuration - * @param tables source tables to capture; capture ids and generations are filled in - * by the connector. One table yields one capture, and that capture is - * the unit of {@link TriggerCdcReadFunction} and - * {@link TriggerCdcAcknowledgeFunction} - * @return per table outcome, never null, with exactly one outcome per requested table (a - * failure is a valid outcome; silence is not) and the request retained through - * {@code TriggerCdcInstallResult#requestedTables(Collection)}. The framework passes - * {@code tables} to {@code validate(tables)} / {@code getCaptureIds(tables)}, so a - * result that does not account for every requested table is rejected rather than read - * as a smaller capture set + * @param connectorContext node-scoped connection context used for the subsequent streamRead + * @param config effective connection-level storage and capture configuration + * @param consumerId stable, nonblank subscription identity, preserved across task restarts + * @param tables source tables; exactly one outcome per requested table, retained in the result + * @return installation and verified capabilities; never null */ - TriggerCdcInstallResult ensure(TapConnectorContext connectorContext, TriggerCdcConfig config, List tables) throws Throwable; + TriggerCdcInstallResult ensure(TapConnectorContext connectorContext, TriggerCdcConfig config, + String consumerId, List tables) throws Throwable; } diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcInspectFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcInspectFunction.java deleted file mode 100644 index 56ebc2fb..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcInspectFunction.java +++ /dev/null @@ -1,30 +0,0 @@ -package io.tapdata.pdk.apis.functions.connector.triggercdc; - -import io.tapdata.pdk.apis.context.TapConnectorContext; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConfig; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcInspectResult; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcTable; -import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; - -import java.util.List; - -/** - * Health and integrity inspection of installed captures. - * - *

Used for periodic monitoring and before resuming a task. It must detect at least: a missing - * or disabled trigger, a missing journal table, a structure drift against the registered schema - * version, backlog size and lag, and journal growth.

- * - *

Drift detection is not DDL capture. Once drift is detected the capture must be marked - * degraded, because the DDL history between two inspections cannot be reconstructed and the - * changes of the missed window cannot be replayed.

- */ -public interface TriggerCdcInspectFunction extends TapConnectorFunction { - /** - * @param connectorContext the connection / node context - * @param config connection level trigger CDC configuration - * @param tables captures to inspect; empty means all captures known to the registry - * @return one report per capture, never null - */ - List inspect(TapConnectorContext connectorContext, TriggerCdcConfig config, List tables) throws Throwable; -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcPlanFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcPlanFunction.java deleted file mode 100644 index d6d03add..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcPlanFunction.java +++ /dev/null @@ -1,30 +0,0 @@ -package io.tapdata.pdk.apis.functions.connector.triggercdc; - -import io.tapdata.pdk.apis.context.TapConnectorContext; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConfig; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcInstallPlan; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcTable; -import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; - -import java.util.List; - -/** - * Builds the installation plan without touching the database. - * - *

Must be side effect free so the UI can show objects, statements, required privileges, - * warnings and blocking problems before anything is created. The same statements feed the - * {@code MANUAL} install path used by a DBA, so they must match what {@code ensure} would run.

- * - *

A plan must report problems instead of skipping them: a table whose types cannot be - * captured, or a table without a primary key, belongs in - * {@link TriggerCdcInstallPlan#getBlockedTables()} so the operator sees the gap.

- */ -public interface TriggerCdcPlanFunction extends TapConnectorFunction { - /** - * @param connectorContext the connection / node context - * @param config connection level trigger CDC configuration; already defaulted - * @param tables source tables the task wants to capture - * @return the plan, never null - */ - TriggerCdcInstallPlan plan(TapConnectorContext connectorContext, TriggerCdcConfig config, List tables) throws Throwable; -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReadFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReadFunction.java deleted file mode 100644 index 0604b348..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReadFunction.java +++ /dev/null @@ -1,68 +0,0 @@ -package io.tapdata.pdk.apis.functions.connector.triggercdc; - -import io.tapdata.pdk.apis.context.TapConnectorContext; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcBatch; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConfig; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcOffset; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcTable; -import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; - -/** - * Reads committed, not yet acknowledged change records for one consumer on one capture. - * - *

Scope of a single call: exactly one capture. {@link TriggerCdcEnsureFunction} installs one - * capture per source table and assigns its {@link TriggerCdcTable#getCaptureId()}, and every - * capture owns its own journal table. A task that captures several tables therefore owns several - * captures and must call {@code read} once per capture, each with its own offset. One call never - * spans two captures: a journal table, a batch token and an offset all belong to exactly one - * capture, and mixing them would make "acknowledged" ambiguous. Merging and ordering the batches - * of different captures is the framework's job, not this function's.

- * - *

This is deliberately not a {@code streamRead}: reliable consumption must not depend on the - * progress of a callback. The read returns a batch plus a token, and only - * {@link TriggerCdcAcknowledgeFunction} moves the position.

- * - *

Paging rules that prevent silent loss:

- *
    - *
  • Never treat {@code event_id > last_event_id} as the cursor. A sequence gives unique - * values, not commit order, so a late committing transaction with a smaller id would be - * skipped forever. Each call must re-check the whole unacknowledged set of this capture; - * ids may be used for paging inside a round, not to permanently exclude smaller ids.
  • - *
  • Only committed rows may be returned. Records of a rolled back transaction must never - * become visible.
  • - *
  • A {@code captureId} that is null or unknown must be rejected, not guessed: a fresh offset - * carries no identity, so the selector is the only thing that can pick the journal.
  • - *
  • A generation that differs from the installed capture must be rejected, never reused. The - * returned batch always echoes the generation actually read, so the framework can persist - * it after the first call.
  • - *
  • When the batch is empty, return an empty batch rather than null.
  • - *
- */ -public interface TriggerCdcReadFunction extends TapConnectorFunction { - /** - * @param connectorContext the connection / node context - * @param config connection level trigger CDC configuration - * @param consumerId stable id of the subscribing task / channel - * @param captureId capture to read, taken from the {@code captureId} of a successful - * {@link TriggerCdcEnsureFunction} result (see - * {@code TriggerCdcInstallResult#getCaptureIds()}). It is the routing - * authority: only the journal of this capture may be read, and the - * returned batch must echo it. One capture per call; a multi table - * task iterates its captures and merges the batches itself. - * @param offset last acknowledged position of this capture for this consumer; - * the framework keeps one offset per {@code (consumerId, captureId)}. - * Its {@link TriggerCdcOffset#getBatchToken()} is the last - * acknowledged batch token, never a merely delivered one. - * {@code null} or a fresh offset means start from the beginning of this - * capture's journal. A non-null - * {@link TriggerCdcOffset#getCaptureId()} must equal {@code captureId}; - * a mismatch must be rejected, because it means a checkpoint of - * another capture was restored here. - * @param batchSize maximum number of records to return in this call; it wins over - * {@link TriggerCdcConfig#getBatchSize()}, which the framework only uses - * as the default it passes in here - * @return the batch of this capture, never null - */ - TriggerCdcBatch read(TapConnectorContext connectorContext, TriggerCdcConfig config, String consumerId, - String captureId, TriggerCdcOffset offset, int batchSize) throws Throwable; -} diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReleaseFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReleaseFunction.java deleted file mode 100644 index 7dc0ab5b..00000000 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcReleaseFunction.java +++ /dev/null @@ -1,34 +0,0 @@ -package io.tapdata.pdk.apis.functions.connector.triggercdc; - -import io.tapdata.pdk.apis.context.TapConnectorContext; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcConfig; -import io.tapdata.pdk.apis.entity.triggercdc.TriggerCdcTable; -import io.tapdata.pdk.apis.functions.connector.TapConnectorFunction; - -import java.util.List; - -/** - * Releases a consumer, and optionally the capture objects themselves. - * - *

Two separate actions on purpose:

- *
    - *
  • Unregistering a consumer stops its claim on journal records, so cleanup may remove them. - * A consumer holds one position per capture, so unregistering must drop all of them - * at once; a leftover position would keep a journal alive forever. This must be an explicit - * operation, never an implicit side effect of a crash or a timeout, otherwise a temporarily - * paused task silently loses its data.
  • - *
  • Dropping objects happens only when the framework owns them and no other consumer - * references the capture. Never drop an entire customer schema.
  • - *
- */ -public interface TriggerCdcReleaseFunction extends TapConnectorFunction { - /** - * @param connectorContext the connection / node context - * @param config connection level trigger CDC configuration - * @param consumerId consumer to unregister; null skips the unregistration step - * @param tables captures to consider; ignored when {@code dropObjects} is false - * @param dropObjects true to also remove the triggers and journal objects Tapdata owns - */ - void release(TapConnectorContext connectorContext, TriggerCdcConfig config, String consumerId, - List tables, boolean dropObjects) throws Throwable; -} diff --git a/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java b/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java new file mode 100644 index 00000000..315b7a60 --- /dev/null +++ b/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java @@ -0,0 +1,169 @@ +package io.tapdata.pdk.apis.entity.triggercdc; + +import io.tapdata.pdk.apis.functions.ConnectorFunctions; +import io.tapdata.pdk.apis.functions.connector.source.StreamReadFunction; +import io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcAcknowledgeFunction; +import io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcCleanupFunction; +import io.tapdata.pdk.apis.functions.connector.triggercdc.TriggerCdcEnsureFunction; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.Collections; + +import static org.junit.jupiter.api.Assertions.*; + +class TriggerCdcContractTest { + private TriggerCdcTable table(String name) { + return TriggerCdcTable.create().sourceTableId(name).tableName(name) + .captureId("capture-" + name).generation(1L); + } + + private TriggerCdcInstallResult result(TriggerCdcTable... tables) { + return TriggerCdcInstallResult.create().capabilities(TriggerCdcCapabilities.create()) + .requestedTables(Arrays.asList(tables)); + } + + @Test + void successFactoryRejectsMissingGeneration() { + TriggerCdcTable table = table("a").generation(null); + assertThrows(IllegalArgumentException.class, () -> TriggerCdcTableResult.success(table)); + } + + @Test + void deserializedSuccessWithoutGenerationIsRejected() { + TriggerCdcTable table = table("a"); + TriggerCdcTableResult outcome = TriggerCdcTableResult.create().sourceTableId("a") + .captureId("capture-a").success(true); + TriggerCdcInstallResult result = result(table).tableResult(outcome); + assertFalse(outcome.isSuccess()); + assertTrue(result.validate().stream().anyMatch(v -> v.contains("generation"))); + assertThrows(IllegalStateException.class, result::getCaptureIds); + } + + @Test + void completeInstallReturnsAllCaptures() { + TriggerCdcTable a = table("a"), b = table("b"); + TriggerCdcInstallResult result = result(a, b).tableResult(TriggerCdcTableResult.success(a)) + .tableResult(TriggerCdcTableResult.success(b)); + assertTrue(result.isSuccess()); + assertEquals(Arrays.asList("capture-a", "capture-b"), result.getCaptureIds()); + } + + @Test + void missingRequestedTableCannotShrinkReadSet() { + TriggerCdcTable a = table("a"), b = table("b"); + TriggerCdcInstallResult result = result(a, b).tableResult(TriggerCdcTableResult.success(a)); + assertFalse(result.isValid()); + assertEquals(Collections.singletonList(b), result.getMissingTables()); + assertThrows(IllegalStateException.class, result::getCaptureIds); + } + + @Test + void callerRequestOverridesConnectorsSelfReportedRequest() { + TriggerCdcTable a = table("a"), b = table("b"); + TriggerCdcInstallResult result = result(a).tableResult(TriggerCdcTableResult.success(a)); + assertTrue(result.isValid()); + assertFalse(result.validate(Arrays.asList(a, b)).isEmpty()); + assertThrows(IllegalStateException.class, () -> result.getCaptureIds(Arrays.asList(a, b))); + } + + @Test + void duplicateCaptureAndDuplicateOutcomeAreRejected() { + TriggerCdcTable a = table("a"), b = table("b").captureId("capture-a"); + assertFalse(result(a, b).tableResult(TriggerCdcTableResult.success(a)) + .tableResult(TriggerCdcTableResult.success(b)).isValid()); + assertFalse(result(a).tableResult(TriggerCdcTableResult.success(a)) + .tableResult(TriggerCdcTableResult.success(a)).isValid()); + } + + @Test + void unrequestedTableAndUnknownRequestAreRejected() { + TriggerCdcTable a = table("a"), b = table("b"); + assertFalse(result(a).tableResult(TriggerCdcTableResult.success(a)) + .tableResult(TriggerCdcTableResult.success(b)).isValid()); + assertFalse(result().tableResult(TriggerCdcTableResult.success(a)).isValid()); + } + + @Test + void partialFailureIsVisibleAndDoesNotReportTaskSuccess() { + TriggerCdcTable a = table("a"), b = table("b"); + TriggerCdcTableResult failure = TriggerCdcTableResult.failure(b, "unsupported", "type"); + TriggerCdcInstallResult result = result(a, b).tableResult(TriggerCdcTableResult.success(a)) + .tableResult(failure); + assertTrue(result.isValid()); + assertFalse(result.isSuccess()); + assertEquals(Collections.singletonList(failure), result.getFailedTables()); + } + + @Test + void capabilitiesMustBeReturnedByEnsure() { + TriggerCdcTable a = table("a"); + TriggerCdcInstallResult result = result(a).capabilities(null) + .tableResult(TriggerCdcTableResult.success(a)); + assertFalse(result.isValid()); + } + + @Test + void forceCleanupRejectsNullEmptyAndBlankConsumerIdsEvenAmongValidIds() { + TriggerCdcConfig config = TriggerCdcConfig.create().forceCleanUnacknowledged(true); + for (String invalid : new String[]{null, "", " "}) { + TriggerCdcCleanupResult result = TriggerCdcCleanupResult.create().deletedCount(1) + .deletedUnacknowledgedCount(1).integrityBrokenConsumer("valid") + .integrityBrokenConsumer(invalid); + assertFalse(result.validate(config).isEmpty()); + } + } + + @Test + void forceCleanupRequiresOptInAndNamedConsumers() { + TriggerCdcCleanupResult result = TriggerCdcCleanupResult.create().deletedCount(1) + .deletedUnacknowledgedCount(1); + assertFalse(result.validate(TriggerCdcConfig.create().withDefaults()).isEmpty()); + TriggerCdcConfig enabled = TriggerCdcConfig.create().forceCleanUnacknowledged(true); + assertFalse(result.validate(enabled).isEmpty()); + result.integrityBrokenConsumer("consumer"); + assertTrue(result.validate(enabled).isEmpty()); + } + + @Test + void cleanupRejectsImpossibleCounts() { + assertFalse(TriggerCdcCleanupResult.create().deletedCount(-1).validate(null).isEmpty()); + assertFalse(TriggerCdcCleanupResult.create().remainingCount(-1).validate(null).isEmpty()); + assertFalse(TriggerCdcCleanupResult.create().deletedUnacknowledgedCount(-1) + .validate(null).isEmpty()); + assertFalse(TriggerCdcCleanupResult.create().deletedCount(0).deletedUnacknowledgedCount(1) + .integrityBrokenConsumer("a").validate(null).isEmpty()); + } + + @Test + void acknowledgedOnlyCleanupDoesNotNeedForceOptIn() { + assertTrue(TriggerCdcCleanupResult.create().deletedCount(3).remainingCount(2) + .validate(TriggerCdcConfig.create().withDefaults()).isEmpty()); + } + + private boolean advertised(ConnectorFunctions functions) { + return functions.getCapabilities().stream() + .anyMatch(c -> TriggerCdcConstants.CAPABILITY_TRIGGER_CDC.equals(c.getId())); + } + + @Test + void capabilityRequiresAllThreeFunctionsAndExistingStreamReader() { + TriggerCdcEnsureFunction ensure = (ctx, config, consumer, tables) -> null; + TriggerCdcAcknowledgeFunction ack = (ctx, config, consumer, offset) -> {}; + TriggerCdcCleanupFunction cleanup = (ctx, config) -> null; + StreamReadFunction read = (ctx, tables, offset, size, consumer) -> {}; + ConnectorFunctions functions = new ConnectorFunctions(); + assertFalse(advertised(functions)); + functions.supportTriggerCdcEnsureFunction(ensure).supportTriggerCdcAcknowledgeFunction(ack) + .supportTriggerCdcCleanupFunction(cleanup); + assertFalse(advertised(functions)); + functions.supportStreamRead(read); + assertTrue(advertised(functions)); + functions.supportTriggerCdcCleanupFunction(null); + assertFalse(advertised(functions)); + functions.supportTriggerCdcCleanupFunction(cleanup).supportTriggerCdcAcknowledgeFunction(null); + assertFalse(advertised(functions)); + functions.supportTriggerCdcAcknowledgeFunction(ack).supportTriggerCdcEnsureFunction(null); + assertFalse(advertised(functions)); + } +} diff --git a/plugin-kit/tapdata-pdk-runner/pom.xml b/plugin-kit/tapdata-pdk-runner/pom.xml index a1dac99a..9aec8146 100644 --- a/plugin-kit/tapdata-pdk-runner/pom.xml +++ b/plugin-kit/tapdata-pdk-runner/pom.xml @@ -4,12 +4,12 @@ io.tapdata plugin-kit - 2.11-SNAPSHOT + 2.12-SNAPSHOT 4.0.0 tapdata-pdk-runner tapdata-pdk-runner - 2.11-SNAPSHOT + 2.12-SNAPSHOT diff --git a/plugin-kit/tapdata-proxy/pom.xml b/plugin-kit/tapdata-proxy/pom.xml index 004d795b..3cba246e 100644 --- a/plugin-kit/tapdata-proxy/pom.xml +++ b/plugin-kit/tapdata-proxy/pom.xml @@ -5,7 +5,7 @@ io.tapdata plugin-kit - 2.11-SNAPSHOT + 2.12-SNAPSHOT tapdata-proxy ${tapdata.proxy.version} diff --git a/plugin-kit/tapdata-tests/pom.xml b/plugin-kit/tapdata-tests/pom.xml index efaca72f..dfb10de8 100644 --- a/plugin-kit/tapdata-tests/pom.xml +++ b/plugin-kit/tapdata-tests/pom.xml @@ -49,7 +49,7 @@ io.tapdata tapdata-pdk-runner - 2.11-SNAPSHOT + 2.12-SNAPSHOT io.tapdata diff --git a/pom.xml b/pom.xml index 4f5793d0..1e070420 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ io.tapdata - 2.11-SNAPSHOT + 2.12-SNAPSHOT 4.0.0 tapdata-common-lib tapdata-common-lib From 0f8ca01094215a0aab9b2188cccc685780ef18b7 Mon Sep 17 00:00:00 2001 From: jarad0628 <25000091+jarad0628@users.noreply.github.com> Date: Thu, 19 Nov 2026 12:34:35 +0000 Subject: [PATCH 3/5] fix(pdk): enforce trigger CDC install and cleanup contracts --- docs/trigger-cdc.md | 6 +- .../triggercdc/TriggerCdcCleanupResult.java | 13 ++- .../triggercdc/TriggerCdcInstallResult.java | 14 +++- .../triggercdc/TriggerCdcContractTest.java | 80 ++++++++++++++++++- 4 files changed, 100 insertions(+), 13 deletions(-) diff --git a/docs/trigger-cdc.md b/docs/trigger-cdc.md index b2fdad2c..1355eb3f 100644 --- a/docs/trigger-cdc.md +++ b/docs/trigger-cdc.md @@ -28,8 +28,10 @@ DDL support or a safe snapshot boundary. Those must be verified by ensure. allocation order is not commit order. Track row membership rather than an event-id high watermark. In-memory delivery tracking can avoid continuous duplicates while a checkpoint is pending; loss of that tracking after restart safely replays unacknowledged events. -5. Only after every destination's writes and the corresponding recoverable checkpoint are - durable, pass each checkpoint entry to acknowledge. Tokens are bound to consumer, capture +5. Persist the complete checkpoint, including captures that have not delivered events. + Only after every destination's writes and that recoverable checkpoint are durable, pass + entries with a valid delivery token to acknowledge; entries without a token stay + in the checkpoint and must not be acknowledged. Tokens are bound to consumer, capture and generation. Duplicate or older acknowledgements are idempotent and cannot regress progress. A crash before acknowledgement may cause replay; targets need idempotent writes. 6. Run bounded cleanup. Delete only rows acknowledged by every registered consumer and old diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java index 16ac9879..045fdc86 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java @@ -40,13 +40,14 @@ public class TriggerCdcCleanupResult implements Serializable { protected long remainingCount; /** * How many of {@link #getDeletedCount()} were not acknowledged by every live consumer — the - * only records that can damage a consumer. Must be {@code 0} outside the force clean opt-in. + * only records that can damage a consumer. Must be explicitly reported, including {@code 0} + * outside the force clean opt-in. A null value means the connector omitted the count. * *

This field exists so the contract is checkable rather than aspirational: without it a * connector could delete unacknowledged rows, report a total {@code deletedCount}, and leave * {@link #getIntegrityBrokenConsumers()} empty with nothing able to tell.

*/ - protected long deletedUnacknowledgedCount; + protected Long deletedUnacknowledgedCount; /** * Consumers whose data is incomplete because this call removed records they had not * acknowledged. Must be empty when {@link #getDeletedUnacknowledgedCount()} is {@code 0}, and @@ -107,6 +108,10 @@ public List validate(TriggerCdcConfig config) { if (remainingCount < 0) { violations.add("remainingCount must not be negative: " + remainingCount); } + if (deletedUnacknowledgedCount == null) { + violations.add("deletedUnacknowledgedCount must be explicitly reported, including 0"); + return violations; + } if (deletedUnacknowledgedCount < 0) { violations.add("deletedUnacknowledgedCount must not be negative: " + deletedUnacknowledgedCount); } @@ -163,11 +168,11 @@ public TriggerCdcCleanupResult remainingCount(long remainingCount) { return this; } - public long getDeletedUnacknowledgedCount() { + public Long getDeletedUnacknowledgedCount() { return deletedUnacknowledgedCount; } - public void setDeletedUnacknowledgedCount(long deletedUnacknowledgedCount) { + public void setDeletedUnacknowledgedCount(Long deletedUnacknowledgedCount) { this.deletedUnacknowledgedCount = deletedUnacknowledgedCount; } diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java index 066b4700..a6c1643a 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java @@ -173,6 +173,12 @@ public List validate(Collection requestedTables) { List violations = new ArrayList<>(); if (capabilities == null) { violations.add("ensure must report the verified database capabilities"); + } else if (capabilities.isTruncateCaptureSupported() + != capabilities.supports(TriggerCdcOperation.TRUNCATE)) { + violations.add("truncateCaptureSupported must agree with the TRUNCATE operation declaration"); + } + if (status == TriggerCdcStatus.REPAIR_REQUIRED || status == TriggerCdcStatus.FAILED) { + violations.add("ensure is not usable in status " + status); } Map requested = indexRequested(requestedTables, violations); if (tableResults == null || tableResults.isEmpty()) { @@ -218,7 +224,7 @@ public List validate(Collection requestedTables) { if (tableResult.getGeneration() == null) { violations.add(owner + " is reported successful but has no generation"); } - String captureId = tableResult.getCaptureId().trim(); + String captureId = tableResult.getCaptureId(); String previousOwner = captureOwners.put(captureId, owner); if (previousOwner != null) { violations.add("captureId " + captureId + " is claimed by both " + previousOwner @@ -291,10 +297,10 @@ private static Map indexRequested(Collection re */ private static String tableKey(String sourceTableId, String tableName) { if (sourceTableId != null && !sourceTableId.trim().isEmpty()) { - return "id:" + sourceTableId.trim(); + return "id:" + sourceTableId; } if (tableName != null && !tableName.trim().isEmpty()) { - return "name:" + tableName.trim(); + return "name:" + tableName; } return null; } @@ -431,7 +437,7 @@ public List getCaptureIds(Collection requestedTables) { List captureIds = new ArrayList<>(tableResults.size()); for (TriggerCdcTableResult tableResult : tableResults) { if (tableResult.isSuccessReported()) { - captureIds.add(tableResult.getCaptureId().trim()); + captureIds.add(tableResult.getCaptureId()); } } return captureIds; diff --git a/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java b/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java index 315b7a60..ea30d0f4 100644 --- a/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java +++ b/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java @@ -127,8 +127,8 @@ void forceCleanupRequiresOptInAndNamedConsumers() { @Test void cleanupRejectsImpossibleCounts() { - assertFalse(TriggerCdcCleanupResult.create().deletedCount(-1).validate(null).isEmpty()); - assertFalse(TriggerCdcCleanupResult.create().remainingCount(-1).validate(null).isEmpty()); + assertFalse(TriggerCdcCleanupResult.create().deletedUnacknowledgedCount(0).deletedCount(-1).validate(null).isEmpty()); + assertFalse(TriggerCdcCleanupResult.create().deletedUnacknowledgedCount(0).remainingCount(-1).validate(null).isEmpty()); assertFalse(TriggerCdcCleanupResult.create().deletedUnacknowledgedCount(-1) .validate(null).isEmpty()); assertFalse(TriggerCdcCleanupResult.create().deletedCount(0).deletedUnacknowledgedCount(1) @@ -137,10 +137,84 @@ void cleanupRejectsImpossibleCounts() { @Test void acknowledgedOnlyCleanupDoesNotNeedForceOptIn() { - assertTrue(TriggerCdcCleanupResult.create().deletedCount(3).remainingCount(2) + assertTrue(TriggerCdcCleanupResult.create().deletedCount(3).remainingCount(2).deletedUnacknowledgedCount(0) .validate(TriggerCdcConfig.create().withDefaults()).isEmpty()); } + @Test + void unusableSubscriptionStatusRejectsOtherwiseCompleteInstall() { + TriggerCdcTable a = table("a"); + for (TriggerCdcStatus status : new TriggerCdcStatus[]{ + TriggerCdcStatus.REPAIR_REQUIRED, TriggerCdcStatus.FAILED}) { + TriggerCdcInstallResult result = result(a).tableResult(TriggerCdcTableResult.success(a)) + .status(status); + assertFalse(result.isSuccess()); + assertTrue(result.validate().stream().anyMatch(v -> v.contains(status.name()))); + assertThrows(IllegalStateException.class, result::getCaptureIds); + } + assertTrue(result(a).tableResult(TriggerCdcTableResult.success(a)) + .status(TriggerCdcStatus.CAPTURING).isSuccess()); + } + + @Test + void cleanupRequiresExplicitUnacknowledgedCountIncludingZero() { + TriggerCdcCleanupResult result = TriggerCdcCleanupResult.create().deletedCount(1); + assertNull(result.getDeletedUnacknowledgedCount()); + assertTrue(result.validate(TriggerCdcConfig.create().withDefaults()).stream() + .anyMatch(v -> v.contains("must be explicitly reported"))); + result.deletedUnacknowledgedCount(0); + assertTrue(result.validate(TriggerCdcConfig.create().withDefaults()).isEmpty()); + result.setDeletedUnacknowledgedCount(null); + assertFalse(result.validate(null).isEmpty()); + result.setDeletedUnacknowledgedCount(0L); + assertTrue(result.validate(null).isEmpty()); + } + + @Test + void captureIdsRetainWhitespaceAndRemainDistinct() { + TriggerCdcTable a = table("a").captureId("journal"); + TriggerCdcTable b = table("b").captureId("journal "); + TriggerCdcInstallResult result = result(a, b).tableResult(TriggerCdcTableResult.success(a)) + .tableResult(TriggerCdcTableResult.success(b)); + assertTrue(result.isSuccess()); + assertEquals(Arrays.asList("journal", "journal "), result.getCaptureIds()); + } + + @Test + void tableNamesWithoutStableIdsMustMatchExactly() { + TriggerCdcTable plain = table("orders").sourceTableId(null); + TriggerCdcTable spaced = table("orders ").sourceTableId(null); + TriggerCdcInstallResult mismatch = result(spaced) + .tableResult(TriggerCdcTableResult.success(plain)); + assertFalse(mismatch.isValid()); + assertEquals(Collections.singletonList(spaced), mismatch.getMissingTables()); + assertThrows(IllegalStateException.class, mismatch::getCaptureIds); + assertTrue(result(plain, spaced).tableResult(TriggerCdcTableResult.success(plain)) + .tableResult(TriggerCdcTableResult.success(spaced)).isSuccess()); + } + + @Test + void stableTableIdsMustMatchExactly() { + TriggerCdcTable a = table("orders"); + TriggerCdcTable other = table("orders").sourceTableId("orders "); + assertFalse(result(other).tableResult(TriggerCdcTableResult.success(a)).isValid()); + } + + @Test + void truncateDeclarationsMustAgreeInBothDirections() { + TriggerCdcTable a = table("a"); + TriggerCdcInstallResult result = result(a).tableResult(TriggerCdcTableResult.success(a)); + assertTrue(result.isSuccess()); + result.capabilities(TriggerCdcCapabilities.create().truncateCaptureSupported(true)); + assertFalse(result.isValid()); + assertThrows(IllegalStateException.class, result::getCaptureIds); + result.capabilities(TriggerCdcCapabilities.create().dmlOperation(TriggerCdcOperation.TRUNCATE)); + assertFalse(result.isValid()); + assertThrows(IllegalStateException.class, result::getCaptureIds); + result.getCapabilities().truncateCaptureSupported(true); + assertTrue(result.isSuccess()); + } + private boolean advertised(ConnectorFunctions functions) { return functions.getCapabilities().stream() .anyMatch(c -> TriggerCdcConstants.CAPABILITY_TRIGGER_CDC.equals(c.getId())); From f8a0ca5784584c3884017afb2f77d7d230469532 Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Fri, 20 Nov 2026 12:12:22 +0000 Subject: [PATCH 4/5] fix(pdk): reject installing captures and preserve cleanup signatures --- .../triggercdc/TriggerCdcCleanupResult.java | 15 ++++++---- .../triggercdc/TriggerCdcInstallResult.java | 3 +- .../triggercdc/TriggerCdcContractTest.java | 28 +++++++++++++++---- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java index 045fdc86..78955379 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCleanupResult.java @@ -41,13 +41,15 @@ public class TriggerCdcCleanupResult implements Serializable { /** * How many of {@link #getDeletedCount()} were not acknowledged by every live consumer — the * only records that can damage a consumer. Must be explicitly reported, including {@code 0} - * outside the force clean opt-in. A null value means the connector omitted the count. + * outside the force clean opt-in. A separate presence flag tracks whether the connector + * reported the count. * *

This field exists so the contract is checkable rather than aspirational: without it a * connector could delete unacknowledged rows, report a total {@code deletedCount}, and leave * {@link #getIntegrityBrokenConsumers()} empty with nothing able to tell.

*/ - protected Long deletedUnacknowledgedCount; + protected long deletedUnacknowledgedCount; + private boolean deletedUnacknowledgedCountReported; /** * Consumers whose data is incomplete because this call removed records they had not * acknowledged. Must be empty when {@link #getDeletedUnacknowledgedCount()} is {@code 0}, and @@ -108,7 +110,7 @@ public List validate(TriggerCdcConfig config) { if (remainingCount < 0) { violations.add("remainingCount must not be negative: " + remainingCount); } - if (deletedUnacknowledgedCount == null) { + if (!deletedUnacknowledgedCountReported) { violations.add("deletedUnacknowledgedCount must be explicitly reported, including 0"); return violations; } @@ -168,16 +170,17 @@ public TriggerCdcCleanupResult remainingCount(long remainingCount) { return this; } - public Long getDeletedUnacknowledgedCount() { + public long getDeletedUnacknowledgedCount() { return deletedUnacknowledgedCount; } - public void setDeletedUnacknowledgedCount(Long deletedUnacknowledgedCount) { + public void setDeletedUnacknowledgedCount(long deletedUnacknowledgedCount) { this.deletedUnacknowledgedCount = deletedUnacknowledgedCount; + this.deletedUnacknowledgedCountReported = true; } public TriggerCdcCleanupResult deletedUnacknowledgedCount(long deletedUnacknowledgedCount) { - this.deletedUnacknowledgedCount = deletedUnacknowledgedCount; + setDeletedUnacknowledgedCount(deletedUnacknowledgedCount); return this; } diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java index a6c1643a..9c68323d 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java @@ -177,7 +177,8 @@ public List validate(Collection requestedTables) { != capabilities.supports(TriggerCdcOperation.TRUNCATE)) { violations.add("truncateCaptureSupported must agree with the TRUNCATE operation declaration"); } - if (status == TriggerCdcStatus.REPAIR_REQUIRED || status == TriggerCdcStatus.FAILED) { + if (status == TriggerCdcStatus.INSTALLING || status == TriggerCdcStatus.REPAIR_REQUIRED + || status == TriggerCdcStatus.FAILED) { violations.add("ensure is not usable in status " + status); } Map requested = indexRequested(requestedTables, violations); diff --git a/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java b/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java index ea30d0f4..f8657f65 100644 --- a/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java +++ b/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java @@ -145,31 +145,49 @@ void acknowledgedOnlyCleanupDoesNotNeedForceOptIn() { void unusableSubscriptionStatusRejectsOtherwiseCompleteInstall() { TriggerCdcTable a = table("a"); for (TriggerCdcStatus status : new TriggerCdcStatus[]{ - TriggerCdcStatus.REPAIR_REQUIRED, TriggerCdcStatus.FAILED}) { + TriggerCdcStatus.INSTALLING, TriggerCdcStatus.REPAIR_REQUIRED, TriggerCdcStatus.FAILED}) { TriggerCdcInstallResult result = result(a).tableResult(TriggerCdcTableResult.success(a)) .status(status); assertFalse(result.isSuccess()); + assertFalse(result.isValid()); assertTrue(result.validate().stream().anyMatch(v -> v.contains(status.name()))); + assertTrue(result.validate(Collections.singletonList(a)).stream() + .anyMatch(v -> v.contains(status.name()))); assertThrows(IllegalStateException.class, result::getCaptureIds); + assertThrows(IllegalStateException.class, + () -> result.getCaptureIds(Collections.singletonList(a))); + } + for (TriggerCdcStatus status : new TriggerCdcStatus[]{TriggerCdcStatus.CAPTURING, + TriggerCdcStatus.SNAPSHOTTING, TriggerCdcStatus.STREAMING, TriggerCdcStatus.DEGRADED}) { + TriggerCdcInstallResult result = result(a).tableResult(TriggerCdcTableResult.success(a)) + .status(status); + assertTrue(result.isSuccess()); + assertEquals(Collections.singletonList("capture-a"), result.getCaptureIds()); } - assertTrue(result(a).tableResult(TriggerCdcTableResult.success(a)) - .status(TriggerCdcStatus.CAPTURING).isSuccess()); } @Test void cleanupRequiresExplicitUnacknowledgedCountIncludingZero() { TriggerCdcCleanupResult result = TriggerCdcCleanupResult.create().deletedCount(1); - assertNull(result.getDeletedUnacknowledgedCount()); + assertEquals(0L, result.getDeletedUnacknowledgedCount()); assertTrue(result.validate(TriggerCdcConfig.create().withDefaults()).stream() .anyMatch(v -> v.contains("must be explicitly reported"))); result.deletedUnacknowledgedCount(0); assertTrue(result.validate(TriggerCdcConfig.create().withDefaults()).isEmpty()); - result.setDeletedUnacknowledgedCount(null); + result = TriggerCdcCleanupResult.create().deletedCount(1); assertFalse(result.validate(null).isEmpty()); result.setDeletedUnacknowledgedCount(0L); assertTrue(result.validate(null).isEmpty()); } + @Test + void cleanupCountAccessorsRetainPrimitiveSignatures() throws NoSuchMethodException { + assertEquals(long.class, TriggerCdcCleanupResult.class + .getMethod("getDeletedUnacknowledgedCount").getReturnType()); + assertEquals(void.class, TriggerCdcCleanupResult.class + .getMethod("setDeletedUnacknowledgedCount", long.class).getReturnType()); + } + @Test void captureIdsRetainWhitespaceAndRemainDistinct() { TriggerCdcTable a = table("a").captureId("journal"); From bcd064942ed268185cd7e7c051e46cc194f5fdea Mon Sep 17 00:00:00 2001 From: OpenAI Codex Date: Fri, 20 Nov 2026 15:33:27 +0000 Subject: [PATCH 5/5] fix(pdk): reject incomplete trigger CDC startup results --- docs/trigger-cdc.md | 32 +++++++++- .../triggercdc/TriggerCdcCapabilities.java | 5 +- .../triggercdc/TriggerCdcInstallResult.java | 61 +++++++++++-------- .../triggercdc/TriggerCdcEnsureFunction.java | 4 +- .../triggercdc/TriggerCdcContractTest.java | 49 ++++++++++++++- 5 files changed, 118 insertions(+), 33 deletions(-) diff --git a/docs/trigger-cdc.md b/docs/trigger-cdc.md index 1355eb3f..6bd9b3ad 100644 --- a/docs/trigger-cdc.md +++ b/docs/trigger-cdc.md @@ -15,8 +15,11 @@ DDL support or a safe snapshot boundary. Those must be verified by ensure. across stop/crash and do not expire them by TTL. 2. Check `validate(originalTables)` and require every table to succeed before starting the task. Persist every capture id and generation. A partial result describes retryable - failures, not permission to silently omit those tables. Reject full-to-incremental - handover if the connector cannot prove a capture boundary. + failures, not permission to silently omit those tables. `validate` checks report structure; + `getCaptureIds(originalTables)` additionally rejects any failed table. A usable result needs + an explicit readable status (CAPTURING, SNAPSHOTTING, STREAMING or DEGRADED), nonblank + database type/version and a STRICT error-policy declaration. DEGRADED still requires + operator attention. Apply the snapshot handover requirements below before starting a snapshot. 3. Invoke streamRead on the same node context. The connector uses the consumer/config/capture mapping retained by ensure and polls committed journal records. The offset is a complete `Map` keyed by capture id, even for a single table. Validate keys, @@ -45,6 +48,31 @@ notification. The engine integration must connect step 5 to durable progress cov sink; until then, it must not acknowledge on callback return or enable journal reclamation based on delivery. This PR provides the PDK contracts, not that engine implementation. +## Snapshot and journal handover + +`captureBoundarySupported` proves only that trigger installation leaves no capture gap. +It is necessary but insufficient for automatic full-to-incremental handover. It does not +identify which journal changes a snapshot already includes, nor authorize a replay position. + +Before enabling automatic handover, the engine and connector must implement and verify a +snapshot/replay protocol for the specific database/version and destination write semantics. +That protocol must either identify and suppress journal changes covered by the snapshot, +or prove overlap replay safe, including updates, deletes, key changes and non-idempotent +writes. For example, a snapshot containing v3 followed by journal v2 then v3 can temporarily +regress the target; idempotent writes alone do not prevent old versions overwriting new ones. +A replay protocol that allows such intermediate states must explicitly define when the target +becomes usable and how it recovers safely after a crash at any intermediate point. + +The handover state must durably bind the snapshot progress/completion and journal replay +position to the consumer, capture ids and generations. Recovery must restore a consistent +pair, retain all journal records still needed for the snapshot/replay protocol, and never +acknowledge overlap merely because the snapshot has finished. Normal acknowledgement still +requires durable destination writes and a recoverable checkpoint as described above. +Validate concurrent changes and crashes during snapshot, overlap replay and transition to +streaming before enabling the protocol. Until this integration is implemented and verified, +automatic full-to-incremental handover must remain disabled, even if captureBoundarySupported +is true. This PDK contract does not itself implement that protocol or add a handover API. + ## Scope There are no separate capability-query, SQL-plan, read, periodic-inspect or release APIs. diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCapabilities.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCapabilities.java index 2ca12f80..1018df62 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCapabilities.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcCapabilities.java @@ -50,7 +50,10 @@ public class TriggerCdcCapabilities implements Serializable { * The connector can prove a capture boundary across installation, so no change committed * while the trigger is being created is lost. * - *

Without it, the framework must not offer an automatic full-then-incremental handover.

+ *

Necessary but insufficient for automatic full-then-incremental handover. The engine and + * connector must also verify snapshot/journal overlap handling and durable recovery of the + * snapshot and replay position. Until that protocol is implemented and verified, automatic + * handover must remain disabled even when this flag is true. See docs/trigger-cdc.md.

*/ protected boolean captureBoundarySupported; /** diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java index 9c68323d..90fed38c 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcInstallResult.java @@ -126,8 +126,9 @@ public boolean isSuccess() { * that claims success carries a non blank, unique capture id. * *

Equivalent to {@code validate().isEmpty()}. A partial install (some tables reported as - * {@code failure}) is still a valid result - those tables simply contribute no capture - but a - * result where no table succeeded is a violation, because its capture list would be + * {@code failure}) is still a valid result. Failed outcomes remain available through + * {@link #getFailedTables()}, while capture accessors reject the partial install. A result + * where no table succeeded is a violation, because its capture list would be * empty and a framework that iterates it would look healthy while reading nothing.

*/ public boolean isValid() { @@ -137,7 +138,7 @@ public boolean isValid() { /** * Checks this result against the rules in this class, using the request recorded through * {@link #requestedTables(Collection)}, and returns one message per violation; an empty list - * means the result is usable. + * means the report is structurally valid, not necessarily that every table succeeded. * *

Equivalent to {@code validate(getRequestedTables())}.

*/ @@ -147,7 +148,8 @@ public List validate() { /** * Checks this result against the tables the caller actually asked {@code ensure} for, and - * returns one message per violation; an empty list means the result is usable. + * returns one message per violation; an empty list means the report is structurally valid, + * not necessarily ready to start. * *

Why the request is required: a report that only lists the tables the connector managed to * install is self consistent and still wrong. If {@code ensure} is called for A and B and only @@ -173,12 +175,23 @@ public List validate(Collection requestedTables) { List violations = new ArrayList<>(); if (capabilities == null) { violations.add("ensure must report the verified database capabilities"); - } else if (capabilities.isTruncateCaptureSupported() - != capabilities.supports(TriggerCdcOperation.TRUNCATE)) { - violations.add("truncateCaptureSupported must agree with the TRUNCATE operation declaration"); + } else { + if (capabilities.getDatabaseType() == null || capabilities.getDatabaseType().trim().isEmpty()) { + violations.add("ensure must report a nonblank databaseType"); + } + if (capabilities.getDatabaseVersion() == null || capabilities.getDatabaseVersion().trim().isEmpty()) { + violations.add("ensure must report a nonblank databaseVersion"); + } + if (!capabilities.supports(TriggerCdcErrorPolicy.STRICT)) { + violations.add("ensure capabilities must declare STRICT error policy"); + } + if (capabilities.isTruncateCaptureSupported() + != capabilities.supports(TriggerCdcOperation.TRUNCATE)) { + violations.add("truncateCaptureSupported must agree with the TRUNCATE operation declaration"); + } } - if (status == TriggerCdcStatus.INSTALLING || status == TriggerCdcStatus.REPAIR_REQUIRED - || status == TriggerCdcStatus.FAILED) { + if (status != TriggerCdcStatus.CAPTURING && status != TriggerCdcStatus.SNAPSHOTTING + && status != TriggerCdcStatus.STREAMING && status != TriggerCdcStatus.DEGRADED) { violations.add("ensure is not usable in status " + status); } Map requested = indexRequested(requestedTables, violations); @@ -395,28 +408,22 @@ public List getFailedTables() { } /** - * Capture ids of the successfully installed tables, in request order. + * Capture ids for a fully successful installation. * - *

This is the authoritative capture list a multi table task iterates: {@code read} and - * {@code acknowledge} work on one capture at a time (one source table is one capture), so the - * framework keeps one {@code TriggerCdcOffset} per entry of this list and per consumer. Failed - * tables contribute no capture id, which is why a partial install must be read as "these - * captures exist", not "the requested set exists".

+ *

This is the authoritative capture list used to start a multi table task. Every requested + * table must succeed; any failed outcome makes this accessor throw, even when the report is + * structurally valid. Use {@link #getFailedTables()} for diagnosis and retry information.

* - *

Fail closed: nothing is skipped and nothing is deduplicated. A successful table without a - * capture id, two tables sharing one capture id, an outcome for a table that was not requested, - * or a requested table with no outcome at all makes the whole list untrustworthy - returning the - * rest would silently leave a table unread - so this method throws instead of filtering. An - * empty list is never returned either: when every table failed there is nothing to read, and a - * silently empty capture list would look like "installed, nothing to do". Both conditions are - * reported by {@link #validate()}.

+ *

Fail closed: missing or duplicate outcomes, missing or duplicate capture ids, unusable + * status and invalid capabilities also cause this accessor to throw. Nothing is filtered out + * and an empty capture list is never returned.

* *

Completeness is checked against the request recorded on this result. The framework normally * also has the request it just made, so it should call * {@link #getCaptureIds(Collection)} with that list: it is the authoritative answer to "what did * we ask for", and it cannot be forged by an incomplete report.

* - * @throws IllegalStateException when {@link #validate()} reports a violation + * @throws IllegalStateException when validation fails or any requested table failed */ public List getCaptureIds() { return getCaptureIds(requestedTables); @@ -427,7 +434,7 @@ public List getCaptureIds() { * actually made rather than the one recorded on the result. * * @param requestedTables the tables handed to {@code ensure}; never {@code null} and never empty - * @throws IllegalStateException when {@link #validate(Collection)} reports a violation + * @throws IllegalStateException when validation fails or any requested table failed */ public List getCaptureIds(Collection requestedTables) { List violations = validate(requestedTables); @@ -437,9 +444,11 @@ public List getCaptureIds(Collection requestedTables) { } List captureIds = new ArrayList<>(tableResults.size()); for (TriggerCdcTableResult tableResult : tableResults) { - if (tableResult.isSuccessReported()) { - captureIds.add(tableResult.getCaptureId()); + if (!tableResult.isSuccessReported()) { + throw new IllegalStateException("install result rejected: every requested table must " + + "succeed before starting; see getFailedTables() for retry information"); } + captureIds.add(tableResult.getCaptureId()); } return captureIds; } diff --git a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcEnsureFunction.java b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcEnsureFunction.java index 71ae68cd..a6154668 100644 --- a/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcEnsureFunction.java +++ b/plugin-kit/tapdata-pdk-api/src/main/java/io/tapdata/pdk/apis/functions/connector/triggercdc/TriggerCdcEnsureFunction.java @@ -19,7 +19,9 @@ *

Database/version-specific capabilities are returned in the install result rather than * obtained through a separate function. Validate requested options and unsupported types before * changing objects. A capture boundary must be verified, not inferred from CREATE TRIGGER - * returning. Without a verified boundary, do not offer automatic full-to-incremental handover.

+ * returning. A verified installation boundary alone does not prove safe snapshot/journal + * handover: overlap handling and durable snapshot/replay recovery must also be implemented and + * verified before automatic full-to-incremental handover is enabled. See docs/trigger-cdc.md.

* *

Call again before resume to verify triggers, journal and schema versions. The registered * {@link io.tapdata.pdk.apis.functions.connector.source.StreamReadFunction} polls those captures, diff --git a/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java b/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java index f8657f65..abb4fa51 100644 --- a/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java +++ b/plugin-kit/tapdata-pdk-api/src/test/java/io/tapdata/pdk/apis/entity/triggercdc/TriggerCdcContractTest.java @@ -18,8 +18,13 @@ private TriggerCdcTable table(String name) { .captureId("capture-" + name).generation(1L); } + private TriggerCdcCapabilities capabilities() { + return TriggerCdcCapabilities.create().databaseType("PostgreSQL").databaseVersion("16") + .errorPolicy(TriggerCdcErrorPolicy.STRICT); + } + private TriggerCdcInstallResult result(TriggerCdcTable... tables) { - return TriggerCdcInstallResult.create().capabilities(TriggerCdcCapabilities.create()) + return TriggerCdcInstallResult.create().status(TriggerCdcStatus.CAPTURING).capabilities(capabilities()) .requestedTables(Arrays.asList(tables)); } @@ -93,6 +98,11 @@ void partialFailureIsVisibleAndDoesNotReportTaskSuccess() { assertTrue(result.isValid()); assertFalse(result.isSuccess()); assertEquals(Collections.singletonList(failure), result.getFailedTables()); + assertThrows(IllegalStateException.class, result::getCaptureIds); + assertThrows(IllegalStateException.class, () -> result.getCaptureIds(Arrays.asList(a, b))); + result.requestedTables(null); + assertTrue(result.validate(Arrays.asList(a, b)).isEmpty()); + assertThrows(IllegalStateException.class, () -> result.getCaptureIds(Arrays.asList(a, b))); } @Test @@ -103,6 +113,39 @@ void capabilitiesMustBeReturnedByEnsure() { assertFalse(result.isValid()); } + @Test + void incompleteCapabilitiesRejectOtherwiseSuccessfulInstall() { + TriggerCdcTable a = table("a"); + TriggerCdcCapabilities[] invalid = { + TriggerCdcCapabilities.create(), + capabilities().databaseType(null), capabilities().databaseType(" "), + capabilities().databaseVersion(null), capabilities().databaseVersion(" "), + TriggerCdcCapabilities.create().databaseType("PostgreSQL").databaseVersion("16") + .errorPolicy(TriggerCdcErrorPolicy.BEST_EFFORT) + }; + for (TriggerCdcCapabilities declaration : invalid) { + TriggerCdcInstallResult result = result(a).capabilities(declaration) + .tableResult(TriggerCdcTableResult.success(a)); + assertFalse(result.isSuccess()); + assertFalse(result.validate(Collections.singletonList(a)).isEmpty()); + assertThrows(IllegalStateException.class, result::getCaptureIds); + assertThrows(IllegalStateException.class, + () -> result.getCaptureIds(Collections.singletonList(a))); + } + } + + @Test + void missingStatusRejectsOtherwiseSuccessfulInstall() { + TriggerCdcTable a = table("a"); + TriggerCdcInstallResult result = result(a).status(null) + .tableResult(TriggerCdcTableResult.success(a)); + assertFalse(result.isSuccess()); + assertFalse(result.isValid()); + assertThrows(IllegalStateException.class, result::getCaptureIds); + assertThrows(IllegalStateException.class, + () -> result.getCaptureIds(Collections.singletonList(a))); + } + @Test void forceCleanupRejectsNullEmptyAndBlankConsumerIdsEvenAmongValidIds() { TriggerCdcConfig config = TriggerCdcConfig.create().forceCleanUnacknowledged(true); @@ -223,10 +266,10 @@ void truncateDeclarationsMustAgreeInBothDirections() { TriggerCdcTable a = table("a"); TriggerCdcInstallResult result = result(a).tableResult(TriggerCdcTableResult.success(a)); assertTrue(result.isSuccess()); - result.capabilities(TriggerCdcCapabilities.create().truncateCaptureSupported(true)); + result.capabilities(capabilities().truncateCaptureSupported(true)); assertFalse(result.isValid()); assertThrows(IllegalStateException.class, result::getCaptureIds); - result.capabilities(TriggerCdcCapabilities.create().dmlOperation(TriggerCdcOperation.TRUNCATE)); + result.capabilities(capabilities().dmlOperation(TriggerCdcOperation.TRUNCATE)); assertFalse(result.isValid()); assertThrows(IllegalStateException.class, result::getCaptureIds); result.getCapabilities().truncateCaptureSupported(true);