From c4c59bc832fb0da162295934adf3f4049078624e Mon Sep 17 00:00:00 2001 From: naivedogger Date: Mon, 20 Jul 2026 17:13:29 +0800 Subject: [PATCH 1/9] [flink] Support latest as stopping offset for log table batch read --- .../source/reader/FlinkSourceSplitReader.java | 24 ++-- .../enumerator/FlinkSourceEnumeratorTest.java | 119 ++++++++++++++++++ .../reader/FlinkSourceSplitReaderTest.java | 64 +++++++++- .../split/SourceSplitSerializerTest.java | 8 ++ 4 files changed, 200 insertions(+), 15 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java index 50e2143962c..6c52f695ca6 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java @@ -258,7 +258,7 @@ private void subscribeLog(SourceSplitBase split, long startingOffset) { Optional stoppingOffsetOpt = logSplit.getStoppingOffset(); if (stoppingOffsetOpt.isPresent()) { Long stoppingOffset = stoppingOffsetOpt.get(); - if (startingOffset >= stoppingOffset) { + if (stoppingOffset == 0 || startingOffset >= stoppingOffset) { // is empty log splits as no log record can be fetched emptyLogSplits.add(split.splitId()); isEmptyLogSplit = true; @@ -468,19 +468,25 @@ private FlinkRecordsWithSplitIds forLogRecords(ScanRecords scanRecords) { splitIdByTableBucket.put(scanBucket, splitId); tableScanBuckets.add(scanBucket); List bucketScanRecords = scanRecords.records(scanBucket); + ScanRecord lastRecord = null; if (!bucketScanRecords.isEmpty()) { - final ScanRecord lastRecord = bucketScanRecords.get(bucketScanRecords.size() - 1); + lastRecord = bucketScanRecords.get(bucketScanRecords.size() - 1); // We keep the maximum message timestamp in the fetch for calculating lags maxConsumerRecordTimestampInFetch = Math.max(maxConsumerRecordTimestampInFetch, lastRecord.timestamp()); + } - // After processing a record with offset of "stoppingOffset - 1", the split reader - // should not continue fetching because the record with stoppingOffset may not - // exist. Keep polling will just block forever - if (lastRecord.logOffset() >= stoppingOffset - 1) { - stoppingOffsets.put(scanBucket, stoppingOffset); - finishedSplits.add(splitId); - } + Long consumedUpToOffset = scanRecords.consumedUpToOffset(scanBucket); + boolean reachedStoppingOffset = + consumedUpToOffset != null + ? consumedUpToOffset >= stoppingOffset + : lastRecord != null && lastRecord.logOffset() >= stoppingOffset - 1; + // After consuming up to the stopping offset, the split reader should not continue + // fetching because the record at the stopping offset is outside this split and may not + // exist. This also handles batches whose records are all filtered out. + if (reachedStoppingOffset) { + stoppingOffsets.put(scanBucket, stoppingOffset); + finishedSplits.add(splitId); } splitRecords.put(splitId, toRecordAndPos(bucketScanRecords.iterator())); } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java index a49dbebc7d8..10f48aee329 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.flink.source.enumerator; +import org.apache.fluss.client.admin.OffsetSpec; import org.apache.fluss.client.initializer.OffsetsInitializer; import org.apache.fluss.client.table.Table; import org.apache.fluss.client.table.writer.UpsertWriter; @@ -1063,6 +1064,124 @@ void testDiscoverPartitionsPeriodically(boolean isPrimaryKeyTable) throws Throwa } } + @Test + void testBatchModeNonLakeLogTable() throws Throwable { + int numSubtasks = DEFAULT_BUCKET_NUM; + long tableId = createTable(DEFAULT_TABLE_PATH, DEFAULT_LOG_TABLE_DESCRIPTOR); + List rows = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + rows.add(row(i, "v" + i)); + } + writeRows(conn, DEFAULT_TABLE_PATH, rows, true); + + List bucketIds = new ArrayList<>(); + for (int bucket = 0; bucket < DEFAULT_BUCKET_NUM; bucket++) { + bucketIds.add(bucket); + } + Map expectedStoppingOffsets = + admin.listOffsets(DEFAULT_TABLE_PATH, bucketIds, new OffsetSpec.LatestSpec()) + .all() + .get(); + + try (MockSplitEnumeratorContext context = + new MockSplitEnumeratorContext<>(numSubtasks)) { + FlinkSourceEnumerator enumerator = + new FlinkSourceEnumerator( + DEFAULT_TABLE_PATH, + flussConf, + false, + false, + context, + OffsetsInitializer.earliest(), + DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, + false, + null, + null, + LeaseContext.DEFAULT, + false); + + enumerator.start(); + for (int i = 0; i < numSubtasks; i++) { + registerReader(context, enumerator, i); + } + context.runNextOneTimeCallable(); + + List assignedSplits = + getReadersAssignments(context).values().stream() + .flatMap(List::stream) + .collect(Collectors.toList()); + assertThat(assignedSplits).hasSize(DEFAULT_BUCKET_NUM); + assertThat(assignedSplits) + .allSatisfy( + split -> { + assertThat(split).isInstanceOf(LogSplit.class); + LogSplit logSplit = split.asLogSplit(); + assertThat(logSplit.getStartingOffset()).isEqualTo(EARLIEST_OFFSET); + assertThat(logSplit.getTableBucket().getTableId()) + .isEqualTo(tableId); + assertThat(logSplit.getStoppingOffset()) + .contains( + expectedStoppingOffsets.get( + logSplit.getTableBucket().getBucket())); + }); + } + } + + @Test + void testBatchModeNonLakePartitionedLogTable() throws Throwable { + int numSubtasks = DEFAULT_BUCKET_NUM; + long tableId = + createTable(DEFAULT_TABLE_PATH, DEFAULT_AUTO_PARTITIONED_LOG_TABLE_DESCRIPTOR); + ZooKeeperClient zooKeeperClient = FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(); + Map partitionNameByIds = + waitUntilPartitions(zooKeeperClient, DEFAULT_TABLE_PATH); + + try (MockSplitEnumeratorContext context = + new MockSplitEnumeratorContext<>(numSubtasks)) { + FlinkSourceEnumerator enumerator = + new FlinkSourceEnumerator( + DEFAULT_TABLE_PATH, + flussConf, + false, + true, + context, + OffsetsInitializer.earliest(), + DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, + false, + null, + null, + LeaseContext.DEFAULT, + false); + + enumerator.start(); + for (int i = 0; i < numSubtasks; i++) { + registerReader(context, enumerator, i); + } + context.runNextOneTimeCallable(); + + List assignedSplits = + getReadersAssignments(context).values().stream() + .flatMap(List::stream) + .collect(Collectors.toList()); + assertThat(assignedSplits).hasSize(partitionNameByIds.size() * DEFAULT_BUCKET_NUM); + Set assignedPartitionNames = new HashSet<>(); + assertThat(assignedSplits) + .allSatisfy( + split -> { + assertThat(split).isInstanceOf(LogSplit.class); + LogSplit logSplit = split.asLogSplit(); + assertThat(logSplit.getTableBucket().getTableId()) + .isEqualTo(tableId); + assertThat(logSplit.getStartingOffset()).isEqualTo(EARLIEST_OFFSET); + // the partitions are empty, so the captured latest offset is 0 + assertThat(logSplit.getStoppingOffset()).contains(0L); + assignedPartitionNames.add(logSplit.getPartitionName()); + }); + assertThat(assignedPartitionNames) + .containsExactlyInAnyOrderElementsOf(partitionNameByIds.values()); + } + } + @Test void testGetSplitOwner() throws Exception { int numSubtasks = 3; diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java index 5b0117a0f73..19f2831ad63 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java @@ -17,6 +17,7 @@ package org.apache.fluss.flink.source.reader; +import org.apache.fluss.client.admin.OffsetSpec; import org.apache.fluss.client.metadata.KvSnapshots; import org.apache.fluss.client.table.Table; import org.apache.fluss.client.table.scanner.ScanRecord; @@ -243,6 +244,54 @@ void testHandleLogSplitChangesAndFetch() throws Exception { } } + @Test + void testBoundedLogSplitStopsAtCapturedLatestOffset() throws Exception { + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("name", DataTypes.STRING()) + .build(); + TableDescriptor tableDescriptor = + TableDescriptor.builder().schema(schema).distributedBy(1).build(); + TablePath tablePath = TablePath.of(DEFAULT_DB, "test-bounded-log-split"); + + long tableId = createTable(tablePath, tableDescriptor); + List initialRows = appendRows(tablePath, 2); + + long stoppingOffset = + admin.listOffsets( + tablePath, + Collections.singletonList(0), + new OffsetSpec.LatestSpec()) + .bucketResult(0) + .get(); + + // These records are written after stoppingOffset was captured. + appendRows(tablePath, 2); + + TableBucket tableBucket = new TableBucket(tableId, 0); + LogSplit split = new LogSplit(tableBucket, null, 0L, stoppingOffset); + + List expected = new ArrayList<>(); + for (int i = 0; i < initialRows.size(); i++) { + expected.add( + new RecordAndPos( + new ScanRecord(i, i, ChangeType.APPEND_ONLY, initialRows.get(i)))); + } + + Map> expectedRecords = new HashMap<>(); + expectedRecords.put(split.splitId(), expected); + + try (FlinkSourceSplitReader splitReader = + createSplitReader(tablePath, schema.getRowType())) { + assignSplitsAndFetchUntilRetrieveRecords( + splitReader, + Collections.singletonList(split), + expectedRecords, + schema.getRowType()); + } + } + @Test void testHandleMixSnapshotLogSplitChangesAndFetch() throws Exception { TablePath tablePath = TablePath.of(DEFAULT_DB, "test-mix-snapshot-log-table"); @@ -338,13 +387,15 @@ void testSubscribeEmptySplits() throws Exception { long tableId = createTable( tablePath, - TableDescriptor.builder().schema(schema).distributedBy(3).build()); + TableDescriptor.builder().schema(schema).distributedBy(4).build()); - // create two empty splits with log start offset equal to end offset + // create three bounded empty splits and one unbounded split LogSplit split1 = new LogSplit(new TableBucket(tableId, 0), null, 0, 0); LogSplit split2 = new LogSplit(new TableBucket(tableId, 1), null, 0, 0); - LogSplit split3 = new LogSplit(new TableBucket(tableId, 2), null, EARLIEST_OFFSET); - List subscribeSplits = Arrays.asList(split1, split2, split3); + LogSplit split3 = new LogSplit(new TableBucket(tableId, 2), null, EARLIEST_OFFSET, 0); + LogSplit split4 = new LogSplit(new TableBucket(tableId, 3), null, EARLIEST_OFFSET); + + List subscribeSplits = Arrays.asList(split1, split2, split3, split4); try (FlinkSourceSplitReader splitReader = createSplitReader(tablePath, schema.getRowType())) { @@ -352,9 +403,10 @@ void testSubscribeEmptySplits() throws Exception { // fetch records RecordsWithSplitIds records = splitReader.fetch(); - // finished splits should be split1,split2 + // finished splits should be split1, split2, split3 assertThat(records.finishedSplits()) - .containsExactlyInAnyOrder(split1.splitId(), split2.splitId()); + .containsExactlyInAnyOrder( + split1.splitId(), split2.splitId(), split3.splitId()); } } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/split/SourceSplitSerializerTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/split/SourceSplitSerializerTest.java index 33cefaaf6cc..c047de34ee7 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/split/SourceSplitSerializerTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/split/SourceSplitSerializerTest.java @@ -87,6 +87,14 @@ void testLogSplitSerde(boolean isPartitioned) throws Exception { SourceSplitBase deserializedSplit = serializer.deserialize(serializer.getVersion(), serialized); assertThat(deserializedSplit).isEqualTo(logSplit); + + LogSplit boundedLogSplit = new LogSplit(bucket, partitionName, 100L, 200L); + + serialized = serializer.serialize(boundedLogSplit); + deserializedSplit = serializer.deserialize(serializer.getVersion(), serialized); + + assertThat(deserializedSplit).isEqualTo(boundedLogSplit); + assertThat(deserializedSplit.asLogSplit().getStoppingOffset()).contains(200L); } @ParameterizedTest From 9146f054a797ae80cb7eb83496c75d8b1e1a559e Mon Sep 17 00:00:00 2001 From: naivedogger Date: Tue, 28 Jul 2026 10:50:07 +0800 Subject: [PATCH 2/9] [flink] Support timestamp as stopping offset via scan.bounded.mode for log tables --- .../fluss/flink/FlinkConnectorOptions.java | 65 ++++++++++++++ .../flink/catalog/FlinkTableFactory.java | 5 ++ .../fluss/flink/source/FlinkSource.java | 6 ++ .../fluss/flink/source/FlinkTableSource.java | 79 +++++++++++++++++ .../enumerator/FlinkSourceEnumerator.java | 87 ++++++++++++++++++- .../utils/FlinkConnectorOptionsUtils.java | 43 +++++++++ .../flink/catalog/FlinkTableFactoryTest.java | 16 ++++ .../enumerator/FlinkSourceEnumeratorTest.java | 72 +++++++++++++++ website/docs/engine-flink/options.md | 2 + website/docs/engine-flink/reads.md | 23 +++++ 10 files changed, 397 insertions(+), 1 deletion(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/FlinkConnectorOptions.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/FlinkConnectorOptions.java index 23d8f0b2e9c..cbbd9d633af 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/FlinkConnectorOptions.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/FlinkConnectorOptions.java @@ -137,6 +137,34 @@ public class FlinkConnectorOptions { + "The format is 'timestamp' or 'yyyy-MM-dd HH:mm:ss'. " + "Like '1678883047356' or '2023-12-09 23:09:12'."); + public static final ConfigOption SCAN_BOUNDED_MODE = + ConfigOptions.key("scan.bounded.mode") + .enumType(ScanBoundedMode.class) + .defaultValue(ScanBoundedMode.UNBOUNDED) + .withDescription( + String.format( + "Bounded mode for the Fluss source. Default is '%s'. In batch " + + "execution mode, '%s' behaves the same as '%s': the " + + "source reads up to the latest log offsets captured " + + "at startup. Currently, bounded modes other than '%s' " + + "are only supported for log tables.", + ScanBoundedMode.UNBOUNDED.value, + ScanBoundedMode.UNBOUNDED.value, + ScanBoundedMode.LATEST_OFFSET.value, + ScanBoundedMode.UNBOUNDED.value)); + + public static final ConfigOption SCAN_BOUNDED_TIMESTAMP = + ConfigOptions.key("scan.bounded.timestamp") + .stringType() + .noDefaultValue() + .withDescription( + "Optional timestamp for Fluss source in case of bounded mode is timestamp. " + + "The source stops before the first record batch whose commit " + + "timestamp is greater than or equal to the given timestamp, i.e. " + + "only records with a commit timestamp smaller than the given " + + "timestamp are read. The format is 'timestamp' or " + + "'yyyy-MM-dd HH:mm:ss'. Like '1678883047356' or '2023-12-09 23:09:12'."); + public static final ConfigOption SCAN_PARTITION_DISCOVERY_INTERVAL = ConfigOptions.key("scan.partition.discovery.interval") .durationType() @@ -333,4 +361,41 @@ public InlineElement getDescription() { return description; } } + + /** Bounded mode for the fluss scanner, see {@link #SCAN_BOUNDED_MODE}. */ + public enum ScanBoundedMode implements DescribedEnum { + UNBOUNDED( + "unbounded", + text( + "In streaming execution mode, the source never stops. In batch execution " + + "mode, the source reads up to the latest log offsets captured " + + "at startup.")), + LATEST_OFFSET( + "latest-offset", + text("Bounded by the latest log offsets captured when the source starts.")), + TIMESTAMP( + "timestamp", + text( + "Bounded by a user-supplied timestamp. The source stops before the first " + + "record batch whose commit timestamp is greater than or equal " + + "to the given timestamp.")); + + private final String value; + private final InlineElement description; + + ScanBoundedMode(String value, InlineElement description) { + this.value = value; + this.description = description; + } + + @Override + public String toString() { + return value; + } + + @Override + public InlineElement getDescription() { + return description; + } + } } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java index 1b082855e6e..2d463084afc 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java @@ -119,6 +119,8 @@ public DynamicTableSource createDynamicTableSource(Context context) { context.getConfiguration().get(TableConfigOptions.LOCAL_TIME_ZONE)); final FlinkConnectorOptionsUtils.StartupOptions startupOptions = FlinkConnectorOptionsUtils.getStartupOptions(tableOptions, timeZone); + final FlinkConnectorOptionsUtils.BoundedOptions boundedOptions = + FlinkConnectorOptionsUtils.getBoundedOptions(tableOptions, timeZone); ResolvedSchema resolvedSchema = context.getCatalogTable().getResolvedSchema(); ResolvedCatalogTable resolvedCatalogTable = context.getCatalogTable(); @@ -161,6 +163,7 @@ public DynamicTableSource createDynamicTableSource(Context context) { partitionKeyIndexes, isStreamingMode, startupOptions, + boundedOptions, tableOptions.get(FlinkConnectorOptions.LOOKUP_ASYNC), tableOptions.get(FlinkConnectorOptions.LOOKUP_INSERT_IF_NOT_EXISTS), cache, @@ -236,6 +239,8 @@ public Set> optionalOptions() { FlinkConnectorOptions.BUCKET_NUMBER, FlinkConnectorOptions.SCAN_STARTUP_MODE, FlinkConnectorOptions.SCAN_STARTUP_TIMESTAMP, + FlinkConnectorOptions.SCAN_BOUNDED_MODE, + FlinkConnectorOptions.SCAN_BOUNDED_TIMESTAMP, FlinkConnectorOptions.SCAN_PARTITION_DISCOVERY_INTERVAL, FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE, FlinkConnectorOptions.SCAN_KV_SNAPSHOT_LEASE_ID, diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java index 85ace290409..a550b99b07e 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java @@ -75,6 +75,7 @@ public class FlinkSource @Nullable private final FlinkRecordEmitter.OutputProjection outputProjection; @Nullable private final int[] projectedFields; protected final OffsetsInitializer offsetsInitializer; + @Nullable protected final OffsetsInitializer stoppingOffsetsInitializer; protected final long scanPartitionDiscoveryIntervalMs; protected final int splitPerAssignmentBatchSize; private final boolean streaming; @@ -208,6 +209,7 @@ public FlinkSource( projectedFields, logRecordBatchFilter, offsetsInitializer, + null, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, deserializationSchema, @@ -243,6 +245,7 @@ public FlinkSource( projectedFields, logRecordBatchFilter, offsetsInitializer, + null, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, deserializationSchema, @@ -262,6 +265,7 @@ public FlinkSource( @Nullable int[] projectedFields, @Nullable Predicate logRecordBatchFilter, OffsetsInitializer offsetsInitializer, + @Nullable OffsetsInitializer stoppingOffsetsInitializer, long scanPartitionDiscoveryIntervalMs, int splitPerAssignmentBatchSize, FlussDeserializationSchema deserializationSchema, @@ -278,6 +282,7 @@ public FlinkSource( this.projectedFields = projectedFields; this.logRecordBatchFilter = logRecordBatchFilter; this.offsetsInitializer = offsetsInitializer; + this.stoppingOffsetsInitializer = stoppingOffsetsInitializer; this.scanPartitionDiscoveryIntervalMs = scanPartitionDiscoveryIntervalMs; this.splitPerAssignmentBatchSize = splitPerAssignmentBatchSize; this.deserializationSchema = deserializationSchema; @@ -304,6 +309,7 @@ public SplitEnumerator createEnumerator( isPartitioned, splitEnumeratorContext, offsetsInitializer, + stoppingOffsetsInitializer, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, streaming, diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java index d2d3c7eaeec..21369283a5a 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java @@ -130,6 +130,7 @@ public class FlinkTableSource private final int[] partitionKeyIndexes; private final boolean streaming; private final FlinkConnectorOptionsUtils.StartupOptions startupOptions; + private final FlinkConnectorOptionsUtils.BoundedOptions boundedOptions; // options for lookup source private final boolean lookupAsync; @@ -221,6 +222,7 @@ public FlinkTableSource( int[] partitionKeyIndexes, boolean streaming, FlinkConnectorOptionsUtils.StartupOptions startupOptions, + FlinkConnectorOptionsUtils.BoundedOptions boundedOptions, boolean lookupAsync, boolean insertIfNotExists, @Nullable LookupCache cache, @@ -239,6 +241,7 @@ public FlinkTableSource( this.partitionKeyIndexes = partitionKeyIndexes; this.streaming = streaming; this.startupOptions = checkNotNull(startupOptions, "startupOptions must not be null"); + this.boundedOptions = checkNotNull(boundedOptions, "boundedOptions must not be null"); this.lookupAsync = lookupAsync; this.insertIfNotExists = insertIfNotExists; @@ -264,6 +267,47 @@ public FlinkTableSource( PushdownUtils.computeAvailableStatsColumns(flussRowType, tableConfig); } + public FlinkTableSource( + TablePath tablePath, + Configuration flussConfig, + TableConfig tableConfig, + org.apache.flink.table.types.logical.RowType tableOutputType, + int[] primaryKeyIndexes, + int[] bucketKeyIndexes, + int[] partitionKeyIndexes, + boolean streaming, + FlinkConnectorOptionsUtils.StartupOptions startupOptions, + boolean lookupAsync, + boolean insertIfNotExists, + @Nullable LookupCache cache, + long scanPartitionDiscoveryIntervalMs, + int splitPerAssignmentBatchSize, + boolean isDataLakeEnabled, + @Nullable MergeEngineType mergeEngineType, + Map tableOptions, + LeaseContext leaseContext) { + this( + tablePath, + flussConfig, + tableConfig, + tableOutputType, + primaryKeyIndexes, + bucketKeyIndexes, + partitionKeyIndexes, + streaming, + startupOptions, + FlinkConnectorOptionsUtils.BoundedOptions.unbounded(), + lookupAsync, + insertIfNotExists, + cache, + scanPartitionDiscoveryIntervalMs, + splitPerAssignmentBatchSize, + isDataLakeEnabled, + mergeEngineType, + tableOptions, + leaseContext); + } + @Override public ChangelogMode getChangelogMode() { if (!streaming) { @@ -397,6 +441,8 @@ public boolean isBounded() { "Unsupported startup mode: " + startupOptions.startupMode); } + OffsetsInitializer stoppingOffsetsInitializer = createStoppingOffsetsInitializer(); + FlinkSource source = new FlinkSource<>( flussConfig, @@ -407,6 +453,7 @@ public boolean isBounded() { projectedFields, logRecordBatchFilter, offsetsInitializer, + stoppingOffsetsInitializer, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, new RowDataDeserializationSchema(), @@ -458,6 +505,37 @@ public boolean isBounded() { } } + /** + * Creates the stopping offsets initializer from the configured bounded options, or returns null + * to fall back to the default behavior, i.e. no stopping offsets in streaming execution mode + * and the latest offsets captured at startup in batch execution mode. + */ + @Nullable + private OffsetsInitializer createStoppingOffsetsInitializer() { + switch (boundedOptions.boundedMode) { + case UNBOUNDED: + return null; + case LATEST_OFFSET: + validateBoundedModeSupported(); + return OffsetsInitializer.latest(); + case TIMESTAMP: + validateBoundedModeSupported(); + return OffsetsInitializer.timestamp(boundedOptions.boundedTimestampMs); + default: + throw new IllegalArgumentException( + "Unsupported bounded mode: " + boundedOptions.boundedMode); + } + } + + private void validateBoundedModeSupported() { + if (hasPrimaryKey()) { + throw new UnsupportedOperationException( + String.format( + "'%s' is currently only supported for log tables.", + FlinkConnectorOptions.SCAN_BOUNDED_MODE.key())); + } + } + @Override public LookupRuntimeProvider getLookupRuntimeProvider(LookupContext context) { LookupNormalizer lookupNormalizer = @@ -512,6 +590,7 @@ public DynamicTableSource copy() { partitionKeyIndexes, streaming, startupOptions, + boundedOptions, lookupAsync, insertIfNotExists, cache, diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java index f774986ef22..ec097f32c74 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java @@ -268,6 +268,7 @@ public FlinkSourceEnumerator( isPartitioned, context, startingOffsetsInitializer, + null, scanPartitionDiscoveryIntervalMs, FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE.defaultValue(), streaming, @@ -291,6 +292,38 @@ public FlinkSourceEnumerator( @Nullable LakeSource lakeSource, LeaseContext leaseContext, boolean checkpointTriggeredBefore) { + this( + tablePath, + flussConf, + hasPrimaryKey, + isPartitioned, + context, + startingOffsetsInitializer, + null, + scanPartitionDiscoveryIntervalMs, + splitPerAssignmentBatchSize, + streaming, + partitionFilters, + lakeSource, + leaseContext, + checkpointTriggeredBefore); + } + + public FlinkSourceEnumerator( + TablePath tablePath, + Configuration flussConf, + boolean hasPrimaryKey, + boolean isPartitioned, + SplitEnumeratorContext context, + OffsetsInitializer startingOffsetsInitializer, + @Nullable OffsetsInitializer stoppingOffsetsInitializer, + long scanPartitionDiscoveryIntervalMs, + int splitPerAssignmentBatchSize, + boolean streaming, + @Nullable Predicate partitionFilters, + @Nullable LakeSource lakeSource, + LeaseContext leaseContext, + boolean checkpointTriggeredBefore) { this( tablePath, flussConf, @@ -301,11 +334,13 @@ public FlinkSourceEnumerator( Collections.emptyMap(), null, startingOffsetsInitializer, + stoppingOffsetsInitializer, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, streaming, partitionFilters, lakeSource, + new WorkerExecutor(context), leaseContext, checkpointTriggeredBefore, false, @@ -380,6 +415,7 @@ public FlinkSourceEnumerator( assignedPartitions, pendingHybridLakeFlussSplits, startingOffsetsInitializer, + null, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, streaming, @@ -419,6 +455,7 @@ public FlinkSourceEnumerator( assignedPartitions, pendingHybridLakeFlussSplits, startingOffsetsInitializer, + null, scanPartitionDiscoveryIntervalMs, FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE.defaultValue(), streaming, @@ -451,6 +488,50 @@ public FlinkSourceEnumerator( boolean checkpointTriggeredBefore, boolean initialDiscoveryFinished, Collection unassignedSplits) { + this( + tablePath, + flussConf, + hasPrimaryKey, + isPartitioned, + context, + assignedTableBuckets, + assignedPartitions, + pendingHybridLakeFlussSplits, + startingOffsetsInitializer, + null, + scanPartitionDiscoveryIntervalMs, + splitPerAssignmentBatchSize, + streaming, + partitionFilters, + lakeSource, + workerExecutor, + leaseContext, + checkpointTriggeredBefore, + initialDiscoveryFinished, + unassignedSplits); + } + + FlinkSourceEnumerator( + TablePath tablePath, + Configuration flussConf, + boolean hasPrimaryKey, + boolean isPartitioned, + SplitEnumeratorContext context, + Set assignedTableBuckets, + Map assignedPartitions, + List pendingHybridLakeFlussSplits, + OffsetsInitializer startingOffsetsInitializer, + @Nullable OffsetsInitializer stoppingOffsetsInitializer, + long scanPartitionDiscoveryIntervalMs, + int splitPerAssignmentBatchSize, + boolean streaming, + @Nullable Predicate partitionFilters, + @Nullable LakeSource lakeSource, + WorkerExecutor workerExecutor, + LeaseContext leaseContext, + boolean checkpointTriggeredBefore, + boolean initialDiscoveryFinished, + Collection unassignedSplits) { checkArgument( splitPerAssignmentBatchSize > 0, "Split assignment batch size must be positive, but was %s.", @@ -473,7 +554,11 @@ public FlinkSourceEnumerator( this.streaming = streaming; this.partitionFilters = partitionFilters; this.stoppingOffsetsInitializer = - streaming ? new NoStoppingOffsetsInitializer() : OffsetsInitializer.latest(); + streaming + ? new NoStoppingOffsetsInitializer() + : (stoppingOffsetsInitializer != null + ? stoppingOffsetsInitializer + : OffsetsInitializer.latest()); this.lakeSource = lakeSource; this.workerExecutor = workerExecutor; this.leaseContext = leaseContext; diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java index f589203b7ad..dfec2018cda 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java @@ -19,6 +19,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.flink.FlinkConnectorOptions; +import org.apache.fluss.flink.FlinkConnectorOptions.ScanBoundedMode; import org.apache.fluss.flink.FlinkConnectorOptions.ScanStartupMode; import org.apache.fluss.flink.sink.shuffle.DistributionMode; import org.apache.fluss.metadata.MergeEngineType; @@ -43,6 +44,8 @@ import static org.apache.fluss.config.ConfigOptions.CLIENT_SCANNER_IO_TMP_DIR; import static org.apache.fluss.config.ConfigOptions.LAKE_TIERING_IO_TMP_DIRS; +import static org.apache.fluss.flink.FlinkConnectorOptions.SCAN_BOUNDED_MODE; +import static org.apache.fluss.flink.FlinkConnectorOptions.SCAN_BOUNDED_TIMESTAMP; import static org.apache.fluss.flink.FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE; import static org.apache.fluss.flink.FlinkConnectorOptions.SCAN_STARTUP_MODE; import static org.apache.fluss.flink.FlinkConnectorOptions.SCAN_STARTUP_TIMESTAMP; @@ -62,6 +65,7 @@ public static ZoneId getLocalTimeZone(String timeZone) { public static void validateTableSourceOptions(ReadableConfig tableOptions) { validateScanStartupMode(tableOptions); + validateScanBoundedMode(tableOptions); validateScanSplitAssignmentBatchSize(tableOptions); } @@ -109,6 +113,20 @@ public static StartupOptions getStartupOptions(ReadableConfig tableOptions, Zone return options; } + public static BoundedOptions getBoundedOptions(ReadableConfig tableOptions, ZoneId timeZone) { + ScanBoundedMode scanBoundedMode = tableOptions.get(SCAN_BOUNDED_MODE); + final BoundedOptions options = new BoundedOptions(); + options.boundedMode = scanBoundedMode; + if (scanBoundedMode == ScanBoundedMode.TIMESTAMP) { + options.boundedTimestampMs = + parseTimestamp( + tableOptions.get(SCAN_BOUNDED_TIMESTAMP), + SCAN_BOUNDED_TIMESTAMP.key(), + timeZone); + } + return options; + } + public static List getBucketKeys(ReadableConfig tableOptions) { Optional bucketKey = tableOptions.getOptional(FlinkConnectorOptions.BUCKET_KEY); if (!bucketKey.isPresent()) { @@ -156,6 +174,18 @@ private static void validateScanStartupMode(ReadableConfig tableOptions) { } } + private static void validateScanBoundedMode(ReadableConfig tableOptions) { + ScanBoundedMode scanBoundedMode = tableOptions.get(SCAN_BOUNDED_MODE); + if (scanBoundedMode == ScanBoundedMode.TIMESTAMP) { + if (!tableOptions.getOptional(SCAN_BOUNDED_TIMESTAMP).isPresent()) { + throw new ValidationException( + String.format( + "'%s' is required in '%s' bounded mode but missing.", + SCAN_BOUNDED_TIMESTAMP.key(), ScanBoundedMode.TIMESTAMP)); + } + } + } + private static void validateScanSplitAssignmentBatchSize(ReadableConfig tableOptions) { int batchSize = tableOptions.get(SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE); if (batchSize <= 0) { @@ -238,4 +268,17 @@ public static class StartupOptions { public ScanStartupMode startupMode; public long startupTimestampMs; } + + /** Fluss bounded options. * */ + public static class BoundedOptions { + public ScanBoundedMode boundedMode; + public long boundedTimestampMs; + + /** Returns the default bounded options, i.e. no user-specified stopping offsets. */ + public static BoundedOptions unbounded() { + BoundedOptions options = new BoundedOptions(); + options.boundedMode = ScanBoundedMode.UNBOUNDED; + return options; + } + } } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/catalog/FlinkTableFactoryTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/catalog/FlinkTableFactoryTest.java index bbf5c29aa8e..532333fca76 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/catalog/FlinkTableFactoryTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/catalog/FlinkTableFactoryTest.java @@ -105,6 +105,22 @@ void testTableSourceOptions() { FlinkConnectorOptions.SCAN_STARTUP_TIMESTAMP.key(), "2023-12-09 23:09:12"); createTableSource(schema, scanModeProperties); + // test scan bounded mode options + Map boundedModeProperties = getBasicOptions(); + boundedModeProperties.put(FlinkConnectorOptions.SCAN_BOUNDED_MODE.key(), "timestamp"); + assertThatThrownBy(() -> createTableSource(schema, boundedModeProperties)) + .isInstanceOf(ValidationException.class) + .hasMessageContaining( + "'scan.bounded.timestamp' is required in 'timestamp' bounded mode but missing."); + boundedModeProperties.put( + FlinkConnectorOptions.SCAN_BOUNDED_TIMESTAMP.key(), "1678883047356"); + createTableSource(schema, boundedModeProperties); + boundedModeProperties.put( + FlinkConnectorOptions.SCAN_BOUNDED_TIMESTAMP.key(), "2023-12-09 23:09:12"); + createTableSource(schema, boundedModeProperties); + boundedModeProperties.put(FlinkConnectorOptions.SCAN_BOUNDED_MODE.key(), "latest-offset"); + createTableSource(schema, boundedModeProperties); + // test split assignment batch size Map splitAssignmentBatchProperties = getBasicOptions(); splitAssignmentBatchProperties.put( diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java index 10f48aee329..647931ded36 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java @@ -1182,6 +1182,78 @@ void testBatchModeNonLakePartitionedLogTable() throws Throwable { } } + @Test + void testBatchModeWithTimestampStoppingOffsets() throws Throwable { + int numSubtasks = DEFAULT_BUCKET_NUM; + createTable(DEFAULT_TABLE_PATH, DEFAULT_LOG_TABLE_DESCRIPTOR); + List rows = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + rows.add(row(i, "v" + i)); + } + writeRows(conn, DEFAULT_TABLE_PATH, rows, true); + + // A stopping timestamp not earlier than all commit timestamps resolves to the latest + // offsets, so the generated splits cover all written records. + List bucketIds = new ArrayList<>(); + for (int bucket = 0; bucket < DEFAULT_BUCKET_NUM; bucket++) { + bucketIds.add(bucket); + } + Map latestOffsets = + admin.listOffsets(DEFAULT_TABLE_PATH, bucketIds, new OffsetSpec.LatestSpec()) + .all() + .get(); + + // wait until the clock strictly advances past the write acknowledgement, so that the + // stopping timestamp is strictly greater than all commit timestamps + long writeAckTime = System.currentTimeMillis(); + long stoppingTimestamp; + do { + stoppingTimestamp = System.currentTimeMillis(); + } while (stoppingTimestamp <= writeAckTime); + + try (MockSplitEnumeratorContext context = + new MockSplitEnumeratorContext<>(numSubtasks)) { + FlinkSourceEnumerator enumerator = + new FlinkSourceEnumerator( + DEFAULT_TABLE_PATH, + flussConf, + false, + false, + context, + OffsetsInitializer.earliest(), + OffsetsInitializer.timestamp(stoppingTimestamp), + DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, + FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE.defaultValue(), + false, + null, + null, + LeaseContext.DEFAULT, + false); + + enumerator.start(); + for (int i = 0; i < numSubtasks; i++) { + registerReader(context, enumerator, i); + } + context.runNextOneTimeCallable(); + + List assignedSplits = + getReadersAssignments(context).values().stream() + .flatMap(List::stream) + .collect(Collectors.toList()); + assertThat(assignedSplits).hasSize(DEFAULT_BUCKET_NUM); + assertThat(assignedSplits) + .allSatisfy( + split -> { + LogSplit logSplit = split.asLogSplit(); + assertThat(logSplit.getStartingOffset()).isEqualTo(EARLIEST_OFFSET); + assertThat(logSplit.getStoppingOffset()) + .contains( + latestOffsets.get( + logSplit.getTableBucket().getBucket())); + }); + } + } + @Test void testGetSplitOwner() throws Exception { int numSubtasks = 3; diff --git a/website/docs/engine-flink/options.md b/website/docs/engine-flink/options.md index d04285ee942..e1a0b5d818e 100644 --- a/website/docs/engine-flink/options.md +++ b/website/docs/engine-flink/options.md @@ -102,6 +102,8 @@ See more details about [ALTER TABLE ... SET](engine-flink/ddl.md#set-properties) |-----------------------------------------------|------------|-------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | scan.startup.mode | Enum | full | The scan startup mode enables you to specify the starting point for data consumption. Fluss currently supports the following `scan.startup.mode` options: `full` (default), earliest, latest, timestamp. See the [Start Reading Position](engine-flink/reads.md#start-reading-position) for more details. | | scan.startup.timestamp | Long | (None) | The timestamp to start reading the data from. This option is only valid when `scan.startup.mode` is set to `timestamp`. The format is 'milli-second-since-epoch' or `yyyy-MM-dd HH:mm:ss`, like `1678883047356` or `2023-12-09 23:09:12`. | +| scan.bounded.mode | Enum | unbounded | The scan bounded mode enables you to specify where the source stops reading. Fluss currently supports the following `scan.bounded.mode` options: `unbounded` (default), `latest-offset`, `timestamp`. Currently, bounded modes other than `unbounded` are only supported for Log Tables. See the [Stop Reading Position](engine-flink/reads.md#stop-reading-position) for more details. | +| scan.bounded.timestamp | Long | (None) | The timestamp to stop reading the data at. This option is only valid when `scan.bounded.mode` is set to `timestamp`. Only records with a commit timestamp smaller than the given timestamp are read. The timestamp must not be in the future. The format is 'milli-second-since-epoch' or `yyyy-MM-dd HH:mm:ss`, like `1678883047356` or `2023-12-09 23:09:12`. | | scan.partition.discovery.interval | Duration | 1min | The time interval for the Fluss source to discover the new partitions for partitioned table while scanning. A non-positive value disables the partition discovery. The default value is 1 minute. Currently, since Fluss Admin#listPartitions(TablePath tablePath) requires a large number of requests to ZooKeeper in server, this option cannot be set too small, as a small value would cause frequent requests and increase server load. In the future, once list partitions is optimized, the default value of this parameter can be reduced. | | scan.kv.snapshot.lease.id | String | UUID | The lease ID used to protect acquired KV snapshots from deletion. If specified, the snapshots will be retained until either the consumer finishes processing all of them or the lease duration expires. By default, this value is set to a randomly generated UUID string if not explicitly provided. | | scan.kv.snapshot.lease.duration | Duration | 1day | The time period how long to wait before expiring the kv snapshot lease to avoid kv snapshot blocking to delete. | diff --git a/website/docs/engine-flink/reads.md b/website/docs/engine-flink/reads.md index 4e4d82e5bc8..9940bd567d9 100644 --- a/website/docs/engine-flink/reads.md +++ b/website/docs/engine-flink/reads.md @@ -396,6 +396,29 @@ SELECT * FROM log_table 'scan.startup.timestamp' = '2023-12-09 23:09:12') */; ``` +### Stop Reading Position + +The config option `scan.bounded.mode` enables you to specify where the source stops reading. It is currently only supported for Log Tables. Fluss supports the following `scan.bounded.mode` options: +- `unbounded` (default): In streaming execution mode, the source never stops. In batch execution mode, the source reads up to the latest log offsets captured when the source starts. +- `latest-offset`: The source stops at the latest log offsets captured when the source starts. In batch execution mode, this behaves the same as `unbounded`. +- `timestamp`: The source stops before the first record batch whose commit timestamp is greater than or equal to the specified time (defined by the configuration item `scan.bounded.timestamp`), i.e. only records with a commit timestamp smaller than the specified time are read. The specified time must not be in the future. + +The following SQL statement reads the `log_table` table up to a specified time. +```sql title="Flink SQL" +SELECT * FROM log_table +/*+ OPTIONS('scan.bounded.mode' = 'timestamp', +'scan.bounded.timestamp' = '2023-12-09 23:09:12') */; +``` + +The start and stop reading positions can be combined to read a time range of the log. +```sql title="Flink SQL" +SELECT * FROM log_table +/*+ OPTIONS('scan.startup.mode' = 'timestamp', +'scan.startup.timestamp' = '2023-12-09 00:00:00', +'scan.bounded.mode' = 'timestamp', +'scan.bounded.timestamp' = '2023-12-10 00:00:00') */; +``` + From aaff5e7f1990679acdb9aff008b94497ffe37c10 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Tue, 28 Jul 2026 14:15:36 +0800 Subject: [PATCH 3/9] [flink] Support bounded streaming read for Fluss source Similar to the Kafka connector's bounded read, a streaming job can now read from a given starting position up to a given stopping position and then finish, which is useful for replaying a bounded time range of the log, backfilling and archiving. - FlinkSource reports BOUNDED when stopping offsets are supplied, and passes them through createEnumerator/restoreEnumerator. - FlinkSourceEnumerator treats a streaming read with stopping offsets as bounded: one-time partition discovery and NoMoreSplits signaling, so the job finishes once all splits reach their stopping offsets. - scan.bounded.mode is supported for log tables, the changelog of primary key tables (earliest/latest/timestamp startup mode) and the $changelog/$binlog virtual tables; the full startup mode of primary key tables and the datalake union read are rejected explicitly. - FlussSourceBuilder#setBounded(OffsetsInitializer) enables bounded streaming reads in the DataStream API. --- .../fluss/flink/FlinkConnectorOptions.java | 12 ++- .../flink/catalog/FlinkTableFactory.java | 6 ++ .../flink/source/BinlogFlinkTableSource.java | 31 +++++++ .../source/ChangelogFlinkTableSource.java | 31 +++++++ .../fluss/flink/source/FlinkSource.java | 8 +- .../fluss/flink/source/FlinkTableSource.java | 44 ++++++---- .../fluss/flink/source/FlussSource.java | 34 ++++++++ .../flink/source/FlussSourceBuilder.java | 49 +++++++++++ .../enumerator/FlinkSourceEnumerator.java | 76 ++++++++++++++--- .../utils/FlinkConnectorOptionsUtils.java | 20 +++++ .../source/BinlogVirtualTableITCase.java | 43 ++++++++++ .../source/ChangelogVirtualTableITCase.java | 33 ++++++++ .../flink/source/FlinkTableSourceITCase.java | 83 +++++++++++++++++++ .../enumerator/FlinkSourceEnumeratorTest.java | 75 +++++++++++++++++ website/docs/engine-flink/datastream.mdx | 23 +++++ website/docs/engine-flink/options.md | 2 +- website/docs/engine-flink/reads.md | 15 +++- 17 files changed, 555 insertions(+), 30 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/FlinkConnectorOptions.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/FlinkConnectorOptions.java index cbbd9d633af..ceffa72a333 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/FlinkConnectorOptions.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/FlinkConnectorOptions.java @@ -146,11 +146,19 @@ public class FlinkConnectorOptions { "Bounded mode for the Fluss source. Default is '%s'. In batch " + "execution mode, '%s' behaves the same as '%s': the " + "source reads up to the latest log offsets captured " - + "at startup. Currently, bounded modes other than '%s' " - + "are only supported for log tables.", + + "at startup. In streaming execution mode, a bounded " + + "mode other than '%s' makes the source stop at the " + + "given stopping offsets and then the job finishes " + + "(a bounded streaming read). Bounded modes other " + + "than '%s' are supported for log tables and the " + + "changelog of primary key tables (earliest/latest/" + + "timestamp startup mode), but not for the full " + + "startup mode of primary key tables or the datalake " + + "union read.", ScanBoundedMode.UNBOUNDED.value, ScanBoundedMode.UNBOUNDED.value, ScanBoundedMode.LATEST_OFFSET.value, + ScanBoundedMode.UNBOUNDED.value, ScanBoundedMode.UNBOUNDED.value)); public static final ConfigOption SCAN_BOUNDED_TIMESTAMP = diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java index 2d463084afc..43051038b97 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/catalog/FlinkTableFactory.java @@ -361,6 +361,8 @@ private DynamicTableSource createChangelogTableSource( context.getConfiguration().get(TableConfigOptions.LOCAL_TIME_ZONE)); final FlinkConnectorOptionsUtils.StartupOptions startupOptions = FlinkConnectorOptionsUtils.getStartupOptions(tableOptions, timeZone); + final FlinkConnectorOptionsUtils.BoundedOptions boundedOptions = + FlinkConnectorOptionsUtils.getBoundedOptions(tableOptions, timeZone); ResolvedCatalogTable resolvedCatalogTable = context.getCatalogTable(); @@ -384,6 +386,7 @@ private DynamicTableSource createChangelogTableSource( partitionKeyIndexes, isStreamingMode, startupOptions, + boundedOptions, partitionDiscoveryIntervalMs, splitAssignmentBatchSize, catalogTableOptions); @@ -415,6 +418,8 @@ private DynamicTableSource createBinlogTableSource( context.getConfiguration().get(TableConfigOptions.LOCAL_TIME_ZONE)); final FlinkConnectorOptionsUtils.StartupOptions startupOptions = FlinkConnectorOptionsUtils.getStartupOptions(tableOptions, timeZone); + final FlinkConnectorOptionsUtils.BoundedOptions boundedOptions = + FlinkConnectorOptionsUtils.getBoundedOptions(tableOptions, timeZone); // Check if the table is partitioned from the internal option boolean isPartitioned = @@ -434,6 +439,7 @@ private DynamicTableSource createBinlogTableSource( isPartitioned, isStreamingMode, startupOptions, + boundedOptions, partitionDiscoveryIntervalMs, splitAssignmentBatchSize, catalogTableOptions); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/BinlogFlinkTableSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/BinlogFlinkTableSource.java index d442dee27ab..6684857caaa 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/BinlogFlinkTableSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/BinlogFlinkTableSource.java @@ -55,6 +55,7 @@ public class BinlogFlinkTableSource private final boolean isPartitioned; private final boolean streaming; private final FlinkConnectorOptionsUtils.StartupOptions startupOptions; + private final FlinkConnectorOptionsUtils.BoundedOptions boundedOptions; private final long scanPartitionDiscoveryIntervalMs; private final int splitPerAssignmentBatchSize; private final Map tableOptions; @@ -93,12 +94,37 @@ public BinlogFlinkTableSource( long scanPartitionDiscoveryIntervalMs, int splitPerAssignmentBatchSize, Map tableOptions) { + this( + tablePath, + flussConfig, + binlogOutputType, + isPartitioned, + streaming, + startupOptions, + FlinkConnectorOptionsUtils.BoundedOptions.unbounded(), + scanPartitionDiscoveryIntervalMs, + splitPerAssignmentBatchSize, + tableOptions); + } + + public BinlogFlinkTableSource( + TablePath tablePath, + Configuration flussConfig, + org.apache.flink.table.types.logical.RowType binlogOutputType, + boolean isPartitioned, + boolean streaming, + FlinkConnectorOptionsUtils.StartupOptions startupOptions, + FlinkConnectorOptionsUtils.BoundedOptions boundedOptions, + long scanPartitionDiscoveryIntervalMs, + int splitPerAssignmentBatchSize, + Map tableOptions) { this.tablePath = tablePath; this.flussConfig = flussConfig; this.binlogOutputType = binlogOutputType; this.isPartitioned = isPartitioned; this.streaming = streaming; this.startupOptions = startupOptions; + this.boundedOptions = boundedOptions; this.scanPartitionDiscoveryIntervalMs = scanPartitionDiscoveryIntervalMs; this.splitPerAssignmentBatchSize = splitPerAssignmentBatchSize; this.tableOptions = tableOptions; @@ -142,6 +168,8 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { } // Create the source with the binlog deserialization schema + OffsetsInitializer stoppingOffsetsInitializer = + FlinkConnectorOptionsUtils.toStoppingOffsetsInitializer(boundedOptions); FlinkSource source = new FlinkSource<>( flussConfig, @@ -152,6 +180,7 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { null, null, offsetsInitializer, + stoppingOffsetsInitializer, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, new BinlogDeserializationSchema(), @@ -160,6 +189,7 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { // $binlog data/partition columns are nested inside before/after ROWs, so no // top-level partition filter is pushable; always scan without one. null, + null, LeaseContext.DEFAULT); return SourceProvider.of(source); @@ -175,6 +205,7 @@ public DynamicTableSource copy() { isPartitioned, streaming, startupOptions, + boundedOptions, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, tableOptions); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/ChangelogFlinkTableSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/ChangelogFlinkTableSource.java index fef9f40ebd2..65938de4185 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/ChangelogFlinkTableSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/ChangelogFlinkTableSource.java @@ -74,6 +74,7 @@ public class ChangelogFlinkTableSource private final int[] partitionKeyIndexes; private final boolean streaming; private final FlinkConnectorOptionsUtils.StartupOptions startupOptions; + private final FlinkConnectorOptionsUtils.BoundedOptions boundedOptions; private final long scanPartitionDiscoveryIntervalMs; private final int splitPerAssignmentBatchSize; private final Map tableOptions; @@ -129,6 +130,30 @@ public ChangelogFlinkTableSource( long scanPartitionDiscoveryIntervalMs, int splitPerAssignmentBatchSize, Map tableOptions) { + this( + tablePath, + flussConfig, + changelogOutputType, + partitionKeyIndexes, + streaming, + startupOptions, + FlinkConnectorOptionsUtils.BoundedOptions.unbounded(), + scanPartitionDiscoveryIntervalMs, + splitPerAssignmentBatchSize, + tableOptions); + } + + public ChangelogFlinkTableSource( + TablePath tablePath, + Configuration flussConfig, + org.apache.flink.table.types.logical.RowType changelogOutputType, + int[] partitionKeyIndexes, + boolean streaming, + FlinkConnectorOptionsUtils.StartupOptions startupOptions, + FlinkConnectorOptionsUtils.BoundedOptions boundedOptions, + long scanPartitionDiscoveryIntervalMs, + int splitPerAssignmentBatchSize, + Map tableOptions) { this.tablePath = tablePath; this.flussConfig = flussConfig; // The changelogOutputType already includes metadata columns from FlinkCatalog @@ -136,6 +161,7 @@ public ChangelogFlinkTableSource( this.partitionKeyIndexes = partitionKeyIndexes; this.streaming = streaming; this.startupOptions = startupOptions; + this.boundedOptions = boundedOptions; this.scanPartitionDiscoveryIntervalMs = scanPartitionDiscoveryIntervalMs; this.splitPerAssignmentBatchSize = splitPerAssignmentBatchSize; this.tableOptions = tableOptions; @@ -198,6 +224,8 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { } // Create the source with the changelog deserialization schema + OffsetsInitializer stoppingOffsetsInitializer = + FlinkConnectorOptionsUtils.toStoppingOffsetsInitializer(boundedOptions); FlinkSource source = new FlinkSource<>( flussConfig, @@ -212,12 +240,14 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { dataProjection, logRecordBatchFilter, offsetsInitializer, + stoppingOffsetsInitializer, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, new ChangelogDeserializationSchema(), FlinkConversions.toFlussRowType(producedDataType), streaming, partitionFilters, + null, LeaseContext.DEFAULT); // Lake source not supported return SourceProvider.of(source); @@ -233,6 +263,7 @@ public DynamicTableSource copy() { partitionKeyIndexes, streaming, startupOptions, + boundedOptions, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, tableOptions); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java index a550b99b07e..9c36b7eea91 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java @@ -296,7 +296,12 @@ public FlinkSource( @Override public Boundedness getBoundedness() { - return streaming ? Boundedness.CONTINUOUS_UNBOUNDED : Boundedness.BOUNDED; + // User-supplied stopping offsets make the source bounded even in streaming execution + // mode (bounded streaming read), so that the job finishes once all splits reach their + // stopping offsets. + return (streaming && stoppingOffsetsInitializer == null) + ? Boundedness.CONTINUOUS_UNBOUNDED + : Boundedness.BOUNDED; } @Override @@ -341,6 +346,7 @@ public SplitEnumerator restoreEnumerator sourceEnumeratorState.getAssignedPartitions(), remainingHybridLakeFlussSplits, offsetsInitializer, + stoppingOffsetsInitializer, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, streaming, diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java index 21369283a5a..7b1935f56e3 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java @@ -512,27 +512,43 @@ public boolean isBounded() { */ @Nullable private OffsetsInitializer createStoppingOffsetsInitializer() { - switch (boundedOptions.boundedMode) { - case UNBOUNDED: - return null; - case LATEST_OFFSET: - validateBoundedModeSupported(); - return OffsetsInitializer.latest(); - case TIMESTAMP: - validateBoundedModeSupported(); - return OffsetsInitializer.timestamp(boundedOptions.boundedTimestampMs); - default: - throw new IllegalArgumentException( - "Unsupported bounded mode: " + boundedOptions.boundedMode); + OffsetsInitializer stoppingOffsetsInitializer = + FlinkConnectorOptionsUtils.toStoppingOffsetsInitializer(boundedOptions); + if (stoppingOffsetsInitializer != null) { + validateBoundedModeSupported(); } + return stoppingOffsetsInitializer; } private void validateBoundedModeSupported() { if (hasPrimaryKey()) { + if (!streaming) { + throw new UnsupportedOperationException( + String.format( + "'%s' is not supported for primary key tables in batch execution mode.", + FlinkConnectorOptions.SCAN_BOUNDED_MODE.key())); + } + if (startupOptions.startupMode == FlinkConnectorOptions.ScanStartupMode.FULL) { + throw new UnsupportedOperationException( + String.format( + "'%s' is not supported for primary key tables in '%s' startup mode, " + + "because the snapshot reading phase has no bounded end. " + + "Use 'earliest', 'latest' or 'timestamp' startup mode to " + + "read the changelog of a primary key table with a bounded end.", + FlinkConnectorOptions.SCAN_BOUNDED_MODE.key(), + FlinkConnectorOptions.ScanStartupMode.FULL)); + } + } + if (isDataLakeEnabled + && startupOptions.startupMode == FlinkConnectorOptions.ScanStartupMode.FULL) { throw new UnsupportedOperationException( String.format( - "'%s' is currently only supported for log tables.", - FlinkConnectorOptions.SCAN_BOUNDED_MODE.key())); + "'%s' is not supported for the datalake union read, i.e. '%s' startup " + + "mode on a datalake-enabled table. Use 'earliest', 'latest' " + + "or 'timestamp' startup mode to read only the Fluss log with " + + "a bounded end.", + FlinkConnectorOptions.SCAN_BOUNDED_MODE.key(), + FlinkConnectorOptions.ScanStartupMode.FULL)); } } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java index 392d77708fb..80546e1e22a 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java @@ -106,6 +106,38 @@ public class FlussSource extends FlinkSource { FlussDeserializationSchema deserializationSchema, boolean streaming, @Nullable LakeSource lakeSource) { + this( + flussConf, + tablePath, + hasPrimaryKey, + isPartitioned, + sourceOutputType, + projectedFields, + logRecordBatchFilter, + offsetsInitializer, + null, + scanPartitionDiscoveryIntervalMs, + splitPerAssignmentBatchSize, + deserializationSchema, + streaming, + lakeSource); + } + + FlussSource( + Configuration flussConf, + TablePath tablePath, + boolean hasPrimaryKey, + boolean isPartitioned, + RowType sourceOutputType, + @Nullable int[] projectedFields, + @Nullable Predicate logRecordBatchFilter, + OffsetsInitializer offsetsInitializer, + @Nullable OffsetsInitializer stoppingOffsetsInitializer, + long scanPartitionDiscoveryIntervalMs, + int splitPerAssignmentBatchSize, + FlussDeserializationSchema deserializationSchema, + boolean streaming, + @Nullable LakeSource lakeSource) { // TODO: Support partition pushDown in datastream super( flussConf, @@ -116,9 +148,11 @@ public class FlussSource extends FlinkSource { projectedFields, logRecordBatchFilter, validateBatchStartupMode(offsetsInitializer, hasPrimaryKey, streaming, tablePath), + stoppingOffsetsInitializer, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, deserializationSchema, + null, streaming, null, lakeSource, diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java index ca691453392..c5a966a7546 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java @@ -83,6 +83,7 @@ public class FlussSourceBuilder { private Long scanPartitionDiscoveryIntervalMs; private Integer splitPerAssignmentBatchSize; private OffsetsInitializer offsetsInitializer; + private OffsetsInitializer stoppingOffsetsInitializer; private boolean bounded; private FlussDeserializationSchema deserializationSchema; @@ -185,6 +186,24 @@ public FlussSourceBuilder setBounded() { return this; } + /** + * Builds a bounded source that stops once it reaches the given stopping offsets and then + * finishes, even in streaming execution mode (a bounded streaming read). Typical usages are + * replaying a bounded time range of the log, backfilling and archiving. + * + *

Supported stopping offsets initializers are {@link OffsetsInitializer#latest()} and {@link + * OffsetsInitializer#timestamp(long)}. + * + * @param stoppingOffsetsInitializer the strategy for determining the stopping offsets + * @return this builder + */ + public FlussSourceBuilder setBounded(OffsetsInitializer stoppingOffsetsInitializer) { + this.stoppingOffsetsInitializer = + checkNotNull( + stoppingOffsetsInitializer, "stoppingOffsetsInitializer must not be null"); + return this; + } + /** * Sets the deserialization schema for converting Fluss records to output records. * @@ -362,6 +381,35 @@ public FlussSource build() { tablePath, fullStartup)); } + // Bounded streaming read support (user-supplied stopping offsets): + // - Log tables and the changelog of primary key tables (earliest/latest/timestamp + // startup mode) are supported. + // - The full startup mode of primary key tables is not supported, because the snapshot + // reading phase has no bounded end. + // - The datalake union read (full startup mode on a datalake-enabled table) is not + // supported, because lake splits have no bounded end. + if (stoppingOffsetsInitializer != null) { + if (hasPrimaryKey && fullStartup) { + throw new IllegalArgumentException( + String.format( + "Bounded read with stopping offsets on primary key table '%s' is " + + "not supported in full startup mode, because the " + + "snapshot reading phase has no bounded end. Use " + + "earliest/latest/timestamp starting offsets to read the " + + "changelog with a bounded end.", + tablePath)); + } + if (lakeEnabled && fullStartup) { + throw new IllegalArgumentException( + String.format( + "Bounded read with stopping offsets on datalake-enabled table '%s' " + + "is not supported in full startup mode (datalake union " + + "read). Use earliest/latest/timestamp starting offsets " + + "to read only the Fluss log with a bounded end.", + tablePath)); + } + } + LakeSource lakeSource = null; if (lakeEnabled && fullStartup) { lakeSource = @@ -393,6 +441,7 @@ public FlussSource build() { projectedFields, logRecordBatchFilter, offsetsInitializer, + stoppingOffsetsInitializer, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, deserializationSchema, diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java index ec097f32c74..8659cc2e470 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java @@ -154,6 +154,13 @@ public class FlinkSourceEnumerator private final OffsetsInitializer startingOffsetsInitializer; private final OffsetsInitializer stoppingOffsetsInitializer; + /** + * Whether this read is bounded, i.e. batch execution mode or a bounded streaming read with + * user-supplied stopping offsets. A bounded read only performs a one-time partition discovery, + * since partitions created after startup are outside the bounded range captured at startup. + */ + private final boolean bounded; + /** * The offsets initializer used for partitions discovered after the initial startup. Following * context, + Set assignedTableBuckets, + Map assignedPartitions, + List pendingHybridLakeFlussSplits, + OffsetsInitializer startingOffsetsInitializer, + @Nullable OffsetsInitializer stoppingOffsetsInitializer, + long scanPartitionDiscoveryIntervalMs, + int splitPerAssignmentBatchSize, + boolean streaming, + @Nullable Predicate partitionFilters, + @Nullable LakeSource lakeSource, + LeaseContext leaseContext, + boolean checkpointTriggeredBefore, + boolean initialDiscoveryFinished, + Collection unassignedSplits) { + this( + tablePath, + flussConf, + hasPrimaryKey, + isPartitioned, + context, + assignedTableBuckets, + assignedPartitions, + pendingHybridLakeFlussSplits, + startingOffsetsInitializer, + stoppingOffsetsInitializer, + scanPartitionDiscoveryIntervalMs, + splitPerAssignmentBatchSize, + streaming, + partitionFilters, + lakeSource, + new WorkerExecutor(context), + leaseContext, + checkpointTriggeredBefore, + initialDiscoveryFinished, + unassignedSplits); + } + FlinkSourceEnumerator( TablePath tablePath, Configuration flussConf, @@ -553,11 +603,14 @@ public FlinkSourceEnumerator( this.scanPartitionDiscoveryIntervalMs = scanPartitionDiscoveryIntervalMs; this.streaming = streaming; this.partitionFilters = partitionFilters; + // The read is bounded if it runs in batch execution mode, or if the user supplied + // stopping offsets for a bounded streaming read. + this.bounded = !streaming || stoppingOffsetsInitializer != null; this.stoppingOffsetsInitializer = - streaming - ? new NoStoppingOffsetsInitializer() - : (stoppingOffsetsInitializer != null - ? stoppingOffsetsInitializer + stoppingOffsetsInitializer != null + ? stoppingOffsetsInitializer + : (streaming + ? new NoStoppingOffsetsInitializer() : OffsetsInitializer.latest()); this.lakeSource = lakeSource; this.workerExecutor = workerExecutor; @@ -626,7 +679,7 @@ public void start() { } } - if (scanPartitionDiscoveryIntervalMs > 0) { + if (scanPartitionDiscoveryIntervalMs > 0 && !bounded) { // should do partition discovery LOG.info( "Starting the FlussSourceEnumerator for table {} " @@ -640,7 +693,9 @@ public void start() { 0, scanPartitionDiscoveryIntervalMs); } else { - // just call once + // Call once for a bounded read or when partition discovery is disabled. For + // a bounded read, partitions created after startup are outside the bounded + // range captured at startup, so continuous discovery is not needed. LOG.info( "Starting the FlussSourceEnumerator for table {} without partition discovery.", tablePath); @@ -1246,7 +1301,7 @@ private static boolean shouldRemoveForDroppedPartition( private void handleSplitsAdd(List splits, Throwable t) { if (t != null) { - if (isPartitioned && streaming && scanPartitionDiscoveryIntervalMs > 0) { + if (isPartitioned && streaming && !bounded && scanPartitionDiscoveryIntervalMs > 0) { // it means continuously read new partition splits, not throw exception, temporally // warn it to avoid job fail. TODO: fix me in #288 LOG.warn("Failed to list splits for {}.", tablePath, t); @@ -1278,9 +1333,10 @@ private void handleSplitsAdd(List splits, Throwable t) { : pendingHybridLakeFlussSplits.size()); if (isPartitioned) { - if (!streaming || scanPartitionDiscoveryIntervalMs <= 0) { - // if not streaming or partition discovery is disabled - // should only add splits only once, no more new splits + if (bounded || scanPartitionDiscoveryIntervalMs <= 0) { + // For a bounded read (batch execution mode or a bounded streaming read) or when + // partition discovery is disabled, splits are only added once, so readers can be + // signaled that no more splits will come and finish eventually. noMoreNewSplits = true; } } else { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java index dfec2018cda..47d2faae4d9 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java @@ -17,6 +17,7 @@ package org.apache.fluss.flink.utils; +import org.apache.fluss.client.initializer.OffsetsInitializer; import org.apache.fluss.config.Configuration; import org.apache.fluss.flink.FlinkConnectorOptions; import org.apache.fluss.flink.FlinkConnectorOptions.ScanBoundedMode; @@ -127,6 +128,25 @@ public static BoundedOptions getBoundedOptions(ReadableConfig tableOptions, Zone return options; } + /** + * Creates the stopping offsets initializer from the given bounded options, or returns null for + * the unbounded mode. + */ + @Nullable + public static OffsetsInitializer toStoppingOffsetsInitializer(BoundedOptions boundedOptions) { + switch (boundedOptions.boundedMode) { + case UNBOUNDED: + return null; + case LATEST_OFFSET: + return OffsetsInitializer.latest(); + case TIMESTAMP: + return OffsetsInitializer.timestamp(boundedOptions.boundedTimestampMs); + default: + throw new IllegalArgumentException( + "Unsupported bounded mode: " + boundedOptions.boundedMode); + } + } + public static List getBucketKeys(ReadableConfig tableOptions) { Optional bucketKey = tableOptions.getOptional(FlinkConnectorOptions.BUCKET_KEY); if (!bucketKey.isPresent()) { diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/BinlogVirtualTableITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/BinlogVirtualTableITCase.java index bb4c21330e8..da6aa14b610 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/BinlogVirtualTableITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/BinlogVirtualTableITCase.java @@ -712,4 +712,47 @@ protected static void waitForCheckpoint(JobID jobId) { Duration.ofSeconds(60), "Timeout waiting for checkpoint for job " + jobIdStr); } + + @Test + public void testBinlogBoundedRead() throws Exception { + tEnv.executeSql( + "CREATE TABLE bounded_binlog_test (" + + " id INT NOT NULL," + + " name STRING," + + " PRIMARY KEY (id) NOT ENFORCED" + + ") WITH ('bucket.num' = '1')"); + TablePath tablePath = TablePath.of(DEFAULT_DB, "bounded_binlog_test"); + + CLOCK.advanceTime(Duration.ofMillis(1000)); + writeRows(conn, tablePath, Arrays.asList(row(1, "Item-1"), row(2, "Item-2")), false); + // the update produces an update_before/update_after pair in the log + writeRows(conn, tablePath, Arrays.asList(row(1, "Item-1-Updated")), false); + CLOCK.advanceTime(Duration.ofMillis(1000)); + long boundedTimestamp = CLOCK.milliseconds(); + // records written at or after the bounded timestamp are not read + writeRows(conn, tablePath, Arrays.asList(row(2, "Item-2-Updated")), false); + + // The stopping offsets are aligned to record batch boundaries, and the update_before/ + // update_after pair of a single update is always written in one record batch, so the + // pair is never split apart by the stopping offset: the last update is either fully + // included (merged into one binlog row) or fully excluded. + String query = + "SELECT _change_type, before.id, before.name, after.id, after.name " + + "FROM bounded_binlog_test$binlog " + + String.format( + "/*+ OPTIONS('scan.bounded.mode' = 'timestamp', " + + "'scan.bounded.timestamp' = '%d') */", + boundedTimestamp); + try (CloseableIterator rowIter = tEnv.executeSql(query).collect()) { + List results = new ArrayList<>(); + while (rowIter.hasNext()) { + results.add(rowIter.next().toString()); + } + assertThat(results) + .containsExactly( + "+I[insert, null, null, 1, Item-1]", + "+I[insert, null, null, 2, Item-2]", + "+I[update, 1, Item-1, 1, Item-1-Updated]"); + } + } } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/ChangelogVirtualTableITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/ChangelogVirtualTableITCase.java index 71974955ece..91aa7d7b219 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/ChangelogVirtualTableITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/ChangelogVirtualTableITCase.java @@ -443,6 +443,39 @@ public void testChangelogVirtualTableWithLogTable() throws Exception { .isEqualTo("+I[insert, 2, 1970-01-01T00:00:02Z, 3, purchase]"); } + @Test + public void testChangelogBoundedRead() throws Exception { + tEnv.executeSql( + "CREATE TABLE bounded_changelog_test (" + + " id INT NOT NULL," + + " name STRING," + + " PRIMARY KEY (id) NOT ENFORCED" + + ") WITH ('bucket.num' = '1')"); + TablePath tablePath = TablePath.of(DEFAULT_DB, "bounded_changelog_test"); + + CLOCK.advanceTime(Duration.ofMillis(1000)); + writeRows(conn, tablePath, Arrays.asList(row(1, "Alice"), row(2, "Bob")), false); + writeRows(conn, tablePath, Arrays.asList(row(1, "Alice-2")), false); + + // the bounded changelog read stops at the latest offsets captured at startup and then + // the job finishes + String query = + "SELECT _change_type, id, name FROM bounded_changelog_test$changelog " + + "/*+ OPTIONS('scan.bounded.mode' = 'latest-offset') */"; + try (CloseableIterator rowIter = tEnv.executeSql(query).collect()) { + List results = new ArrayList<>(); + while (rowIter.hasNext()) { + results.add(rowIter.next().toString()); + } + assertThat(results) + .containsExactly( + "+I[insert, 1, Alice]", + "+I[insert, 2, Bob]", + "+I[update_before, 1, Alice]", + "+I[update_after, 1, Alice-2]"); + } + } + @Test public void testProjectionOnChangelogTable() throws Exception { // Create a primary key table with 1 bucket and extra columns to test projection diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceITCase.java index f19f20163e8..25c052c7e16 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlinkTableSourceITCase.java @@ -75,6 +75,7 @@ import static org.apache.fluss.flink.source.testutils.FlinkRowAssertionsUtils.assertQueryResultExactOrder; import static org.apache.fluss.flink.source.testutils.FlinkRowAssertionsUtils.assertResultsIgnoreOrder; +import static org.apache.fluss.flink.source.testutils.FlinkRowAssertionsUtils.collectBatchRows; import static org.apache.fluss.flink.source.testutils.FlinkRowAssertionsUtils.collectRowsWithTimeout; import static org.apache.fluss.flink.utils.FlinkTestBase.waitUntilPartitions; import static org.apache.fluss.flink.utils.FlinkTestBase.writeRows; @@ -695,6 +696,88 @@ void testReadPrimaryKeyPartitionedTable(boolean isAutoPartition) throws Exceptio assertResultsIgnoreOrder(rowIter, expectedRowValues, true); } + @Test + void testLogTableBoundedReadLatestOffset() throws Exception { + tEnv.executeSql("create table bounded_latest_log_table (a int, b varchar)"); + TablePath tablePath = TablePath.of(DEFAULT_DB, "bounded_latest_log_table"); + writeRows(conn, tablePath, Arrays.asList(row(1, "v1"), row(2, "v2"), row(3, "v3")), true); + + // in streaming execution mode, the bounded source stops at the latest offsets captured + // at startup and then the job finishes + try (CloseableIterator rowIter = + tEnv.executeSql( + "select * from bounded_latest_log_table " + + "/*+ OPTIONS('scan.bounded.mode' = 'latest-offset') */") + .collect()) { + assertThat(collectBatchRows(rowIter)) + .containsExactlyInAnyOrder("+I[1, v1]", "+I[2, v2]", "+I[3, v3]"); + } + } + + @Test + void testLogTableBoundedReadTimestampRange() throws Exception { + tEnv.executeSql("create table bounded_ts_range_log_table (a int, b varchar)"); + TablePath tablePath = TablePath.of(DEFAULT_DB, "bounded_ts_range_log_table"); + + writeRows(conn, tablePath, Arrays.asList(row(1, "v1"), row(2, "v2")), true); + CLOCK.advanceTime(Duration.ofMillis(100)); + long startTimestamp = CLOCK.milliseconds(); + writeRows(conn, tablePath, Arrays.asList(row(3, "v3"), row(4, "v4")), true); + CLOCK.advanceTime(Duration.ofMillis(100)); + long stopTimestamp = CLOCK.milliseconds(); + writeRows(conn, tablePath, Arrays.asList(row(5, "v5"), row(6, "v6")), true); + + // replay only the records committed in the time range [startTimestamp, stopTimestamp) + String options = + String.format( + " /*+ OPTIONS('scan.startup.mode' = 'timestamp', " + + "'scan.startup.timestamp' = '%d', " + + "'scan.bounded.mode' = 'timestamp', " + + "'scan.bounded.timestamp' = '%d') */", + startTimestamp, stopTimestamp); + try (CloseableIterator rowIter = + tEnv.executeSql("select * from bounded_ts_range_log_table" + options).collect()) { + assertThat(collectBatchRows(rowIter)) + .containsExactlyInAnyOrder("+I[3, v3]", "+I[4, v4]"); + } + } + + @Test + void testPrimaryKeyTableChangelogBoundedRead() throws Exception { + tEnv.executeSql( + "create table bounded_pk_changelog_table (a int not null primary key not enforced," + + " b varchar) with ('bucket.num' = '1')"); + TablePath tablePath = TablePath.of(DEFAULT_DB, "bounded_pk_changelog_table"); + writeRows(conn, tablePath, Arrays.asList(row(1, "v1"), row(2, "v2")), false); + writeRows(conn, tablePath, Arrays.asList(row(1, "v11")), false); + + // read the changelog of the primary key table from the earliest offset up to the latest + // offsets captured at startup, then the job finishes + try (CloseableIterator rowIter = + tEnv.executeSql( + "select * from bounded_pk_changelog_table " + + "/*+ OPTIONS('scan.startup.mode' = 'earliest', " + + "'scan.bounded.mode' = 'latest-offset') */") + .collect()) { + assertThat(collectBatchRows(rowIter)) + .containsExactly("+I[1, v1]", "+I[2, v2]", "-U[1, v1]", "+U[1, v11]"); + } + } + + @Test + void testPrimaryKeyTableFullStartupBoundedReadThrows() { + tEnv.executeSql( + "create table bounded_pk_full_table " + + "(a int not null primary key not enforced, b varchar)"); + assertThatThrownBy( + () -> + tEnv.executeSql( + "select * from bounded_pk_full_table " + + "/*+ OPTIONS('scan.bounded.mode' = 'latest-offset') */")) + .hasStackTraceContaining( + "'scan.bounded.mode' is not supported for primary key tables in 'full' startup mode"); + } + @Test void testReadTimestampGreaterThanMaxTimestamp() throws Exception { tEnv.executeSql("create table timestamp_table (a int, b varchar) "); diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java index 647931ded36..313a7d79735 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java @@ -1254,6 +1254,81 @@ void testBatchModeWithTimestampStoppingOffsets() throws Throwable { } } + @Test + void testBoundedStreamingReadSignalsNoMoreSplits() throws Throwable { + int numSubtasks = 3; + createTable(DEFAULT_TABLE_PATH, DEFAULT_AUTO_PARTITIONED_LOG_TABLE_DESCRIPTOR); + ZooKeeperClient zooKeeperClient = FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(); + Map partitionNameByIds = + waitUntilPartitions(zooKeeperClient, DEFAULT_TABLE_PATH); + List rows = new ArrayList<>(); + for (String partitionName : partitionNameByIds.values()) { + for (int i = 0; i < 5; i++) { + rows.add(row(i, partitionName)); + } + } + writeRows(conn, DEFAULT_TABLE_PATH, rows, true); + + try (MockSplitEnumeratorContext context = + new MockSplitEnumeratorContext<>(numSubtasks)) { + // a streaming read with user-supplied stopping offsets is a bounded read + FlinkSourceEnumerator enumerator = + new FlinkSourceEnumerator( + DEFAULT_TABLE_PATH, + flussConf, + false, + true, + context, + OffsetsInitializer.earliest(), + OffsetsInitializer.latest(), + DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, + FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE.defaultValue(), + true, + null, + null, + LeaseContext.DEFAULT, + false); + + enumerator.start(); + for (int i = 0; i < numSubtasks; i++) { + registerReader(context, enumerator, i); + } + + // a bounded streaming read only performs a one-time partition discovery, even though + // the partition discovery interval is positive + assertThat(context.getPeriodicCallables()).isEmpty(); + // discover the partitions and then initialize the splits + context.runNextOneTimeCallable(); + context.runNextOneTimeCallable(); + + List assignedSplits = + getReadersAssignments(context).values().stream() + .flatMap(List::stream) + .collect(Collectors.toList()); + assertThat(assignedSplits) + .hasSize(partitionNameByIds.size() * DEFAULT_BUCKET_NUM) + .allSatisfy( + split -> { + LogSplit logSplit = split.asLogSplit(); + assertThat(logSplit.getStartingOffset()).isEqualTo(EARLIEST_OFFSET); + assertThat(logSplit.getStoppingOffset()).isPresent(); + }); + // the stopping offsets are the latest offsets captured at startup, which sum up to + // the total number of written records + long totalStoppingOffset = + assignedSplits.stream() + .mapToLong(split -> split.asLogSplit().getStoppingOffset().get()) + .sum(); + assertThat(totalStoppingOffset).isEqualTo(rows.size()); + + // all splits are added at once for a bounded read, so all readers have been signaled + // that no more splits will come, which lets the job finish eventually + for (int i = 0; i < numSubtasks; i++) { + assertThat(context.hasNoMoreSplits(i)).isTrue(); + } + } + } + @Test void testGetSplitOwner() throws Exception { int numSubtasks = 3; diff --git a/website/docs/engine-flink/datastream.mdx b/website/docs/engine-flink/datastream.mdx index 483f4b73037..ef63c3bfd77 100644 --- a/website/docs/engine-flink/datastream.mdx +++ b/website/docs/engine-flink/datastream.mdx @@ -95,6 +95,7 @@ The `FlussSourceBuilder` provides several methods for configuring the source con * **setProjectedFields(String... projectedFieldNames):** Sets the fields to project from the table (if not specified, all fields are included) * **setScanPartitionDiscoveryIntervalMs(long intervalMs):** Sets the interval for discovering new partitions (default: from configuration) * **setStartingOffsets(OffsetsInitializer initializer):** Sets the strategy for determining starting offsets (default: `OffsetsInitializer.full()`) +* **setBounded(OffsetsInitializer stoppingOffsetsInitializer):** Makes the source bounded: it stops once it reaches the given stopping offsets and then finishes, even in streaming execution mode (see [Bounded Reads](#bounded-reads)) * **setFlussConfig(Configuration flussConf):** Sets custom Fluss configuration properties ### Offset Initializers @@ -128,6 +129,28 @@ FlussSource source = FlussSource.builder() .build(); ``` +### Bounded Reads +By default, the source is unbounded and keeps reading new records. Calling `setBounded(OffsetsInitializer)` makes the source stop once it reaches the given stopping offsets and then finish, even in streaming execution mode (a bounded streaming read). Typical usages are replaying a bounded time range of the log, backfilling and archiving. Supported stopping offsets initializers are `OffsetsInitializer.latest()` and `OffsetsInitializer.timestamp(long)`. + +Bounded reads are supported for log tables and the changelog of primary key tables (earliest/latest/timestamp starting offsets), but not for the full startup mode of primary key tables or the datalake union read. + +**Example:** +```java +// Replay a bounded time range of the log: from one hour ago up to now +FlussSource source = FlussSource.builder() + .setStartingOffsets(OffsetsInitializer.timestamp(System.currentTimeMillis() - 3600 * 1000)) + .setBounded(OffsetsInitializer.timestamp(System.currentTimeMillis())) + // other configuration... + .build(); + +// Read up to the latest offsets captured at startup and then finish +FlussSource source = FlussSource.builder() + .setStartingOffsets(OffsetsInitializer.earliest()) + .setBounded(OffsetsInitializer.latest()) + // other configuration... + .build(); +``` + ### Deserialization Schemas The `FlussDeserializationSchema` interface is used to convert Fluss records to your desired output type. Fluss provides some built-in implementations: diff --git a/website/docs/engine-flink/options.md b/website/docs/engine-flink/options.md index e1a0b5d818e..7a0c5a5dd6c 100644 --- a/website/docs/engine-flink/options.md +++ b/website/docs/engine-flink/options.md @@ -102,7 +102,7 @@ See more details about [ALTER TABLE ... SET](engine-flink/ddl.md#set-properties) |-----------------------------------------------|------------|-------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | scan.startup.mode | Enum | full | The scan startup mode enables you to specify the starting point for data consumption. Fluss currently supports the following `scan.startup.mode` options: `full` (default), earliest, latest, timestamp. See the [Start Reading Position](engine-flink/reads.md#start-reading-position) for more details. | | scan.startup.timestamp | Long | (None) | The timestamp to start reading the data from. This option is only valid when `scan.startup.mode` is set to `timestamp`. The format is 'milli-second-since-epoch' or `yyyy-MM-dd HH:mm:ss`, like `1678883047356` or `2023-12-09 23:09:12`. | -| scan.bounded.mode | Enum | unbounded | The scan bounded mode enables you to specify where the source stops reading. Fluss currently supports the following `scan.bounded.mode` options: `unbounded` (default), `latest-offset`, `timestamp`. Currently, bounded modes other than `unbounded` are only supported for Log Tables. See the [Stop Reading Position](engine-flink/reads.md#stop-reading-position) for more details. | +| scan.bounded.mode | Enum | unbounded | The scan bounded mode enables you to specify where the source stops reading. Fluss currently supports the following `scan.bounded.mode` options: `unbounded` (default), `latest-offset`, `timestamp`. Bounded modes other than `unbounded` make the source bounded even in streaming execution mode (a bounded streaming read), and are supported for Log Tables and the changelog of Primary Key Tables (`earliest`/`latest`/`timestamp` startup mode). See the [Stop Reading Position](engine-flink/reads.md#stop-reading-position) for more details. | | scan.bounded.timestamp | Long | (None) | The timestamp to stop reading the data at. This option is only valid when `scan.bounded.mode` is set to `timestamp`. Only records with a commit timestamp smaller than the given timestamp are read. The timestamp must not be in the future. The format is 'milli-second-since-epoch' or `yyyy-MM-dd HH:mm:ss`, like `1678883047356` or `2023-12-09 23:09:12`. | | scan.partition.discovery.interval | Duration | 1min | The time interval for the Fluss source to discover the new partitions for partitioned table while scanning. A non-positive value disables the partition discovery. The default value is 1 minute. Currently, since Fluss Admin#listPartitions(TablePath tablePath) requires a large number of requests to ZooKeeper in server, this option cannot be set too small, as a small value would cause frequent requests and increase server load. In the future, once list partitions is optimized, the default value of this parameter can be reduced. | | scan.kv.snapshot.lease.id | String | UUID | The lease ID used to protect acquired KV snapshots from deletion. If specified, the snapshots will be retained until either the consumer finishes processing all of them or the lease duration expires. By default, this value is set to a randomly generated UUID string if not explicitly provided. | diff --git a/website/docs/engine-flink/reads.md b/website/docs/engine-flink/reads.md index 9940bd567d9..cee06ee237a 100644 --- a/website/docs/engine-flink/reads.md +++ b/website/docs/engine-flink/reads.md @@ -398,7 +398,11 @@ SELECT * FROM log_table ### Stop Reading Position -The config option `scan.bounded.mode` enables you to specify where the source stops reading. It is currently only supported for Log Tables. Fluss supports the following `scan.bounded.mode` options: +The config option `scan.bounded.mode` enables you to specify where the source stops reading. A bounded mode other than `unbounded` makes the source bounded even in streaming execution mode: the job finishes once the source reaches the stopping position (a bounded streaming read). Typical usages are replaying a bounded time range of the log, backfilling and archiving. + +It is supported for Log Tables, the changelog of Primary Key Tables (`earliest`, `latest` or `timestamp` startup mode) and the `$changelog`/`$binlog` virtual tables, but not for the `full` startup mode of Primary Key Tables (the snapshot reading phase has no bounded end) or the datalake union read. + +Fluss supports the following `scan.bounded.mode` options: - `unbounded` (default): In streaming execution mode, the source never stops. In batch execution mode, the source reads up to the latest log offsets captured when the source starts. - `latest-offset`: The source stops at the latest log offsets captured when the source starts. In batch execution mode, this behaves the same as `unbounded`. - `timestamp`: The source stops before the first record batch whose commit timestamp is greater than or equal to the specified time (defined by the configuration item `scan.bounded.timestamp`), i.e. only records with a commit timestamp smaller than the specified time are read. The specified time must not be in the future. @@ -410,7 +414,7 @@ SELECT * FROM log_table 'scan.bounded.timestamp' = '2023-12-09 23:09:12') */; ``` -The start and stop reading positions can be combined to read a time range of the log. +The start and stop reading positions can be combined to replay a time range of the log, even in streaming execution mode. ```sql title="Flink SQL" SELECT * FROM log_table /*+ OPTIONS('scan.startup.mode' = 'timestamp', @@ -419,6 +423,13 @@ SELECT * FROM log_table 'scan.bounded.timestamp' = '2023-12-10 00:00:00') */; ``` +The following SQL statement reads the changelog of a primary key table up to the latest log offsets captured when the source starts, and then finishes. +```sql title="Flink SQL" +SELECT * FROM pk_table +/*+ OPTIONS('scan.startup.mode' = 'earliest', +'scan.bounded.mode' = 'latest-offset') */; +``` + From c65265650629129be818e0239e5f79b25960c33a Mon Sep 17 00:00:00 2001 From: naivedogger Date: Mon, 10 Aug 2026 11:21:16 +0800 Subject: [PATCH 4/9] [flink] Fix bounded streaming source termination --- .../flink/source/FlussSourceBuilder.java | 12 +- .../enumerator/FlinkSourceEnumerator.java | 81 ++++--- .../FlussOnlyBatchSplitGenerator.java | 14 +- .../flink/source/FlussSourceBuilderTest.java | 19 ++ .../enumerator/FlinkSourceEnumeratorTest.java | 207 ++++++++++++++++++ 5 files changed, 297 insertions(+), 36 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java index c5a966a7546..9585df16333 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java @@ -20,8 +20,10 @@ import org.apache.fluss.client.Connection; import org.apache.fluss.client.ConnectionFactory; import org.apache.fluss.client.admin.Admin; +import org.apache.fluss.client.initializer.LatestOffsetsInitializer; import org.apache.fluss.client.initializer.OffsetsInitializer; import org.apache.fluss.client.initializer.SnapshotOffsetsInitializer; +import org.apache.fluss.client.initializer.TimestampOffsetsInitializer; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.flink.FlinkConnectorOptions; @@ -44,6 +46,7 @@ import java.util.concurrent.ExecutionException; import static org.apache.flink.util.Preconditions.checkNotNull; +import static org.apache.fluss.utils.Preconditions.checkArgument; /** * Builder class for creating {@link FlussSource} instances. @@ -198,9 +201,16 @@ public FlussSourceBuilder setBounded() { * @return this builder */ public FlussSourceBuilder setBounded(OffsetsInitializer stoppingOffsetsInitializer) { - this.stoppingOffsetsInitializer = + OffsetsInitializer checkedStoppingOffsetsInitializer = checkNotNull( stoppingOffsetsInitializer, "stoppingOffsetsInitializer must not be null"); + checkArgument( + checkedStoppingOffsetsInitializer instanceof LatestOffsetsInitializer + || checkedStoppingOffsetsInitializer instanceof TimestampOffsetsInitializer, + "Only OffsetsInitializer.latest() and OffsetsInitializer.timestamp(...) are " + + "supported as stopping offsets, but was %s.", + checkedStoppingOffsetsInitializer.getClass().getName()); + this.stoppingOffsetsInitializer = checkedStoppingOffsetsInitializer; return this; } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java index 8659cc2e470..6d31502f480 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java @@ -619,6 +619,7 @@ public FlinkSourceEnumerator( this.splitPerAssignmentBatchSize = splitPerAssignmentBatchSize; this.initialDiscoveryFinished = initialDiscoveryFinished; this.unassignedSplits = new ArrayList<>(unassignedSplits); + this.noMoreNewSplits = initialDiscoveryFinished && hasFiniteSplitSet(); } @Override @@ -679,7 +680,7 @@ public void start() { } } - if (scanPartitionDiscoveryIntervalMs > 0 && !bounded) { + if (isPeriodicPartitionDiscoveryEnabled()) { // should do partition discovery LOG.info( "Starting the FlussSourceEnumerator for table {} " @@ -765,7 +766,7 @@ private void startInStreamModeForNonPartitionedTable() { + "{} splits already restored from checkpoint state.", tablePath, pendingSplitAssignment.values().stream().mapToInt(List::size).sum()); - initialDiscoveryFinished = true; + markInitialDiscoveryFinished(); return; } @@ -854,8 +855,13 @@ private void checkPartitionChanges(Set partitionInfos, Throwable return; } if (t != null) { - LOG.error("Failed to list partitions for {}", tablePath, t); - return; + if (isPeriodicPartitionDiscoveryEnabled()) { + LOG.warn("Failed to list partitions for {}. Will retry.", tablePath, t); + return; + } + throw new FlinkRuntimeException( + String.format("Failed to list partitions for %s", tablePath), + ExceptionUtils.stripCompletionException(t)); } LOG.debug( @@ -871,7 +877,10 @@ private void checkPartitionChanges(Set partitionInfos, Throwable // to track), mark initial discovery as finished immediately since there are // no splits that need to be persisted in state first. if (!initialDiscoveryFinished) { - initialDiscoveryFinished = true; + markInitialDiscoveryFinished(); + } + if (noMoreNewSplits) { + assignPendingSplits(context.registeredReaders().keySet()); } LOG.debug("No partition changes detected for table {}", tablePath); return; @@ -1181,15 +1190,28 @@ private List getLogSplit( Map stoppingOffsets = stoppingOffsetsInitializer.getBucketOffsets( partitionName, bucketsNeedInitOffset, bucketOffsetsRetriever); - startingOffsets.forEach( - (bucketId, startingOffset) -> - splits.add( - new LogSplit( - new TableBucket( - tableInfo.getTableId(), partitionId, bucketId), - partitionName, - startingOffset, - stoppingOffsets.get(bucketId)))); + for (Integer bucketId : bucketsNeedInitOffset) { + Long startingOffset = startingOffsets.get(bucketId); + Long stoppingOffset = stoppingOffsets.get(bucketId); + checkState( + startingOffset != null, + "Starting offset should be present for bucket %s.", + bucketId); + checkState( + stoppingOffset != null + && (stoppingOffset == LogSplit.NO_STOPPING_OFFSET + || stoppingOffset >= 0), + "Stopping offset for bucket %s must be non-negative or the no-stopping " + + "sentinel, but was %s.", + bucketId, + stoppingOffset); + splits.add( + new LogSplit( + new TableBucket(tableInfo.getTableId(), partitionId, bucketId), + partitionName, + startingOffset, + stoppingOffset)); + } } return splits; } @@ -1299,12 +1321,25 @@ private static boolean shouldRemoveForDroppedPartition( return removedPartitionsMap.containsKey(split.getTableBucket().getPartitionId()); } + private boolean isPeriodicPartitionDiscoveryEnabled() { + return isPartitioned && streaming && !bounded && scanPartitionDiscoveryIntervalMs > 0; + } + + private boolean hasFiniteSplitSet() { + return !isPeriodicPartitionDiscoveryEnabled(); + } + + private void markInitialDiscoveryFinished() { + initialDiscoveryFinished = true; + noMoreNewSplits = hasFiniteSplitSet(); + } + private void handleSplitsAdd(List splits, Throwable t) { if (t != null) { - if (isPartitioned && streaming && !bounded && scanPartitionDiscoveryIntervalMs > 0) { + if (isPeriodicPartitionDiscoveryEnabled()) { // it means continuously read new partition splits, not throw exception, temporally // warn it to avoid job fail. TODO: fix me in #288 - LOG.warn("Failed to list splits for {}.", tablePath, t); + LOG.warn("Failed to list splits for {}. Will retry.", tablePath, t); return; } else { throw new FlinkRuntimeException( @@ -1313,7 +1348,7 @@ private void handleSplitsAdd(List splits, Throwable t) { } } - initialDiscoveryFinished = true; + markInitialDiscoveryFinished(); if (pendingHybridLakeFlussSplits != null) { // removed from the pendingHybridLakeFlussSplits since this split already be moved to // unassignedSplits @@ -1332,18 +1367,6 @@ private void handleSplitsAdd(List splits, Throwable t) { ? "null" : pendingHybridLakeFlussSplits.size()); - if (isPartitioned) { - if (bounded || scanPartitionDiscoveryIntervalMs <= 0) { - // For a bounded read (batch execution mode or a bounded streaming read) or when - // partition discovery is disabled, splits are only added once, so readers can be - // signaled that no more splits will come and finish eventually. - noMoreNewSplits = true; - } - } else { - // if not partitioned, only will add splits only once, - // so, noMoreNewPartitionSplits should be set to true - noMoreNewSplits = true; - } doHandleSplitsAdd(splits); } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlussOnlyBatchSplitGenerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlussOnlyBatchSplitGenerator.java index 4036f047a52..8e29d729bfb 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlussOnlyBatchSplitGenerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlussOnlyBatchSplitGenerator.java @@ -146,9 +146,10 @@ private List getBatchSnapshotAndLogSplits( Long logStoppingOffset = stoppingOffsets.get(bucketId); checkState( - logStoppingOffset != null, - "Stopping offset should be present for bucket %s.", - bucketId); + logStoppingOffset != null && logStoppingOffset >= 0, + "Stopping offset for bucket %s must be non-negative, but was %s.", + bucketId, + logStoppingOffset); splits.add( new HybridSnapshotLogSplit( tableBucket, @@ -190,9 +191,10 @@ private List getLogSplits( "Starting offset should be present for bucket %s.", bucketId); checkState( - stoppingOffset != null, - "Stopping offset should be present for bucket %s.", - bucketId); + stoppingOffset != null && stoppingOffset >= 0, + "Stopping offset for bucket %s must be non-negative, but was %s.", + bucketId, + stoppingOffset); splits.add( new LogSplit( new TableBucket(tableInfo.getTableId(), partitionId, bucketId), diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlussSourceBuilderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlussSourceBuilderTest.java index 0f54fc04b4d..e31026de0c1 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlussSourceBuilderTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlussSourceBuilderTest.java @@ -62,6 +62,25 @@ public void testBuildWithValidConfiguration() { assertThat(source).isNotNull(); } + @Test + public void testRejectUnsupportedStoppingOffsetsInitializer() { + FlussSourceBuilder builder = FlussSource.builder(); + + assertThatThrownBy(() -> builder.setBounded(OffsetsInitializer.earliest())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining( + "Only OffsetsInitializer.latest() and " + + "OffsetsInitializer.timestamp(...) are supported"); + assertThatThrownBy(() -> builder.setBounded(OffsetsInitializer.full())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining( + "Only OffsetsInitializer.latest() and " + + "OffsetsInitializer.timestamp(...) are supported"); + + assertThat(builder.setBounded(OffsetsInitializer.latest())).isSameAs(builder); + assertThat(builder.setBounded(OffsetsInitializer.timestamp(1L))).isSameAs(builder); + } + @Test public void testMissingBootstrapServers() { // Given diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java index 313a7d79735..8eae3a8ab1e 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java @@ -64,6 +64,7 @@ import org.apache.flink.api.connector.source.SplitsAssignment; import org.apache.flink.api.connector.source.mocks.MockSplitEnumeratorContext; import org.apache.flink.table.data.RowData; +import org.apache.flink.util.FlinkRuntimeException; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -82,6 +83,8 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.concurrent.Callable; +import java.util.function.BiConsumer; import java.util.stream.Collectors; import static org.apache.fluss.client.table.scanner.log.LogScanner.EARLIEST_OFFSET; @@ -1135,6 +1138,12 @@ void testBatchModeNonLakePartitionedLogTable() throws Throwable { ZooKeeperClient zooKeeperClient = FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(); Map partitionNameByIds = waitUntilPartitions(zooKeeperClient, DEFAULT_TABLE_PATH); + partitionNameByIds + .keySet() + .forEach( + partitionId -> + FLUSS_CLUSTER_EXTENSION.waitUntilTablePartitionReady( + tableId, partitionId)); try (MockSplitEnumeratorContext context = new MockSplitEnumeratorContext<>(numSubtasks)) { @@ -1329,6 +1338,204 @@ void testBoundedStreamingReadSignalsNoMoreSplits() throws Throwable { } } + @Test + void testBoundedStreamingRestoreSignalsNoMoreSplits() throws Throwable { + int numSubtasks = 3; + createTable(DEFAULT_TABLE_PATH, DEFAULT_AUTO_PARTITIONED_LOG_TABLE_DESCRIPTOR); + Map partitions = + waitUntilPartitions( + FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(), DEFAULT_TABLE_PATH); + + try (MockSplitEnumeratorContext context = + new MockSplitEnumeratorContext<>(numSubtasks); + MockWorkExecutor workExecutor = new MockWorkExecutor(context); + FlinkSourceEnumerator enumerator = + new FlinkSourceEnumerator( + DEFAULT_TABLE_PATH, + flussConf, + false, + true, + context, + Collections.emptySet(), + partitions, + null, + OffsetsInitializer.earliest(), + OffsetsInitializer.latest(), + DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, + FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE + .defaultValue(), + true, + null, + null, + workExecutor, + LeaseContext.DEFAULT, + true, + true, + Collections.emptyList())) { + enumerator.start(); + for (int i = 0; i < numSubtasks; i++) { + registerReader(context, enumerator, i); + assertThat(context.hasNoMoreSplits(i)).isTrue(); + } + + // The restored partition set has not changed. The one-time discovery must not clear + // the restored terminal state. + workExecutor.runNextOneTimeCallable(); + for (int i = 0; i < numSubtasks; i++) { + assertThat(context.hasNoMoreSplits(i)).isTrue(); + } + } + } + + @Test + void testBoundedNonPartitionedRestoreSignalsNoMoreSplits() throws Exception { + long tableId = createTable(DEFAULT_TABLE_PATH, DEFAULT_LOG_TABLE_DESCRIPTOR); + LogSplit restoredSplit = new LogSplit(new TableBucket(tableId, 0), null, 0L, 0L); + + try (MockSplitEnumeratorContext context = + new MockSplitEnumeratorContext<>(1); + MockWorkExecutor workExecutor = new MockWorkExecutor(context); + FlinkSourceEnumerator enumerator = + new FlinkSourceEnumerator( + DEFAULT_TABLE_PATH, + flussConf, + false, + false, + context, + Collections.emptySet(), + Collections.emptyMap(), + null, + OffsetsInitializer.earliest(), + OffsetsInitializer.latest(), + DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, + FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE + .defaultValue(), + true, + null, + null, + workExecutor, + LeaseContext.DEFAULT, + true, + true, + Collections.singletonList(restoredSplit))) { + enumerator.start(); + registerReader(context, enumerator, 0); + + assertThat(getReadersAssignments(context).get(0)).containsExactly(restoredSplit); + assertThat(context.hasNoMoreSplits(0)).isTrue(); + } + } + + @Test + void testBoundedStreamingReadWithNoPartitionsSignalsNoMoreSplits() throws Throwable { + TablePath tablePath = TablePath.of(DEFAULT_DB, "bounded-empty-partition-table"); + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("partition_col", DataTypes.STRING()) + .build(); + TableDescriptor tableDescriptor = + TableDescriptor.builder() + .schema(schema) + .partitionedBy("partition_col") + .distributedBy(DEFAULT_BUCKET_NUM, "id") + .build(); + createTable(tablePath, tableDescriptor); + + try (MockSplitEnumeratorContext context = + new MockSplitEnumeratorContext<>(2); + FlinkSourceEnumerator enumerator = + new FlinkSourceEnumerator( + tablePath, + flussConf, + false, + true, + context, + OffsetsInitializer.earliest(), + OffsetsInitializer.latest(), + DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, + FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE + .defaultValue(), + true, + null, + null, + LeaseContext.DEFAULT, + false)) { + enumerator.start(); + registerReader(context, enumerator, 0); + registerReader(context, enumerator, 1); + + context.runNextOneTimeCallable(); + + assertThat(context.getSplitsAssignmentSequence()).isEmpty(); + assertThat(context.hasNoMoreSplits(0)).isTrue(); + assertThat(context.hasNoMoreSplits(1)).isTrue(); + } + } + + @Test + void testBoundedPartitionDiscoveryFailureFailsJob() throws Exception { + TablePath tablePath = TablePath.of(DEFAULT_DB, "bounded-partition-discovery-failure"); + Schema schema = + Schema.newBuilder() + .column("id", DataTypes.INT()) + .column("partition_col", DataTypes.STRING()) + .build(); + TableDescriptor tableDescriptor = + TableDescriptor.builder() + .schema(schema) + .partitionedBy("partition_col") + .distributedBy(DEFAULT_BUCKET_NUM, "id") + .build(); + createTable(tablePath, tableDescriptor); + + try (MockSplitEnumeratorContext context = + new MockSplitEnumeratorContext<>(1)) { + WorkerExecutor failingWorkerExecutor = + new WorkerExecutor(context) { + @Override + public void callAsync( + Callable callable, BiConsumer handler) { + context.callAsync( + () -> { + throw new FlinkRuntimeException( + "expected partition discovery failure"); + }, + handler); + } + }; + + try (FlinkSourceEnumerator enumerator = + new FlinkSourceEnumerator( + tablePath, + flussConf, + false, + true, + context, + Collections.emptySet(), + Collections.emptyMap(), + null, + OffsetsInitializer.earliest(), + OffsetsInitializer.latest(), + DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, + FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE.defaultValue(), + true, + null, + null, + failingWorkerExecutor, + LeaseContext.DEFAULT, + false, + false, + Collections.emptyList())) { + enumerator.start(); + + assertThatThrownBy(context::runNextOneTimeCallable) + .isInstanceOf(FlinkRuntimeException.class) + .hasMessage("Failed to list partitions for " + tablePath); + } + } + } + @Test void testGetSplitOwner() throws Exception { int numSubtasks = 3; From 588de9cd993bdb2e1d9cb7d29b87658cb7c90250 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Tue, 18 Aug 2026 12:00:48 +0800 Subject: [PATCH 5/9] [flink] Skip partition discovery after bounded restore --- .../enumerator/FlinkSourceEnumerator.java | 8 +++++++- .../enumerator/FlinkSourceEnumeratorTest.java | 18 +++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java index 6d31502f480..dac0d3a0953 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java @@ -665,6 +665,7 @@ public void start() { addSplitToPendingAssignments(unassignedSplits); } + boolean restoredBoundedPartitionSet = bounded && initialDiscoveryFinished; if (isPartitioned) { if (streaming) { if (lakeSource != null) { @@ -680,7 +681,12 @@ public void start() { } } - if (isPeriodicPartitionDiscoveryEnabled()) { + if (restoredBoundedPartitionSet) { + LOG.info( + "Skipping partition discovery for restored bounded source of table {}.", + tablePath); + assignPendingSplits(context.registeredReaders().keySet()); + } else if (isPeriodicPartitionDiscoveryEnabled()) { // should do partition discovery LOG.info( "Starting the FlussSourceEnumerator for table {} " diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java index 8eae3a8ab1e..bf76eeb43ff 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java @@ -1339,12 +1339,17 @@ void testBoundedStreamingReadSignalsNoMoreSplits() throws Throwable { } @Test - void testBoundedStreamingRestoreSignalsNoMoreSplits() throws Throwable { + void testBoundedStreamingRestoreSkipsPartitionDiscovery() throws Throwable { int numSubtasks = 3; createTable(DEFAULT_TABLE_PATH, DEFAULT_AUTO_PARTITIONED_LOG_TABLE_DESCRIPTOR); - Map partitions = - waitUntilPartitions( - FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(), DEFAULT_TABLE_PATH); + ZooKeeperClient zooKeeperClient = FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(); + Map partitions = waitUntilPartitions(zooKeeperClient, DEFAULT_TABLE_PATH); + + // Simulate a partition created after the checkpoint represented by the restored state. + createPartitions( + zooKeeperClient, + DEFAULT_TABLE_PATH, + Collections.singletonList("created-after-checkpoint")); try (MockSplitEnumeratorContext context = new MockSplitEnumeratorContext<>(numSubtasks); @@ -1378,9 +1383,8 @@ void testBoundedStreamingRestoreSignalsNoMoreSplits() throws Throwable { assertThat(context.hasNoMoreSplits(i)).isTrue(); } - // The restored partition set has not changed. The one-time discovery must not clear - // the restored terminal state. - workExecutor.runNextOneTimeCallable(); + assertThat(workExecutor.getOneTimeCallables()).isEmpty(); + assertThat(context.getSplitsAssignmentSequence()).isEmpty(); for (int i = 0; i < numSubtasks; i++) { assertThat(context.hasNoMoreSplits(i)).isTrue(); } From 4db45b4d4074c2f8fc8ef417c97ecc9920258646 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Tue, 18 Aug 2026 13:57:35 +0800 Subject: [PATCH 6/9] [flink] Keep bounded source changes scoped --- .../FlussOnlyBatchSplitGenerator.java | 14 +++++------ .../source/reader/FlinkSourceSplitReader.java | 24 +++++++------------ .../reader/FlinkSourceSplitReaderTest.java | 15 +++++------- 3 files changed, 21 insertions(+), 32 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlussOnlyBatchSplitGenerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlussOnlyBatchSplitGenerator.java index 8e29d729bfb..4036f047a52 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlussOnlyBatchSplitGenerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlussOnlyBatchSplitGenerator.java @@ -146,10 +146,9 @@ private List getBatchSnapshotAndLogSplits( Long logStoppingOffset = stoppingOffsets.get(bucketId); checkState( - logStoppingOffset != null && logStoppingOffset >= 0, - "Stopping offset for bucket %s must be non-negative, but was %s.", - bucketId, - logStoppingOffset); + logStoppingOffset != null, + "Stopping offset should be present for bucket %s.", + bucketId); splits.add( new HybridSnapshotLogSplit( tableBucket, @@ -191,10 +190,9 @@ private List getLogSplits( "Starting offset should be present for bucket %s.", bucketId); checkState( - stoppingOffset != null && stoppingOffset >= 0, - "Stopping offset for bucket %s must be non-negative, but was %s.", - bucketId, - stoppingOffset); + stoppingOffset != null, + "Stopping offset should be present for bucket %s.", + bucketId); splits.add( new LogSplit( new TableBucket(tableInfo.getTableId(), partitionId, bucketId), diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java index 6c52f695ca6..50e2143962c 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java @@ -258,7 +258,7 @@ private void subscribeLog(SourceSplitBase split, long startingOffset) { Optional stoppingOffsetOpt = logSplit.getStoppingOffset(); if (stoppingOffsetOpt.isPresent()) { Long stoppingOffset = stoppingOffsetOpt.get(); - if (stoppingOffset == 0 || startingOffset >= stoppingOffset) { + if (startingOffset >= stoppingOffset) { // is empty log splits as no log record can be fetched emptyLogSplits.add(split.splitId()); isEmptyLogSplit = true; @@ -468,25 +468,19 @@ private FlinkRecordsWithSplitIds forLogRecords(ScanRecords scanRecords) { splitIdByTableBucket.put(scanBucket, splitId); tableScanBuckets.add(scanBucket); List bucketScanRecords = scanRecords.records(scanBucket); - ScanRecord lastRecord = null; if (!bucketScanRecords.isEmpty()) { - lastRecord = bucketScanRecords.get(bucketScanRecords.size() - 1); + final ScanRecord lastRecord = bucketScanRecords.get(bucketScanRecords.size() - 1); // We keep the maximum message timestamp in the fetch for calculating lags maxConsumerRecordTimestampInFetch = Math.max(maxConsumerRecordTimestampInFetch, lastRecord.timestamp()); - } - Long consumedUpToOffset = scanRecords.consumedUpToOffset(scanBucket); - boolean reachedStoppingOffset = - consumedUpToOffset != null - ? consumedUpToOffset >= stoppingOffset - : lastRecord != null && lastRecord.logOffset() >= stoppingOffset - 1; - // After consuming up to the stopping offset, the split reader should not continue - // fetching because the record at the stopping offset is outside this split and may not - // exist. This also handles batches whose records are all filtered out. - if (reachedStoppingOffset) { - stoppingOffsets.put(scanBucket, stoppingOffset); - finishedSplits.add(splitId); + // After processing a record with offset of "stoppingOffset - 1", the split reader + // should not continue fetching because the record with stoppingOffset may not + // exist. Keep polling will just block forever + if (lastRecord.logOffset() >= stoppingOffset - 1) { + stoppingOffsets.put(scanBucket, stoppingOffset); + finishedSplits.add(splitId); + } } splitRecords.put(splitId, toRecordAndPos(bucketScanRecords.iterator())); } diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java index 19f2831ad63..cc71ddb763f 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java @@ -387,15 +387,13 @@ void testSubscribeEmptySplits() throws Exception { long tableId = createTable( tablePath, - TableDescriptor.builder().schema(schema).distributedBy(4).build()); + TableDescriptor.builder().schema(schema).distributedBy(3).build()); - // create three bounded empty splits and one unbounded split + // create two empty splits with log start offset equal to end offset LogSplit split1 = new LogSplit(new TableBucket(tableId, 0), null, 0, 0); LogSplit split2 = new LogSplit(new TableBucket(tableId, 1), null, 0, 0); - LogSplit split3 = new LogSplit(new TableBucket(tableId, 2), null, EARLIEST_OFFSET, 0); - LogSplit split4 = new LogSplit(new TableBucket(tableId, 3), null, EARLIEST_OFFSET); - - List subscribeSplits = Arrays.asList(split1, split2, split3, split4); + LogSplit split3 = new LogSplit(new TableBucket(tableId, 2), null, EARLIEST_OFFSET); + List subscribeSplits = Arrays.asList(split1, split2, split3); try (FlinkSourceSplitReader splitReader = createSplitReader(tablePath, schema.getRowType())) { @@ -403,10 +401,9 @@ void testSubscribeEmptySplits() throws Exception { // fetch records RecordsWithSplitIds records = splitReader.fetch(); - // finished splits should be split1, split2, split3 + // finished splits should be split1,split2 assertThat(records.finishedSplits()) - .containsExactlyInAnyOrder( - split1.splitId(), split2.splitId(), split3.splitId()); + .containsExactlyInAnyOrder(split1.splitId(), split2.splitId()); } } From a44926af607f991c853fd39f9b948b95436f47ba Mon Sep 17 00:00:00 2001 From: naivedogger Date: Tue, 18 Aug 2026 15:58:25 +0800 Subject: [PATCH 7/9] [flink] Refine bounded streaming source changes --- .../enumerator/FlinkSourceEnumerator.java | 49 +++-- .../utils/FlinkConnectorOptionsUtils.java | 2 +- .../source/BinlogVirtualTableITCase.java | 6 +- .../source/ChangelogVirtualTableITCase.java | 6 +- .../flink/source/FlussSourceBuilderTest.java | 18 +- .../enumerator/FlinkSourceEnumeratorTest.java | 196 +++++++----------- .../reader/FlinkSourceSplitReaderTest.java | 49 ----- .../split/SourceSplitSerializerTest.java | 8 - website/docs/engine-flink/datastream.mdx | 4 +- 9 files changed, 122 insertions(+), 216 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java index dac0d3a0953..ffb85026798 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java @@ -619,7 +619,7 @@ public FlinkSourceEnumerator( this.splitPerAssignmentBatchSize = splitPerAssignmentBatchSize; this.initialDiscoveryFinished = initialDiscoveryFinished; this.unassignedSplits = new ArrayList<>(unassignedSplits); - this.noMoreNewSplits = initialDiscoveryFinished && hasFiniteSplitSet(); + this.noMoreNewSplits = initialDiscoveryFinished && isBoundedStreamingRead(); } @Override @@ -665,7 +665,7 @@ public void start() { addSplitToPendingAssignments(unassignedSplits); } - boolean restoredBoundedPartitionSet = bounded && initialDiscoveryFinished; + boolean restoredBoundedPartitionSet = isBoundedStreamingRead() && initialDiscoveryFinished; if (isPartitioned) { if (streaming) { if (lakeSource != null) { @@ -772,7 +772,10 @@ private void startInStreamModeForNonPartitionedTable() { + "{} splits already restored from checkpoint state.", tablePath, pendingSplitAssignment.values().stream().mapToInt(List::size).sum()); - markInitialDiscoveryFinished(); + initialDiscoveryFinished = true; + if (isBoundedStreamingRead()) { + noMoreNewSplits = true; + } return; } @@ -861,13 +864,13 @@ private void checkPartitionChanges(Set partitionInfos, Throwable return; } if (t != null) { - if (isPeriodicPartitionDiscoveryEnabled()) { - LOG.warn("Failed to list partitions for {}. Will retry.", tablePath, t); - return; + if (isBoundedStreamingRead()) { + throw new FlinkRuntimeException( + String.format("Failed to list partitions for %s", tablePath), + ExceptionUtils.stripCompletionException(t)); } - throw new FlinkRuntimeException( - String.format("Failed to list partitions for %s", tablePath), - ExceptionUtils.stripCompletionException(t)); + LOG.error("Failed to list partitions for {}", tablePath, t); + return; } LOG.debug( @@ -883,9 +886,10 @@ private void checkPartitionChanges(Set partitionInfos, Throwable // to track), mark initial discovery as finished immediately since there are // no splits that need to be persisted in state first. if (!initialDiscoveryFinished) { - markInitialDiscoveryFinished(); + initialDiscoveryFinished = true; } - if (noMoreNewSplits) { + if (isBoundedStreamingRead()) { + noMoreNewSplits = true; assignPendingSplits(context.registeredReaders().keySet()); } LOG.debug("No partition changes detected for table {}", tablePath); @@ -1331,13 +1335,8 @@ private boolean isPeriodicPartitionDiscoveryEnabled() { return isPartitioned && streaming && !bounded && scanPartitionDiscoveryIntervalMs > 0; } - private boolean hasFiniteSplitSet() { - return !isPeriodicPartitionDiscoveryEnabled(); - } - - private void markInitialDiscoveryFinished() { - initialDiscoveryFinished = true; - noMoreNewSplits = hasFiniteSplitSet(); + private boolean isBoundedStreamingRead() { + return streaming && bounded; } private void handleSplitsAdd(List splits, Throwable t) { @@ -1345,7 +1344,7 @@ private void handleSplitsAdd(List splits, Throwable t) { if (isPeriodicPartitionDiscoveryEnabled()) { // it means continuously read new partition splits, not throw exception, temporally // warn it to avoid job fail. TODO: fix me in #288 - LOG.warn("Failed to list splits for {}. Will retry.", tablePath, t); + LOG.warn("Failed to list splits for {}.", tablePath, t); return; } else { throw new FlinkRuntimeException( @@ -1354,7 +1353,7 @@ private void handleSplitsAdd(List splits, Throwable t) { } } - markInitialDiscoveryFinished(); + initialDiscoveryFinished = true; if (pendingHybridLakeFlussSplits != null) { // removed from the pendingHybridLakeFlussSplits since this split already be moved to // unassignedSplits @@ -1373,6 +1372,16 @@ private void handleSplitsAdd(List splits, Throwable t) { ? "null" : pendingHybridLakeFlussSplits.size()); + if (isPartitioned) { + if (!streaming || bounded || scanPartitionDiscoveryIntervalMs <= 0) { + // A batch or bounded streaming read has a finite split set. An unbounded streaming + // read also has a finite split set when periodic partition discovery is disabled. + noMoreNewSplits = true; + } + } else { + // A non-partitioned table only adds splits once. + noMoreNewSplits = true; + } doHandleSplitsAdd(splits); } diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java index 47d2faae4d9..361ddc1151e 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java @@ -289,7 +289,7 @@ public static class StartupOptions { public long startupTimestampMs; } - /** Fluss bounded options. * */ + /** Fluss bounded options. */ public static class BoundedOptions { public ScanBoundedMode boundedMode; public long boundedTimestampMs; diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/BinlogVirtualTableITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/BinlogVirtualTableITCase.java index da6aa14b610..75627809206 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/BinlogVirtualTableITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/BinlogVirtualTableITCase.java @@ -744,11 +744,7 @@ public void testBinlogBoundedRead() throws Exception { + "'scan.bounded.timestamp' = '%d') */", boundedTimestamp); try (CloseableIterator rowIter = tEnv.executeSql(query).collect()) { - List results = new ArrayList<>(); - while (rowIter.hasNext()) { - results.add(rowIter.next().toString()); - } - assertThat(results) + assertThat(collectBatchRows(rowIter)) .containsExactly( "+I[insert, null, null, 1, Item-1]", "+I[insert, null, null, 2, Item-2]", diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/ChangelogVirtualTableITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/ChangelogVirtualTableITCase.java index 91aa7d7b219..a3fe6b27b9f 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/ChangelogVirtualTableITCase.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/ChangelogVirtualTableITCase.java @@ -463,11 +463,7 @@ public void testChangelogBoundedRead() throws Exception { "SELECT _change_type, id, name FROM bounded_changelog_test$changelog " + "/*+ OPTIONS('scan.bounded.mode' = 'latest-offset') */"; try (CloseableIterator rowIter = tEnv.executeSql(query).collect()) { - List results = new ArrayList<>(); - while (rowIter.hasNext()) { - results.add(rowIter.next().toString()); - } - assertThat(results) + assertThat(collectBatchRows(rowIter)) .containsExactly( "+I[insert, 1, Alice]", "+I[insert, 2, Bob]", diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlussSourceBuilderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlussSourceBuilderTest.java index e31026de0c1..5c107cd9355 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlussSourceBuilderTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlussSourceBuilderTest.java @@ -26,6 +26,7 @@ import org.apache.fluss.types.RowType; import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.connector.source.Boundedness; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.function.Executable; @@ -60,6 +61,7 @@ public void testBuildWithValidConfiguration() { // Then assertThat(source).isNotNull(); + assertThat(source.getBoundedness()).isEqualTo(Boundedness.CONTINUOUS_UNBOUNDED); } @Test @@ -77,10 +79,24 @@ public void testRejectUnsupportedStoppingOffsetsInitializer() { "Only OffsetsInitializer.latest() and " + "OffsetsInitializer.timestamp(...) are supported"); - assertThat(builder.setBounded(OffsetsInitializer.latest())).isSameAs(builder); assertThat(builder.setBounded(OffsetsInitializer.timestamp(1L))).isSameAs(builder); } + @Test + public void testBuildBoundedStreamingSource() { + FlussSource source = + FlussSource.builder() + .setBootstrapServers(bootstrapServers) + .setDatabase(DEFAULT_DB) + .setTable(DEFAULT_TABLE_PATH.getTableName()) + .setStartingOffsets(OffsetsInitializer.earliest()) + .setBounded(OffsetsInitializer.latest()) + .setDeserializationSchema(new TestDeserializationSchema()) + .build(); + + assertThat(source.getBoundedness()).isEqualTo(Boundedness.BOUNDED); + } + @Test public void testMissingBootstrapServers() { // Given diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java index bf76eeb43ff..d35c4e554e4 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java @@ -1067,130 +1067,6 @@ void testDiscoverPartitionsPeriodically(boolean isPrimaryKeyTable) throws Throwa } } - @Test - void testBatchModeNonLakeLogTable() throws Throwable { - int numSubtasks = DEFAULT_BUCKET_NUM; - long tableId = createTable(DEFAULT_TABLE_PATH, DEFAULT_LOG_TABLE_DESCRIPTOR); - List rows = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - rows.add(row(i, "v" + i)); - } - writeRows(conn, DEFAULT_TABLE_PATH, rows, true); - - List bucketIds = new ArrayList<>(); - for (int bucket = 0; bucket < DEFAULT_BUCKET_NUM; bucket++) { - bucketIds.add(bucket); - } - Map expectedStoppingOffsets = - admin.listOffsets(DEFAULT_TABLE_PATH, bucketIds, new OffsetSpec.LatestSpec()) - .all() - .get(); - - try (MockSplitEnumeratorContext context = - new MockSplitEnumeratorContext<>(numSubtasks)) { - FlinkSourceEnumerator enumerator = - new FlinkSourceEnumerator( - DEFAULT_TABLE_PATH, - flussConf, - false, - false, - context, - OffsetsInitializer.earliest(), - DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, - false, - null, - null, - LeaseContext.DEFAULT, - false); - - enumerator.start(); - for (int i = 0; i < numSubtasks; i++) { - registerReader(context, enumerator, i); - } - context.runNextOneTimeCallable(); - - List assignedSplits = - getReadersAssignments(context).values().stream() - .flatMap(List::stream) - .collect(Collectors.toList()); - assertThat(assignedSplits).hasSize(DEFAULT_BUCKET_NUM); - assertThat(assignedSplits) - .allSatisfy( - split -> { - assertThat(split).isInstanceOf(LogSplit.class); - LogSplit logSplit = split.asLogSplit(); - assertThat(logSplit.getStartingOffset()).isEqualTo(EARLIEST_OFFSET); - assertThat(logSplit.getTableBucket().getTableId()) - .isEqualTo(tableId); - assertThat(logSplit.getStoppingOffset()) - .contains( - expectedStoppingOffsets.get( - logSplit.getTableBucket().getBucket())); - }); - } - } - - @Test - void testBatchModeNonLakePartitionedLogTable() throws Throwable { - int numSubtasks = DEFAULT_BUCKET_NUM; - long tableId = - createTable(DEFAULT_TABLE_PATH, DEFAULT_AUTO_PARTITIONED_LOG_TABLE_DESCRIPTOR); - ZooKeeperClient zooKeeperClient = FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(); - Map partitionNameByIds = - waitUntilPartitions(zooKeeperClient, DEFAULT_TABLE_PATH); - partitionNameByIds - .keySet() - .forEach( - partitionId -> - FLUSS_CLUSTER_EXTENSION.waitUntilTablePartitionReady( - tableId, partitionId)); - - try (MockSplitEnumeratorContext context = - new MockSplitEnumeratorContext<>(numSubtasks)) { - FlinkSourceEnumerator enumerator = - new FlinkSourceEnumerator( - DEFAULT_TABLE_PATH, - flussConf, - false, - true, - context, - OffsetsInitializer.earliest(), - DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, - false, - null, - null, - LeaseContext.DEFAULT, - false); - - enumerator.start(); - for (int i = 0; i < numSubtasks; i++) { - registerReader(context, enumerator, i); - } - context.runNextOneTimeCallable(); - - List assignedSplits = - getReadersAssignments(context).values().stream() - .flatMap(List::stream) - .collect(Collectors.toList()); - assertThat(assignedSplits).hasSize(partitionNameByIds.size() * DEFAULT_BUCKET_NUM); - Set assignedPartitionNames = new HashSet<>(); - assertThat(assignedSplits) - .allSatisfy( - split -> { - assertThat(split).isInstanceOf(LogSplit.class); - LogSplit logSplit = split.asLogSplit(); - assertThat(logSplit.getTableBucket().getTableId()) - .isEqualTo(tableId); - assertThat(logSplit.getStartingOffset()).isEqualTo(EARLIEST_OFFSET); - // the partitions are empty, so the captured latest offset is 0 - assertThat(logSplit.getStoppingOffset()).contains(0L); - assignedPartitionNames.add(logSplit.getPartitionName()); - }); - assertThat(assignedPartitionNames) - .containsExactlyInAnyOrderElementsOf(partitionNameByIds.values()); - } - } - @Test void testBatchModeWithTimestampStoppingOffsets() throws Throwable { int numSubtasks = DEFAULT_BUCKET_NUM; @@ -1380,7 +1256,6 @@ void testBoundedStreamingRestoreSkipsPartitionDiscovery() throws Throwable { enumerator.start(); for (int i = 0; i < numSubtasks; i++) { registerReader(context, enumerator, i); - assertThat(context.hasNoMoreSplits(i)).isTrue(); } assertThat(workExecutor.getOneTimeCallables()).isEmpty(); @@ -1391,6 +1266,77 @@ void testBoundedStreamingRestoreSkipsPartitionDiscovery() throws Throwable { } } + @Test + void testUnboundedRestoreWithDisabledPartitionDiscoveryDiscoversBeforeFinishing() + throws Throwable { + int numSubtasks = 3; + createTable(DEFAULT_TABLE_PATH, DEFAULT_AUTO_PARTITIONED_LOG_TABLE_DESCRIPTOR); + ZooKeeperClient zooKeeperClient = FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(); + Map restoredPartitions = + waitUntilPartitions(zooKeeperClient, DEFAULT_TABLE_PATH); + Map newPartitions = + createPartitions( + zooKeeperClient, + DEFAULT_TABLE_PATH, + Collections.singletonList("created-after-checkpoint")); + + try (MockSplitEnumeratorContext context = + new MockSplitEnumeratorContext<>(numSubtasks); + MockWorkExecutor workExecutor = new MockWorkExecutor(context); + FlinkSourceEnumerator enumerator = + new FlinkSourceEnumerator( + DEFAULT_TABLE_PATH, + flussConf, + false, + true, + context, + Collections.emptySet(), + restoredPartitions, + null, + OffsetsInitializer.earliest(), + null, + 0L, + FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE + .defaultValue(), + true, + null, + null, + workExecutor, + LeaseContext.DEFAULT, + true, + true, + Collections.emptyList())) { + enumerator.start(); + for (int i = 0; i < numSubtasks; i++) { + registerReader(context, enumerator, i); + assertThat(context.hasNoMoreSplits(i)).isFalse(); + } + + // List partitions first. The newly created partition still needs split initialization, + // so readers must not be told that no more splits are available yet. + workExecutor.runNextOneTimeCallable(); + for (int i = 0; i < numSubtasks; i++) { + assertThat(context.hasNoMoreSplits(i)).isFalse(); + } + + // Initialize and assign the splits for the newly discovered partition. + workExecutor.runNextOneTimeCallable(); + List assignedSplits = + getReadersAssignments(context).values().stream() + .flatMap(List::stream) + .collect(Collectors.toList()); + assertThat(assignedSplits) + .hasSize(newPartitions.size() * DEFAULT_BUCKET_NUM) + .allSatisfy( + split -> + assertThat(split.getPartitionName()) + .isIn(newPartitions.values())); + for (int i = 0; i < numSubtasks; i++) { + assertThat(context.hasNoMoreSplits(i)).isTrue(); + } + } + } + @Test void testBoundedNonPartitionedRestoreSignalsNoMoreSplits() throws Exception { long tableId = createTable(DEFAULT_TABLE_PATH, DEFAULT_LOG_TABLE_DESCRIPTOR); diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java index cc71ddb763f..5b0117a0f73 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java @@ -17,7 +17,6 @@ package org.apache.fluss.flink.source.reader; -import org.apache.fluss.client.admin.OffsetSpec; import org.apache.fluss.client.metadata.KvSnapshots; import org.apache.fluss.client.table.Table; import org.apache.fluss.client.table.scanner.ScanRecord; @@ -244,54 +243,6 @@ void testHandleLogSplitChangesAndFetch() throws Exception { } } - @Test - void testBoundedLogSplitStopsAtCapturedLatestOffset() throws Exception { - Schema schema = - Schema.newBuilder() - .column("id", DataTypes.INT()) - .column("name", DataTypes.STRING()) - .build(); - TableDescriptor tableDescriptor = - TableDescriptor.builder().schema(schema).distributedBy(1).build(); - TablePath tablePath = TablePath.of(DEFAULT_DB, "test-bounded-log-split"); - - long tableId = createTable(tablePath, tableDescriptor); - List initialRows = appendRows(tablePath, 2); - - long stoppingOffset = - admin.listOffsets( - tablePath, - Collections.singletonList(0), - new OffsetSpec.LatestSpec()) - .bucketResult(0) - .get(); - - // These records are written after stoppingOffset was captured. - appendRows(tablePath, 2); - - TableBucket tableBucket = new TableBucket(tableId, 0); - LogSplit split = new LogSplit(tableBucket, null, 0L, stoppingOffset); - - List expected = new ArrayList<>(); - for (int i = 0; i < initialRows.size(); i++) { - expected.add( - new RecordAndPos( - new ScanRecord(i, i, ChangeType.APPEND_ONLY, initialRows.get(i)))); - } - - Map> expectedRecords = new HashMap<>(); - expectedRecords.put(split.splitId(), expected); - - try (FlinkSourceSplitReader splitReader = - createSplitReader(tablePath, schema.getRowType())) { - assignSplitsAndFetchUntilRetrieveRecords( - splitReader, - Collections.singletonList(split), - expectedRecords, - schema.getRowType()); - } - } - @Test void testHandleMixSnapshotLogSplitChangesAndFetch() throws Exception { TablePath tablePath = TablePath.of(DEFAULT_DB, "test-mix-snapshot-log-table"); diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/split/SourceSplitSerializerTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/split/SourceSplitSerializerTest.java index c047de34ee7..33cefaaf6cc 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/split/SourceSplitSerializerTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/split/SourceSplitSerializerTest.java @@ -87,14 +87,6 @@ void testLogSplitSerde(boolean isPartitioned) throws Exception { SourceSplitBase deserializedSplit = serializer.deserialize(serializer.getVersion(), serialized); assertThat(deserializedSplit).isEqualTo(logSplit); - - LogSplit boundedLogSplit = new LogSplit(bucket, partitionName, 100L, 200L); - - serialized = serializer.serialize(boundedLogSplit); - deserializedSplit = serializer.deserialize(serializer.getVersion(), serialized); - - assertThat(deserializedSplit).isEqualTo(boundedLogSplit); - assertThat(deserializedSplit.asLogSplit().getStoppingOffset()).contains(200L); } @ParameterizedTest diff --git a/website/docs/engine-flink/datastream.mdx b/website/docs/engine-flink/datastream.mdx index ef63c3bfd77..48db2e1af54 100644 --- a/website/docs/engine-flink/datastream.mdx +++ b/website/docs/engine-flink/datastream.mdx @@ -137,14 +137,14 @@ Bounded reads are supported for log tables and the changelog of primary key tabl **Example:** ```java // Replay a bounded time range of the log: from one hour ago up to now -FlussSource source = FlussSource.builder() +FlussSource timeRangeSource = FlussSource.builder() .setStartingOffsets(OffsetsInitializer.timestamp(System.currentTimeMillis() - 3600 * 1000)) .setBounded(OffsetsInitializer.timestamp(System.currentTimeMillis())) // other configuration... .build(); // Read up to the latest offsets captured at startup and then finish -FlussSource source = FlussSource.builder() +FlussSource latestSource = FlussSource.builder() .setStartingOffsets(OffsetsInitializer.earliest()) .setBounded(OffsetsInitializer.latest()) // other configuration... From 9cb6785b429dcb7840023a4de3786f141a214d6b Mon Sep 17 00:00:00 2001 From: naivedogger Date: Tue, 18 Aug 2026 17:29:08 +0800 Subject: [PATCH 8/9] retrigger ci From ee96ba5cf7f173d7fba9e6520a04157de148161d Mon Sep 17 00:00:00 2001 From: naivedogger Date: Wed, 19 Aug 2026 19:27:36 +0800 Subject: [PATCH 9/9] [flink] Align bounded source with Kafka semantics --- .../flink/source/BinlogFlinkTableSource.java | 3 +- .../source/ChangelogFlinkTableSource.java | 3 +- .../fluss/flink/source/FlinkSource.java | 23 +- .../fluss/flink/source/FlinkTableSource.java | 14 +- .../fluss/flink/source/FlussSource.java | 10 +- .../flink/source/FlussSourceBuilder.java | 20 +- .../enumerator/FlinkSourceEnumerator.java | 86 +++---- .../utils/FlinkConnectorOptionsUtils.java | 31 ++- .../flink/source/FlussSourceBuilderTest.java | 14 ++ .../enumerator/FlinkSourceEnumeratorTest.java | 235 +----------------- .../utils/FlinkConnectorOptionsUtilTest.java | 25 ++ 11 files changed, 148 insertions(+), 316 deletions(-) diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/BinlogFlinkTableSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/BinlogFlinkTableSource.java index 6684857caaa..8c7c7141121 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/BinlogFlinkTableSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/BinlogFlinkTableSource.java @@ -169,7 +169,7 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { // Create the source with the binlog deserialization schema OffsetsInitializer stoppingOffsetsInitializer = - FlinkConnectorOptionsUtils.toStoppingOffsetsInitializer(boundedOptions); + FlinkConnectorOptionsUtils.toStoppingOffsetsInitializer(streaming, boundedOptions); FlinkSource source = new FlinkSource<>( flussConfig, @@ -181,6 +181,7 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { null, offsetsInitializer, stoppingOffsetsInitializer, + FlinkConnectorOptionsUtils.toBoundedness(streaming, boundedOptions), scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, new BinlogDeserializationSchema(), diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/ChangelogFlinkTableSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/ChangelogFlinkTableSource.java index 65938de4185..195dd73765b 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/ChangelogFlinkTableSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/ChangelogFlinkTableSource.java @@ -225,7 +225,7 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { // Create the source with the changelog deserialization schema OffsetsInitializer stoppingOffsetsInitializer = - FlinkConnectorOptionsUtils.toStoppingOffsetsInitializer(boundedOptions); + FlinkConnectorOptionsUtils.toStoppingOffsetsInitializer(streaming, boundedOptions); FlinkSource source = new FlinkSource<>( flussConfig, @@ -241,6 +241,7 @@ public ScanRuntimeProvider getScanRuntimeProvider(ScanContext scanContext) { logRecordBatchFilter, offsetsInitializer, stoppingOffsetsInitializer, + FlinkConnectorOptionsUtils.toBoundedness(streaming, boundedOptions), scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, new ChangelogDeserializationSchema(), diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java index 9c36b7eea91..e2041814f59 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java @@ -17,6 +17,7 @@ package org.apache.fluss.flink.source; +import org.apache.fluss.client.initializer.NoStoppingOffsetsInitializer; import org.apache.fluss.client.initializer.OffsetsInitializer; import org.apache.fluss.config.Configuration; import org.apache.fluss.flink.FlinkConnectorOptions; @@ -75,10 +76,11 @@ public class FlinkSource @Nullable private final FlinkRecordEmitter.OutputProjection outputProjection; @Nullable private final int[] projectedFields; protected final OffsetsInitializer offsetsInitializer; - @Nullable protected final OffsetsInitializer stoppingOffsetsInitializer; + protected final OffsetsInitializer stoppingOffsetsInitializer; protected final long scanPartitionDiscoveryIntervalMs; protected final int splitPerAssignmentBatchSize; private final boolean streaming; + private final Boundedness boundedness; private final FlussDeserializationSchema deserializationSchema; @Nullable private final Predicate partitionFilters; @Nullable private final LakeSource lakeSource; @@ -209,7 +211,8 @@ public FlinkSource( projectedFields, logRecordBatchFilter, offsetsInitializer, - null, + streaming ? new NoStoppingOffsetsInitializer() : OffsetsInitializer.latest(), + streaming ? Boundedness.CONTINUOUS_UNBOUNDED : Boundedness.BOUNDED, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, deserializationSchema, @@ -245,7 +248,8 @@ public FlinkSource( projectedFields, logRecordBatchFilter, offsetsInitializer, - null, + streaming ? new NoStoppingOffsetsInitializer() : OffsetsInitializer.latest(), + streaming ? Boundedness.CONTINUOUS_UNBOUNDED : Boundedness.BOUNDED, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, deserializationSchema, @@ -265,7 +269,8 @@ public FlinkSource( @Nullable int[] projectedFields, @Nullable Predicate logRecordBatchFilter, OffsetsInitializer offsetsInitializer, - @Nullable OffsetsInitializer stoppingOffsetsInitializer, + OffsetsInitializer stoppingOffsetsInitializer, + Boundedness boundedness, long scanPartitionDiscoveryIntervalMs, int splitPerAssignmentBatchSize, FlussDeserializationSchema deserializationSchema, @@ -283,6 +288,7 @@ public FlinkSource( this.logRecordBatchFilter = logRecordBatchFilter; this.offsetsInitializer = offsetsInitializer; this.stoppingOffsetsInitializer = stoppingOffsetsInitializer; + this.boundedness = boundedness; this.scanPartitionDiscoveryIntervalMs = scanPartitionDiscoveryIntervalMs; this.splitPerAssignmentBatchSize = splitPerAssignmentBatchSize; this.deserializationSchema = deserializationSchema; @@ -296,12 +302,7 @@ public FlinkSource( @Override public Boundedness getBoundedness() { - // User-supplied stopping offsets make the source bounded even in streaming execution - // mode (bounded streaming read), so that the job finishes once all splits reach their - // stopping offsets. - return (streaming && stoppingOffsetsInitializer == null) - ? Boundedness.CONTINUOUS_UNBOUNDED - : Boundedness.BOUNDED; + return boundedness; } @Override @@ -315,6 +316,7 @@ public SplitEnumerator createEnumerator( splitEnumeratorContext, offsetsInitializer, stoppingOffsetsInitializer, + boundedness, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, streaming, @@ -347,6 +349,7 @@ public SplitEnumerator restoreEnumerator remainingHybridLakeFlussSplits, offsetsInitializer, stoppingOffsetsInitializer, + boundedness, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, streaming, diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java index 7b1935f56e3..714ef921b51 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java @@ -454,6 +454,7 @@ public boolean isBounded() { logRecordBatchFilter, offsetsInitializer, stoppingOffsetsInitializer, + FlinkConnectorOptionsUtils.toBoundedness(streaming, boundedOptions), scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, new RowDataDeserializationSchema(), @@ -505,19 +506,12 @@ public boolean isBounded() { } } - /** - * Creates the stopping offsets initializer from the configured bounded options, or returns null - * to fall back to the default behavior, i.e. no stopping offsets in streaming execution mode - * and the latest offsets captured at startup in batch execution mode. - */ - @Nullable + /** Creates the stopping offsets initializer from the configured bounded options. */ private OffsetsInitializer createStoppingOffsetsInitializer() { - OffsetsInitializer stoppingOffsetsInitializer = - FlinkConnectorOptionsUtils.toStoppingOffsetsInitializer(boundedOptions); - if (stoppingOffsetsInitializer != null) { + if (boundedOptions.boundedMode != FlinkConnectorOptions.ScanBoundedMode.UNBOUNDED) { validateBoundedModeSupported(); } - return stoppingOffsetsInitializer; + return FlinkConnectorOptionsUtils.toStoppingOffsetsInitializer(streaming, boundedOptions); } private void validateBoundedModeSupported() { diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java index 80546e1e22a..17f6f16bb99 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java @@ -18,6 +18,7 @@ package org.apache.fluss.flink.source; import org.apache.fluss.annotation.VisibleForTesting; +import org.apache.fluss.client.initializer.NoStoppingOffsetsInitializer; import org.apache.fluss.client.initializer.OffsetsInitializer; import org.apache.fluss.client.initializer.SnapshotOffsetsInitializer; import org.apache.fluss.config.Configuration; @@ -30,6 +31,8 @@ import org.apache.fluss.predicate.Predicate; import org.apache.fluss.types.RowType; +import org.apache.flink.api.connector.source.Boundedness; + import javax.annotation.Nullable; /** @@ -115,7 +118,8 @@ public class FlussSource extends FlinkSource { projectedFields, logRecordBatchFilter, offsetsInitializer, - null, + streaming ? new NoStoppingOffsetsInitializer() : OffsetsInitializer.latest(), + streaming ? Boundedness.CONTINUOUS_UNBOUNDED : Boundedness.BOUNDED, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, deserializationSchema, @@ -132,7 +136,8 @@ public class FlussSource extends FlinkSource { @Nullable int[] projectedFields, @Nullable Predicate logRecordBatchFilter, OffsetsInitializer offsetsInitializer, - @Nullable OffsetsInitializer stoppingOffsetsInitializer, + OffsetsInitializer stoppingOffsetsInitializer, + Boundedness boundedness, long scanPartitionDiscoveryIntervalMs, int splitPerAssignmentBatchSize, FlussDeserializationSchema deserializationSchema, @@ -149,6 +154,7 @@ public class FlussSource extends FlinkSource { logRecordBatchFilter, validateBatchStartupMode(offsetsInitializer, hasPrimaryKey, streaming, tablePath), stoppingOffsetsInitializer, + boundedness, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, deserializationSchema, diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java index 9585df16333..08945afeb7c 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java @@ -21,6 +21,7 @@ import org.apache.fluss.client.ConnectionFactory; import org.apache.fluss.client.admin.Admin; import org.apache.fluss.client.initializer.LatestOffsetsInitializer; +import org.apache.fluss.client.initializer.NoStoppingOffsetsInitializer; import org.apache.fluss.client.initializer.OffsetsInitializer; import org.apache.fluss.client.initializer.SnapshotOffsetsInitializer; import org.apache.fluss.client.initializer.TimestampOffsetsInitializer; @@ -36,6 +37,7 @@ import org.apache.fluss.predicate.Predicate; import org.apache.fluss.types.RowType; +import org.apache.flink.api.connector.source.Boundedness; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -86,8 +88,13 @@ public class FlussSourceBuilder { private Long scanPartitionDiscoveryIntervalMs; private Integer splitPerAssignmentBatchSize; private OffsetsInitializer offsetsInitializer; - private OffsetsInitializer stoppingOffsetsInitializer; + private OffsetsInitializer stoppingOffsetsInitializer = new NoStoppingOffsetsInitializer(); + + // This flag preserves the execution semantics of the legacy no-argument setBounded(), which + // switches the source to batch mode. The parameterized setBounded(OffsetsInitializer) keeps + // streaming execution semantics and only makes the source bounded. private boolean bounded; + private Boundedness boundedness = Boundedness.CONTINUOUS_UNBOUNDED; private FlussDeserializationSchema deserializationSchema; private String bootstrapServers; @@ -180,12 +187,17 @@ public FlussSourceBuilder setStartingOffsets(OffsetsInitializer offsetsInit * Builds a bounded source for batch execution. The source reads up to the latest offsets at job * startup and then finishes; combined with the default {@link OffsetsInitializer#full()} on a * datalake-enabled table this performs a bounded union read of the lake snapshot and the Fluss - * log. If not called, the source is unbounded (streaming). + * log. + * + *

This overload is retained for compatibility and switches the source to batch execution + * semantics. Use {@link #setBounded(OffsetsInitializer)} for a bounded streaming read. * * @return this builder */ public FlussSourceBuilder setBounded() { this.bounded = true; + this.boundedness = Boundedness.BOUNDED; + this.stoppingOffsetsInitializer = OffsetsInitializer.latest(); return this; } @@ -211,6 +223,7 @@ public FlussSourceBuilder setBounded(OffsetsInitializer stoppingOffsetsInit + "supported as stopping offsets, but was %s.", checkedStoppingOffsetsInitializer.getClass().getName()); this.stoppingOffsetsInitializer = checkedStoppingOffsetsInitializer; + this.boundedness = Boundedness.BOUNDED; return this; } @@ -398,7 +411,7 @@ public FlussSource build() { // reading phase has no bounded end. // - The datalake union read (full startup mode on a datalake-enabled table) is not // supported, because lake splits have no bounded end. - if (stoppingOffsetsInitializer != null) { + if (!bounded && boundedness == Boundedness.BOUNDED) { if (hasPrimaryKey && fullStartup) { throw new IllegalArgumentException( String.format( @@ -452,6 +465,7 @@ public FlussSource build() { logRecordBatchFilter, offsetsInitializer, stoppingOffsetsInitializer, + boundedness, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, deserializationSchema, diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java index ffb85026798..c0552e2f2d8 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumerator.java @@ -56,6 +56,7 @@ import org.apache.flink.annotation.Internal; import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.connector.source.Boundedness; import org.apache.flink.api.connector.source.SourceEvent; import org.apache.flink.api.connector.source.SplitEnumerator; import org.apache.flink.api.connector.source.SplitEnumeratorContext; @@ -151,16 +152,10 @@ public class FlinkSourceEnumerator private final long scanPartitionDiscoveryIntervalMs; private final boolean streaming; + private final Boundedness boundedness; private final OffsetsInitializer startingOffsetsInitializer; private final OffsetsInitializer stoppingOffsetsInitializer; - /** - * Whether this read is bounded, i.e. batch execution mode or a bounded streaming read with - * user-supplied stopping offsets. A bounded read only performs a one-time partition discovery, - * since partitions created after startup are outside the bounded range captured at startup. - */ - private final boolean bounded; - /** * The offsets initializer used for partitions discovered after the initial startup. Following * context, OffsetsInitializer startingOffsetsInitializer, - @Nullable OffsetsInitializer stoppingOffsetsInitializer, + OffsetsInitializer stoppingOffsetsInitializer, + Boundedness boundedness, long scanPartitionDiscoveryIntervalMs, int splitPerAssignmentBatchSize, boolean streaming, @@ -342,6 +340,7 @@ public FlinkSourceEnumerator( null, startingOffsetsInitializer, stoppingOffsetsInitializer, + boundedness, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, streaming, @@ -422,7 +421,8 @@ public FlinkSourceEnumerator( assignedPartitions, pendingHybridLakeFlussSplits, startingOffsetsInitializer, - null, + streaming ? new NoStoppingOffsetsInitializer() : OffsetsInitializer.latest(), + streaming ? Boundedness.CONTINUOUS_UNBOUNDED : Boundedness.BOUNDED, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, streaming, @@ -445,7 +445,8 @@ public FlinkSourceEnumerator( Map assignedPartitions, List pendingHybridLakeFlussSplits, OffsetsInitializer startingOffsetsInitializer, - @Nullable OffsetsInitializer stoppingOffsetsInitializer, + OffsetsInitializer stoppingOffsetsInitializer, + Boundedness boundedness, long scanPartitionDiscoveryIntervalMs, int splitPerAssignmentBatchSize, boolean streaming, @@ -466,6 +467,7 @@ public FlinkSourceEnumerator( pendingHybridLakeFlussSplits, startingOffsetsInitializer, stoppingOffsetsInitializer, + boundedness, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, streaming, @@ -505,7 +507,8 @@ public FlinkSourceEnumerator( assignedPartitions, pendingHybridLakeFlussSplits, startingOffsetsInitializer, - null, + streaming ? new NoStoppingOffsetsInitializer() : OffsetsInitializer.latest(), + streaming ? Boundedness.CONTINUOUS_UNBOUNDED : Boundedness.BOUNDED, scanPartitionDiscoveryIntervalMs, FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE.defaultValue(), streaming, @@ -548,7 +551,8 @@ public FlinkSourceEnumerator( assignedPartitions, pendingHybridLakeFlussSplits, startingOffsetsInitializer, - null, + streaming ? new NoStoppingOffsetsInitializer() : OffsetsInitializer.latest(), + streaming ? Boundedness.CONTINUOUS_UNBOUNDED : Boundedness.BOUNDED, scanPartitionDiscoveryIntervalMs, splitPerAssignmentBatchSize, streaming, @@ -571,7 +575,8 @@ public FlinkSourceEnumerator( Map assignedPartitions, List pendingHybridLakeFlussSplits, OffsetsInitializer startingOffsetsInitializer, - @Nullable OffsetsInitializer stoppingOffsetsInitializer, + OffsetsInitializer stoppingOffsetsInitializer, + Boundedness boundedness, long scanPartitionDiscoveryIntervalMs, int splitPerAssignmentBatchSize, boolean streaming, @@ -602,16 +607,9 @@ public FlinkSourceEnumerator( this.newDiscoveryOffsetsInitializer = OffsetsInitializer.earliest(); this.scanPartitionDiscoveryIntervalMs = scanPartitionDiscoveryIntervalMs; this.streaming = streaming; + this.boundedness = checkNotNull(boundedness); this.partitionFilters = partitionFilters; - // The read is bounded if it runs in batch execution mode, or if the user supplied - // stopping offsets for a bounded streaming read. - this.bounded = !streaming || stoppingOffsetsInitializer != null; - this.stoppingOffsetsInitializer = - stoppingOffsetsInitializer != null - ? stoppingOffsetsInitializer - : (streaming - ? new NoStoppingOffsetsInitializer() - : OffsetsInitializer.latest()); + this.stoppingOffsetsInitializer = checkNotNull(stoppingOffsetsInitializer); this.lakeSource = lakeSource; this.workerExecutor = workerExecutor; this.leaseContext = leaseContext; @@ -619,7 +617,6 @@ public FlinkSourceEnumerator( this.splitPerAssignmentBatchSize = splitPerAssignmentBatchSize; this.initialDiscoveryFinished = initialDiscoveryFinished; this.unassignedSplits = new ArrayList<>(unassignedSplits); - this.noMoreNewSplits = initialDiscoveryFinished && isBoundedStreamingRead(); } @Override @@ -665,7 +662,6 @@ public void start() { addSplitToPendingAssignments(unassignedSplits); } - boolean restoredBoundedPartitionSet = isBoundedStreamingRead() && initialDiscoveryFinished; if (isPartitioned) { if (streaming) { if (lakeSource != null) { @@ -681,12 +677,7 @@ public void start() { } } - if (restoredBoundedPartitionSet) { - LOG.info( - "Skipping partition discovery for restored bounded source of table {}.", - tablePath); - assignPendingSplits(context.registeredReaders().keySet()); - } else if (isPeriodicPartitionDiscoveryEnabled()) { + if (isPeriodicPartitionDiscoveryEnabled()) { // should do partition discovery LOG.info( "Starting the FlussSourceEnumerator for table {} " @@ -700,9 +691,7 @@ public void start() { 0, scanPartitionDiscoveryIntervalMs); } else { - // Call once for a bounded read or when partition discovery is disabled. For - // a bounded read, partitions created after startup are outside the bounded - // range captured at startup, so continuous discovery is not needed. + // Call once for a bounded read or when partition discovery is disabled. LOG.info( "Starting the FlussSourceEnumerator for table {} without partition discovery.", tablePath); @@ -773,7 +762,7 @@ private void startInStreamModeForNonPartitionedTable() { tablePath, pendingSplitAssignment.values().stream().mapToInt(List::size).sum()); initialDiscoveryFinished = true; - if (isBoundedStreamingRead()) { + if (!isPeriodicPartitionDiscoveryEnabled()) { noMoreNewSplits = true; } return; @@ -864,11 +853,6 @@ private void checkPartitionChanges(Set partitionInfos, Throwable return; } if (t != null) { - if (isBoundedStreamingRead()) { - throw new FlinkRuntimeException( - String.format("Failed to list partitions for %s", tablePath), - ExceptionUtils.stripCompletionException(t)); - } LOG.error("Failed to list partitions for {}", tablePath, t); return; } @@ -888,7 +872,7 @@ private void checkPartitionChanges(Set partitionInfos, Throwable if (!initialDiscoveryFinished) { initialDiscoveryFinished = true; } - if (isBoundedStreamingRead()) { + if (!isPeriodicPartitionDiscoveryEnabled()) { noMoreNewSplits = true; assignPendingSplits(context.registeredReaders().keySet()); } @@ -1332,11 +1316,10 @@ private static boolean shouldRemoveForDroppedPartition( } private boolean isPeriodicPartitionDiscoveryEnabled() { - return isPartitioned && streaming && !bounded && scanPartitionDiscoveryIntervalMs > 0; - } - - private boolean isBoundedStreamingRead() { - return streaming && bounded; + return isPartitioned + && streaming + && boundedness == Boundedness.CONTINUOUS_UNBOUNDED + && scanPartitionDiscoveryIntervalMs > 0; } private void handleSplitsAdd(List splits, Throwable t) { @@ -1372,14 +1355,7 @@ private void handleSplitsAdd(List splits, Throwable t) { ? "null" : pendingHybridLakeFlussSplits.size()); - if (isPartitioned) { - if (!streaming || bounded || scanPartitionDiscoveryIntervalMs <= 0) { - // A batch or bounded streaming read has a finite split set. An unbounded streaming - // read also has a finite split set when periodic partition discovery is disabled. - noMoreNewSplits = true; - } - } else { - // A non-partitioned table only adds splits once. + if (!isPeriodicPartitionDiscoveryEnabled()) { noMoreNewSplits = true; } doHandleSplitsAdd(splits); diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java index 361ddc1151e..a90f240ec8f 100644 --- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java +++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtils.java @@ -17,6 +17,7 @@ package org.apache.fluss.flink.utils; +import org.apache.fluss.client.initializer.NoStoppingOffsetsInitializer; import org.apache.fluss.client.initializer.OffsetsInitializer; import org.apache.fluss.config.Configuration; import org.apache.fluss.flink.FlinkConnectorOptions; @@ -25,6 +26,7 @@ import org.apache.fluss.flink.sink.shuffle.DistributionMode; import org.apache.fluss.metadata.MergeEngineType; +import org.apache.flink.api.connector.source.Boundedness; import org.apache.flink.configuration.ConfigurationUtils; import org.apache.flink.configuration.ReadableConfig; import org.apache.flink.table.api.ValidationException; @@ -128,15 +130,11 @@ public static BoundedOptions getBoundedOptions(ReadableConfig tableOptions, Zone return options; } - /** - * Creates the stopping offsets initializer from the given bounded options, or returns null for - * the unbounded mode. - */ - @Nullable + /** Creates the stopping offsets initializer from the given bounded options. */ public static OffsetsInitializer toStoppingOffsetsInitializer(BoundedOptions boundedOptions) { switch (boundedOptions.boundedMode) { case UNBOUNDED: - return null; + return new NoStoppingOffsetsInitializer(); case LATEST_OFFSET: return OffsetsInitializer.latest(); case TIMESTAMP: @@ -147,6 +145,27 @@ public static OffsetsInitializer toStoppingOffsetsInitializer(BoundedOptions bou } } + /** + * Creates the stopping offsets initializer for the execution mode and bounded options. + * + *

Batch execution remains bounded by the latest offsets when no explicit bounded mode is + * configured. + */ + public static OffsetsInitializer toStoppingOffsetsInitializer( + boolean streaming, BoundedOptions boundedOptions) { + if (!streaming && boundedOptions.boundedMode == ScanBoundedMode.UNBOUNDED) { + return OffsetsInitializer.latest(); + } + return toStoppingOffsetsInitializer(boundedOptions); + } + + /** Returns the Flink source boundedness for the execution mode and bounded options. */ + public static Boundedness toBoundedness(boolean streaming, BoundedOptions boundedOptions) { + return streaming && boundedOptions.boundedMode == ScanBoundedMode.UNBOUNDED + ? Boundedness.CONTINUOUS_UNBOUNDED + : Boundedness.BOUNDED; + } + public static List getBucketKeys(ReadableConfig tableOptions) { Optional bucketKey = tableOptions.getOptional(FlinkConnectorOptions.BUCKET_KEY); if (!bucketKey.isPresent()) { diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlussSourceBuilderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlussSourceBuilderTest.java index 5c107cd9355..550ce59052c 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlussSourceBuilderTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/FlussSourceBuilderTest.java @@ -97,6 +97,20 @@ public void testBuildBoundedStreamingSource() { assertThat(source.getBoundedness()).isEqualTo(Boundedness.BOUNDED); } + @Test + public void testBuildLegacyBoundedSource() { + FlussSource source = + FlussSource.builder() + .setBootstrapServers(bootstrapServers) + .setDatabase(DEFAULT_DB) + .setTable(DEFAULT_TABLE_PATH.getTableName()) + .setBounded() + .setDeserializationSchema(new TestDeserializationSchema()) + .build(); + + assertThat(source.getBoundedness()).isEqualTo(Boundedness.BOUNDED); + } + @Test public void testMissingBootstrapServers() { // Given diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java index d35c4e554e4..97a11ae08b5 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/enumerator/FlinkSourceEnumeratorTest.java @@ -58,13 +58,13 @@ import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.RowType; +import org.apache.flink.api.connector.source.Boundedness; import org.apache.flink.api.connector.source.ReaderInfo; import org.apache.flink.api.connector.source.SourceEvent; import org.apache.flink.api.connector.source.SplitEnumerator; import org.apache.flink.api.connector.source.SplitsAssignment; import org.apache.flink.api.connector.source.mocks.MockSplitEnumeratorContext; import org.apache.flink.table.data.RowData; -import org.apache.flink.util.FlinkRuntimeException; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -83,8 +83,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.concurrent.Callable; -import java.util.function.BiConsumer; import java.util.stream.Collectors; import static org.apache.fluss.client.table.scanner.log.LogScanner.EARLIEST_OFFSET; @@ -152,6 +150,9 @@ void testPkTableNoSnapshotSplits() throws Throwable { Map> actualAssignment = getReadersAssignments(context); assertThat(actualAssignment).isEqualTo(expectedAssignment); + for (int i = 0; i < numSubtasks; i++) { + assertThat(context.hasNoMoreSplits(i)).isTrue(); + } } } @@ -1107,6 +1108,7 @@ void testBatchModeWithTimestampStoppingOffsets() throws Throwable { context, OffsetsInitializer.earliest(), OffsetsInitializer.timestamp(stoppingTimestamp), + Boundedness.BOUNDED, DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE.defaultValue(), false, @@ -1166,6 +1168,7 @@ void testBoundedStreamingReadSignalsNoMoreSplits() throws Throwable { context, OffsetsInitializer.earliest(), OffsetsInitializer.latest(), + Boundedness.BOUNDED, DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE.defaultValue(), true, @@ -1214,168 +1217,6 @@ void testBoundedStreamingReadSignalsNoMoreSplits() throws Throwable { } } - @Test - void testBoundedStreamingRestoreSkipsPartitionDiscovery() throws Throwable { - int numSubtasks = 3; - createTable(DEFAULT_TABLE_PATH, DEFAULT_AUTO_PARTITIONED_LOG_TABLE_DESCRIPTOR); - ZooKeeperClient zooKeeperClient = FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(); - Map partitions = waitUntilPartitions(zooKeeperClient, DEFAULT_TABLE_PATH); - - // Simulate a partition created after the checkpoint represented by the restored state. - createPartitions( - zooKeeperClient, - DEFAULT_TABLE_PATH, - Collections.singletonList("created-after-checkpoint")); - - try (MockSplitEnumeratorContext context = - new MockSplitEnumeratorContext<>(numSubtasks); - MockWorkExecutor workExecutor = new MockWorkExecutor(context); - FlinkSourceEnumerator enumerator = - new FlinkSourceEnumerator( - DEFAULT_TABLE_PATH, - flussConf, - false, - true, - context, - Collections.emptySet(), - partitions, - null, - OffsetsInitializer.earliest(), - OffsetsInitializer.latest(), - DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, - FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE - .defaultValue(), - true, - null, - null, - workExecutor, - LeaseContext.DEFAULT, - true, - true, - Collections.emptyList())) { - enumerator.start(); - for (int i = 0; i < numSubtasks; i++) { - registerReader(context, enumerator, i); - } - - assertThat(workExecutor.getOneTimeCallables()).isEmpty(); - assertThat(context.getSplitsAssignmentSequence()).isEmpty(); - for (int i = 0; i < numSubtasks; i++) { - assertThat(context.hasNoMoreSplits(i)).isTrue(); - } - } - } - - @Test - void testUnboundedRestoreWithDisabledPartitionDiscoveryDiscoversBeforeFinishing() - throws Throwable { - int numSubtasks = 3; - createTable(DEFAULT_TABLE_PATH, DEFAULT_AUTO_PARTITIONED_LOG_TABLE_DESCRIPTOR); - ZooKeeperClient zooKeeperClient = FLUSS_CLUSTER_EXTENSION.getZooKeeperClient(); - Map restoredPartitions = - waitUntilPartitions(zooKeeperClient, DEFAULT_TABLE_PATH); - Map newPartitions = - createPartitions( - zooKeeperClient, - DEFAULT_TABLE_PATH, - Collections.singletonList("created-after-checkpoint")); - - try (MockSplitEnumeratorContext context = - new MockSplitEnumeratorContext<>(numSubtasks); - MockWorkExecutor workExecutor = new MockWorkExecutor(context); - FlinkSourceEnumerator enumerator = - new FlinkSourceEnumerator( - DEFAULT_TABLE_PATH, - flussConf, - false, - true, - context, - Collections.emptySet(), - restoredPartitions, - null, - OffsetsInitializer.earliest(), - null, - 0L, - FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE - .defaultValue(), - true, - null, - null, - workExecutor, - LeaseContext.DEFAULT, - true, - true, - Collections.emptyList())) { - enumerator.start(); - for (int i = 0; i < numSubtasks; i++) { - registerReader(context, enumerator, i); - assertThat(context.hasNoMoreSplits(i)).isFalse(); - } - - // List partitions first. The newly created partition still needs split initialization, - // so readers must not be told that no more splits are available yet. - workExecutor.runNextOneTimeCallable(); - for (int i = 0; i < numSubtasks; i++) { - assertThat(context.hasNoMoreSplits(i)).isFalse(); - } - - // Initialize and assign the splits for the newly discovered partition. - workExecutor.runNextOneTimeCallable(); - List assignedSplits = - getReadersAssignments(context).values().stream() - .flatMap(List::stream) - .collect(Collectors.toList()); - assertThat(assignedSplits) - .hasSize(newPartitions.size() * DEFAULT_BUCKET_NUM) - .allSatisfy( - split -> - assertThat(split.getPartitionName()) - .isIn(newPartitions.values())); - for (int i = 0; i < numSubtasks; i++) { - assertThat(context.hasNoMoreSplits(i)).isTrue(); - } - } - } - - @Test - void testBoundedNonPartitionedRestoreSignalsNoMoreSplits() throws Exception { - long tableId = createTable(DEFAULT_TABLE_PATH, DEFAULT_LOG_TABLE_DESCRIPTOR); - LogSplit restoredSplit = new LogSplit(new TableBucket(tableId, 0), null, 0L, 0L); - - try (MockSplitEnumeratorContext context = - new MockSplitEnumeratorContext<>(1); - MockWorkExecutor workExecutor = new MockWorkExecutor(context); - FlinkSourceEnumerator enumerator = - new FlinkSourceEnumerator( - DEFAULT_TABLE_PATH, - flussConf, - false, - false, - context, - Collections.emptySet(), - Collections.emptyMap(), - null, - OffsetsInitializer.earliest(), - OffsetsInitializer.latest(), - DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, - FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE - .defaultValue(), - true, - null, - null, - workExecutor, - LeaseContext.DEFAULT, - true, - true, - Collections.singletonList(restoredSplit))) { - enumerator.start(); - registerReader(context, enumerator, 0); - - assertThat(getReadersAssignments(context).get(0)).containsExactly(restoredSplit); - assertThat(context.hasNoMoreSplits(0)).isTrue(); - } - } - @Test void testBoundedStreamingReadWithNoPartitionsSignalsNoMoreSplits() throws Throwable { TablePath tablePath = TablePath.of(DEFAULT_DB, "bounded-empty-partition-table"); @@ -1403,6 +1244,7 @@ void testBoundedStreamingReadWithNoPartitionsSignalsNoMoreSplits() throws Throwa context, OffsetsInitializer.earliest(), OffsetsInitializer.latest(), + Boundedness.BOUNDED, DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE .defaultValue(), @@ -1423,69 +1265,6 @@ void testBoundedStreamingReadWithNoPartitionsSignalsNoMoreSplits() throws Throwa } } - @Test - void testBoundedPartitionDiscoveryFailureFailsJob() throws Exception { - TablePath tablePath = TablePath.of(DEFAULT_DB, "bounded-partition-discovery-failure"); - Schema schema = - Schema.newBuilder() - .column("id", DataTypes.INT()) - .column("partition_col", DataTypes.STRING()) - .build(); - TableDescriptor tableDescriptor = - TableDescriptor.builder() - .schema(schema) - .partitionedBy("partition_col") - .distributedBy(DEFAULT_BUCKET_NUM, "id") - .build(); - createTable(tablePath, tableDescriptor); - - try (MockSplitEnumeratorContext context = - new MockSplitEnumeratorContext<>(1)) { - WorkerExecutor failingWorkerExecutor = - new WorkerExecutor(context) { - @Override - public void callAsync( - Callable callable, BiConsumer handler) { - context.callAsync( - () -> { - throw new FlinkRuntimeException( - "expected partition discovery failure"); - }, - handler); - } - }; - - try (FlinkSourceEnumerator enumerator = - new FlinkSourceEnumerator( - tablePath, - flussConf, - false, - true, - context, - Collections.emptySet(), - Collections.emptyMap(), - null, - OffsetsInitializer.earliest(), - OffsetsInitializer.latest(), - DEFAULT_SCAN_PARTITION_DISCOVERY_INTERVAL_MS, - FlinkConnectorOptions.SCAN_SPLIT_ASSIGNMENT_BATCH_SIZE.defaultValue(), - true, - null, - null, - failingWorkerExecutor, - LeaseContext.DEFAULT, - false, - false, - Collections.emptyList())) { - enumerator.start(); - - assertThatThrownBy(context::runNextOneTimeCallable) - .isInstanceOf(FlinkRuntimeException.class) - .hasMessage("Failed to list partitions for " + tablePath); - } - } - } - @Test void testGetSplitOwner() throws Exception { int numSubtasks = 3; diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtilTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtilTest.java index 2fe355c25a6..cee97c59abd 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtilTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConnectorOptionsUtilTest.java @@ -17,8 +17,12 @@ package org.apache.fluss.flink.utils; +import org.apache.fluss.client.initializer.LatestOffsetsInitializer; +import org.apache.fluss.client.initializer.NoStoppingOffsetsInitializer; import org.apache.fluss.config.Configuration; +import org.apache.fluss.flink.FlinkConnectorOptions.ScanBoundedMode; +import org.apache.flink.api.connector.source.Boundedness; import org.apache.flink.table.api.ValidationException; import org.junit.jupiter.api.Test; @@ -83,6 +87,27 @@ void testValidateSplitAssignmentBatchSize() { .hasMessage("'scan.split.assignment.batch-size' must be positive, but was 0."); } + @Test + void testStoppingOffsetsAndBoundedness() { + FlinkConnectorOptionsUtils.BoundedOptions boundedOptions = + FlinkConnectorOptionsUtils.BoundedOptions.unbounded(); + + assertThat(FlinkConnectorOptionsUtils.toStoppingOffsetsInitializer(boundedOptions)) + .isInstanceOf(NoStoppingOffsetsInitializer.class); + assertThat(FlinkConnectorOptionsUtils.toStoppingOffsetsInitializer(false, boundedOptions)) + .isInstanceOf(LatestOffsetsInitializer.class); + assertThat(FlinkConnectorOptionsUtils.toBoundedness(true, boundedOptions)) + .isEqualTo(Boundedness.CONTINUOUS_UNBOUNDED); + assertThat(FlinkConnectorOptionsUtils.toBoundedness(false, boundedOptions)) + .isEqualTo(Boundedness.BOUNDED); + + boundedOptions.boundedMode = ScanBoundedMode.LATEST_OFFSET; + assertThat(FlinkConnectorOptionsUtils.toStoppingOffsetsInitializer(true, boundedOptions)) + .isInstanceOf(LatestOffsetsInitializer.class); + assertThat(FlinkConnectorOptionsUtils.toBoundedness(true, boundedOptions)) + .isEqualTo(Boundedness.BOUNDED); + } + @Test void testGetClientScannerIoTmpDir() { Configuration flussConfig =