From 3ddb419b678ed8680afe8401cdb0a0bd3bd7e6b7 Mon Sep 17 00:00:00 2001 From: Purushottam Sinha Date: Wed, 29 Jul 2026 22:43:31 +0530 Subject: [PATCH] [FLINK-40256][docs] Include sub-package config options in the configuration reference ConfigurationOptionLocator discovers ConfigOptions from a hard-coded list of packages and reads each with Files.newDirectoryStream, which does not recurse into sub-packages. Options outside that list are dropped from the generated configuration reference without any error, and ConfigOptionsDocsCompletenessITCase cannot detect it because it derives its expectations from the same list. All seven state.backend.rocksdb.manual-compaction.* options were affected: they carry @Documentation.Section(EXPERT_ROCKSDB) but live in org.apache.flink.state.rocksdb.sstmerge, a sub-package of a searched package, so the feature shipped in 1.20 had no documented configuration. Add a location for the sub-package and regenerate the affected tables. RocksDBManualCompactionOptions needs a stability annotation because becoming discoverable also subjects it to ConfigOptionsDocGenerator#verifyClassAnnotation; @PublicEvolving matches RocksDBOptions and RocksDBConfigurableOptions in the same module. Two option descriptions were missing a space between concatenated sentences, which is now user-visible, so fix those too. Add ConfigurationOptionLocatorTest to prevent recurrence: it scans the source tree and fails when a @Documentation.Section option sits in a package the locator does not search. Repo-wide it needs no exclusions. Generated-by: Claude Code (claude-opus-5) --- .../generated/expert_rocksdb_section.html | 42 +++++ ...cksdb_manual_compaction_configuration.html | 54 +++++++ .../docs/util/ConfigurationOptionLocator.java | 8 + .../util/ConfigurationOptionLocatorTest.java | 146 ++++++++++++++++++ .../RocksDBManualCompactionOptions.java | 6 +- 5 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 docs/layouts/shortcodes/generated/rocksdb_manual_compaction_configuration.html create mode 100644 flink-docs/src/test/java/org/apache/flink/docs/util/ConfigurationOptionLocatorTest.java diff --git a/docs/layouts/shortcodes/generated/expert_rocksdb_section.html b/docs/layouts/shortcodes/generated/expert_rocksdb_section.html index eac1d574e26d9c..e34bf48952e991 100644 --- a/docs/layouts/shortcodes/generated/expert_rocksdb_section.html +++ b/docs/layouts/shortcodes/generated/expert_rocksdb_section.html @@ -20,6 +20,48 @@ String The local directory (on the TaskManager) where RocksDB puts its files. Per default, it will be <WORKING_DIR>/tmp. See process.taskmanager.working-dir for more details. + +
state.backend.rocksdb.manual-compaction.max-auto-compactions
+ 30 + Integer + The maximum number of automatic compactions running for manual compaction to start. If the actual number is higher, manual compaction won't be started to avoid delaying automatic ones. + + +
state.backend.rocksdb.manual-compaction.max-file-size-to-compact
+ 50 kb + MemorySize + The maximum size of individual input files + + +
state.backend.rocksdb.manual-compaction.max-files-to-compact
+ 30 + Integer + The maximum number of input files to compact together in a single compaction run + + +
state.backend.rocksdb.manual-compaction.max-output-file-size
+ 64 mb + MemorySize + The maximum output file size + + +
state.backend.rocksdb.manual-compaction.max-parallel-compactions
+ 5 + Integer + The maximum number of manual compactions to start. Note that only one of them can run at a time as of v8.10.0; all the others will be waiting + + +
state.backend.rocksdb.manual-compaction.min-files-to-compact
+ 5 + Integer + The minimum number of input files to compact together in a single compaction run + + +
state.backend.rocksdb.manual-compaction.min-interval
+ 0 ms + Duration + The minimum interval between manual compactions. Zero disables manual compactions +
state.backend.rocksdb.options-factory
(none) diff --git a/docs/layouts/shortcodes/generated/rocksdb_manual_compaction_configuration.html b/docs/layouts/shortcodes/generated/rocksdb_manual_compaction_configuration.html new file mode 100644 index 00000000000000..fce0dd16f6a037 --- /dev/null +++ b/docs/layouts/shortcodes/generated/rocksdb_manual_compaction_configuration.html @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
KeyDefaultTypeDescription
state.backend.rocksdb.manual-compaction.max-auto-compactions
30IntegerThe maximum number of automatic compactions running for manual compaction to start. If the actual number is higher, manual compaction won't be started to avoid delaying automatic ones.
state.backend.rocksdb.manual-compaction.max-file-size-to-compact
50 kbMemorySizeThe maximum size of individual input files
state.backend.rocksdb.manual-compaction.max-files-to-compact
30IntegerThe maximum number of input files to compact together in a single compaction run
state.backend.rocksdb.manual-compaction.max-output-file-size
64 mbMemorySizeThe maximum output file size
state.backend.rocksdb.manual-compaction.max-parallel-compactions
5IntegerThe maximum number of manual compactions to start. Note that only one of them can run at a time as of v8.10.0; all the others will be waiting
state.backend.rocksdb.manual-compaction.min-files-to-compact
5IntegerThe minimum number of input files to compact together in a single compaction run
state.backend.rocksdb.manual-compaction.min-interval
0 msDurationThe minimum interval between manual compactions. Zero disables manual compactions
diff --git a/flink-docs/src/main/java/org/apache/flink/docs/util/ConfigurationOptionLocator.java b/flink-docs/src/main/java/org/apache/flink/docs/util/ConfigurationOptionLocator.java index ed2d987be504a9..7001b86e48239f 100644 --- a/flink-docs/src/main/java/org/apache/flink/docs/util/ConfigurationOptionLocator.java +++ b/flink-docs/src/main/java/org/apache/flink/docs/util/ConfigurationOptionLocator.java @@ -68,6 +68,9 @@ public class ConfigurationOptionLocator { new OptionsClassLocation( "flink-state-backends/flink-statebackend-rocksdb", "org.apache.flink.state.rocksdb"), + new OptionsClassLocation( + "flink-state-backends/flink-statebackend-rocksdb", + "org.apache.flink.state.rocksdb.sstmerge"), new OptionsClassLocation( "flink-state-backends/flink-statebackend-forst", "org.apache.flink.state.forst"), @@ -132,6 +135,11 @@ public ConfigurationOptionLocator(OptionsClassLocation[] locations, String pathP this.pathPrefix = pathPrefix; } + @VisibleForTesting + static OptionsClassLocation[] getLocations() { + return LOCATIONS; + } + public void discoverOptionsAndApply( Path rootDir, BiConsumerWithException, Collection, ? extends Exception> diff --git a/flink-docs/src/test/java/org/apache/flink/docs/util/ConfigurationOptionLocatorTest.java b/flink-docs/src/test/java/org/apache/flink/docs/util/ConfigurationOptionLocatorTest.java new file mode 100644 index 00000000000000..d12e6daccb989b --- /dev/null +++ b/flink-docs/src/test/java/org/apache/flink/docs/util/ConfigurationOptionLocatorTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.docs.util; + +import org.apache.flink.annotation.docs.Documentation; + +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link ConfigurationOptionLocator}. */ +class ConfigurationOptionLocatorTest { + + private static final String SOURCE_ROOT = "src/main/java"; + + private static final String SECTION_ANNOTATION = "@Documentation.Section"; + + private static final Set PRUNED_DIRECTORIES = + Collections.unmodifiableSet( + new HashSet<>(Arrays.asList("target", "node_modules", ".git"))); + + /** Mirrors the file names {@link ConfigurationOptionLocator} recognizes. */ + private static final Pattern OPTIONS_CLASS_FILE_NAME = + Pattern.compile("[a-zA-Z]*(?:Options|Config|Parameters)\\.java"); + + /** + * Verifies that every option annotated with {@link Documentation.Section} sits in a package + * that {@link ConfigurationOptionLocator} actually searches. + * + *

The annotation is an explicit statement that the option belongs in the generated + * configuration reference, but discovery is driven by a hard-coded list of packages and does + * not recurse into sub-packages. An option outside that list is therefore dropped from the + * reference without any error, and {@code ConfigOptionsDocsCompletenessITCase} cannot catch it + * because it derives its expectations from the same list. + */ + @Test + void testSectionAnnotatedOptionsAreAllDiscoverable() throws IOException { + final Path rootDir = Paths.get(Utils.getProjectRootDir()).toAbsolutePath().normalize(); + + final Set searchedPackages = + Arrays.stream(ConfigurationOptionLocator.getLocations()) + .map( + location -> + location.getModule() + + '/' + + location.getPackage().replace('.', '/')) + .collect(Collectors.toSet()); + + final List undiscoverable = new ArrayList<>(); + for (Path optionsClass : findSectionAnnotatedOptionClasses(rootDir)) { + final String relativePath = toUnixPath(rootDir.relativize(optionsClass)); + final String modulePath = relativePath.substring(0, relativePath.indexOf(SOURCE_ROOT)); + final String packagePath = + relativePath.substring( + modulePath.length() + SOURCE_ROOT.length() + 1, + relativePath.lastIndexOf('/')); + + if (!searchedPackages.contains(modulePath + packagePath)) { + undiscoverable.add(relativePath); + } + } + + assertThat(undiscoverable) + .as( + "The options in these classes are annotated with @Documentation.Section but " + + "cannot be found by %s, so they are silently missing from the " + + "generated configuration reference. Add an %s entry for the " + + "containing package to %s#LOCATIONS.", + ConfigurationOptionLocator.class.getSimpleName(), + OptionsClassLocation.class.getSimpleName(), + ConfigurationOptionLocator.class.getSimpleName()) + .isEmpty(); + } + + private static List findSectionAnnotatedOptionClasses(Path rootDir) throws IOException { + final List optionClasses = new ArrayList<>(); + + Files.walkFileTree( + rootDir, + new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory( + Path dir, BasicFileAttributes attributes) { + return PRUNED_DIRECTORIES.contains(dir.getFileName().toString()) + ? FileVisitResult.SKIP_SUBTREE + : FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attributes) + throws IOException { + if (OPTIONS_CLASS_FILE_NAME.matcher(file.getFileName().toString()).matches() + && toUnixPath(file).contains('/' + SOURCE_ROOT + '/') + && isSectionAnnotated(file)) { + optionClasses.add(file); + } + return FileVisitResult.CONTINUE; + } + }); + + return optionClasses; + } + + private static boolean isSectionAnnotated(Path file) throws IOException { + try (Stream lines = Files.lines(file)) { + return lines.anyMatch(line -> line.contains(SECTION_ANNOTATION)); + } + } + + private static String toUnixPath(Path path) { + return path.toString().replace(path.getFileSystem().getSeparator(), "/"); + } +} diff --git a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/sstmerge/RocksDBManualCompactionOptions.java b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/sstmerge/RocksDBManualCompactionOptions.java index c328dc5a589a47..81c7a7e77f0251 100644 --- a/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/sstmerge/RocksDBManualCompactionOptions.java +++ b/flink-state-backends/flink-statebackend-rocksdb/src/main/java/org/apache/flink/state/rocksdb/sstmerge/RocksDBManualCompactionOptions.java @@ -18,6 +18,7 @@ package org.apache.flink.state.rocksdb.sstmerge; +import org.apache.flink.annotation.PublicEvolving; import org.apache.flink.annotation.docs.Documentation; import org.apache.flink.configuration.ConfigOption; import org.apache.flink.configuration.ConfigOptions; @@ -26,6 +27,7 @@ import java.time.Duration; /** Configuration options for manual compaction for the RocksDB backend. */ +@PublicEvolving public class RocksDBManualCompactionOptions { @Documentation.Section(Documentation.Sections.EXPERT_ROCKSDB) @@ -42,7 +44,7 @@ public class RocksDBManualCompactionOptions { .intType() .defaultValue(5) .withDescription( - "The maximum number of manual compactions to start." + "The maximum number of manual compactions to start. " + "Note that only one of them can run at a time as of v8.10.0; all the others will be waiting"); @Documentation.Section(Documentation.Sections.EXPERT_ROCKSDB) @@ -81,6 +83,6 @@ public class RocksDBManualCompactionOptions { .intType() .defaultValue(30) .withDescription( - "The maximum number of automatic compactions running for manual compaction to start." + "The maximum number of automatic compactions running for manual compaction to start. " + "If the actual number is higher, manual compaction won't be started to avoid delaying automatic ones."); }