diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..0f200bb63 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,24 @@ +version: 2 +updates: + - package-ecosystem: maven + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 5 + groups: + maven-minor-and-patch: + patterns: + - "*" + update-types: + - minor + - patch + + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + open-pull-requests-limit: 3 + groups: + github-actions: + patterns: + - "*" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6668456ee..f0347ae47 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -4,11 +4,13 @@ Closes ## Checklist Delete items not relevant to your PR: - [ ] Closes # - [ ] Unit and integration tests covering the common scenarios were added - [ ] A human-readable description of the changes was provided to include in CHANGELOG -- [ ] For significant changes, documentation in https://github.com/ClickHouse/clickhouse-docs was updated with further explanations or tutorials +- [ ] For significant changes, the authoritative source under `docs/clickhouse-docs/` was updated for publication diff --git a/.github/workflows/api-compatibility.yml b/.github/workflows/api-compatibility.yml new file mode 100644 index 000000000..33ab38e25 --- /dev/null +++ b/.github/workflows/api-compatibility.yml @@ -0,0 +1,75 @@ +name: API compatibility + +on: + pull_request: + paths: + - "**/pom.xml" + - "**/src/main/java/**" + - "VERSION" + - ".github/workflows/api-compatibility.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + compare-last-ga: + name: Compare public API with last GA + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out repository and release tags + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Install JDKs and Maven cache + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: | + 11 + 17 + cache: maven + - name: Set up JDK toolchain + shell: bash + run: | + mkdir -p "$HOME/.m2" + cat > "$HOME/.m2/toolchains.xml" < + + + jdk + 11 + ${JAVA_HOME_11_X64} + + + jdk + 17 + ${JAVA_HOME_17_X64} + + + EOF + - name: Resolve last GA version + id: baseline + shell: bash + run: | + tag="$( + git tag --list 'v[0-9]*' --sort=-version:refname | + awk '/^v[0-9]+\.[0-9]+\.[0-9]+$/ { print; exit }' + )" + test -n "$tag" || { + echo "No vMAJOR.MINOR.PATCH GA tag was found" >&2 + exit 1 + } + echo "version=${tag#v}" >> "$GITHUB_OUTPUT" + echo "Comparing against $tag" + - name: Validate development version + run: python3 scripts/release/validate_version.py + - name: Check binary compatibility + run: | + mvn --batch-mode --no-transfer-progress \ + -DskipTests -Dmaven.javadoc.skip=true install + mvn --batch-mode --no-transfer-progress \ + --projects clickhouse-data,clickhouse-client,clickhouse-http-client,client-v2,jdbc-v2,clickhouse-jdbc,clickhouse-r2dbc \ + -Dapi.compatibility.version="${{ steps.baseline.outputs.version }}" \ + -Papi-compatibility japicmp:cmp diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2f1539b1c..ae2502093 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,6 +47,10 @@ jobs: git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \ origin pull/${{ github.event.inputs.pr }}/merge:merged-pr && git checkout merged-pr if: github.event.inputs.pr != '' + - name: Validate release version metadata + run: | + python3 -m unittest discover -s scripts/release -p 'test_*.py' + python3 scripts/release/validate_version.py - name: Install JDK 8 and Maven uses: actions/setup-java@v4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d748fbb38..59319580d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,17 +2,25 @@ name: Release on: workflow_dispatch: + inputs: + publish: + description: "Publish to Maven Central and create the GitHub release" + required: true + default: false + type: boolean env: CH_VERSION: "25.8" jobs: release: - name: "Build and Publish Artifact" + name: Verify release artifacts runs-on: "ubuntu-latest" - + timeout-minutes: 60 + outputs: + version: ${{ steps.version.outputs.version }} permissions: - contents: write + contents: read steps: - name: Check out Git repository @@ -20,70 +28,146 @@ jobs: - name: Read release version id: version shell: bash - run: echo "version=$(cat VERSION)" >> "$GITHUB_OUTPUT" - - name: Install JDK and Maven + run: | + python3 scripts/release/validate_version.py + echo "version=$(cat VERSION)" >> "$GITHUB_OUTPUT" + - name: Install JDKs and Maven cache uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: | - 8 + 11 17 cache: 'maven' - server-id: central - server-username: MAVEN_USERNAME - server-password: MAVEN_PASSWORD - gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} - gpg-passphrase: MAVEN_GPG_PASSPHRASE - name: Setup Toolchain shell: bash run: | - mkdir -p $HOME/.m2 \ - && cat << EOF > $HOME/.m2/toolchains.xml + mkdir -p "$HOME/.m2" + cat > "$HOME/.m2/toolchains.xml" << EOF + + jdk + + 11 + + + ${JAVA_HOME_11_X64} + + jdk 17 - ${{ env.JAVA_HOME }} + ${JAVA_HOME_17_X64} EOF - - name: Update Configuration + - name: Verify release build + env: + TEST_CLICKHOUSE_IMAGE_VERSION: ${{ env.CH_VERSION }} run: | - sed -i 's|.*|${{ steps.version.outputs.version }}|g' pom.xml - find . -type f -name "simplelogger.*" -exec rm -fv '{}' \; - - name: Release Maven package + mvn --batch-mode --no-transfer-progress \ + -Drevision="${{ steps.version.outputs.version }}" \ + -Dgpg.skip=true -DskipITs \ + -P release clean verify + - name: Inventory release artifacts + run: | + python3 scripts/release/inventory_artifacts.py \ + --version "${{ steps.version.outputs.version }}" + - name: Upload dry-run artifacts + uses: actions/upload-artifact@v4 + with: + name: release-${{ steps.version.outputs.version }} + path: release-assets/ + if-no-files-found: error + + publish: + name: Publish verified release + needs: release + if: ${{ inputs.publish }} + runs-on: ubuntu-latest + timeout-minutes: 45 + environment: release + permissions: + contents: write + steps: + - name: Check out verified commit + uses: actions/checkout@v4 + - name: Download verified release artifacts + uses: actions/download-artifact@v4 + with: + name: release-${{ needs.release.outputs.version }} + path: release-assets + - name: Install JDK 11 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 11 + - name: Configure Maven Central signing + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + cache: maven + server-id: central + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} + gpg-passphrase: MAVEN_GPG_PASSPHRASE + - name: Set up JDK toolchain + shell: bash + run: | + mkdir -p "$HOME/.m2" + cat > "$HOME/.m2/toolchains.xml" << EOF + + + + jdk + 11 + ${JAVA_HOME_11_X64} + + + jdk + 17 + ${JAVA_HOME_17_X64} + + + EOF + - name: Publish verified version to Maven Central env: MAVEN_USERNAME: ${{ secrets.SONATYPE_TOKEN_USER }} MAVEN_PASSWORD: ${{ secrets.SONATYPE_TOKEN }} MAVEN_GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - TEST_CLICKHOUSE_IMAGE_VERSION: ${{ env.CH_VERSION }} - run: mvn -q --batch-mode -P release clean deploy - - name: Create Pre-release on Github + run: | + mvn --batch-mode --no-transfer-progress \ + -Drevision="${{ needs.release.outputs.version }}" \ + -DskipTests \ + -P release deploy + - name: Create RC or GA GitHub release + shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh release create "v${{ steps.version.outputs.version }}" \ - --prerelease \ - --title "Release v${{ steps.version.outputs.version }}" \ - LICENSE \ - clickhouse-client/target/clickhouse*.jar \ - clickhouse-data/target/clickhouse*.jar \ - clickhouse-http-client/target/clickhouse*.jar \ - clickhouse-jdbc/target/clickhouse*.jar \ - clickhouse-jdbc/target/bundle/clickhouse*.jar \ - clickhouse-r2dbc/target/clickhouse*.jar \ - client-v2/target/client-v2*.jar \ - jdbc-v2/target/jdbc-v2*.jar + version="${{ needs.release.outputs.version }}" + release_flags=(--generate-notes --target "$GITHUB_SHA") + if [[ "$version" == *-* ]]; then + release_flags+=(--prerelease --latest=false) + else + release_flags+=(--latest) + fi + gh release create "v${version}" \ + --title "Release v${version}" \ + "${release_flags[@]}" \ + release-assets/* # Mirror Tableau TDVT after release (non-blocking). tdvt: name: "Tableau TDVT" - needs: release - if: ${{ needs.release.result == 'success' && startsWith(github.repository, 'ClickHouse/') }} + needs: publish + if: ${{ needs.publish.result == 'success' && startsWith(github.repository, 'ClickHouse/') }} runs-on: "ubuntu-latest" timeout-minutes: 150 # the TDVT matrix (latest/head/cloud, single-threaded) runs ~1h; allow for a queue steps: diff --git a/.gitignore b/.gitignore index b5cc91a03..fe04fb8ce 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,7 @@ jdbc-v2/src/main/antlr4/com/clickhouse/jdbc/internal/ClickHouseLexer.tokens # test configuration **/test/resources/test.properties performance/jmh-simple-results.json +performance/results/ *.csv *.sql *.json diff --git a/AGENTS.md b/AGENTS.md index 8660a32a4..1c5c9679b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,7 +40,7 @@ Prefer targeted Maven commands over full-repo runs. Also compile examples or packaging modules (e.g. `packages/clickhouse-jdbc-all`) when a change affects examples, packaging, public APIs, or other user-facing behavior — running the touched module's tests alone isn't enough. -Run benchmarks under `clickhouse-benchmark` when a change touches a performance-sensitive path (see +Run benchmarks under `performance` when a change touches a performance-sensitive path (see `docs/ai-review.md`); this is optional and only worth doing when performance is actually in question. Avoid broad dependency, formatting, or unrelated cleanup churn unless required by the task. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 80ab28bdc..05c1a0623 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -214,11 +214,10 @@ mvn -DskipTests -Pnative clean package Run benchmarks when the change may affect performance: ```bash -cd clickhouse-benchmark -mvn clean package +mvn -DskipTests -pl packages/clickhouse-jdbc-all -am install +mvn -f performance/pom.xml clean package -# single thread mode -java -DdbHost=localhost -jar target/benchmarks.jar -t 1 \ - -p client=clickhouse-jdbc -p connection=reuse \ - -p statement=prepared -p type=default Query.selectInt8 +# list available benchmarks +java -cp performance/target/jmh-benchmark-0.1-SNAPSHOT-jar-with-dependencies.jar \ + org.openjdk.jmh.Main -l ``` diff --git a/README.md b/README.md index a70c8cddf..5c06e03cb 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Nightly Builds: https://s01.oss.sonatype.org/content/repositories/snapshots/com/ ### Examples -[Begin-with Usage Examples](../../tree/main/examples/client-v2) +[Begin-with Usage Examples](examples/client-v2) [Spring Demo Service](https://github.com/ClickHouse/clickhouse-java/tree/main/examples/demo-spring-service) @@ -105,7 +105,7 @@ Nightly Builds: https://s01.oss.sonatype.org/content/repositories/snapshots/com/ ### Examples -See [JDBC examples](../../tree/main/examples/jdbc) +See [JDBC examples](examples/jdbc) ## R2DBC Driver diff --git a/clickhouse-data/src/test/java/com/clickhouse/data/format/BinaryStreamUtilsTest.java b/clickhouse-data/src/test/java/com/clickhouse/data/format/BinaryStreamUtilsTest.java index 4af1887b9..3c7ffb3d7 100644 --- a/clickhouse-data/src/test/java/com/clickhouse/data/format/BinaryStreamUtilsTest.java +++ b/clickhouse-data/src/test/java/com/clickhouse/data/format/BinaryStreamUtilsTest.java @@ -2,6 +2,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.EOFException; import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; @@ -1963,4 +1964,19 @@ public void testVarInt() throws IOException { Assert.assertEquals(out.toByteArray(), new byte[] { (byte) -128, (byte) 1 }); Assert.assertEquals(BinaryStreamUtils.readVarInt(generateInput(-128, 1)), 128); } + + @DataProvider(name = "truncatedVarInts") + private Object[][] truncatedVarInts() { + return new Object[][] { + {new byte[0]}, + {new byte[] {(byte) 0x80}}, + {new byte[] {(byte) 0x80, (byte) 0x80, (byte) 0x80}}, + }; + } + + @Test(dataProvider = "truncatedVarInts", groups = { "unit" }) + public void testTruncatedVarIntIsRejected(byte[] bytes) { + Assert.expectThrows(EOFException.class, + () -> BinaryStreamUtils.readVarInt(new ByteArrayInputStream(bytes))); + } } diff --git a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java index 07f8e4f54..0715b7f09 100644 --- a/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java +++ b/client-v2/src/main/java/com/clickhouse/client/api/internal/HttpAPIClientHelper.java @@ -33,6 +33,7 @@ import org.apache.hc.client5.http.impl.classic.RequestFailedException; import org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager; import org.apache.hc.client5.http.impl.io.DefaultManagedHttpClientConnection; +import org.apache.hc.client5.http.impl.io.DefaultManagedHttpClientConnection; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; import org.apache.hc.client5.http.io.HttpClientConnectionManager; @@ -105,6 +106,10 @@ import java.net.URISyntaxException; import java.net.URLEncoder; import java.net.UnknownHostException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Base64; @@ -1374,8 +1379,24 @@ public void bind(SSLSocket sslSocket, Socket socket) throws IOException { } public static class MeteredManagedHttpClientConnectionFactory extends NoWireLoggingConnectionFactory { + private static final AtomicLong CONNECTION_COUNTER = new AtomicLong(); + + private final Http1Config http1Config; + private final CharCodingConfig charCodingConfig; + private final DefaultHttpResponseParserFactory responseParserFactory; + private final boolean collectMetrics; public MeteredManagedHttpClientConnectionFactory(Http1Config http1Config, CharCodingConfig charCodingConfig, DefaultHttpResponseParserFactory defaultHttpResponseParserFactory) { + this(http1Config, charCodingConfig, defaultHttpResponseParserFactory, true); + } + + private MeteredManagedHttpClientConnectionFactory(Http1Config http1Config, CharCodingConfig charCodingConfig, + DefaultHttpResponseParserFactory defaultHttpResponseParserFactory, + boolean collectMetrics) { super(http1Config, charCodingConfig, defaultHttpResponseParserFactory); + this.http1Config = http1Config; + this.charCodingConfig = charCodingConfig; + this.responseParserFactory = defaultHttpResponseParserFactory; + this.collectMetrics = collectMetrics; } ConcurrentLinkedQueue times = new ConcurrentLinkedQueue<>(); @@ -1385,10 +1406,36 @@ public MeteredManagedHttpClientConnectionFactory(Http1Config http1Config, CharCo public ManagedHttpClientConnection createConnection(Socket socket) throws IOException { long startT = System.currentTimeMillis(); try { - return super.createConnection(socket); + CharsetDecoder charDecoder = null; + CharsetEncoder charEncoder = null; + Charset charset = charCodingConfig.getCharset(); + if (charset != null) { + CodingErrorAction malformedInputAction = charCodingConfig.getMalformedInputAction() != null + ? charCodingConfig.getMalformedInputAction() : CodingErrorAction.REPORT; + CodingErrorAction unmappableInputAction = charCodingConfig.getUnmappableInputAction() != null + ? charCodingConfig.getUnmappableInputAction() : CodingErrorAction.REPORT; + charDecoder = charset.newDecoder() + .onMalformedInput(malformedInputAction) + .onUnmappableCharacter(unmappableInputAction); + charEncoder = charset.newEncoder() + .onMalformedInput(malformedInputAction) + .onUnmappableCharacter(unmappableInputAction); + } + ManagedHttpClientConnection connection = new WireLoggingDisabledManagedHttpClientConnection( + "http-outgoing-" + CONNECTION_COUNTER.getAndIncrement(), + charDecoder, + charEncoder, + http1Config, + responseParserFactory); + if (socket != null) { + connection.bind(socket); + } + return connection; } finally { - long endT = System.currentTimeMillis(); - times.add(endT - startT); + if (collectMetrics) { + long endT = System.currentTimeMillis(); + times.add(endT - startT); + } } } @@ -1406,6 +1453,39 @@ public long getTime() { } } + private static final class WireLoggingDisabledManagedHttpClientConnection + extends DefaultManagedHttpClientConnection { + private WireLoggingDisabledManagedHttpClientConnection( + String id, + CharsetDecoder charDecoder, + CharsetEncoder charEncoder, + Http1Config http1Config, + DefaultHttpResponseParserFactory responseParserFactory) { + super( + id, + charDecoder, + charEncoder, + http1Config, + DefaultContentLengthStrategy.INSTANCE, + DefaultContentLengthStrategy.INSTANCE, + NoResponseOutOfOrderStrategy.INSTANCE, + new DefaultHttpRequestWriterFactory(http1Config), + responseParserFactory); + } + + @Override + public void bind(Socket socket) throws IOException { + // Apache's implementation replaces SocketHolder with its raw-byte logging wrapper when the wire + // logger is DEBUG-enabled. Bind the plain holder directly so logger configuration cannot enable it. + super.bind(new SocketHolder(socket)); + } + + @Override + public void bind(SSLSocket sslSocket, Socket socket) throws IOException { + super.bind(new SocketHolder(sslSocket, socket)); + } + } + public static class CustomSSLConnectionFactory extends SSLConnectionSocketFactory { private final SNIHostName defaultSNI; diff --git a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java index 0f46013b7..ccabb8314 100644 --- a/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java +++ b/client-v2/src/test/java/com/clickhouse/client/api/data_formats/internal/BinaryStreamReaderTests.java @@ -51,6 +51,45 @@ public void testCachedByteAllocator() { } } + @DataProvider(name = "invalidVarInts") + private Object[][] invalidVarInts() { + return new Object[][] { + {"truncated", new byte[] {(byte) 0x80}, null}, + {"unterminated", new byte[] { + (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80 + }, "too large for int"}, + {"int overflow", new byte[] { + (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x80, (byte) 0x08 + }, "too large for int"}, + }; + } + + @Test(dataProvider = "invalidVarInts") + public void testInvalidVarIntIsRejected(String label, byte[] bytes, String expectedMessage) { + IOException ex = Assert.expectThrows(IOException.class, + () -> BinaryStreamReader.readVarInt(new ByteArrayInputStream(bytes))); + if (expectedMessage != null) { + Assert.assertTrue(ex.getMessage().contains(expectedMessage), label + ": " + ex.getMessage()); + } + } + + @DataProvider(name = "truncatedLengthPrefixedValues") + private Object[][] truncatedLengthPrefixedValues() { + return new Object[][] { + {"String payload", "String", new byte[] {2, 'a'}}, + {"Array fixed-width element", "Array(Int32)", new byte[] {2, 1, 0, 0, 0}}, + {"Array variable-width element", "Array(String)", new byte[] {1, 2, 'a'}}, + }; + } + + @Test(dataProvider = "truncatedLengthPrefixedValues") + public void testTruncatedLengthPrefixedValueIsRejected(String label, String typeName, byte[] bytes) { + BinaryStreamReader reader = qbitReader(bytes); + IOException ex = Assert.expectThrows(IOException.class, + () -> reader.readValue(ClickHouseColumn.of("v", typeName))); + Assert.assertNotNull(ex, label); + } + @Test(dataProvider = "dateTestData") void readDateZonedDateTimeNoTimeZone(ZonedDateTime zdt, ZoneId writeTZ, ZoneId readTZ, ZonedDateTime expectedZDT) throws IOException diff --git a/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpWireLoggingTest.java b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpWireLoggingTest.java new file mode 100644 index 000000000..3265d6985 --- /dev/null +++ b/client-v2/src/test/java/com/clickhouse/client/api/internal/HttpWireLoggingTest.java @@ -0,0 +1,110 @@ +package com.clickhouse.client.api.internal; + +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.ClientConfigProperties; +import com.clickhouse.client.api.enums.Protocol; +import com.github.tomakehurst.wiremock.WireMockServer; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import org.apache.hc.core5.http.HttpHeaders; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.concurrent.TimeUnit; + +public class HttpWireLoggingTest { + private static final String USERNAME = "wire-user-canary"; + private static final String PASSWORD = "wire-password-canary"; + private static final String BASIC_TOKEN = Base64.getEncoder().encodeToString( + (USERNAME + ":" + PASSWORD).getBytes(StandardCharsets.UTF_8)); + private static final String BEARER_TOKEN = "wire-bearer-token-canary"; + private static final String CUSTOM_HEADER = "X-Sensitive-Canary"; + private static final String CUSTOM_VALUE = "wire-custom-header-canary"; + private static final String RESPONSE_VALUE = "wire-response-canary"; + + @DataProvider(name = "connectionManagersAndAuthentication") + public Object[][] connectionManagersAndAuthentication() { + return new Object[][] { + {true, false}, + {true, true}, + {false, false}, + {false, true} + }; + } + + @Test(dataProvider = "connectionManagersAndAuthentication") + public void testSensitiveValuesAreNotWrittenToWireLog(boolean pooled, boolean bearer) throws Exception { + WireMockServer server = new WireMockServer(WireMockConfiguration.options().dynamicPort()); + server.start(); + try { + String expectedAuthorization = bearer + ? "Bearer " + BEARER_TOKEN + : "Basic " + BASIC_TOKEN; + server.stubFor(WireMock.post(WireMock.anyUrl()) + .withHeader(HttpHeaders.AUTHORIZATION, WireMock.equalTo(expectedAuthorization)) + .withHeader(CUSTOM_HEADER, WireMock.equalTo(CUSTOM_VALUE)) + .willReturn(WireMock.aResponse() + .withStatus(200) + .withBody(RESPONSE_VALUE))); + + Assert.assertTrue(LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME).isDebugEnabled()); + Assert.assertTrue(LoggerFactory.getLogger("org.apache.hc.client5.http.wire").isDebugEnabled()); + String logs = captureStdErr(() -> { + Client.Builder builder = new Client.Builder() + .addEndpoint(Protocol.HTTP, "localhost", server.port(), false) + .setOption(ClientConfigProperties.CONNECTION_POOL_ENABLED.getKey(), String.valueOf(pooled)) + .compressServerResponse(false) + .httpHeader(CUSTOM_HEADER, CUSTOM_VALUE); + if (bearer) { + builder.useBearerTokenAuth(BEARER_TOKEN); + } else { + builder.setUsername(USERNAME).setPassword(PASSWORD); + } + try (Client client = builder.build()) { + client.execute("SELECT 1").get(5, TimeUnit.SECONDS); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + server.verify(WireMock.postRequestedFor(WireMock.anyUrl())); + Assert.assertFalse(logs.contains("org.apache.hc.client5.http.wire"), logs); + Assert.assertFalse(logs.contains(USERNAME), logs); + Assert.assertFalse(logs.contains(PASSWORD), logs); + Assert.assertFalse(logs.contains(BASIC_TOKEN), logs); + Assert.assertFalse(logs.contains(BEARER_TOKEN), logs); + Assert.assertFalse(logs.contains(expectedAuthorization), logs); + Assert.assertFalse(logs.contains(CUSTOM_VALUE), logs); + Assert.assertFalse(logs.contains(RESPONSE_VALUE), logs); + Assert.assertTrue(logs.contains("client compression:"), logs); + Assert.assertTrue(logs.contains(pooled + ? "PoolingHttpClientConnectionManager" + : "BasicHttpClientConnectionManager"), logs); + Assert.assertTrue(LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME).isDebugEnabled()); + Assert.assertTrue(LoggerFactory.getLogger("org.apache.hc.client5.http.wire").isDebugEnabled()); + } finally { + server.stop(); + } + } + + private static synchronized String captureStdErr(Runnable action) { + PrintStream original = System.err; + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + PrintStream capture = new PrintStream(buffer, true); + System.setErr(capture); + try { + action.run(); + } finally { + capture.flush(); + System.setErr(original); + } + return new String(buffer.toByteArray(), StandardCharsets.UTF_8); + } +} diff --git a/client-v2/src/test/resources/simplelogger.properties b/client-v2/src/test/resources/simplelogger.properties index f647ee22d..e98cf61b8 100644 --- a/client-v2/src/test/resources/simplelogger.properties +++ b/client-v2/src/test/resources/simplelogger.properties @@ -1,9 +1,9 @@ -org.slf4j.simpleLogger.defaultLogLevel=info -org.slf4j.simpleLogger.log.com.clickhouse.client=info -# Apache HttpClient logs every request header, including Authorization, at DEBUG. -# Keep these loggers at INFO so that credentials never reach the test output. +org.slf4j.simpleLogger.defaultLogLevel=debug +org.slf4j.simpleLogger.log.com.clickhouse.client=debug +# Parsed Apache header diagnostics can include Authorization, so keep that category at INFO. +# Raw wire logging is deliberately DEBUG to exercise client-v2's connection-level suppression. org.slf4j.simpleLogger.log.org.apache.hc.client5.http.headers=info -org.slf4j.simpleLogger.log.org.apache.hc.client5.http.wire=info +org.slf4j.simpleLogger.log.org.apache.hc.client5.http.wire=debug org.slf4j.simpleLogger.showDateTime=true org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z org.slf4j.simpleLogger.showThreadName=true diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..e65aa504a --- /dev/null +++ b/docs/README.md @@ -0,0 +1,22 @@ +# Documentation layers + +Use the narrowest authoritative layer for a change: + +1. `docs/clickhouse-docs/*.mdx` is the source for the published ClickHouse website pages. +2. `docs/integration-*.md` provides repository-local integration guidance and links to the published reference for exhaustive details. +3. `docs/features.md` is the compatibility contract for stable `client-v2` and `jdbc-v2` behavior. +4. Module and root `README.md` files are landing pages and quick starts, not exhaustive references. +5. Public Javadocs own symbol-level API behavior. + +Release-specific compatibility and migration notes belong in `docs/releases/`. The +[`migration-helpers`](../migration-helpers/) module contains utilities for translating legacy client and JDBC +configuration. + +When documentation changes, run: + +```shell +python3 scripts/check-relative-links.py +``` + +The checker validates relative links in the maintained root, contribution, pull-request-template, and `docs/` +Markdown sources. External URLs and website-root routes are intentionally excluded. diff --git a/docs/clickhouse-docs/r2dbc.mdx b/docs/clickhouse-docs/r2dbc.mdx index 452ee9ff8..ffeccf445 100644 --- a/docs/clickhouse-docs/r2dbc.mdx +++ b/docs/clickhouse-docs/r2dbc.mdx @@ -23,20 +23,14 @@ integration: ```xml com.clickhouse - clickhouse-r2dbc - 0.7.1 - - all - - - * - * - - + 0.11.0 ``` +The current artifact implements R2DBC SPI 1.0. Use the optional `all` classifier only when a self-contained shaded +artifact is required; managed builds should use the standard artifact and its declared dependencies. + ### Connect to ClickHouse {#connect-to-clickhouse} ```java showLineNumbers diff --git a/docs/features.md b/docs/features.md index 1bd3588ca..a621d51cb 100644 --- a/docs/features.md +++ b/docs/features.md @@ -74,7 +74,7 @@ Compatibility-sensitive traits: - JDBC URL parsing: Accepts `jdbc:clickhouse:` and `jdbc:ch:` URLs with host, port, optional HTTP path, optional database, and query parameters. - SSL URL support: Supports HTTPS connections through URL and property configuration, including default protocol and port handling. The `ssl_mode` property selects the verification strictness (`disabled`, `trust`, `verify_ca`, `strict`); values are case-insensitive and the traditional JDBC value `none` is accepted as an alias for `trust`. Root CA and client certificate/key may be supplied as a file path or as inline PEM content. The `ssl_cipher_suites` property (a comma-separated list) restricts the negotiated TLS cipher suites and is forwarded to the underlying `client-v2` transport. - Custom SSL context via properties: A fully pre-built `javax.net.ssl.SSLContext` may be passed as a live object in the connection `Properties` under the `ssl_context` key (added with `Properties.put`, since it is not a string). It is forwarded to the underlying `client-v2` transport and used as is; trust/key material properties (e.g. `sslrootcert`, `sslcert`) cannot be combined with it and are rejected, while `ssl_mode` then only controls hostname verification. This supports diskless, in-memory TLS material behind connection pools that only expose `java.util.Properties`. -- Driver and client properties: Separates JDBC-specific properties from passthrough client options used by the underlying `client-v2` transport. A passthrough client option that has no default value is reported with a `null` default by `Driver#getPropertyInfo` and is not sent to the transport unless the connection sets it — this is the case for the socket buffer properties `socket_rcvbuf` and `socket_sndbuf`, which are left to the operating system. +- Driver and client properties: Separates JDBC-specific properties from passthrough client options used by the underlying `client-v2` transport. `Driver#getPropertyInfo` reports a non-empty description for every known JDBC and client property. A passthrough client option that has no default value is reported with a `null` default and is not sent to the transport unless the connection sets it — this is the case for the socket buffer properties `socket_rcvbuf` and `socket_sndbuf`, which are left to the operating system. - Metrics recording: The `jdbc_metrics_recorder` property names the `client-v2` `MetricsRecorder` implementation the connection reports the metrics of its operations to. A JDBC URL carries strings only, so the value is the fully-qualified class name of a recorder with a public no-argument constructor, and each connection creates its own instance and registers it with its own client. A class that is not on the classpath, does not implement `MetricsRecorder`, or cannot be instantiated is rejected when the connection is created. When the property is not set, nothing is recorded. - DataSource support: Provides a JDBC `DataSource` implementation backed by the same driver configuration model. - Connection lifecycle: Supports connection close, validity checks, ping-based health checks, and network timeout management. `Connection#getNetworkTimeout()` returns the underlying client's current network timeout, initially configured by `socket_timeout`, and `setNetworkTimeout` updates that client-level value so subsequent requests from existing as well as new statements use it. When a connection pool saves the timeout, sets another one, and restores the saved one, the configured `socket_timeout` stays in effect. `setNetworkTimeout(executor, 0)` disables the socket timeout. diff --git a/docs/integration-client.md b/docs/integration-client.md index 4e1d291bb..eb85a6db0 100644 --- a/docs/integration-client.md +++ b/docs/integration-client.md @@ -693,7 +693,7 @@ public void streamEvents(Client client) throws Exception { The `QueryResponse` object also provides direct access to the underlying input stream via `response.getInputStream()`. This is incredibly useful when you want to read text formats using third-party libraries (like Jackson for JSON or OpenCSV for CSV). -Because of this direct stream access, it is also possible to stream columnar formats directly into memory structures like **Apache Arrow**. (See our [examples directory](../examples/src/main/java/com/clickhouse/examples) for reference implementations). +Because of this direct stream access, it is also possible to stream columnar formats directly into memory structures like **Apache Arrow**. (See the [client examples](../examples/) for reference implementations). ### Parameterized queries diff --git a/docs/integration-index.md b/docs/integration-index.md index 047476b53..2ae9c194d 100644 --- a/docs/integration-index.md +++ b/docs/integration-index.md @@ -45,6 +45,9 @@ will cause problems. They are usually found only after something is broken. We highlight breaking and significant changes in release notes. See also release specific migration guides in `docs/releases/` in this repo. Migration guide for V1-to-V2 is in main documentation separetly for [Client](https://clickhouse.com/docs/integrations/language-clients/java/client#migration_guide) and [JDBC](https://clickhouse.com/docs/integrations/language-clients/java/jdbc#migration-guide). +For automated configuration conversion, see the +[`migration-helpers`](../migration-helpers/) module and its +[`ConfigurationMigrationHelper`](../migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java). Minor versions may have many changes, and some of them need a preview from your side. In this case, we release an `-rc` version and let you know. This version is **only** for preview and not for production use. The preview lasts for a few weeks to let everyone send their feedback. If changes are needed, we will release a new `-rc` and repeat the cycle. diff --git a/docs/integration-jdbc.md b/docs/integration-jdbc.md index efecf5c63..f6075a7d7 100644 --- a/docs/integration-jdbc.md +++ b/docs/integration-jdbc.md @@ -2,7 +2,7 @@ This guide is a **step-by-step integration path** for the **JDBC Driver V2** (`jdbc-v2`, published as `com.clickhouse:clickhouse-jdbc`). It is written to be used as context for building an application or a downstream integration spec: each step states the decisions you must make, how to configure them, and the common pitfalls to avoid. -**Prerequisites:** Read [integration-common.md](integration-common.md) to understand the JDBC trade-offs before committing to this path. +**Prerequisites:** Read the [integration overview](integration-index.md) to understand the JDBC trade-offs before committing to this path. > **Architecture in one line.** Every JDBC `Connection` wraps a `client-v2` [`Client`](../client-v2/src/main/java/com/clickhouse/client/api/Client.java) internally. JDBC is not a separate protocol stack — it is a `java.sql.*` façade over the Java Client. @@ -775,7 +775,8 @@ public int getParameterCount(PreparedStatement ps) throws SQLException { ### Type mapping -The driver maps ClickHouse types to JDBC types (e.g. `UInt64` → `NUMERIC`, `Tuple` → `STRUCT`); see [type_mapping.md](../type_mapping.md). Override defaults: +The driver maps ClickHouse types to JDBC types (e.g. `UInt64` → `NUMERIC`, `Tuple` → `STRUCT`); see the +[JDBC type mapping reference](clickhouse-docs/jdbc.mdx#jdbc-type-mapping). Override defaults: ```java @@ -840,6 +841,10 @@ Key JDBC-specific properties (see [`DriverProperties`](../jdbc-v2/src/main/java/ | `jdbc_metadata_use_show_statements` | `true` | `DatabaseMetaData` schemas, tables and columns via `SHOW` / `DESCRIBE` instead of `system.*` tables | | `jdbc_multi_dimensional_array_element_type` | `true` | `Array#getBaseType()` / `getBaseTypeName()` of a multi-dimensional array return the next array level; `false` returns the innermost non-array type | +Tools and connection pools can discover both JDBC and forwarded client options through +`java.sql.Driver#getPropertyInfo`. The returned entries include defaults and descriptions; the enums linked above +remain the source for property keys used in application code. + ## Observability & Monitoring @@ -911,10 +916,11 @@ For a complete sample demonstrating JDBC driver setup and telemetry in a Spring **Related documents in this repository:** -- [integration-common.md](integration-common.md) — choosing JDBC vs Client +- [integration-index.md](integration-index.md) — choosing JDBC vs Client - [integration-client.md](integration-client.md) — Java Client integration path - [integration-ops.md](integration-ops.md) — operations and observability guide - [authentication.md](authentication.md) — full authentication and TLS reference - [features.md](features.md) — compatibility contract -- [type_mapping.md](../type_mapping.md) — JDBC type mapping recommendations -- [integration-testing.md](integration-testing.md) — integration testing recommendations and practices \ No newline at end of file +- [JDBC type mapping](clickhouse-docs/jdbc.mdx#jdbc-type-mapping) — JDBC type mapping recommendations +- [integration-testing.md](integration-testing.md) — integration testing recommendations and practices +- [migration-helpers](../migration-helpers/) — helpers for converting legacy client and JDBC configuration diff --git a/docs/releasing.md b/docs/releasing.md new file mode 100644 index 000000000..c844bfdf8 --- /dev/null +++ b/docs/releasing.md @@ -0,0 +1,88 @@ +# Maintainer release runbook + +This runbook covers release candidates (RCs) and general-availability (GA) +releases. The release workflow always builds and verifies first. Publishing is a +separate, explicit workflow-dispatch choice. + +## Prepare the release commit + +1. Start from an up-to-date, protected release branch and confirm required CI is + green. +2. Choose a version: + - RC: `MAJOR.MINOR.PATCH-rcN`, for example `0.12.0-rc1`. + - GA: `MAJOR.MINOR.PATCH`, for example `0.12.0`. +3. Run `./release.sh `. This updates `VERSION`, the root development + revision, examples, and performance projects to `-SNAPSHOT`. +4. Run `python3 scripts/release/validate_version.py`. +5. Review release notes and commit the version changes. The workflow releases + the exact commit selected in GitHub Actions; do not publish from an + unreviewed commit. + +`VERSION` contains the release version without `-SNAPSHOT`. In source control, +the root Maven `revision` must be exactly `VERSION-SNAPSHOT`. The workflow +passes `-Drevision=` for release builds instead of editing the POM. + +## Dry run (required) + +1. Open the **Release** workflow in GitHub Actions. +2. Select the release commit or branch. +3. Leave **Publish to Maven Central and create the GitHub release** unchecked. +4. Run the workflow and download the `release-` artifact. +5. Confirm: + - unit tests and the Maven `verify` lifecycle passed; + - `SHA256SUMS` lists every attached asset; + - main JARs exist for all modules, including + `packages/clickhouse-jdbc-all`; + - `bom.json` and `bom.xml` are present and describe the complete reactor; + - the version and commit are the intended release. + +The dry run does not use publishing credentials and does not create a tag, +GitHub release, or Maven Central deployment. + +## Publish + +1. Re-run **Release** for the same reviewed commit. +2. Check **Publish to Maven Central and create the GitHub release**. +3. Approve the protected GitHub `release` environment if one is configured. +4. Wait for the verify step to pass. Only then does the workflow sign and + publish the reactor to Maven Central. +5. Confirm the Maven Central deployment is published before announcing the + release. +6. Confirm the GitHub release: + - an RC is marked as a prerelease and is not `latest`; + - a GA is not marked as a prerelease and is marked `latest`; + - the tag is `v` and points to the selected workflow commit; + - release assets include `clickhouse-jdbc-all`, both CycloneDX SBOM formats, + and `SHA256SUMS`. + +If Maven Central succeeds but GitHub release creation fails, do not rerun the +publish step blindly. Verify Central state first, then create the GitHub release +from the downloaded dry-run inventory or rerun only after confirming that the +Central plugin will not create a duplicate deployment. + +## API compatibility gate + +The **API compatibility** workflow finds the newest tag matching +`vMAJOR.MINOR.PATCH`; prerelease tags are deliberately excluded. Japicmp then +checks public binary compatibility for reactor JARs against that GA version. +The shaded `clickhouse-jdbc-all` distribution is excluded because its source +modules are checked directly. + +Missing classes from optional dependencies are ignored to avoid environment +noise. Any intentional binary break still requires maintainer review and an +explicit compatibility decision; do not weaken the gate to accommodate an +unreviewed break. + +## Repository and external controls + +Repository automation cannot enforce organization settings. Maintainers should +configure: + +- a protected `release` environment with required reviewers; +- least-privilege Maven Central and GPG secrets; +- branch protection requiring build and API compatibility checks; +- GitHub artifact and log retention appropriate for release audit needs; +- Maven Central account protections and recovery procedures. + +Trusted publishing/provenance attestations should be added when the Maven +Central and GitHub organization configuration supports them end to end. diff --git a/docs/serde-ownership-matrix.md b/docs/serde-ownership-matrix.md new file mode 100644 index 000000000..23e4eded2 --- /dev/null +++ b/docs/serde-ownership-matrix.md @@ -0,0 +1,55 @@ +# Serialization ownership matrix + +This matrix records which implementation owns each serialization path. It is a +maintenance map, not a promise that every ClickHouse type is supported by every +format. Public support remains documented in `docs/features.md`. + +## Stack and format ownership + +| Stack | Format | Read owner | Write owner | Type parsing / conversion owner | +|---|---|---|---|---| +| `clickhouse-data` (legacy v1) | RowBinary | `ClickHouseRowBinaryProcessor` and `BinaryDataProcessor` | `ClickHouseRowBinaryProcessor` and `BinaryDataProcessor` | `ClickHouseColumn`, `ClickHouseValue`, and value classes | +| `clickhouse-data` (legacy v1) | TabSeparated / text | `ClickHouseTabSeparatedProcessor` and `TextDataProcessor` | `ClickHouseTabSeparatedProcessor` and `TextDataProcessor` | `ClickHouseColumn`, `ClickHouseValue`, and value classes | +| `client-v2` | RowBinary variants | `RowBinaryFormatReader` / `RowBinaryWithNames*FormatReader` delegate values to `BinaryStreamReader` | `RowBinaryFormatWriter` delegates values to `SerializerUtils`; `RowBinaryFormatSerializer` owns primitive framing | `ClickHouseColumn` supplies metadata; `BinaryStreamReader` and `SerializerUtils` own runtime conversion | +| `client-v2` | Native | `NativeFormatReader` owns block/column layout and delegates scalar values to `BinaryStreamReader` | Not an insert format | `NativeFormatReader` for columnar framing; `BinaryStreamReader` for values | +| `client-v2` | JSONEachRow | `JSONEachRowFormatReader` and the selected `JsonParser` | Server-side HTTP text path; no matching client serializer | `ProcessParser` and `ValueConverters` | + +Code that fixes a v2 result decoding problem belongs in `BinaryStreamReader` +(or `NativeFormatReader` when the problem is columnar framing), not in the +legacy `ClickHouseColumn` value path. Formatting a value for a request parameter +belongs in the relevant converter, not in HTTP transport code. + +## Type / format / operation matrix + +Legend: **owned** means there is a dedicated implementation path; **delegated** +means wrapper semantics are composed from child columns; **limited** means the +path is intentionally partial or version-dependent; **n/a** means the operation +is not provided by that stack/format. + +| Type family | Legacy RowBinary read/write | v2 RowBinary read | v2 RowBinary write | v2 Native read | Deterministic coverage anchor | +|---|---|---|---|---|---| +| Fixed-width numeric, Bool, Decimal | owned | owned | owned | owned | `BinaryStreamUtilsTest`, `SerializerUtilsTest`, `BinaryStreamReaderTests` | +| String, FixedString, UUID, IP | owned | owned | owned | owned | binary/string and nested nullable cases in `SerializerUtilsTest` | +| Date, DateTime, Time64 | owned | owned | owned | owned | timezone cases in `BinaryStreamReaderTests` | +| Nullable | delegated to nested serializer | delegated; RowBinary marker is per value | delegated; marker emitted before nested value | delegated; null map is read separately | middle-of-schema cases in `SerializerUtilsTest` | +| Array, Map, Tuple, Nested | delegated recursively | delegated recursively | delegated recursively | format-specific container framing plus delegated values | nested nullable and trailing fixed-width guards in `SerializerUtilsTest` | +| Enum / LowCardinality | owned | owned | owned | owned, with Native dictionary framing | enum width/alignment cases in `BinaryStreamReaderTests` | +| SimpleAggregateFunction | delegated to underlying type | delegated | delegated | delegated | middle-of-schema tests in `SerializerUtilsTest` and `BinaryStreamReaderTests` | +| Variant | n/a in the legacy value stack | owned, including null discriminator | owned; runtime alternative selection | owned, version-dependent server metadata | null and Dynamic-wrapped Variant tests in `BinaryStreamReaderTests`; integration coverage in `DataTypeTests` | +| Dynamic / JSON | n/a in the legacy value stack | owned, server-version dependent | limited and server-version dependent | owned, including runtime type metadata | dynamic type-encoding alignment cases in `BinaryStreamReaderTests` | +| Point/Ring/Polygon families | owned for concrete geometry types | owned for concrete types | owned for concrete types | owned | `ClickHouseRowBinaryProcessorTest` and geometry integration tests | +| Geometry sum type | n/a | owned as Variant-like data | owned, server-version dependent | owned, server-version dependent | geometry integration tests in `DataTypeTests` | +| QBit | n/a | owned | owned | owned with bit-plane framing | checked-in Native golden bytes in `BinaryStreamReaderTests` | +| AggregateFunction state | limited to supported bitmap/state shapes | limited | not generally supported | limited | explicit skipped/negative cases; no general round-trip claim | + +## Test rules and known gaps + +- Put nullable or variable-width values in the middle of a schema and keep a + fixed-width field after them; assert the complete row. +- Exercise malformed length prefixes without allocating from untrusted sizes. +- Prefer the live v2 `BinaryStreamReader` path for result decoding regressions. +- Keep golden bytes only when their server version and stripped framing are + documented beside the fixture. +- General AggregateFunction states, all Variant alternative combinations, and + all server-version-specific Geometry encodings remain incomplete. Add focused + rows here when support or deterministic coverage is added. diff --git a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java index 056bf0742..55b7a0500 100644 --- a/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java +++ b/jdbc-v2/src/main/java/com/clickhouse/jdbc/internal/JdbcConfiguration.java @@ -12,6 +12,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; @@ -35,6 +37,9 @@ public class JdbcConfiguration { public static final Logger LOG = LoggerFactory.getLogger(JdbcConfiguration.class); + private static final String PROPERTY_DESCRIPTIONS_RESOURCE = + "com/clickhouse/jdbc/driver-property-descriptions.properties"; + private static final Properties PROPERTY_DESCRIPTIONS = loadPropertyDescriptions(); private static final String PREFIX_CLICKHOUSE = "jdbc:clickhouse:"; private static final String PREFIX_CLICKHOUSE_SHORT = "jdbc:ch:"; @@ -426,9 +431,9 @@ private void buildFinalProperties(Map urlProperties, Properties DriverPropertyInfo propertyInfo = propertyInfos.get(clientProp.getKey()); if (propertyInfo == null) { propertyInfo = new DriverPropertyInfo(clientProp.getKey(), clientProp.getDefaultValue()); - // TODO: read description from resource file propertyInfos.put(clientProp.getKey(), propertyInfo); } + propertyInfo.description = getPropertyDescription(clientProp.getKey()); } // Fill list of driver properties information, add not specified properties, @@ -439,11 +444,29 @@ private void buildFinalProperties(Map urlProperties, Properties propertyInfo = new DriverPropertyInfo(driverProp.getKey(), driverProp.getDefaultValue()); propertyInfos.put(driverProp.getKey(), propertyInfo); } + propertyInfo.description = getPropertyDescription(driverProp.getKey()); } listOfProperties = propertyInfos.values().stream().sorted(Comparator.comparing(o -> o.name)).collect(Collectors.toList()); } + private static Properties loadPropertyDescriptions() { + Properties descriptions = new Properties(); + ClassLoader classLoader = JdbcConfiguration.class.getClassLoader(); + try (InputStream input = classLoader.getResourceAsStream(PROPERTY_DESCRIPTIONS_RESOURCE)) { + if (input != null) { + descriptions.load(input); + } + } catch (IOException e) { + LOG.warn("Unable to load JDBC property descriptions", e); + } + return descriptions; + } + + private static String getPropertyDescription(String key) { + return PROPERTY_DESCRIPTIONS.getProperty(key, "ClickHouse configuration property."); + } + /** * Returns a list of driver property information. * @return a list of driver property information for the driver diff --git a/jdbc-v2/src/main/resources/com/clickhouse/jdbc/driver-property-descriptions.properties b/jdbc-v2/src/main/resources/com/clickhouse/jdbc/driver-property-descriptions.properties new file mode 100644 index 000000000..940aeeae4 --- /dev/null +++ b/jdbc-v2/src/main/resources/com/clickhouse/jdbc/driver-property-descriptions.properties @@ -0,0 +1,89 @@ +session_db_roles=Database roles enabled for the session. +clickhouse_setting_log_comment=Log comment sent as a ClickHouse server setting. +http_use_basic_auth=Use HTTP Basic authentication for user credentials. +user=ClickHouse user name. +password=ClickHouse user password. +max_open_connections=Maximum number of active connections in the HTTP connection pool. +http_keep_alive_timeout=HTTP keep-alive timeout override in milliseconds. +use_server_time_zone=Use the server time zone for date and time values. +use_time_zone=Time zone used for date and time values. +server_version=Server version override used by the client. +server_time_zone=Server time zone override used by the client. +async=Run client operations asynchronously. +connection_ttl=Maximum connection lifetime in milliseconds. +connection_timeout=Timeout for establishing a connection in milliseconds. +connection_reuse_strategy=Strategy used to reuse pooled HTTP connections. +socket_timeout=Socket operation timeout in milliseconds. +socket_rcvbuf=Socket receive buffer size in bytes. +socket_sndbuf=Socket send buffer size in bytes. +socket_reuseaddr=Enable the SO_REUSEADDR socket option. +socket_keepalive=Enable the SO_KEEPALIVE socket option. +socket_tcp_nodelay=Enable the TCP_NODELAY socket option. +socket_linger=Socket linger timeout in seconds. +database=Default ClickHouse database. +compress=Enable compressed server responses. +decompress=Enable compression for client request bodies. +client.use_http_compression=Enable HTTP content compression. +compression.lz4.uncompressed_buffer_size=Uncompressed buffer size used by LZ4 compression. +compression.method=Compression algorithm used by the client. +disable_native_compression=Disable ClickHouse native compression. +proxy_type=Proxy type used for HTTP connections. +proxy_host=Proxy server host name. +proxy_port=Proxy server port. +proxy_user=Proxy authentication user name. +proxy_password=Proxy authentication password. +max_execution_time=Maximum server query execution time in seconds. +trust_store=Path to the TLS trust store. +key_store_type=TLS key store type. +ssl_key_store=Path to the TLS key store. +key_store_password=Password for the TLS key store. +ssl_key=Path to or PEM content of the TLS client private key. +sslrootcert=Path to or PEM content of the TLS root certificate. +sslcert=Path to or PEM content of the TLS client certificate. +ssl_mode=TLS certificate and host verification mode. +ssl_context=Application-provided javax.net.ssl.SSLContext object. +retry=Maximum number of client retry attempts. +format=ClickHouse input and output format. +max_threads_per_client=Maximum number of client worker threads. +query_id=Query identifier sent to ClickHouse. +client_network_buffer_size=Client network buffer size in bytes. +access_token=Access token used for authentication. +ssl_authentication=Enable TLS client-certificate authentication. +connection_pool_enabled=Enable the HTTP connection pool. +connection_request_timeout=Timeout for obtaining a pooled connection in milliseconds. +client_retry_on_failures=Failure categories that trigger a client retry. +client_name=Application name included in the HTTP User-Agent. +product_name=Deprecated alias for client_name. +bearer_token=Bearer token used for authentication. +app_compressed_data=Indicate that application-provided data is already compressed. +metrics_name=Name used for the HTTP connection-pool metrics group. +client.http.cookies_enabled=Enable storage and reuse of HTTP cookies. +client_allow_binary_reader_to_reuse_buffers=Allow binary readers to reuse preallocated buffers. +type_hint_mapping=Custom mapping from ClickHouse type hints to Java types. +binary_string_support=Preserve raw bytes for top-level String and FixedString values. +ssl_socket_sni=TLS Server Name Indication value. +client.http.use_form_request_for_query=Send query parameters in the HTTP request body. +json_disable_number_quoting=Disable quoting of JSON numeric values. +custom_settings_prefix=Prefix recognized for custom ClickHouse server settings. +ssl_cipher_suites=Comma-separated TLS cipher suites enabled for connections. +client_id_metadata=Ordered metadata entries included in the HTTP User-Agent. +jdbc_ignore_unsupported_values=Ignore unsupported JDBC methods and values instead of throwing. +jdbc_schema_term=Term used for schemas in JDBC metadata. +ssl=Use HTTPS for the JDBC connection. +default_query_settings=Default settings applied to JDBC query operations. +beta.row_binary_for_simple_insert=Use RowBinary for supported prepared-statement inserts. +jdbc_resultset_auto_close=Close the current result set before executing another statement. +jdbc_use_max_result_rows=Enforce Statement max rows with the server max_result_rows setting. +jdbc_sql_parser=SQL parser implementation used by the JDBC driver. +jdbc_query_id_generator=Supplier class used to generate query identifiers. +remember_last_set_roles=Remember roles selected by SET ROLE statements. +custom_http_params=Deprecated comma-separated custom HTTP parameters. +custom_settings=Deprecated comma-separated ClickHouse server settings. +use_server_time_zone_for_dates=Deprecated date time-zone option retained for compatibility. +http_connection_provider=Deprecated HTTP provider option retained for compatibility. +jdbc_cluster_name=Cluster used for cluster-wide statement cancellation. +jdbc_type_mappings=Custom ClickHouse-to-Java type mappings. +typeMappings=Deprecated alias for jdbc_type_mappings. +jdbc_json_parser_factory=JsonParserFactory implementation used for JSONEachRow results. +jdbc_metrics_recorder=MetricsRecorder implementation used for JDBC operations. +jdbc_metadata_use_show_statements=Use SHOW and DESCRIBE statements for JDBC metadata discovery. diff --git a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DriverTest.java b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DriverTest.java index 59f75883f..79e6c6485 100644 --- a/jdbc-v2/src/test/java/com/clickhouse/jdbc/DriverTest.java +++ b/jdbc-v2/src/test/java/com/clickhouse/jdbc/DriverTest.java @@ -88,6 +88,23 @@ public void testGetPropertyInfo(String url, Properties props, Map propertiesByName = new HashMap<>(); + for (DriverPropertyInfo property : properties) { + propertiesByName.put(property.name, property); + Assert.assertNotNull(property.description, property.name); + Assert.assertFalse(property.description.trim().isEmpty(), property.name); + Assert.assertNotEquals(property.description, "ClickHouse configuration property.", property.name); + } + + assertEquals(propertiesByName.get(ClientConfigProperties.USER.getKey()).description, + "ClickHouse user name."); + assertEquals(propertiesByName.get(DriverProperties.SQL_PARSER.getKey()).description, + "SQL parser implementation used by the JDBC driver."); + } + @DataProvider(name = "testGetPropertyInfoDP") public Object[][] testGetPropertyInfoDP() { return new Object[][]{ diff --git a/packages/clickhouse-jdbc-all/pom.xml b/packages/clickhouse-jdbc-all/pom.xml index e610e4829..d51f0e071 100644 --- a/packages/clickhouse-jdbc-all/pom.xml +++ b/packages/clickhouse-jdbc-all/pom.xml @@ -17,6 +17,11 @@ JDBC driver for ClickHouse https://github.com/ClickHouse/clickhouse-java/tree/main/clickhouse-jdbc-all + + + true + + com.clickhouse diff --git a/performance/Dockerfile.write-stream b/performance/Dockerfile.write-stream new file mode 100644 index 000000000..e8f30da51 --- /dev/null +++ b/performance/Dockerfile.write-stream @@ -0,0 +1,54 @@ +FROM eclipse-temurin:11-jdk AS jdk11 + +FROM maven:3.9.11-eclipse-temurin-17 AS build + +COPY --from=jdk11 /opt/java/openjdk /opt/java/openjdk-11 + +RUN mkdir -p /root/.m2 \ + && printf '%s\n' \ + '' \ + '' \ + ' jdk11/opt/java/openjdk-11' \ + ' jdk17/opt/java/openjdk' \ + '' \ + > /root/.m2/toolchains.xml + +WORKDIR /source +COPY . . + +ARG GIT_COMMIT=unknown +RUN GIT_COMMIT_ABBREV="$(printf '%.7s' "$GIT_COMMIT")" \ + && mvn -B -DskipTests \ + -Dmaven.gitcommitid.skip=true \ + -Dgit.commit.id.full="$GIT_COMMIT" \ + -Dgit.commit.id.abbrev="$GIT_COMMIT_ABBREV" \ + -pl packages/clickhouse-jdbc-all -am install +RUN mvn -B -DskipTests -pl packages/clickhouse-jdbc-all -am install +RUN mvn -B -DskipTests -f performance/pom.xml package + +FROM eclipse-temurin:17-jre + +ARG TARGETARCH +ARG ASYNC_PROFILER_VERSION=3.0 + +RUN apt-get update \ + && apt-get install --no-install-recommends -y ca-certificates curl \ + && case "${TARGETARCH}" in \ + amd64) profiler_arch=x64 ;; \ + arm64) profiler_arch=arm64 ;; \ + *) echo "Unsupported architecture: ${TARGETARCH}" >&2; exit 1 ;; \ + esac \ + && curl --fail --location --silent --show-error \ + "https://github.com/async-profiler/async-profiler/releases/download/v${ASYNC_PROFILER_VERSION}/async-profiler-${ASYNC_PROFILER_VERSION}-linux-${profiler_arch}.tar.gz" \ + | tar --extract --gzip --directory /opt \ + && mv "/opt/async-profiler-${ASYNC_PROFILER_VERSION}-linux-${profiler_arch}" /opt/async-profiler \ + && apt-get purge -y curl \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=build /source/performance/target/jmh-benchmark-0.1-SNAPSHOT-jar-with-dependencies.jar \ + /opt/benchmark/benchmarks.jar + +WORKDIR /results + +ENTRYPOINT ["java", "-cp", "/opt/benchmark/benchmarks.jar", "org.openjdk.jmh.Main"] diff --git a/performance/README.md b/performance/README.md index 369f875d8..c4e818511 100644 --- a/performance/README.md +++ b/performance/README.md @@ -1,91 +1,195 @@ - -## JMH Benchmarks - - -### Dependencies - - - -### How to Run - - -#### Generating Dataset - -```shell -mvn compile exec:exec -Dexec.executable=java -Dexec.args="-classpath %classpath com.clickhouse.benchmark.data.DataSetGenerator \ --input sample_dataset.sql -name default -rows 10" -``` - -#### Target Server - -By default the benchmarks start a local ClickHouse Docker container and run -against it. To run against an existing remote server instead (ClickHouse Cloud -or any self-hosted instance), set `CLICKHOUSE_URL`: - -```shell -export CLICKHOUSE_URL="https://default:my-password@abc123.clickhouse.cloud:8443?cluster=true" -mvn compile exec:exec -``` - -```shell -export CLICKHOUSE_URL="http://default@localhost:8123" -mvn compile exec:exec -``` - -The URL is parsed as: `://[:]@[:][?cluster=true]` -- scheme (`http`/`https`) selects whether SSL is used -- username/password come from the URL's user-info; if the password is omitted - the client connects without one, and if the username is omitted it defaults - to `default` -- port is optional; if omitted it defaults to `8443` for `https` and `8123` - for `http` -- `cluster` is optional (default `false`) and tells the benchmarks whether the - remote server is part of a replicated cluster (e.g. ClickHouse Cloud). When - `true`, a `SYSTEM SYNC REPLICA` is issued after writes. Leave it unset for a - plain standalone remote server — its tables aren't replicated and it will - reject that statement. - -When `CLICKHOUSE_URL` is unset, the local Docker container is started -automatically and no other configuration is needed. - -#### Running Benchmarks - -With default settings : -```shell -mvn compile exec:exec -``` - -With custom measurement iterations: -```shell -mvn compile exec:exec -Dexec.executable=java -Dexec.args="-classpath %classpath com.clickhouse.benchmark.BenchmarkRunner -m 3" -``` - -Other options: -- "-d" - dataset name or file path (like `file://default.csv`) -- "-l" - dataset limits to test coma separated (ex.: `-l 10000,10000`) -- "-m" - number of measurement iterations -- "-t" - time in seconds per iteration -- "-b" - benchmark mask coma separated. Ex.: `-b writer,reader,i`. Default : `-b i,q` - - "all" - Run alpl benchmarks - - "i" - InsertClient - insert operation benchmarks - - "q" - QueryClient - query operation benchmarks - - "ci" - ConcurrentInsertClient - concurrent version of insert benchmarks - - "cq" - ConcurrentQueryClient - concurrent version of query benchmarks - - "lz" - Compression - LZ4 output stream benchmarks (no server involved) - - "comp" - Compression - query/insert compression matrix of clients, methods, algorithms and formats (see below) - - "writer" - Serializer - serialization only logic benchmarks - - "reader" - DeSerilalizer - deserialization only logic benchmarks - - "mixed" - MixedWorkload - - "jq" - JDBCQuery - query operations using JDBC - - "ji" - JDBCInsert - insert operation using JDBC - -Compression matrix filters (used with `-b comp` or `-b all`, each defaults to all values): -- "-cc" - clients: `v1,v2` -- "-cm" - compression methods: `http` (`Content-Encoding`/`Accept-Encoding`), `native` (ClickHouse block compression, `use_http_compression = false`) -- "-ca" - algorithms: `lz4,zstd,snappy,brotli` -- "-cf" - formats: `RowBinaryWithNamesAndTypes,JSONEachRow` (any ClickHouse format name, e.g. `RowBinary`, is accepted) - -Ex.: `-b comp -cc v2 -cm http -ca zstd,lz4 -cf JSONEachRow`. - -Combinations a client doesn't support are skipped: V1 sends LZ4 only natively and other algorithms only over HTTP, + +# JMH benchmarks + +The `performance` module is a standalone Java 11 JMH project. Build the +repository artifacts it benchmarks before packaging the module: + +```shell +mvn -DskipTests -pl packages/clickhouse-jdbc-all -am install +mvn -f performance/pom.xml clean package +``` + +The shipping libraries remain Java 8 compatible; Java 11 is required only to +compile and run this benchmark module. + +## How to run + + +#### Generating Dataset + +```shell +mvn compile exec:exec -Dexec.executable=java -Dexec.args="-classpath %classpath com.clickhouse.benchmark.data.DataSetGenerator \ +-input sample_dataset.sql -name default -rows 10" +``` + +#### Target Server + +By default the benchmarks start a local ClickHouse Docker container and run +against it. To run against an existing remote server instead (ClickHouse Cloud +or any self-hosted instance), set `CLICKHOUSE_URL`: + +```shell +export CLICKHOUSE_URL="https://default:my-password@abc123.clickhouse.cloud:8443?cluster=true" +mvn compile exec:exec +``` + +```shell +export CLICKHOUSE_URL="http://default@localhost:8123" +mvn compile exec:exec +``` + +The URL is parsed as: `://[:]@[:][?cluster=true]` +- scheme (`http`/`https`) selects whether SSL is used +- username/password come from the URL's user-info; if the password is omitted + the client connects without one, and if the username is omitted it defaults + to `default` +- port is optional; if omitted it defaults to `8443` for `https` and `8123` + for `http` +- `cluster` is optional (default `false`) and tells the benchmarks whether the + remote server is part of a replicated cluster (e.g. ClickHouse Cloud). When + `true`, a `SYSTEM SYNC REPLICA` is issued after writes. Leave it unset for a + plain standalone remote server — its tables aren't replicated and it will + reject that statement. + +When `CLICKHOUSE_URL` is unset, the local Docker container is started +automatically and no other configuration is needed. + +### Reproducing the V1 write-stream spin + +`WriteStream` isolates the HTTP request-body handoff from serialization and +server storage. Each invocation sends the same 25,000 pre-serialized +`FixedString(256)` RowBinary rows (6.4 MB) to an `ENGINE = Null` table. Three +benchmark threads model three concurrent sink tasks. The methods are: + +- `v1NonBlocking`: the default V1 callback stream and its non-blocking queue; +- `v1BlockingQueue`: the same V1 path with only `use_blocking_queue` changed; +- `v2Direct`: the V2 callback stream. + +From the repository root, run the CPU-capped benchmark and profiler with: + +```shell +mkdir -p performance/results/write-stream +docker compose -f performance/compose.write-stream.yml up \ + --build --abort-on-container-exit --exit-code-from benchmark benchmark +docker compose -f performance/compose.write-stream.yml down +``` + +The benchmark container has a 3.5-CPU CFS quota and a 6 GiB heap. ClickHouse +runs in a separate container, and the Null table removes persistence and merge +work. The run writes these files under `performance/results/write-stream`: + +- `jmh-results.json` and `jmh-results.out`; +- one `cpu-.html` async-profiler flame graph per JMH fork. + +JMH reports rows/second, not batches/second, because each invocation is +declared as 25,000 operations. The Docker setup intentionally uses +async-profiler's `itimer` event: it samples on-CPU time without requiring +host `perf_event` permissions. + +#### Why V1 spends CPU while waiting + +A V1 `.data(writer)` request does not write the callback's bytes directly to +the HTTP entity output stream. It creates a producer/consumer handoff: + +1. `DelegatedInputStream` runs the writer callback on a `ClickHouseWorker` + thread and writes buffers to `NonBlockingPipedOutputStream`. +2. The request thread enters `ClickHouseHttpEntity.writeTo`, then + `ClickHouseHttpConnection.postData`, and drains the corresponding input + stream through `ClickHouseInputStream.pipe`. +3. `NonBlockingInputStream.updateBuffer` repeatedly calls + `AdaptiveQueue.DefaultQueue.poll` until the producer publishes another + buffer. When a timeout is configured, every failed poll also calls + `System.currentTimeMillis`. + +There is no park, wait, or backoff in that loop. With three inserts in flight, +the producer threads do useful work while three consumers can remain runnable +and consume their CPU quota polling empty queues. The expected V1 flame graph +therefore contains the following cluster: + +```text +[vdso] / __clock_gettime + <- System.currentTimeMillis + <- NonBlockingInputStream.updateBuffer + <- ClickHouseInputStream.pipe + <- ClickHouseHttpConnection.postData + <- ClickHouseHttpEntity.writeTo + +AdaptiveQueue$DefaultQueue.poll + <- NonBlockingInputStream.updateBuffer + <- ClickHouseInputStream.pipe + <- ClickHouseHttpConnection.postData + <- ClickHouseHttpEntity.writeTo +``` + +The blocking V1 method is a causal control. It retains the V1 HTTP and piped +handoff but replaces the spin-poll with a blocking queue. Its throughput and +profile show how much of the default V1 result comes from waiting policy rather +than payload size, HTTP, or ClickHouse. + +The blocking control is not expected to reach V2 throughput. It still pays for +the producer thread, queue synchronization, intermediate buffers, and the pipe +copy. Compare non-blocking V1 with blocking V1 to isolate the wait policy; +compare blocking V1 with V2 to see the remaining handoff overhead. + +V2 does not optimize that loop; it removes the handoff from the insert path. +`HttpAPIClientHelper` gives Apache HttpClient 5 an `EntityTemplate` whose +`IOCallback` writes the body directly to the connection output +stream. Consequently, the V2 profile should have no `NonBlocking*`, +`AdaptiveQueue`, or `ClickHouseInputStream.pipe` request-body frames. + +The CPU quota is essential to interpreting throughput. Without a quota, V1 can +burn otherwise idle cores and finish at roughly the same rate, making the +wasted CPU visible only in a profile. Under a quota, spin time displaces useful +work and lowers rows/second. + +This is a mechanism benchmark, not a Kafka Connect benchmark. It deliberately +excludes Avro conversion, Schema Registry, connector batching, and real +RowBinary serialization. It should reproduce and attribute the capped-CPU +penalty, but the exact V1/V2 ratio is hardware- and scheduling-dependent and is +not expected to equal 2.5 on every host. + +#### Running Benchmarks + +With default settings : +```shell +mvn compile exec:exec +``` + +With custom measurement iterations: +```shell +mvn compile exec:exec -Dexec.executable=java -Dexec.args="-classpath %classpath com.clickhouse.benchmark.BenchmarkRunner -m 3" +``` + +Other options: +- "-d" - dataset name or file path (like `file://default.csv`) +- "-l" - comma-separated dataset limits to test (for example, `-l 10000,10000`) +- "-m" - number of measurement iterations +- "-t" - time in seconds per iteration +- "-b" - comma-separated benchmark mask. Example: `-b writer,reader,i`. Default: `-b i,q` + - "all" - run all general benchmarks + - "i" - InsertClient - insert operation benchmarks + - "q" - QueryClient - query operation benchmarks + - "ci" - ConcurrentInsertClient - concurrent version of insert benchmarks + - "cq" - ConcurrentQueryClient - concurrent version of query benchmarks + - "lz" - Compression - LZ4 output stream benchmarks (no server involved) + - "comp" - Compression - query/insert compression matrix of clients, methods, algorithms and formats (see below) + - "writer" - Serializer - serialization only logic benchmarks + - "reader" - Deserializer - deserialization-only logic benchmarks + - "mixed" - MixedWorkload + - "jq" - JDBCQuery - query operations using JDBC + - "ji" - JDBCInsert - insert operation using JDBC + - "ws" - WriteStream - isolated V1/V2 HTTP request-body handoff + +`ws` uses throughput mode with a second time unit and must be selected by +itself. It is excluded from `-b all`; run it explicitly with `-b ws`. + +Compression matrix filters (used with `-b comp` or `-b all`, each defaults to all values): +- "-cc" - clients: `v1,v2` +- "-cm" - compression methods: `http` (`Content-Encoding`/`Accept-Encoding`), `native` (ClickHouse block compression, `use_http_compression = false`) +- "-ca" - algorithms: `lz4,zstd,snappy,brotli` +- "-cf" - formats: `RowBinaryWithNamesAndTypes,JSONEachRow` (any ClickHouse format name, e.g. `RowBinary`, is accepted) + +Ex.: `-b comp -cc v2 -cm http -ca zstd,lz4 -cf JSONEachRow`. + +Combinations a client doesn't support are skipped: V1 sends LZ4 only natively and other algorithms only over HTTP, V2 native mode is LZ4 only, V2 can't compress requests with brotli, and snappy works with neither client. \ No newline at end of file diff --git a/performance/compare-jmh.py b/performance/compare-jmh.py new file mode 100755 index 000000000..b513ed3ea --- /dev/null +++ b/performance/compare-jmh.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Compare matching primary metrics in two JMH JSON result files.""" + +import argparse +import csv +import json +import sys + + +def result_key(result): + params = tuple(sorted((result.get("params") or {}).items())) + return result["benchmark"], params + + +def load_results(path): + with open(path, encoding="utf-8") as source: + results = json.load(source) + return {result_key(result): result for result in results} + + +def display_name(key): + benchmark, params = key + if not params: + return benchmark + return "{} [{}]".format( + benchmark, ", ".join("{}={}".format(name, value) for name, value in params) + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Compare point estimates from matching JMH JSON results." + ) + parser.add_argument("baseline", help="baseline JMH JSON file") + parser.add_argument("candidate", help="candidate JMH JSON file") + args = parser.parse_args() + + baseline = load_results(args.baseline) + candidate = load_results(args.candidate) + shared = sorted(set(baseline).intersection(candidate)) + if not shared: + parser.error("the files contain no matching benchmark and parameter combinations") + + writer = csv.writer(sys.stdout) + writer.writerow( + [ + "benchmark", + "mode", + "unit", + "baseline", + "candidate", + "candidate/baseline", + "change_percent", + ] + ) + for key in shared: + base_result = baseline[key] + candidate_result = candidate[key] + base_metric = base_result["primaryMetric"] + candidate_metric = candidate_result["primaryMetric"] + if base_result["mode"] != candidate_result["mode"]: + raise ValueError("mode differs for {}".format(display_name(key))) + if base_metric["scoreUnit"] != candidate_metric["scoreUnit"]: + raise ValueError("score unit differs for {}".format(display_name(key))) + + base_score = float(base_metric["score"]) + candidate_score = float(candidate_metric["score"]) + if base_score == 0: + raise ValueError("baseline score is zero for {}".format(display_name(key))) + ratio = candidate_score / base_score + writer.writerow( + [ + display_name(key), + base_result["mode"], + base_metric["scoreUnit"], + "{:.9g}".format(base_score), + "{:.9g}".format(candidate_score), + "{:.6f}".format(ratio), + "{:+.3f}".format((ratio - 1.0) * 100.0), + ] + ) + + baseline_only = len(set(baseline).difference(candidate)) + candidate_only = len(set(candidate).difference(baseline)) + if baseline_only or candidate_only: + print( + "warning: skipped {} baseline-only and {} candidate-only results".format( + baseline_only, candidate_only + ), + file=sys.stderr, + ) + + +if __name__ == "__main__": + main() diff --git a/performance/compose.write-stream.yml b/performance/compose.write-stream.yml new file mode 100644 index 000000000..fcaf28551 --- /dev/null +++ b/performance/compose.write-stream.yml @@ -0,0 +1,57 @@ +services: + clickhouse: + image: clickhouse/clickhouse-server:25.8 + environment: + CLICKHOUSE_PASSWORD: benchmark + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + healthcheck: + test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost:8123/ping"] + interval: 2s + timeout: 2s + retries: 60 + + benchmark: + build: + context: .. + dockerfile: performance/Dockerfile.write-stream + args: + GIT_COMMIT: ${GIT_COMMIT:-unknown} + cpus: 3.5 + mem_limit: 8g + environment: + CLICKHOUSE_URL: http://default:benchmark@clickhouse:8123 + depends_on: + clickhouse: + condition: service_healthy + volumes: + - ./results/write-stream:/results + command: + - "com.clickhouse.benchmark.clients.WriteStream.(v1NonBlocking|v1BlockingQueue|v2Direct)" + - "-f" + - "1" + - "-wi" + - "1" + - "-w" + - "5s" + - "-i" + - "5" + - "-r" + - "20s" + - "-t" + - "3" + - "-bm" + - "thrpt" + - "-tu" + - "s" + - "-prof" + - "gc" + - "-prof" + - "mempool" + - "-rf" + - "json" + - "-rff" + - "/results/jmh-results.json" + - "-o" + - "/results/jmh-results.out" + - "-jvmArgs" + - "-Xms6g -Xmx6g -agentpath:/opt/async-profiler/lib/libasyncProfiler.so=start,event=itimer,interval=1ms,file=/results/cpu-%p.html" diff --git a/performance/pom.xml b/performance/pom.xml index 8c50b1365..cfd642821 100644 --- a/performance/pom.xml +++ b/performance/pom.xml @@ -167,7 +167,7 @@ - com.clickhouse.com.clickhouse.benchmark.BenchmarkRunner + com.clickhouse.benchmark.BenchmarkRunner diff --git a/performance/src/main/java/com/clickhouse/benchmark/BenchmarkRunner.java b/performance/src/main/java/com/clickhouse/benchmark/BenchmarkRunner.java index e18639241..d1b61fe12 100644 --- a/performance/src/main/java/com/clickhouse/benchmark/BenchmarkRunner.java +++ b/performance/src/main/java/com/clickhouse/benchmark/BenchmarkRunner.java @@ -10,6 +10,7 @@ import com.clickhouse.benchmark.clients.MixedWorkload; import com.clickhouse.benchmark.clients.QueryClient; import com.clickhouse.benchmark.clients.Serializers; +import com.clickhouse.benchmark.clients.WriteStream; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.profile.GCProfiler; @@ -56,6 +57,13 @@ public static void main(String[] args) throws Exception { final String outputFile = String.format("jmh-results-%s-%s.out", env, time); final String datasetName = options.getOrDefault("-d", "file://default.csv"); final String[] limits = options.getOrDefault("-l", "300000,100000,10000").split(","); + String testMask = options.getOrDefault("-b", "q,i"); + String[] testMaskParts = testMask.split(","); + boolean runAll = testMaskParts[0].equalsIgnoreCase("all"); + boolean writeStreamOnly = testMaskParts.length == 1 && "ws".equalsIgnoreCase(testMaskParts[0]); + if (!writeStreamOnly && Arrays.asList(testMaskParts).contains("ws")) { + throw new IllegalArgumentException("The ws benchmark uses throughput mode and must be run by itself"); + } System.out.println("Measurement iterations: " + measurementIterations); System.out.println("Measurement time: " + measurementTime + "s"); @@ -66,9 +74,6 @@ public static void main(String[] args) throws Exception { ChainedOptionsBuilder optBuilder = new OptionsBuilder() .forks(1) // must be a fork. No fork only for debugging - .mode(Mode.SampleTime) - .param("datasetSourceName", datasetName) - .param("limit", limits) .timeUnit(TimeUnit.MILLISECONDS) .addProfiler(GCProfiler.class) .addProfiler(MemPoolProfiler.class) @@ -82,15 +87,22 @@ public static void main(String[] args) throws Exception { .result(resultFile) .shouldFailOnError(true); - String testMask = options.getOrDefault("-b", "q,i"); - String[] testMaskParts = testMask.split(","); + if (writeStreamOnly) { + optBuilder.mode(Mode.Throughput) + .timeUnit(TimeUnit.SECONDS); + } else { + optBuilder.mode(Mode.SampleTime) + .param("datasetSourceName", datasetName) + .param("limit", limits); + } SortedSet benchmarks = new TreeSet<>(); - boolean runAll = testMaskParts[0].equalsIgnoreCase("all"); if (runAll) { BENCHMARK_FLAGS.values().forEach((b) -> { - optBuilder.include(b); - benchmarks.add(b); + if (!b.equals(WriteStream.class.getName())) { + optBuilder.include(b); + benchmarks.add(b); + } }); } else { for (String p : testMaskParts) { @@ -134,6 +146,7 @@ private static Map buildBenchmarkFlags() { map.put("mixed", MixedWorkload.class.getName()); map.put("jq", JDBCQuery.class.getName()); map.put("ji", JDBCInsert.class.getName()); + map.put("ws", WriteStream.class.getName()); return map; } diff --git a/performance/src/main/java/com/clickhouse/benchmark/clients/DataTypes.java b/performance/src/main/java/com/clickhouse/benchmark/clients/DataTypes.java index a49f271da..1ae33ad38 100644 --- a/performance/src/main/java/com/clickhouse/benchmark/clients/DataTypes.java +++ b/performance/src/main/java/com/clickhouse/benchmark/clients/DataTypes.java @@ -43,7 +43,7 @@ public void setUpIteration(DataState dataState) { LOGGER.error("Failed to init data for components benchmark", e); } - if (dataState.syntheticDataSet != null) { + if (dataState.syntheticDataSet == null) { dataState.syntheticDataSet = new SyntheticDataSet(dataState.limit); } } diff --git a/performance/src/main/java/com/clickhouse/benchmark/clients/WriteStream.java b/performance/src/main/java/com/clickhouse/benchmark/clients/WriteStream.java new file mode 100644 index 000000000..a6e74a64c --- /dev/null +++ b/performance/src/main/java/com/clickhouse/benchmark/clients/WriteStream.java @@ -0,0 +1,156 @@ +package com.clickhouse.benchmark.clients; + +import com.clickhouse.benchmark.TestEnvironment; +import com.clickhouse.client.ClickHouseClient; +import com.clickhouse.client.ClickHouseCredentials; +import com.clickhouse.client.ClickHouseNodeSelector; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseResponse; +import com.clickhouse.client.api.Client; +import com.clickhouse.client.api.enums.Protocol; +import com.clickhouse.client.api.insert.InsertResponse; +import com.clickhouse.client.api.insert.InsertSettings; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.data.ClickHouseFormat; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OperationsPerInvocation; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; + +import java.util.Arrays; +import java.util.concurrent.TimeUnit; + +import static com.clickhouse.benchmark.TestEnvironment.DB_NAME; +import static com.clickhouse.benchmark.TestEnvironment.getHost; +import static com.clickhouse.benchmark.TestEnvironment.getPassword; +import static com.clickhouse.benchmark.TestEnvironment.getPort; +import static com.clickhouse.benchmark.TestEnvironment.getServer; +import static com.clickhouse.benchmark.TestEnvironment.getUsername; +import static com.clickhouse.benchmark.TestEnvironment.isSsl; + +/** + * Isolates the HTTP request-body handoff by sending the same pre-serialized RowBinary rows through both clients. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@Threads(3) +public class WriteStream { + private static final int ROWS_PER_BATCH = 25_000; + private static final int ROW_SIZE = 256; + private static final String TABLE_NAME = "write_stream_null"; + + private ClickHouseClient clientV1NonBlocking; + private ClickHouseClient clientV1Blocking; + private Client clientV2; + private byte[][] rows; + + @Setup(Level.Trial) + public void setup() { + TestEnvironment.setupEnvironment(); + BenchmarkBase.runQuery(String.format( + "CREATE TABLE `%s`.`%s` (`payload` FixedString(%d)) ENGINE = Null", + DB_NAME, TABLE_NAME, ROW_SIZE)); + + clientV1NonBlocking = createClientV1(false); + clientV1Blocking = createClientV1(true); + clientV2 = new Client.Builder() + .addEndpoint(Protocol.HTTP, getHost(), getPort(), isSsl()) + .setUsername(getUsername()) + .setPassword(getPassword()) + .setDefaultDatabase(DB_NAME) + .setMaxRetries(0) + .build(); + + rows = new byte[ROWS_PER_BATCH][ROW_SIZE]; + for (int i = 0; i < rows.length; i++) { + Arrays.fill(rows[i], (byte) i); + } + } + + @TearDown(Level.Trial) + public void tearDown() throws Exception { + Exception failure = null; + for (AutoCloseable closeable : new AutoCloseable[] {clientV1NonBlocking, clientV1Blocking, clientV2}) { + if (closeable == null) { + continue; + } + try { + closeable.close(); + } catch (Exception e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + try { + TestEnvironment.cleanupEnvironment(); + } catch (RuntimeException e) { + if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + if (failure != null) { + throw failure; + } + } + + @Benchmark + @OperationsPerInvocation(ROWS_PER_BATCH) + public void v1NonBlocking() throws Exception { + insertV1(clientV1NonBlocking); + } + + @Benchmark + @OperationsPerInvocation(ROWS_PER_BATCH) + public void v1BlockingQueue() throws Exception { + insertV1(clientV1Blocking); + } + + @Benchmark + @OperationsPerInvocation(ROWS_PER_BATCH) + public void v2Direct() throws Exception { + try (InsertResponse response = clientV2.insert(TABLE_NAME, out -> { + for (byte[] row : rows) { + out.write(row); + } + }, ClickHouseFormat.RowBinary, new InsertSettings()).get()) { + response.getWrittenRows(); + } + } + + private void insertV1(ClickHouseClient client) throws Exception { + try (ClickHouseResponse response = client.read(getServer()) + .write() + .option(ClickHouseClientOption.ASYNC, false) + .format(ClickHouseFormat.RowBinary) + .query(BenchmarkBase.getInsertQuery(TABLE_NAME)) + .data(out -> { + for (byte[] row : rows) { + out.write(row); + } + }) + .executeAndWait()) { + response.getSummary(); + } + } + + private static ClickHouseClient createClientV1(boolean useBlockingQueue) { + return ClickHouseClient.builder() + .option(ClickHouseClientOption.USE_BLOCKING_QUEUE, useBlockingQueue) + .defaultCredentials(ClickHouseCredentials.fromUserAndPassword(getUsername(), getPassword())) + .nodeSelector(ClickHouseNodeSelector.of(ClickHouseProtocol.HTTP)) + .build(); + } +} diff --git a/pom.xml b/pom.xml index 10b82f3cd..0b4d005e8 100644 --- a/pom.xml +++ b/pom.xml @@ -121,6 +121,7 @@ 3.6.0 3.5.0 3.14.0 + 2.9.1 3.1.1 3.3.0 3.1.0 @@ -132,6 +133,7 @@ 0.8.12 3.4.2 3.11.2 + 0.18.3 0.9.23 1.7.1 0.6.1 @@ -152,6 +154,7 @@ 1.8 1.8 1.8 + false false ${skipTests} ${skipTests} @@ -368,6 +371,11 @@ os-maven-plugin ${os-plugin.version} + + org.cyclonedx + cyclonedx-maven-plugin + ${cyclonedx-plugin.version} + org.apache.maven.plugins maven-antrun-plugin @@ -398,6 +406,11 @@ maven-enforcer-plugin ${enforcer-plugin.version} + + com.github.siom79.japicmp + japicmp-maven-plugin + ${japicmp-plugin.version} + org.codehaus.mojo exec-maven-plugin @@ -849,6 +862,29 @@ release + + org.cyclonedx + cyclonedx-maven-plugin + false + + all + bom + 1.6 + true + false + false + false + + + + release-sbom + package + + makeAggregateBom + + + + org.apache.maven.plugins maven-toolchains-plugin @@ -953,6 +989,43 @@ true central + true + PUBLISHED + + + + + + + api-compatibility + + + + com.github.siom79.japicmp + japicmp-maven-plugin + + ${japicmp.skip} + + + ${project.groupId} + ${project.artifactId} + ${api.compatibility.version} + jar + + + + + ${project.build.directory}/${project.build.finalName}.jar + + + + public + true + false + true + false + true + diff --git a/release.sh b/release.sh index 4508b09a5..2bf8fa032 100755 --- a/release.sh +++ b/release.sh @@ -1,20 +1,27 @@ #!/bin/sh +set -eu -echo $1 -RELEASE_VERSION=$1 +RELEASE_VERSION=${1:-} if [ -z "$RELEASE_VERSION" ]; then - echo "Usage: $0 " + echo "Usage: $0 " >&2 exit 1 fi -echo "Release version: $RELEASE_VERSION" -# write version to VERSION file (consumed by the release workflow) +case "$RELEASE_VERSION" in + *-SNAPSHOT) + echo "Release version must not contain -SNAPSHOT" >&2 + exit 1 + ;; +esac + +echo "Preparing development revision for release: $RELEASE_VERSION" printf '%s\n' "$RELEASE_VERSION" > VERSION -# update version in main pom.xml -sed -i "s|.*<\/clickhouse-java.version>|${RELEASE_VERSION}<\/clickhouse-java.version>|g" pom.xml +sed -i "s|.*|${RELEASE_VERSION}-SNAPSHOT|" pom.xml +find ./examples -type f -name pom.xml -exec \ + sed -i "s|.*|${RELEASE_VERSION}-SNAPSHOT|" '{}' \; +find ./performance -type f -name pom.xml -exec \ + sed -i "s|.*|${RELEASE_VERSION}-SNAPSHOT|" '{}' \; -# udpate examples with new version -find ./examples/ -type f -name "pom.xml" -exec sed -i "s|.*<\/clickhouse-java.version>|${RELEASE_VERSION}-SNAPSHOT<\/clickhouse-java.version>|g" '{}' \; -find ./performance/ -type f -name "pom.xml" -exec sed -i "s|.*<\/ch.jdbc.revision>|${RELEASE_VERSION}-SNAPSHOT<\/ch.jdbc.revision>|g" '{}' \; +python3 scripts/release/validate_version.py diff --git a/scripts/check-relative-links.py b/scripts/check-relative-links.py new file mode 100755 index 000000000..4d75d1402 --- /dev/null +++ b/scripts/check-relative-links.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Check relative Markdown links in maintained repository documentation.""" + +from __future__ import print_function + +import argparse +import re +import sys +from pathlib import Path +from urllib.parse import unquote + + +LINK_PATTERN = re.compile(r"!?\[[^\]]*\]\(([^)\s]+)(?:\s+['\"][^)]*['\"])?\)") +FENCE_PATTERN = re.compile(r"^\s*(```|~~~)") +IGNORED_PREFIXES = ("#", "/", "http://", "https://", "mailto:", "tel:", "data:") + + +def maintained_files(root): + candidates = [ + root / "README.md", + root / "CONTRIBUTING.md", + root / ".github" / "pull_request_template.md", + ] + for docs_root in (root / "docs",): + if docs_root.exists(): + candidates.extend(docs_root.rglob("*.md")) + candidates.extend(docs_root.rglob("*.mdx")) + return sorted(path for path in candidates if path.is_file()) + + +def links_in(path): + in_fence = False + with path.open(encoding="utf-8") as source: + for line_number, line in enumerate(source, 1): + if FENCE_PATTERN.match(line): + in_fence = not in_fence + continue + if in_fence: + continue + for match in LINK_PATTERN.finditer(line): + yield line_number, match.group(1).strip("<>") + + +def check_file(root, path): + failures = [] + for line_number, destination in links_in(path): + if destination.startswith(IGNORED_PREFIXES): + continue + destination = unquote(destination.split("#", 1)[0].split("?", 1)[0]) + if not destination: + continue + target = (path.parent / destination).resolve() + if not target.exists(): + failures.append( + "{}:{}: {} -> {}".format( + path.relative_to(root), line_number, destination, target + ) + ) + return failures + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("files", nargs="*", help="Markdown files to check") + args = parser.parse_args() + + root = Path(__file__).resolve().parents[1] + files = [Path(name).resolve() for name in args] if args.files else maintained_files(root) + failures = [] + for path in files: + failures.extend(check_file(root, path)) + + if failures: + print("Broken relative documentation links:", file=sys.stderr) + for failure in failures: + print(" " + failure, file=sys.stderr) + return 1 + + print("Checked {} documentation files; relative links are valid.".format(len(files))) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/inventory_artifacts.py b/scripts/release/inventory_artifacts.py new file mode 100644 index 000000000..8c8ea1b16 --- /dev/null +++ b/scripts/release/inventory_artifacts.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Verify and stage the GitHub release artifact set.""" + +import argparse +import hashlib +import shutil +import sys +from pathlib import Path + + +MODULES = ( + "clickhouse-client", + "clickhouse-data", + "clickhouse-http-client", + "clickhouse-jdbc", + "clickhouse-r2dbc", + "client-v2", + "jdbc-v2", + "packages/clickhouse-jdbc-all", +) + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--version", required=True) + parser.add_argument("--output", type=Path, default=Path("release-assets")) + args = parser.parse_args() + + candidates = [] + missing = [] + for module in MODULES: + target = Path(module) / "target" + main_jar = target / "{}-{}.jar".format(Path(module).name, args.version) + if not main_jar.is_file(): + missing.append(str(main_jar)) + artifact_prefix = "{}-{}".format(Path(module).name, args.version) + candidates.extend( + path + for path in sorted(target.glob("*.jar")) + if path.name.startswith(artifact_prefix) + and not path.name.startswith(artifact_prefix + "-SNAPSHOT") + ) + + jdbc_bundle = Path("clickhouse-jdbc/target/bundle") + candidates.extend( + path + for path in sorted( + jdbc_bundle.glob("clickhouse-jdbc-{}*.jar".format(args.version)) + ) + if not path.name.startswith( + "clickhouse-jdbc-{}-SNAPSHOT".format(args.version) + ) + ) + + sboms = [Path("target/bom.json"), Path("target/bom.xml")] + for sbom in sboms: + if not sbom.is_file(): + missing.append(str(sbom)) + else: + candidates.append(sbom) + + if missing: + for path in missing: + print("missing required release artifact: " + path, file=sys.stderr) + return 1 + + if args.output.exists() and any(args.output.iterdir()): + print("release asset output is not empty: " + str(args.output), file=sys.stderr) + return 1 + args.output.mkdir(parents=True, exist_ok=True) + copied = {} + for source in candidates: + destination = args.output / source.name + if destination.name in copied and copied[destination.name] != source: + print( + "duplicate release asset name: {} and {}".format( + copied[destination.name], source + ), + file=sys.stderr, + ) + return 1 + shutil.copy2(str(source), str(destination)) + copied[destination.name] = source + + manifest = args.output / "SHA256SUMS" + lines = [ + "{} {}".format(sha256(path), path.name) + for path in sorted(args.output.iterdir()) + if path.name != manifest.name + ] + manifest.write_text("\n".join(lines) + "\n", encoding="utf-8") + print("staged {} release assets in {}".format(len(lines), args.output)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/test_validate_version.py b/scripts/release/test_validate_version.py new file mode 100644 index 000000000..3c097c9b1 --- /dev/null +++ b/scripts/release/test_validate_version.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +import tempfile +import unittest +from pathlib import Path + +from validate_version import validate + + +POM = """\ + + {} + +""" + + +class ValidateVersionTest(unittest.TestCase): + def validate_fixture(self, version, revision, release=False): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + version_file = root / "VERSION" + pom_file = root / "pom.xml" + version_file.write_text(version + "\n", encoding="utf-8") + pom_file.write_text(POM.format(revision), encoding="utf-8") + return validate(version_file, pom_file, release) + + def test_development_revision_matches_release_version(self): + self.assertEqual( + [], self.validate_fixture("1.2.3-rc1", "1.2.3-rc1-SNAPSHOT") + ) + + def test_release_revision_must_not_be_snapshot(self): + self.assertEqual([], self.validate_fixture("1.2.3", "1.2.3", release=True)) + self.assertTrue( + self.validate_fixture("1.2.3", "1.2.3-SNAPSHOT", release=True) + ) + + def test_rejects_snapshot_in_version_file(self): + self.assertTrue( + self.validate_fixture("1.2.3-SNAPSHOT", "1.2.3-SNAPSHOT") + ) + + def test_rejects_revision_drift(self): + self.assertTrue(self.validate_fixture("1.2.3", "1.2.4-SNAPSHOT")) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/release/validate_version.py b/scripts/release/validate_version.py new file mode 100644 index 000000000..db0a65a86 --- /dev/null +++ b/scripts/release/validate_version.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Validate VERSION against the Maven CI-friendly root revision.""" + +import argparse +import re +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + + +VERSION_RE = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z][0-9A-Za-z.-]*)?$") +MAVEN_NAMESPACE = {"m": "http://maven.apache.org/POM/4.0.0"} + + +def validate(version_file: Path, pom_file: Path, release: bool): + errors = [] + version = version_file.read_text(encoding="utf-8").strip() + if not VERSION_RE.fullmatch(version): + errors.append( + "VERSION must be a release version such as 1.2.3 or 1.2.3-rc1 " + "(SNAPSHOT is not allowed)" + ) + + try: + root = ET.parse(str(pom_file)).getroot() + except ET.ParseError as exc: + return errors + ["cannot parse {}: {}".format(pom_file, exc)] + + revision_node = root.find("m:properties/m:revision", MAVEN_NAMESPACE) + if revision_node is None or revision_node.text is None: + errors.append("root pom.xml must define properties/revision") + return errors + + revision = revision_node.text.strip() + expected = version if release else version + "-SNAPSHOT" + if revision != expected: + errors.append( + "root revision is {!r}; expected {!r} from VERSION".format( + revision, expected + ) + ) + return errors + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--version-file", type=Path, default=Path("VERSION")) + parser.add_argument("--pom", type=Path, default=Path("pom.xml")) + parser.add_argument( + "--release", + action="store_true", + help="require an exact release revision instead of the development SNAPSHOT", + ) + args = parser.parse_args() + + errors = validate(args.version_file, args.pom, args.release) + if errors: + for error in errors: + print("version validation failed: " + error, file=sys.stderr) + return 1 + + mode = "release" if args.release else "development" + print("VERSION and root revision are consistent ({})".format(mode)) + return 0 + + +if __name__ == "__main__": + sys.exit(main())