diff --git a/docs/generated/catalog_configuration.html b/docs/generated/catalog_configuration.html index bc7226d1b07d..443fce991042 100644 --- a/docs/generated/catalog_configuration.html +++ b/docs/generated/catalog_configuration.html @@ -206,6 +206,24 @@ String Uri of metastore server. + +
user-agent.extended
+ (none) + String + Free-form text appended to Paimon's unified User-Agent, sent on REST and object storage requests. For OSS, fs.oss.user.agent.extended takes precedence; for REST, header.User-Agent replaces the whole value. + + +
user-agent.features
+ (none) + String + Space-separated features of Paimon's unified User-Agent, sent on REST and object storage requests. For OSS, fs.oss.user.agent.features takes precedence; for REST, header.User-Agent replaces the whole value. + + +
user-agent.module
+ (none) + String + The module of Paimon's unified User-Agent, module(transport;features) extended, sent on REST and object storage requests. Defaults to the Paimon client and its version. For OSS, fs.oss.user.agent.module takes precedence; for REST, header.User-Agent replaces the whole value. +
warehouse
(none) diff --git a/paimon-api/pom.xml b/paimon-api/pom.xml index 50314960eaf5..c3159a56121d 100644 --- a/paimon-api/pom.xml +++ b/paimon-api/pom.xml @@ -71,6 +71,19 @@ under the License. + + org.codehaus.mojo + templating-maven-plugin + + + filter-sources + + filter-sources + + + + + org.apache.maven.plugins maven-jar-plugin diff --git a/paimon-api/src/main/java-templates/org/apache/paimon/utils/BuildVersions.java b/paimon-api/src/main/java-templates/org/apache/paimon/utils/BuildVersions.java new file mode 100644 index 000000000000..97d731a1681f --- /dev/null +++ b/paimon-api/src/main/java-templates/org/apache/paimon/utils/BuildVersions.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.utils; + +/** Versions written in at build time, so reading them needs no class loader lookup. */ +public final class BuildVersions { + + public static final String PAIMON = "${project.version}"; + + public static final String HTTP_CLIENT = "${apache.hc.client.version}"; + + private BuildVersions() {} +} diff --git a/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java b/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java index bda512a944b8..fc923cb85489 100644 --- a/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java +++ b/paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java @@ -208,6 +208,31 @@ public class CatalogOptions { + "Paimon can read and write to external storage paths, such as OSS or S3. " + "In order to access these external paths correctly, you also need to configure the corresponding access key and secret key."); + public static final ConfigOption USER_AGENT_MODULE = + ConfigOptions.key("user-agent.module") + .stringType() + .noDefaultValue() + .withDescription( + "The module of Paimon's unified User-Agent, module(transport;features) extended, " + + "sent on REST and object storage requests. Defaults to the Paimon client and its version. " + + "For OSS, fs.oss.user.agent.module takes precedence; for REST, header.User-Agent replaces the whole value."); + + public static final ConfigOption USER_AGENT_FEATURES = + ConfigOptions.key("user-agent.features") + .stringType() + .noDefaultValue() + .withDescription( + "Space-separated features of Paimon's unified User-Agent, sent on REST and object storage requests. " + + "For OSS, fs.oss.user.agent.features takes precedence; for REST, header.User-Agent replaces the whole value."); + + public static final ConfigOption USER_AGENT_EXTENDED = + ConfigOptions.key("user-agent.extended") + .stringType() + .noDefaultValue() + .withDescription( + "Free-form text appended to Paimon's unified User-Agent, sent on REST and object storage requests. " + + "For OSS, fs.oss.user.agent.extended takes precedence; for REST, header.User-Agent replaces the whole value."); + public static final ConfigOption FILE_IO_ALLOW_CACHE = ConfigOptions.key("file-io.allow-cache") .booleanType() diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java index 1444c8118d14..620918dd89d7 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java @@ -18,9 +18,12 @@ package org.apache.paimon.rest; +import org.apache.paimon.options.Options; import org.apache.paimon.rest.interceptor.LoggingInterceptor; import org.apache.paimon.rest.interceptor.TimingInterceptor; +import org.apache.paimon.utils.BuildVersions; import org.apache.paimon.utils.SensitiveConfigUtils; +import org.apache.paimon.utils.StringUtils; import org.apache.hc.client5.http.classic.methods.HttpDelete; import org.apache.hc.client5.http.classic.methods.HttpGet; @@ -55,6 +58,10 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.paimon.options.CatalogOptions.USER_AGENT_EXTENDED; +import static org.apache.paimon.options.CatalogOptions.USER_AGENT_FEATURES; +import static org.apache.paimon.options.CatalogOptions.USER_AGENT_MODULE; + /** Utils for {@link HttpClientBuilder}. */ public class HttpClientUtils { @@ -80,12 +87,40 @@ public static HttpClientBuilder createLoggingBuilder() { public static HttpClientBuilder createBuilder() { HttpClientBuilder clientBuilder = HttpClients.custom(); clientBuilder.setDefaultRequestConfig(DEFAULT_REQUEST_CONFIG); + // A request's own User-Agent header, e.g. from header.User-Agent, still wins. + clientBuilder.setUserAgent(userAgent(new Options())); clientBuilder.setConnectionManager(configureConnectionManager()); clientBuilder.setRetryStrategy(new ExponentialHttpRequestRetryStrategy(5)); return clientBuilder; } + /** + * Paimon's unified User-Agent, {@code module(Apache-HttpClient/;features) extended}. + */ + public static String userAgent(Options options) { + String module = options.get(USER_AGENT_MODULE); + StringBuilder builder = + new StringBuilder( + StringUtils.isNullOrWhitespaceOnly(module) + ? "Paimon/" + BuildVersions.PAIMON + : module.trim()) + .append("(Apache-HttpClient/") + .append(BuildVersions.HTTP_CLIENT); + String features = options.get(USER_AGENT_FEATURES); + if (!StringUtils.isNullOrWhitespaceOnly(features)) { + for (String feature : features.trim().split("\\s+")) { + builder.append(';').append(feature); + } + } + builder.append(')'); + String extended = options.get(USER_AGENT_EXTENDED); + if (!StringUtils.isNullOrWhitespaceOnly(extended)) { + builder.append(' ').append(extended.trim()); + } + return builder.toString(); + } + private static HttpClientConnectionManager configureConnectionManager() { PoolingHttpClientConnectionManagerBuilder connectionManagerBuilder = PoolingHttpClientConnectionManagerBuilder.create(); diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 8246cb153335..77a871d40007 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -129,6 +129,7 @@ import javax.annotation.Nullable; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -174,6 +175,7 @@ public class RESTApi { public static final String HEADER_PREFIX = "header."; + private static final String USER_AGENT_HEADER = "User-Agent"; /** * Optional header carrying the URL-encoded {@link Identifier} JSON of the table which initiated * a dependency read. @@ -240,15 +242,28 @@ public RESTApi(Options options, boolean configRequired) { ResourcePaths.config(), queryParams, ConfigResponse.class, - new RESTAuthFunction(baseHeaders, authProvider)) + new RESTAuthFunction( + withUserAgent(baseHeaders, options), + authProvider)) .merge(options.toMap())); baseHeaders.putAll(extractPrefixMap(options, HEADER_PREFIX)); } - this.restAuthFunction = new RESTAuthFunction(baseHeaders, authProvider); + this.restAuthFunction = + new RESTAuthFunction(withUserAgent(baseHeaders, options), authProvider); this.options = options; this.resourcePaths = ResourcePaths.forCatalogProperties(options); } + /** Adds Paimon's unified User-Agent unless {@code header.User-Agent} is set. */ + private static Map withUserAgent(Map headers, Options options) { + if (headers.keySet().stream().anyMatch(USER_AGENT_HEADER::equalsIgnoreCase)) { + return headers; + } + Map result = new HashMap<>(headers); + result.put(USER_AGENT_HEADER, HttpClientUtils.userAgent(options)); + return result; + } + /** Get the configured options which has been merged from REST Server. */ public Options options() { return options; diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/HttpClientUtilsTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/HttpClientUtilsTest.java index 5b1c3c27004c..f8ae28fbfef7 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/HttpClientUtilsTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/HttpClientUtilsTest.java @@ -18,11 +18,15 @@ package org.apache.paimon.rest; +import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.Options; +import org.apache.paimon.utils.BuildVersions; import org.apache.paimon.utils.SensitiveConfigUtils; import com.sun.net.httpserver.HttpExchange; import com.sun.net.httpserver.HttpHandler; import com.sun.net.httpserver.HttpServer; +import org.apache.hc.core5.util.VersionInfo; import org.assertj.core.api.ThrowableAssert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -62,6 +66,42 @@ public void tearDown() { } } + @Test + public void testUserAgent() throws Exception { + AtomicReference userAgent = new AtomicReference<>(); + registerHandler( + "/ua", + exchange -> { + userAgent.set(exchange.getRequestHeaders().getFirst("User-Agent")); + respond(exchange, 200, new byte[0]); + }); + + assertThat(HttpClientUtils.exists(url("/ua"))).isTrue(); + assertThat(userAgent.get()) + .isEqualTo(HttpClientUtils.userAgent(new Options())) + .isEqualTo( + "Paimon/" + + BuildVersions.PAIMON + + "(Apache-HttpClient/" + + BuildVersions.HTTP_CLIENT + + ")"); + Options options = new Options(); + options.set(CatalogOptions.USER_AGENT_MODULE, "MyApp/1.0"); + options.set(CatalogOptions.USER_AGENT_FEATURES, " Flink Spark "); + options.set(CatalogOptions.USER_AGENT_EXTENDED, "vvr"); + assertThat(HttpClientUtils.userAgent(options)) + .isEqualTo( + "MyApp/1.0(Apache-HttpClient/" + + BuildVersions.HTTP_CLIENT + + ";Flink;Spark) vvr"); + assertThat(BuildVersions.PAIMON).matches("\\d+\\.\\d+\\S*"); + assertThat(BuildVersions.HTTP_CLIENT) + .isEqualTo( + VersionInfo.loadVersionInfo( + "org.apache.hc.client5", getClass().getClassLoader()) + .getRelease()); + } + @Test public void testExistsReturnsTrueForAvailableResource() throws Exception { registerHandler( diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java index 0054af3d1652..b878a198f7aa 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java @@ -68,6 +68,7 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.BuildVersions; import org.apache.paimon.utils.InstantiationUtil; import org.apache.paimon.utils.JsonSerdeUtil; @@ -1631,6 +1632,53 @@ void testBaseHeadersInRequests() throws Exception { checkHeader(customHeaderName, customHeaderValue); } + @Test + void testDefaultUserAgentInRequests() throws Exception { + restCatalogServer.clearReceivedHeaders(); + initCatalog(false).listDatabases(); + + assertThat(restCatalogServer.getReceivedHeaders()) + .isNotEmpty() + .allSatisfy( + headers -> + assertThat(headers) + .containsEntry( + "user-agent", + HttpClientUtils.userAgent(new Options()))); + } + + @Test + void testCatalogWideUserAgentOptionsInRequests() throws Exception { + options.set(CatalogOptions.USER_AGENT_FEATURES, "Flink"); + options.set(CatalogOptions.USER_AGENT_EXTENDED, "vvr"); + restCatalogServer.clearReceivedHeaders(); + initCatalog(false).listDatabases(); + + String expected = + "Paimon/" + + BuildVersions.PAIMON + + "(Apache-HttpClient/" + + BuildVersions.HTTP_CLIENT + + ";Flink) vvr"; + assertThat(restCatalogServer.getReceivedHeaders()) + .isNotEmpty() + .allSatisfy(headers -> assertThat(headers).containsEntry("user-agent", expected)); + } + + @Test + void testConfiguredUserAgentWins() throws Exception { + options.set(RESTCatalogOptions.HTTP_USER_AGENT, "starrocks/user"); + options.set(CatalogOptions.USER_AGENT_EXTENDED, "vvr"); + restCatalogServer.clearReceivedHeaders(); + initCatalog(false).listDatabases(); + + assertThat(restCatalogServer.getReceivedHeaders()) + .isNotEmpty() + .allSatisfy( + headers -> + assertThat(headers).containsEntry("user-agent", "starrocks/user")); + } + @Test void testReadViaHeaderOnDependencyTableAndDataTokenRequests() throws Exception { Identifier root = Identifier.create("db", "root"); diff --git a/paimon-vfs/paimon-vfs-hadoop/src/main/java/org/apache/paimon/vfs/hadoop/PaimonVirtualFileSystem.java b/paimon-vfs/paimon-vfs-hadoop/src/main/java/org/apache/paimon/vfs/hadoop/PaimonVirtualFileSystem.java index 58d67e1f6db1..4b20cb2f46fd 100644 --- a/paimon-vfs/paimon-vfs-hadoop/src/main/java/org/apache/paimon/vfs/hadoop/PaimonVirtualFileSystem.java +++ b/paimon-vfs/paimon-vfs-hadoop/src/main/java/org/apache/paimon/vfs/hadoop/PaimonVirtualFileSystem.java @@ -24,6 +24,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.rest.RESTCatalogOptions; import org.apache.paimon.rest.responses.GetDatabaseResponse; +import org.apache.paimon.utils.StringUtils; import org.apache.paimon.vfs.VFSCatalogIdentifier; import org.apache.paimon.vfs.VFSDatabaseIdentifier; import org.apache.paimon.vfs.VFSIdentifier; @@ -58,7 +59,7 @@ public class PaimonVirtualFileSystem extends FileSystem { private VFSOperations vfsOperations; private Configuration conf; - private static final String USER_AGENT = "HadoopPVFS"; + private static final String USER_AGENT_FEATURE = "HadoopPVFS"; private static final long DEFAULT_BLOCK_SIZE = 128 * 1024 * 1024L; @Override @@ -96,8 +97,13 @@ private void initVFSOperations() { options.set(RESTCatalogOptions.URI, endpoint); } - // Set user agent - options.set(RESTCatalogOptions.HTTP_USER_AGENT, USER_AGENT); + // PVFS leads the features of Paimon's unified User-Agent. + String features = options.get(CatalogOptions.USER_AGENT_FEATURES); + options.set( + CatalogOptions.USER_AGENT_FEATURES, + StringUtils.isNullOrWhitespaceOnly(features) + ? USER_AGENT_FEATURE + : USER_AGENT_FEATURE + " " + features.trim()); vfsOperations = new VFSOperations(options); } diff --git a/paimon-vfs/paimon-vfs-hadoop/src/test/java/org/apache/paimon/vfs/hadoop/MockRestVirtualFileSystemTest.java b/paimon-vfs/paimon-vfs-hadoop/src/test/java/org/apache/paimon/vfs/hadoop/MockRestVirtualFileSystemTest.java index 46f8bd17fe95..e0b74ccd1091 100644 --- a/paimon-vfs/paimon-vfs-hadoop/src/test/java/org/apache/paimon/vfs/hadoop/MockRestVirtualFileSystemTest.java +++ b/paimon-vfs/paimon-vfs-hadoop/src/test/java/org/apache/paimon/vfs/hadoop/MockRestVirtualFileSystemTest.java @@ -20,6 +20,8 @@ import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.HttpClientUtils; import org.apache.paimon.rest.RESTCatalog; import org.apache.paimon.rest.RESTCatalogInternalOptions; import org.apache.paimon.rest.RESTCatalogOptions; @@ -35,15 +37,19 @@ import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableMap; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.junit.Assert; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import java.io.IOException; import java.util.Map; import java.util.UUID; +import static org.assertj.core.api.Assertions.assertThat; + /** Test for {@link PaimonVirtualFileSystem} with Mock Rest Server. */ public class MockRestVirtualFileSystemTest extends VirtualFileSystemTest { protected RESTCatalogServer restCatalogServer; @@ -110,11 +116,42 @@ private RESTCatalog initCatalog(boolean enableDataToken) throws IOException { return new RESTCatalog(CatalogContext.create(options)); } - protected void initFs() throws Exception { + @Test + public void testUserAgent() throws Exception { + assertPvfsUserAgent(vfs, "HadoopPVFS"); + + Configuration conf = pvfsConf(); + conf.set("fs.pvfs.user-agent.features", "Flink"); + PaimonVirtualFileSystem withFeatures = new PaimonVirtualFileSystem(); + withFeatures.initialize(vfsRoot.toUri(), conf); + assertPvfsUserAgent(withFeatures, "HadoopPVFS Flink"); + } + + private void assertPvfsUserAgent(FileSystem fs, String features) throws IOException { + restCatalogServer.clearReceivedHeaders(); + fs.exists(new Path(vfsRoot, "test_db")); + + Options expected = new Options(); + expected.set(CatalogOptions.USER_AGENT_FEATURES, features); + assertThat(restCatalogServer.getReceivedHeaders()) + .isNotEmpty() + .allSatisfy( + headers -> + assertThat(headers) + .containsEntry( + "user-agent", HttpClientUtils.userAgent(expected))); + } + + private Configuration pvfsConf() { Configuration conf = new Configuration(); conf.set("fs.pvfs.uri", restCatalogServer.getUrl()); conf.set("fs.pvfs.token.provider", AuthProviderEnum.BEAR.identifier()); conf.set("fs.pvfs.token", initToken); + return conf; + } + + protected void initFs() throws Exception { + Configuration conf = pvfsConf(); this.vfs = new PaimonVirtualFileSystem(); this.vfsRoot = new Path("pvfs://" + restWarehouse + "/"); this.vfs.initialize(vfsRoot.toUri(), conf); diff --git a/pom.xml b/pom.xml index 0b5512bd31d1..ee9ba053991a 100644 --- a/pom.xml +++ b/pom.xml @@ -980,6 +980,12 @@ under the License. 1.7 + + org.codehaus.mojo + templating-maven-plugin + 3.0.0 + + org.apache.maven.plugins maven-compiler-plugin