[rest] Unify the REST User-Agent format

docs/generated/catalog_configuration.html

@@ -206,6 +206,24 @@
             <td>String</td>
             <td>Uri of metastore server.</td>
         </tr>
+        <tr>
+            <td><h5>user-agent.extended</h5></td>
+            <td style="word-wrap: break-word;">(none)</td>
+            <td>String</td>
+            <td>Free-form text appended to Paimon's unified User-Agent, sent on REST and object storage requests. For OSS, fs.oss.user.agent.extended takes precedence; for REST, header.User-Agent replaces the whole value.</td>
+        </tr>
+        <tr>
+            <td><h5>user-agent.features</h5></td>
+            <td style="word-wrap: break-word;">(none)</td>
+            <td>String</td>
+            <td>Space-separated features of Paimon's unified User-Agent, sent on REST and object storage requests. For OSS, fs.oss.user.agent.features takes precedence; for REST, header.User-Agent replaces the whole value.</td>
+        </tr>
+        <tr>
+            <td><h5>user-agent.module</h5></td>
+            <td style="word-wrap: break-word;">(none)</td>
+            <td>String</td>
+            <td>The module of Paimon's unified User-Agent, module(transport;features) extended, sent on REST and object storage requests. Defaults to the Paimon client and its version. For OSS, fs.oss.user.agent.module takes precedence; for REST, header.User-Agent replaces the whole value.</td>
+        </tr>
         <tr>
             <td><h5>warehouse</h5></td>
             <td style="word-wrap: break-word;">(none)</td>

paimon-api/pom.xml

@@ -71,6 +71,19 @@ under the License.
 
     <build>
         <plugins>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>templating-maven-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <id>filter-sources</id>
+                        <goals>
+                            <goal>filter-sources</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>

paimon-api/src/main/java-templates/org/apache/paimon/utils/BuildVersions.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.utils;
+
+/** Versions written in at build time, so reading them needs no class loader lookup. */
+public final class BuildVersions {
+
+    public static final String PAIMON = "${project.version}";
+
+    public static final String HTTP_CLIENT = "${apache.hc.client.version}";
+
+    private BuildVersions() {}
+}

paimon-api/src/main/java/org/apache/paimon/options/CatalogOptions.java

@@ -208,6 +208,31 @@ public class CatalogOptions {
                                     + "Paimon can read and write to external storage paths, such as OSS or S3. "
                                     + "In order to access these external paths correctly, you also need to configure the corresponding access key and secret key.");
 
+    public static final ConfigOption<String> USER_AGENT_MODULE =
+            ConfigOptions.key("user-agent.module")
+                    .stringType()
+                    .noDefaultValue()
+                    .withDescription(
+                            "The module of Paimon's unified User-Agent, module(transport;features) extended, "
+                                    + "sent on REST and object storage requests. Defaults to the Paimon client and its version. "
+                                    + "For OSS, fs.oss.user.agent.module takes precedence; for REST, header.User-Agent replaces the whole value.");
+
+    public static final ConfigOption<String> USER_AGENT_FEATURES =
+            ConfigOptions.key("user-agent.features")
+                    .stringType()
+                    .noDefaultValue()
+                    .withDescription(
+                            "Space-separated features of Paimon's unified User-Agent, sent on REST and object storage requests. "
+                                    + "For OSS, fs.oss.user.agent.features takes precedence; for REST, header.User-Agent replaces the whole value.");
+
+    public static final ConfigOption<String> USER_AGENT_EXTENDED =
+            ConfigOptions.key("user-agent.extended")
+                    .stringType()
+                    .noDefaultValue()
+                    .withDescription(
+                            "Free-form text appended to Paimon's unified User-Agent, sent on REST and object storage requests. "
+                                    + "For OSS, fs.oss.user.agent.extended takes precedence; for REST, header.User-Agent replaces the whole value.");
+
     public static final ConfigOption<Boolean> FILE_IO_ALLOW_CACHE =
             ConfigOptions.key("file-io.allow-cache")
                     .booleanType()

paimon-api/src/main/java/org/apache/paimon/rest/HttpClientUtils.java

@@ -18,9 +18,12 @@
 
 package org.apache.paimon.rest;
 
+import org.apache.paimon.options.Options;
 import org.apache.paimon.rest.interceptor.LoggingInterceptor;
 import org.apache.paimon.rest.interceptor.TimingInterceptor;
+import org.apache.paimon.utils.BuildVersions;
 import org.apache.paimon.utils.SensitiveConfigUtils;
+import org.apache.paimon.utils.StringUtils;
 
 import org.apache.hc.client5.http.classic.methods.HttpDelete;
 import org.apache.hc.client5.http.classic.methods.HttpGet;
@@ -55,6 +58,10 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import static org.apache.paimon.options.CatalogOptions.USER_AGENT_EXTENDED;
+import static org.apache.paimon.options.CatalogOptions.USER_AGENT_FEATURES;
+import static org.apache.paimon.options.CatalogOptions.USER_AGENT_MODULE;
+
 /** Utils for {@link HttpClientBuilder}. */
 public class HttpClientUtils {
 
@@ -80,12 +87,40 @@
     public static HttpClientBuilder createBuilder() {
         HttpClientBuilder clientBuilder = HttpClients.custom();
         clientBuilder.setDefaultRequestConfig(DEFAULT_REQUEST_CONFIG);
+        // A request's own User-Agent header, e.g. from header.User-Agent, still wins.
+        clientBuilder.setUserAgent(userAgent(new Options()));
 
         clientBuilder.setConnectionManager(configureConnectionManager());
         clientBuilder.setRetryStrategy(new ExponentialHttpRequestRetryStrategy(5));
         return clientBuilder;
     }
 
+    /**
+     * Paimon's unified User-Agent, {@code module(Apache-HttpClient/<version>;features) extended}.
+     */
+    public static String userAgent(Options options) {
+        String module = options.get(USER_AGENT_MODULE);
+        StringBuilder builder =
+                new StringBuilder(
+                                StringUtils.isNullOrWhitespaceOnly(module)
+                                        ? "Paimon/" + BuildVersions.PAIMON
+                                        : module.trim())
+                        .append("(Apache-HttpClient/")
+                        .append(BuildVersions.HTTP_CLIENT);
+        String features = options.get(USER_AGENT_FEATURES);
+        if (!StringUtils.isNullOrWhitespaceOnly(features)) {
+            for (String feature : features.trim().split("\\s+")) {
+                builder.append(';').append(feature);
+            }
+        }
+        builder.append(')');
+        String extended = options.get(USER_AGENT_EXTENDED);
+        if (!StringUtils.isNullOrWhitespaceOnly(extended)) {
+            builder.append(' ').append(extended.trim());
+        }
+        return builder.toString();
+    }
+
     private static HttpClientConnectionManager configureConnectionManager() {
         PoolingHttpClientConnectionManagerBuilder connectionManagerBuilder =
                 PoolingHttpClientConnectionManagerBuilder.create();
@@ -219,7 +254,7 @@
     private static CloseableHttpResponse execute(ClassicHttpRequest request, String uri)
             throws IOException {
         try {
             return DEFAULT_HTTP_CLIENT.execute(request);
         } catch (IOException | RuntimeException e) {
             throw new IOException(
                     "HTTP request failed for uri: " + SensitiveConfigUtils.sanitizeUri(uri));

paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java

@@ -129,6 +129,7 @@
 import javax.annotation.Nullable;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -174,6 +175,7 @@
 public class RESTApi {
 
     public static final String HEADER_PREFIX = "header.";
+    private static final String USER_AGENT_HEADER = "User-Agent";
     /**
      * Optional header carrying the URL-encoded {@link Identifier} JSON of the table which initiated
      * a dependency read.
@@ -240,15 +242,28 @@ public RESTApi(Options options, boolean configRequired) {
                                             ResourcePaths.config(),
                                             queryParams,
                                             ConfigResponse.class,
-                                            new RESTAuthFunction(baseHeaders, authProvider))
+                                            new RESTAuthFunction(
+                                                    withUserAgent(baseHeaders, options),
+                                                    authProvider))
                                     .merge(options.toMap()));
             baseHeaders.putAll(extractPrefixMap(options, HEADER_PREFIX));
         }
-        this.restAuthFunction = new RESTAuthFunction(baseHeaders, authProvider);
+        this.restAuthFunction =
+                new RESTAuthFunction(withUserAgent(baseHeaders, options), authProvider);
         this.options = options;
         this.resourcePaths = ResourcePaths.forCatalogProperties(options);
     }
 
+    /** Adds Paimon's unified User-Agent unless {@code header.User-Agent} is set. */
+    private static Map<String, String> withUserAgent(Map<String, String> headers, Options options) {
+        if (headers.keySet().stream().anyMatch(USER_AGENT_HEADER::equalsIgnoreCase)) {
+            return headers;
+        }
+        Map<String, String> result = new HashMap<>(headers);
+        result.put(USER_AGENT_HEADER, HttpClientUtils.userAgent(options));
+        return result;
+    }
+
     /** Get the configured options which has been merged from REST Server. */
     public Options options() {
         return options;

paimon-api/src/test/java/org/apache/paimon/rest/HttpClientUtilsTest.java

@@ -18,11 +18,15 @@
 
 package org.apache.paimon.rest;
 
+import org.apache.paimon.options.CatalogOptions;
+import org.apache.paimon.options.Options;
+import org.apache.paimon.utils.BuildVersions;
 import org.apache.paimon.utils.SensitiveConfigUtils;
 
 import com.sun.net.httpserver.HttpExchange;
 import com.sun.net.httpserver.HttpHandler;
 import com.sun.net.httpserver.HttpServer;
+import org.apache.hc.core5.util.VersionInfo;
 import org.assertj.core.api.ThrowableAssert;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
@@ -62,6 +66,42 @@ public void tearDown() {
         }
     }
 
+    @Test
+    public void testUserAgent() throws Exception {
+        AtomicReference<String> userAgent = new AtomicReference<>();
+        registerHandler(
+                "/ua",
+                exchange -> {
+                    userAgent.set(exchange.getRequestHeaders().getFirst("User-Agent"));
+                    respond(exchange, 200, new byte[0]);
+                });
+
+        assertThat(HttpClientUtils.exists(url("/ua"))).isTrue();
+        assertThat(userAgent.get())
+                .isEqualTo(HttpClientUtils.userAgent(new Options()))
+                .isEqualTo(
+                        "Paimon/"
+                                + BuildVersions.PAIMON
+                                + "(Apache-HttpClient/"
+                                + BuildVersions.HTTP_CLIENT
+                                + ")");
+        Options options = new Options();
+        options.set(CatalogOptions.USER_AGENT_MODULE, "MyApp/1.0");
+        options.set(CatalogOptions.USER_AGENT_FEATURES, " Flink  Spark ");
+        options.set(CatalogOptions.USER_AGENT_EXTENDED, "vvr");
+        assertThat(HttpClientUtils.userAgent(options))
+                .isEqualTo(
+                        "MyApp/1.0(Apache-HttpClient/"
+                                + BuildVersions.HTTP_CLIENT
+                                + ";Flink;Spark) vvr");
+        assertThat(BuildVersions.PAIMON).matches("\\d+\\.\\d+\\S*");
+        assertThat(BuildVersions.HTTP_CLIENT)
+                .isEqualTo(
+                        VersionInfo.loadVersionInfo(
+                                        "org.apache.hc.client5", getClass().getClassLoader())
+                                .getRelease());
+    }
+
     @Test
     public void testExistsReturnsTrueForAvailableResource() throws Exception {
         registerHandler(

paimon-core/src/test/java/org/apache/paimon/rest/MockRESTCatalogTest.java

@@ -68,6 +68,7 @@
 import org.apache.paimon.types.DataField;
 import org.apache.paimon.types.DataTypes;
 import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.BuildVersions;
 import org.apache.paimon.utils.InstantiationUtil;
 import org.apache.paimon.utils.JsonSerdeUtil;
 
@@ -1631,6 +1632,53 @@ void testBaseHeadersInRequests() throws Exception {
         checkHeader(customHeaderName, customHeaderValue);
     }
 
+    @Test
+    void testDefaultUserAgentInRequests() throws Exception {
+        restCatalogServer.clearReceivedHeaders();
+        initCatalog(false).listDatabases();
+
+        assertThat(restCatalogServer.getReceivedHeaders())
+                .isNotEmpty()
+                .allSatisfy(
+                        headers ->
+                                assertThat(headers)
+                                        .containsEntry(
+                                                "user-agent",
+                                                HttpClientUtils.userAgent(new Options())));
+    }
+
+    @Test
+    void testCatalogWideUserAgentOptionsInRequests() throws Exception {
+        options.set(CatalogOptions.USER_AGENT_FEATURES, "Flink");
+        options.set(CatalogOptions.USER_AGENT_EXTENDED, "vvr");
+        restCatalogServer.clearReceivedHeaders();
+        initCatalog(false).listDatabases();
+
+        String expected =
+                "Paimon/"
+                        + BuildVersions.PAIMON
+                        + "(Apache-HttpClient/"
+                        + BuildVersions.HTTP_CLIENT
+                        + ";Flink) vvr";
+        assertThat(restCatalogServer.getReceivedHeaders())
+                .isNotEmpty()
+                .allSatisfy(headers -> assertThat(headers).containsEntry("user-agent", expected));
+    }
+
+    @Test
+    void testConfiguredUserAgentWins() throws Exception {
+        options.set(RESTCatalogOptions.HTTP_USER_AGENT, "starrocks/user");
+        options.set(CatalogOptions.USER_AGENT_EXTENDED, "vvr");
+        restCatalogServer.clearReceivedHeaders();
+        initCatalog(false).listDatabases();
+
+        assertThat(restCatalogServer.getReceivedHeaders())
+                .isNotEmpty()
+                .allSatisfy(
+                        headers ->
+                                assertThat(headers).containsEntry("user-agent", "starrocks/user"));
+    }
+
     @Test
     void testReadViaHeaderOnDependencyTableAndDataTokenRequests() throws Exception {
         Identifier root = Identifier.create("db", "root");

paimon-vfs/paimon-vfs-hadoop/src/main/java/org/apache/paimon/vfs/hadoop/PaimonVirtualFileSystem.java

@@ -24,6 +24,7 @@
 import org.apache.paimon.options.Options;
 import org.apache.paimon.rest.RESTCatalogOptions;
 import org.apache.paimon.rest.responses.GetDatabaseResponse;
+import org.apache.paimon.utils.StringUtils;
 import org.apache.paimon.vfs.VFSCatalogIdentifier;
 import org.apache.paimon.vfs.VFSDatabaseIdentifier;
 import org.apache.paimon.vfs.VFSIdentifier;
@@ -58,7 +59,7 @@ public class PaimonVirtualFileSystem extends FileSystem {
     private VFSOperations vfsOperations;
     private Configuration conf;
 
-    private static final String USER_AGENT = "HadoopPVFS";
+    private static final String USER_AGENT_FEATURE = "HadoopPVFS";
     private static final long DEFAULT_BLOCK_SIZE = 128 * 1024 * 1024L;
 
     @Override
@@ -96,8 +97,13 @@ private void initVFSOperations() {
             options.set(RESTCatalogOptions.URI, endpoint);
         }
 
-        // Set user agent
-        options.set(RESTCatalogOptions.HTTP_USER_AGENT, USER_AGENT);
+        // PVFS leads the features of Paimon's unified User-Agent.
+        String features = options.get(CatalogOptions.USER_AGENT_FEATURES);
+        options.set(
+                CatalogOptions.USER_AGENT_FEATURES,
+                StringUtils.isNullOrWhitespaceOnly(features)
+                        ? USER_AGENT_FEATURE
+                        : USER_AGENT_FEATURE + " " + features.trim());
 
         vfsOperations = new VFSOperations(options);
     }