diff --git a/CHANGELOG.md b/CHANGELOG.md index 861c20280..d35dd6600 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ ### New Features +- **[migration-helpers]** Added `migration-helpers` module containing `ConfigurationMigrationHelper` and `ConfigPropertyCache` to convert configuration properties and connection URLs from v1 (0.7.1) format to v2 (0.9.8+) format (automatically prefixing ClickHouse server settings with `clickhouse_setting_`, custom headers with `http_header_`, and mapping renamed property keys). + - **[client-v2]** Added an OpenTelemetry implementation of the observability SPI. `Client.Builder.setSpanRecorder(new OpenTelemetrySpanRecorder(openTelemetry))` reports every client operation and every transport request as an OpenTelemetry `CLIENT` span: an operation span is diff --git a/migration-helpers/migrating_properties.md b/migration-helpers/migrating_properties.md new file mode 100644 index 000000000..2fb0f2054 --- /dev/null +++ b/migration-helpers/migrating_properties.md @@ -0,0 +1,133 @@ +# Migration Notes for ugprade from 0.6.x to 0.9.x + +## Host specification + +- No multihost allowed in jdbc URL like `jdbc:ch://host1,host2:8123/`. +- `ssl_mode` was redefined in `0.10.0`. In `0.9.x` it is handled by JDBC and can be only `STRICT` +- HTTP protocol is not guessed by port anymore. Default is plain HTTP. Otherwise should be like `jdbc:ch:https://cloud.com:8443/ + +## Authentication + +- Plain user-password is unchanged. +- New SSL modes for self-signed certificates added in `0.10.0`. +- `http_use_basic_auth` (default: `true`) to send authentication credentials. was introduced in V2 and backported to V1 + + +## Protocol Configuration + +- `protocol` - deprecated. Only http supported. Can be ignored in JDBC case because URL defines protocol. + +### Connection + +- `sslcerttype` - is deprecated. But will be re-introduced soon with a new name. Can be ignored if X.509 requested +- `sslkeyalg` - is deprecated. But will be re-introduced soon with a new name. Can be ignored if RSA requested. +- `sslprotocol` - is deprecated. Currently only latest protocol is available. Can be ignored. +- `custom_socket_factory` - is deprecated. +- `custom_socket_factory_options` - is deprecated. +- `connect_timeout` - is replaced with `connection_request_timeout` and `connection_timeout`. V1 used same timeout for getting + connection from pool and timeouting establishing new connection. +- `ssl` - is deprecated. Ignored. +- `sslmode` - replaced by `ssl_mode` with more values. + +### TCP Socket Configuration + +- `socket_ip_tos` - is deprecated + +### HTTP Configuration + +- `http_connection_provider` - is deprecated and has to be ignored. +- `custom_http_headers` - is deprecated. Custom headers should be set by one and with `http_header_` prefix. +- `custom_http_params` - is deprecated. If custom http parameter is clickhouse setting it should be set with `clickhouse_setting_` prefix. + There is another case when query parameters have user define meaning. The should be with prefix set in DB configuration like `custom_` (see more https://clickhouse.com/docs/reference/settings/server-settings/settings/custom#custom_settings_prefixes). + +- `http_server_default_response` - is deprecated. Can be ignored for JDBC case. +- `receive_query_progress` - is deprecated. Not supported and can be ignored for JDBC case. +- `send_http_client_id` - is deprecated. Can be ignored for JDBC case. +- `wait_end_of_query` - is really a server setting - should be prefixed with `clickhouse_setting_` +- `remember_last_set_roles` - valid for JDBC only. List of roles should be set via `session_db_roles` if working with client directly. +- `ahc_validate_after_inactivity` - is deprecated. Can be ignored. Validation made automatically. +- `ahc_retry_on_failure` - is deprecated. Two new properties `retry` (for number of retries) and `client_retry_on_failures` (to configure when to retry. Possible values: `NoHttpResponse`, `ConnectTimeout`, `ConnectionRequestTimeout`, `ServerRetryable`) + +- `alive_timeout` and `http_keep_alive` - are deprecated and joined into `http_keep_alive_timeout`. + + +## Client Operation Side + +- `use_compilation` - is deprecated. +- `debug_measure_request_time` - is deprecated. + + +### Multithreading +- `async` - this defined if each operation is run in separate thread. V2 switched to `false` by default. +- `max_scheduler_threads` - is deprecated. scheduler is set via configuration and defined by user. +- `max_threads` - is deprecated. +- `max_requests` - is deprecated. +- `thread_keepalive_timeout` - is deprecated. +- `max_core_thread_ttl` - is deprecated. + + +### Server Endpoints + +- `auto_discovery` - is deprecated. +- `load_balancing_policy` - is deprecated. Load balancing is not part of Client main functionality. +- `load_balancing_tags` - is deprecated. +- `health_check_interval` - is deprecated. +- `health_check_method` - is deprecated. +- `node_discovery_interval` - is deprecated. +- `node_discovery_limit` - is deprecated. +- `node_check_interval` - is deprecated. +- `node_group_size` - is deprecated. +- `check_all_nodes` - is deprecated. +- `version` - is replaced by `server_version`. +- `server_revision` - is replaced by `server_version`. +- `failover` - is deprecated. + +### Server Interaction + +- `custom_settings` - is deprecated. Was used to define client wide list of server settings. Now each settings should be set separatly and + with `clickhouse_setting_` prefix. +- `time_zone` - is replaced by `server_time_zone` +- `auto_session` - is deprecated. Sessions are created using client API. JDBC has no direct control over it. +- `log_leading_comment` - is deprecated and was applicable for JDBC. When true JDBC was parsing leading comment and sent to + server via `log_comment`. +- `max_execution_time` - Should be replaced with server setting (`clickhouse_setting_max_execution_time`). However V2 + client has similar setting with another meaning for async operations. +- `max_result_rows` - Should be replaced with server setting (`clickhouse_setting_max_result_rows`). +- `result_overflow_mode` - Should be replaced with server setting (`clickhouse_setting_result_overflow_mode`). +- `product_name` - replaced by `client_name`. +- `rename_response_column` - is deprecated. +- `transaction_timeout` - is deprecated. + +### Sessions +- `repeat_on_session_lock` - is deprecated. But need to be implemented as part of retry logic. +- `session_id` - Should be replaced with server setting (`clickhouse_setting_session_id`). +- `session_check` - Should be replaced with server setting (`clickhouse_setting_session_check`). +- `session_timeout` - Should be replaced with server setting (`clickhouse_setting_session_timeout`). + + +### Data Transfer + +- `buffering` - is deprecated. +- `buffer_size` - is deprecated. +- `buffer_queue_variation` - is deprecated. +- `use_blocking_queue` - is deprecated. +- `read_buffer_size` - is deprecated. +- `write_buffer_size` - is deprecated. +- `request_chunk_size` - is deprecated. +- `request_buffering` - is deprecated. +- `response_buffering`- is deprecated. +- `compress_algorithm` - is deprecated. +- `decompress_algorithm` - is deprecated. +- `compress_level` - is deprecated. +- `decompress_level` - is deprecated. +- `max_buffer_size` - is deprecated. +- `max_mapper_cache` - is deprecated. +- `max_queued_buffers` - is deprecated. +- `max_queued_requests` - is deprecated. +- `rounding_mode` - is deprecated. +- `srv_resolve` - is deprecated. +- `reuse_value_wrapper` - is deprecated. +- `widen_unsigned_types` - is deprecated. +- `use_binary_string` - replaced with `binary_string_support`. applicable only for `0.10.0` +- `use_objects_in_arrays` - is deprecated. +- `use_server_time_zone_for_dates` - is deprecated. diff --git a/migration-helpers/pom.xml b/migration-helpers/pom.xml new file mode 100644 index 000000000..f86333bcf --- /dev/null +++ b/migration-helpers/pom.xml @@ -0,0 +1,91 @@ + + + 4.0.0 + + + com.clickhouse + clickhouse-java + ${revision} + + + migration-helpers + jar + + ClickHouse Migration Helpers + Helper utilities for migrating from ClickHouse v1 client/driver to v2 + https://github.com/ClickHouse/clickhouse-java/tree/main/migration-helpers + + + + ${project.parent.groupId} + client-v2 + ${revision} + + + + ${project.parent.groupId} + jdbc-v2 + ${revision} + + + + ${project.parent.groupId} + clickhouse-client + ${revision} + + + + ${project.parent.groupId} + clickhouse-jdbc + ${revision} + + + + org.slf4j + slf4j-api + ${slf4j.version} + + + + + org.slf4j + slf4j-simple + ${slf4j.version} + + + + + ${project.parent.groupId} + clickhouse-client + ${revision} + test-jar + test + + + org.testcontainers + testcontainers + test + + + org.testng + testng + ${testng.version} + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.8 + 1.8 + + + + + diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java new file mode 100644 index 000000000..ca9366ad7 --- /dev/null +++ b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigPropertyCache.java @@ -0,0 +1,248 @@ +package com.clickhouse.migration.config; + +import java.io.InputStream; +import java.io.InputStreamReader; +import java.lang.reflect.Array; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Cache for v1 and v2 configuration properties, loaded from resource files and pre-loaded into memory. + */ +public class ConfigPropertyCache { + + private static final Logger log = LoggerFactory.getLogger(ConfigPropertyCache.class); + + private static final String V1_KNOWN_RESOURCE = "/com/clickhouse/migration/config/v1-known-properties.properties"; + private static final String V2_KNOWN_RESOURCE = "/com/clickhouse/migration/config/v2-known-properties.properties"; + private static final String V1_DEPRECATED_RESOURCE = "/com/clickhouse/migration/config/v1-deprecated-properties.properties"; + private static final String MAPPINGS_RESOURCE = "/com/clickhouse/migration/config/v1-to-v2-mappings.properties"; + + private static final ConfigPropertyCache INSTANCE = new ConfigPropertyCache(); + + private final Set v1KnownProperties; + private final Set v2KnownProperties; + private final Set v1DeprecatedProperties; + private final Map v1ToV2Mappings; + + /** + * Gets the singleton instance of {@link ConfigPropertyCache}. + * + * @return cache singleton instance + */ + public static ConfigPropertyCache getInstance() { + return INSTANCE; + } + + private ConfigPropertyCache() { + Set v1Props = new HashSet<>(); + Set v2Props = new HashSet<>(); + Set deprecatedProps = new HashSet<>(); + Map mappings = new HashMap<>(); + + // 1. Load properties from resource files + loadPropertiesResource(V1_KNOWN_RESOURCE, v1Props, null); + loadPropertiesResource(V2_KNOWN_RESOURCE, v2Props, null); + loadPropertiesResource(V1_DEPRECATED_RESOURCE, deprecatedProps, null); + loadPropertiesResource(MAPPINGS_RESOURCE, null, mappings); + + // 2. Pre-load / enrich with runtime enum keys from v1 and v2 if present on classpath + enrichWithRuntimeEnums(v1Props, v2Props); + + this.v1KnownProperties = Collections.unmodifiableSet(v1Props); + this.v2KnownProperties = Collections.unmodifiableSet(v2Props); + this.v1DeprecatedProperties = Collections.unmodifiableSet(deprecatedProps); + this.v1ToV2Mappings = Collections.unmodifiableMap(mappings); + + log.debug("Pre-loaded {} v1 properties, {} v2 properties, {} deprecated properties, {} mappings into cache.", + v1KnownProperties.size(), v2KnownProperties.size(), v1DeprecatedProperties.size(), v1ToV2Mappings.size()); + } + + private void loadPropertiesResource(String resourcePath, Set targetSet, Map targetMap) { + try (InputStream in = getClass().getResourceAsStream(resourcePath)) { + if (in != null) { + Properties props = new Properties(); + try (InputStreamReader reader = new InputStreamReader(in, StandardCharsets.UTF_8)) { + props.load(reader); + } + for (String key : props.stringPropertyNames()) { + if (targetSet != null) { + targetSet.add(key.trim()); + } + if (targetMap != null) { + targetMap.put(key.trim(), props.getProperty(key).trim()); + } + } + } else { + log.warn("Migration resource file not found on classpath: {}", resourcePath); + } + } catch (Exception e) { + log.error("Failed to load migration resource file: {}", resourcePath, e); + } + } + + private void enrichWithRuntimeEnums(Set v1Props, Set v2Props) { + // v2 ClientConfigProperties and ClientConfigurationProperties + loadEnumKeysFromClasspath("com.clickhouse.client.api.ClientConfigProperties", v2Props); + loadEnumKeysFromClasspath("com.clickhouse.client.api.ClientConfigurationProperties", v2Props); + + // v2 DriverProperties + loadEnumKeysFromClasspath("com.clickhouse.jdbc.DriverProperties", v2Props); + + // v1 ClickHouseClientOption + loadEnumKeysFromClasspath("com.clickhouse.client.config.ClickHouseClientOption", v1Props); + + // v1 ClickHouseHttpOption + loadEnumKeysFromClasspath("com.clickhouse.client.http.config.ClickHouseHttpOption", v1Props); + + // v1 JdbcConfig + loadJdbcConfigFromClasspath(v1Props); + } + + private void loadEnumKeysFromClasspath(String className, Set targetSet) { + try { + Class clazz = Class.forName(className, false, getClass().getClassLoader()); + if (clazz.isEnum()) { + Object[] constants = clazz.getEnumConstants(); + if (constants != null) { + Method getKeyMethod = null; + try { + getKeyMethod = clazz.getMethod("getKey"); + } catch (NoSuchMethodException ignored) { + // ignore if getKey() is missing + } + + for (Object obj : constants) { + if (obj != null) { + if (getKeyMethod != null) { + try { + Object keyObj = getKeyMethod.invoke(obj); + if (keyObj != null) { + targetSet.add(keyObj.toString()); + } + } catch (Exception e) { + targetSet.add(obj.toString()); + } + } else { + targetSet.add(obj.toString()); + } + } + } + } + } + } catch (Throwable t) { + log.debug("Class {} is not present on classpath or could not be loaded: {}", className, t.getMessage()); + } + } + + private void loadJdbcConfigFromClasspath(Set v1Props) { + try { + Class clazz = Class.forName("com.clickhouse.jdbc.JdbcConfig", false, getClass().getClassLoader()); + Method getDriverPropertiesMethod = clazz.getMethod("getDriverProperties"); + Object driverProps = getDriverPropertiesMethod.invoke(null); + if (driverProps != null && driverProps.getClass().isArray()) { + int length = Array.getLength(driverProps); + for (int i = 0; i < length; i++) { + Object info = Array.get(driverProps, i); + if (info != null) { + Field nameField = info.getClass().getField("name"); + Object nameObj = nameField.get(info); + if (nameObj != null) { + v1Props.add(nameObj.toString()); + } + } + } + } + } catch (Throwable t) { + log.debug("Could not inspect JdbcConfig properties: {}", t.getMessage()); + } + } + + /** + * Checks if the key is a known v1 configuration property. + * + * @param key property name + * @return true if key is known in v1 + */ + public boolean isV1KnownProperty(String key) { + return key != null && v1KnownProperties.contains(key); + } + + /** + * Checks if the key is a known v2 configuration property. + * + * @param key property name + * @return true if key is known in v2 + */ + public boolean isV2KnownProperty(String key) { + return key != null && v2KnownProperties.contains(key); + } + + /** + * Checks if the property is deprecated in v2 without direct conversion. + * + * @param key property name + * @return true if property is deprecated + */ + public boolean isDeprecatedProperty(String key) { + return key != null && (v1DeprecatedProperties.contains(key) || v1DeprecatedProperties.contains(key.toLowerCase())); + } + + /** + * Gets the mapped v2 key name for a given v1 property key. + * + * @param v1Key property name in v1 format + * @return mapped property name in v2 format, or original key if no explicit mapping exists + */ + public String getV2MappedKey(String v1Key) { + if (v1Key == null) { + return null; + } + return v1ToV2Mappings.getOrDefault(v1Key, v1Key); + } + + /** + * Gets the unmodifiable set of known v1 property keys. + * + * @return set of v1 property keys + */ + public Set getV1KnownProperties() { + return v1KnownProperties; + } + + /** + * Gets the unmodifiable set of known v2 property keys. + * + * @return set of v2 property keys + */ + public Set getV2KnownProperties() { + return v2KnownProperties; + } + + /** + * Gets the unmodifiable set of deprecated v1 property keys without conversion. + * + * @return set of deprecated property keys + */ + public Set getV1DeprecatedProperties() { + return v1DeprecatedProperties; + } + + /** + * Gets the unmodifiable map of v1-to-v2 property mappings. + * + * @return map of v1-to-v2 property mappings + */ + public Map getV1ToV2Mappings() { + return v1ToV2Mappings; + } +} diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java new file mode 100644 index 000000000..7544fd144 --- /dev/null +++ b/migration-helpers/src/main/java/com/clickhouse/migration/config/ConfigurationMigrationHelper.java @@ -0,0 +1,242 @@ +package com.clickhouse.migration.config; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Migration helper for converting ClickHouse configuration properties and connection URLs + * from v1 (0.7.1) format to v2 (0.9.8+) format. + */ +public class ConfigurationMigrationHelper { + + private static final Logger log = LoggerFactory.getLogger(ConfigurationMigrationHelper.class); + + /** + * Prefix used for ClickHouse server settings in v2. + */ + public static final String SERVER_SETTING_PREFIX = "clickhouse_setting_"; + + /** + * Prefix used for custom HTTP headers in v2. + */ + public static final String HTTP_HEADER_PREFIX = "http_header_"; + + /** + * Converts a {@link Properties} object from v1 format to v2 format. + * + * @param v1Properties source properties in v1 format + * @return converted properties in v2 format + */ + public static Properties convertProperties(Properties v1Properties) { + if (v1Properties == null) { + return new Properties(); + } + Properties v2Properties = new Properties(); + Map convertedMap = convertMap(propertiesToMap(v1Properties)); + for (Map.Entry entry : convertedMap.entrySet()) { + if (entry.getKey() != null && entry.getValue() != null) { + v2Properties.setProperty(entry.getKey(), entry.getValue()); + } + } + return v2Properties; + } + + /** + * Converts a configuration map from v1 format to v2 format. + * Unprefixed ClickHouse server settings are automatically prefixed with {@code clickhouse_setting_}. + * Custom HTTP headers are prefixed with {@code http_header_}. + * Renamed v1 client/driver properties are mapped to their corresponding v2 names. + * + * @param v1Config map of configuration key-values in v1 format + * @return converted map in v2 format + */ + public static Map convertMap(Map v1Config) { + if (v1Config == null) { + return new LinkedHashMap<>(); + } + ConfigPropertyCache cache = ConfigPropertyCache.getInstance(); + Map v2Config = new LinkedHashMap<>(); + + for (Map.Entry entry : v1Config.entrySet()) { + String origKey = entry.getKey(); + String value = entry.getValue(); + + if (origKey == null) { + continue; + } + + String key = origKey.trim(); + + // 1. If key already starts with clickhouse_setting_ or http_header_, preserve it as is. + if (key.toLowerCase().startsWith(SERVER_SETTING_PREFIX) || key.toLowerCase().startsWith(HTTP_HEADER_PREFIX)) { + v2Config.put(key, value); + continue; + } + + // 2. Handle legacy composite setting properties: custom_settings, custom_http_params, custom_params + if ("custom_settings".equalsIgnoreCase(key) || "custom_http_params".equalsIgnoreCase(key) || "custom_params".equalsIgnoreCase(key)) { + parseAndAddKeyValuePairs(value, SERVER_SETTING_PREFIX, v2Config); + continue; + } + + // Handle legacy composite header properties: custom_http_headers, custom_headers + if ("custom_http_headers".equalsIgnoreCase(key) || "custom_headers".equalsIgnoreCase(key)) { + parseAndAddKeyValuePairs(value, HTTP_HEADER_PREFIX, v2Config); + continue; + } + + // 3. Check for mapped renamed key (e.g. connect_timeout -> connection_timeout) + String mappedKey = cache.getV2MappedKey(key); + boolean isMapped = mappedKey != null && !mappedKey.equalsIgnoreCase(key); + + if (isMapped) { + v2Config.put(mappedKey, value); + continue; + } + + // 4. Check if key is deprecated in v2 without conversion + if (cache.isDeprecatedProperty(key)) { + log.debug("Property '{}' is deprecated in v2 without conversion and will be ignored.", key); + continue; + } + + // 5. If key is a known v2 property, keep as client/driver property + if (cache.isV2KnownProperty(key)) { + v2Config.put(key, value); + } else { + // 6. Unrecognized key: in v1 this was implicitly treated as a ClickHouse server setting. + // In v2, it must be explicitly prefixed with clickhouse_setting_ + String serverSettingKey = SERVER_SETTING_PREFIX + key; + v2Config.put(serverSettingKey, value); + } + } + + return v2Config; + } + + /** + * Converts an Object-valued configuration map from v1 to v2 format. + * + * @param v1Config map with Object values + * @return converted map with Object values + */ + public static Map convertObjectMap(Map v1Config) { + if (v1Config == null) { + return new LinkedHashMap<>(); + } + Map strMap = new LinkedHashMap<>(); + for (Map.Entry entry : v1Config.entrySet()) { + if (entry.getKey() != null) { + strMap.put(entry.getKey(), entry.getValue() != null ? entry.getValue().toString() : null); + } + } + Map convertedStrMap = convertMap(strMap); + Map result = new LinkedHashMap<>(); + for (Map.Entry entry : convertedStrMap.entrySet()) { + result.put(entry.getKey(), entry.getValue()); + } + return result; + } + + /** + * Converts a connection URL containing query parameters from v1 format to v2 format. + * Example: + * {@code jdbc:clickhouse://localhost:8123/db?max_threads=8&connect_timeout=5000} + * -> {@code jdbc:clickhouse://localhost:8123/db?clickhouse_setting_max_threads=8&connection_timeout=5000} + * + * @param url connection string/URL + * @return converted connection string/URL in v2 format + */ + public static String convertUrl(String url) { + if (url == null || url.trim().isEmpty()) { + return url; + } + + int queryIndex = url.indexOf('?'); + if (queryIndex < 0 || queryIndex == url.length() - 1) { + return url; + } + + String baseUrl = url.substring(0, queryIndex); + String queryString = url.substring(queryIndex + 1); + + Map queryParams = parseQueryString(queryString); + Map convertedParams = convertMap(queryParams); + + if (convertedParams.isEmpty()) { + return baseUrl; + } + + StringBuilder sb = new StringBuilder(baseUrl).append('?'); + boolean first = true; + for (Map.Entry entry : convertedParams.entrySet()) { + if (!first) { + sb.append('&'); + } + sb.append(entry.getKey()); + if (entry.getValue() != null) { + sb.append('=').append(entry.getValue()); + } + first = false; + } + + return sb.toString(); + } + + private static Map propertiesToMap(Properties props) { + Map map = new LinkedHashMap<>(); + for (String name : props.stringPropertyNames()) { + map.put(name, props.getProperty(name)); + } + return map; + } + + private static void parseAndAddKeyValuePairs(String valueStr, String prefix, Map targetMap) { + if (valueStr == null || valueStr.trim().isEmpty()) { + return; + } + String[] pairs = valueStr.split(","); + for (String pair : pairs) { + String trimmed = pair.trim(); + if (trimmed.isEmpty()) { + continue; + } + int eqIndex = trimmed.indexOf('='); + if (eqIndex > 0) { + String k = trimmed.substring(0, eqIndex).trim(); + String v = trimmed.substring(eqIndex + 1).trim(); + if (!k.isEmpty()) { + if (!k.toLowerCase().startsWith(prefix)) { + k = prefix + k; + } + targetMap.put(k, v); + } + } + } + } + + private static Map parseQueryString(String queryString) { + Map map = new LinkedHashMap<>(); + if (queryString == null || queryString.trim().isEmpty()) { + return map; + } + String[] pairs = queryString.split("&"); + for (String pair : pairs) { + if (pair.isEmpty()) { + continue; + } + int eqIdx = pair.indexOf('='); + if (eqIdx >= 0) { + String k = pair.substring(0, eqIdx); + String v = pair.substring(eqIdx + 1); + map.put(k, v); + } else { + map.put(pair, ""); + } + } + return map; + } +} diff --git a/migration-helpers/src/main/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExample.java b/migration-helpers/src/main/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExample.java new file mode 100644 index 000000000..54b598b53 --- /dev/null +++ b/migration-helpers/src/main/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExample.java @@ -0,0 +1,97 @@ +package com.clickhouse.migration.examples; + +import com.clickhouse.migration.config.ConfigurationMigrationHelper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +/** + * Example demonstrating how to migrate v1 JDBC configuration properties and connection URLs + * to v2 format using {@link ConfigurationMigrationHelper} before establishing a JDBC connection. + */ +public class JdbcConfigurationMigrationExample { + + private static final Logger log = LoggerFactory.getLogger(JdbcConfigurationMigrationExample.class); + + /** + * Converts legacy v1 connection properties to v2 format and creates a JDBC connection. + * + * @param url connection URL (e.g., "jdbc:clickhouse://localhost:8123/default") + * @param v1Properties legacy properties containing v1 option names and un-prefixed server settings + * @return active JDBC connection + * @throws SQLException if a database access error occurs + */ + public Connection createConnectionWithConvertedProperties(String url, Properties v1Properties) throws SQLException { + // Convert v1 properties (un-prefixed server settings, renamed keys, custom_settings) to v2 format + Properties v2Properties = ConfigurationMigrationHelper.convertProperties(v1Properties); + + // Connect using standard JDBC DriverManager with converted v2 properties + return DriverManager.getConnection(url, v2Properties); + } + + /** + * Converts a legacy v1 connection URL (containing query parameters) to v2 format + * and creates a JDBC connection. + * + * @param v1Url v1 connection URL containing query parameters (e.g. {@code "jdbc:clickhouse://localhost:8123/default?max_threads=8&connect_timeout=5000"}) + * @return active JDBC connection + * @throws SQLException if a database access error occurs + */ + public Connection createConnectionWithConvertedUrl(String v1Url) throws SQLException { + // Convert v1 URL query parameters to v2 format (e.g. max_threads -> clickhouse_setting_max_threads) + String v2Url = ConfigurationMigrationHelper.convertUrl(v1Url); + + // Connect using standard JDBC DriverManager with the converted v2 URL + return DriverManager.getConnection(v2Url); + } + + /** + * Demonstrates complete workflow: migrating v1 configuration and executing a query with the v2 JDBC driver. + * + * @throws SQLException if a database access error occurs + */ + public void executeQueryWithMigratedConfig() throws SQLException { + // 1. Build legacy v1 properties + Properties v1Props = new Properties(); + v1Props.setProperty("user", "default"); + v1Props.setProperty("password", ""); + v1Props.setProperty("connect_timeout", "10000"); // Renamed in v2 to connection_timeout + v1Props.setProperty("max_threads", "4"); // Server setting in v1; needs clickhouse_setting_ prefix in v2 + v1Props.setProperty("custom_settings", "join_use_nulls=1"); // Legacy custom_settings property + + // 2. Convert to v2 properties + Properties v2Props = ConfigurationMigrationHelper.convertProperties(v1Props); + log.info("v2Props: {}", v2Props); + + // 3. Connect and execute query with try-with-resources + String url = "jdbc:clickhouse://localhost:8123/default"; + try (Connection conn = DriverManager.getConnection(url, v2Props); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT 1")) { + + while (rs.next()) { + int value = rs.getInt(1); + // process result + } + } + } + + /** + * Main entry point demonstrating configuration migration execution. + * + * @param args command line arguments + */ + public static void main(String[] args) { + try { + new JdbcConfigurationMigrationExample().executeQueryWithMigratedConfig(); + } catch (Exception e) { + log.error("failed to query with migration config", e); + } + } +} diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties new file mode 100644 index 000000000..fa6b73323 --- /dev/null +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-deprecated-properties.properties @@ -0,0 +1,58 @@ +# Deprecated v1 properties without conversion in v2 +protocol=protocol +sslcerttype=sslcerttype +sslkeyalg=sslkeyalg +sslprotocol=sslprotocol +custom_socket_factory=custom_socket_factory +custom_socket_factory_options=custom_socket_factory_options +socket_ip_tos=socket_ip_tos +http_connection_provider=http_connection_provider +http_server_default_response=http_server_default_response +receive_query_progress=receive_query_progress +send_http_client_id=send_http_client_id +ahc_validate_after_inactivity=ahc_validate_after_inactivity +ahc_retry_on_failure=ahc_retry_on_failure +use_compilation=use_compilation +debug_measure_request_time=debug_measure_request_time +max_scheduler_threads=max_scheduler_threads +max_requests=max_requests +thread_keepalive_timeout=thread_keepalive_timeout +max_core_thread_ttl=max_core_thread_ttl +auto_discovery=auto_discovery +load_balancing_policy=load_balancing_policy +load_balancing_tags=load_balancing_tags +health_check_interval=health_check_interval +health_check_method=health_check_method +node_discovery_interval=node_discovery_interval +node_discovery_limit=node_discovery_limit +node_check_interval=node_check_interval +node_group_size=node_group_size +check_all_nodes=check_all_nodes +failover=failover +auto_session=auto_session +log_leading_comment=log_leading_comment +rename_response_column=rename_response_column +transaction_timeout=transaction_timeout +repeat_on_session_lock=repeat_on_session_lock +buffering=buffering +buffer_queue_variation=buffer_queue_variation +use_blocking_queue=use_blocking_queue +read_buffer_size=read_buffer_size +write_buffer_size=write_buffer_size +request_chunk_size=request_chunk_size +request_buffering=request_buffering +response_buffering=response_buffering +compress_algorithm=compress_algorithm +decompress_algorithm=decompress_algorithm +compress_level=compress_level +decompress_level=decompress_level +max_buffer_size=max_buffer_size +max_mapper_cache=max_mapper_cache +max_queued_buffers=max_queued_buffers +max_queued_requests=max_queued_requests +rounding_mode=rounding_mode +srv_resolve=srv_resolve +reuse_value_wrapper=reuse_value_wrapper +widen_unsigned_types=widen_unsigned_types +use_objects_in_arrays=use_objects_in_arrays +use_server_time_zone_for_dates=use_server_time_zone_for_dates diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-known-properties.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-known-properties.properties new file mode 100644 index 000000000..dc13fac38 --- /dev/null +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-known-properties.properties @@ -0,0 +1,131 @@ +# Known client and driver options in v1 (0.7.1) +async=async +auto_discovery=auto_discovery +custom_settings=custom_settings +custom_socket_factory=custom_socket_factory +custom_socket_factory_options=custom_socket_factory_options +load_balancing_policy=load_balancing_policy +load_balancing_tags=load_balancing_tags +health_check_interval=health_check_interval +health_check_method=health_check_method +node_discovery_interval=node_discovery_interval +node_discovery_limit=node_discovery_limit +node_check_interval=node_check_interval +node_group_size=node_group_size +check_all_nodes=check_all_nodes +buffer_size=buffer_size +buffer_queue_variation=buffer_queue_variation +read_buffer_size=read_buffer_size +write_buffer_size=write_buffer_size +request_chunk_size=request_chunk_size +request_buffering=request_buffering +response_buffering=response_buffering +client_name=client_name +compress=compress +decompress=decompress +compress_algorithm=compress_algorithm +decompress_algorithm=decompress_algorithm +compress_level=compress_level +decompress_level=decompress_level +connect_timeout=connect_timeout +database=database +failover=failover +format=format +log_leading_comment=log_leading_comment +max_buffer_size=max_buffer_size +max_mapper_cache=max_mapper_cache +max_execution_time=max_execution_time +max_queued_buffers=max_queued_buffers +max_queued_requests=max_queued_requests +max_result_rows=max_result_rows +result_overflow_mode=result_overflow_mode +max_threads_per_client=max_threads_per_client +max_core_thread_ttl=max_core_thread_ttl +product_name=product_name +rename_response_column=rename_response_column +retry=retry +repeat_on_session_lock=repeat_on_session_lock +reuse_value_wrapper=reuse_value_wrapper +server_revision=server_revision +server_time_zone=server_time_zone +server_version=server_version +session_id=session_id +session_check=session_check +session_timeout=session_timeout +socket_timeout=socket_timeout +socket_reuseaddr=socket_reuseaddr +socket_keepalive=socket_keepalive +socket_linger=socket_linger +socket_ip_tos=socket_ip_tos +socket_tcp_nodelay=socket_tcp_nodelay +socket_rcvbuf=socket_rcvbuf +socket_sndbuf=socket_sndbuf +ssl=ssl +sslmode=sslmode +sslrootcert=sslrootcert +sslcert=sslcert +sslkey=sslkey +key_store_type=key_store_type +trust_store=trust_store +key_store_password=key_store_password +transaction_timeout=transaction_timeout +widen_unsigned_types=widen_unsigned_types +use_binary_string=use_binary_string +use_blocking_queue=use_blocking_queue +use_compilation=use_compilation +use_objects_in_arrays=use_objects_in_arrays +proxy_type=proxy_type +proxy_host=proxy_host +proxy_port=proxy_port +proxy_username=proxy_username +proxy_password=proxy_password +use_server_time_zone=use_server_time_zone +use_server_time_zone_for_dates=use_server_time_zone_for_dates +use_time_zone=use_time_zone +query_id=query_id +connection_ttl=connection_ttl +debug_measure_request_time=debug_measure_request_time +ssl_socket_sni=ssl_socket_sni +http_connection_provider=http_connection_provider +custom_http_headers=custom_http_headers +custom_headers=custom_headers +custom_http_params=custom_http_params +custom_params=custom_params +http_server_default_response=http_server_default_response +http_keep_alive=http_keep_alive +max_open_connections=max_open_connections +receive_query_progress=receive_query_progress +send_http_client_id=send_http_client_id +wait_end_of_query=wait_end_of_query +remember_last_set_roles=remember_last_set_roles +ahc_validate_after_inactivity=ahc_validate_after_inactivity +ahc_retry_on_failure=ahc_retry_on_failure +connection_reuse_strategy=connection_reuse_strategy +alive_timeout=alive_timeout +http_use_basic_auth=http_use_basic_auth +autoCommit=autoCommit +createDatabaseIfNotExist=createDatabaseIfNotExist +continueBatchOnError=continueBatchOnError +databaseTerm=databaseTerm +dialect=dialect +externalDatabase=externalDatabase +fetchSize=fetchSize +localFile=localFile +jdbcCompliant=jdbcCompliant +namedParameter=namedParameter +nullAsDefault=nullAsDefault +transactionSupport=transactionSupport +typeMappings=typeMappings +wrapperObject=wrapperObject +user=user +password=password +protocol=protocol +sslcerttype=sslcerttype +sslkeyalg=sslkeyalg +sslprotocol=sslprotocol +max_scheduler_threads=max_scheduler_threads +max_requests=max_requests +thread_keepalive_timeout=thread_keepalive_timeout +buffering=buffering +time_zone=time_zone +version=version diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties new file mode 100644 index 000000000..2fb9d807b --- /dev/null +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v1-to-v2-mappings.properties @@ -0,0 +1,15 @@ +# Property mappings from v1 keys to v2 keys +connect_timeout=connection_timeout +buffer_size=client_network_buffer_size +sslmode=ssl_mode +sslkey=ssl_key +proxy_username=proxy_user +alive_timeout=http_keep_alive_timeout +http_keep_alive=http_keep_alive_timeout +version=server_version +server_revision=server_version +time_zone=server_time_zone +product_name=client_name +use_binary_string=binary_string_support +typeMappings=jdbc_type_mappings +databaseTerm=jdbc_schema_term diff --git a/migration-helpers/src/main/resources/com/clickhouse/migration/config/v2-known-properties.properties b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v2-known-properties.properties new file mode 100644 index 000000000..1457254e8 --- /dev/null +++ b/migration-helpers/src/main/resources/com/clickhouse/migration/config/v2-known-properties.properties @@ -0,0 +1,85 @@ +# Known client and driver options in v2 (0.9.8) +session_db_roles=session_db_roles +http_use_basic_auth=http_use_basic_auth +user=user +password=password +max_open_connections=max_open_connections +http_keep_alive_timeout=http_keep_alive_timeout +use_server_time_zone=use_server_time_zone +use_time_zone=use_time_zone +server_version=server_version +server_time_zone=server_time_zone +async=async +connection_ttl=connection_ttl +connection_timeout=connection_timeout +connection_reuse_strategy=connection_reuse_strategy +socket_timeout=socket_timeout +socket_rcvbuf=socket_rcvbuf +socket_sndbuf=socket_sndbuf +socket_reuseaddr=socket_reuseaddr +socket_keepalive=socket_keepalive +socket_tcp_nodelay=socket_tcp_nodelay +socket_linger=socket_linger +database=database +compress=compress +decompress=decompress +client.use_http_compression=client.use_http_compression +compression.lz4.uncompressed_buffer_size=compression.lz4.uncompressed_buffer_size +disable_native_compression=disable_native_compression +proxy_type=proxy_type +proxy_host=proxy_host +proxy_port=proxy_port +proxy_user=proxy_user +proxy_password=proxy_password +max_execution_time=max_execution_time +trust_store=trust_store +key_store_type=key_store_type +ssl_key_store=ssl_key_store +key_store_password=key_store_password +ssl_key=ssl_key +sslrootcert=sslrootcert +sslcert=sslcert +ssl_mode=ssl_mode +ssl_context=ssl_context +retry=retry +format=format +max_threads_per_client=max_threads_per_client +query_id=query_id +client_network_buffer_size=client_network_buffer_size +access_token=access_token +ssl_authentication=ssl_authentication +connection_pool_enabled=connection_pool_enabled +connection_request_timeout=connection_request_timeout +client_retry_on_failures=client_retry_on_failures +client_name=client_name +product_name=product_name +bearer_token=bearer_token +app_compressed_data=app_compressed_data +metrics_name=metrics_name +client.http.cookies_enabled=client.http.cookies_enabled +client_allow_binary_reader_to_reuse_buffers=client_allow_binary_reader_to_reuse_buffers +type_hint_mapping=type_hint_mapping +binary_string_support=binary_string_support +ssl_socket_sni=ssl_socket_sni +client.http.use_form_request_for_query=client.http.use_form_request_for_query +json_disable_number_quoting=json_disable_number_quoting +custom_settings_prefix=custom_settings_prefix +ssl_cipher_suites=ssl_cipher_suites + +# JDBC Driver V2 specific properties +jdbc_ignore_unsupported_values=jdbc_ignore_unsupported_values +jdbc_schema_term=jdbc_schema_term +ssl=ssl +default_query_settings=default_query_settings +beta.row_binary_for_simple_insert=beta.row_binary_for_simple_insert +jdbc_resultset_auto_close=jdbc_resultset_auto_close +jdbc_use_max_result_rows=jdbc_use_max_result_rows +jdbc_sql_parser=jdbc_sql_parser +jdbc_query_id_generator=jdbc_query_id_generator +remember_last_set_roles=remember_last_set_roles +custom_http_params=custom_http_params +custom_settings=custom_settings +jdbc_cluster_name=jdbc_cluster_name +jdbc_type_mappings=jdbc_type_mappings +typeMappings=typeMappings +jdbc_json_parser_factory=jdbc_json_parser_factory diff --git a/migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseJdbcMigrationIntegrationTest.java b/migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseJdbcMigrationIntegrationTest.java new file mode 100644 index 000000000..f08b155e4 --- /dev/null +++ b/migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseJdbcMigrationIntegrationTest.java @@ -0,0 +1,94 @@ +package com.clickhouse.migration.config; + +import com.clickhouse.client.BaseIntegrationTest; +import com.clickhouse.client.ClickHouseProtocol; +import com.clickhouse.client.ClickHouseServerForTest; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +public class ClickHouseJdbcMigrationIntegrationTest extends BaseIntegrationTest { + + @BeforeClass + public static void setUpContainer() { + ClickHouseServerForTest.beforeSuite(); + try { + Class.forName("com.clickhouse.jdbc.Driver"); + } catch (ClassNotFoundException e) { + throw new RuntimeException("ClickHouse JDBC Driver not found on classpath", e); + } + } + + @AfterClass + public static void tearDownContainer() { + ClickHouseServerForTest.afterSuite(); + } + + @Test + public void testMinimalJdbcConnectivityWithConvertedProperties() throws SQLException { + String hostAndPort = ClickHouseServerForTest.getClickHouseAddress(ClickHouseProtocol.HTTP, false); + String database = ClickHouseServerForTest.getDatabase(); + String url = "jdbc:clickhouse:http://" + hostAndPort + "/" + database; + + // Legacy v1 properties containing renamed properties, un-prefixed server settings, and deprecated properties + Properties v1Props = new Properties(); + v1Props.setProperty("user", ClickHouseServerForTest.getUsername()); + v1Props.setProperty("password", ClickHouseServerForTest.getPassword()); + v1Props.setProperty("connect_timeout", "5000"); // Renamed to connection_timeout + v1Props.setProperty("buffer_size", "65536"); // Renamed to client_network_buffer_size + v1Props.setProperty("max_threads", "4"); // Server setting -> clickhouse_setting_max_threads + v1Props.setProperty("protocol", "http"); // Deprecated -> ignored + v1Props.setProperty("use_compilation", "true"); // Deprecated -> ignored + v1Props.setProperty("custom_settings", "join_use_nulls=1"); + + Properties v2Props = ConfigurationMigrationHelper.convertProperties(v1Props); + + // Verify minimal JDBC connectivity with converted properties + try (Connection conn = DriverManager.getConnection(url, v2Props); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT 1")) { + + Assert.assertTrue(rs.next(), "ResultSet should contain at least one row."); + Assert.assertEquals(rs.getInt(1), 1, "First column should be 1."); + } + } + + @Test + public void testMinimalJdbcConnectivityWithConvertedUrl() throws SQLException { + String hostAndPort = ClickHouseServerForTest.getClickHouseAddress(ClickHouseProtocol.HTTP, false); + String database = ClickHouseServerForTest.getDatabase(); + String password = ClickHouseServerForTest.getPassword(); + String username = ClickHouseServerForTest.getUsername(); + + // Legacy v1 connection URL containing query parameters + String v1Url = "jdbc:clickhouse:http://" + hostAndPort + "/" + database + + "?user=" + username + + "&password=" + password + + "&connect_timeout=5000&max_threads=4&protocol=http&use_compilation=true"; + + String v2Url = ConfigurationMigrationHelper.convertUrl(v1Url); + + Assert.assertFalse(v2Url.contains("protocol="), "Converted URL should not contain deprecated protocol param."); + Assert.assertFalse(v2Url.contains("use_compilation="), "Converted URL should not contain deprecated use_compilation param."); + Assert.assertTrue(v2Url.contains("connection_timeout=5000"), "Converted URL should contain connection_timeout=5000."); + Assert.assertTrue(v2Url.contains("clickhouse_setting_max_threads=4"), "Converted URL should contain clickhouse_setting_max_threads=4."); + + // Verify minimal JDBC connectivity with converted URL + try (Connection conn = DriverManager.getConnection(v2Url); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("SELECT 1, 'test_container'")) { + + Assert.assertTrue(rs.next(), "ResultSet should contain at least one row."); + Assert.assertEquals(rs.getInt(1), 1, "First column should be 1."); + Assert.assertEquals(rs.getString(2), "test_container", "Second column should match 'test_container'."); + } + } +} diff --git a/migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseOptionMigrationTest.java b/migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseOptionMigrationTest.java new file mode 100644 index 000000000..58bbd88ed --- /dev/null +++ b/migration-helpers/src/test/java/com/clickhouse/migration/config/ClickHouseOptionMigrationTest.java @@ -0,0 +1,130 @@ +package com.clickhouse.migration.config; + +import com.clickhouse.client.api.Client; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.config.ClickHouseDefaults; +import com.clickhouse.client.http.config.ClickHouseHttpOption; +import com.clickhouse.config.ClickHouseOption; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +public class ClickHouseOptionMigrationTest { + + @Test + public void testAllClickHouseOptionDerivativesAreConvertedOrCleanedUp() { + Map v1OptionsMap = new LinkedHashMap<>(); + + // 1. Add all enum options from ClickHouseClientOption + for (ClickHouseClientOption opt : ClickHouseClientOption.values()) { + if (opt.getKey() != null) { + v1OptionsMap.put(opt.getKey(), getSampleValueForOption(opt.getKey())); + } + } + + // 2. Add all enum options from ClickHouseHttpOption + for (ClickHouseHttpOption opt : ClickHouseHttpOption.values()) { + if (opt.getKey() != null) { + v1OptionsMap.put(opt.getKey(), getSampleValueForOption(opt.getKey())); + } + } + + // 3. Add all enum options from ClickHouseDefaults + for (ClickHouseDefaults opt : ClickHouseDefaults.values()) { + if (opt.getKey() != null) { + v1OptionsMap.put(opt.getKey(), getSampleValueForOption(opt.getKey())); + } + } + + // 4. Add all known v1 options from cache + ConfigPropertyCache cache = ConfigPropertyCache.getInstance(); + for (String v1Key : cache.getV1KnownProperties()) { + v1OptionsMap.put(v1Key, getSampleValueForOption(v1Key)); + } + + Assert.assertFalse(v1OptionsMap.isEmpty(), "v1OptionsMap should contain ClickHouseOption derivatives."); + + // Convert options map + Map convertedMap = ConfigurationMigrationHelper.convertMap(v1OptionsMap); + Set deprecatedProps = cache.getV1DeprecatedProperties(); + + // Verify conversion rules for every single v1 option + for (Map.Entry entry : v1OptionsMap.entrySet()) { + String origKey = entry.getKey(); + + if (deprecatedProps.contains(origKey) || deprecatedProps.contains(origKey.toLowerCase())) { + // Deprecated options without conversion must NOT be in the converted map + Assert.assertFalse(convertedMap.containsKey(origKey), + "Deprecated option '" + origKey + "' should have been cleaned up/ignored."); + Assert.assertFalse(convertedMap.containsKey(ConfigurationMigrationHelper.SERVER_SETTING_PREFIX + origKey), + "Deprecated option '" + origKey + "' should not be converted to a server setting."); + } else if ("custom_settings".equalsIgnoreCase(origKey) || "custom_http_params".equalsIgnoreCase(origKey) || "custom_params".equalsIgnoreCase(origKey) || "custom_http_headers".equalsIgnoreCase(origKey) || "custom_headers".equalsIgnoreCase(origKey)) { + // Legacy composite properties are unpacked into individual clickhouse_setting_ / http_header_ entries + Assert.assertFalse(convertedMap.containsKey(origKey), + "Composite option '" + origKey + "' should be unpacked rather than remaining as a raw property."); + } else { + String mappedKey = cache.getV2MappedKey(origKey); + if (mappedKey != null && !mappedKey.equalsIgnoreCase(origKey)) { + // Mapped keys should be converted to their v2 name + Assert.assertTrue(convertedMap.containsKey(mappedKey), + "Mapped option '" + origKey + "' -> '" + mappedKey + "' should be present in converted map."); + } else if (cache.isV2KnownProperty(origKey)) { + // Known v2 property should remain present + Assert.assertTrue(convertedMap.containsKey(origKey), + "Known v2 option '" + origKey + "' should be present in converted map."); + } else { + // Un-prefixed server settings should receive clickhouse_setting_ prefix + String expectedSettingKey = ConfigurationMigrationHelper.SERVER_SETTING_PREFIX + origKey; + Assert.assertTrue(convertedMap.containsKey(expectedSettingKey), + "Unrecognized option '" + origKey + "' should be prefixed with " + expectedSettingKey); + } + } + } + } + + @Test + public void testClientInstantiationWithConvertedOptionsMap() { + Map v1Props = new LinkedHashMap<>(); + v1Props.put("user", "default"); + v1Props.put("password", "secret"); + v1Props.put("database", "default"); + v1Props.put("connect_timeout", "5000"); + v1Props.put("buffer_size", "65536"); + v1Props.put("max_threads", "8"); + v1Props.put("protocol", "http"); // deprecated - ignored + v1Props.put("use_compilation", "true"); // deprecated - ignored + v1Props.put("custom_settings", "join_use_nulls=1"); + v1Props.put("custom_http_headers", "X-App-Name=test"); + + Map convertedMap = ConfigurationMigrationHelper.convertMap(v1Props); + + // Client in client-v2 should be successfully instantiated with converted options + try (Client client = new Client.Builder() + .addEndpoint("http://localhost:8123") + .setOptions(convertedMap) + .build()) { + + Assert.assertNotNull(client, "Client instance should be successfully created."); + } + } + + private String getSampleValueForOption(String key) { + if ("connect_timeout".equalsIgnoreCase(key) || "socket_timeout".equalsIgnoreCase(key) || "alive_timeout".equalsIgnoreCase(key)) { + return "5000"; + } + if ("buffer_size".equalsIgnoreCase(key) || "read_buffer_size".equalsIgnoreCase(key)) { + return "65536"; + } + if ("ssl".equalsIgnoreCase(key) || "async".equalsIgnoreCase(key) || "compress".equalsIgnoreCase(key)) { + return "true"; + } + if ("port".equalsIgnoreCase(key)) { + return "8123"; + } + return "sample_value"; + } +} diff --git a/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java b/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java new file mode 100644 index 000000000..722a5fdef --- /dev/null +++ b/migration-helpers/src/test/java/com/clickhouse/migration/config/ConfigurationMigrationHelperTest.java @@ -0,0 +1,170 @@ +package com.clickhouse.migration.config; + +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +public class ConfigurationMigrationHelperTest { + + @DataProvider(name = "propertyConversionData") + public Object[][] providePropertyConversionData() { + return new Object[][]{ + // Standard known v2 properties should remain as-is + {"user", "default", "user", "default"}, + {"password", "secret", "password", "secret"}, + {"database", "analytics", "database", "analytics"}, + {"ssl", "true", "ssl", "true"}, + {"async", "false", "async", "false"}, + + // Unprefixed server settings in v1 must be prefixed with clickhouse_setting_ in v2 + {"max_threads", "8", "clickhouse_setting_max_threads", "8"}, + {"date_time_input_format", "best_effort", "clickhouse_setting_date_time_input_format", "best_effort"}, + {"join_use_nulls", "1", "clickhouse_setting_join_use_nulls", "1"}, + + // Renamed properties in v1 should be converted to v2 names + {"connect_timeout", "10000", "connection_timeout", "10000"}, + {"buffer_size", "65536", "client_network_buffer_size", "65536"}, + {"sslmode", "strict", "ssl_mode", "strict"}, + {"sslkey", "/path/to/key", "ssl_key", "/path/to/key"}, + {"proxy_username", "puser", "proxy_user", "puser"}, + {"alive_timeout", "60000", "http_keep_alive_timeout", "60000"}, + {"http_keep_alive", "60000", "http_keep_alive_timeout", "60000"}, + {"version", "23.8", "server_version", "23.8"}, + {"server_revision", "54460", "server_version", "54460"}, + {"time_zone", "UTC", "server_time_zone", "UTC"}, + {"product_name", "my-app", "client_name", "my-app"}, + {"use_binary_string", "true", "binary_string_support", "true"}, + + // Existing v2 prefixed properties should be preserved + {"clickhouse_setting_max_execution_time", "60", "clickhouse_setting_max_execution_time", "60"}, + {"http_header_X-Custom-Header", "custom-val", "http_header_X-Custom-Header", "custom-val"} + }; + } + + @Test(dataProvider = "propertyConversionData") + public void testConvertSingleProperty(String inputKey, String inputValue, String expectedKey, String expectedValue) { + Map input = new HashMap<>(); + input.put(inputKey, inputValue); + + Map result = ConfigurationMigrationHelper.convertMap(input); + + Assert.assertEquals(result.size(), 1); + Assert.assertTrue(result.containsKey(expectedKey), "Expected key missing: " + expectedKey); + Assert.assertEquals(result.get(expectedKey), expectedValue); + } + + @Test + public void testConvertCustomSettingsAndHeaders() { + Map input = new LinkedHashMap<>(); + input.put("user", "my_user"); + input.put("custom_settings", "max_threads=4, join_use_nulls=1"); + input.put("custom_http_headers", "X-Trace-Id=123, X-App-Name=demo"); + + Map result = ConfigurationMigrationHelper.convertMap(input); + + Assert.assertEquals(result.get("user"), "my_user"); + Assert.assertEquals(result.get("clickhouse_setting_max_threads"), "4"); + Assert.assertEquals(result.get("clickhouse_setting_join_use_nulls"), "1"); + Assert.assertEquals(result.get("http_header_X-Trace-Id"), "123"); + Assert.assertEquals(result.get("http_header_X-App-Name"), "demo"); + } + + @Test + public void testConvertPropertiesObject() { + Properties v1Props = new Properties(); + v1Props.setProperty("user", "default"); + v1Props.setProperty("connect_timeout", "5000"); + v1Props.setProperty("max_threads", "16"); + + Properties v2Props = ConfigurationMigrationHelper.convertProperties(v1Props); + + Assert.assertEquals(v2Props.getProperty("user"), "default"); + Assert.assertEquals(v2Props.getProperty("connection_timeout"), "5000"); + Assert.assertEquals(v2Props.getProperty("clickhouse_setting_max_threads"), "16"); + } + + @DataProvider(name = "urlConversionData") + public Object[][] provideUrlConversionData() { + return new Object[][]{ + { + "jdbc:clickhouse://localhost:8123/default?user=default&connect_timeout=5000&max_threads=8", + "jdbc:clickhouse://localhost:8123/default?user=default&connection_timeout=5000&clickhouse_setting_max_threads=8" + }, + { + "http://localhost:8123/?ssl=true&date_time_input_format=best_effort", + "http://localhost:8123/?ssl=true&clickhouse_setting_date_time_input_format=best_effort" + }, + { + "jdbc:clickhouse://localhost:8123/db", + "jdbc:clickhouse://localhost:8123/db" + } + }; + } + + @Test(dataProvider = "urlConversionData") + public void testConvertUrl(String inputUrl, String expectedUrl) { + String resultUrl = ConfigurationMigrationHelper.convertUrl(inputUrl); + Assert.assertEquals(resultUrl, expectedUrl); + } + + @Test + public void testDeprecatedPropertiesWithoutConversionAreIgnored() { + Map input = new LinkedHashMap<>(); + input.put("user", "default"); + input.put("protocol", "http"); + input.put("use_compilation", "true"); + input.put("socket_ip_tos", "0"); + input.put("http_connection_provider", "custom"); + input.put("auto_discovery", "true"); + input.put("buffering", "true"); + input.put("max_requests", "10"); + input.put("failover", "2"); + + Map result = ConfigurationMigrationHelper.convertMap(input); + + Assert.assertEquals(result.size(), 1); + Assert.assertEquals(result.get("user"), "default"); + Assert.assertFalse(result.containsKey("protocol")); + Assert.assertFalse(result.containsKey("clickhouse_setting_protocol")); + Assert.assertFalse(result.containsKey("use_compilation")); + Assert.assertFalse(result.containsKey("buffering")); + } + + @Test + public void testConvertUrlWithDeprecatedProperties() { + String inputUrl = "jdbc:clickhouse://localhost:8123/default?protocol=http&use_compilation=true&connect_timeout=5000"; + String expectedUrl = "jdbc:clickhouse://localhost:8123/default?connection_timeout=5000"; + + String resultUrl = ConfigurationMigrationHelper.convertUrl(inputUrl); + Assert.assertEquals(resultUrl, expectedUrl); + } + + @Test + public void testCacheInitializationAndPreload() { + ConfigPropertyCache cache = ConfigPropertyCache.getInstance(); + + Assert.assertTrue(cache.isV1KnownProperty("connect_timeout")); + Assert.assertTrue(cache.isV2KnownProperty("connection_timeout")); + Assert.assertTrue(cache.isV2KnownProperty("user")); + Assert.assertTrue(cache.isDeprecatedProperty("protocol")); + Assert.assertTrue(cache.isDeprecatedProperty("use_compilation")); + + Assert.assertEquals(cache.getV2MappedKey("connect_timeout"), "connection_timeout"); + Assert.assertEquals(cache.getV2MappedKey("buffer_size"), "client_network_buffer_size"); + } + + @Test + public void testClasspathReflectionHandlesMissingClassesGracefully() { + ConfigPropertyCache cache = ConfigPropertyCache.getInstance(); + + // Verify cache instance is non-null and functioning even when checking optional classpath classes + Assert.assertNotNull(cache, "Cache should initialize without throwing exceptions when checking classpath enums."); + Assert.assertNotNull(cache.getV2KnownProperties()); + Assert.assertNotNull(cache.getV1KnownProperties()); + } +} diff --git a/migration-helpers/src/test/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExampleTest.java b/migration-helpers/src/test/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExampleTest.java new file mode 100644 index 000000000..77a1e053e --- /dev/null +++ b/migration-helpers/src/test/java/com/clickhouse/migration/examples/JdbcConfigurationMigrationExampleTest.java @@ -0,0 +1,28 @@ +package com.clickhouse.migration.examples; + +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.Properties; + +public class JdbcConfigurationMigrationExampleTest { + + @Test + public void testExampleMethods() throws Exception { + JdbcConfigurationMigrationExample example = new JdbcConfigurationMigrationExample(); + + Properties v1Props = new Properties(); + v1Props.setProperty("user", "default"); + v1Props.setProperty("connect_timeout", "5000"); + v1Props.setProperty("max_threads", "8"); + + // Verify conversion logic works as demonstrated in example + String v1Url = "jdbc:clickhouse://localhost:8123/default?connect_timeout=5000&max_threads=8"; + String convertedUrl = com.clickhouse.migration.config.ConfigurationMigrationHelper.convertUrl(v1Url); + Assert.assertEquals(convertedUrl, "jdbc:clickhouse://localhost:8123/default?connection_timeout=5000&clickhouse_setting_max_threads=8"); + + Properties convertedProps = com.clickhouse.migration.config.ConfigurationMigrationHelper.convertProperties(v1Props); + Assert.assertEquals(convertedProps.getProperty("connection_timeout"), "5000"); + Assert.assertEquals(convertedProps.getProperty("clickhouse_setting_max_threads"), "8"); + } +}