diff --git a/Chart.yaml b/Chart.yaml index 2275c26..8185d59 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 name: pgdog -version: v0.97 +version: v0.98 appVersion: "v0.1.63" diff --git a/templates/config.yaml b/templates/config.yaml index 804959f..3a11b39 100644 --- a/templates/config.yaml +++ b/templates/config.yaml @@ -144,6 +144,9 @@ data: {{- if hasKey .Values "passthroughAuth" }} passthrough_auth = {{ .Values.passthroughAuth | quote }} {{- end }} + {{- if hasKey .Values "passthroughAuthDebounceDelay" }} + passthrough_auth_debounce_delay = {{ include "pgdog.intval" .Values.passthroughAuthDebounceDelay }} + {{- end }} {{- if hasKey .Values "connectTimeout" }} connect_timeout = {{ include "pgdog.intval" .Values.connectTimeout }} {{- end }} @@ -156,6 +159,9 @@ data: {{- if hasKey .Values "queryTimeout" }} query_timeout = {{ include "pgdog.intval" .Values.queryTimeout }} {{- end }} + {{- if hasKey .Values "queryCancellationTimeout" }} + query_cancellation_timeout = {{ include "pgdog.intval" .Values.queryCancellationTimeout }} + {{- end }} {{- if hasKey .Values "clientIdleInTransactionTimeout" }} client_idle_in_transaction_timeout = {{ include "pgdog.intval" .Values.clientIdleInTransactionTimeout }} {{- end }} @@ -197,6 +203,9 @@ data: {{- if hasKey .Values "reshardingParallelCopies" }} resharding_parallel_copies = {{ include "pgdog.intval" .Values.reshardingParallelCopies }} {{- end }} + {{- if hasKey .Values "reshardingParallelWithinTableCopies" }} + resharding_parallel_within_table_copies = {{ include "pgdog.intval" .Values.reshardingParallelWithinTableCopies }} + {{- end }} {{- if hasKey .Values "reshardingCopyRetryMaxAttempts" }} resharding_copy_retry_max_attempts = {{ include "pgdog.intval" .Values.reshardingCopyRetryMaxAttempts }} {{- end }} @@ -308,9 +317,6 @@ data: {{- if hasKey .Values "cutoverTimeoutAction" }} cutover_timeout_action = {{ .Values.cutoverTimeoutAction | quote }} {{- end }} - {{- if hasKey .Values "cutoverSaveConfig" }} - cutover_save_config = {{ .Values.cutoverSaveConfig }} - {{- end }} {{- range .Values.databases }} [[databases]] @@ -410,6 +416,9 @@ data: {{- if hasKey . "name" }} name = {{ .name | quote }} {{- end }} + {{- if hasKey . "prefix" }} + prefix = {{ .prefix }} + {{- end }} column = {{ .column | quote }} {{- with .dataType }} data_type = {{ . | quote }} @@ -543,6 +552,17 @@ data: {{- if hasKey .Values.qos "writeBackoffQueueSize" }} write_backoff_queue_size = {{ include "pgdog.intval" .Values.qos.writeBackoffQueueSize }} {{- end }} + {{- with .Values.qos.unit }} + [qos.unit] + regex = {{ required "qos.unit.regex is required" .regex | quote }} + template = {{ required "qos.unit.template is required" .template | quote }} + {{- if hasKey . "regexSizeLimit" }} + regex_size_limit = {{ include "pgdog.intval" .regexSizeLimit }} + {{- end }} + {{- if hasKey . "maxUnitLength" }} + max_unit_length = {{ include "pgdog.intval" .maxUnitLength }} + {{- end }} + {{- end }} {{- end }} {{- with .Values.queryStats }} @@ -597,17 +617,14 @@ data: {{- if hasKey . "activeQueriesInterval" }} active_queries_interval = {{ include "pgdog.intval" .activeQueriesInterval }} {{- end }} - {{- if hasKey . "errorsInterval" }} - errors_interval = {{ include "pgdog.intval" .errorsInterval }} - {{- end }} {{- if hasKey . "requestTimeout" }} request_timeout = {{ include "pgdog.intval" .requestTimeout }} {{- end }} - {{- if hasKey . "queryTimingsChunkSize" }} - query_timings_chunk_size = {{ include "pgdog.intval" .queryTimingsChunkSize }} + {{- if hasKey . "configRefreshInterval" }} + config_refresh_interval = {{ include "pgdog.intval" .configRefreshInterval }} {{- end }} - {{- if hasKey . "queryTimingsNewQueryQueueSize" }} - query_timings_new_query_queue_size = {{ include "pgdog.intval" .queryTimingsNewQueryQueueSize }} + {{- if hasKey . "taskUpdatesInterval" }} + task_updates_interval = {{ include "pgdog.intval" .taskUpdatesInterval }} {{- end }} {{- end }} {{- end }} @@ -629,6 +646,19 @@ data: {{- if hasKey .Values.rewrite "nonDeterministicFunctions" }} non_deterministic_functions = {{ .Values.rewrite.nonDeterministicFunctions | quote }} {{- end }} + {{- if hasKey .Values.rewrite "subqueries" }} + subqueries = {{ .Values.rewrite.subqueries | quote }} + {{- end }} + {{- end }} + + {{- with .Values.resharding }} + [resharding] + {{- if hasKey . "postDataValidation" }} + post_data_validation = {{ .postDataValidation | quote }} + {{- end }} + {{- if hasKey . "twoPcCleanupTimeout" }} + two_pc_cleanup_timeout = {{ include "pgdog.intval" .twoPcCleanupTimeout }} + {{- end }} {{- end }} {{- if .Values.multiTenant }} diff --git a/test/values-control.yaml b/test/values-control.yaml index 250959c..be0a286 100644 --- a/test/values-control.yaml +++ b/test/values-control.yaml @@ -7,10 +7,9 @@ control: metricsInterval: 2000 statsInterval: 10000 activeQueriesInterval: 5000 - errorsInterval: 5000 requestTimeout: 2000 - queryTimingsChunkSize: 50 - queryTimingsNewQueryQueueSize: 500 + configRefreshInterval: 30000 + taskUpdatesInterval: 2000 # Basic database config for testing control integration databases: diff --git a/test/values-pgdog-config-extras.yaml b/test/values-pgdog-config-extras.yaml index 689c09a..a3f84c8 100644 --- a/test/values-pgdog-config-extras.yaml +++ b/test/values-pgdog-config-extras.yaml @@ -5,6 +5,8 @@ # query_parsers table, database lb_weight/resharding_only, multi_tenant, otel. listenBacklog: 4_096 +passthroughAuthDebounceDelay: 0 +queryCancellationTimeout: 12_000 banReplicaLag: 60_000 banReplicaLagBytes: 10_000_000 @@ -16,6 +18,11 @@ twoPhaseCommitWalCheckpointInterval: 60 reshardingCopyRetryMaxAttempts: 3 reshardingCopyRetryMinDelay: 1_000 +reshardingParallelWithinTableCopies: 4 + +resharding: + postDataValidation: before_cutover + twoPcCleanupTimeout: 0 rewriteShardKeyUpdates: error @@ -27,7 +34,6 @@ cutoverReplicationLagThreshold: 0 cutoverLastTransactionDelay: 1_000 cutoverTimeout: 30_000 cutoverTimeoutAction: abort -cutoverSaveConfig: false tcpKeepalive: true tcpUserTimeout: 30_000 @@ -69,3 +75,13 @@ queryParsers: - database: primary level: "off" engine: pg_query_raw + +shardedTables: + - database: shard0 + name: events_ + prefix: true + column: tenant_id + - database: shard0 + name: users + prefix: false + column: id diff --git a/test/values-qos-backoff.yaml b/test/values-qos-backoff.yaml index 5bf1e6f..c966560 100644 --- a/test/values-qos-backoff.yaml +++ b/test/values-qos-backoff.yaml @@ -1,8 +1,16 @@ # Test explicitly configured write backoff settings, including zero jitter. qos: + enabled: true + maxEntries: 10_000 writeBackoffDelay: 100 writeBackoffDelayJitter: 0 writeBackoffQueueSize: 50 + unit: + # Exercise TOML escaping of regex backslashes and quotes, and literal $ groups. + regex: 'module="(?P\w+)".*?op="(?P\w+)"' + template: '${module}.${op}' + regexSizeLimit: 2_097_152 + maxUnitLength: 0 databases: - name: primary diff --git a/test/values-rewrite.yaml b/test/values-rewrite.yaml index bb42a75..e113988 100644 --- a/test/values-rewrite.yaml +++ b/test/values-rewrite.yaml @@ -4,6 +4,7 @@ rewrite: shardKey: "rewrite" splitInserts: "ignore" primaryKey: "rewrite" + subqueries: "rewrite" # Basic database config for testing rewrite databases: diff --git a/values.yaml b/values.yaml index 6f290c7..81f836a 100644 --- a/values.yaml +++ b/values.yaml @@ -74,6 +74,22 @@ port: 6432 # authTokenCacheSize sets auth_token_cache_size in the [general] section. # Omitted unless explicitly set. # authTokenCacheSize: 2000 +# Passthrough authentication debounce delay in milliseconds (default: 1000). +# passthroughAuthDebounceDelay: 1000 +# Maximum time to send a cancellation request and wait for PostgreSQL to +# acknowledge it, in milliseconds (default: 5000). +# queryCancellationTimeout: 5000 +# Number of parallel source COPY streams per table (default: 1). +# reshardingParallelWithinTableCopies: 1 +# Resharding validation and failed two-phase commit cleanup settings. +# resharding: +# # off, during_replication, before_cutover, or after_cutover +# postDataValidation: during_replication +# twoPcCleanupTimeout: 600000 # milliseconds +# Query rewrite behavior for subqueries (default: ignore; experimental upstream). +# rewrite: +# enabled: true +# subqueries: rewrite # replicas indicates how many instances of PgDog will run (HA). replicas: 2 @@ -331,6 +347,7 @@ shardedSchemas: [] # shardedTables contains the list of sharded table entries in pgdog.toml # Each entry requires: database and column; name and dataType are optional. +# Set prefix: true to match table names starting with name (default: false). # When dataType is omitted, PgDog defaults to bigint. # An entry may also carry a list/range/default mapping via the optional `mapping` field. # Each mapping rule has a target `shard`; the rule type is inferred from the fields set: @@ -654,6 +671,17 @@ grafanaRemoteWrite: # maxBackoff is the maximum retry delay maxBackoff: 5s +# QoS (PgDog EE). Optional unit rules extract a unit from SQL block comments. +# Regex capture groups are substituted into template by PgDog. +# qos: +# enabled: true +# maxEntries: 10000 +# unit: +# regex: "module='(?P[^']*)'.*?op='(?P[^']*)'" +# template: '${module}.${op}' +# regexSizeLimit: 1048576 # bytes of compiled regex memory +# maxUnitLength: 128 # bytes in a recorded unit + # Query stats (PgDog EE) queryStats: # enabled enables pgdog query stats (EE version only) @@ -690,14 +718,12 @@ control: statsInterval: 5000 # activeQueriesInterval defines how often to push active queries (in milliseconds) activeQueriesInterval: 5000 - # errorsInterval defines how often to push errors (in milliseconds) - errorsInterval: 5000 # requestTimeout defines the timeout for control plane requests (in milliseconds) requestTimeout: 1000 - # queryTimingsChunkSize defines the chunk size for query timings - queryTimingsChunkSize: 25 - # queryTimingsNewQueryQueueSize defines the queue size for new query timings - queryTimingsNewQueryQueueSize: 1000 + # configRefreshInterval defines how often to push configuration (in milliseconds) + configRefreshInterval: 60000 + # taskUpdatesInterval defines how often to push resharding tasks (in milliseconds) + taskUpdatesInterval: 1000 # LSN check configuration for replication failover auto mode (in milliseconds) # See: https://docs.pgdog.dev/features/load-balancer/replication-failover/