From 0c2229c8f24700c8e2167af2dc59489627ad0e3a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 19 Nov 2026 13:49:22 -0800 Subject: [PATCH 1/2] feat: add a bunch of settings --- Chart.yaml | 2 +- templates/config.yaml | 50 ++++++++++++++++++++++------ test/test.sh | 40 ++++++++++++++++++++++ test/values-control.yaml | 5 ++- test/values-pgdog-config-extras.yaml | 18 +++++++++- test/values-qos-backoff.yaml | 8 +++++ test/values-rewrite.yaml | 1 + values.yaml | 38 +++++++++++++++++---- 8 files changed, 141 insertions(+), 21 deletions(-) diff --git a/Chart.yaml b/Chart.yaml index 2275c26..8185d59 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -1,4 +1,4 @@ apiVersion: v1 name: pgdog -version: v0.97 +version: v0.98 appVersion: "v0.1.63" diff --git a/templates/config.yaml b/templates/config.yaml index 804959f..3a11b39 100644 --- a/templates/config.yaml +++ b/templates/config.yaml @@ -144,6 +144,9 @@ data: {{- if hasKey .Values "passthroughAuth" }} passthrough_auth = {{ .Values.passthroughAuth | quote }} {{- end }} + {{- if hasKey .Values "passthroughAuthDebounceDelay" }} + passthrough_auth_debounce_delay = {{ include "pgdog.intval" .Values.passthroughAuthDebounceDelay }} + {{- end }} {{- if hasKey .Values "connectTimeout" }} connect_timeout = {{ include "pgdog.intval" .Values.connectTimeout }} {{- end }} @@ -156,6 +159,9 @@ data: {{- if hasKey .Values "queryTimeout" }} query_timeout = {{ include "pgdog.intval" .Values.queryTimeout }} {{- end }} + {{- if hasKey .Values "queryCancellationTimeout" }} + query_cancellation_timeout = {{ include "pgdog.intval" .Values.queryCancellationTimeout }} + {{- end }} {{- if hasKey .Values "clientIdleInTransactionTimeout" }} client_idle_in_transaction_timeout = {{ include "pgdog.intval" .Values.clientIdleInTransactionTimeout }} {{- end }} @@ -197,6 +203,9 @@ data: {{- if hasKey .Values "reshardingParallelCopies" }} resharding_parallel_copies = {{ include "pgdog.intval" .Values.reshardingParallelCopies }} {{- end }} + {{- if hasKey .Values "reshardingParallelWithinTableCopies" }} + resharding_parallel_within_table_copies = {{ include "pgdog.intval" .Values.reshardingParallelWithinTableCopies }} + {{- end }} {{- if hasKey .Values "reshardingCopyRetryMaxAttempts" }} resharding_copy_retry_max_attempts = {{ include "pgdog.intval" .Values.reshardingCopyRetryMaxAttempts }} {{- end }} @@ -308,9 +317,6 @@ data: {{- if hasKey .Values "cutoverTimeoutAction" }} cutover_timeout_action = {{ .Values.cutoverTimeoutAction | quote }} {{- end }} - {{- if hasKey .Values "cutoverSaveConfig" }} - cutover_save_config = {{ .Values.cutoverSaveConfig }} - {{- end }} {{- range .Values.databases }} [[databases]] @@ -410,6 +416,9 @@ data: {{- if hasKey . "name" }} name = {{ .name | quote }} {{- end }} + {{- if hasKey . "prefix" }} + prefix = {{ .prefix }} + {{- end }} column = {{ .column | quote }} {{- with .dataType }} data_type = {{ . | quote }} @@ -543,6 +552,17 @@ data: {{- if hasKey .Values.qos "writeBackoffQueueSize" }} write_backoff_queue_size = {{ include "pgdog.intval" .Values.qos.writeBackoffQueueSize }} {{- end }} + {{- with .Values.qos.unit }} + [qos.unit] + regex = {{ required "qos.unit.regex is required" .regex | quote }} + template = {{ required "qos.unit.template is required" .template | quote }} + {{- if hasKey . "regexSizeLimit" }} + regex_size_limit = {{ include "pgdog.intval" .regexSizeLimit }} + {{- end }} + {{- if hasKey . "maxUnitLength" }} + max_unit_length = {{ include "pgdog.intval" .maxUnitLength }} + {{- end }} + {{- end }} {{- end }} {{- with .Values.queryStats }} @@ -597,17 +617,14 @@ data: {{- if hasKey . "activeQueriesInterval" }} active_queries_interval = {{ include "pgdog.intval" .activeQueriesInterval }} {{- end }} - {{- if hasKey . "errorsInterval" }} - errors_interval = {{ include "pgdog.intval" .errorsInterval }} - {{- end }} {{- if hasKey . "requestTimeout" }} request_timeout = {{ include "pgdog.intval" .requestTimeout }} {{- end }} - {{- if hasKey . "queryTimingsChunkSize" }} - query_timings_chunk_size = {{ include "pgdog.intval" .queryTimingsChunkSize }} + {{- if hasKey . "configRefreshInterval" }} + config_refresh_interval = {{ include "pgdog.intval" .configRefreshInterval }} {{- end }} - {{- if hasKey . "queryTimingsNewQueryQueueSize" }} - query_timings_new_query_queue_size = {{ include "pgdog.intval" .queryTimingsNewQueryQueueSize }} + {{- if hasKey . "taskUpdatesInterval" }} + task_updates_interval = {{ include "pgdog.intval" .taskUpdatesInterval }} {{- end }} {{- end }} {{- end }} @@ -629,6 +646,19 @@ data: {{- if hasKey .Values.rewrite "nonDeterministicFunctions" }} non_deterministic_functions = {{ .Values.rewrite.nonDeterministicFunctions | quote }} {{- end }} + {{- if hasKey .Values.rewrite "subqueries" }} + subqueries = {{ .Values.rewrite.subqueries | quote }} + {{- end }} + {{- end }} + + {{- with .Values.resharding }} + [resharding] + {{- if hasKey . "postDataValidation" }} + post_data_validation = {{ .postDataValidation | quote }} + {{- end }} + {{- if hasKey . "twoPcCleanupTimeout" }} + two_pc_cleanup_timeout = {{ include "pgdog.intval" .twoPcCleanupTimeout }} + {{- end }} {{- end }} {{- if .Values.multiTenant }} diff --git a/test/test.sh b/test/test.sh index 3439762..bbe15f6 100755 --- a/test/test.sh +++ b/test/test.sh @@ -198,5 +198,45 @@ helm template test-release "$CHART_DIR" --set-json 'lifecycle={"postStart":{"exe expect_fail "lifecycle.preStop has no explicit terminationGracePeriodSeconds" -f "$TEST_DIR/values-pod-extensions.yaml" --set terminationGracePeriodSeconds=null echo " lifecycle, extraContainers, shareProcessNamespace, automountServiceAccountToken and service.exposeMetrics render as set, and only when set" +echo "" +echo "==> Validating recent pgdog-config settings..." +config_json() { + helm template test-release "$CHART_DIR" "$@" \ + | yq -r 'select(.kind == "ConfigMap" and .metadata.name == "test-release-pgdog") | .data["pgdog.toml"]' \ + | yq -p toml -o json -I=0 +} +extras=$(config_json -f "$TEST_DIR/values-pgdog-config-extras.yaml") +expect "general timeouts and parallel copies preserve integers and zero" \ + "$(echo "$extras" | yq -p json -o json -I=0 '[.general.passthrough_auth_debounce_delay, .general.query_cancellation_timeout, .general.resharding_parallel_within_table_copies]')" \ + '[0,12000,4]' +expect "resharding options are in their own table" \ + "$(echo "$extras" | yq -p json -o json -I=0 '.resharding')" \ + '{"post_data_validation":"before_cutover","two_pc_cleanup_timeout":0}' +expect "sharded table prefixes preserve true and false" \ + "$(echo "$extras" | yq -p json -o json -I=0 '[.sharded_tables[].prefix]')" '[true,false]' +expect "removed cutover option is not emitted" \ + "$(config_json --set cutoverSaveConfig=true | yq -p json '.general | has("cutover_save_config")')" 'false' + +qos=$(config_json -f "$TEST_DIR/values-qos-backoff.yaml") +expect "QoS units preserve regex escapes and literal template groups" \ + "$(echo "$qos" | yq -p json -o json -I=0 '[.qos.unit.regex, .qos.unit.template, .qos.unit.regex_size_limit, .qos.unit.max_unit_length]')" \ + "$(yq -o json -I=0 '[.qos.unit.regex, .qos.unit.template, .qos.unit.regexSizeLimit, .qos.unit.maxUnitLength]' "$TEST_DIR/values-qos-backoff.yaml")" +expect "QoS backoff remains in the parent table" \ + "$(echo "$qos" | yq -p json -o json -I=0 '[.qos.enabled, .qos.max_entries, .qos.write_backoff_delay, .qos.write_backoff_delay_jitter, .qos.write_backoff_queue_size]')" \ + '[true,10000,100,0,50]' +expect "optional QoS limits use PgDog defaults when omitted" \ + "$(config_json -f "$TEST_DIR/values-qos-backoff.yaml" --set qos.unit.regexSizeLimit=null --set qos.unit.maxUnitLength=null | yq -p json -o json -I=0 '[.qos.unit | has("regex_size_limit"), has("max_unit_length")]')" \ + '[false,false]' +expect_fail "QoS unit has no regex" -f "$TEST_DIR/values-qos-backoff.yaml" --set qos.unit.regex=null +expect_fail "QoS unit has no template" -f "$TEST_DIR/values-qos-backoff.yaml" --set qos.unit.template=null +expect "subquery rewriting renders" \ + "$(config_json -f "$TEST_DIR/values-rewrite.yaml" | yq -p json '.rewrite.subqueries')" 'rewrite' +expect "control uses current interval fields" \ + "$(config_json -f "$TEST_DIR/values-control.yaml" | yq -p json -o json -I=0 '[.control.config_refresh_interval, .control.task_updates_interval, (.control | has("errors_interval")), (.control | has("query_timings_chunk_size")), (.control | has("query_timings_new_query_queue_size"))]')" \ + '[30000,2000,false,false,false]' +expect "optional new settings stay omitted by default" \ + "$(config_json | yq -p json -o json -I=0 '[has("qos"), has("resharding"), has("rewrite"), (.general | has("query_cancellation_timeout")), (.general | has("passthrough_auth_debounce_delay")), (.general | has("resharding_parallel_within_table_copies"))]')" \ + '[false,false,false,false,false,false]' + echo "" echo "==> All tests passed!" diff --git a/test/values-control.yaml b/test/values-control.yaml index 250959c..be0a286 100644 --- a/test/values-control.yaml +++ b/test/values-control.yaml @@ -7,10 +7,9 @@ control: metricsInterval: 2000 statsInterval: 10000 activeQueriesInterval: 5000 - errorsInterval: 5000 requestTimeout: 2000 - queryTimingsChunkSize: 50 - queryTimingsNewQueryQueueSize: 500 + configRefreshInterval: 30000 + taskUpdatesInterval: 2000 # Basic database config for testing control integration databases: diff --git a/test/values-pgdog-config-extras.yaml b/test/values-pgdog-config-extras.yaml index 689c09a..a3f84c8 100644 --- a/test/values-pgdog-config-extras.yaml +++ b/test/values-pgdog-config-extras.yaml @@ -5,6 +5,8 @@ # query_parsers table, database lb_weight/resharding_only, multi_tenant, otel. listenBacklog: 4_096 +passthroughAuthDebounceDelay: 0 +queryCancellationTimeout: 12_000 banReplicaLag: 60_000 banReplicaLagBytes: 10_000_000 @@ -16,6 +18,11 @@ twoPhaseCommitWalCheckpointInterval: 60 reshardingCopyRetryMaxAttempts: 3 reshardingCopyRetryMinDelay: 1_000 +reshardingParallelWithinTableCopies: 4 + +resharding: + postDataValidation: before_cutover + twoPcCleanupTimeout: 0 rewriteShardKeyUpdates: error @@ -27,7 +34,6 @@ cutoverReplicationLagThreshold: 0 cutoverLastTransactionDelay: 1_000 cutoverTimeout: 30_000 cutoverTimeoutAction: abort -cutoverSaveConfig: false tcpKeepalive: true tcpUserTimeout: 30_000 @@ -69,3 +75,13 @@ queryParsers: - database: primary level: "off" engine: pg_query_raw + +shardedTables: + - database: shard0 + name: events_ + prefix: true + column: tenant_id + - database: shard0 + name: users + prefix: false + column: id diff --git a/test/values-qos-backoff.yaml b/test/values-qos-backoff.yaml index 5bf1e6f..c966560 100644 --- a/test/values-qos-backoff.yaml +++ b/test/values-qos-backoff.yaml @@ -1,8 +1,16 @@ # Test explicitly configured write backoff settings, including zero jitter. qos: + enabled: true + maxEntries: 10_000 writeBackoffDelay: 100 writeBackoffDelayJitter: 0 writeBackoffQueueSize: 50 + unit: + # Exercise TOML escaping of regex backslashes and quotes, and literal $ groups. + regex: 'module="(?P\w+)".*?op="(?P\w+)"' + template: '${module}.${op}' + regexSizeLimit: 2_097_152 + maxUnitLength: 0 databases: - name: primary diff --git a/test/values-rewrite.yaml b/test/values-rewrite.yaml index bb42a75..e113988 100644 --- a/test/values-rewrite.yaml +++ b/test/values-rewrite.yaml @@ -4,6 +4,7 @@ rewrite: shardKey: "rewrite" splitInserts: "ignore" primaryKey: "rewrite" + subqueries: "rewrite" # Basic database config for testing rewrite databases: diff --git a/values.yaml b/values.yaml index 6f290c7..81f836a 100644 --- a/values.yaml +++ b/values.yaml @@ -74,6 +74,22 @@ port: 6432 # authTokenCacheSize sets auth_token_cache_size in the [general] section. # Omitted unless explicitly set. # authTokenCacheSize: 2000 +# Passthrough authentication debounce delay in milliseconds (default: 1000). +# passthroughAuthDebounceDelay: 1000 +# Maximum time to send a cancellation request and wait for PostgreSQL to +# acknowledge it, in milliseconds (default: 5000). +# queryCancellationTimeout: 5000 +# Number of parallel source COPY streams per table (default: 1). +# reshardingParallelWithinTableCopies: 1 +# Resharding validation and failed two-phase commit cleanup settings. +# resharding: +# # off, during_replication, before_cutover, or after_cutover +# postDataValidation: during_replication +# twoPcCleanupTimeout: 600000 # milliseconds +# Query rewrite behavior for subqueries (default: ignore; experimental upstream). +# rewrite: +# enabled: true +# subqueries: rewrite # replicas indicates how many instances of PgDog will run (HA). replicas: 2 @@ -331,6 +347,7 @@ shardedSchemas: [] # shardedTables contains the list of sharded table entries in pgdog.toml # Each entry requires: database and column; name and dataType are optional. +# Set prefix: true to match table names starting with name (default: false). # When dataType is omitted, PgDog defaults to bigint. # An entry may also carry a list/range/default mapping via the optional `mapping` field. # Each mapping rule has a target `shard`; the rule type is inferred from the fields set: @@ -654,6 +671,17 @@ grafanaRemoteWrite: # maxBackoff is the maximum retry delay maxBackoff: 5s +# QoS (PgDog EE). Optional unit rules extract a unit from SQL block comments. +# Regex capture groups are substituted into template by PgDog. +# qos: +# enabled: true +# maxEntries: 10000 +# unit: +# regex: "module='(?P[^']*)'.*?op='(?P[^']*)'" +# template: '${module}.${op}' +# regexSizeLimit: 1048576 # bytes of compiled regex memory +# maxUnitLength: 128 # bytes in a recorded unit + # Query stats (PgDog EE) queryStats: # enabled enables pgdog query stats (EE version only) @@ -690,14 +718,12 @@ control: statsInterval: 5000 # activeQueriesInterval defines how often to push active queries (in milliseconds) activeQueriesInterval: 5000 - # errorsInterval defines how often to push errors (in milliseconds) - errorsInterval: 5000 # requestTimeout defines the timeout for control plane requests (in milliseconds) requestTimeout: 1000 - # queryTimingsChunkSize defines the chunk size for query timings - queryTimingsChunkSize: 25 - # queryTimingsNewQueryQueueSize defines the queue size for new query timings - queryTimingsNewQueryQueueSize: 1000 + # configRefreshInterval defines how often to push configuration (in milliseconds) + configRefreshInterval: 60000 + # taskUpdatesInterval defines how often to push resharding tasks (in milliseconds) + taskUpdatesInterval: 1000 # LSN check configuration for replication failover auto mode (in milliseconds) # See: https://docs.pgdog.dev/features/load-balancer/replication-failover/ From 96539567fed04a9dcb2497dfe602bf8d02c5f46a Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 19 Nov 2026 13:51:39 -0800 Subject: [PATCH 2/2] remove slop --- test/test.sh | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/test/test.sh b/test/test.sh index bbe15f6..3439762 100755 --- a/test/test.sh +++ b/test/test.sh @@ -198,45 +198,5 @@ helm template test-release "$CHART_DIR" --set-json 'lifecycle={"postStart":{"exe expect_fail "lifecycle.preStop has no explicit terminationGracePeriodSeconds" -f "$TEST_DIR/values-pod-extensions.yaml" --set terminationGracePeriodSeconds=null echo " lifecycle, extraContainers, shareProcessNamespace, automountServiceAccountToken and service.exposeMetrics render as set, and only when set" -echo "" -echo "==> Validating recent pgdog-config settings..." -config_json() { - helm template test-release "$CHART_DIR" "$@" \ - | yq -r 'select(.kind == "ConfigMap" and .metadata.name == "test-release-pgdog") | .data["pgdog.toml"]' \ - | yq -p toml -o json -I=0 -} -extras=$(config_json -f "$TEST_DIR/values-pgdog-config-extras.yaml") -expect "general timeouts and parallel copies preserve integers and zero" \ - "$(echo "$extras" | yq -p json -o json -I=0 '[.general.passthrough_auth_debounce_delay, .general.query_cancellation_timeout, .general.resharding_parallel_within_table_copies]')" \ - '[0,12000,4]' -expect "resharding options are in their own table" \ - "$(echo "$extras" | yq -p json -o json -I=0 '.resharding')" \ - '{"post_data_validation":"before_cutover","two_pc_cleanup_timeout":0}' -expect "sharded table prefixes preserve true and false" \ - "$(echo "$extras" | yq -p json -o json -I=0 '[.sharded_tables[].prefix]')" '[true,false]' -expect "removed cutover option is not emitted" \ - "$(config_json --set cutoverSaveConfig=true | yq -p json '.general | has("cutover_save_config")')" 'false' - -qos=$(config_json -f "$TEST_DIR/values-qos-backoff.yaml") -expect "QoS units preserve regex escapes and literal template groups" \ - "$(echo "$qos" | yq -p json -o json -I=0 '[.qos.unit.regex, .qos.unit.template, .qos.unit.regex_size_limit, .qos.unit.max_unit_length]')" \ - "$(yq -o json -I=0 '[.qos.unit.regex, .qos.unit.template, .qos.unit.regexSizeLimit, .qos.unit.maxUnitLength]' "$TEST_DIR/values-qos-backoff.yaml")" -expect "QoS backoff remains in the parent table" \ - "$(echo "$qos" | yq -p json -o json -I=0 '[.qos.enabled, .qos.max_entries, .qos.write_backoff_delay, .qos.write_backoff_delay_jitter, .qos.write_backoff_queue_size]')" \ - '[true,10000,100,0,50]' -expect "optional QoS limits use PgDog defaults when omitted" \ - "$(config_json -f "$TEST_DIR/values-qos-backoff.yaml" --set qos.unit.regexSizeLimit=null --set qos.unit.maxUnitLength=null | yq -p json -o json -I=0 '[.qos.unit | has("regex_size_limit"), has("max_unit_length")]')" \ - '[false,false]' -expect_fail "QoS unit has no regex" -f "$TEST_DIR/values-qos-backoff.yaml" --set qos.unit.regex=null -expect_fail "QoS unit has no template" -f "$TEST_DIR/values-qos-backoff.yaml" --set qos.unit.template=null -expect "subquery rewriting renders" \ - "$(config_json -f "$TEST_DIR/values-rewrite.yaml" | yq -p json '.rewrite.subqueries')" 'rewrite' -expect "control uses current interval fields" \ - "$(config_json -f "$TEST_DIR/values-control.yaml" | yq -p json -o json -I=0 '[.control.config_refresh_interval, .control.task_updates_interval, (.control | has("errors_interval")), (.control | has("query_timings_chunk_size")), (.control | has("query_timings_new_query_queue_size"))]')" \ - '[30000,2000,false,false,false]' -expect "optional new settings stay omitted by default" \ - "$(config_json | yq -p json -o json -I=0 '[has("qos"), has("resharding"), has("rewrite"), (.general | has("query_cancellation_timeout")), (.general | has("passthrough_auth_debounce_delay")), (.general | has("resharding_parallel_within_table_copies"))]')" \ - '[false,false,false,false,false,false]' - echo "" echo "==> All tests passed!"