From ffe58a7cb8b889d898a194e63244af3dbaf70b13 Mon Sep 17 00:00:00 2001 From: Bharat Kathi Date: Thu, 4 Jun 2026 22:40:54 -0700 Subject: [PATCH 1/3] feat(clickhouse): add gr-clickhouse EC2 module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New infra/modules/clickhouse-ec2 mirroring the postgres-ec2 shape: Graviton AL2023, EIP, separate gp3 EBS data volume on prevent_destroy, SG ingress on 8123 (HTTP) + 9000 (native) from EKS node SG + admin CIDRs - user-data formats /var/lib/clickhouse, installs Docker, drops a users.d admin user (sha256_hex'd password, access_management=1) and removes the default no-password user, then runs clickhouse/clickhouse-server:24.12 with --ulimit nofile=262144 per the official docs requirement - Deploys as gr-clickhouse in prod on t4g.xlarge (16 GiB RAM) with 200 GB data volume; columnar OLAP store for CAN telemetry + Epic Shelter ingest, complementing the transactional gr-postgres - Cloudflare A record gr-clickhouse.gauchoracing.com → EIP, gray-cloud - Outputs clickhouse_admin_password (sensitive) for downstream Secret population --- infra/environments/prod/main.tf | 37 ++++ infra/environments/prod/outputs.tf | 16 ++ infra/modules/clickhouse-ec2/main.tf | 162 ++++++++++++++++++ infra/modules/clickhouse-ec2/outputs.tf | 35 ++++ .../modules/clickhouse-ec2/user-data.sh.tftpl | 74 ++++++++ infra/modules/clickhouse-ec2/variables.tf | 61 +++++++ 6 files changed, 385 insertions(+) create mode 100644 infra/modules/clickhouse-ec2/main.tf create mode 100644 infra/modules/clickhouse-ec2/outputs.tf create mode 100644 infra/modules/clickhouse-ec2/user-data.sh.tftpl create mode 100644 infra/modules/clickhouse-ec2/variables.tf diff --git a/infra/environments/prod/main.tf b/infra/environments/prod/main.tf index 2944f75..492bae1 100644 --- a/infra/environments/prod/main.tf +++ b/infra/environments/prod/main.tf @@ -153,6 +153,43 @@ resource "cloudflare_dns_record" "gr_mqtt" { proxied = false } +# ClickHouse on EC2 — analytics store for CAN telemetry + Epic Shelter +# ingest. Sentinel + transactional mapache state stay on gr-postgres; +# anything column-store-shaped (signals, aggregates, gr25_message +# successor tables) lands here. t4g.xlarge (16 GiB) gives meaningful +# RAM headroom — ClickHouse benefits from RAM more than Postgres did, +# and the t4g.medium gr-postgres OOM was a fresh reminder. +# +# Read the admin password via: +# terraform output -raw clickhouse_admin_password +module "clickhouse" { + source = "../../modules/clickhouse-ec2" + + name = "gr-clickhouse" + vpc_id = module.vpc.vpc_id + subnet_id = module.vpc.public_subnet_ids[0] + availability_zone = "us-west-2a" + + instance_type = "t4g.xlarge" + data_volume_size_gb = 200 + + associate_public_ip = true + admin_cidr_blocks = ["0.0.0.0/0"] + + allowed_security_group_ids = [ + module.eks.node_security_group_id, + ] +} + +resource "cloudflare_dns_record" "gr_clickhouse" { + zone_id = data.cloudflare_zone.gauchoracing.id + name = "gr-clickhouse" + type = "A" + content = module.clickhouse.public_ip + ttl = 300 + proxied = false +} + # Per-hostname SSL/TLS override. The zone defaults to "Flexible" (CF # talks HTTP to origin), but our ALB-backed Ingresses run HTTPS-only # with the imported Origin CA cert — Flexible there causes a redirect diff --git a/infra/environments/prod/outputs.tf b/infra/environments/prod/outputs.tf index fbe996b..5d0aceb 100644 --- a/infra/environments/prod/outputs.tf +++ b/infra/environments/prod/outputs.tf @@ -64,3 +64,19 @@ output "mqtt_password_tcm26" { value = module.mqtt.mqtt_password_tcm26 sensitive = true } + +output "clickhouse_private_ip" { + description = "Private IP of the ClickHouse EC2. In-cluster pods connect to this on 8123 (HTTP) / 9000 (native)." + value = module.clickhouse.private_ip +} + +output "clickhouse_public_ip" { + description = "Public IP of the ClickHouse EC2 (EIP). External admins connect to this on 8123/9000, or use gr-clickhouse.gauchoracing.com." + value = module.clickhouse.public_ip +} + +output "clickhouse_admin_password" { + description = "Generated ClickHouse admin password. Read with `terraform output -raw clickhouse_admin_password`." + value = module.clickhouse.admin_password + sensitive = true +} diff --git a/infra/modules/clickhouse-ec2/main.tf b/infra/modules/clickhouse-ec2/main.tf new file mode 100644 index 0000000..d3b3c5d --- /dev/null +++ b/infra/modules/clickhouse-ec2/main.tf @@ -0,0 +1,162 @@ +# ClickHouse on a single EC2 in the EKS VPC. Mirrors the postgres-ec2 +# module shape — pods reach it via private IP through the AWS split-horizon +# DNS trick, external admin clients via the EIP + gr-clickhouse hostname. +# +# Columnar OLAP store for telemetry: CAN signals from gr26, Epic Shelter +# ingest output, future analytics queries. Postgres keeps the small +# transactional state (users, vehicles, jobs, sessions). +# +# Backups are NOT included — add a snapshot policy before this holds +# unrecoverable data. ClickHouse data on a dedicated EBS volume so the +# instance can be replaced without losing the database. + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 6.0" + } + random = { + source = "hashicorp/random" + version = "~> 3.0" + } + } +} + +data "aws_ami" "al2023_arm64" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["al2023-ami-2023.*-arm64"] + } + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +resource "random_password" "admin" { + length = 32 + special = false +} + +resource "aws_security_group" "this" { + name = var.name + description = "ClickHouse for ${var.name}" + vpc_id = var.vpc_id + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "${var.name}" + } +} + +# Ingress on the HTTP + native protocol ports from each allowed SG +# (typically the EKS node SG). 8123 for HTTP clients (query service, +# clickhouse-client --http); 9000 for the native binary protocol +# (clickhouse-client, the Go and Python drivers). +locals { + ingress_ports = [8123, 9000] +} + +resource "aws_security_group_rule" "ingress_sg" { + for_each = { + for pair in setproduct(var.allowed_security_group_ids, local.ingress_ports) : + "${pair[0]}-${pair[1]}" => { sg = pair[0], port = pair[1] } + } + type = "ingress" + from_port = each.value.port + to_port = each.value.port + protocol = "tcp" + source_security_group_id = each.value.sg + security_group_id = aws_security_group.this.id + description = "ClickHouse :${each.value.port} from ${each.value.sg}" +} + +# Ingress from arbitrary CIDR blocks (admin laptops, the public internet, +# etc.). 32-char random admin password + sha256_hex on the wire is the only +# gate; tighten the CIDR list later when a known set of admin IPs exists. +resource "aws_security_group_rule" "ingress_cidr" { + for_each = length(var.admin_cidr_blocks) > 0 ? toset([for p in local.ingress_ports : tostring(p)]) : [] + type = "ingress" + from_port = tonumber(each.value) + to_port = tonumber(each.value) + protocol = "tcp" + cidr_blocks = var.admin_cidr_blocks + security_group_id = aws_security_group.this.id + description = "ClickHouse :${each.value} from admin CIDRs" +} + +resource "aws_ebs_volume" "data" { + availability_zone = var.availability_zone + size = var.data_volume_size_gb + type = "gp3" + encrypted = true + + tags = { + Name = "${var.name}-data" + } + + # Preserve the volume across instance replacements. ClickHouse data + # is the whole reason this server exists; never destroy by accident. + lifecycle { + prevent_destroy = true + } +} + +resource "aws_instance" "this" { + ami = data.aws_ami.al2023_arm64.id + instance_type = var.instance_type + subnet_id = var.subnet_id + vpc_security_group_ids = [aws_security_group.this.id] + availability_zone = var.availability_zone + associate_public_ip_address = var.associate_public_ip + + root_block_device { + volume_size = 20 + volume_type = "gp3" + encrypted = true + } + + user_data = templatefile("${path.module}/user-data.sh.tftpl", { + clickhouse_version = var.clickhouse_version + admin_user = var.admin_user + admin_password_sha256 = sha256(random_password.admin.result) + }) + + # Re-rendering user-data shouldn't recreate the instance — the data + # volume preserves state, and admin credentials are generated once + # and persist in TF state. AMI bumps similarly ignored. + lifecycle { + ignore_changes = [user_data, ami] + } + + tags = { + Name = "${var.name}" + Role = "clickhouse" + } +} + +resource "aws_volume_attachment" "data" { + device_name = "/dev/sdf" # appears as /dev/nvme1n1 inside the instance + volume_id = aws_ebs_volume.data.id + instance_id = aws_instance.this.id +} + +resource "aws_eip" "this" { + count = var.associate_public_ip ? 1 : 0 + domain = "vpc" + instance = aws_instance.this.id + + tags = { + Name = "${var.name}" + } +} diff --git a/infra/modules/clickhouse-ec2/outputs.tf b/infra/modules/clickhouse-ec2/outputs.tf new file mode 100644 index 0000000..2125dda --- /dev/null +++ b/infra/modules/clickhouse-ec2/outputs.tf @@ -0,0 +1,35 @@ +output "instance_id" { + description = "EC2 instance ID. Useful for SSM/console access." + value = aws_instance.this.id +} + +output "private_ip" { + description = "Private IP address. In-cluster pods can connect to this on 8123/9000." + value = aws_instance.this.private_ip +} + +output "public_ip" { + description = "EIP-assigned public IP, if associate_public_ip = true; null otherwise. External admin clients dial this." + value = try(aws_eip.this[0].public_ip, null) +} + +output "public_dns" { + description = "EC2 public DNS hostname. AWS split-horizon DNS resolves it to the private IP from inside the VPC and the EIP from outside, so pods + admins can both use this single name." + value = try("ec2-${replace(aws_eip.this[0].public_ip, ".", "-")}.us-west-2.compute.amazonaws.com", null) +} + +output "security_group_id" { + description = "Security group ID. Add ingress rules for any additional callers." + value = aws_security_group.this.id +} + +output "admin_user" { + description = "Admin username created on first boot. Pair with admin_password." + value = var.admin_user +} + +output "admin_password" { + description = "Generated admin password (32-char random). Read via `terraform output -raw clickhouse_admin_password` and put into the k8s Secret + clickhouse-client config." + value = random_password.admin.result + sensitive = true +} diff --git a/infra/modules/clickhouse-ec2/user-data.sh.tftpl b/infra/modules/clickhouse-ec2/user-data.sh.tftpl new file mode 100644 index 0000000..90044a5 --- /dev/null +++ b/infra/modules/clickhouse-ec2/user-data.sh.tftpl @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# Wait for the data volume to attach. AWS asynchronously attaches secondary +# volumes; cloud-init can start before /dev/nvme1n1 exists. +DATA_DEVICE="/dev/nvme1n1" +for _ in $(seq 1 30); do + [ -b "$DATA_DEVICE" ] && break + sleep 2 +done + +# Format only if not already a filesystem — prevents wipe on instance +# replacement when the EBS volume is preserved. +if ! blkid "$DATA_DEVICE" >/dev/null 2>&1; then + mkfs.ext4 -L clickhouse-data "$DATA_DEVICE" +fi + +mkdir -p /var/lib/clickhouse +echo "LABEL=clickhouse-data /var/lib/clickhouse ext4 defaults,nofail 0 2" >> /etc/fstab +mount /var/lib/clickhouse + +# Official clickhouse-server image runs as uid 101. Hand the data dir over +# before the container starts; otherwise initdb fails on permission errors. +chown 101:101 /var/lib/clickhouse + +dnf install -y docker +systemctl enable --now docker + +# users.d overrides — merged with the image's default users.xml at startup. +# Drops the default user (no password, full access) and adds a sha256-hashed +# admin with access_management so further users can be created via SQL. +mkdir -p /etc/clickhouse-server/users.d + +cat >/etc/clickhouse-server/users.d/admin.xml < + + + <${admin_user}> + ${admin_password_sha256} + + ::/0 + + default + default + 1 + + + +EOF + +cat >/etc/clickhouse-server/users.d/remove-default-user.xml <<'EOF' + + + + + + +EOF + +chmod 600 /etc/clickhouse-server/users.d/admin.xml + +# --network=host: nothing else on this box uses 8123 or 9000, so skip the +# docker-proxy NAT layer. --restart=always: container survives reboots +# (docker.service is enabled above). --ulimit nofile=262144: ClickHouse +# wants high fd limits for many concurrent table parts; the official docs +# call this out as a hard requirement. +docker run -d \ + --name clickhouse \ + --restart=always \ + --network=host \ + --ulimit nofile=262144:262144 \ + -v /var/lib/clickhouse:/var/lib/clickhouse \ + -v /etc/clickhouse-server/users.d:/etc/clickhouse-server/users.d:ro \ + clickhouse/clickhouse-server:${clickhouse_version} diff --git a/infra/modules/clickhouse-ec2/variables.tf b/infra/modules/clickhouse-ec2/variables.tf new file mode 100644 index 0000000..5f74a7b --- /dev/null +++ b/infra/modules/clickhouse-ec2/variables.tf @@ -0,0 +1,61 @@ +variable "name" { + description = "Friendly name for the ClickHouse instance. Used in resource Name tags + SG name." + type = string +} + +variable "vpc_id" { + description = "VPC the EC2 lives in. Should be the same VPC as EKS so pods reach it via private IP." + type = string +} + +variable "subnet_id" { + description = "Subnet to launch into. Pick a public subnet if associate_public_ip = true." + type = string +} + +variable "availability_zone" { + description = "AZ for the instance + EBS data volume. Must match the AZ of the chosen subnet — EBS is AZ-scoped." + type = string +} + +variable "instance_type" { + description = "EC2 instance type. ClickHouse benefits significantly from RAM (mark cache, query memory); t4g.large (8 GiB) is the floor, t4g.xlarge (16 GiB) is comfortable. r-series gets expensive but is RAM-optimized." + type = string + default = "t4g.large" +} + +variable "data_volume_size_gb" { + description = "Size of the EBS data volume in GB. ClickHouse compresses CAN telemetry ~10:1 vs row stores, so 200 GB lasts a long time. Resize via TF when needed; gp3 supports online grow." + type = number + default = 200 +} + +variable "clickhouse_version" { + description = "Pinned clickhouse/clickhouse-server image tag. Bump deliberately; major versions occasionally change defaults around merge tree storage." + type = string + default = "24.12" +} + +variable "admin_user" { + description = "Admin user created in users.d on first boot. Has access_management = 1 so it can grant further users from SQL." + type = string + default = "admin" +} + +variable "allowed_security_group_ids" { + description = "Security group IDs allowed to connect on 8123 + 9000. Typically the EKS node SG." + type = list(string) + default = [] +} + +variable "admin_cidr_blocks" { + description = "CIDR blocks allowed to connect to 8123 + 9000 directly. Use specific IPs (your laptop, office) for least exposure, or \"0.0.0.0/0\" to leave the admin password as the only gate." + type = list(string) + default = [] +} + +variable "associate_public_ip" { + description = "If true, the instance gets a public IP via an EIP. Required when allowing inbound connections from outside the VPC." + type = bool + default = false +} From 8cc8a0fe529e9694c15d45e28a4960a4644d890e Mon Sep 17 00:00:00 2001 From: Bharat Kathi Date: Thu, 4 Jun 2026 22:48:26 -0700 Subject: [PATCH 2/3] feat(clickhouse): bump gr-clickhouse to r8g.xlarge Graviton4 RAM-optimized: 4 vCPU / 32 GiB (vs t4g.xlarge's 16 GiB) and dedicated CPU instead of t-series credit-burstable. ClickHouse query memory + mark cache get the bigger pool, and big aggregations don't risk throttling. ~$55/mo more on-demand. --- infra/environments/prod/main.tf | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/infra/environments/prod/main.tf b/infra/environments/prod/main.tf index 492bae1..0a9174d 100644 --- a/infra/environments/prod/main.tf +++ b/infra/environments/prod/main.tf @@ -156,9 +156,13 @@ resource "cloudflare_dns_record" "gr_mqtt" { # ClickHouse on EC2 — analytics store for CAN telemetry + Epic Shelter # ingest. Sentinel + transactional mapache state stay on gr-postgres; # anything column-store-shaped (signals, aggregates, gr25_message -# successor tables) lands here. t4g.xlarge (16 GiB) gives meaningful -# RAM headroom — ClickHouse benefits from RAM more than Postgres did, -# and the t4g.medium gr-postgres OOM was a fresh reminder. +# successor tables) lands here. +# +# r8g.xlarge (Graviton4, 4 vCPU, 32 GiB) — RAM-optimized for the mark +# cache + per-query memory budget that columnar scans depend on, and +# dedicated CPU (no t-series credit accounting) so a big aggregation +# doesn't tip into throttling. Steeper hourly than t4g.xlarge but +# avoiding another OOM is worth ~$55/mo. # # Read the admin password via: # terraform output -raw clickhouse_admin_password @@ -170,7 +174,7 @@ module "clickhouse" { subnet_id = module.vpc.public_subnet_ids[0] availability_zone = "us-west-2a" - instance_type = "t4g.xlarge" + instance_type = "r8g.xlarge" data_volume_size_gb = 200 associate_public_ip = true From 179a8529b9511abe421de3cc0aadc2e692768481 Mon Sep 17 00:00:00 2001 From: Bharat Kathi Date: Thu, 4 Jun 2026 22:54:35 -0700 Subject: [PATCH 3/3] chore(clickhouse): align module defaults with prod values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - instance_type: t4g.large → r8g.xlarge (Graviton4 RAM-optimized) - clickhouse_version: 24.12 → 26.3-alpine Module default == what prod actually runs. Future callers can still opt back to smaller/cheaper by overriding. --- infra/modules/clickhouse-ec2/variables.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/modules/clickhouse-ec2/variables.tf b/infra/modules/clickhouse-ec2/variables.tf index 5f74a7b..5740765 100644 --- a/infra/modules/clickhouse-ec2/variables.tf +++ b/infra/modules/clickhouse-ec2/variables.tf @@ -19,9 +19,9 @@ variable "availability_zone" { } variable "instance_type" { - description = "EC2 instance type. ClickHouse benefits significantly from RAM (mark cache, query memory); t4g.large (8 GiB) is the floor, t4g.xlarge (16 GiB) is comfortable. r-series gets expensive but is RAM-optimized." + description = "EC2 instance type. ClickHouse benefits significantly from RAM (mark cache, query memory) and dedicated CPU. r8g.xlarge (Graviton4, 4 vCPU, 32 GiB) is the prod default; r-series RAM-optimized, dedicated cores (no t-series credit accounting). Drop to t4g.xlarge if cost matters more than steady-state perf." type = string - default = "t4g.large" + default = "r8g.xlarge" } variable "data_volume_size_gb" { @@ -31,9 +31,9 @@ variable "data_volume_size_gb" { } variable "clickhouse_version" { - description = "Pinned clickhouse/clickhouse-server image tag. Bump deliberately; major versions occasionally change defaults around merge tree storage." + description = "Pinned clickhouse/clickhouse-server image tag. Defaults to the Alpine variant — smaller image, musl libc; the regular Ubuntu-based tag drops the `-alpine` suffix. Bump deliberately; major versions occasionally change defaults around merge tree storage." type = string - default = "24.12" + default = "26.3-alpine" } variable "admin_user" {