diff --git a/README.md b/README.md index 6973a80..dfdb2d1 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,8 @@ It covers the full lifecycle from infrastructure provisioning to GitOps-managed | **Thanos** | Long-term metric storage - Prometheus sidecar ships data to S3; Query, Compactor, and StoreGateway provide durable retention | | **HPA + metrics-server** | Horizontal pod autoscaling based on CPU utilization | | **Loki + Fluent Bit** | Centralized log aggregation backed by S3 object storage, queryable in Grafana | +| **OTel Collector (agent + gateway)** | Two-tier telemetry pipeline: a DaemonSet agent on every node receives spans from local pods; a central Deployment gateway batches and forwards to Tempo | +| **Grafana Tempo** | Distributed trace storage - receives spans from the OTel gateway via OTLP gRPC, queryable in Grafana with trace-to-log correlation through Loki; backed by S3 | | **Velero** | Cluster backup and restore - backs up Kubernetes resources and EBS volume snapshots to S3 | | **Kyverno** | Admission controller - evaluates `ClusterPolicy` rules on every pod admission and writes `PolicyReport` objects; all policies run in Audit mode by default | | **Policy Reporter** | Web UI for Kyverno `PolicyReport` and `ClusterPolicyReport` objects - accessible via `kubectl port-forward` | @@ -72,6 +74,7 @@ See [docs/cost.md](docs/cost.md) for a full per-component breakdown, cost-contro | [gitops/monitoring/README.md](gitops/monitoring/README.md) | Prometheus, Grafana, Thanos, Prometheus Adapter, ServiceMonitor, EBS persistence | | [gitops/alerts/README.md](gitops/alerts/README.md) | PrometheusRules, Slack alerting, silencing, grouping | | [gitops/logs/README.md](gitops/logs/README.md) | Loki S3 backend, Fluent Bit, log querying in Grafana | +| [gitops/tracing/README.md](gitops/tracing/README.md) | OTel Collector agent/gateway pipeline, Tempo S3 backend, trace querying and log correlation in Grafana | | [gitops/security/README.md](gitops/security/README.md) | Kyverno policies, Policy Reporter UI, inspecting policy reports | | **Design** | | | [docs/design-decisions.md](docs/design-decisions.md) | Why each architectural choice was made - tool selection, configuration defaults, and demo trade-offs | @@ -194,7 +197,7 @@ bash terraform/scripts/upgrade.sh 2. Generates or accepts ArgoCD bcrypt hash and Grafana password 3. Runs `terraform init` → `terraform validate` → `terraform apply` 4. Updates kubeconfig -5. Re-applies the ArgoCD cluster secret with the latest Terraform outputs (account ID, VPC ID, domain, Karpenter instance profile, Thanos/Loki/Velero bucket names) +5. Re-applies the ArgoCD cluster secret with the latest Terraform outputs (account ID, VPC ID, domain, Karpenter instance profile, Thanos/Loki/Velero/Tempo bucket names) 6. Force-annotates `argocd-admin`, `grafana-admin`, and `alertmanager-webhook` ExternalSecrets to trigger an immediate re-sync from Secrets Manager --- @@ -211,7 +214,9 @@ bash terraform/scripts/upgrade.sh | 6 | HPA reacts to load | `kubectl get hpa -n simple-time-service -w` while running `python3 scripts/load_test.py` | | 7 | Slack alert fires | Fire test alert (see [gitops/alerts/README.md](gitops/alerts/README.md#testing-the-slack-receiver)) - appears in `#alerts-test` within 30s | | 8 | Loki API reachable | `kubectl port-forward svc/loki-gateway -n logging 3100:80` → `curl 'http://localhost:3100/loki/api/v1/labels'` | -| 9 | Logs in Grafana | Explore → Loki datasource → `{namespace="simple-time-service"}` | +| 9 | Logs in Grafana | Explore → Loki datasource → `{namespace="simple-time-service"}` - log lines include `trace_id` / `span_id` fields in JSON | +| 9a | OTel pipeline healthy | `kubectl get pods -n tracing` - `otel-gateway` (Deployment) and `otel-agent` (DaemonSet) all `Running` | +| 9b | Traces in Grafana | Explore → Tempo datasource → **Search** → service `simple-time-service` - spans visible; clicking a trace shows correlated Loki logs | | 10 | Thanos Query healthy | `kubectl get pods -n monitoring -l app.kubernetes.io/name=thanos-query` - `Running`; add Thanos datasource in Grafana pointing to `thanos-query.monitoring.svc:9090` | | 11 | EBS volumes provisioned | `kubectl get pv` - PVs for Prometheus (20Gi), Alertmanager (2Gi), Thanos Compactor (10Gi), StoreGateway (5Gi) all `Bound` | | 12 | Velero running | `kubectl get pods -n velero` - `Running`; `kubectl get backupstoragelocation -n velero` - `Available` | @@ -345,6 +350,14 @@ bash terraform/scripts/upgrade.sh │ │ │ └── grafana-loki-datasource.yaml # ApplicationSet - Loki datasource ConfigMap (sync-wave 4) │ │ └── fluent-bit/ │ │ └── fluent-bit.yaml # ApplicationSet - Fluent Bit DaemonSet (sync-wave 4) +│ ├── tracing/ +│ │ ├── README.md # OTel Collector pipeline, Tempo S3 backend, trace querying and log correlation +│ │ ├── tempo/ +│ │ │ ├── tempo.yaml # ApplicationSet - Tempo trace store S3 backend (sync-wave 6) +│ │ │ └── grafana-tempo-datasource.yaml # ApplicationSet - Tempo datasource ConfigMap + Loki correlation (sync-wave 8) +│ │ └── otel-collector/ +│ │ ├── otel-collector-gateway.yaml # ApplicationSet - OTel Collector Deployment gateway (sync-wave 6) +│ │ └── otel-collector-agent.yaml # ApplicationSet - OTel Collector DaemonSet agent (sync-wave 7) │ └── security/ │ └── kyverno/ │ ├── kyverno.yaml # ApplicationSet - Kyverno admission controller (sync-wave 3) @@ -376,6 +389,7 @@ bash terraform/scripts/upgrade.sh ├── thanos.tf # S3 bucket + IRSA roles for Thanos Prometheus sidecar, Compactor, StoreGateway ├── loki.tf # S3 bucket + IRSA role for Loki object storage ├── velero.tf # S3 bucket + IRSA role for Velero backups + ├── tempo.tf # S3 bucket + IRSA role for Tempo trace storage ├── secrets-manager.tf # AWS Secrets Manager secrets (ArgoCD, Grafana, Slack) ├── .tflint.hcl # TFLint AWS ruleset plugin config ├── .checkov.yaml # Checkov global skip rules (documented false positives) @@ -426,7 +440,7 @@ The cleanup script runs six steps in order: 1. **Kubernetes resources** - deletes the root app, all ArgoCD Applications and ApplicationSets, Ingresses, LoadBalancer Services, Karpenter NodePools and NodeClaims, PersistentVolumes, and all namespaces labelled `app.kubernetes.io/part-of=eks-gitops-platform` (including `security`) (every namespace created by the `cluster-namespaces` chart) 2. **AWS Load Balancers** - deletes any leftover ALB/NLB/Classic ELBs tagged for the cluster that were not removed by step 1 3. **EBS volumes and snapshots** - deletes EBS volumes tagged `kubernetes.io/cluster/=owned` (CSI Driver PVCs with Retain policy), then deletes EBS snapshots tagged for the cluster or with a `velero.io/backup` tag -4. **S3 buckets** - empties all versioned S3 buckets in the same region whose name contains the cluster name (Thanos, Loki, Velero), draining all object versions and delete markers in batches so that Terraform can delete the buckets cleanly +4. **S3 buckets** - empties all versioned S3 buckets in the same region whose name contains the cluster name (Thanos, Loki, Velero, Tempo), draining all object versions and delete markers in batches so that Terraform can delete the buckets cleanly 5. **Kyverno CRDs** - deletes all CRDs in the `kyverno.io` and `wgpolicyk8s.io` API groups; this is handled automatically by `cleanup.sh` using a label selector 6. **Terraform destroy** - removes all AWS resources provisioned by Terraform diff --git a/app/README.md b/app/README.md index fc96266..8f94a38 100644 --- a/app/README.md +++ b/app/README.md @@ -25,6 +25,9 @@ A minimal Python microservice that returns the current UTC timestamp and the cal - **Framework**: FastAPI + Uvicorn - **Container**: single-stage build based on `python:3.12-slim`, kept small by clearing pip cache. - **ASGI server**: Uvicorn (production-grade async Python server) +- **Metrics**: `prometheus-fastapi-instrumentator` — `/metrics` endpoint scraped by Prometheus via ServiceMonitor +- **Tracing**: OpenTelemetry SDK with OTLP HTTP exporter — a `TracerProvider` is always configured so every request gets a real span; spans are exported to Grafana Tempo only when `OTEL_EXPORTER_OTLP_ENDPOINT` is set +- **Logging**: JSON-structured logs written to stdout; `trace_id` and `span_id` are injected into every log line for every HTTP request --- @@ -36,7 +39,7 @@ A minimal Python microservice that returns the current UTC timestamp and the cal docker compose up --build ``` -The service is available at `http://localhost:8080`. +The service is available at `http://localhost:8080`. Logs include `trace_id` and `span_id` on every request even without a collector. ### Docker only @@ -66,13 +69,12 @@ Expected response: The image is published on Docker Hub under two distinct tags: -| Tag | Metrics endpoint | Use when | -|-----|-----------------|----------| -| `v1` | No | You don't need Prometheus metrics | -| `latest` | Yes (`/metrics`) | You want Prometheus scraping via ServiceMonitor | +| Tag | Metrics | Tracing | Use when | +|-----|---------|---------|----------| +| `v1` | No | No | Pinned baseline — original service without instrumentation | +| `latest` | Yes (`/metrics`) | Yes | Tracks `main` — full instrumentation; deploy with the Helm chart | -> **`v1`** is a pinned baseline - the original service with no instrumentation. -> **`latest`** tracks `main` and includes the Prometheus `/metrics` endpoint exposed by `prometheus-fastapi-instrumentator`. Use this tag when deploying with the Helm chart and `serviceMonitor.enabled=true`. +> **`latest`** includes the Prometheus `/metrics` endpoint and OpenTelemetry tracing. Tracing is off by default and activates only when `OTEL_EXPORTER_OTLP_ENDPOINT` is set in the pod's environment. Use this tag when deploying with the Helm chart and `serviceMonitor.enabled=true`. ```bash # No metrics (baseline) @@ -166,6 +168,49 @@ To enforce that the scan must pass before a PR can be merged, configure a branch --- +## Observability + +### Structured logging + +All logs are written to stdout as JSON. Every log line includes: + +| Field | Example | Notes | +|-------|---------|-------| +| `timestamp` | `2026-04-07T12:00:00.000000+00:00` | ISO-8601 UTC | +| `level` | `INFO` | | +| `logger` | `simple-time-service` | | +| `message` | `Request completed: ...` | | +| `trace_id` | `4bf92f3577b34da6...` | Present on every HTTP request | +| `span_id` | `00f067aa0ba902b7` | Present on every HTTP request | + +`trace_id` and `span_id` are injected from the active OpenTelemetry span on every request. When `OTEL_EXPORTER_OTLP_ENDPOINT` is set, the same span is also exported to Tempo, enabling a direct Loki → Tempo link from any log line. + +### Distributed tracing + +The platform uses a two-tier OpenTelemetry Collector topology: + +``` +Pod → OTel Agent (DaemonSet, node-local, :4318) → OTel Gateway (Deployment) → Tempo +``` + +The Helm chart injects the node IP at runtime via the downward API so the pod always reaches its local agent: + +| Variable | Set by | Value | +|----------|--------|-------| +| `NODE_IP` | Downward API (`status.hostIP`) | Node's internal IP | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | Helm template | `http://$(NODE_IP):4318` | +| `OTEL_SERVICE_NAME` | Helm values | `simple-time-service` | + +All three are injected automatically when `tracing.enabled: true` in `charts/simple-time-service/values.yaml`. The `/health` and `/metrics` endpoints are excluded from tracing to avoid noise. + +Locally, `trace_id` appears in logs without any collector. To also export spans, point the endpoint at a local OTel Collector or Tempo: + +```bash +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 docker compose up --build +``` + +--- + ## Prometheus metrics exposed Metrics are emitted by [`prometheus-fastapi-instrumentator`](https://github.com/trallnag/prometheus-fastapi-instrumentator) and the standard Python `prometheus_client` collectors. diff --git a/app/requirements.txt b/app/requirements.txt index 7b6855c..199410c 100644 --- a/app/requirements.txt +++ b/app/requirements.txt @@ -1,4 +1,8 @@ fastapi==0.136.3 uvicorn[standard]==0.34.0 prometheus-fastapi-instrumentator==7.0.0 -starlette==0.52.1 \ No newline at end of file +starlette==0.52.1 +opentelemetry-api==1.28.0 +opentelemetry-sdk==1.28.0 +opentelemetry-instrumentation-fastapi==0.49b0 +opentelemetry-exporter-otlp-proto-http==1.28.0 diff --git a/app/src/app.py b/app/src/app.py index ec857d8..9328f2a 100644 --- a/app/src/app.py +++ b/app/src/app.py @@ -1,8 +1,16 @@ +import json import logging +import os from datetime import datetime, timezone from fastapi import FastAPI, Request from fastapi.responses import JSONResponse +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor from prometheus_fastapi_instrumentator import Instrumentator app = FastAPI(title="simple-time-service") @@ -10,28 +18,51 @@ Instrumentator().instrument(app).expose(app, include_in_schema=False) +class JSONFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + span = trace.get_current_span() + ctx = span.get_span_context() + log_record = { + "timestamp": datetime.fromtimestamp(record.created, tz=timezone.utc).isoformat(), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + } + if ctx.is_valid: + log_record["trace_id"] = format(ctx.trace_id, "032x") + log_record["span_id"] = format(ctx.span_id, "016x") + return json.dumps(log_record) + + def configure_logging() -> logging.Logger: - """ - Configure and return the application logger. - - Logs are written to stdout. - """ - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s [%(name)s] %(message)s", - ) + handler = logging.StreamHandler() + handler.setFormatter(JSONFormatter()) + root = logging.getLogger() + root.handlers.clear() + root.addHandler(handler) + root.setLevel(logging.INFO) logger = logging.getLogger("simple-time-service") logger.setLevel(logging.INFO) return logger +def configure_tracing() -> None: + service_name = os.environ.get("OTEL_SERVICE_NAME", "simple-time-service") + provider = TracerProvider(resource=Resource.create({"service.name": service_name})) + endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") + if endpoint: + provider.add_span_processor( + BatchSpanProcessor(OTLPSpanExporter(endpoint=f"{endpoint}/v1/traces")) + ) + trace.set_tracer_provider(provider) + + +configure_tracing() logger = configure_logging() +FastAPIInstrumentor.instrument_app(app, excluded_urls="health,metrics") def get_client_ip(request: Request) -> str: - """ - Return the originating client IP address from the incoming request. - """ x_forwarded_for = request.headers.get("x-forwarded-for", "") if x_forwarded_for: return x_forwarded_for.split(",")[0].strip() @@ -48,9 +79,6 @@ def get_client_ip(request: Request) -> str: @app.middleware("http") async def log_requests(request: Request, call_next): - """ - Log incoming requests and response status codes. - """ client_ip = get_client_ip(request) logger.info( "Incoming request: method=%s path=%s remote_addr=%s user_agent=%s", @@ -74,9 +102,6 @@ async def log_requests(request: Request, call_next): @app.get("/") async def get_service_metadata(request: Request): - """ - Return the current UTC timestamp and resolved client IP address. - """ response = { "timestamp": datetime.now(timezone.utc).isoformat(), "ip": get_client_ip(request), @@ -88,8 +113,5 @@ async def get_service_metadata(request: Request): @app.get("/health") async def health(): - """ - Return a simple health response. - """ logger.info("Health check successful") return JSONResponse(content={"status": "ok"}, status_code=200) diff --git a/charts/simple-time-service/README.md b/charts/simple-time-service/README.md index 3d8fbba..d0becbe 100644 --- a/charts/simple-time-service/README.md +++ b/charts/simple-time-service/README.md @@ -117,6 +117,10 @@ kubectl delete namespace simple-time-service | serviceMonitor.labels | object | `{}` | Extra labels added to the ServiceMonitor (used to target a specific Prometheus instance) | | serviceMonitor.path | string | `"/metrics"` | Metrics endpoint path | | tolerations | list | `[]` | Pod tolerations | +| tracing.agentPort | int | `4318` | OTLP HTTP port on the node-local OTel Collector agent; used only when `endpoint` is empty | +| tracing.enabled | bool | `false` | Enable OpenTelemetry tracing | +| tracing.endpoint | string | `""` | Direct OTLP HTTP endpoint (e.g. `http://otel-collector.tracing.svc.cluster.local:4318`). When empty, uses the node-local DaemonSet agent via `status.hostIP:agentPort` | +| tracing.serviceName | string | `"simple-time-service"` | Service name reported in traces and correlated in Grafana Tempo | --- @@ -139,6 +143,19 @@ helm install simple-time-service charts/simple-time-service \ --set serviceMonitor.enabled=true ``` +### Deploy with tracing enabled + +Requires the OTel Collector DaemonSet to be running on each node (deployed via `gitops/tracing/`). + +```bash +helm install simple-time-service charts/simple-time-service \ + --set image.tag=latest \ + --set serviceMonitor.enabled=true \ + --set tracing.enabled=true +``` + +The chart injects `NODE_IP` (via the downward API) and constructs `OTEL_EXPORTER_OTLP_ENDPOINT=http://$(NODE_IP):4318` automatically so pods always reach the node-local agent without any manual endpoint configuration. + --- ## Updating this README diff --git a/charts/simple-time-service/README.md.gotmpl b/charts/simple-time-service/README.md.gotmpl index 8299588..42948eb 100644 --- a/charts/simple-time-service/README.md.gotmpl +++ b/charts/simple-time-service/README.md.gotmpl @@ -81,6 +81,19 @@ helm install simple-time-service charts/simple-time-service \ --set serviceMonitor.enabled=true ``` +### Deploy with tracing enabled + +Requires the OTel Collector DaemonSet to be running on each node (deployed via `gitops/tracing/`). + +```bash +helm install simple-time-service charts/simple-time-service \ + --set image.tag=latest \ + --set serviceMonitor.enabled=true \ + --set tracing.enabled=true +``` + +The chart injects `NODE_IP` (via the downward API) and constructs `OTEL_EXPORTER_OTLP_ENDPOINT=http://$(NODE_IP):4318` automatically so pods always reach the node-local agent without any manual endpoint configuration. + --- ## Updating this README diff --git a/charts/simple-time-service/templates/deployment.yaml b/charts/simple-time-service/templates/deployment.yaml index 910351c..b89321b 100644 --- a/charts/simple-time-service/templates/deployment.yaml +++ b/charts/simple-time-service/templates/deployment.yaml @@ -29,6 +29,19 @@ spec: - name: simple-time-service image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- if .Values.tracing.enabled }} + env: + {{- if not .Values.tracing.endpoint }} + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- end }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ if .Values.tracing.endpoint }}{{ .Values.tracing.endpoint | quote }}{{ else }}"http://$(NODE_IP):{{ .Values.tracing.agentPort }}"{{ end }} + - name: OTEL_SERVICE_NAME + value: {{ .Values.tracing.serviceName | quote }} + {{- end }} ports: - name: http containerPort: {{ .Values.service.targetPort }} diff --git a/charts/simple-time-service/values.yaml b/charts/simple-time-service/values.yaml index f3effe3..4c3c864 100644 --- a/charts/simple-time-service/values.yaml +++ b/charts/simple-time-service/values.yaml @@ -134,6 +134,16 @@ serviceMonitor: # -- Extra labels added to the ServiceMonitor (used to target a specific Prometheus instance) labels: {} +tracing: + # -- Enable OpenTelemetry tracing + enabled: false + # -- Direct OTLP HTTP endpoint (e.g. `http://otel-collector.tracing.svc.cluster.local:4318`). When empty, uses the node-local DaemonSet agent via `status.hostIP:agentPort` + endpoint: "" + # -- OTLP HTTP port on the node-local OTel Collector agent; used only when `endpoint` is empty + agentPort: 4318 + # -- Service name reported in traces and correlated in Grafana Tempo + serviceName: simple-time-service + # -- Extra annotations added to every pod podAnnotations: {} diff --git a/docs/cost.md b/docs/cost.md index 2aed471..5cef22d 100644 --- a/docs/cost.md +++ b/docs/cost.md @@ -16,7 +16,7 @@ Tear down the cluster when not in use. At rest, only S3, Route 53, and Secrets M | NAT Gateway | 1 gateway + minimal data transfer | ~$38 | | Application Load Balancers | 2 ALBs (ArgoCD + Grafana/app) | ~$43 | | EBS volumes | Prometheus 50 GB + Thanos 20 GB (`gp3`) | ~$7 | -| S3 | Loki logs, Thanos blocks, Velero backups | ~$3–5 | +| S3 | Loki logs, Thanos blocks, Velero backups, Tempo traces | ~$3–6 | | Secrets Manager | ~5 secrets | ~$2 | | Route 53 | 1 hosted zone | ~$1 | | **Total** | | **~$320–325/month** | diff --git a/docs/design-decisions.md b/docs/design-decisions.md index 72a78b7..2f615c1 100644 --- a/docs/design-decisions.md +++ b/docs/design-decisions.md @@ -163,9 +163,9 @@ Setting it as the cluster default means any workload that does not explicitly re --- -## 9. S3-Backed Storage for Loki, Thanos, and Velero +## 9. S3-Backed Storage for Loki, Thanos, Velero, and Tempo -**Decision:** All three long-term storage concerns - logs (Loki), metrics (Thanos), and backups (Velero) - are backed by dedicated S3 buckets, provisioned in Terraform ([terraform/loki.tf](../terraform/loki.tf), [terraform/thanos.tf](../terraform/thanos.tf), [terraform/velero.tf](../terraform/velero.tf)) and referenced via cluster annotations. +**Decision:** All four long-term storage concerns - logs (Loki), metrics (Thanos), backups (Velero), and traces (Tempo) - are backed by dedicated S3 buckets, provisioned in Terraform ([terraform/loki.tf](../terraform/loki.tf), [terraform/thanos.tf](../terraform/thanos.tf), [terraform/velero.tf](../terraform/velero.tf), [terraform/tempo.tf](../terraform/tempo.tf)) and referenced via cluster annotations. **Alternatives considered:** - EBS-only storage (no S3). @@ -181,7 +181,7 @@ Setting it as the cluster default means any workload that does not explicitly re **Why separate buckets:** -Separate buckets allow independent IAM policies scoped to each workload's IRSA role, independent lifecycle policies (Thanos needs tiered downsampling; Velero needs backup retention windows; Loki needs chunk expiry), and independent encryption/access audit trails. A single shared bucket would require prefix-based IAM policies that are harder to reason about and easier to misconfigure. +Separate buckets allow independent IAM policies scoped to each workload's IRSA role, independent lifecycle policies (Thanos needs tiered downsampling; Velero needs backup retention windows; Loki needs chunk expiry; Tempo needs trace retention), and independent encryption/access audit trails. A single shared bucket would require prefix-based IAM policies that are harder to reason about and easier to misconfigure. Bucket names flow into Helm values via ArgoCD cluster secret annotations - the mechanism is covered in [decision 21](#21-terraformgitops-boundary-and-cluster-annotation-bridge). @@ -358,7 +358,7 @@ ArgoCD applies all resources in a sync together by default. Without waves, a `Pr | 3 | Karpenter, Velero, Kyverno | Controllers before their CRD instances (NodePools, Schedules, ClusterPolicies) | | 4 | Karpenter NodePools, ClusterSecretStore | NodePools after Karpenter is running; ClusterSecretStore after ESO is ready | | 5 | All ExternalSecrets, Kyverno policies | Secrets before consumers; policies before workloads | -| 6–8 | Loki, Prometheus, Grafana, workloads | Observability stack before application workloads | +| 6–8 | Loki, Tempo, Prometheus, Grafana, workloads | Observability stack before application workloads | | 9–10 | Prometheus Adapter, AlertManager config, Thanos | Depends on Prometheus sidecar being live | The wave 2 → wave 6 gap for ALB Ingress resources is explained in [decision 16](#16-aws-load-balancer-controller-instead-of-nginx-ingress). @@ -413,12 +413,46 @@ Conversely, Kubernetes config (Helm values, Ingress rules, PrometheusRules) chan **How the bridge works:** -After `terraform apply`, the bootstrap script writes a cluster secret into ArgoCD with annotations containing Terraform outputs: `vpcId`, `clusterName`, `region`, `accountId`, `thanosBucketName`, `lokiBucketName`, `veleroBucketName`, `domainName`, and IRSA role ARNs. ApplicationSet `clusterDecisionResource` generators read these annotations and inject them as template variables into every child Application's Helm values. +After `terraform apply`, the bootstrap script writes a cluster secret into ArgoCD with annotations containing Terraform outputs: `vpcId`, `clusterName`, `region`, `accountId`, `thanosBucketName`, `lokiBucketName`, `veleroBucketName`, `tempoBucketName`, `domainName`, and IRSA role ARNs. ApplicationSet `clusterDecisionResource` generators read these annotations and inject them as template variables into every child Application's Helm values. This means the GitOps manifests contain no hardcoded AWS account IDs, bucket names, or ARNs. The same manifests work across dev, staging, and production clusters by simply changing the cluster secret annotations - Terraform outputs flow automatically into all downstream Helm values without any manifest changes. --- +## 22. OTel Collector Two-Tier Topology (DaemonSet Agent + Deployment Gateway) + +**Decision:** Traces are not exported directly from application pods to Tempo. Instead, a two-tier OpenTelemetry Collector pipeline sits between them: a DaemonSet agent on every node receives spans from local pods, and a central Deployment gateway batches and forwards to Tempo ([gitops/tracing/otel-collector/](../gitops/tracing/otel-collector/)). + +**Alternatives considered:** +- App → Tempo directly (OTLP HTTP). +- App → single shared Collector Deployment (no DaemonSet). + +**Why not direct to Tempo:** + +Exporting directly couples every instrumented pod to Tempo's address, port, and availability. If Tempo is restarting, being upgraded, or temporarily unavailable, the application SDK's send buffer fills and spans are dropped unless retry/queue logic is explicitly configured in each application. Moving that concern to a collector process that runs independently of the application means the SDK can stay simple (fire-and-forget OTLP) and the collector handles retries, buffering, and backend routing. + +Directly connecting also prevents any future enrichment or fanout without touching application code. Routing to a second backend (e.g. a vendor APM tool alongside Tempo) requires a collector in the path; retrofitting one later is harder than having it there from the start. + +**Why a DaemonSet agent, not a single Deployment:** + +A single shared Collector Deployment would mean every pod on every node sends spans over the network to a single pod on a different node. Under load, this concentrates both network traffic and resource pressure on one target. It also introduces a single point of failure for span collection — if the collector pod restarts, spans from the entire cluster queue or drop until it recovers. + +The DaemonSet pattern keeps the first hop node-local: each pod sends spans to `$(status.hostIP):4318`, which is the agent running on the same physical node. No cross-node traffic is incurred for span ingestion. The agent is small (50m CPU / 128Mi memory) and runs a short-lived buffer with a 200ms batch window before forwarding gRPC to the gateway. + +**Why still have a central gateway:** + +The agent is intentionally thin — short buffers, fast batching, minimal processing. Centralising durable buffering, larger batches (1s / 1000 spans), and the Tempo export in a single gateway pod means: + +- The agent failure domain is scoped to one node. If it crashes, only pods on that node are affected until it restarts (typically seconds on a DaemonSet). +- The gateway is the single point that speaks to Tempo. Tempo connection details, TLS config, or backend changes only need to be updated in one place. +- Backpressure from Tempo (slow writes, storage pressure) is absorbed at the gateway, not propagated back to every node agent simultaneously. + +**Trade-off:** + +The two-tier setup adds a hop and two separate Helm releases to manage. For a single-node development cluster it is over-engineered. For a multi-node cluster where workload nodes turn over frequently (Karpenter), the DaemonSet's node-local placement and independent lifecycle justify the added complexity. + +--- + ## Demo Trade-offs The following are intentional shortcuts taken for a demonstration environment. Each entry notes what was simplified and what the production-ready alternative looks like. diff --git a/docs/security.md b/docs/security.md index 048ff43..671ca60 100644 --- a/docs/security.md +++ b/docs/security.md @@ -21,6 +21,7 @@ No workload uses a node-level IAM role for AWS API access. Every component that | `prometheus` | `monitoring` | S3 read/write - Thanos bucket only (sidecar ship path) | | `thanos-compactor`, `thanos-storegateway` | `monitoring` | S3 read/write - Thanos bucket only | | `velero` | `velero` | S3 access on Velero bucket; `ec2:CreateSnapshot`, `ec2:DeleteSnapshot`, `ec2:Describe*` (AWS does not support resource ARNs for snapshot operations) | +| `tempo` | `tracing` | S3 `GetObject`, `PutObject`, `DeleteObject` - Tempo bucket only | | `karpenter` | `karpenter` | Pod Identity Association - newer mechanism, no OIDC annotation required | Inline Checkov skips (`CKV_AWS_111`, `CKV_AWS_356`) document the two IAM policies where AWS does not support resource-level ARNs. @@ -121,13 +122,13 @@ Policy Reporter (chart `2.24.1`) provides a web UI for browsing policy violation Three buckets store observability data and backups. All share the same baseline controls: -| Control | Loki | Thanos | Velero | -|---------|------|--------|--------| -| Encryption | SSE-S3 (AES-256) | SSE-S3 (AES-256) | SSE-S3 (AES-256) | -| Versioning | Enabled | Enabled | Enabled | -| Public access block | All four flags enabled | All four flags enabled | All four flags enabled | -| Access logging | Disabled | Disabled | Disabled | -| Static credentials | None - IRSA only | None - IRSA only | None - IRSA only | +| Control | Loki | Thanos | Velero | Tempo | +|---------|------|--------|--------|-------| +| Encryption | SSE-S3 (AES-256) | SSE-S3 (AES-256) | SSE-S3 (AES-256) | SSE-S3 (AES-256) | +| Versioning | Enabled | Enabled | Enabled | Enabled | +| Public access block | All four flags enabled | All four flags enabled | All four flags enabled | All four flags enabled | +| Access logging | Disabled | Disabled | Disabled | Disabled | +| Static credentials | None - IRSA only | None - IRSA only | None - IRSA only | None - IRSA only | SSE-KMS and access logging are intentionally deferred for a personal demo environment. Both are tracked in [Production Hardening](#production-hardening). The rationale for each skipped Checkov check is documented in `terraform/.checkov.yaml`. diff --git a/gitops/README.md b/gitops/README.md index 2827e99..5619e19 100644 --- a/gitops/README.md +++ b/gitops/README.md @@ -71,12 +71,20 @@ gitops/ │ ├── simple-time-service-alerts.yaml # ApplicationSet - PrometheusRule (alert expressions) (wave 10) │ ├── alertmanager-webhook-secret.yaml # ApplicationSet - ExternalSecret for Slack webhook (wave 5) │ └── alertmanager-slack.yaml # ApplicationSet - AlertmanagerConfig (Slack routing) (wave 9) -└── logs/ - ├── loki/ - │ ├── loki.yaml # ApplicationSet - Loki single-binary log store (wave 6) - │ └── grafana-loki-datasource.yaml # ApplicationSet - Loki datasource ConfigMap (wave 8) - └── fluent-bit/ - └── fluent-bit.yaml # ApplicationSet - Fluent Bit DaemonSet (wave 8) +├── logs/ +│ ├── loki/ +│ │ ├── loki.yaml # ApplicationSet - Loki single-binary log store (wave 6) +│ │ └── grafana-loki-datasource.yaml # ApplicationSet - Loki datasource ConfigMap (wave 8) +│ └── fluent-bit/ +│ └── fluent-bit.yaml # ApplicationSet - Fluent Bit DaemonSet (wave 8) +├── tracing/ +│ ├── README.md # OTel Collector pipeline, Tempo S3 backend, trace querying and log correlation +│ ├── tempo/ +│ │ ├── tempo.yaml # ApplicationSet - Tempo trace store S3 backend (wave 6) +│ │ └── grafana-tempo-datasource.yaml # ApplicationSet - Tempo datasource ConfigMap + Loki correlation (wave 8) +│ └── otel-collector/ +│ ├── otel-collector-gateway.yaml # ApplicationSet - OTel Collector Deployment gateway → Tempo (wave 6) +│ └── otel-collector-agent.yaml # ApplicationSet - OTel Collector DaemonSet agent → gateway (wave 7) └── security/ └── kyverno/ ├── kyverno.yaml # ApplicationSet - Kyverno admission controller (wave 3) @@ -98,6 +106,7 @@ gitops/ | [monitoring/README.md](monitoring/README.md) | Prometheus, Grafana, Thanos, Prometheus Adapter, ServiceMonitor | | [alerts/README.md](alerts/README.md) | PrometheusRules, Slack setup, testing, silencing, grouping | | [logs/README.md](logs/README.md) | Loki S3 backend, Fluent Bit, Grafana datasource, LogQL queries | +| [tracing/README.md](tracing/README.md) | OTel Collector agent/gateway pipeline, Tempo S3 backend, trace querying and Loki correlation | | [security/README.md](security/README.md) | Kyverno policies, Policy Reporter UI, policy reports | --- @@ -199,10 +208,14 @@ Any push to `main` affecting `gitops/` or `charts/` is automatically applied wit | argocd-ingress | argocd | platform | 6 | ALB Ingress at `argocd.platform.` | | policy-reporter | security | security | 6 | Policy Reporter UI (port-forward access) | | loki | logging | observability | 6 | Loki log store | +| tempo | tracing | observability | 6 | Tempo trace store (S3 backend) | +| otel-collector-gateway | tracing | observability | 6 | OTel Collector Deployment — batches spans and exports to Tempo | | prometheus | monitoring | observability | 7 | kube-prometheus-stack | +| otel-collector-agent | tracing | observability | 7 | OTel Collector DaemonSet — receives spans from local pods, forwards to gateway | | simple-time-service | simple-time-service | workloads | 8 | SimpleTimeService Helm chart (HPA, ALB Ingress) | | fluent-bit | logging | observability | 8 | Log collector DaemonSet | | grafana-loki-datasource | monitoring | observability | 8 | Loki datasource ConfigMap | +| grafana-tempo-datasource | monitoring | observability | 8 | Tempo datasource ConfigMap (trace-to-log correlation) | | grafana-dashboard | monitoring | observability | 8 | SimpleTimeService dashboard ConfigMap | | prometheus-adapter | monitoring | observability | 9 | Custom metrics API bridge (enables custom-metric HPA) | | alertmanager-slack | monitoring | observability | 9 | AlertmanagerConfig CRD | @@ -225,8 +238,8 @@ ArgoCD advances through sync waves sequentially, waiting for all resources in th | 3 | Karpenter, Velero, **Kyverno** | | 4 | ClusterSecretStore, Karpenter NodePools | | 5 | All ExternalSecrets, VeleroSchedule, Kyverno ClusterPolicies | -| 6 | **ArgoCD Ingress**, Loki, Policy Reporter UI | -| 7 | Prometheus (kube-prometheus-stack) | +| 6 | **ArgoCD Ingress**, Loki, **Tempo**, OTel Gateway, Policy Reporter UI | +| 7 | Prometheus (kube-prometheus-stack), **OTel Agent** DaemonSet | | 8 | SimpleTimeService, Fluent Bit, Grafana datasource + dashboard | | 9 | Prometheus Adapter, AlertmanagerConfig | | 10 | PrometheusRules, Thanos | @@ -244,7 +257,7 @@ All ApplicationSets are scoped to one of five AppProjects defined in `gitops/arg | `bootstrap` | `bootstrap-project.yaml` | `argocd` | None - root-app only deploys namespace-scoped ArgoCD resources | root-app only | | `namespaces` | `namespaces-project.yaml` | `*` | `Namespace` only | cluster-namespaces (creates namespaces + ResourceQuotas) | | `platform` | `platform-project.yaml` | `argocd`, `kube-system`, `karpenter`, `external-secrets`, `external-dns`, `reloader`, `velero`, `kube-node-lease` | All (`*/*`) - infra tools install CRDs and cluster RBAC | ArgoCD self-management, networking, autoscaling, secrets, storage, backup | -| `observability` | `observability-project.yaml` | `monitoring`, `logging`, `kube-system` | `CustomResourceDefinition`, `ClusterRole`, `ClusterRoleBinding` | Prometheus, Grafana, Thanos, Loki, Fluent Bit, alerts | +| `observability` | `observability-project.yaml` | `monitoring`, `logging`, `tracing`, `kube-system` | `CustomResourceDefinition`, `ClusterRole`, `ClusterRoleBinding` | Prometheus, Grafana, Thanos, Loki, Fluent Bit, Tempo, alerts | | `security` | `security-project.yaml` | `security` | `CustomResourceDefinition`, `ClusterRole`, `ClusterRoleBinding`, webhooks, `ClusterPolicy` | Kyverno admission controller, ClusterPolicies, Policy Reporter | | `workloads` | `workloads-project.yaml` | `simple-time-service` | None | Application workloads | diff --git a/gitops/app/simple-time-service/simple-time-service.yaml b/gitops/app/simple-time-service/simple-time-service.yaml index 1300e1f..b6090da 100644 --- a/gitops/app/simple-time-service/simple-time-service.yaml +++ b/gitops/app/simple-time-service/simple-time-service.yaml @@ -55,6 +55,10 @@ spec: - path: / pathType: Prefix + tracing: + enabled: true + serviceName: simple-time-service + serviceMonitor: enabled: true diff --git a/gitops/argocd/projects/observability-project.yaml b/gitops/argocd/projects/observability-project.yaml index 0065a85..60ec246 100644 --- a/gitops/argocd/projects/observability-project.yaml +++ b/gitops/argocd/projects/observability-project.yaml @@ -8,12 +8,14 @@ metadata: app.kubernetes.io/managed-by: argocd app.kubernetes.io/component: monitoring spec: - description: Observability stack - metrics, logs, and alerting (Prometheus, Grafana, Thanos, Loki, Fluent Bit) + description: Observability stack - metrics, logs, traces, and alerting (Prometheus, Grafana, Thanos, Loki, Tempo, Fluent Bit) sourceRepos: - https://prometheus-community.github.io/helm-charts - registry-1.docker.io/bitnamicharts - https://grafana-community.github.io/helm-charts + - https://grafana.github.io/helm-charts + - https://open-telemetry.github.io/opentelemetry-helm-charts - https://fluent.github.io/helm-charts # GitOps repo (dashboards, datasources, alert rules, secrets) - https://github.com/Beem0807/eks-gitops-platform.git @@ -23,6 +25,8 @@ spec: namespace: monitoring - server: https://kubernetes.default.svc namespace: logging + - server: https://kubernetes.default.svc + namespace: tracing - server: https://kubernetes.default.svc namespace: kube-system diff --git a/gitops/logs/README.md b/gitops/logs/README.md index 04ebff4..30e4022 100644 --- a/gitops/logs/README.md +++ b/gitops/logs/README.md @@ -83,9 +83,11 @@ A ConfigMap with label `grafana_datasource: "1"` deployed into the `monitoring` | Setting | Value | |---------|-------| | Name | `Loki` | +| UID | `loki` | | Type | `loki` | | URL | `http://loki-gateway.logging.svc.cluster.local` | | Default | No (Prometheus remains the default) | +| Derived field | Regex `"trace_id":"([a-f0-9]{32})"` — adds a **TraceID** link on matching JSON log lines that opens the span in Tempo | --- diff --git a/gitops/logs/loki/grafana-loki-datasource.yaml b/gitops/logs/loki/grafana-loki-datasource.yaml index fe88b05..ce85714 100644 --- a/gitops/logs/loki/grafana-loki-datasource.yaml +++ b/gitops/logs/loki/grafana-loki-datasource.yaml @@ -47,10 +47,17 @@ spec: datasources: - name: Loki type: loki + uid: loki access: proxy url: http://loki-gateway.logging.svc.cluster.local isDefault: false editable: true + jsonData: + derivedFields: + - datasourceUid: tempo + matcherRegex: '"trace_id":"([a-f0-9]{32})"' + name: TraceID + url: "$${__value.raw}" destination: server: '{{.server}}' namespace: monitoring diff --git a/gitops/namespaces/namespaces.yaml b/gitops/namespaces/namespaces.yaml index 33ba01e..a69edce 100644 --- a/gitops/namespaces/namespaces.yaml +++ b/gitops/namespaces/namespaces.yaml @@ -58,6 +58,7 @@ spec: - name: logging - name: simple-time-service - name: security + - name: tracing destination: server: '{{.server}}' namespace: argocd diff --git a/gitops/tracing/README.md b/gitops/tracing/README.md new file mode 100644 index 0000000..1f97d44 --- /dev/null +++ b/gitops/tracing/README.md @@ -0,0 +1,205 @@ +# Tracing - OTel Collector and Tempo + +The `gitops/tracing/` directory deploys the full distributed tracing pipeline via ArgoCD. + +| File | What it deploys | +|------|----------------| +| `tempo/tempo.yaml` | Grafana Tempo in monolithic mode - central trace store backed by S3 | +| `tempo/grafana-tempo-datasource.yaml` | Grafana Tempo datasource ConfigMap with Loki trace-to-log correlation, auto-provisioned via `charts/raw` | +| `otel-collector/otel-collector-gateway.yaml` | OTel Collector Deployment - central gateway that receives spans from all agents, batches, and exports to Tempo | +| `otel-collector/otel-collector-agent.yaml` | OTel Collector DaemonSet - node-local agent that receives spans from co-located pods and forwards to the gateway | + +--- + +## Architecture + +``` +Pod (simple-time-service) + │ OTLP HTTP → status.hostIP:4318 (node-local, no cross-node hop) + ▼ +OTel Agent (DaemonSet · one pod per node · hostPort 4318) + │ OTLP gRPC → otel-gateway.tracing.svc.cluster.local:4317 + ▼ +OTel Gateway (Deployment · tracing namespace · batch 1 s / 1 000 spans) + │ OTLP gRPC → tempo.tracing.svc.cluster.local:4317 + ▼ +Tempo (Deployment · tracing namespace) + │ writes trace blocks to S3 via IRSA · 30-day retention + ▼ +S3 bucket: -tempo-traces-- + +Grafana ◀── Tempo datasource (query port :3100) + tracesToLogsV2 ──▶ Loki (uid: loki) +``` + +The two-tier collector topology keeps inter-node traffic off the path for every span write. Pods send to `$(status.hostIP):4318` — always the local agent — so the only cross-node traffic is the agent-to-gateway leg, which is batched. + +Sync-wave ordering ensures the gateway is healthy before the agent tries to forward to it, and both are ready before the application workload starts emitting spans: + +| Component | Sync wave | Reason | +|-----------|-----------|--------| +| Tempo | 6 | Storage backend before any span is accepted | +| OTel Gateway | 6 | Accepts from agents; must exist before agents start | +| OTel Agent | 7 | Forwards to gateway; deploying after guarantees the endpoint resolves | +| Grafana Tempo datasource | 8 | After Grafana (wave 7) and after Loki datasource (wave 8) | + +--- + +## OTel Collector Gateway + +Runs as a single-replica **Deployment** in the `tracing` namespace. Tolerates `app=core:NoSchedule` so it lands on a managed node group node alongside the other platform services. + +| Setting | Value | +|---------|-------| +| Mode | `deployment` | +| Receivers | OTLP gRPC `:4317`, OTLP HTTP `:4318` | +| Processors | `memory_limiter` (256 MiB) → `batch` (1 s / 1 000 spans) | +| Exporter | OTLP gRPC → `tempo.tracing.svc.cluster.local:4317` (insecure, in-cluster) | +| Resources | 100m / 256Mi request · 500m / 512Mi limit | +| Service name | `otel-gateway` (via `fullnameOverride`) | + +The `memory_limiter` runs before `batch` so backpressure propagates upstream before the process OOMs. + +--- + +## OTel Collector Agent + +Runs as a **DaemonSet** in the `tracing` namespace — one pod per node. Tolerates all `NoSchedule` taints so it runs on both managed core nodes and Karpenter workload nodes. + +| Setting | Value | +|---------|-------| +| Mode | `daemonset` | +| Receiver | OTLP HTTP `:4318` (also exposed as `hostPort: 4318`) | +| Processors | `memory_limiter` (128 MiB) → `batch` (200 ms / 128 spans) | +| Exporter | OTLP gRPC → `otel-gateway.tracing.svc.cluster.local:4317` (insecure, in-cluster) | +| Resources | 50m / 128Mi request · 200m / 256Mi limit | +| Service name | `otel-agent` (via `fullnameOverride`) | + +The `hostPort: 4318` binding is what makes `http://$(NODE_IP):4318` work from application pods. The Helm chart for `simple-time-service` injects `NODE_IP` via the downward API (`status.hostIP`) and constructs the endpoint automatically when `tracing.enabled: true`. + +--- + +## Tempo + +Runs in **monolithic mode** (single-process, all components) in the `tracing` namespace. + +| Setting | Value | Notes | +|---------|-------|-------| +| Mode | Monolithic | All components in one process - suitable for demos | +| Storage | S3 (`backend: s3`) | Trace blocks written durably to S3 | +| Retention | 30 days (`retention: 720h`) | Configurable in `tempo/tempo.yaml` | +| IRSA | `-tempo-irsa` | Grants scoped S3 read/write; no static credentials | +| Bucket | `-tempo-traces--` | Provisioned by `terraform/tempo.tf` | +| OTLP gRPC | `:4317` | Receives spans from the OTel gateway | +| Query API | `:3100` | Used by the Grafana datasource | + +--- + +## Grafana Tempo datasource + +A ConfigMap with label `grafana_datasource: "1"` deployed into the `monitoring` namespace. Grafana's sidecar detects it and provisions the datasource automatically - no manual steps needed. + +| Setting | Value | +|---------|-------| +| Name | `Tempo` | +| UID | `tempo` | +| URL | `http://tempo.tracing.svc.cluster.local:3100` | +| Node graph | Enabled | +| `tracesToLogsV2.datasourceUid` | `loki` — clicking a span opens the correlated Loki log stream | +| `lokiSearch.datasourceUid` | `loki` — enables the **Logs** tab in the Tempo explorer | +| `serviceMap.datasourceUid` | `thanos` — service graph overlay from Prometheus metrics | + +The Loki datasource is given `uid: loki` in `gitops/logs/loki/grafana-loki-datasource.yaml` so the reference above resolves correctly. + +Correlation is **bidirectional**: + +| Direction | Mechanism | How | +|-----------|-----------|-----| +| Trace → Logs | `tracesToLogsV2` in Tempo datasource | Open a trace → **Logs** tab → Grafana runs `{job=~".+"} \| json \| trace_id = ""` against Loki | +| Log → Trace | `derivedFields` in Loki datasource | A log line with `"trace_id":"..."` gets a clickable **TraceID** link that opens the span in Tempo | + +`derivedFields` applies a regex to each log line. Non-JSON lines that don't match are displayed unchanged — no errors, no broken layout. + +--- + +## Instrumenting an application + +The `simple-time-service` Helm chart handles all OTel wiring. Enable tracing in `gitops/app/simple-time-service/simple-time-service.yaml`: + +```yaml +tracing: + enabled: true + serviceName: simple-time-service # appears in Tempo search +``` + +The chart injects three env vars into the pod: + +```yaml +env: + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: http://$(NODE_IP):4318 # always the local agent + - name: OTEL_SERVICE_NAME + value: simple-time-service +``` + +The app (`app/src/app.py`) uses `opentelemetry-instrumentation-fastapi` to auto-instrument all HTTP routes and `JSONFormatter` to inject `trace_id` and `span_id` into every log line when a span is active. + +For a new service, mirror this pattern: enable the OTel SDK with `OTEL_EXPORTER_OTLP_ENDPOINT` set to the node-local agent and `OTEL_SERVICE_NAME` set to something meaningful. + +--- + +## Querying traces in Grafana + +```bash +kubectl port-forward svc/prometheus-grafana -n monitoring 3000:80 +``` + +Open [http://localhost:3000](http://localhost:3000) → **Explore** → select **Tempo** datasource. + +**Search by service:** +- Tab: **Search** +- Service Name: `simple-time-service` +- Click any trace → span waterfall + +**Search by trace ID** (from a JSON log line in Loki): + +1. Explore → **Loki** → `{namespace="simple-time-service"}` +2. Find a log line with `"trace_id": "..."` and click the **Tempo** link that appears in the derived fields panel + +**From Grafana Explore (split view):** + +Use the split-pane view to open Loki and Tempo side-by-side. Clicking a trace ID in the Loki panel opens the matching Tempo trace in the right pane. + +--- + +## Verifying the pipeline + +```bash +# All tracing pods healthy +kubectl get pods -n tracing + +# Agent is running on every node +kubectl get pods -n tracing -l app.kubernetes.io/name=opentelemetry-collector \ + -o wide + +# Gateway is ready +kubectl rollout status deploy/otel-gateway -n tracing + +# Tempo is ready +kubectl rollout status deploy/tempo -n tracing + +# Send a test span to the local agent from outside the cluster (replace NODE_IP) +NODE_IP=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}') +kubectl run otlp-test --rm -it --image=curlimages/curl --restart=Never -- \ + curl -s -X POST "http://${NODE_IP}:4318/v1/traces" \ + -H "Content-Type: application/json" \ + -d '{"resourceSpans":[]}' + +# Check Tempo received spans +kubectl port-forward svc/tempo -n tracing 3100:3100 +curl http://localhost:3100/ready +``` diff --git a/gitops/tracing/otel-collector/otel-collector-agent.yaml b/gitops/tracing/otel-collector/otel-collector-agent.yaml new file mode 100644 index 0000000..045fa8b --- /dev/null +++ b/gitops/tracing/otel-collector/otel-collector-agent.yaml @@ -0,0 +1,96 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: otel-collector-agent + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "7" + labels: + app.kubernetes.io/part-of: eks-gitops-platform + app.kubernetes.io/managed-by: argocd + app.kubernetes.io/component: tracing +spec: + goTemplate: true + goTemplateOptions: ["missingkey=error"] + generators: + - clusters: + selector: + matchLabels: + argocd.argoproj.io/secret-type: cluster + template: + metadata: + name: 'otel-collector-agent-{{.nameNormalized}}' + labels: + app.kubernetes.io/part-of: eks-gitops-platform + app.kubernetes.io/managed-by: argocd + app.kubernetes.io/component: tracing + spec: + project: observability + source: + repoURL: https://open-telemetry.github.io/opentelemetry-helm-charts + chart: opentelemetry-collector + targetRevision: 0.108.0 + helm: + releaseName: otel-agent + values: | + fullnameOverride: otel-agent + + mode: daemonset + + config: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + processors: + memory_limiter: + check_interval: 5s + limit_mib: 128 + batch: + timeout: 200ms + send_batch_size: 128 + exporters: + otlp: + endpoint: otel-gateway.tracing.svc.cluster.local:4317 + tls: + insecure: true + service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp] + + ports: + otlp-http: + enabled: true + containerPort: 4318 + servicePort: 4318 + hostPort: 4318 + protocol: TCP + + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 200m + memory: 256Mi + + # Run on every node - core nodes and Karpenter workload nodes + tolerations: + - operator: Exists + effect: NoSchedule + + serviceMonitor: + enabled: true + destination: + server: '{{.server}}' + namespace: tracing + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/gitops/tracing/otel-collector/otel-collector-gateway.yaml b/gitops/tracing/otel-collector/otel-collector-gateway.yaml new file mode 100644 index 0000000..297de05 --- /dev/null +++ b/gitops/tracing/otel-collector/otel-collector-gateway.yaml @@ -0,0 +1,92 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: otel-collector-gateway + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "6" + labels: + app.kubernetes.io/part-of: eks-gitops-platform + app.kubernetes.io/managed-by: argocd + app.kubernetes.io/component: tracing +spec: + goTemplate: true + goTemplateOptions: ["missingkey=error"] + generators: + - clusters: + selector: + matchLabels: + argocd.argoproj.io/secret-type: cluster + template: + metadata: + name: 'otel-collector-gateway-{{.nameNormalized}}' + labels: + app.kubernetes.io/part-of: eks-gitops-platform + app.kubernetes.io/managed-by: argocd + app.kubernetes.io/component: tracing + spec: + project: observability + source: + repoURL: https://open-telemetry.github.io/opentelemetry-helm-charts + chart: opentelemetry-collector + targetRevision: 0.108.0 + helm: + releaseName: otel-gateway + values: | + fullnameOverride: otel-gateway + + mode: deployment + replicaCount: 1 + + config: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + processors: + memory_limiter: + check_interval: 5s + limit_mib: 256 + batch: + timeout: 1s + send_batch_size: 1000 + exporters: + otlp: + endpoint: tempo.tracing.svc.cluster.local:4317 + tls: + insecure: true + service: + pipelines: + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp] + + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + + tolerations: + - key: "app" + operator: "Equal" + value: "core" + effect: "NoSchedule" + + serviceMonitor: + enabled: true + destination: + server: '{{.server}}' + namespace: tracing + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/gitops/tracing/tempo/grafana-tempo-datasource.yaml b/gitops/tracing/tempo/grafana-tempo-datasource.yaml new file mode 100644 index 0000000..20a101f --- /dev/null +++ b/gitops/tracing/tempo/grafana-tempo-datasource.yaml @@ -0,0 +1,75 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: grafana-tempo-datasource + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "8" + labels: + app.kubernetes.io/part-of: eks-gitops-platform + app.kubernetes.io/managed-by: argocd + app.kubernetes.io/component: tracing +spec: + goTemplate: true + goTemplateOptions: ["missingkey=error"] + generators: + - clusters: + selector: + matchLabels: + argocd.argoproj.io/secret-type: cluster + template: + metadata: + name: 'grafana-tempo-datasource-{{.nameNormalized}}' + labels: + app.kubernetes.io/part-of: eks-gitops-platform + app.kubernetes.io/managed-by: argocd + app.kubernetes.io/component: tracing + spec: + project: observability + source: + repoURL: https://github.com/Beem0807/eks-gitops-platform.git + targetRevision: main + path: charts/raw + helm: + releaseName: grafana-tempo-datasource + values: | + resources: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: grafana-tempo-datasource + namespace: monitoring + labels: + grafana_datasource: "1" + data: + tempo-datasource.yaml: | + apiVersion: 1 + datasources: + - name: Tempo + type: tempo + uid: tempo + access: proxy + url: http://tempo.tracing.svc.cluster.local:3100 + isDefault: false + editable: true + jsonData: + nodeGraph: + enabled: true + tracesToLogsV2: + datasourceUid: loki + filterByTraceID: true + filterBySpanID: false + customQuery: false + lokiSearch: + datasourceUid: loki + serviceMap: + datasourceUid: thanos + destination: + server: '{{.server}}' + namespace: monitoring + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/gitops/tracing/tempo/tempo.yaml b/gitops/tracing/tempo/tempo.yaml new file mode 100644 index 0000000..3bc7282 --- /dev/null +++ b/gitops/tracing/tempo/tempo.yaml @@ -0,0 +1,81 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: tempo + namespace: argocd + annotations: + argocd.argoproj.io/sync-wave: "6" + labels: + app.kubernetes.io/part-of: eks-gitops-platform + app.kubernetes.io/managed-by: argocd + app.kubernetes.io/component: tracing +spec: + goTemplate: true + goTemplateOptions: ["missingkey=error"] + generators: + - clusters: + selector: + matchLabels: + argocd.argoproj.io/secret-type: cluster + values: + accountId: '{{index .metadata.annotations "aws-account-id"}}' + region: '{{index .metadata.labels "region"}}' + clusterName: '{{index .metadata.annotations "cluster-name"}}' + tempoBucketName: '{{index .metadata.annotations "tempo-bucket-name"}}' + template: + metadata: + name: 'tempo-{{.nameNormalized}}' + labels: + app.kubernetes.io/part-of: eks-gitops-platform + app.kubernetes.io/managed-by: argocd + app.kubernetes.io/component: tracing + spec: + project: observability + source: + repoURL: https://grafana.github.io/helm-charts + chart: tempo + targetRevision: 1.10.3 + helm: + releaseName: tempo + values: | + serviceAccount: + create: true + annotations: + eks.amazonaws.com/role-arn: arn:aws:iam::{{ .values.accountId }}:role/{{ .values.clusterName }}-tempo-irsa + + tempo: + storage: + trace: + backend: s3 + s3: + bucket: {{ .values.tempoBucketName }} + region: {{ .values.region }} + insecure: false + + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + + retention: 720h + + tolerations: + - key: "app" + operator: "Equal" + value: "core" + effect: "NoSchedule" + + serviceMonitor: + enabled: true + destination: + server: '{{.server}}' + namespace: tracing + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/terraform/README.md b/terraform/README.md index e6225fc..a94dd4f 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -26,10 +26,11 @@ This directory provisions the AWS infrastructure: a VPC and an EKS cluster. The | Node security group | NLB NodePort rule optional (`enable_nlb_nodeport_rule`); Karpenter discovery tag when `enable_karpenter_discovery_tags=true` | | EKS add-ons | `coredns`, `kube-proxy`, `vpc-cni`, `eks-pod-identity-agent` managed by the EKS module | | Karpenter | IAM controller role + EKS Pod Identity association + node IAM role + instance profile + SQS interruption queue | -| IRSA roles | Cluster Autoscaler, AWS Load Balancer Controller, ExternalDNS, External Secrets Operator, EBS CSI Driver controller, Thanos Prometheus sidecar, Thanos Compactor + StoreGateway, Loki, Velero | +| IRSA roles | Cluster Autoscaler, AWS Load Balancer Controller, ExternalDNS, External Secrets Operator, EBS CSI Driver controller, Thanos Prometheus sidecar, Thanos Compactor + StoreGateway, Loki, Velero, Tempo | | Thanos S3 bucket | `-thanos-metrics--` - versioned, AES256-encrypted, public access blocked | | Loki S3 bucket | `-loki-logs--` - versioned, AES256-encrypted, public access blocked | | Velero S3 bucket | `-velero-backups--` - versioned, AES256-encrypted, public access blocked | +| Tempo S3 bucket | `-tempo-traces--` - versioned, AES256-encrypted, public access blocked | | Secrets Manager | `argocd-admin`, `grafana-admin`, `alertmanager-webhook` secrets provisioned by Terraform | Modules used: [`terraform-aws-modules/eks/aws`](https://registry.terraform.io/modules/terraform-aws-modules/eks/aws/latest), [`terraform-aws-modules/vpc/aws`](https://registry.terraform.io/modules/terraform-aws-modules/vpc/aws/latest), and [`terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts`](https://registry.terraform.io/modules/terraform-aws-modules/iam/aws/latest). @@ -209,7 +210,7 @@ bash terraform/scripts/bootstrap.sh ### upgrade.sh - re-apply Terraform on a running cluster -Use `upgrade.sh` when you need to apply Terraform changes or rotate secrets without reinstalling ArgoCD. It re-runs Terraform, refreshes the ArgoCD cluster secret with the latest outputs (including the Thanos bucket name), and force-annotates all three ExternalSecrets to trigger an immediate re-sync. +Use `upgrade.sh` when you need to apply Terraform changes or rotate secrets without reinstalling ArgoCD. It re-runs Terraform, refreshes the ArgoCD cluster secret with the latest outputs (including Thanos/Loki/Velero/Tempo bucket names), and force-annotates all three ExternalSecrets to trigger an immediate re-sync. ```bash export TF_VAR_alertmanager_slack_webhook_url="https://hooks.slack.com/..." @@ -223,7 +224,7 @@ bash terraform/scripts/upgrade.sh 1. **Kubernetes resources** - deletes the root app, all ArgoCD Applications and ApplicationSets, Ingresses, LoadBalancer Services, Karpenter NodePools and NodeClaims, PersistentVolumes, and all platform namespaces (including `velero`) 2. **AWS Load Balancers** - deletes leftover ALB/NLB/Classic ELBs tagged for the cluster 3. **EBS volumes and snapshots** - deletes EBS volumes tagged `kubernetes.io/cluster/=owned` (CSI Driver PVCs left behind by the `Retain` reclaim policy), then deletes EBS snapshots tagged for the cluster or carrying a `velero.io/backup` tag -4. **S3 buckets** - empties all versioned S3 buckets in the same region whose name contains the cluster name (Thanos, Loki, Velero), draining all object versions and delete markers in batches so that Terraform can delete the buckets cleanly +4. **S3 buckets** - empties all versioned S3 buckets in the same region whose name contains the cluster name (Thanos, Loki, Velero, Tempo), draining all object versions and delete markers in batches so that Terraform can delete the buckets cleanly 5. **Terraform destroy** - removes all remaining AWS resources --- diff --git a/terraform/outputs.tf b/terraform/outputs.tf index bac0a05..4f1d653 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -37,4 +37,8 @@ output "loki_bucket_name" { output "velero_bucket_name" { value = aws_s3_bucket.velero.bucket +} + +output "tempo_bucket_name" { + value = aws_s3_bucket.tempo.bucket } \ No newline at end of file diff --git a/terraform/scripts/bootstrap.sh b/terraform/scripts/bootstrap.sh index e46b78d..017bf9b 100755 --- a/terraform/scripts/bootstrap.sh +++ b/terraform/scripts/bootstrap.sh @@ -154,6 +154,7 @@ KARPENTER_INSTANCE_PROFILE_NAME="$(terraform output -raw karpenter_instance_prof THANOS_BUCKET_NAME="$(terraform output -raw thanos_bucket_name 2>/dev/null || true)" LOKI_BUCKET_NAME="$(terraform output -raw loki_bucket_name 2>/dev/null || true)" VELERO_BUCKET_NAME="$(terraform output -raw velero_bucket_name 2>/dev/null || true)" +TEMPO_BUCKET_NAME="$(terraform output -raw tempo_bucket_name 2>/dev/null || true)" echo "Updating kubeconfig..." aws eks update-kubeconfig \ @@ -241,6 +242,7 @@ metadata: thanos-bucket-name: "${THANOS_BUCKET_NAME}" loki-bucket-name: "${LOKI_BUCKET_NAME}" velero-bucket-name: "${VELERO_BUCKET_NAME}" + tempo-bucket-name: "${TEMPO_BUCKET_NAME}" type: Opaque stringData: name: in-cluster-local diff --git a/terraform/tempo.tf b/terraform/tempo.tf new file mode 100644 index 0000000..a7a94a5 --- /dev/null +++ b/terraform/tempo.tf @@ -0,0 +1,97 @@ +resource "aws_s3_bucket" "tempo" { + bucket = "${var.cluster_name}-tempo-traces-${data.aws_caller_identity.current.account_id}-${var.aws_region}" + + tags = merge(var.tags, { + Name = "${var.cluster_name}-tempo-traces" + }) +} + +resource "aws_s3_bucket_versioning" "tempo" { + bucket = aws_s3_bucket.tempo.id + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "tempo" { + bucket = aws_s3_bucket.tempo.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_public_access_block" "tempo" { + bucket = aws_s3_bucket.tempo.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +data "aws_iam_policy_document" "tempo_s3" { + statement { + sid = "TempoBucketAccess" + effect = "Allow" + + actions = [ + "s3:ListBucket", + "s3:GetBucketLocation", + "s3:ListBucketMultipartUploads" + ] + + resources = [ + aws_s3_bucket.tempo.arn + ] + } + + statement { + sid = "TempoObjectAccess" + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts" + ] + + resources = [ + "${aws_s3_bucket.tempo.arn}/*" + ] + } +} + +resource "aws_iam_policy" "tempo_s3" { + name = "${var.cluster_name}-tempo-s3" + description = "S3 access policy for Tempo trace storage" + policy = data.aws_iam_policy_document.tempo_s3.json + + tags = var.tags +} + +module "tempo_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts" + version = "~> 6.0" + + name = "${var.cluster_name}-tempo-irsa" + use_name_prefix = false + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["tracing:tempo"] + } + } + + policies = { + tempo_s3 = aws_iam_policy.tempo_s3.arn + } + + tags = var.tags +}