diff --git a/.github/labeler.yml b/.github/labeler.yml index 9601c0aaa..11a7b60e3 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,5 +1,9 @@ -'package: wallet/backend': packages/wallet/backend/**/* -'package: wallet/frontend': packages/wallet/frontend/**/* +'package: wallet/backend': + - packages/wallet/backend/**/* + - packages/shared/telemetry/**/* +'package: wallet/frontend': + - packages/wallet/frontend/**/* + - packages/shared/telemetry/**/* 'package: boutique/backend': packages/boutique/backend/**/* 'package: boutique/frontend': packages/boutique/frontend/**/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2cd1d6584..1a391878b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: steps: - uses: actions/checkout@v5 - uses: ./.github/workflows/setup - - run: pnpm wallet:backend build && pnpm wallet:backend test --detectOpenHandles --forceExit + - run: pnpm wallet:backend build && pnpm --filter @shared/telemetry test && pnpm wallet:backend test --detectOpenHandles --forceExit test-boutique-backend: name: BOUTIQUE - Test backend diff --git a/README.md b/README.md index 732a5d8f8..39ab4f1f2 100644 --- a/README.md +++ b/README.md @@ -156,6 +156,7 @@ Upon executing the above commands the following will be available: - [https://mockgatehub.testnet.test](https://mockgatehub.testnet.test) - Mock GateHub service used for local funding and related sandbox flows. - [https://rafiki-frontend.testnet.test](https://rafiki-frontend.testnet.test) - Rafiki frontend UI. - [https://rafiki-backend.testnet.test](https://rafiki-backend.testnet.test) - Rafiki backend service. +- [https://grafana.testnet.test](https://grafana.testnet.test) - Grafana with the wallet's metrics and traces. Opt-in: `pnpm dev:observability`. See `pnpm local:help`. ### Local Playground on Rhyza diff --git a/helm/testnet-wallet/README.md b/helm/testnet-wallet/README.md index 5ee0e8635..ebc6de652 100644 --- a/helm/testnet-wallet/README.md +++ b/helm/testnet-wallet/README.md @@ -134,6 +134,55 @@ merging it, so the environment's values file must also write out, in full: `env/cards-playground/wallet/testnet-wallet.yaml` in `testnet-deploy` is the worked example. +### OpenTelemetry + +The backend and the frontend's Node.js server can push traces and metrics over +OTLP/gRPC. The chart runs one OpenTelemetry collector per release, built the +same way as the collector in the merchant and issuer charts. The browser sends +nothing. + +Telemetry is off by default. One block turns it on for both components: + +```yaml +config: + telemetry: + enabled: true + serviceEnv: cards-playground +``` + +With `enabled: true` the chart: + +- adds the `TELEMETRY_*` and `SERVICE_*` keys to both ConfigMaps. They are not + contentMap entries; `templates/_helpers.tpl` renders them. See + `packages/shared/telemetry` for each key. +- points both components at `-otel-collector:4317`, unless + `config.telemetry.endpoint` names another collector. The render fails if + `otelCollector.enabled` is false and no endpoint is set. +- sets `SERVICE_VERSION` to the component's image tag. + +Each pod gets `POD_NAME`, `POD_NAMESPACE` and `OTEL_RESOURCE_ATTRIBUTES` through +`extraEnv`, not `env`, so an environment that rewrites `env` keeps them. +`service.instance.id` keeps the series of two replicas apart. + +The collector: + +- exposes the metrics on a Prometheus endpoint (`metrics`, port 8889) and its + own metrics on `telemetry` (port 8888). A ServiceMonitor scrapes both. The + chart renders the ServiceMonitor only where the Prometheus Operator CRDs + exist; `helm template` needs `--api-versions +monitoring.coreos.com/v1/ServiceMonitor` to show it. +- forwards traces to `otelCollector.traces.endpoint`. Empty drops them. On + ilf-1 the Tempo address is `core-tempo.monitoring.svc.cluster.local:4317`, + and the namespace needs `network.allowTracingEgress` in its + deploy-environment. + +`otelCollector.config` replaces the generated collector config wholesale. + +The metrics follow the OpenTelemetry semantic conventions. Examples: +`http_server_request_duration_seconds` (with `http_route` on the backend), +`http_client_request_duration_seconds`, `db_client_operation_duration_seconds` +and `nodejs_eventloop_*`. + ## Image Tags By default both deployments use the chart's `appVersion` as the image tag. To pin a specific tag for one image: diff --git a/helm/testnet-wallet/templates/_helpers.tpl b/helm/testnet-wallet/templates/_helpers.tpl new file mode 100644 index 000000000..99a3299d8 --- /dev/null +++ b/helm/testnet-wallet/templates/_helpers.tpl @@ -0,0 +1,194 @@ +{{/* +testnet-wallet.otelCollectorName is the collector's Deployment and ConfigMap +name. +*/}} +{{- define "testnet-wallet.otelCollectorName" -}} +{{- printf "%s-%s" (include "common.fullname" .) .Values.otelCollector.name -}} +{{- end -}} + +{{/* +testnet-wallet.otelCollectorServiceName is the collector Service's name, the +host the backend and the frontend dial. common.svc names it after +otelCollector.service.name, which need not match otelCollector.name. +*/}} +{{- define "testnet-wallet.otelCollectorServiceName" -}} +{{- printf "%s-%s" (include "common.fullname" .) .Values.otelCollector.service.name -}} +{{- end -}} + +{{/* +testnet-wallet.otelCollectorContainerPort is the collector container port with +the given name. The collector config listens on these ports, so the container +ports are the one place to move a listener. Fails the render if the port is +missing. + +Usage: include "testnet-wallet.otelCollectorContainerPort" (list . "otlp-grpc") +*/}} +{{- define "testnet-wallet.otelCollectorContainerPort" -}} +{{- $top := index . 0 -}} +{{- $name := index . 1 -}} +{{- $port := 0 -}} +{{- range $top.Values.otelCollector.ports -}} +{{- if eq .name $name -}} +{{- $port = .containerPort -}} +{{- end -}} +{{- end -}} +{{- if eq (int $port) 0 -}} +{{- fail (printf "otelCollector.ports must contain a port named %s — the collector config listens on it" $name) -}} +{{- end -}} +{{- $port -}} +{{- end -}} + +{{/* +testnet-wallet.otelCollectorOTLPPort is the collector Service's OTLP/gRPC port, +read off the port named `otlp-grpc`, so that moving it moves the endpoint the +backend and the frontend dial. +*/}} +{{- define "testnet-wallet.otelCollectorOTLPPort" -}} +{{- $port := 0 -}} +{{- range .Values.otelCollector.service.ports -}} +{{- if eq .name "otlp-grpc" -}} +{{- $port = .port -}} +{{- end -}} +{{- end -}} +{{- if eq (int $port) 0 -}} +{{- fail "otelCollector.service.ports must contain a port named otlp-grpc — the backend and the frontend dial the collector on it" -}} +{{- end -}} +{{- $port -}} +{{- end -}} + +{{/* +testnet-wallet.telemetryEndpoint is the OTLP/gRPC target both components send +to: config.telemetry.endpoint when set, otherwise this release's collector. +*/}} +{{- define "testnet-wallet.telemetryEndpoint" -}} +{{- $telemetry := .Values.config.telemetry -}} +{{- if $telemetry.endpoint -}} +{{- $telemetry.endpoint -}} +{{- else -}} +{{- if not .Values.otelCollector.enabled -}} +{{- fail "config.telemetry.enabled is true, but otelCollector.enabled is false and no config.telemetry.endpoint is set — enable the collector, or point the endpoint at one outside this release" -}} +{{- end -}} +{{- printf "%s:%v" (include "testnet-wallet.otelCollectorServiceName" .) (include "testnet-wallet.otelCollectorOTLPPort" .) -}} +{{- end -}} +{{- end -}} + +{{/* +testnet-wallet.telemetryData renders the TELEMETRY_* and SERVICE_* ConfigMap +keys for one component, "backend" or "frontend". Both read the same +config.telemetry block, so the two halves of the wallet are always on or off +together and send to the same collector. With telemetry off it renders +nothing, and the applications keep their own default, which is off. + +SERVICE_VERSION is the component's image tag, resolved the way the common +chart resolves it, so a trace can be read against the build that produced it. +*/}} +{{- define "testnet-wallet.telemetryData" -}} +{{- $top := index . 0 -}} +{{- $component := index . 1 -}} +{{- $telemetry := $top.Values.config.telemetry -}} +{{- if $telemetry.enabled -}} +{{- $deployment := index $top.Values.deployments $component -}} +{{- $image := $deployment.image | default dict -}} +{{- $imageOverride := $top.Values.imageOverride | default dict -}} +{{- $version := coalesce $telemetry.serviceVersion $image.tag $top.Values.versionOverride $imageOverride.tag $top.Chart.AppVersion -}} +TELEMETRY_ENABLED: "true" +TELEMETRY_ENDPOINT: {{ include "testnet-wallet.telemetryEndpoint" $top | quote }} +TELEMETRY_INSECURE: {{ $telemetry.insecure | toString | quote }} +TELEMETRY_TRACES_ENABLED: {{ $telemetry.traces.enabled | toString | quote }} +TELEMETRY_TRACES_SAMPLE_RATIO: {{ $telemetry.traces.sampleRatio | toString | quote }} +TELEMETRY_METRICS_ENABLED: {{ $telemetry.metrics.enabled | toString | quote }} +TELEMETRY_METRICS_INTERVAL: {{ $telemetry.metrics.interval | quote }} +SERVICE_NAME: {{ index $telemetry.serviceNames $component | quote }} +SERVICE_VERSION: {{ $version | quote }} +SERVICE_ENV: {{ $telemetry.serviceEnv | quote }} +{{- end -}} +{{- end -}} + +{{/* +testnet-wallet.otelCollectorConfig renders the collector's own config file. Set +otelCollector.config to replace it wholesale. The generated pipeline is the one +the merchant and issuer charts use: + + OTLP in (gRPC and HTTP) + -> metrics out on the Prometheus exporter, which Prometheus scrapes + through the ServiceMonitor + -> traces out over OTLP to otelCollector.traces.endpoint + +Three details it does not show on its face: + + * With no traces endpoint the pipeline exports to `nop`. A pipeline must + have an exporter, and dropping the pipeline would unregister the OTLP + trace service, so the wallet would log a failed export on every batch. + + * memory_limiter runs first, shedding load before the batcher can queue + enough to push the heap past the container memory limit. Its percentages + are of that limit, so raise the limit before raising them. + + * resource_to_telemetry_conversion turns resource attributes into labels: + service_name, service_version, service_instance_id, + deployment_environment_name. One collector serves the backend and the + frontend, and every replica of each, so without service_instance_id + replicas emit identical label sets and overwrite each other's samples. +*/}} +{{- define "testnet-wallet.otelCollectorConfig" -}} +{{- if .Values.otelCollector.config -}} +{{- toYaml .Values.otelCollector.config -}} +{{- else -}} +{{- $c := .Values.otelCollector -}} +extensions: + health_check: + endpoint: 0.0.0.0:{{ include "testnet-wallet.otelCollectorContainerPort" (list . "health") }} + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:{{ include "testnet-wallet.otelCollectorContainerPort" (list . "otlp-grpc") }} + http: + endpoint: 0.0.0.0:{{ include "testnet-wallet.otelCollectorContainerPort" (list . "otlp-http") }} + +processors: + memory_limiter: + check_interval: 1s + limit_percentage: {{ $c.memoryLimiter.limitPercentage }} + spike_limit_percentage: {{ $c.memoryLimiter.spikeLimitPercentage }} + batch: + timeout: {{ $c.batch.timeout }} + send_batch_size: {{ $c.batch.sendBatchSize }} + +exporters: + prometheus: + endpoint: 0.0.0.0:{{ include "testnet-wallet.otelCollectorContainerPort" (list . "metrics") }} + resource_to_telemetry_conversion: + enabled: true + metric_expiration: {{ $c.metricExpiration }} +{{- if $c.traces.endpoint }} + otlp/traces: + endpoint: {{ $c.traces.endpoint | quote }} + tls: + insecure: {{ $c.traces.insecure }} +{{- else }} + nop: {} +{{- end }} + +service: + extensions: [health_check] + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: {{ include "testnet-wallet.otelCollectorContainerPort" (list . "telemetry") }} + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [{{ if $c.traces.endpoint }}otlp/traces{{ else }}nop{{ end }}] +{{- end -}} +{{- end -}} diff --git a/helm/testnet-wallet/templates/configMap.backend.yaml b/helm/testnet-wallet/templates/configMap.backend.yaml index 0c9ffe515..24399852a 100644 --- a/helm/testnet-wallet/templates/configMap.backend.yaml +++ b/helm/testnet-wallet/templates/configMap.backend.yaml @@ -7,12 +7,22 @@ entry would emit `ATALLA_HOST: ""` in every environment that leaves it unset and stop the backend from starting. Here the whole block appears only when config.backend.hsm.enabled is true. +The override also adds the TELEMETRY_* and SERVICE_* keys when +config.telemetry.enabled is true. See testnet-wallet.telemetryData. + common.utils.merge merges maps deeply, so these keys join the contentMap's `data` rather than replacing it. */}} {{- define "testnet-wallet.backend.ConfigMap" -}} {{- $top := first . }} {{- $hsm := $top.Values.config.backend.hsm }} +{{- $telemetry := include "testnet-wallet.telemetryData" (list $top "backend") }} +{{- if or $hsm.enabled $telemetry }} +data: +{{- end }} +{{- with $telemetry }} + {{- . | nindent 2 }} +{{- end }} {{- if $hsm.enabled }} {{- if not $hsm.atalla.host }} {{- fail "config.backend.hsm.atalla.host is required when config.backend.hsm.enabled is true" }} @@ -20,7 +30,6 @@ common.utils.merge merges maps deeply, so these keys join the contentMap's {{- if not $hsm.atalla.port }} {{- fail "config.backend.hsm.atalla.port is required when config.backend.hsm.enabled is true" }} {{- end }} -data: HSM_ENABLED: "true" HSM_SERVER_ROLE: {{ $hsm.serverRole | quote }} {{- /* ISSUER_GRPC_HOST and MERCHANT_GRPC_HOST are omitted: the backend falls diff --git a/helm/testnet-wallet/templates/configMap.frontend.yaml b/helm/testnet-wallet/templates/configMap.frontend.yaml index 3079bf729..9ece07be0 100644 --- a/helm/testnet-wallet/templates/configMap.frontend.yaml +++ b/helm/testnet-wallet/templates/configMap.frontend.yaml @@ -1,3 +1,12 @@ +{{/* +The override adds the TELEMETRY_* and SERVICE_* keys when +config.telemetry.enabled is true. See testnet-wallet.telemetryData. +*/}} {{- define "testnet-wallet.frontend.ConfigMap" -}} +{{- $top := first . }} +{{- with include "testnet-wallet.telemetryData" (list $top "frontend") }} +data: + {{- . | nindent 2 }} +{{- end }} {{- end }} {{- include "common.configMapper" (list . .Values.configMaps.frontend "testnet-wallet.frontend.ConfigMap") }} diff --git a/helm/testnet-wallet/templates/configMap.otel-collector.yaml b/helm/testnet-wallet/templates/configMap.otel-collector.yaml new file mode 100644 index 000000000..790038990 --- /dev/null +++ b/helm/testnet-wallet/templates/configMap.otel-collector.yaml @@ -0,0 +1,10 @@ +{{- if .Values.otelCollector.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "testnet-wallet.otelCollectorName" . }} + {{- include "common.metadata" (list . .Values.otelCollector) | nindent 2 }} +data: + config.yaml: | + {{- include "testnet-wallet.otelCollectorConfig" . | nindent 4 }} +{{- end }} diff --git a/helm/testnet-wallet/templates/deployment.otel-collector.yaml b/helm/testnet-wallet/templates/deployment.otel-collector.yaml new file mode 100644 index 000000000..586057970 --- /dev/null +++ b/helm/testnet-wallet/templates/deployment.otel-collector.yaml @@ -0,0 +1,19 @@ +{{- define "testnet-wallet.otelCollector.deployment" -}} +{{- $top := first . }} +{{- $values := $top.Values.otelCollector }} +{{- $configHash := include "testnet-wallet.otelCollectorConfig" $top | sha256sum }} +{{- $builtinVolumes := list + (dict "name" "otel-collector-config" "configMap" (dict "name" (include "testnet-wallet.otelCollectorName" $top))) +}} +spec: + template: + metadata: + annotations: + ilf-otel-collector-config-sha256: {{ $configHash | quote }} + spec: + volumes: + {{- toYaml (concat $builtinVolumes ($values.volumes | default list)) | nindent 8 }} +{{- end }} +{{- if .Values.otelCollector.enabled }} +{{- include "common.deployment" (list . .Values.otelCollector .Values.otelCollector.autoscaling .Values.otelCollector.serviceAccount "testnet-wallet.otelCollector.deployment") }} +{{- end }} diff --git a/helm/testnet-wallet/templates/service.otel-collector.yaml b/helm/testnet-wallet/templates/service.otel-collector.yaml new file mode 100644 index 000000000..010a6f2fb --- /dev/null +++ b/helm/testnet-wallet/templates/service.otel-collector.yaml @@ -0,0 +1,5 @@ +{{- define "testnet-wallet.otelCollector.service" -}} +{{- end }} +{{- if .Values.otelCollector.enabled }} +{{- include "common.svc" (list . .Values.otelCollector.service .Values.otelCollector "testnet-wallet.otelCollector.service") }} +{{- end }} diff --git a/helm/testnet-wallet/templates/serviceMonitor.otel-collector.yaml b/helm/testnet-wallet/templates/serviceMonitor.otel-collector.yaml new file mode 100644 index 000000000..8fac5684e --- /dev/null +++ b/helm/testnet-wallet/templates/serviceMonitor.otel-collector.yaml @@ -0,0 +1,28 @@ +{{- /* +Rendered only where the Prometheus Operator CRDs exist, so the chart still +installs on a cluster without them. Argo CD and `helm install` both pass the +cluster's API versions; `helm template` needs `--api-versions +monitoring.coreos.com/v1/ServiceMonitor`. +*/}} +{{- if and .Values.otelCollector.enabled .Values.otelCollector.serviceMonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "testnet-wallet.otelCollectorName" . }} + {{- include "common.metadata" (list . .Values.otelCollector) | nindent 2 }} +spec: + # Selects the Service, which common.svc labels after otelCollector.service. + selector: + matchLabels: + {{- include "common.selectorLabels" (list . .Values.otelCollector.service) | nindent 6 }} + endpoints: + # The backend's and the frontend's metrics, converted from OTLP. Resource + # attributes arrive as labels, so each series names the pod it came from. + - port: metrics + path: /metrics + interval: {{ .Values.otelCollector.serviceMonitor.interval }} + # The collector's own metrics, where a dropped batch shows up. + - port: telemetry + path: /metrics + interval: {{ .Values.otelCollector.serviceMonitor.interval }} +{{- end }} diff --git a/helm/testnet-wallet/tests/configMap.otel-collector_test.yaml b/helm/testnet-wallet/tests/configMap.otel-collector_test.yaml new file mode 100644 index 000000000..c3eb2bdd6 --- /dev/null +++ b/helm/testnet-wallet/tests/configMap.otel-collector_test.yaml @@ -0,0 +1,195 @@ +suite: testnet-wallet otel-collector - configmap +templates: + - configMap.otel-collector.yaml +set: + fullnameOverride: test-testnet-wallet +release: + name: test-testnet-wallet +tests: + - it: 'a collector ConfigMap is created by default' + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ConfigMap + - equal: + path: metadata.name + value: test-testnet-wallet-otel-collector + + - it: 'no ConfigMap is created when the collector is disabled' + set: + otelCollector: + enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: 'the OTLP receiver listens on both protocols' + asserts: + - matchRegex: + path: data['config.yaml'] + pattern: 'grpc:\n endpoint: 0\.0\.0\.0:4317' + - matchRegex: + path: data['config.yaml'] + pattern: 'http:\n endpoint: 0\.0\.0\.0:4318' + + - it: 'the prometheus exporter converts resource attributes to labels' + asserts: + - matchRegex: + path: data['config.yaml'] + pattern: 'endpoint: 0\.0\.0\.0:8889' + # Without this, every replica emits the same label set and the samples + # overwrite each other. + - matchRegex: + path: data['config.yaml'] + pattern: 'resource_to_telemetry_conversion:\n enabled: true' + + - it: 'with no traces endpoint the traces pipeline exports to nop' + asserts: + - matchRegex: + path: data['config.yaml'] + pattern: 'nop: \{\}' + - matchRegex: + path: data['config.yaml'] + pattern: 'exporters: \[nop\]' + - notMatchRegex: + path: data['config.yaml'] + pattern: 'otlp/traces' + + - it: 'a traces endpoint replaces nop with an OTLP exporter' + set: + otelCollector: + traces: + endpoint: core-tempo.monitoring.svc.cluster.local:4317 + asserts: + - matchRegex: + path: data['config.yaml'] + pattern: 'otlp/traces:\n endpoint: "core-tempo\.monitoring\.svc\.cluster\.local:4317"' + - matchRegex: + path: data['config.yaml'] + pattern: 'exporters: \[otlp/traces\]' + - notMatchRegex: + path: data['config.yaml'] + pattern: 'nop:' + + - it: 'traces TLS can be turned on for a collector that terminates it' + set: + otelCollector: + traces: + endpoint: otel-gateway.example:4317 + insecure: false + asserts: + - matchRegex: + path: data['config.yaml'] + pattern: 'tls:\n insecure: false' + + - it: 'the collector exposes its own metrics for the second scrape endpoint' + asserts: + - matchRegex: + path: data['config.yaml'] + pattern: 'prometheus:\n host: 0\.0\.0\.0\n port: 8888' + + - it: 'otelCollector.config replaces the generated config wholesale' + set: + otelCollector: + # Deliberately nothing like the generated pipeline, so the assertions + # below cannot pass by accident. + config: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:14317 + exporters: + debug: {} + service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] + asserts: + - matchRegex: + path: data['config.yaml'] + pattern: '0\.0\.0\.0:14317' + - notMatchRegex: + path: data['config.yaml'] + pattern: 'memory_limiter' + - notMatchRegex: + path: data['config.yaml'] + pattern: 'health_check' + + - it: 'batch and memory limiter settings are overridable' + set: + otelCollector: + batch: + timeout: 1s + sendBatchSize: 64 + memoryLimiter: + limitPercentage: 70 + spikeLimitPercentage: 10 + metricExpiration: 2m + asserts: + - matchRegex: + path: data['config.yaml'] + pattern: 'timeout: 1s' + - matchRegex: + path: data['config.yaml'] + pattern: 'send_batch_size: 64' + - matchRegex: + path: data['config.yaml'] + pattern: 'limit_percentage: 70' + - matchRegex: + path: data['config.yaml'] + pattern: 'spike_limit_percentage: 10' + - matchRegex: + path: data['config.yaml'] + pattern: 'metric_expiration: 2m' + + - it: 'the listeners follow the container ports, found by name' + set: + otelCollector: + ports: + - name: otlp-grpc + containerPort: 14317 + - name: otlp-http + containerPort: 14318 + - name: metrics + containerPort: 18889 + - name: telemetry + containerPort: 18888 + - name: health + containerPort: 23133 + asserts: + - matchRegex: + path: data['config.yaml'] + pattern: 'grpc:\n endpoint: 0\.0\.0\.0:14317' + - matchRegex: + path: data['config.yaml'] + pattern: 'http:\n endpoint: 0\.0\.0\.0:14318' + - matchRegex: + path: data['config.yaml'] + pattern: 'prometheus:\n endpoint: 0\.0\.0\.0:18889' + - matchRegex: + path: data['config.yaml'] + pattern: 'port: 18888' + - matchRegex: + path: data['config.yaml'] + pattern: 'health_check:\n endpoint: 0\.0\.0\.0:23133' + + - it: 'a container without an otlp-grpc port fails the render' + set: + otelCollector: + ports: + - name: otlp-http + containerPort: 4318 + - name: metrics + containerPort: 8889 + - name: telemetry + containerPort: 8888 + - name: health + containerPort: 13133 + asserts: + - failedTemplate: + errorMessage: >- + otelCollector.ports must contain a port named otlp-grpc — the + collector config listens on it diff --git a/helm/testnet-wallet/tests/deployment.backend_test.yaml b/helm/testnet-wallet/tests/deployment.backend_test.yaml index 844ab0641..8a66dbd35 100644 --- a/helm/testnet-wallet/tests/deployment.backend_test.yaml +++ b/helm/testnet-wallet/tests/deployment.backend_test.yaml @@ -44,21 +44,20 @@ tests: - equal: path: data.PORT value: '4003' - - equal: + - notExists: path: data.METRICS_PORT - value: '9464' - equal: path: data.COOKIE_NAME value: testnet.cookie - - it: 'should expose the metrics port on the container' + - it: 'exposes only the http port; metrics leave through the collector' template: deployment.backend.yaml asserts: + - lengthEqual: + path: spec.template.spec.containers[0].ports + count: 1 - equal: - path: spec.template.spec.containers[0].ports[1].name - value: metrics - - equal: - path: spec.template.spec.containers[0].ports[1].containerPort - value: 9464 + path: spec.template.spec.containers[0].ports[0].name + value: http - it: 'should handle accessSepaKey as a config' template: configMap.backend.yaml set: diff --git a/helm/testnet-wallet/tests/deployment.otel-collector_test.yaml b/helm/testnet-wallet/tests/deployment.otel-collector_test.yaml new file mode 100644 index 000000000..b6f75f2f0 --- /dev/null +++ b/helm/testnet-wallet/tests/deployment.otel-collector_test.yaml @@ -0,0 +1,125 @@ +suite: testnet-wallet otel-collector - deployment +templates: + - deployment.otel-collector.yaml +set: + fullnameOverride: test-testnet-wallet +release: + name: test-testnet-wallet +tests: + - it: 'a collector Deployment is created by default' + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Deployment + - equal: + path: metadata.name + value: test-testnet-wallet-otel-collector + + - it: 'no Deployment is created when the collector is disabled' + set: + otelCollector: + enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: 'the image tag is pinned, not taken from the chart appVersion' + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.162.0 + + - it: 'versionOverride does not reach the collector image' + set: + versionOverride: v1.2.3 + asserts: + - equal: + path: spec.template.spec.containers[0].image + value: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.162.0 + + - it: 'the collector runs its config from the mounted ConfigMap' + asserts: + - equal: + path: spec.template.spec.containers[0].command + value: + - /otelcol-contrib + - equal: + path: spec.template.spec.containers[0].args + value: + - --config=/conf/config.yaml + - contains: + path: spec.template.spec.containers[0].volumeMounts + content: + name: otel-collector-config + mountPath: /conf + readOnly: true + - contains: + path: spec.template.spec.volumes + content: + name: otel-collector-config + configMap: + name: test-testnet-wallet-otel-collector + + - it: 'extra volumes are added alongside the config volume, not instead of it' + set: + otelCollector: + volumes: + - name: scratch + emptyDir: {} + asserts: + - lengthEqual: + path: spec.template.spec.volumes + count: 2 + - contains: + path: spec.template.spec.volumes + content: + name: otel-collector-config + configMap: + name: test-testnet-wallet-otel-collector + + - it: 'every port the config listens on is declared on the container' + asserts: + - contains: + path: spec.template.spec.containers[0].ports + content: + name: otlp-grpc + containerPort: 4317 + protocol: TCP + - contains: + path: spec.template.spec.containers[0].ports + content: + name: metrics + containerPort: 8889 + protocol: TCP + - contains: + path: spec.template.spec.containers[0].ports + content: + name: telemetry + containerPort: 8888 + protocol: TCP + - contains: + path: spec.template.spec.containers[0].ports + content: + name: health + containerPort: 13133 + protocol: TCP + + - it: 'the probes target the health_check extension' + asserts: + - equal: + path: spec.template.spec.containers[0].livenessProbe.httpGet + value: + path: / + port: health + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet + value: + path: / + port: health + + - it: 'the pod template carries a hash of the collector config, so a config change rolls the pods' + asserts: + - matchRegex: + path: spec.template.metadata.annotations['ilf-otel-collector-config-sha256'] + pattern: '^[0-9a-f]{64}$' diff --git a/helm/testnet-wallet/tests/service.otel-collector_test.yaml b/helm/testnet-wallet/tests/service.otel-collector_test.yaml new file mode 100644 index 000000000..4b8b087bc --- /dev/null +++ b/helm/testnet-wallet/tests/service.otel-collector_test.yaml @@ -0,0 +1,63 @@ +suite: testnet-wallet otel-collector - service +templates: + - service.otel-collector.yaml +set: + fullnameOverride: test-testnet-wallet +release: + name: test-testnet-wallet +tests: + - it: 'a collector Service is created by default' + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Service + - equal: + path: metadata.name + value: test-testnet-wallet-otel-collector + - equal: + path: spec.type + value: ClusterIP + + - it: 'no Service is created when the collector is disabled' + set: + otelCollector: + enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: 'the Service selects the collector pods, not the whole release' + asserts: + - equal: + path: spec.selector + value: + app.kubernetes.io/name: test-testnet-wallet-otel-collector + app.kubernetes.io/instance: test-testnet-wallet + + - it: 'the OTLP port the wallet dials is exposed' + asserts: + - contains: + path: spec.ports + content: + name: otlp-grpc + port: 4317 + targetPort: otlp-grpc + protocol: TCP + + - it: 'both scrape ports are exposed for the ServiceMonitor' + asserts: + - contains: + path: spec.ports + content: + name: metrics + port: 8889 + targetPort: metrics + protocol: TCP + - contains: + path: spec.ports + content: + name: telemetry + port: 8888 + targetPort: telemetry + protocol: TCP diff --git a/helm/testnet-wallet/tests/serviceMonitor.otel-collector-no-crd_test.yaml b/helm/testnet-wallet/tests/serviceMonitor.otel-collector-no-crd_test.yaml new file mode 100644 index 000000000..18b4f3014 --- /dev/null +++ b/helm/testnet-wallet/tests/serviceMonitor.otel-collector-no-crd_test.yaml @@ -0,0 +1,13 @@ +suite: testnet-wallet otel-collector - serviceMonitor without the CRDs +templates: + - serviceMonitor.otel-collector.yaml +set: + fullnameOverride: test-testnet-wallet +release: + name: test-testnet-wallet +tests: + # The chart must still install on a cluster without the Prometheus Operator. + - it: 'no ServiceMonitor on a cluster without the Prometheus Operator CRDs' + asserts: + - hasDocuments: + count: 0 diff --git a/helm/testnet-wallet/tests/serviceMonitor.otel-collector_test.yaml b/helm/testnet-wallet/tests/serviceMonitor.otel-collector_test.yaml new file mode 100644 index 000000000..3a9abef9f --- /dev/null +++ b/helm/testnet-wallet/tests/serviceMonitor.otel-collector_test.yaml @@ -0,0 +1,90 @@ +suite: testnet-wallet otel-collector - serviceMonitor +templates: + - serviceMonitor.otel-collector.yaml +set: + fullnameOverride: test-testnet-wallet +release: + name: test-testnet-wallet +# Every test but the last runs against a cluster with the Prometheus Operator +# CRDs. +capabilities: + apiVersions: + - monitoring.coreos.com/v1/ServiceMonitor +tests: + - it: 'a ServiceMonitor is created by default' + asserts: + - hasDocuments: + count: 1 + - isKind: + of: ServiceMonitor + - equal: + path: metadata.name + value: test-testnet-wallet-otel-collector + + - it: 'it scrapes the service metrics and the collector own metrics' + asserts: + - lengthEqual: + path: spec.endpoints + count: 2 + - equal: + path: spec.endpoints[0] + value: + port: metrics + path: /metrics + interval: 30s + - equal: + path: spec.endpoints[1] + value: + port: telemetry + path: /metrics + interval: 30s + + - it: 'it selects the collector pods' + asserts: + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: test-testnet-wallet-otel-collector + app.kubernetes.io/instance: test-testnet-wallet + + - it: 'the scrape interval is overridable on both endpoints' + set: + otelCollector: + serviceMonitor: + interval: 15s + asserts: + - equal: + path: spec.endpoints[0].interval + value: 15s + - equal: + path: spec.endpoints[1].interval + value: 15s + + - it: 'no ServiceMonitor when it is turned off' + set: + otelCollector: + serviceMonitor: + enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: 'no ServiceMonitor when the collector is disabled' + set: + otelCollector: + enabled: false + asserts: + - hasDocuments: + count: 0 + + - it: 'it selects the Service by its own name when that differs' + set: + otelCollector: + service: + name: collector + asserts: + - equal: + path: spec.selector.matchLabels + value: + app.kubernetes.io/name: test-testnet-wallet-collector + app.kubernetes.io/instance: test-testnet-wallet diff --git a/helm/testnet-wallet/tests/services_test.yaml b/helm/testnet-wallet/tests/services_test.yaml index 775cf7b69..4181f9feb 100644 --- a/helm/testnet-wallet/tests/services_test.yaml +++ b/helm/testnet-wallet/tests/services_test.yaml @@ -22,15 +22,9 @@ tests: - equal: path: spec.ports[0].targetPort value: 4003 - - equal: - path: spec.ports[1].name - value: metrics - - equal: - path: spec.ports[1].port - value: 9464 - - equal: - path: spec.ports[1].targetPort - value: 9464 + - lengthEqual: + path: spec.ports + count: 1 - it: 'frontend service should be created with correct name' template: service.frontend.yaml asserts: diff --git a/helm/testnet-wallet/tests/telemetry-no-collector.configMap.backend_test.yaml b/helm/testnet-wallet/tests/telemetry-no-collector.configMap.backend_test.yaml new file mode 100644 index 000000000..d34ea49ef --- /dev/null +++ b/helm/testnet-wallet/tests/telemetry-no-collector.configMap.backend_test.yaml @@ -0,0 +1,41 @@ +# helm-unittest reports a failure against the template that raised it, so +# each template that renders the telemetry keys gets a suite of its own. +suite: testnet-wallet - OpenTelemetry without a collector (configMap.backend) +templates: + - configMap.backend.yaml +set: + fullnameOverride: test-testnet-wallet +release: + name: test-testnet-wallet +tests: + - it: 'enabling telemetry with no collector and no endpoint fails the render' + set: + config: + telemetry: + enabled: true + otelCollector: + enabled: false + asserts: + - failedTemplate: + errorMessage: >- + config.telemetry.enabled is true, but otelCollector.enabled is false + and no config.telemetry.endpoint is set — enable the collector, or + point the endpoint at one outside this release + + - it: 'a collector Service without an otlp-grpc port fails the render' + set: + config: + telemetry: + enabled: true + otelCollector: + service: + ports: + - name: metrics + port: 8889 + targetPort: metrics + protocol: TCP + asserts: + - failedTemplate: + errorMessage: >- + otelCollector.service.ports must contain a port named otlp-grpc — + the backend and the frontend dial the collector on it diff --git a/helm/testnet-wallet/tests/telemetry-no-collector.configMap.frontend_test.yaml b/helm/testnet-wallet/tests/telemetry-no-collector.configMap.frontend_test.yaml new file mode 100644 index 000000000..50c678530 --- /dev/null +++ b/helm/testnet-wallet/tests/telemetry-no-collector.configMap.frontend_test.yaml @@ -0,0 +1,23 @@ +# helm-unittest reports a failure against the template that raised it, so +# each template that renders the telemetry keys gets a suite of its own. +suite: testnet-wallet - OpenTelemetry without a collector (configMap.frontend) +templates: + - configMap.frontend.yaml +set: + fullnameOverride: test-testnet-wallet +release: + name: test-testnet-wallet +tests: + - it: 'enabling telemetry with no collector and no endpoint fails the render' + set: + config: + telemetry: + enabled: true + otelCollector: + enabled: false + asserts: + - failedTemplate: + errorMessage: >- + config.telemetry.enabled is true, but otelCollector.enabled is false + and no config.telemetry.endpoint is set — enable the collector, or + point the endpoint at one outside this release diff --git a/helm/testnet-wallet/tests/telemetry_test.yaml b/helm/testnet-wallet/tests/telemetry_test.yaml new file mode 100644 index 000000000..e61c2b6ad --- /dev/null +++ b/helm/testnet-wallet/tests/telemetry_test.yaml @@ -0,0 +1,312 @@ +# config.telemetry drives both components. The key names are a contract with +# packages/shared/telemetry/src/config.ts. +suite: testnet-wallet - OpenTelemetry wiring +templates: + - configMap.backend.yaml + - configMap.frontend.yaml + - deployment.backend.yaml + - deployment.frontend.yaml +set: + fullnameOverride: test-testnet-wallet +release: + name: test-testnet-wallet +chart: + appVersion: v1.4.0 +tests: + - it: 'telemetry is off by default and writes no keys' + template: configMap.backend.yaml + asserts: + - notExists: + path: data.TELEMETRY_ENABLED + - notExists: + path: data.TELEMETRY_ENDPOINT + - notExists: + path: data.SERVICE_NAME + # The contentMap keys are still there. + - equal: + path: data.NODE_ENV + value: production + + - it: 'telemetry is off by default in the frontend too' + template: configMap.frontend.yaml + asserts: + - notExists: + path: data.TELEMETRY_ENABLED + - equal: + path: data.NODE_ENV + value: production + + - it: 'enabling telemetry points the backend at this release collector' + template: configMap.backend.yaml + set: + config: + telemetry: + enabled: true + asserts: + - equal: + path: data.TELEMETRY_ENABLED + value: 'true' + - equal: + path: data.TELEMETRY_ENDPOINT + value: test-testnet-wallet-otel-collector:4317 + - equal: + path: data.TELEMETRY_INSECURE + value: 'true' + - equal: + path: data.TELEMETRY_TRACES_ENABLED + value: 'true' + # Numbers must reach the ConfigMap as strings. + - equal: + path: data.TELEMETRY_TRACES_SAMPLE_RATIO + value: '1' + - equal: + path: data.TELEMETRY_METRICS_ENABLED + value: 'true' + - equal: + path: data.TELEMETRY_METRICS_INTERVAL + value: 15s + - equal: + path: data.SERVICE_NAME + value: wallet-backend + - equal: + path: data.SERVICE_ENV + value: production + - equal: + path: data.SERVICE_VERSION + value: v1.4.0 + # The override must merge with the contentMap, not replace it. + - equal: + path: data.NODE_ENV + value: production + + - it: 'enabling telemetry points the frontend at the same collector' + template: configMap.frontend.yaml + set: + config: + telemetry: + enabled: true + asserts: + - equal: + path: data.TELEMETRY_ENABLED + value: 'true' + - equal: + path: data.TELEMETRY_ENDPOINT + value: test-testnet-wallet-otel-collector:4317 + - equal: + path: data.SERVICE_NAME + value: wallet-frontend + - equal: + path: data.SERVICE_VERSION + value: v1.4.0 + - equal: + path: data.BACKEND_URL + value: https://api.interledger-test.dev + + - it: 'an explicit endpoint wins over this release collector' + template: configMap.frontend.yaml + set: + config: + telemetry: + enabled: true + endpoint: otel-collector.observability:4317 + asserts: + - equal: + path: data.TELEMETRY_ENDPOINT + value: otel-collector.observability:4317 + + - it: 'moving the collector otlp-grpc port moves the endpoint' + template: configMap.backend.yaml + set: + config: + telemetry: + enabled: true + otelCollector: + service: + ports: + - name: otlp-grpc + port: 14317 + targetPort: otlp-grpc + protocol: TCP + asserts: + - equal: + path: data.TELEMETRY_ENDPOINT + value: test-testnet-wallet-otel-collector:14317 + + - it: 'the endpoint names the collector Service, which may differ from the Deployment' + template: configMap.backend.yaml + set: + config: + telemetry: + enabled: true + otelCollector: + service: + name: collector + asserts: + - equal: + path: data.TELEMETRY_ENDPOINT + value: test-testnet-wallet-collector:4317 + + - it: 'telemetry may be enabled without the collector when an endpoint names one elsewhere' + template: configMap.backend.yaml + set: + config: + telemetry: + enabled: true + endpoint: otel-collector.observability:4317 + otelCollector: + enabled: false + asserts: + - equal: + path: data.TELEMETRY_ENDPOINT + value: otel-collector.observability:4317 + + - it: 'service.version follows the component image tag' + template: configMap.backend.yaml + set: + config: + telemetry: + enabled: true + deployments: + backend: + image: + tag: v2.0.0 + asserts: + - equal: + path: data.SERVICE_VERSION + value: v2.0.0 + + - it: 'service.version follows versionOverride' + template: configMap.frontend.yaml + set: + versionOverride: v3.0.0 + config: + telemetry: + enabled: true + asserts: + - equal: + path: data.SERVICE_VERSION + value: v3.0.0 + + - it: 'an explicit serviceVersion wins over the image tag' + template: configMap.backend.yaml + set: + config: + telemetry: + enabled: true + serviceVersion: v9.9.9 + deployments: + backend: + image: + tag: v2.0.0 + asserts: + - equal: + path: data.SERVICE_VERSION + value: v9.9.9 + + - it: 'the sampling, interval, names and environment are passed through' + template: configMap.backend.yaml + set: + config: + telemetry: + enabled: true + insecure: false + serviceEnv: cards-playground + serviceNames: + backend: wallet-b-backend + traces: + enabled: false + sampleRatio: 0.25 + metrics: + interval: 30s + asserts: + - equal: + path: data.TELEMETRY_INSECURE + value: 'false' + - equal: + path: data.TELEMETRY_TRACES_ENABLED + value: 'false' + - equal: + path: data.TELEMETRY_TRACES_SAMPLE_RATIO + value: '0.25' + - equal: + path: data.TELEMETRY_METRICS_INTERVAL + value: 30s + - equal: + path: data.SERVICE_NAME + value: wallet-b-backend + - equal: + path: data.SERVICE_ENV + value: cards-playground + + - it: 'telemetry and the HSM adapter block land in the same ConfigMap' + template: configMap.backend.yaml + set: + config: + telemetry: + enabled: true + backend: + hsm: + enabled: true + atalla: + host: '10.0.0.1' + port: 13506 + asserts: + - equal: + path: data.TELEMETRY_ENABLED + value: 'true' + - equal: + path: data.HSM_ENABLED + value: 'true' + - equal: + path: data.ATALLA_HOST + value: '10.0.0.1' + - equal: + path: data.NODE_ENV + value: production + + - it: 'the backend pod carries its identity for the OpenTelemetry resource' + template: deployment.backend.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - contains: + path: spec.template.spec.containers[0].env + content: + name: OTEL_RESOURCE_ATTRIBUTES + value: service.instance.id=$(POD_NAME),k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE) + + - it: 'the pod identity survives an environment that empties env' + template: deployment.backend.yaml + set: + deployments: + backend: + env: [] + asserts: + - lengthEqual: + path: spec.template.spec.containers[0].env + count: 3 + - equal: + path: spec.template.spec.containers[0].env[2].name + value: OTEL_RESOURCE_ATTRIBUTES + + - it: 'the frontend pod carries its identity, after the variables it references' + template: deployment.frontend.yaml + asserts: + # Kubernetes expands $(POD_NAME) only from variables defined earlier. + - equal: + path: spec.template.spec.containers[0].env[0].name + value: BACKEND_INTERNAL_URL + - equal: + path: spec.template.spec.containers[0].env[1].name + value: POD_NAME + - equal: + path: spec.template.spec.containers[0].env[2].name + value: POD_NAMESPACE + - equal: + path: spec.template.spec.containers[0].env[3].name + value: OTEL_RESOURCE_ATTRIBUTES diff --git a/helm/testnet-wallet/values.yaml b/helm/testnet-wallet/values.yaml index 41e32d155..16077961d 100644 --- a/helm/testnet-wallet/values.yaml +++ b/helm/testnet-wallet/values.yaml @@ -55,7 +55,6 @@ config: nodeEnv: production logLevel: info port: 4003 - metricsPort: 9464 cookie: name: 'testnet.cookie' @@ -227,6 +226,42 @@ config: clientCertPath: /etc/atalla/tls/client.crt clientKeyPath: /etc/atalla/tls/client.key + # --------------------------------------------------------------------------- + # OpenTelemetry + # --------------------------------------------------------------------------- + # One block for both components. With `enabled: true`, the backend and the + # frontend's Node.js server push traces and metrics over OTLP/gRPC to this + # release's collector (see `otelCollector`). The browser sends nothing. + # + # The chart writes these into each component's ConfigMap as TELEMETRY_* and + # SERVICE_* variables. See packages/shared/telemetry for what each one does. + telemetry: + enabled: false + # The collector's OTLP/gRPC target: host:port, no scheme. Left empty, the + # chart fills in this release's own collector, and fails the render if + # otelCollector.enabled is false. + endpoint: '' + # The in-release collector does not terminate TLS. + insecure: true + # The service.name of each component's traces and metrics. + serviceNames: + backend: wallet-backend + frontend: wallet-frontend + # deployment.environment.name + serviceEnv: production + # service.version. Empty means the component's image tag. + serviceVersion: '' + traces: + enabled: true + # Head sampling, 0 to 1. A trace that arrives already sampled is always + # followed, so lowering this thins only traces that begin here. + sampleRatio: 1.0 + metrics: + enabled: true + # Keep at or below otelCollector.serviceMonitor.interval, or Prometheus + # reads the same values twice. + interval: 15s + deployments: backend: name: backend @@ -337,13 +372,28 @@ deployments: name: '{{ .Values.config.backend.secretName | default (printf "%s-backend" (include "common.fullname" .)) }}' key: '{{ .Values.config.backend.rafiki.adminApiSecret.secretKeyRef.key }}' + # Pod identity for the OpenTelemetry resource. service.instance.id is what + # separates one replica from another: one collector serves them all, so + # without it their metrics carry identical labels and overwrite each + # other. Kubernetes expands the $(...) references; no shell is involved. + # This is `extraEnv`, not `env`, so an environment that rewrites `env` + # keeps it. + extraEnv: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OTEL_RESOURCE_ATTRIBUTES + value: service.instance.id=$(POD_NAME),k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE) + ports: - name: http containerPort: 4003 protocol: TCP - - name: metrics - containerPort: 9464 - protocol: TCP resources: requests: memory: 256Mi @@ -375,6 +425,23 @@ deployments: envFrom: - configMapRef: name: '{{ include "common.fullname" . }}-frontend' + # Pod identity for the OpenTelemetry resource. service.instance.id is what + # separates one replica from another: one collector serves them all, so + # without it their metrics carry identical labels and overwrite each + # other. Kubernetes expands the $(...) references; no shell is involved. + # This is `extraEnv`, not `env`, so an environment that rewrites `env` + # keeps it. + extraEnv: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OTEL_RESOURCE_ATTRIBUTES + value: service.instance.id=$(POD_NAME),k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(POD_NAMESPACE) ports: - name: http containerPort: 4003 @@ -407,10 +474,6 @@ services: targetPort: 4003 protocol: TCP name: http - - port: 9464 - targetPort: 9464 - protocol: TCP - name: metrics frontend: name: frontend-service @@ -423,6 +486,157 @@ services: protocol: TCP name: http +# ----------------------------------------------------------------------------- +# OpenTelemetry collector +# ----------------------------------------------------------------------------- +# One collector per release, the same one the merchant and issuer charts run. +# The backend and the frontend push traces and metrics to it over OTLP. It +# exposes the metrics on a Prometheus endpoint, scraped through the +# ServiceMonitor below, and forwards the traces to `traces.endpoint`. +# +# Scraping the collector means one target holds both signals for both +# components, and each series carries the resource attributes of the pod that +# produced it. +# ----------------------------------------------------------------------------- +otelCollector: + name: otel-collector + # On by default, like the merchant and issuer charts, even while + # config.telemetry.enabled is false: the collector then runs and receives + # nothing. Turn it off only with telemetry off, or with + # config.telemetry.endpoint set to a collector outside this release; the + # render fails otherwise. + enabled: true + # One replica. The collector holds only in-flight batches, and a second + # would split a pod's series across two Prometheus endpoints, which makes a + # rate() over a restart read wrong. + replicaCount: 1 + + # Pinned explicitly. Without a tag the common library falls back to + # versionOverride or the chart's appVersion, which is a wallet release and + # means nothing here. + image: + repository: ghcr.io/open-telemetry/opentelemetry-collector-releases + name: opentelemetry-collector-contrib + tag: '0.162.0' + + # Where traces go after the collector. Empty drops them: the pipeline still + # accepts spans, so the wallet sees a healthy export and nothing is stored. + # + # On ilf-1 this is core-tempo.monitoring.svc.cluster.local:4317, left empty + # here on purpose. A tenant namespace reaches those ports only when its + # deploy-environment sets `network.allowTracingEgress`, a reviewed change in + # clusternation-deploy. An endpoint a namespace cannot reach buys export + # failures, not traces. + traces: + endpoint: '' + insecure: true + + # Replaces the generated collector config wholesale. Everything below feeds + # the generated one and is ignored once this is set. + config: {} + + batch: + timeout: 5s + sendBatchSize: 1024 + + # Percentages of the container memory limit. + memoryLimiter: + limitPercentage: 80 + spikeLimitPercentage: 25 + + # How long the Prometheus exporter keeps a series after the pod that + # produced it stops reporting. Must comfortably exceed the scrape interval. + metricExpiration: 5m + + # The collector config listens on these, found by name: otlp-grpc, + # otlp-http, metrics, telemetry and health. Move a listener here. + ports: + - name: otlp-grpc + containerPort: 4317 + protocol: TCP + - name: otlp-http + containerPort: 4318 + protocol: TCP + # The Prometheus exporter: the backend's and the frontend's metrics. + - name: metrics + containerPort: 8889 + protocol: TCP + # The collector's own metrics. + - name: telemetry + containerPort: 8888 + protocol: TCP + - name: health + containerPort: 13133 + protocol: TCP + + service: + name: otel-collector + type: ClusterIP + # A port named `otlp-grpc` must be here: the chart reads the telemetry + # endpoint off it. + ports: + - name: otlp-grpc + port: 4317 + targetPort: otlp-grpc + protocol: TCP + - name: otlp-http + port: 4318 + targetPort: otlp-http + protocol: TCP + - name: metrics + port: 8889 + targetPort: metrics + protocol: TCP + - name: telemetry + port: 8888 + targetPort: telemetry + protocol: TCP + annotations: {} + + # Scrapes both metrics ports. Rendered only where the Prometheus Operator + # CRDs exist. + serviceMonitor: + enabled: true + interval: 30s + + # The collector's health_check extension answers on /. + livenessProbe: + httpGet: + path: / + port: health + readinessProbe: + httpGet: + path: / + port: health + + command: + - /otelcol-contrib + args: + - --config=/conf/config.yaml + + volumeMounts: + - name: otel-collector-config + mountPath: /conf + readOnly: true + # The deployment template prepends the config volume. + volumes: [] + + # Sized for the whole release's telemetry, not one pod's. memoryLimiter is a + # percentage of the limit set here. + resources: + requests: + memory: 128Mi + cpu: 50m + limits: + memory: 512Mi + cpu: 500m + + serviceAccount: {} + autoscaling: + enabled: false + env: [] + envFrom: [] + # This section of the values file defines Kubernetes config maps that will be created # for each service (backend and frontend). configMaps: @@ -460,8 +674,6 @@ configMaps: valueRef: config.backend.nodeEnv - key: PORT valueRef: config.backend.port - - key: METRICS_PORT - valueRef: config.backend.metricsPort - key: COOKIE_NAME valueRef: config.backend.cookie.name - key: COOKIE_TTL diff --git a/local/config/grafana/dashboards/testnet-wallet.json b/local/config/grafana/dashboards/testnet-wallet.json new file mode 100644 index 000000000..f116d46d6 --- /dev/null +++ b/local/config/grafana/dashboards/testnet-wallet.json @@ -0,0 +1,605 @@ +{ + "uid": "testnet-wallet", + "title": "Testnet Wallet", + "tags": ["testnet", "wallet", "opentelemetry"], + "description": "The wallet backend and frontend server, from the OpenTelemetry metrics the collector exposes.", + "editable": true, + "schemaVersion": 39, + "time": { + "from": "now-30m", + "to": "now" + }, + "refresh": "30s", + "templating": { + "list": [ + { + "name": "service", + "label": "Service", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "query": { + "query": "label_values(target_info, service_name)", + "refId": "service" + }, + "definition": "label_values(target_info, service_name)", + "includeAll": true, + "multi": true, + "allValue": ".*", + "current": { + "selected": true, + "text": ["All"], + "value": ["$__all"] + }, + "refresh": 2, + "sort": 1 + } + ] + }, + "panels": [ + { + "type": "row", + "id": 1, + "title": "Incoming HTTP", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 2, + "title": "Requests per second, by route", + "description": "http.server.request.duration count. The backend labels each series with its Express route; the frontend has no route label, so its requests form one series per status.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (service_name, http_route) (rate(http_server_request_duration_seconds_count{service_name=~\"$service\"}[$__rate_interval]))", + "legendFormat": "{{service_name}} {{http_route}}" + } + ] + }, + { + "type": "timeseries", + "id": 3, + "title": "Latency p95, by route", + "description": "95th percentile of http.server.request.duration.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, service_name, http_route) (rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\"}[$__rate_interval])))", + "legendFormat": "{{service_name}} {{http_route}}" + } + ] + }, + { + "type": "timeseries", + "id": 4, + "title": "Responses per second, by status", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (service_name, http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name=~\"$service\"}[$__rate_interval]))", + "legendFormat": "{{service_name}} {{http_response_status_code}}" + } + ] + }, + { + "type": "timeseries", + "id": 5, + "title": "Server errors (5xx), share of requests", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (service_name) (rate(http_server_request_duration_seconds_count{service_name=~\"$service\", http_response_status_code=~\"5..\"}[$__rate_interval])) / sum by (service_name) (rate(http_server_request_duration_seconds_count{service_name=~\"$service\"}[$__rate_interval]))", + "legendFormat": "{{service_name}}" + } + ] + }, + { + "type": "row", + "id": 6, + "title": "Outgoing HTTP", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 7, + "title": "Latency p95, by destination", + "description": "95th percentile of http.client.request.duration: GateHub, Rafiki, and the frontend's calls to the backend.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, service_name, server_address) (rate(http_client_request_duration_seconds_bucket{service_name=~\"$service\"}[$__rate_interval])))", + "legendFormat": "{{service_name}} \u2192 {{server_address}}" + } + ] + }, + { + "type": "timeseries", + "id": 8, + "title": "Failed requests per second, by destination", + "description": "Requests with an error.type: a status of 400 or more, or a connection failure.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (service_name, server_address, error_type) (rate(http_client_request_duration_seconds_count{service_name=~\"$service\", error_type!=\"\"}[$__rate_interval]))", + "legendFormat": "{{service_name}} \u2192 {{server_address}} {{error_type}}" + } + ] + }, + { + "type": "row", + "id": 9, + "title": "Postgres", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 10, + "title": "Queries per second, by operation", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "sum by (service_name, db_operation_name) (rate(db_client_operation_duration_seconds_count{service_name=~\"$service\"}[$__rate_interval]))", + "legendFormat": "{{service_name}} {{db_operation_name}}" + } + ] + }, + { + "type": "timeseries", + "id": 11, + "title": "Query latency p95, by operation", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le, service_name, db_operation_name) (rate(db_client_operation_duration_seconds_bucket{service_name=~\"$service\"}[$__rate_interval])))", + "legendFormat": "{{service_name}} {{db_operation_name}}" + } + ] + }, + { + "type": "row", + "id": 12, + "title": "Node.js runtime", + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "panels": [] + }, + { + "type": "timeseries", + "id": 13, + "title": "Event loop utilization", + "description": "Share of time the event loop was busy. Near 1 means requests queue behind CPU work.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 36 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "avg by (service_name, service_instance_id) (nodejs_eventloop_utilization_ratio{service_name=~\"$service\"})", + "legendFormat": "{{service_name}} {{service_instance_id}}" + } + ] + }, + { + "type": "timeseries", + "id": 14, + "title": "Event loop delay p99", + "description": "", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 36 + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "fillOpacity": 10, + "showPoints": "never", + "stacking": { + "mode": "none" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": ["mean", "max"] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "max by (service_name, service_instance_id) (nodejs_eventloop_delay_p99_seconds{service_name=~\"$service\"})", + "legendFormat": "{{service_name}} {{service_instance_id}}" + } + ] + } + ] +} diff --git a/local/config/grafana/provisioning/dashboards/dashboards.yaml b/local/config/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 000000000..17594fa82 --- /dev/null +++ b/local/config/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +providers: + - name: wallet + folder: Testnet + type: file + disableDeletion: true + options: + path: /var/lib/grafana/dashboards diff --git a/local/config/grafana/provisioning/datasources/datasources.yaml b/local/config/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 000000000..b289b26c1 --- /dev/null +++ b/local/config/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,26 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + + # The same name and uid as on ilf-1, so a saved query moves between the two. + - name: Tempo + type: tempo + uid: tempo + access: proxy + url: http://tempo:3200 + jsonData: + # From a span to the request metrics of the service that produced it. + tracesToMetrics: + datasourceUid: prometheus + tags: + - key: service.name + value: service_name + queries: + - name: Request rate + query: sum by (http_route) (rate(http_server_request_duration_seconds_count{$$__tags}[5m])) diff --git a/local/config/otel-collector/config.yaml b/local/config/otel-collector/config.yaml new file mode 100644 index 000000000..9f5209df2 --- /dev/null +++ b/local/config/otel-collector/config.yaml @@ -0,0 +1,55 @@ +# The pipeline the testnet-wallet chart generates +# (helm/testnet-wallet/templates/_helpers.tpl, testnet-wallet.otelCollectorConfig), +# with traces sent to the local Tempo. Keep the two in step. +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + memory_limiter: + check_interval: 1s + # Of the container memory limit in observability.yaml, as in the chart. + limit_percentage: 80 + spike_limit_percentage: 25 + batch: + timeout: 5s + send_batch_size: 1024 + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + resource_to_telemetry_conversion: + enabled: true + metric_expiration: 5m + otlp/traces: + endpoint: tempo:4317 + tls: + insecure: true + +service: + extensions: [health_check] + telemetry: + metrics: + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [prometheus] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlp/traces] diff --git a/local/config/prometheus/prometheus.yml b/local/config/prometheus/prometheus.yml new file mode 100644 index 000000000..aef2849c5 --- /dev/null +++ b/local/config/prometheus/prometheus.yml @@ -0,0 +1,14 @@ +# Scrapes the collector's two endpoints, as the chart's ServiceMonitor does on +# the cluster. +global: + scrape_interval: 15s + +scrape_configs: + # The wallet's metrics, converted from OTLP. + - job_name: otel-collector-metrics + static_configs: + - targets: ['otel-collector:8889'] + # The collector's own metrics, where a dropped batch shows up. + - job_name: otel-collector-telemetry + static_configs: + - targets: ['otel-collector:8888'] diff --git a/local/config/tempo/tempo.yaml b/local/config/tempo/tempo.yaml new file mode 100644 index 000000000..11f0bb986 --- /dev/null +++ b/local/config/tempo/tempo.yaml @@ -0,0 +1,28 @@ +# Single binary, OTLP in only: the shape of core-tempo on ilf-1. Traces live in +# the container and go with it. +stream_over_http_enabled: true + +server: + http_listen_port: 3200 + log_level: warn + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +storage: + trace: + backend: local + wal: + path: /tmp/tempo/wal + local: + path: /tmp/tempo/blocks + +compactor: + compaction: + block_retention: 24h diff --git a/local/docker-compose.rhyza.yml b/local/docker-compose.rhyza.yml index 971182dd6..b6c6d387f 100644 --- a/local/docker-compose.rhyza.yml +++ b/local/docker-compose.rhyza.yml @@ -10,6 +10,7 @@ include: - redis.yaml - mockgatehub.yaml - rhyza.yaml + - observability.yaml networks: testnet: diff --git a/local/docker-compose.yml b/local/docker-compose.yml index 0af62b50c..b9147aeff 100644 --- a/local/docker-compose.yml +++ b/local/docker-compose.yml @@ -4,6 +4,7 @@ include: - redis.yaml - mockgatehub.yaml - rafiki.yaml + - observability.yaml networks: testnet: diff --git a/local/observability.yaml b/local/observability.yaml new file mode 100644 index 000000000..b4af5eb2a --- /dev/null +++ b/local/observability.yaml @@ -0,0 +1,94 @@ +# Traces and metrics for the wallet, the same path they take on ilf-1: +# +# wallet backend, wallet frontend (on the host) +# -> otel-collector (OTLP on localhost:4317) +# -> metrics on its Prometheus exporter, scraped by prometheus +# -> traces to tempo +# grafana reads both: https://grafana.testnet.test or http://localhost:3300 +# +# The collector config mirrors the one the testnet-wallet chart generates, so a +# metric or label seen here is the one Prometheus on the cluster gets. +# +# Opt-in, behind the `observability` profile, so `pnpm local:up` does not start +# it. `pnpm dev:observability` (or `dev:rhyza:observability`) starts it with the +# rest of the stack and runs the apps with TELEMETRY_ENABLED=true; +# `pnpm local:up:observability` and `local:down:observability` start and stop +# it on its own. Nothing here keeps data once its containers are removed. +services: + otel-collector: + container_name: otel-collector-local + profiles: [observability] + image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:0.162.0 + command: ['--config=/etc/otelcol/config.yaml'] + volumes: + - ./config/otel-collector/config.yaml:/etc/otelcol/config.yaml:ro + ports: + # The apps run on the host and send here. + - '4317:4317' + - '4318:4318' + # memory_limiter's percentages are of this, as on the cluster. + mem_limit: 512m + depends_on: + - tempo + restart: unless-stopped + networks: + - testnet + + prometheus: + container_name: prometheus-local + profiles: [observability] + image: prom/prometheus:v3.15.0 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.retention.time=2d + volumes: + - ./config/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + ports: + - '9090:9090' + depends_on: + - otel-collector + restart: unless-stopped + networks: + - testnet + + tempo: + container_name: tempo-local + profiles: [observability] + image: grafana/tempo:2.10.8 + command: ['-config.file=/etc/tempo/tempo.yaml'] + volumes: + - ./config/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + restart: unless-stopped + networks: + - testnet + + grafana: + container_name: grafana-local + profiles: [observability] + image: grafana/grafana:13.2.3 + environment: + # Local only: no login, everyone is an admin. + GF_AUTH_ANONYMOUS_ENABLED: 'true' + GF_AUTH_ANONYMOUS_ORG_ROLE: Admin + GF_AUTH_DISABLE_LOGIN_FORM: 'true' + GF_ANALYTICS_REPORTING_ENABLED: 'false' + GF_ANALYTICS_CHECK_FOR_UPDATES: 'false' + GF_NEWS_NEWS_FEED_ENABLED: 'false' + volumes: + - ./config/grafana/provisioning:/etc/grafana/provisioning:ro + - ./config/grafana/dashboards:/var/lib/grafana/dashboards:ro + ports: + - '3300:3000' + depends_on: + - prometheus + - tempo + restart: unless-stopped + networks: + - testnet + labels: + - traefik.enable=true + - traefik.docker.network=testnet + - traefik.http.routers.grafana.rule=Host(`grafana.testnet.test`) + - traefik.http.routers.grafana.entrypoints=websecure + - traefik.http.routers.grafana.tls=true + - traefik.http.services.grafana.loadbalancer.server.port=3000 diff --git a/local/scripts/local-tools.sh b/local/scripts/local-tools.sh index 261711b2a..1764f53af 100644 --- a/local/scripts/local-tools.sh +++ b/local/scripts/local-tools.sh @@ -23,6 +23,7 @@ HOSTS=( "api.boutique.test" "rafiki-card-service.testnet.test" "mockgatehub.testnet.test" + "grafana.testnet.test" # Rhyza stack -- see local/rhyza.yaml "ilp.testnet.test" "connector.testnet.test" @@ -296,6 +297,15 @@ local:reset:rhyza Stop stack and remove volumes (ALL volumes, incl. Pos local:logs:rhyza Follow stack logs local:rhyza-assets Seed assets via the Rhyza REST Admin API local:reset:rhyza-data Drop only Rhyza's Kafka + MongoDB state (keeps Postgres) + +Metrics and traces (opt-in) -- see local/observability.yaml +dev:observability pnpm dev, plus Grafana, Prometheus, Tempo and the collector +dev:rhyza:observability The same on the Rhyza stack +local:up:observability Start only those containers, next to a running stack +local:down:observability Stop and remove them + Grafana: https://grafana.testnet.test (or http://localhost:3300) + The apps send only with TELEMETRY_ENABLED=true; the dev: + commands set it. EOF } diff --git a/package.json b/package.json index 8d61d1846..54b8e9ec4 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "build": "pnpm -r build", "checks": "pnpm prettier:check && pnpm lint:check", "test": "pnpm test:unit", - "test:unit": "pnpm boutique:backend test && pnpm wallet:backend test", + "test:unit": "pnpm boutique:backend test && pnpm --filter @shared/telemetry test && pnpm wallet:backend test", "clean": "pnpm clean:modules && pnpm clean:builds && pnpm local:down && pnpm local:reset && pnpm local:reset:rhyza-data", "clean:builds": "find . \\( -name \"dist\" -o -name \".next\" \\) -type d -prune -exec rm -rf '{}' +", "clean:modules": "find . -name 'node_modules' -type d -prune -exec rm -rf '{}' +", @@ -23,6 +23,8 @@ "dev:debug": "pnpm local:up:debug && pnpm dev:apps", "dev:lite": "pnpm local:up:lite && pnpm dev:apps", "dev:rhyza": "pnpm local:up:rhyza && pnpm dev:apps", + "dev:observability": "pnpm local:up && pnpm local:up:observability && cross-env TELEMETRY_ENABLED=true pnpm dev:apps", + "dev:rhyza:observability": "pnpm local:up:rhyza && pnpm local:up:observability && cross-env TELEMETRY_ENABLED=true pnpm dev:apps", "format": "pnpm prettier:write && pnpm lint:fix", "lint": "pnpm lint:check", "lint:check": "eslint --max-warnings=0 .", @@ -35,9 +37,11 @@ "local:build": "pnpm compose build", "local:rebuild": "pnpm compose build --no-cache", "local:up": "pnpm compose up -d --build --remove-orphans", - "local:down": "pnpm compose down", - "local:reset": "pnpm compose down -v", + "local:down": "pnpm compose --profile observability down", + "local:reset": "pnpm compose --profile observability down -v", "local:logs": "pnpm compose logs -f", + "local:up:observability": "pnpm compose --profile observability up -d otel-collector prometheus tempo grafana", + "local:down:observability": "pnpm compose --profile observability rm --stop --force otel-collector prometheus tempo grafana", "local:certs": "bash ./local/scripts/local-tools.sh certs", "local:hosts": "bash ./local/scripts/local-tools.sh hosts", "local:trust": "bash ./local/scripts/local-tools.sh trust", @@ -46,8 +50,8 @@ "local:trust:linux": "bash ./local/scripts/local-tools.sh trust-linux", "local:rafiki-assets": "node ./local/scripts/rafiki-setup.js", "local:up:rhyza": "pnpm compose:rhyza up -d --remove-orphans", - "local:down:rhyza": "pnpm compose:rhyza down", - "local:reset:rhyza": "pnpm compose:rhyza down -v", + "local:down:rhyza": "pnpm compose:rhyza --profile observability down", + "local:reset:rhyza": "pnpm compose:rhyza --profile observability down -v", "local:logs:rhyza": "pnpm compose:rhyza logs -f", "local:setup:rhyza": "pnpm local:hosts && pnpm local:trust && pnpm local:up:rhyza && pnpm local:rhyza-assets", "local:rhyza-assets": "node ./local/scripts/rhyza-setup.js", diff --git a/packages/shared/backend/package.json b/packages/shared/backend/package.json index fbf958c49..054c880a9 100644 --- a/packages/shared/backend/package.json +++ b/packages/shared/backend/package.json @@ -25,7 +25,6 @@ "typescript": "^5.9.3" }, "dependencies": { - "@prometheus-io/client": "^0.16.1", "awilix": "^12.0.5", "axios": "^1.12.2", "express": "^4.21.2", diff --git a/packages/shared/backend/src/middleware/index.ts b/packages/shared/backend/src/middleware/index.ts index 1624c7720..f31c50f2f 100644 --- a/packages/shared/backend/src/middleware/index.ts +++ b/packages/shared/backend/src/middleware/index.ts @@ -1,2 +1 @@ export * from './errorHandler' -export * from './metrics' diff --git a/packages/shared/backend/src/middleware/metrics.ts b/packages/shared/backend/src/middleware/metrics.ts deleted file mode 100644 index 8245a4e41..000000000 --- a/packages/shared/backend/src/middleware/metrics.ts +++ /dev/null @@ -1,89 +0,0 @@ -import { - Registry, - Histogram, - Gauge, - collectDefaultMetrics -} from '@prometheus-io/client' -import type { Request, Response, NextFunction } from 'express' -import http, { type Server } from 'http' - -export function createMetricsRegistry(): Registry { - const register = new Registry() - collectDefaultMetrics({ register, eventLoopMonitoringPrecision: 5 }) - return register -} - -export function createHttpMetrics(register: Registry) { - const httpRequestDuration = new Histogram({ - name: 'http_request_duration_seconds', - help: 'Duration of HTTP requests in seconds', - labelNames: ['method', 'route', 'status_code'] as const, - buckets: [0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30], - registers: [register] - }) - - const httpRequestsInFlight = new Gauge({ - name: 'http_requests_in_flight', - help: 'Number of in-flight HTTP requests', - labelNames: ['method'] as const, - registers: [register] - }) - - const httpMetricsMiddleware = ( - req: Request, - res: Response, - next: NextFunction - ) => { - const end = httpRequestDuration.startTimer() - httpRequestsInFlight.inc({ method: req.method }) - - // 'finish' doesn't fire if the client disconnects early (aborted request, - // timeout) — 'close' covers that case, but also fires after a normal - // 'finish'. Guard so the gauge/histogram are finalized exactly once. - let finalized = false - const finalize = () => { - if (finalized) return - finalized = true - httpRequestsInFlight.dec({ method: req.method }) - // req.route is unset for router.use('*', ...) catch-alls (used for 404s - // in both apps), so without this fallback every unmatched/scanned path - // would become its own label series (unbounded cardinality). - const route = req.route?.path - ? `${req.baseUrl}${req.route.path}` - : 'unmatched' - end({ method: req.method, route, status_code: String(res.statusCode) }) - } - res.once('finish', finalize) - res.once('close', finalize) - next() - } - - return { httpMetricsMiddleware, httpRequestDuration, httpRequestsInFlight } -} - -// Serves /metrics on its own port, separate from the app's main HTTP port, -// so Prometheus scraping never has to be threaded through cors/helmet/auth -// middleware ordering on the app router. -export function startMetricsServer(register: Registry, port: number): Server { - const server = http.createServer((req, res) => { - if (req.url !== '/metrics') { - res.statusCode = 404 - res.end() - return - } - register - .metrics() - .then((body) => { - res.setHeader('Content-Type', register.contentType) - res.end(body) - }) - .catch((e) => { - res.statusCode = 500 - res.end(e instanceof Error ? e.message : 'Internal Server Error') - }) - }) - server.listen(port) - return server -} - -export type { Registry } from '@prometheus-io/client' diff --git a/packages/shared/backend/tests/middleware/metrics.test.ts b/packages/shared/backend/tests/middleware/metrics.test.ts deleted file mode 100644 index df2223b25..000000000 --- a/packages/shared/backend/tests/middleware/metrics.test.ts +++ /dev/null @@ -1,131 +0,0 @@ -import { Request, Response } from 'express' -import { createRequest, createResponse } from 'node-mocks-http' -import { EventEmitter } from 'events' -import http from 'http' -import { - createMetricsRegistry, - createHttpMetrics, - startMetricsServer -} from '@/middleware/metrics' - -// httpMetricsMiddleware relies on the real 'finish' event; node-mocks-http's -// default response stubs out EventEmitter, so it must be opted back in. -const createMockResponse = () => - createResponse({ eventEmitter: EventEmitter }) - -// server.address() is null until the socket has actually bound (.listen() -// only schedules that async) and can also be a string (unix socket) rather -// than an AddressInfo — wait for 'listening' and check the shape before -// reading .port, instead of assuming it's already there. -function getListeningPort(server: http.Server): Promise { - return new Promise((resolve, reject) => { - server.once('error', reject) - server.once('listening', () => { - const address = server.address() - if (address === null || typeof address === 'string') { - reject(new Error('Expected server to be listening on a port')) - return - } - resolve(address.port) - }) - }) -} - -function get( - port: number, - path: string -): Promise<{ status: number; body: string }> { - return new Promise((resolve, reject) => { - http - .get(`http://localhost:${port}${path}`, (res) => { - let body = '' - res.on('data', (chunk) => (body += chunk)) - res.on('end', () => resolve({ status: res.statusCode ?? 0, body })) - }) - .on('error', reject) - }) -} - -describe('metrics middleware', (): void => { - it('startMetricsServer exposes Prometheus exposition format on /metrics', async (): Promise => { - const register = createMetricsRegistry() - // port 0 lets the OS assign a free ephemeral port - const server = startMetricsServer(register, 0) - const port = await getListeningPort(server) - - try { - const { status, body } = await get(port, '/metrics') - expect(status).toBe(200) - expect(body).toContain('process_cpu_user_seconds_total') - } finally { - server.close() - } - }) - - it('startMetricsServer 404s on any other path', async (): Promise => { - const register = createMetricsRegistry() - const server = startMetricsServer(register, 0) - const port = await getListeningPort(server) - - try { - const { status } = await get(port, '/does-not-exist') - expect(status).toBe(404) - } finally { - server.close() - } - }) - - it('records a request with a route label when req.route is set', async (): Promise => { - const register = createMetricsRegistry() - const { httpMetricsMiddleware, httpRequestDuration } = - createHttpMetrics(register) - const req = createRequest({ - method: 'GET', - route: { path: '/accounts/:id' }, - baseUrl: '' - }) - const res = createMockResponse() - const next = jest.fn() - - httpMetricsMiddleware(req, res, next) - res.statusCode = 200 - res.emit('finish') - - expect(next).toHaveBeenCalled() - const metric = await httpRequestDuration.get() - const sample = metric.values.find((v) => v.metricName?.endsWith('_count')) - expect(sample?.labels).toMatchObject({ - method: 'GET', - route: '/accounts/:id', - status_code: '200' - }) - }) - - it('falls back to an "unmatched" route label when req.route is unset', async (): Promise => { - const register = createMetricsRegistry() - const { httpMetricsMiddleware, httpRequestDuration } = - createHttpMetrics(register) - const req = createRequest({ method: 'GET' }) - const res = createMockResponse() - - httpMetricsMiddleware(req, res, jest.fn()) - res.statusCode = 404 - res.emit('finish') - - const secondReq = createRequest({ method: 'GET' }) - const secondRes = createMockResponse() - httpMetricsMiddleware(secondReq, secondRes, jest.fn()) - secondRes.statusCode = 404 - secondRes.emit('finish') - - const metric = await httpRequestDuration.get() - const unmatchedSamples = metric.values.filter( - (v) => - v.metricName?.endsWith('_count') && - (v.labels as Record).route === 'unmatched' - ) - // Both bogus requests must collapse into a single label series, not two. - expect(unmatchedSamples).toHaveLength(1) - expect(unmatchedSamples[0].value).toBe(2) - }) -}) diff --git a/packages/shared/telemetry/README.md b/packages/shared/telemetry/README.md new file mode 100644 index 000000000..3ba92d4bd --- /dev/null +++ b/packages/shared/telemetry/README.md @@ -0,0 +1,69 @@ +# @shared/telemetry + +This package sets up OpenTelemetry for the wallet backend and the wallet frontend server. Both +components use it, so their traces and metrics have the same settings, resource attributes and +redaction rules. + +## Why it is a separate package + +- The wallet frontend image does not contain `@shared/backend`. +- The entry point of `@shared/backend` loads Express, knex and winston. An instrumentation can + patch a module only before that module loads. This package loads none of those modules, so an + application can start telemetry first. + +## What it does + +`startTelemetry(config, options)` installs the global tracer and meter providers. + +- It sends traces and metrics over OTLP/gRPC to the collector at `TELEMETRY_ENDPOINT`. +- It adds `service.name`, `service.version` and `deployment.environment.name` to every span and + metric. It also reads `OTEL_RESOURCE_ATTRIBUTES`. The Helm chart uses that variable for the pod + identity. +- It always records the Node.js runtime metrics. The application supplies the other + instrumentations, for example HTTP, Express or Postgres. +- It removes secrets from URLs before export. It drops the query string. It also replaces the + path segments in `WALLET_SECRET_PATHS`, for example a password reset token. +- It can stop some spans from starting a trace. The backend uses this for the Postgres queries of + its polling loop. Those queries still count in the Postgres metrics. +- With `TELEMETRY_ENABLED=false` it installs nothing, and the OpenTelemetry API stays a no-op. + +The exporters connect when they first send data. A collector that is not available does not stop +the application. `shutdown()` sends the data that is still in the buffers. + +## Settings + +The application reads these environment variables. The names match the `telemetry:` block of +merchant-api and issuer-api. + +| Variable | Default | Purpose | +| ------------------------------- | ------- | ----------------------------------------------------------- | +| `TELEMETRY_ENABLED` | `false` | Turns traces and metrics on. | +| `TELEMETRY_ENDPOINT` | (empty) | The collector, as `host:port`. Do not use a URL. | +| `TELEMETRY_INSECURE` | `false` | Sends without TLS. The collector in the chart needs `true`. | +| `TELEMETRY_TRACES_ENABLED` | `true` | Turns traces on. | +| `TELEMETRY_TRACES_SAMPLE_RATIO` | `1` | Share of new traces to keep, from 0 to 1. | +| `TELEMETRY_METRICS_ENABLED` | `true` | Turns metrics on. | +| `TELEMETRY_METRICS_INTERVAL` | `15s` | Time between two metric exports. | +| `SERVICE_NAME` | per app | `wallet-backend` or `wallet-frontend`. | +| `SERVICE_VERSION` | (empty) | The image tag. The chart sets it. | +| `SERVICE_ENV` | `dev` | The environment name, for example `cards-playground`. | + +The application checks these values at start-up. A wrong value stops the process with a message +that names the variable. + +## Where it is used + +- `packages/wallet/backend/src/telemetry.ts`. `index.ts` imports this file first. +- `packages/wallet/frontend/src/instrumentation.node.ts`. Next.js runs this file when the server + starts. + +To see the data locally, run `pnpm dev:observability`. See `pnpm local:help`. + +## Tests + +```sh +pnpm --filter @shared/telemetry test +``` + +One test starts an OTLP receiver in the test process. It checks that a span and a metric reach the +collector endpoint. diff --git a/packages/shared/telemetry/jest.config.json b/packages/shared/telemetry/jest.config.json new file mode 100644 index 000000000..30bedf984 --- /dev/null +++ b/packages/shared/telemetry/jest.config.json @@ -0,0 +1,9 @@ +{ + "preset": "ts-jest", + "clearMocks": true, + "testEnvironment": "node", + "modulePathIgnorePatterns": ["/dist/"], + "moduleNameMapper": { + "@/(.*)": "/src/$1" + } +} diff --git a/packages/shared/telemetry/package.json b/packages/shared/telemetry/package.json new file mode 100644 index 000000000..d11cc98fc --- /dev/null +++ b/packages/shared/telemetry/package.json @@ -0,0 +1,39 @@ +{ + "name": "@shared/telemetry", + "version": "1.0.0", + "description": "OpenTelemetry set-up shared by the wallet backend and the wallet frontend server", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist/**/*" + ], + "scripts": { + "build": "tsc --build tsconfig.build.json && tsc-alias -p tsconfig.build.json", + "test": "jest --passWithNoTests --maxWorkers=75%" + }, + "author": "", + "license": "ISC", + "devDependencies": { + "@grpc/grpc-js": "^1.14.5", + "@types/jest": "^29.5.14", + "@types/node": "^24.13.1", + "jest": "^29.7.0", + "ts-jest": "^29.4.4", + "tsc-alias": "^1.8.16", + "typescript": "^5.9.3" + }, + "dependencies": { + "@opentelemetry/api": "^1.9.1", + "@opentelemetry/context-async-hooks": "^2.12.0", + "@opentelemetry/core": "^2.12.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "^0.223.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.223.0", + "@opentelemetry/instrumentation": "^0.223.0", + "@opentelemetry/instrumentation-runtime-node": "^0.36.0", + "@opentelemetry/resources": "^2.12.0", + "@opentelemetry/sdk-metrics": "^2.12.0", + "@opentelemetry/sdk-trace-node": "^2.12.0", + "@opentelemetry/semantic-conventions": "^1.43.0", + "zod": "^3.25.76" + } +} diff --git a/packages/shared/telemetry/src/config.ts b/packages/shared/telemetry/src/config.ts new file mode 100644 index 000000000..9235ddaa2 --- /dev/null +++ b/packages/shared/telemetry/src/config.ts @@ -0,0 +1,177 @@ +import { z } from 'zod' + +// The same settings as the `telemetry:` block of merchant-api and issuer-api, +// read from environment variables. The Helm chart sets them, and fills in +// TELEMETRY_ENDPOINT with the release's own collector when it is left empty. + +// An empty variable means "not set". Without this, z.coerce.number() would +// read TELEMETRY_TRACES_SAMPLE_RATIO= as 0 and silently sample nothing. +const emptyAsUnset = (value: unknown) => (value === '' ? undefined : value) + +const booleanString = (defaultValue: 'true' | 'false') => + z + .enum(['true', 'false']) + .default(defaultValue) + .transform((value) => value === 'true') + +const DURATION = /^(\d+(?:\.\d+)?)(ms|s|m)$/ +const DURATION_UNIT_MS = { ms: 1, s: 1000, m: 60_000 } as const + +// Takes the Go duration subset the charts use: "500ms", "15s", "1m". +export function parseDuration(value: string): number | undefined { + const match = DURATION.exec(value.trim()) + if (!match) return undefined + const unit = match[2] as keyof typeof DURATION_UNIT_MS + return Number(match[1]) * DURATION_UNIT_MS[unit] +} + +export const telemetryEnvShape = { + TELEMETRY_ENABLED: booleanString('false'), + // A gRPC target, `host:port`. Not a URL. + TELEMETRY_ENDPOINT: z.string().trim().default(''), + TELEMETRY_INSECURE: booleanString('false'), + TELEMETRY_TRACES_ENABLED: booleanString('true'), + TELEMETRY_TRACES_SAMPLE_RATIO: z.preprocess( + emptyAsUnset, + z.coerce.number().default(1) + ), + TELEMETRY_METRICS_ENABLED: booleanString('true'), + TELEMETRY_METRICS_INTERVAL: z.string().default('15s'), + SERVICE_NAME: z.string().trim().optional(), + SERVICE_VERSION: z.string().trim().default(''), + SERVICE_ENV: z.string().trim().default('dev') +} + +const telemetryEnvSchema = z.object(telemetryEnvShape) + +export type TelemetryEnv = z.infer + +// A gRPC target here is `host:port` or `[ipv6]:port`. The exporter adds the +// scheme from TELEMETRY_INSECURE. An http(s) URL is the usual mistake, +// because the OTEL_* variables take one. +const URL_SCHEME = /^[a-z][a-z0-9+.-]*:\/\//i +const HOST_PORT = /^(?:\[[0-9a-f:.]+\]|[^\s/:[\]]+):\d{1,5}$/i + +function telemetryIssues( + env: TelemetryEnv +): { path: string; message: string }[] { + // A disabled block is not checked, the same as in merchant-api. + if (!env.TELEMETRY_ENABLED) return [] + + const issues: { path: string; message: string }[] = [] + + if (!env.TELEMETRY_TRACES_ENABLED && !env.TELEMETRY_METRICS_ENABLED) { + issues.push({ + path: 'TELEMETRY_ENABLED', + message: + 'telemetry is enabled, but TELEMETRY_TRACES_ENABLED and TELEMETRY_METRICS_ENABLED are both false' + }) + } + + const endpoint = env.TELEMETRY_ENDPOINT + if (!endpoint) { + issues.push({ + path: 'TELEMETRY_ENDPOINT', + message: 'telemetry is enabled, but TELEMETRY_ENDPOINT is not set' + }) + } else if (URL_SCHEME.test(endpoint)) { + issues.push({ + path: 'TELEMETRY_ENDPOINT', + message: `TELEMETRY_ENDPOINT must be a gRPC target (host:port), not a URL: ${endpoint}` + }) + } else if (!HOST_PORT.test(endpoint)) { + issues.push({ + path: 'TELEMETRY_ENDPOINT', + message: `TELEMETRY_ENDPOINT must be a gRPC target (host:port): ${endpoint}` + }) + } + + const ratio = env.TELEMETRY_TRACES_SAMPLE_RATIO + if (Number.isNaN(ratio) || ratio < 0 || ratio > 1) { + issues.push({ + path: 'TELEMETRY_TRACES_SAMPLE_RATIO', + message: `TELEMETRY_TRACES_SAMPLE_RATIO must be between 0 and 1: ${ratio}` + }) + } + + const interval = parseDuration(env.TELEMETRY_METRICS_INTERVAL) + if (interval === undefined || interval <= 0) { + issues.push({ + path: 'TELEMETRY_METRICS_INTERVAL', + message: `TELEMETRY_METRICS_INTERVAL must be a duration greater than 0, such as 15s: ${env.TELEMETRY_METRICS_INTERVAL}` + }) + } + + return issues +} + +// For a schema that spreads `telemetryEnvShape` into its own object, such as +// the wallet backend's env.ts. Call it from that schema's superRefine. +export function refineTelemetryEnv(env: TelemetryEnv, ctx: z.RefinementCtx) { + for (const issue of telemetryIssues(env)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + path: [issue.path], + message: issue.message + }) + } +} + +export interface TelemetryConfig { + enabled: boolean + // Always `host:port`. + endpoint: string + insecure: boolean + traces: { enabled: boolean; sampleRatio: number } + metrics: { enabled: boolean; intervalMillis: number } + service: { name: string; version: string; env: string } +} + +export interface TelemetryDefaults { + serviceName: string +} + +// Expects an env that has passed refineTelemetryEnv. +export function telemetryConfigFromEnv( + env: TelemetryEnv, + defaults: TelemetryDefaults +): TelemetryConfig { + return { + enabled: env.TELEMETRY_ENABLED, + endpoint: env.TELEMETRY_ENDPOINT, + insecure: env.TELEMETRY_INSECURE, + traces: { + enabled: env.TELEMETRY_TRACES_ENABLED, + sampleRatio: env.TELEMETRY_TRACES_SAMPLE_RATIO + }, + metrics: { + enabled: env.TELEMETRY_METRICS_ENABLED, + intervalMillis: parseDuration(env.TELEMETRY_METRICS_INTERVAL) ?? 15_000 + }, + service: { + name: env.SERVICE_NAME || defaults.serviceName, + version: env.SERVICE_VERSION, + env: env.SERVICE_ENV + } + } +} + +// Parses and checks the telemetry variables on their own. Throws with every +// problem in one message. +export function parseTelemetryEnv( + processEnv: NodeJS.ProcessEnv, + defaults: TelemetryDefaults +): TelemetryConfig { + const result = telemetryEnvSchema + .superRefine(refineTelemetryEnv) + .safeParse(processEnv) + + if (!result.success) { + const problems = result.error.issues + .map((issue) => `${issue.path.join('.')}: ${issue.message}`) + .join('; ') + throw new Error(`invalid telemetry configuration: ${problems}`) + } + + return telemetryConfigFromEnv(result.data, defaults) +} diff --git a/packages/shared/telemetry/src/index.ts b/packages/shared/telemetry/src/index.ts new file mode 100644 index 000000000..874ff5d26 --- /dev/null +++ b/packages/shared/telemetry/src/index.ts @@ -0,0 +1,5 @@ +export * from './config' +export * from './redact' +export * from './sampler' +export * from './secretPaths' +export * from './telemetry' diff --git a/packages/shared/telemetry/src/redact.ts b/packages/shared/telemetry/src/redact.ts new file mode 100644 index 000000000..06927e747 --- /dev/null +++ b/packages/shared/telemetry/src/redact.ts @@ -0,0 +1,61 @@ +import type { Span, SpanProcessor } from '@opentelemetry/sdk-trace-node' + +// A path segment that carries a secret, and what to write in its place. For +// example [/\/reset-password\/[^/:][^/]*/, '/reset-password/{token}']. The +// `[^/:]` leaves a route template's `:token` alone. +export type PathRedaction = [pattern: RegExp, replacement: string] + +export const REDACTED = 'REDACTED' + +// The URL attributes the HTTP, undici and Next.js instrumentations write, +// under the stable and the older semantic conventions. Next.js also copies +// the raw request line, query included, into next.span_name. +const URL_ATTRIBUTES = [ + 'url.full', + 'url.path', + 'url.original', + 'http.target', + 'http.url', + 'next.span_name' +] as const + +// Drops the query string's content and rewrites each secret path segment. The +// query goes as a whole: the wallet passes interaction ids and nonces there, +// and a list of known keys would miss the next one. +export function redactUrl(value: string, paths: PathRedaction[]): string { + const queryOrFragment = value.search(/[?#]/) + let path = queryOrFragment === -1 ? value : value.slice(0, queryOrFragment) + for (const [pattern, replacement] of paths) { + path = path.replace(pattern, replacement) + } + return value[queryOrFragment] === '?' ? `${path}?${REDACTED}` : path +} + +// Rewrites the URL attributes and the name of every span as it ends, before +// any exporter sees it. Must come before the exporting span processor. +export class UrlRedactingSpanProcessor implements SpanProcessor { + constructor(private readonly paths: PathRedaction[] = []) {} + + onStart(): void {} + + onEnding(span: Span): void { + for (const key of URL_ATTRIBUTES) { + const value = span.attributes[key] + if (typeof value === 'string') { + span.setAttribute(key, redactUrl(value, this.paths)) + } + } + if (typeof span.attributes['url.query'] === 'string') { + span.setAttribute('url.query', REDACTED) + } + + const name = redactUrl(span.name, this.paths) + if (name !== span.name) span.updateName(name) + } + + onEnd(): void {} + + async forceFlush(): Promise {} + + async shutdown(): Promise {} +} diff --git a/packages/shared/telemetry/src/sampler.ts b/packages/shared/telemetry/src/sampler.ts new file mode 100644 index 000000000..4b7d75c27 --- /dev/null +++ b/packages/shared/telemetry/src/sampler.ts @@ -0,0 +1,49 @@ +import type { Attributes, Context, Link, SpanKind } from '@opentelemetry/api' +import { + type Sampler, + SamplingDecision, + type SamplingResult +} from '@opentelemetry/sdk-trace-node' + +// Decides, by span name, which spans may not start a trace of their own. +export type DropRootSpan = (spanName: string) => boolean + +// The root sampler: drops a span that would start a trace when `drop` says +// so, and leaves every other decision to `delegate`. Wrapped in +// ParentBasedSampler it only sees spans without a parent, so the same span +// inside a request is kept. +// +// Sampling decides only what is traced. Instrumentations record their +// metrics either way, so a dropped query still counts in +// db.client.operation.duration. +export class DropRootSpanSampler implements Sampler { + constructor( + private readonly drop: DropRootSpan, + private readonly delegate: Sampler + ) {} + + shouldSample( + context: Context, + traceId: string, + spanName: string, + spanKind: SpanKind, + attributes: Attributes, + links: Link[] + ): SamplingResult { + if (this.drop(spanName)) { + return { decision: SamplingDecision.NOT_RECORD } + } + return this.delegate.shouldSample( + context, + traceId, + spanName, + spanKind, + attributes, + links + ) + } + + toString(): string { + return `DropRootSpanSampler{${this.delegate.toString()}}` + } +} diff --git a/packages/shared/telemetry/src/secretPaths.ts b/packages/shared/telemetry/src/secretPaths.ts new file mode 100644 index 000000000..2bf8c3dc0 --- /dev/null +++ b/packages/shared/telemetry/src/secretPaths.ts @@ -0,0 +1,23 @@ +import type { PathRedaction } from './redact' + +// Wallet URLs whose path carries a secret. Both components pass this list: the +// frontend serves the pages and calls the backend endpoints, so its spans hold +// both kinds. Add a route here when it puts a token in the path. +// +// `[^/:[]` leaves a route template alone: Express's `:token` and Next.js's +// `[token]` are no secret, and they name the span. +const SEGMENT = '[^/:[][^/]*' + +export const WALLET_SECRET_PATHS: PathRedaction[] = [ + // Backend: password reset and e-mail verification links. + [new RegExp(`/reset-password/${SEGMENT}`), '/reset-password/{token}'], + [new RegExp(`/verify-email/${SEGMENT}`), '/verify-email/{token}'], + // Backend: an Open Payments grant interaction and its nonce. + [ + new RegExp(`/grant-interactions/${SEGMENT}/${SEGMENT}`), + '/grant-interactions/{interactionId}/{nonce}' + ], + // Frontend: the pages those links open. + [new RegExp(`/auth/reset/${SEGMENT}`), '/auth/reset/{token}'], + [new RegExp(`/auth/verify/${SEGMENT}`), '/auth/verify/{token}'] +] diff --git a/packages/shared/telemetry/src/telemetry.ts b/packages/shared/telemetry/src/telemetry.ts new file mode 100644 index 000000000..6d729c5e0 --- /dev/null +++ b/packages/shared/telemetry/src/telemetry.ts @@ -0,0 +1,240 @@ +import { + context, + diag, + DiagLogLevel, + metrics, + propagation, + trace +} from '@opentelemetry/api' +import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks' +import { + CompositePropagator, + W3CBaggagePropagator, + W3CTraceContextPropagator +} from '@opentelemetry/core' +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-grpc' +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc' +import { + type Instrumentation, + registerInstrumentations +} from '@opentelemetry/instrumentation' +import { RuntimeNodeInstrumentation } from '@opentelemetry/instrumentation-runtime-node' +import { + defaultResource, + detectResources, + envDetector, + type Resource, + resourceFromAttributes +} from '@opentelemetry/resources' +import { + MeterProvider, + PeriodicExportingMetricReader +} from '@opentelemetry/sdk-metrics' +import { + BatchSpanProcessor, + NodeTracerProvider, + ParentBasedSampler, + TraceIdRatioBasedSampler +} from '@opentelemetry/sdk-trace-node' +import { + ATTR_SERVICE_NAME, + ATTR_SERVICE_VERSION +} from '@opentelemetry/semantic-conventions' +import type { TelemetryConfig } from './config' +import { type PathRedaction, UrlRedactingSpanProcessor } from './redact' +import { type DropRootSpan, DropRootSpanSampler } from './sampler' + +// Not yet in the stable semantic conventions export. +const ATTR_DEPLOYMENT_ENVIRONMENT_NAME = 'deployment.environment.name' + +// The OTLP exporters give up on one export after this long. The metric reader +// needs a timeout no longer than its interval. +const EXPORT_TIMEOUT_MILLIS = 10_000 + +export interface TelemetryLogger { + info(message: string): void + warn(message: string): void +} + +export interface TelemetryOptions { + // Called only when telemetry is enabled. An instrumentation patches its + // module when it is constructed, so with telemetry off nothing is patched. + instrumentations?: () => Instrumentation[] + // The application logger usually does not exist yet, because winston must + // load after its instrumentation. Defaults to the console. + logger?: TelemetryLogger + // Path segments that carry a secret, such as a password reset token. Every + // span's URL attributes are rewritten with these before export, and the + // query string is always dropped. See redact.ts. + redactUrlPaths?: PathRedaction[] + // Spans that may not start a trace, by name: work outside any request, such + // as a polling loop's queries. They are still traced inside a request, and + // their metrics are still recorded. See sampler.ts. + dropRootSpan?: DropRootSpan +} + +export interface Telemetry { + // Flushes and stops every pipeline. Safe to call more than once. + shutdown(): Promise +} + +const noopTelemetry: Telemetry = { shutdown: async () => {} } + +export function buildResource(config: TelemetryConfig): Resource { + const attributes: Record = { + [ATTR_SERVICE_NAME]: config.service.name, + [ATTR_DEPLOYMENT_ENVIRONMENT_NAME]: config.service.env + } + if (config.service.version) { + attributes[ATTR_SERVICE_VERSION] = config.service.version + } + + // OTEL_RESOURCE_ATTRIBUTES is read, so the chart can add the pod identity. + // The configured service attributes win over it. + return defaultResource() + .merge(detectResources({ detectors: [envDetector] })) + .merge(resourceFromAttributes(attributes)) +} + +// The gRPC exporters pick their credentials from the URL scheme. +function exporterUrl(config: TelemetryConfig): string { + return `${config.insecure ? 'http' : 'https'}://${config.endpoint}` +} + +// Installs the global tracer and meter providers. With telemetry disabled it +// installs nothing, and the OpenTelemetry API stays a no-op. The exporters +// connect lazily, so a collector that is down does not stop the service. +export function startTelemetry( + config: TelemetryConfig, + options: TelemetryOptions = {} +): Telemetry { + const logger = options.logger ?? console + + if (!config.enabled) { + logger.info('telemetry is disabled') + return noopTelemetry + } + + // Export failures are logged, and do not stop the service. + diag.setLogger( + { + error: (message, ...args) => + logger.warn(`telemetry: ${message} ${args.join(' ')}`.trim()), + warn: (message, ...args) => + logger.warn(`telemetry: ${message} ${args.join(' ')}`.trim()), + info: () => {}, + debug: () => {}, + verbose: () => {} + }, + DiagLogLevel.WARN + ) + + const contextManager = new AsyncLocalStorageContextManager().enable() + context.setGlobalContextManager(contextManager) + propagation.setGlobalPropagator( + new CompositePropagator({ + propagators: [new W3CTraceContextPropagator(), new W3CBaggagePropagator()] + }) + ) + + const resource = buildResource(config) + const url = exporterUrl(config) + const shutdowns: (() => Promise)[] = [ + async () => { + context.disable() + propagation.disable() + diag.disable() + } + ] + + let tracerProvider: NodeTracerProvider | undefined + if (config.traces.enabled) { + tracerProvider = new NodeTracerProvider({ + resource, + sampler: new ParentBasedSampler({ + root: options.dropRootSpan + ? new DropRootSpanSampler( + options.dropRootSpan, + new TraceIdRatioBasedSampler(config.traces.sampleRatio) + ) + : new TraceIdRatioBasedSampler(config.traces.sampleRatio) + }), + spanProcessors: [ + // Must come first, so the exporter only sees redacted spans. + new UrlRedactingSpanProcessor(options.redactUrlPaths), + new BatchSpanProcessor( + new OTLPTraceExporter({ url, timeoutMillis: EXPORT_TIMEOUT_MILLIS }) + ) + ] + }) + trace.setGlobalTracerProvider(tracerProvider) + const provider = tracerProvider + shutdowns.push(async () => { + await provider.shutdown() + trace.disable() + }) + } + + let meterProvider: MeterProvider | undefined + if (config.metrics.enabled) { + meterProvider = new MeterProvider({ + resource, + readers: [ + new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ + url, + timeoutMillis: EXPORT_TIMEOUT_MILLIS + }), + exportIntervalMillis: config.metrics.intervalMillis, + exportTimeoutMillis: Math.min( + config.metrics.intervalMillis, + EXPORT_TIMEOUT_MILLIS + ) + }) + ] + }) + metrics.setGlobalMeterProvider(meterProvider) + const provider = meterProvider + shutdowns.push(async () => { + await provider.shutdown() + metrics.disable() + }) + } + + const instrumentations = [ + ...(config.metrics.enabled ? [new RuntimeNodeInstrumentation()] : []), + ...(options.instrumentations?.() ?? []) + ] + const unregister = registerInstrumentations({ + instrumentations, + tracerProvider, + meterProvider + }) + shutdowns.push(async () => unregister()) + + logger.info( + `telemetry is enabled: endpoint=${config.endpoint} traces=${config.traces.enabled} metrics=${config.metrics.enabled}` + ) + + let done: Promise | undefined + return { + shutdown: () => { + // Stop the instrumentations first, the providers next, and the globals + // last. Every step runs, even when an earlier one fails. + done ??= (async () => { + const errors: unknown[] = [] + for (const stop of [...shutdowns].reverse()) { + try { + await stop() + } catch (error) { + errors.push(error) + } + } + if (errors.length > 0) { + throw new AggregateError(errors, 'telemetry shutdown failed') + } + })() + return done + } + } +} diff --git a/packages/shared/telemetry/tests/config.test.ts b/packages/shared/telemetry/tests/config.test.ts new file mode 100644 index 000000000..cabe56224 --- /dev/null +++ b/packages/shared/telemetry/tests/config.test.ts @@ -0,0 +1,184 @@ +import { parseDuration, parseTelemetryEnv } from '@/config' + +const defaults = { serviceName: 'wallet-test' } + +const enabled = { + TELEMETRY_ENABLED: 'true', + TELEMETRY_ENDPOINT: 'wallet-otel-collector:4317', + TELEMETRY_INSECURE: 'true' +} + +describe('parseTelemetryEnv', (): void => { + it('is disabled by default, with the merchant-api defaults', (): void => { + expect(parseTelemetryEnv({}, defaults)).toEqual({ + enabled: false, + endpoint: '', + insecure: false, + traces: { enabled: true, sampleRatio: 1 }, + metrics: { enabled: true, intervalMillis: 15_000 }, + service: { name: 'wallet-test', version: '', env: 'dev' } + }) + }) + + it('accepts an enabled block with an endpoint', (): void => { + const config = parseTelemetryEnv( + { + ...enabled, + TELEMETRY_TRACES_SAMPLE_RATIO: '0.25', + TELEMETRY_METRICS_INTERVAL: '30s', + SERVICE_NAME: 'wallet-backend', + SERVICE_VERSION: 'v1.3.0', + SERVICE_ENV: 'cards-playground' + }, + defaults + ) + + expect(config).toEqual({ + enabled: true, + endpoint: 'wallet-otel-collector:4317', + insecure: true, + traces: { enabled: true, sampleRatio: 0.25 }, + metrics: { enabled: true, intervalMillis: 30_000 }, + service: { + name: 'wallet-backend', + version: 'v1.3.0', + env: 'cards-playground' + } + }) + }) + + it('ignores a disabled block, even with invalid values', (): void => { + expect(() => + parseTelemetryEnv( + { + TELEMETRY_ENABLED: 'false', + TELEMETRY_ENDPOINT: 'http://not-checked:4317', + TELEMETRY_TRACES_SAMPLE_RATIO: '7', + TELEMETRY_METRICS_INTERVAL: '0s', + TELEMETRY_TRACES_ENABLED: 'false', + TELEMETRY_METRICS_ENABLED: 'false' + }, + defaults + ) + ).not.toThrow() + }) + + it('rejects an enabled block without an endpoint', (): void => { + expect(() => + parseTelemetryEnv({ TELEMETRY_ENABLED: 'true' }, defaults) + ).toThrow('TELEMETRY_ENDPOINT is not set') + }) + + it.each([ + 'http://wallet-otel-collector:4317', + 'https://wallet-otel-collector:4317', + 'dns:///wallet-otel-collector:4317' + ])('rejects an endpoint that is a URL: %s', (endpoint): void => { + expect(() => + parseTelemetryEnv({ ...enabled, TELEMETRY_ENDPOINT: endpoint }, defaults) + ).toThrow('not a URL') + }) + + it.each(['wallet-otel-collector', 'wallet-otel-collector:port', ':4317'])( + 'rejects an endpoint without a port: %s', + (endpoint): void => { + expect(() => + parseTelemetryEnv( + { ...enabled, TELEMETRY_ENDPOINT: endpoint }, + defaults + ) + ).toThrow('must be a gRPC target') + } + ) + + it.each([ + 'localhost:4317', + '10.0.0.7:4317', + '[::1]:4317', + 'wallet-otel-collector.depenv-cards-playground.svc.cluster.local:4317' + ])('accepts the gRPC target %s', (endpoint): void => { + expect( + parseTelemetryEnv({ ...enabled, TELEMETRY_ENDPOINT: endpoint }, defaults) + .endpoint + ).toBe(endpoint) + }) + + it.each(['-0.1', '1.5', 'half'])( + 'rejects the sample ratio %s', + (ratio): void => { + expect(() => + parseTelemetryEnv( + { ...enabled, TELEMETRY_TRACES_SAMPLE_RATIO: ratio }, + defaults + ) + ).toThrow('TELEMETRY_TRACES_SAMPLE_RATIO') + } + ) + + it.each(['0s', '15', 'soon', '-1s'])( + 'rejects the metrics interval %s', + (interval): void => { + expect(() => + parseTelemetryEnv( + { ...enabled, TELEMETRY_METRICS_INTERVAL: interval }, + defaults + ) + ).toThrow('TELEMETRY_METRICS_INTERVAL') + } + ) + + it('rejects an enabled block with neither signal', (): void => { + expect(() => + parseTelemetryEnv( + { + ...enabled, + TELEMETRY_TRACES_ENABLED: 'false', + TELEMETRY_METRICS_ENABLED: 'false' + }, + defaults + ) + ).toThrow('both false') + }) + + it('names every problem in one error', (): void => { + expect(() => + parseTelemetryEnv( + { + TELEMETRY_ENABLED: 'true', + TELEMETRY_TRACES_SAMPLE_RATIO: '2' + }, + defaults + ) + ).toThrow(/TELEMETRY_ENDPOINT.*TELEMETRY_TRACES_SAMPLE_RATIO/) + }) + + it('reads an empty sample ratio as unset, not as 0', (): void => { + expect( + parseTelemetryEnv( + { ...enabled, TELEMETRY_TRACES_SAMPLE_RATIO: '' }, + defaults + ).traces.sampleRatio + ).toBe(1) + }) + + it('rejects a boolean that is not "true" or "false"', (): void => { + expect(() => + parseTelemetryEnv({ TELEMETRY_ENABLED: 'yes' }, defaults) + ).toThrow('TELEMETRY_ENABLED') + }) +}) + +describe('parseDuration', (): void => { + it.each([ + ['500ms', 500], + ['15s', 15_000], + ['1.5s', 1_500], + ['2m', 120_000] + ])('reads %s as %d ms', (value, millis): void => { + expect(parseDuration(value)).toBe(millis) + }) + + it.each(['', '15', '1h', 's'])('does not read %p', (value): void => { + expect(parseDuration(value)).toBeUndefined() + }) +}) diff --git a/packages/shared/telemetry/tests/redact.test.ts b/packages/shared/telemetry/tests/redact.test.ts new file mode 100644 index 000000000..8d8eb0087 --- /dev/null +++ b/packages/shared/telemetry/tests/redact.test.ts @@ -0,0 +1,147 @@ +import { + InMemorySpanExporter, + NodeTracerProvider, + SimpleSpanProcessor +} from '@opentelemetry/sdk-trace-node' +import { + type PathRedaction, + REDACTED, + redactUrl, + UrlRedactingSpanProcessor +} from '@/redact' +import { WALLET_SECRET_PATHS } from '@/secretPaths' + +// `[^/:]` skips a route template's `:token`, which is no secret. +const paths: PathRedaction[] = [ + [/\/reset-password\/[^/:][^/]*/, '/reset-password/{token}'], + [ + /\/grant-interactions\/[^/:][^/]*\/[^/:][^/]*/, + '/grant-interactions/{interactionId}/{nonce}' + ] +] + +describe('redactUrl', (): void => { + it.each([ + ['/reset-password/abc123', '/reset-password/{token}'], + ['/reset-password/abc123/validate', '/reset-password/{token}/validate'], + [ + 'http://wallet-backend:4003/reset-password/abc123/validate', + 'http://wallet-backend:4003/reset-password/{token}/validate' + ], + [ + '/grant-interactions/int-1/nonce-2/', + '/grant-interactions/{interactionId}/{nonce}/' + ], + // The query string's content always goes. + [ + '/grant-interactions?interactId=int-1&nonce=nonce-2', + `/grant-interactions?${REDACTED}` + ], + ['/reset-password/abc123?x=1#y', `/reset-password/{token}?${REDACTED}`], + // A fragment is dropped; it never reaches a server anyway. + ['/accounts#top', '/accounts'], + ['/accounts/abc', '/accounts/abc'], + ['GET /reset-password/:token', 'GET /reset-password/:token'] + ])('rewrites %s as %s', (value, expected): void => { + expect(redactUrl(value, paths)).toBe(expected) + }) +}) + +describe('UrlRedactingSpanProcessor', (): void => { + const exporter = new InMemorySpanExporter() + const provider = new NodeTracerProvider({ + spanProcessors: [ + new UrlRedactingSpanProcessor(paths), + new SimpleSpanProcessor(exporter) + ] + }) + const tracer = provider.getTracer('test') + + afterEach((): void => exporter.reset()) + afterAll(async (): Promise => provider.shutdown()) + + it('rewrites the URL attributes before the exporter sees them', (): void => { + tracer + .startSpan('GET', { + attributes: { + 'url.path': '/reset-password/abc123/validate', + 'url.query': 'interactId=int-1', + 'url.full': + 'http://wallet-backend:4003/reset-password/abc123/validate?a=b', + 'http.target': '/reset-password/abc123', + 'http.route': '/reset-password/:token/validate', + 'http.request.method': 'GET' + } + }) + .end() + + const [span] = exporter.getFinishedSpans() + expect(span.attributes).toEqual({ + 'url.path': '/reset-password/{token}/validate', + 'url.query': REDACTED, + 'url.full': `http://wallet-backend:4003/reset-password/{token}/validate?${REDACTED}`, + 'http.target': '/reset-password/{token}', + 'http.route': '/reset-password/:token/validate', + 'http.request.method': 'GET' + }) + }) + + it('rewrites the raw request line Next.js copies into next.span_name', (): void => { + tracer + .startSpan('GET /grant-interactions', { + attributes: { + 'next.span_name': 'GET /grant-interactions?interactId=int-1&nonce=n' + } + }) + .end() + + expect(exporter.getFinishedSpans()[0].attributes['next.span_name']).toBe( + `GET /grant-interactions?${REDACTED}` + ) + }) + + it('rewrites a span name that holds a raw path', (): void => { + tracer.startSpan('GET /reset-password/abc123').end() + + expect(exporter.getFinishedSpans()[0].name).toBe( + 'GET /reset-password/{token}' + ) + }) + + it('leaves a span without URL attributes alone', (): void => { + tracer + .startSpan('pg.query:SELECT wallet_backend', { + attributes: { 'db.query.text': 'select * from users where id = $1' } + }) + .end() + + const [span] = exporter.getFinishedSpans() + expect(span.name).toBe('pg.query:SELECT wallet_backend') + expect(span.attributes).toEqual({ + 'db.query.text': 'select * from users where id = $1' + }) + }) +}) + +describe('WALLET_SECRET_PATHS', (): void => { + it.each([ + ['/reset-password/abc123/validate', '/reset-password/{token}/validate'], + ['/verify-email/abc123', '/verify-email/{token}'], + [ + '/grant-interactions/int-1/nonce-2/', + '/grant-interactions/{interactionId}/{nonce}/' + ], + ['/auth/reset/abc123', '/auth/reset/{token}'], + ['/auth/verify/abc123', '/auth/verify/{token}'], + [ + 'http://wallet-backend:4003/verify-email/abc123', + 'http://wallet-backend:4003/verify-email/{token}' + ], + // Route templates name spans and carry no secret. + ['POST /reset-password/:token', 'POST /reset-password/:token'], + ['GET /auth/reset/[token]', 'GET /auth/reset/[token]'], + ['/accounts/abc/wallet-addresses', '/accounts/abc/wallet-addresses'] + ])('rewrites %s as %s', (value, expected): void => { + expect(redactUrl(value, WALLET_SECRET_PATHS)).toBe(expected) + }) +}) diff --git a/packages/shared/telemetry/tests/sampler.test.ts b/packages/shared/telemetry/tests/sampler.test.ts new file mode 100644 index 000000000..29d3aacca --- /dev/null +++ b/packages/shared/telemetry/tests/sampler.test.ts @@ -0,0 +1,68 @@ +import { context, trace } from '@opentelemetry/api' +import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks' +import { + AlwaysOnSampler, + InMemorySpanExporter, + NodeTracerProvider, + ParentBasedSampler, + SimpleSpanProcessor +} from '@opentelemetry/sdk-trace-node' +import { DropRootSpanSampler } from '@/sampler' + +describe('DropRootSpanSampler', (): void => { + const exporter = new InMemorySpanExporter() + const provider = new NodeTracerProvider({ + sampler: new ParentBasedSampler({ + root: new DropRootSpanSampler( + (name) => name.startsWith('pg.'), + new AlwaysOnSampler() + ) + }), + spanProcessors: [new SimpleSpanProcessor(exporter)] + }) + const tracer = provider.getTracer('test') + const contextManager = new AsyncLocalStorageContextManager().enable() + + beforeAll((): void => { + context.setGlobalContextManager(contextManager) + }) + afterEach((): void => exporter.reset()) + afterAll(async (): Promise => { + context.disable() + await provider.shutdown() + }) + + it('drops a matching span that would start a trace', (): void => { + const span = tracer.startSpan('pg.query:SELECT wallet_backend') + expect(span.isRecording()).toBe(false) + span.end() + + expect(exporter.getFinishedSpans()).toHaveLength(0) + }) + + it('keeps the same span inside a request', (): void => { + tracer.startActiveSpan('GET /accounts/:id', (request) => { + tracer.startSpan('pg.query:SELECT wallet_backend').end() + request.end() + }) + + expect(exporter.getFinishedSpans().map((span) => span.name)).toEqual([ + 'pg.query:SELECT wallet_backend', + 'GET /accounts/:id' + ]) + }) + + it('leaves other root spans to the delegate', (): void => { + tracer.startSpan('GET /me').end() + + expect(exporter.getFinishedSpans()).toHaveLength(1) + }) + + it('drops the children of a dropped root as well', (): void => { + const dropped = tracer.startSpan('pg.connect') + + context.with(trace.setSpan(context.active(), dropped), () => { + expect(tracer.startSpan('pg.query:SELECT').isRecording()).toBe(false) + }) + }) +}) diff --git a/packages/shared/telemetry/tests/telemetry.test.ts b/packages/shared/telemetry/tests/telemetry.test.ts new file mode 100644 index 000000000..76a4ba181 --- /dev/null +++ b/packages/shared/telemetry/tests/telemetry.test.ts @@ -0,0 +1,242 @@ +import { metrics, trace } from '@opentelemetry/api' +import { + Server, + ServerCredentials, + type UntypedServiceImplementation +} from '@grpc/grpc-js' +import { ATTR_SERVICE_NAME } from '@opentelemetry/semantic-conventions' +import type { TelemetryConfig } from '@/config' +import { buildResource, startTelemetry } from '@/telemetry' + +const silent = { info: jest.fn(), warn: jest.fn() } + +const config = (overrides: Partial = {}): TelemetryConfig => ({ + enabled: true, + // Nothing listens on port 1. + endpoint: '127.0.0.1:1', + insecure: true, + traces: { enabled: true, sampleRatio: 1 }, + metrics: { enabled: true, intervalMillis: 60_000 }, + service: { name: 'wallet-test', version: 'v9.9.9', env: 'test' }, + ...overrides +}) + +// An OTLP gRPC receiver that keeps the raw request bytes. Protobuf stores +// strings as UTF-8, so a test can look for an attribute value in them. +async function startReceiver() { + const received = { traces: [] as Buffer[], metrics: [] as Buffer[] } + const exportMethod = (path: string) => ({ + path, + requestStream: false, + responseStream: false, + requestSerialize: (value: Buffer) => value, + requestDeserialize: (value: Buffer) => value, + // An empty message is a valid Export*ServiceResponse. + responseSerialize: () => Buffer.alloc(0), + responseDeserialize: (value: Buffer) => value + }) + const handler = + (into: Buffer[]): UntypedServiceImplementation[string] => + ( + call: { request: Buffer }, + callback: (error: null, response: Buffer) => void + ) => { + into.push(call.request) + callback(null, Buffer.alloc(0)) + } + + const server = new Server() + server.addService( + { + Export: exportMethod( + '/opentelemetry.proto.collector.trace.v1.TraceService/Export' + ) + }, + { Export: handler(received.traces) } + ) + server.addService( + { + Export: exportMethod( + '/opentelemetry.proto.collector.metrics.v1.MetricsService/Export' + ) + }, + { Export: handler(received.metrics) } + ) + + const port = await new Promise((resolve, reject) => + server.bindAsync( + '127.0.0.1:0', + ServerCredentials.createInsecure(), + (error, boundPort) => (error ? reject(error) : resolve(boundPort)) + ) + ) + + return { + endpoint: `127.0.0.1:${port}`, + received, + stop: () => + new Promise((resolve) => server.tryShutdown(() => resolve())) + } +} + +describe('startTelemetry', (): void => { + const originalResourceAttributes = process.env.OTEL_RESOURCE_ATTRIBUTES + + afterEach((): void => { + if (originalResourceAttributes === undefined) { + delete process.env.OTEL_RESOURCE_ATTRIBUTES + } else { + process.env.OTEL_RESOURCE_ATTRIBUTES = originalResourceAttributes + } + }) + + it('installs nothing and builds no instrumentation when disabled', async (): Promise => { + const instrumentations = jest.fn(() => []) + const telemetry = startTelemetry(config({ enabled: false }), { + instrumentations, + logger: silent + }) + + expect(instrumentations).not.toHaveBeenCalled() + expect(trace.getTracer('test').startSpan('span').isRecording()).toBe(false) + expect(metrics.getMeterProvider().constructor.name).toBe( + 'NoopMeterProvider' + ) + expect(silent.info).toHaveBeenCalledWith('telemetry is disabled') + + await telemetry.shutdown() + }) + + it('starts without a reachable collector', async (): Promise => { + const telemetry = startTelemetry(config(), { logger: silent }) + + const span = trace.getTracer('test').startSpan('span') + expect(span.isRecording()).toBe(true) + span.end() + + // The final flush cannot reach the collector. Shutdown still finishes, + // and reports the failed export instead of hiding it. + const startedAt = Date.now() + await expect(telemetry.shutdown()).rejects.toThrow( + 'telemetry shutdown failed' + ) + expect(Date.now() - startedAt).toBeLessThan(15_000) + }, 30_000) + + it('builds the instrumentations when enabled', async (): Promise => { + const instrumentations = jest.fn(() => []) + const telemetry = startTelemetry( + config({ + traces: { enabled: true, sampleRatio: 1 }, + metrics: { enabled: false, intervalMillis: 60_000 } + }), + { + instrumentations, + logger: silent + } + ) + + expect(instrumentations).toHaveBeenCalledTimes(1) + await telemetry.shutdown() + }) + + it('has a shutdown that is safe to call more than once', async (): Promise => { + const telemetry = startTelemetry( + config({ metrics: { enabled: false, intervalMillis: 60_000 } }), + { logger: silent } + ) + + const first = telemetry.shutdown() + const second = telemetry.shutdown() + expect(second).toBe(first) + await first + await expect(telemetry.shutdown()).resolves.toBeUndefined() + }) + + it('removes the global providers on shutdown', async (): Promise => { + const telemetry = startTelemetry( + config({ metrics: { enabled: false, intervalMillis: 60_000 } }), + { logger: silent } + ) + await telemetry.shutdown() + + expect(trace.getTracer('test').startSpan('span').isRecording()).toBe(false) + }) + + it('records nothing with a sample ratio of 0', async (): Promise => { + const telemetry = startTelemetry( + config({ + traces: { enabled: true, sampleRatio: 0 }, + metrics: { enabled: false, intervalMillis: 60_000 } + }), + { logger: silent } + ) + + expect(trace.getTracer('test').startSpan('span').isRecording()).toBe(false) + await telemetry.shutdown() + }) + + it('exports traces and metrics to the endpoint, with the service identity', async (): Promise => { + const receiver = await startReceiver() + try { + const telemetry = startTelemetry( + config({ endpoint: receiver.endpoint }), + { logger: silent } + ) + + trace.getTracer('test').startSpan('exported-span').end() + metrics.getMeter('test').createCounter('exported_counter').add(1) + + // Shutdown flushes both pipelines. + await telemetry.shutdown() + + const traces = Buffer.concat(receiver.received.traces).toString('utf8') + const exportedMetrics = Buffer.concat(receiver.received.metrics).toString( + 'utf8' + ) + expect(traces).toContain('exported-span') + expect(traces).toContain('wallet-test') + expect(exportedMetrics).toContain('exported_counter') + expect(exportedMetrics).toContain('wallet-test') + expect(silent.warn).not.toHaveBeenCalled() + } finally { + await receiver.stop() + } + }, 30_000) +}) + +describe('buildResource', (): void => { + afterEach((): void => { + delete process.env.OTEL_RESOURCE_ATTRIBUTES + }) + + it('carries the service identity', (): void => { + const { attributes } = buildResource(config()) + + expect(attributes).toMatchObject({ + [ATTR_SERVICE_NAME]: 'wallet-test', + 'service.version': 'v9.9.9', + 'deployment.environment.name': 'test' + }) + }) + + it('leaves out an empty version', (): void => { + const { attributes } = buildResource( + config({ service: { name: 'wallet-test', version: '', env: 'test' } }) + ) + + expect(attributes).not.toHaveProperty('service.version') + }) + + it('adds OTEL_RESOURCE_ATTRIBUTES, but the configured identity wins', (): void => { + process.env.OTEL_RESOURCE_ATTRIBUTES = + 'k8s.pod.name=wallet-backend-abc,service.name=from-env' + + const { attributes } = buildResource(config()) + + expect(attributes).toMatchObject({ + 'k8s.pod.name': 'wallet-backend-abc', + [ATTR_SERVICE_NAME]: 'wallet-test' + }) + }) +}) diff --git a/packages/shared/telemetry/tsconfig.build.json b/packages/shared/telemetry/tsconfig.build.json new file mode 100644 index 000000000..2f89a6611 --- /dev/null +++ b/packages/shared/telemetry/tsconfig.build.json @@ -0,0 +1,15 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ESNext"], + "outDir": "./dist", + "rootDir": "./src", + "sourceMap": true, + "baseUrl": ".", + "paths": { + "@/*": ["./src/*"] + } + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "**/node_modules/*"] +} diff --git a/packages/shared/telemetry/tsconfig.json b/packages/shared/telemetry/tsconfig.json new file mode 100644 index 000000000..f4892c1af --- /dev/null +++ b/packages/shared/telemetry/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ESNext"], + "sourceMap": true, + "baseUrl": ".", + "paths": { + "@/*": ["./src/*"] + } + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "**/node_modules/*"] +} diff --git a/packages/wallet/backend/.env.example b/packages/wallet/backend/.env.example index 01a66a057..8e63e2f05 100644 --- a/packages/wallet/backend/.env.example +++ b/packages/wallet/backend/.env.example @@ -94,6 +94,22 @@ # ATALLA_CLIENT_CERT_PATH=/etc/atalla/tls/client.crt # ATALLA_CLIENT_KEY_PATH=/etc/atalla/tls/client.key +########## OPENTELEMETRY ####################################################################### +## Off by default. When enabled, traces and metrics go over OTLP/gRPC to the +## collector at TELEMETRY_ENDPOINT, a host:port with no scheme. The process +## stops at start-up if a value is wrong. See packages/shared/telemetry. + +# TELEMETRY_ENABLED=false +# TELEMETRY_ENDPOINT=localhost:4317 +# TELEMETRY_INSECURE=true +# TELEMETRY_TRACES_ENABLED=true +# TELEMETRY_TRACES_SAMPLE_RATIO=1 +# TELEMETRY_METRICS_ENABLED=true +# TELEMETRY_METRICS_INTERVAL=15s +# SERVICE_NAME=wallet-backend +# SERVICE_VERSION= +# SERVICE_ENV=dev + # STRIPE_SECRET_KEY=notset # STRIPE_WEBHOOK_SECRET=notset # USE_STRIPE=false diff --git a/packages/wallet/backend/.env.local b/packages/wallet/backend/.env.local index 534f2eaf9..dd3b8d8d8 100644 --- a/packages/wallet/backend/.env.local +++ b/packages/wallet/backend/.env.local @@ -1,4 +1,12 @@ PORT=3003 +# Traces and metrics to the local collector (local/observability.yaml). Off +# here, because that stack is opt-in: `pnpm dev:observability` starts it and +# sets TELEMETRY_ENABLED=true, which wins over this file. See Grafana at +# https://grafana.testnet.test. +TELEMETRY_ENABLED=false +TELEMETRY_ENDPOINT=localhost:4317 +TELEMETRY_INSECURE=true +SERVICE_ENV=local NODE_ENV=development NODE_TLS_REJECT_UNAUTHORIZED=0 DATABASE_URL=postgres://wallet_backend:wallet_backend@localhost:15434/wallet_backend diff --git a/packages/wallet/backend/Dockerfile.prod b/packages/wallet/backend/Dockerfile.prod index cf8fc3a4e..d52eaf031 100644 --- a/packages/wallet/backend/Dockerfile.prod +++ b/packages/wallet/backend/Dockerfile.prod @@ -14,6 +14,7 @@ COPY package.json pnpm-lock.yaml pnpm-workspace.yaml .npmrc .nvmrc ./ COPY packages/wallet/backend/package.json ./packages/wallet/backend/package.json COPY packages/wallet/shared/package.json ./packages/wallet/shared/package.json COPY packages/shared/backend/package.json ./packages/shared/backend/package.json +COPY packages/shared/telemetry/package.json ./packages/shared/telemetry/package.json RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install -r --frozen-lockfile \ | grep -v "cross-device link not permitted\|Falling back to copying packages from store" @@ -26,9 +27,10 @@ COPY package.json pnpm-lock.yaml pnpm-workspace.yaml .npmrc .nvmrc tsconfig.base COPY packages/wallet/backend ./packages/wallet/backend COPY packages/wallet/shared ./packages/wallet/shared COPY packages/shared/backend ./packages/shared/backend +COPY packages/shared/telemetry ./packages/shared/telemetry RUN pnpm wallet:backend build -RUN rm -rf ./node_modules ./packages/wallet/backend/node_modules ./packages/wallet/shared/node_modules ./packages/shared/backend/node_modules +RUN rm -rf ./node_modules ./packages/wallet/backend/node_modules ./packages/wallet/shared/node_modules ./packages/shared/backend/node_modules ./packages/shared/telemetry/node_modules RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install -r --frozen-lockfile --prod \ | grep -v "cross-device link not permitted\|Falling back to copying packages from store" @@ -51,4 +53,8 @@ COPY --from=builder /testnet/packages/shared/backend/node_modules ./packages/sha COPY --from=builder /testnet/packages/shared/backend/dist ./packages/shared/backend/dist COPY --from=builder /testnet/packages/shared/backend/package.json ./packages/shared/backend/package.json +COPY --from=builder /testnet/packages/shared/telemetry/node_modules ./packages/shared/telemetry/node_modules +COPY --from=builder /testnet/packages/shared/telemetry/dist ./packages/shared/telemetry/dist +COPY --from=builder /testnet/packages/shared/telemetry/package.json ./packages/shared/telemetry/package.json + CMD ["node", "packages/wallet/backend/dist/index.js"] diff --git a/packages/wallet/backend/package.json b/packages/wallet/backend/package.json index e545a2c67..1cd60b3d1 100644 --- a/packages/wallet/backend/package.json +++ b/packages/wallet/backend/package.json @@ -2,7 +2,7 @@ "name": "@wallet/backend", "scripts": { "start": "sh -c 'set --; [ -f .env.local ] && set -- \"$@\" --env-file=.env.local; [ -f .env ] && set -- \"$@\" --env-file=.env; node \"$@\" -r tsconfig-paths/register dist/index.js'", - "build:deps": "pnpm --filter @shared/backend build && pnpm --filter @wallet/shared build", + "build:deps": "pnpm --filter @shared/backend build && pnpm --filter @shared/telemetry build && pnpm --filter @wallet/shared build", "build": "pnpm build:deps && tsc --build tsconfig.build.json && tsc-alias -p tsconfig.build.json", "dev": "nodemon --watch ./src --watch ../shared/src --ext ts,json --delay 1 --exec \"sh -c 'set --; [ -f .env.local ] && set -- \\$@ --env-file=.env.local; [ -f .env ] && set -- \\$@ --env-file=.env; pnpm run build && node \\$@ ./dist/index.js'\"", "test": "jest --passWithNoTests --maxWorkers=75%", @@ -10,13 +10,19 @@ }, "dependencies": { "@google-cloud/logging-winston": "^6.0.1", - "@interledger/tlv-kit": "^1.1.0", "@interledger/hsm-atalla-issuer": "^0.2.0", "@interledger/hsm-atalla-merchant": "^0.2.0", "@interledger/hsm-atalla-transport": "^0.2.0", - "@prometheus-io/client": "^0.16.1", + "@interledger/tlv-kit": "^1.1.0", + "@opentelemetry/instrumentation-express": "^0.71.0", + "@opentelemetry/instrumentation-http": "^0.223.0", + "@opentelemetry/instrumentation-ioredis": "^0.71.0", + "@opentelemetry/instrumentation-pg": "^0.75.0", + "@opentelemetry/instrumentation-undici": "^0.33.0", + "@opentelemetry/instrumentation-winston": "^0.67.0", "@sendgrid/mail": "^8.1.6", "@shared/backend": "workspace:*", + "@shared/telemetry": "workspace:^", "@wallet/shared": "workspace:*", "awilix": "^12.0.5", "axios": "^1.12.2", diff --git a/packages/wallet/backend/src/app.ts b/packages/wallet/backend/src/app.ts index e676d5d68..cd7e5a916 100644 --- a/packages/wallet/backend/src/app.ts +++ b/packages/wallet/backend/src/app.ts @@ -48,13 +48,7 @@ import { SocketService } from './socket/service' import { GrantService } from '@/grant/service' import { AwilixContainer } from 'awilix' import { Cradle } from '@/createContainer' -import { - Forbidden, - initErrorHandler, - RedisClient, - createHttpMetrics, - startMetricsServer -} from '@shared/backend' +import { Forbidden, initErrorHandler, RedisClient } from '@shared/backend' import { GateHubController } from '@/gatehub/controller' import { GateHubClient } from '@/gatehub/client' import { GateHubService } from '@/gatehub/service' @@ -109,7 +103,6 @@ export interface Bindings { export class App { private server!: Server - private metricsServer!: Server constructor(private container: AwilixContainer) {} @@ -120,7 +113,6 @@ export class App { const knex = this.container.resolve('knex') const socketService = this.container.resolve('socketService') const hsmAtallaService = this.container.resolve('hsmAtallaService') - const metricsRegistry = this.container.resolve('metricsRegistry') await knex.migrate.latest({ directory: __dirname + '/../migrations' @@ -130,9 +122,6 @@ export class App { this.server = express.listen(env.PORT) logger.info(`Server started on port ${env.PORT}`) - this.metricsServer = startMetricsServer(metricsRegistry, env.METRICS_PORT) - logger.info(`Metrics server started on port ${env.METRICS_PORT}`) - // Log the browser-facing auth configuration on every boot. These values // fail silently when wrong — a bad cookie domain or a missing CORS origin // produces no error anywhere, just users who cannot stay logged in — so @@ -149,7 +138,6 @@ export class App { await hsmAtallaService.start().catch((error) => { this.server.close() - this.metricsServer?.close() throw error }) @@ -159,7 +147,6 @@ export class App { public stop = async (): Promise => { const hsmAtallaService = this.container.resolve('hsmAtallaService') this.server.close() - this.metricsServer?.close() await hsmAtallaService.stop() } @@ -178,7 +165,6 @@ export class App { const env = this.container.resolve('env') const logger = this.container.resolve('logger') - const metricsRegistry = this.container.resolve('metricsRegistry') const authController = this.container.resolve('authController') const userController = this.container.resolve('userController') const walletAddressController = this.container.resolve( @@ -215,9 +201,6 @@ export class App { ? this.container.resolve('adminNotificationController') : undefined - const { httpMetricsMiddleware } = createHttpMetrics(metricsRegistry) - app.use(httpMetricsMiddleware) - app.use( cors({ origin: getFrontendOrigins(env.RAFIKI_MONEY_FRONTEND_HOST), diff --git a/packages/wallet/backend/src/config/env.ts b/packages/wallet/backend/src/config/env.ts index ca59add45..383782c3d 100644 --- a/packages/wallet/backend/src/config/env.ts +++ b/packages/wallet/backend/src/config/env.ts @@ -1,5 +1,6 @@ import { z } from 'zod' import { validateTlsFile } from '@/hsm/atalla/tls-files' +import { refineTelemetryEnv, telemetryEnvShape } from '@shared/telemetry' const booleanString = (defaultValue: 'true' | 'false') => z @@ -10,7 +11,6 @@ const booleanString = (defaultValue: 'true' | 'false') => export const envSchema = z .object({ PORT: z.coerce.number().default(3000), - METRICS_PORT: z.coerce.number().default(9464), NODE_ENV: z.string().default('development'), DATABASE_URL: z .string() @@ -114,9 +114,13 @@ export const envSchema = z .transform((val) => { const trimmed = val?.trim() return trimmed && trimmed.length > 0 ? trimmed : undefined - }) + }), + // TELEMETRY_* and SERVICE_*. See packages/shared/telemetry. + ...telemetryEnvShape }) .superRefine((env, ctx) => { + refineTelemetryEnv(env, ctx) + if (!env.HSM_ENABLED) return for (const key of ['ATALLA_HOST', 'ATALLA_PORT'] as const) { diff --git a/packages/wallet/backend/src/createContainer.ts b/packages/wallet/backend/src/createContainer.ts index 3f10309a6..767ed4c9d 100644 --- a/packages/wallet/backend/src/createContainer.ts +++ b/packages/wallet/backend/src/createContainer.ts @@ -47,12 +47,7 @@ import { createRhyzaAdminClient, createRhyzaIdpClient } from '@/config/rhyza' import { WalletAddressKeyController } from '@/walletAddressKeys/controller' import { WalletAddressKeyService } from '@/walletAddressKeys/service' import { generateKnex } from '@/config/knex' -import { - asClassSingletonWithLogger, - RedisClient, - createMetricsRegistry, - type Registry -} from '@shared/backend' +import { asClassSingletonWithLogger, RedisClient } from '@shared/backend' import { generateLogger } from '@/config/logger' import { GraphQLClient } from 'graphql-request' import { KratosService } from './rafiki/kratos.service' @@ -75,7 +70,6 @@ export interface Cradle { env: Env logger: Logger knex: Knex - metricsRegistry: Registry sessionService: SessionService emailService: EmailService userService: UserService @@ -138,7 +132,6 @@ export async function createContainer( container.register({ env: asValue(env), logger: asValue(logger), - metricsRegistry: asFunction(createMetricsRegistry).singleton(), knex: asFunction(generateKnex).singleton(), sessionService: asClass(SessionService).singleton(), emailService: asClassSingletonWithLogger(EmailService, logger), diff --git a/packages/wallet/backend/src/index.ts b/packages/wallet/backend/src/index.ts index 50a385078..394b54151 100644 --- a/packages/wallet/backend/src/index.ts +++ b/packages/wallet/backend/src/index.ts @@ -1,9 +1,14 @@ +// Must stay the first import. See telemetry.ts. +import { telemetry } from './telemetry' import { App } from './app' import { env } from './config/env' import { createContainer } from '@/createContainer' import console from 'console' import process from 'process' +// Kubernetes sends SIGKILL 30 seconds after SIGTERM by default. +const SHUTDOWN_TIMEOUT_MS = 10_000 + BigInt.prototype.toJSON = function (this: bigint) { return this.toString() } @@ -11,6 +16,31 @@ export const start = async (app: App): Promise => { await app.startServer() } +// Stops the app, then flushes the telemetry that is still buffered. +const handleShutdown = (app: App) => async (signal: NodeJS.Signals) => { + console.log(`Received ${signal}, shutting down`) + setTimeout(() => { + console.log('Shutdown timed out') + process.exit(1) + }, SHUTDOWN_TIMEOUT_MS).unref() + + let exitCode = 0 + try { + await app.stop() + } catch (e) { + console.log('Error on stopping the app') + console.log(e) + exitCode = 1 + } + try { + await telemetry.shutdown() + } catch (e) { + console.log('Error on flushing telemetry') + console.log(e) + } + process.exit(exitCode) +} + async function bootstrap() { const container = await createContainer(env) const app = new App(container) @@ -22,8 +52,13 @@ async function bootstrap() { } catch (e) { console.log('Error on starting the app') console.log(e) + await telemetry.shutdown().catch(() => {}) process.exit(1) } + + const shutdown = handleShutdown(app) + process.once('SIGTERM', shutdown) + process.once('SIGINT', shutdown) } if (!module.parent) { diff --git a/packages/wallet/backend/src/telemetry.ts b/packages/wallet/backend/src/telemetry.ts new file mode 100644 index 000000000..8e92b9bdf --- /dev/null +++ b/packages/wallet/backend/src/telemetry.ts @@ -0,0 +1,51 @@ +// index.ts imports this file first. An instrumentation patches a module when +// that module is loaded, so this must run before express, pg, ioredis and +// winston are imported. Import only the environment here. +import { + startTelemetry, + telemetryConfigFromEnv, + WALLET_SECRET_PATHS +} from '@shared/telemetry' +import { + ExpressInstrumentation, + ExpressLayerType +} from '@opentelemetry/instrumentation-express' +import { HttpInstrumentation } from '@opentelemetry/instrumentation-http' +import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis' +import { PgInstrumentation } from '@opentelemetry/instrumentation-pg' +import { UndiciInstrumentation } from '@opentelemetry/instrumentation-undici' +import { WinstonInstrumentation } from '@opentelemetry/instrumentation-winston' +import { env } from '@/config/env' + +export const telemetry = startTelemetry( + telemetryConfigFromEnv(env, { serviceName: 'wallet-backend' }), + { + // The application logger is silent in tests; so is this. + logger: env.NODE_ENV === 'test' ? { info() {}, warn() {} } : console, + // Password reset and e-mail verification tokens travel in the path. + redactUrlPaths: WALLET_SECRET_PATHS, + // The pending-transaction loop queries Postgres every few seconds, outside + // any request. Its queries would each start a trace of their own and bury + // the requests in Tempo. They still count in the Postgres metrics. + dropRootSpan: (name) => + name.startsWith('pg.') || name.startsWith('pg-pool.'), + instrumentations: () => [ + // HTTP server and client spans, and the http.server.request.duration and + // http.client.request.duration metrics. + new HttpInstrumentation(), + // Sets http.route on the HTTP spans and metrics, and adds a span for the + // route handler. A span for every middleware and router layer would add + // ten spans to each request and say little; http.route is still set. + new ExpressInstrumentation({ + ignoreLayersType: [ExpressLayerType.MIDDLEWARE, ExpressLayerType.ROUTER] + }), + // Query arguments are not recorded, because they carry PII. + new PgInstrumentation(), + new IORedisInstrumentation(), + // fetch() + new UndiciInstrumentation(), + // Adds trace_id and span_id to every log line written inside a span. + new WinstonInstrumentation({ disableLogSending: true }) + ] + } +) diff --git a/packages/wallet/backend/tests/app.ts b/packages/wallet/backend/tests/app.ts index 95c1a56fb..7c199c20a 100644 --- a/packages/wallet/backend/tests/app.ts +++ b/packages/wallet/backend/tests/app.ts @@ -17,7 +17,6 @@ export const createApp = async ( const knex = await container.resolve('knex') env.PORT = 0 - env.METRICS_PORT = 0 const app = new App(container) await start(app) diff --git a/packages/wallet/backend/tests/config/env.test.ts b/packages/wallet/backend/tests/config/env.test.ts index 3737257e7..4252bb76c 100644 --- a/packages/wallet/backend/tests/config/env.test.ts +++ b/packages/wallet/backend/tests/config/env.test.ts @@ -138,3 +138,56 @@ describe('env — Atalla TLS paths', () => { expect(errorsFor(overrides, 'ATALLA_CLIENT_KEY_PATH')).toHaveLength(0) }) }) + +describe('env — telemetry', () => { + const parse = (overrides: Record) => + envSchema.safeParse({ ...BASE, ...overrides }) + + const enabled = { + TELEMETRY_ENABLED: 'true', + TELEMETRY_ENDPOINT: 'testnet-wallet-otel-collector:4317', + TELEMETRY_INSECURE: 'true' + } + + it('is disabled by default', (): void => { + const result = parse({}) + expect(result.success).toBe(true) + expect(result.data).toMatchObject({ + TELEMETRY_ENABLED: false, + TELEMETRY_ENDPOINT: '', + TELEMETRY_METRICS_INTERVAL: '15s', + TELEMETRY_TRACES_SAMPLE_RATIO: 1 + }) + }) + + it('accepts an enabled block with an endpoint', (): void => { + expect(parse(enabled).success).toBe(true) + }) + + it('rejects an enabled block without an endpoint', (): void => { + const result = parse({ TELEMETRY_ENABLED: 'true' }) + expect(result.success).toBe(false) + expect(result.error?.flatten().fieldErrors.TELEMETRY_ENDPOINT).toEqual([ + 'telemetry is enabled, but TELEMETRY_ENDPOINT is not set' + ]) + }) + + it('rejects an endpoint that is a URL', (): void => { + const result = parse({ + ...enabled, + TELEMETRY_ENDPOINT: 'http://testnet-wallet-otel-collector:4317' + }) + expect(result.success).toBe(false) + expect(result.error?.flatten().fieldErrors.TELEMETRY_ENDPOINT).toEqual([ + expect.stringContaining('not a URL') + ]) + }) + + it('checks telemetry when the HSM is disabled', (): void => { + const result = parse({ ...enabled, TELEMETRY_TRACES_SAMPLE_RATIO: '2' }) + expect(result.success).toBe(false) + expect( + result.error?.flatten().fieldErrors.TELEMETRY_TRACES_SAMPLE_RATIO + ).toHaveLength(1) + }) +}) diff --git a/packages/wallet/frontend/.env.example b/packages/wallet/frontend/.env.example index bfa9260d1..aa71b65f2 100644 --- a/packages/wallet/frontend/.env.example +++ b/packages/wallet/frontend/.env.example @@ -43,3 +43,12 @@ # NODE_ENV=production # GATEHUB_ENV=production # FEATURES_ENABLED=true + +########## OPENTELEMETRY ######################################################################### +## The server only, read once at start-up. The same variables as the backend; +## see packages/shared/telemetry. A wrong value stops the server. + +# TELEMETRY_ENABLED=false +# TELEMETRY_ENDPOINT=localhost:4317 +# TELEMETRY_INSECURE=true +# SERVICE_NAME=wallet-frontend diff --git a/packages/wallet/frontend/.env.local b/packages/wallet/frontend/.env.local index 3197db57a..fe285eaad 100644 --- a/packages/wallet/frontend/.env.local +++ b/packages/wallet/frontend/.env.local @@ -11,3 +11,12 @@ OPEN_PAYMENTS_HOST=https://rafiki-backend.testnet.test GATEHUB_ENV=sandbox THEME=dark FEATURES_ENABLED=false + +# Traces and metrics to the local collector (local/observability.yaml). Off +# here, because that stack is opt-in: `pnpm dev:observability` starts it and +# sets TELEMETRY_ENABLED=true, which wins over this file. See Grafana at +# https://grafana.testnet.test. +TELEMETRY_ENABLED=false +TELEMETRY_ENDPOINT=localhost:4317 +TELEMETRY_INSECURE=true +SERVICE_ENV=local diff --git a/packages/wallet/frontend/Dockerfile.prod b/packages/wallet/frontend/Dockerfile.prod index 7b34a708e..1dc2c6b5e 100644 --- a/packages/wallet/frontend/Dockerfile.prod +++ b/packages/wallet/frontend/Dockerfile.prod @@ -14,6 +14,7 @@ RUN apk add --no-cache \ COPY package.json pnpm-lock.yaml pnpm-workspace.yaml .npmrc .nvmrc ./ COPY packages/wallet/frontend/package.json ./packages/wallet/frontend/package.json COPY packages/wallet/shared/package.json ./packages/wallet/shared/package.json +COPY packages/shared/telemetry/package.json ./packages/shared/telemetry/package.json RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm fetch RUN --mount=type=cache,id=pnpm,target=/root/.local/share/pnpm/store pnpm install -r --frozen-lockfile @@ -26,10 +27,13 @@ ENV NEXT_TELEMETRY_DISABLED=1 COPY package.json pnpm-lock.yaml pnpm-workspace.yaml .npmrc .nvmrc tsconfig.base.json eslint.config.mjs ./ COPY packages/wallet/frontend /testnet/packages/wallet/frontend COPY packages/wallet/shared ./packages/wallet/shared +# Webpack bundles it into the server build. The runner stage does not need it. +COPY packages/shared/telemetry ./packages/shared/telemetry COPY --from=base /testnet/node_modules ./node_modules COPY --from=base /testnet/packages/wallet/frontend/node_modules ./packages/wallet/frontend/node_modules COPY --from=base /testnet/packages/wallet/shared/node_modules ./packages/wallet/shared/node_modules +COPY --from=base /testnet/packages/shared/telemetry/node_modules ./packages/shared/telemetry/node_modules # This build takes no configuration. Every URL, the theme and the feature flags # are read from the environment per request, so one image serves every diff --git a/packages/wallet/frontend/next.config.js b/packages/wallet/frontend/next.config.js index 388ecee4c..87f152868 100644 --- a/packages/wallet/frontend/next.config.js +++ b/packages/wallet/frontend/next.config.js @@ -13,7 +13,9 @@ const nextConfig = { poweredByHeader: false, // ESLint 9.x removed options (useEslintrc, extensions) that Next.js 14 // passes internally. Linting is handled separately via `pnpm lint:check`. - eslint: { ignoreDuringBuilds: true } + eslint: { ignoreDuringBuilds: true }, + // Loads src/instrumentation.ts, which starts OpenTelemetry on the server. + experimental: { instrumentationHook: true } } module.exports = withBundleAnalyzer(nextConfig) diff --git a/packages/wallet/frontend/package.json b/packages/wallet/frontend/package.json index c84b1fab0..9bd3c3438 100644 --- a/packages/wallet/frontend/package.json +++ b/packages/wallet/frontend/package.json @@ -2,21 +2,25 @@ "name": "@wallet/frontend", "scripts": { "dev": "next dev -p 4003", - "build:deps": "pnpm --filter @wallet/shared build", + "build:deps": "pnpm --filter @shared/telemetry build && pnpm --filter @wallet/shared build", "build": "pnpm build:deps && next build", "start": "next start" }, "dependencies": { "@headlessui/react": "^2.2.9", "@hookform/resolvers": "^3.10.0", + "@opentelemetry/instrumentation-http": "^0.223.0", + "@opentelemetry/instrumentation-undici": "^0.33.0", "@radix-ui/react-toast": "^1.2.14", "@radix-ui/react-tooltip": "^1.2.7", + "@shared/telemetry": "workspace:^", "@types/node-rsa": "^1.1.4", "@wallet/shared": "workspace:*", "class-variance-authority": "^0.7.1", "jspdf": "^4.2.1", "ky": "^1.11.0", "next": "14.2.32", + "next-qrcode": "^2.5.1", "next-themes": "^0.4.6", "node-rsa": "^1.1.1", "nprogress": "^0.2.0", @@ -30,8 +34,7 @@ "tailwind-merge": "^2.6.0", "tailwindcss-animate": "^1.0.7", "valtio": "^2.1.8", - "zod": "^3.25.76", - "next-qrcode": "^2.5.1" + "zod": "^3.25.76" }, "devDependencies": { "@next/bundle-analyzer": "^13.4.12", diff --git a/packages/wallet/frontend/src/instrumentation.node.ts b/packages/wallet/frontend/src/instrumentation.node.ts new file mode 100644 index 000000000..5474a4325 --- /dev/null +++ b/packages/wallet/frontend/src/instrumentation.node.ts @@ -0,0 +1,52 @@ +/** + * Sends the server's traces and metrics to the same collector as the wallet + * backend. The settings are the backend's TELEMETRY_* and SERVICE_* variables; + * see packages/shared/telemetry. + */ +import { + parseTelemetryEnv, + startTelemetry, + WALLET_SECRET_PATHS +} from '@shared/telemetry' +import { HttpInstrumentation } from '@opentelemetry/instrumentation-http' +import { UndiciInstrumentation } from '@opentelemetry/instrumentation-undici' + +let config +try { + config = parseTelemetryEnv(process.env, { serviceName: 'wallet-frontend' }) +} catch (error) { + // The same rule as the backend: a bad setting stops the pod. + console.error(`[telemetry] Refusing to start. ${(error as Error).message}`) + process.exit(1) +} + +const telemetry = startTelemetry(config, { + // The reset and verification pages, and the backend calls they make, carry + // their token in the path. + redactUrlPaths: WALLET_SECRET_PATHS, + instrumentations: () => [ + // Server spans, and the http.server.request.duration metric. + new HttpInstrumentation(), + // fetch() from server-side rendering to the backend, so a page request + // and the backend calls it makes share one trace. + new UndiciInstrumentation() + ] +}) + +// Flushes the last batch on SIGTERM and SIGINT. Next.js's own handler closes the +// server and then calls process.exit, which would drop whatever is still +// buffered. This holds that exit back until the flush finishes, so the server +// still drains as before. With telemetry off nothing is registered. +if (config.enabled) { + const flushBeforeExit = () => { + const exit = process.exit.bind(process) + const flushed = telemetry.shutdown().catch((error: Error) => { + console.error(`[telemetry] Flush on shutdown failed. ${error.message}`) + }) + process.exit = ((code?: number) => { + void flushed.finally(() => exit(code)) + }) as typeof process.exit + } + process.once('SIGTERM', flushBeforeExit) + process.once('SIGINT', flushBeforeExit) +} diff --git a/packages/wallet/frontend/src/instrumentation.ts b/packages/wallet/frontend/src/instrumentation.ts new file mode 100644 index 000000000..c929962a0 --- /dev/null +++ b/packages/wallet/frontend/src/instrumentation.ts @@ -0,0 +1,9 @@ +/** + * Next.js calls `register()` once when the server starts. Telemetry covers the + * Node.js server only: the edge middleware and the browser send nothing. + */ +export async function register() { + if (process.env.NEXT_RUNTIME === 'nodejs') { + await import('./instrumentation.node') + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a26135907..e757aad4f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -269,9 +269,6 @@ importers: packages/shared/backend: dependencies: - '@prometheus-io/client': - specifier: ^0.16.1 - version: 0.16.1 awilix: specifier: ^12.0.5 version: 12.0.5 @@ -319,6 +316,67 @@ importers: specifier: ^5.9.3 version: 5.9.3 + packages/shared/telemetry: + dependencies: + '@opentelemetry/api': + specifier: ^1.9.1 + version: 1.9.1 + '@opentelemetry/context-async-hooks': + specifier: ^2.12.0 + version: 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/core': + specifier: ^2.12.0 + version: 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-metrics-otlp-grpc': + specifier: ^0.223.0 + version: 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-trace-otlp-grpc': + specifier: ^0.223.0 + version: 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': + specifier: ^0.223.0 + version: 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-runtime-node': + specifier: ^0.36.0 + version: 0.36.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': + specifier: ^2.12.0 + version: 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': + specifier: ^2.12.0 + version: 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-node': + specifier: ^2.12.0 + version: 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': + specifier: ^1.43.0 + version: 1.43.0 + zod: + specifier: ^3.25.76 + version: 3.25.76 + devDependencies: + '@grpc/grpc-js': + specifier: ^1.14.5 + version: 1.14.5 + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 + '@types/node': + specifier: ^24.13.1 + version: 24.13.1 + jest: + specifier: ^29.7.0 + version: 29.7.0(@types/node@24.13.1)(babel-plugin-macros@3.1.0) + ts-jest: + specifier: ^29.4.4 + version: 29.4.4(@babel/core@7.25.2)(@jest/transform@29.7.0)(@jest/types@29.6.3)(babel-jest@29.7.0(@babel/core@7.25.2))(jest-util@29.7.0)(jest@29.7.0(@types/node@24.13.1)(babel-plugin-macros@3.1.0))(typescript@5.9.3) + tsc-alias: + specifier: ^1.8.16 + version: 1.8.16 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + packages/wallet/backend: dependencies: '@google-cloud/logging-winston': @@ -336,15 +394,33 @@ importers: '@interledger/tlv-kit': specifier: ^1.1.0 version: 1.1.0 - '@prometheus-io/client': - specifier: ^0.16.1 - version: 0.16.1 + '@opentelemetry/instrumentation-express': + specifier: ^0.71.0 + version: 0.71.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-http': + specifier: ^0.223.0 + version: 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-ioredis': + specifier: ^0.71.0 + version: 0.71.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-pg': + specifier: ^0.75.0 + version: 0.75.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-undici': + specifier: ^0.33.0 + version: 0.33.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-winston': + specifier: ^0.67.0 + version: 0.67.0(@opentelemetry/api@1.9.1) '@sendgrid/mail': specifier: ^8.1.6 version: 8.1.6 '@shared/backend': specifier: workspace:* version: link:../../shared/backend + '@shared/telemetry': + specifier: workspace:^ + version: link:../../shared/telemetry '@wallet/shared': specifier: workspace:* version: link:../shared @@ -490,12 +566,21 @@ importers: '@hookform/resolvers': specifier: ^3.10.0 version: 3.10.0(react-hook-form@7.64.0(react@18.3.1)) + '@opentelemetry/instrumentation-http': + specifier: ^0.223.0 + version: 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation-undici': + specifier: ^0.33.0 + version: 0.33.0(@opentelemetry/api@1.9.1) '@radix-ui/react-toast': specifier: ^1.2.14 version: 1.2.14(@types/react-dom@18.3.7(@types/react@18.3.25))(@types/react@18.3.25)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@radix-ui/react-tooltip': specifier: ^1.2.7 version: 1.2.7(@types/react-dom@18.3.7(@types/react@18.3.25))(@types/react@18.3.25)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + '@shared/telemetry': + specifier: workspace:^ + version: link:../../shared/telemetry '@types/node-rsa': specifier: ^1.1.4 version: 1.1.4 @@ -513,7 +598,7 @@ importers: version: 1.11.0 next: specifier: 14.2.32 - version: 14.2.32(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) + version: 14.2.32(@opentelemetry/api@1.9.1)(@playwright/test@1.58.2)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) next-qrcode: specifier: ^2.5.1 version: 2.5.1(react@18.3.1) @@ -1591,8 +1676,8 @@ packages: peerDependencies: graphql: ^0.8.0 || ^0.9.0 || ^0.10.0 || ^0.11.0 || ^0.12.0 || ^0.13.0 || ^14.0.0 || ^15.0.0 || ^16.0.0 || ^17.0.0 - '@grpc/grpc-js@1.11.1': - resolution: {integrity: sha512-gyt/WayZrVPH2w/UTLansS7F9Nwld472JxxaETamrM8HNlsa+jSLNyKAZmhxI2Me4c3mQHFiS1wWHDY1g1Kthw==} + '@grpc/grpc-js@1.14.5': + resolution: {integrity: sha512-7VZM+SVdEcUUqSQeNI3zM8Qs/BhQKZndPo2h5VkYkAM8Iz0wJIa8mKV5ekQGqG8UUsnkQ0NMxIxwkIHYvj0qOw==} engines: {node: '>=12.10.0'} '@grpc/proto-loader@0.7.13': @@ -1600,6 +1685,11 @@ packages: engines: {node: '>=6'} hasBin: true + '@grpc/proto-loader@0.8.1': + resolution: {integrity: sha512-wtF6h+DY6M3YaDBPAmvuuA6jV8Sif9MjtOI5euKFWRgCDl5PeDpPsHR9u2l6St5ceY8AZgoNDww5+HvEsXFsGg==} + engines: {node: '>=6'} + hasBin: true + '@headlessui/react@2.2.9': resolution: {integrity: sha512-Mb+Un58gwBn0/yWZfyrCh0TJyurtT+dETj7YHleylHk5od3dv2XqETPGWMyQ5/7sYN7oWdyM1u9MvC0OC8UmzQ==} engines: {node: '>=10'} @@ -1963,10 +2053,160 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} - '@opentelemetry/api@1.9.0': - resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} + '@opentelemetry/api-logs@0.223.0': + resolution: {integrity: sha512-Oepm6etVPFCn0VWB1t833iOTgz1eYBDUPxcrOR/XvwzD1+Ag9VzSPSrrKAFNJ/haatj+8CXbG47pZXvdNQKDJw==} + engines: {node: '>=8.0.0'} + + '@opentelemetry/api@1.9.1': + resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} engines: {node: '>=8.0.0'} + '@opentelemetry/context-async-hooks@2.12.0': + resolution: {integrity: sha512-iK/1nD7mmZVkuYjVjkdFBiCvzntJdFny1S3CYbUYyOq7V5enL6XX+gJ7eh965L1LnS11VA8mQ7Ej/ck04SB0MQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/core@2.12.0': + resolution: {integrity: sha512-1HsSAuvT4/my0QrXWsyXFdWjKaPeCHsismKZgEEIb8NK13LP6DrMRlsUbvD/0F9mDWK3xDom9AMv3IscxgOnOQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/exporter-metrics-otlp-grpc@0.223.0': + resolution: {integrity: sha512-QneYDf2PWYAqmikS1/jeG8xCv0Q2EOSBm0RRM7IFU9zEnKLu7qxdlsz9t3YjxiAo7XzqfjTsJZys5qhYZAQ2PQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-metrics-otlp-http@0.223.0': + resolution: {integrity: sha512-hNC8yai/+xCNyTuiOy6JFepXXA/sb/2jCzQRY7c2cUWz6owaidihlHUpO20B9/dj8JKPNo8rcn7ikTy7Cj+JaQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/exporter-trace-otlp-grpc@0.223.0': + resolution: {integrity: sha512-gYDt98a44cvdbJNL+fFCFO7k7qSGaWGZAb6WUXGCKL0RiRhF96N6jlH4wXMzS1daZoV5AMHZZixjaLpQBMhpvA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-express@0.71.0': + resolution: {integrity: sha512-U/qhsPrcAHr5PxeoAsHng9fZWK/v+Fx+wvqOTj5kOTASarlv+/098UdE1dD4livq+dE/l50YXMKeORE1lsAHfA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-http@0.223.0': + resolution: {integrity: sha512-lQkvLeRjEHwDjc2NEEAPUlk4x/0qfjhbZh0U5d2/5rPPRQ7TNvMmOtd/ForpVVPGUBfhtEDrWhqumr7vLMjBtA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-ioredis@0.71.0': + resolution: {integrity: sha512-hPaLhyiwVqV6hcnVNK95ilmzKdEVSJVoRdWbE1Q8ABasTpJLy5CGBzHYwyDgYVulLzdLKJy23U5yRVwPt5zieA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-pg@0.75.0': + resolution: {integrity: sha512-YleNiIjYYEmIsH5ECDpq15ybFFXgcjdVhnh2vG3ij2xAApBMhgysuKtl6jM/KAelTl6rKZnREhrteGYmGBIPYg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-runtime-node@0.36.0': + resolution: {integrity: sha512-UL1AvU90pDTmtzjm5MkZ3IgaLDfZSKRxRPM6Zb3EZ5R3JIl6i8v+yPdvR4bozFGW4+2jT7AcK1mdeWZeb1XQwg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation-undici@0.33.0': + resolution: {integrity: sha512-eR4Y9kApgnKmiIoygMtyfABTL4bwAUlt00/Nxkkr5FSC28kcM6MxzTp5mNw+KD1XcVt6dBj3Sz5aoekbTqY9GQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.7.0 + + '@opentelemetry/instrumentation-winston@0.67.0': + resolution: {integrity: sha512-AhOQwJYfh3KzLupSOmb0fX+IcFvfPOWe0SXeKMpwFXRc0taolMtRofzpmi6sTBLcSAyHBw/0KEHKxeuIwqVkqQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/instrumentation@0.223.0': + resolution: {integrity: sha512-tttBo88TbwK1zpsH6VeJoxLQopxOjW3cnrP50fsQJRZftkSKDDwjolXXBvow2jy0LKXlNRkDQh0NT6yI/q+BEQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-exporter-base@0.223.0': + resolution: {integrity: sha512-HlJMrJXjOqdQeoYNDHEvFTpaY2p9uWDK1GPl5AZ4wI5easzfoppTduOv47M8eiF+4ew42vfI/Xvt7THYgESnRw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-grpc-exporter-base@0.223.0': + resolution: {integrity: sha512-62DuT9qzwFEFPq1aguQ+P6cWC4sJPwMxCj8wZ3Q+mH8LpPcV6zdv5gj1YtGIGZF5FcKt9JnDVraey0CEnu9MtQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-transformer@0.223.0': + resolution: {integrity: sha512-oxmmPL5/MbSsO8K588V9n/XLIe7FZTK30Fo+/U/lAnt0ZIK/hoPM03n2WpSTZVbUBGTUE7aLyxnAzh7nE2PD4Q==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/redis-common@0.38.3': + resolution: {integrity: sha512-VCghU1JYs/4gP6Gqf/xro9MEsZ7LrMv2uONVsaESKL38ZOB9BqnI98FfS23wjMnHlpuE+TTaWSoAVNpTwYXzjw==} + engines: {node: ^18.19.0 || >=20.6.0} + + '@opentelemetry/resources@2.12.0': + resolution: {integrity: sha512-+rSRoOdln6NhXTVcBdHSlZxJDNzwxadeweK2ltZO3tVtfp6aklABc06xclDk6ovRR+EiGgIcSbh/6NgjFhnb3A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-logs@0.223.0': + resolution: {integrity: sha512-ApiAfUiUh6fmJt1GbsBuoeDn8D8IL42UvGyPUr8o1Z89ZeIiOCE1rZ78KkIUjEX3OV8Oorl9ez5Qjo43ZynDyw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + + '@opentelemetry/sdk-metrics@2.12.0': + resolution: {integrity: sha512-mVMaEHwCUJuY3S4Esn+nT0yQA3qCEGh+SOPJsXygU0Gl453WcsDNjtEa8EfdewMwoLyr1tlofB4fXytVpiXuow==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.9.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@2.12.0': + resolution: {integrity: sha512-JOERgZCxRM06FEJ5kEWxlSRChah6MmXDx3KwCruqEv/iEyLkTokCSemf2y1w8oIHtfrr8EVTpzebq/31vn2Cpg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-trace-node@2.12.0': + resolution: {integrity: sha512-9DufH1IlmQMoKAfot5y/FhTP1RnmhLb5K6eKmJO/bvzPjYwPAazoiAUbTZFdPbqxOXN0FEbhZx/e6SB1Og4jmg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/sdk-trace@2.12.0': + resolution: {integrity: sha512-ImVuRQ6faOsTi1Wna88s2avgrrf/iDgXPigIeynkUhjstMuoQUDOJXgkez2j1mlU4btjXcXBGa+q7qKFYR3GsA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/semantic-conventions@1.43.0': + resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==} + engines: {node: '>=14'} + + '@opentelemetry/sql-common@0.42.0': + resolution: {integrity: sha512-nwUwUU+8O8a4bnLqk6CodWeegGMEANgC94KTAhXcpGWLrW/2/hek/0ajNbjXnSOoNuCX+nteUPs46HFHhou9Xw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.1.0 + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -1979,10 +2219,6 @@ packages: '@polka/url@1.0.0-next.25': resolution: {integrity: sha512-j7P6Rgr3mmtdkeDGTe0E/aYyWEWVtc5yFXtHCRHs28/jptDEWfaVOc5T7cblqy1XKPPfCxJc/8DwQ5YgLOZOVQ==} - '@prometheus-io/client@0.16.1': - resolution: {integrity: sha512-XkFPsGFGoSs/UMg/vh0QLDpanl/tsrOV5PLHdpGe8Ho+ir/Llhytr15j6y8ZwLrpIGzK9gTVVpIalA6C4UjUZw==} - engines: {node: ^22 || ^24 || >=26} - '@protobufjs/aspromise@1.1.2': resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} @@ -1992,12 +2228,21 @@ packages: '@protobufjs/codegen@2.0.4': resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + '@protobufjs/eventemitter@1.1.0': resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + '@protobufjs/fetch@1.1.0': resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + '@protobufjs/float@1.0.2': resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} @@ -2013,6 +2258,9 @@ packages: '@protobufjs/utf8@1.1.0': resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + '@protobufjs/utf8@1.1.2': + resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==} + '@radix-ui/primitive@1.1.2': resolution: {integrity: sha512-XnbHrrprsNqZKQhStrSwgRUQzoCI1glLzdw79xiZPoofhGICeZRSQ3dIxAKH1gb3OHfNf4d6f+vAv3kil2eggA==} @@ -2763,6 +3011,12 @@ packages: '@types/parse-json@4.0.2': resolution: {integrity: sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==} + '@types/pg-pool@2.0.7': + resolution: {integrity: sha512-U4CwmGVQcbEuqpyju8/ptOKg6gEC+Tqsvj2xS9o1g71bUh8twxnC6ZL5rZKCsGN0iyH0CwgUyc9VR5owNQF9Ng==} + + '@types/pg@8.15.6': + resolution: {integrity: sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ==} + '@types/pg@8.20.0': resolution: {integrity: sha512-bEPFOaMAHTEP1EzpvHTbmwR8UsFyHSKsRisLIHVMXnpNefSbGA1bD6CVy+qKjGSqmZqNqBDV2azOBo8TgkcVow==} @@ -3222,9 +3476,6 @@ packages: resolution: {integrity: sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==} engines: {node: '>=8'} - bintrees@1.0.2: - resolution: {integrity: sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==} - bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} @@ -3387,6 +3638,9 @@ packages: cjs-module-lexer@1.3.1: resolution: {integrity: sha512-a3KdPAANPbNE4ZUv9h6LckSl9zLsYOP4MBmhIPkRaeyybt+r4UghLvq+xw/YwUcC1gqylCkL4rdVs3Lwupjm4Q==} + cjs-module-lexer@2.3.0: + resolution: {integrity: sha512-lmNyBzi6iYiqyrExTctfmjd8tInx+dHu9PBomTfH5RXS53c27JMFJgzNv5TyAnm3PgGf+MLy0+Ws41/HlVxdKQ==} + class-transformer@0.5.1: resolution: {integrity: sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw==} @@ -3940,6 +4194,9 @@ packages: resolution: {integrity: sha512-uDn+FE1yrDzyC0pCo961B2IHbdM8y/ACZsKD4dG6WqrjV53BADjwa7D+1aom2rsNVfLyDgU/eigvlJGJ08OQ4w==} engines: {node: '>= 0.4'} + es-module-lexer@3.0.3: + resolution: {integrity: sha512-yWGe1O/j+almnV4hOYrmZo8u58JdnmXEMneRQAYkWtqSiT86yoMIEKF4rpqEDj52tbYl3xceBD3DQIBwcsC9Pw==} + es-object-atoms@1.0.0: resolution: {integrity: sha512-MZ4iQ6JwHOBQjahnjwaC1ZtIBH+2ohjamzAO3oaHcXYup7qxjF2fixyH+Q71voWHeOkI2q/TnJao/KfXYIZWbw==} engines: {node: '>= 0.4'} @@ -4246,6 +4503,9 @@ packages: resolution: {integrity: sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==} engines: {node: '>= 6'} + forwarded-parse@2.1.2: + resolution: {integrity: sha512-alTFZZQDKMporBH77856pXgzhEzaUVmLCDk+egLgIgHst3Tpndzz8MnKe+GzRJRfvVdn69HhpW7cmXzvtLvJAw==} + forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -4589,6 +4849,10 @@ packages: resolution: {integrity: sha512-P9J71vT5nLlDeV8FHs5nNxaLbrpfAV5cF5srvbZfpwpcJoM/xZR3hiv+q+SAnuSmuGbXMWud063iIMx/V/EWZQ==} engines: {node: '>=12.2'} + import-in-the-middle@3.5.2: + resolution: {integrity: sha512-WseIA/4o56+GYO3RmlOPvejYi771OFeJEeK/ejROsr9bdjW+E5NXTOD1Di8TL40Qgg3JJX9aa97Hf8iiSVIKZw==} + engines: {node: '>=18'} + import-local@3.2.0: resolution: {integrity: sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==} engines: {node: '>=8'} @@ -5175,6 +5439,9 @@ packages: long@5.2.3: resolution: {integrity: sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + loose-envify@1.4.0: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true @@ -5310,6 +5577,9 @@ packages: engines: {node: '>=10'} hasBin: true + module-details-from-path@1.0.4: + resolution: {integrity: sha512-EGWKgxALGMgzvxYF1UyGTy0HXX/2vHLkw6+NvDKW2jypWbHpjQuj4UMcqQWXHERJhVGKikolT06G3bcKe4fi7w==} + moment@2.30.1: resolution: {integrity: sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==} @@ -5931,6 +6201,10 @@ packages: resolution: {integrity: sha512-RXyHaACeqXeqAKGLDl68rQKbmObRsTIn4TYVUUug1KfS47YWCo5MacGITEryugIgZqORCvJWEk4l449POg5Txg==} engines: {node: '>=12.0.0'} + protobufjs@7.6.6: + resolution: {integrity: sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==} + engines: {node: '>=12.0.0'} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -6177,6 +6451,10 @@ packages: resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} engines: {node: '>=0.10.0'} + require-in-the-middle@8.0.1: + resolution: {integrity: sha512-QT7FVMXfWOYFbeRBF6nu+I6tr2Tf3u0q8RIEjNob/heKY/nh7drD/k7eeMFmSQgnTtCzLDcCu/XEnpW2wk4xCQ==} + engines: {node: '>=9.3.0 || >=8.10.0 <9.0.0'} + require-main-filename@2.0.0: resolution: {integrity: sha512-NKN5kMDylKuldxYLSUfrbo5Tuzh4hd+2E8NPPX02mZtn1VuREQToYe/ZdlJy+J3uCpfaiGF05e7B8W0iXbQHmg==} @@ -6652,9 +6930,6 @@ packages: resolution: {integrity: sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==} engines: {node: '>=8.0.0'} - tdigest@0.1.3: - resolution: {integrity: sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==} - teeny-request@9.0.0: resolution: {integrity: sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==} engines: {node: '>=14'} @@ -8001,7 +8276,7 @@ snapshots: '@google-cloud/paginator': 5.0.2 '@google-cloud/projectify': 4.0.0 '@google-cloud/promisify': 4.0.0 - '@opentelemetry/api': 1.9.0 + '@opentelemetry/api': 1.9.1 arrify: 2.0.1 dot-prop: 6.0.1 eventid: 2.0.1 @@ -8490,9 +8765,9 @@ snapshots: dependencies: graphql: 16.11.0 - '@grpc/grpc-js@1.11.1': + '@grpc/grpc-js@1.14.5': dependencies: - '@grpc/proto-loader': 0.7.13 + '@grpc/proto-loader': 0.8.1 '@js-sdsl/ordered-map': 4.4.2 '@grpc/proto-loader@0.7.13': @@ -8502,6 +8777,13 @@ snapshots: protobufjs: 7.3.2 yargs: 17.7.2 + '@grpc/proto-loader@0.8.1': + dependencies: + lodash.camelcase: 4.3.0 + long: 5.2.3 + protobufjs: 7.6.6 + yargs: 17.7.2 + '@headlessui/react@2.2.9(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': dependencies: '@floating-ui/react': 0.26.23(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -8931,7 +9213,194 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.17.1 - '@opentelemetry/api@1.9.0': {} + '@opentelemetry/api-logs@0.223.0': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@opentelemetry/api@1.9.1': {} + + '@opentelemetry/context-async-hooks@2.12.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@opentelemetry/core@2.12.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/exporter-metrics-otlp-grpc@0.223.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/exporter-metrics-otlp-http': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-grpc-exporter-base': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-metrics-otlp-http@0.223.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-exporter-base': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 2.12.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/exporter-trace-otlp-grpc@0.223.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/otlp-exporter-base': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-grpc-exporter-base': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.12.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/instrumentation-express@0.71.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-http@0.223.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + forwarded-parse: 2.1.2 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-ioredis@0.71.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/instrumentation': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/redis-common': 0.38.3 + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-pg@0.75.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + '@opentelemetry/sql-common': 0.42.0(@opentelemetry/api@1.9.1) + '@types/pg': 8.15.6 + '@types/pg-pool': 2.0.7 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-runtime-node@0.36.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.223.0 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.223.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-undici@0.33.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/instrumentation': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation-winston@0.67.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.223.0 + '@opentelemetry/instrumentation': 0.223.0(@opentelemetry/api@1.9.1) + transitivePeerDependencies: + - supports-color + + '@opentelemetry/instrumentation@0.223.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.223.0 + import-in-the-middle: 3.5.2 + require-in-the-middle: 8.0.1 + transitivePeerDependencies: + - supports-color + + '@opentelemetry/otlp-exporter-base@0.223.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/otlp-grpc-exporter-base@0.223.0(@opentelemetry/api@1.9.1)': + dependencies: + '@grpc/grpc-js': 1.14.5 + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-exporter-base': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.223.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/otlp-transformer@0.223.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.223.0 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.223.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.12.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/redis-common@0.38.3': {} + + '@opentelemetry/resources@2.12.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-logs@0.223.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.223.0 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-metrics@2.12.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.12.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/sdk-trace-base@2.12.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/sdk-trace-node@2.12.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/context-async-hooks': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.12.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/sdk-trace@2.12.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.12.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.43.0 + + '@opentelemetry/semantic-conventions@1.43.0': {} + + '@opentelemetry/sql-common@0.42.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.12.0(@opentelemetry/api@1.9.1) '@pkgjs/parseargs@0.11.0': optional: true @@ -8942,24 +9411,27 @@ snapshots: '@polka/url@1.0.0-next.25': {} - '@prometheus-io/client@0.16.1': - dependencies: - '@opentelemetry/api': 1.9.0 - tdigest: 0.1.3 - '@protobufjs/aspromise@1.1.2': {} '@protobufjs/base64@1.1.2': {} '@protobufjs/codegen@2.0.4': {} + '@protobufjs/codegen@2.0.5': {} + '@protobufjs/eventemitter@1.1.0': {} + '@protobufjs/eventemitter@1.1.1': {} + '@protobufjs/fetch@1.1.0': dependencies: '@protobufjs/aspromise': 1.1.2 '@protobufjs/inquire': 1.1.0 + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/float@1.0.2': {} '@protobufjs/inquire@1.1.0': {} @@ -8970,6 +9442,8 @@ snapshots: '@protobufjs/utf8@1.1.0': {} + '@protobufjs/utf8@1.1.2': {} + '@radix-ui/primitive@1.1.2': {} '@radix-ui/react-arrow@1.1.7(@types/react-dom@18.3.7(@types/react@18.3.25))(@types/react@18.3.25)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': @@ -9689,6 +10163,16 @@ snapshots: '@types/parse-json@4.0.2': {} + '@types/pg-pool@2.0.7': + dependencies: + '@types/pg': 8.20.0 + + '@types/pg@8.15.6': + dependencies: + '@types/node': 24.13.1 + pg-protocol: 1.10.3 + pg-types: 2.2.0 + '@types/pg@8.20.0': dependencies: '@types/node': 24.13.1 @@ -10270,8 +10754,6 @@ snapshots: binary-extensions@2.3.0: {} - bintrees@1.0.2: {} - bl@4.1.0: dependencies: buffer: 5.7.1 @@ -10495,6 +10977,8 @@ snapshots: cjs-module-lexer@1.3.1: {} + cjs-module-lexer@2.3.0: {} + class-transformer@0.5.1: {} class-variance-authority@0.7.1: @@ -10870,7 +11354,7 @@ snapshots: dockerode@4.0.7: dependencies: '@balena/dockerignore': 1.0.2 - '@grpc/grpc-js': 1.11.1 + '@grpc/grpc-js': 1.14.5 '@grpc/proto-loader': 0.7.13 docker-modem: 5.0.6 protobufjs: 7.3.2 @@ -11069,6 +11553,8 @@ snapshots: iterator.prototype: 1.1.4 safe-array-concat: 1.1.3 + es-module-lexer@3.0.3: {} + es-object-atoms@1.0.0: dependencies: es-errors: 1.3.0 @@ -11476,6 +11962,8 @@ snapshots: hasown: 2.0.2 mime-types: 2.1.35 + forwarded-parse@2.1.2: {} + forwarded@0.2.0: {} fraction.js@4.3.7: {} @@ -11667,7 +12155,7 @@ snapshots: google-gax@4.3.9: dependencies: - '@grpc/grpc-js': 1.11.1 + '@grpc/grpc-js': 1.14.5 '@grpc/proto-loader': 0.7.13 '@types/long': 4.0.2 abort-controller: 3.0.0 @@ -11883,6 +12371,12 @@ snapshots: import-from@4.0.0: {} + import-in-the-middle@3.5.2: + dependencies: + cjs-module-lexer: 2.3.0 + es-module-lexer: 3.0.3 + module-details-from-path: 1.0.4 + import-local@3.2.0: dependencies: pkg-dir: 4.2.0 @@ -12688,6 +13182,8 @@ snapshots: long@5.2.3: {} + long@5.3.2: {} + loose-envify@1.4.0: dependencies: js-tokens: 4.0.0 @@ -12790,6 +13286,8 @@ snapshots: mkdirp@1.0.4: {} + module-details-from-path@1.0.4: {} + moment@2.30.1: {} motion-dom@11.18.1: @@ -12839,7 +13337,7 @@ snapshots: react: 18.3.1 react-dom: 18.3.1(react@18.3.1) - next@14.2.32(@opentelemetry/api@1.9.0)(@playwright/test@1.58.2)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + next@14.2.32(@opentelemetry/api@1.9.1)(@playwright/test@1.58.2)(react-dom@18.3.1(react@18.3.1))(react@18.3.1): dependencies: '@next/env': 14.2.32 '@swc/helpers': 0.5.5 @@ -12860,7 +13358,7 @@ snapshots: '@next/swc-win32-arm64-msvc': 14.2.32 '@next/swc-win32-ia32-msvc': 14.2.32 '@next/swc-win32-x64-msvc': 14.2.32 - '@opentelemetry/api': 1.9.0 + '@opentelemetry/api': 1.9.1 '@playwright/test': 1.58.2 transitivePeerDependencies: - '@babel/core' @@ -13373,6 +13871,20 @@ snapshots: '@types/node': 24.13.1 long: 5.2.3 + protobufjs@7.6.6: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 24.13.1 + long: 5.3.2 + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 @@ -13655,6 +14167,13 @@ snapshots: require-from-string@2.0.2: {} + require-in-the-middle@8.0.1: + dependencies: + debug: 4.4.3 + module-details-from-path: 1.0.4 + transitivePeerDependencies: + - supports-color + require-main-filename@2.0.0: {} resolve-cwd@3.0.0: @@ -14279,10 +14798,6 @@ snapshots: tarn@3.0.2: {} - tdigest@0.1.3: - dependencies: - bintrees: 1.0.2 - teeny-request@9.0.0: dependencies: http-proxy-agent: 5.0.0