feat: telemetry using otel pattern

.github/labeler.yml

@@ -1,5 +1,9 @@
-'package: wallet/backend': packages/wallet/backend/**/*
-'package: wallet/frontend': packages/wallet/frontend/**/*
+'package: wallet/backend':
+  - packages/wallet/backend/**/*
+  - packages/shared/telemetry/**/*
+'package: wallet/frontend':
+  - packages/wallet/frontend/**/*
+  - packages/shared/telemetry/**/*
 'package: boutique/backend': packages/boutique/backend/**/*
 'package: boutique/frontend': packages/boutique/frontend/**/*
 

.github/workflows/ci.yml

@@ -73,7 +73,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
       - uses: ./.github/workflows/setup
-      - run: pnpm wallet:backend build && pnpm wallet:backend test --detectOpenHandles --forceExit
+      - run: pnpm wallet:backend build && pnpm --filter @shared/telemetry test && pnpm wallet:backend test --detectOpenHandles --forceExit
 
   test-boutique-backend:
     name: BOUTIQUE - Test backend

README.md

@@ -156,6 +156,7 @@ Upon executing the above commands the following will be available:
 - [https://mockgatehub.testnet.test](https://mockgatehub.testnet.test) - Mock GateHub service used for local funding and related sandbox flows.
 - [https://rafiki-frontend.testnet.test](https://rafiki-frontend.testnet.test) - Rafiki frontend UI.
 - [https://rafiki-backend.testnet.test](https://rafiki-backend.testnet.test) - Rafiki backend service.
+- [https://grafana.testnet.test](https://grafana.testnet.test) - Grafana with the wallet's metrics and traces. Opt-in: `pnpm dev:observability`. See `pnpm local:help`.
 
 ### Local Playground on Rhyza
 

helm/testnet-wallet/README.md

@@ -134,6 +134,55 @@ merging it, so the environment's values file must also write out, in full:
 `env/cards-playground/wallet/testnet-wallet.yaml` in `testnet-deploy` is the
 worked example.
 
+### OpenTelemetry
+
+The backend and the frontend's Node.js server can push traces and metrics over
+OTLP/gRPC. The chart runs one OpenTelemetry collector per release, built the
+same way as the collector in the merchant and issuer charts. The browser sends
+nothing.
+
+Telemetry is off by default. One block turns it on for both components:
+
+```yaml
+config:
+  telemetry:
+    enabled: true
+    serviceEnv: cards-playground
+```
+
+With `enabled: true` the chart:
+
+- adds the `TELEMETRY_*` and `SERVICE_*` keys to both ConfigMaps. They are not
+  contentMap entries; `templates/_helpers.tpl` renders them. See
+  `packages/shared/telemetry` for each key.
+- points both components at `<release>-otel-collector:4317`, unless
+  `config.telemetry.endpoint` names another collector. The render fails if
+  `otelCollector.enabled` is false and no endpoint is set.
+- sets `SERVICE_VERSION` to the component's image tag.
+
+Each pod gets `POD_NAME`, `POD_NAMESPACE` and `OTEL_RESOURCE_ATTRIBUTES` through
+`extraEnv`, not `env`, so an environment that rewrites `env` keeps them.
+`service.instance.id` keeps the series of two replicas apart.
+
+The collector:
+
+- exposes the metrics on a Prometheus endpoint (`metrics`, port 8889) and its
+  own metrics on `telemetry` (port 8888). A ServiceMonitor scrapes both. The
+  chart renders the ServiceMonitor only where the Prometheus Operator CRDs
+  exist; `helm template` needs `--api-versions
+monitoring.coreos.com/v1/ServiceMonitor` to show it.
+- forwards traces to `otelCollector.traces.endpoint`. Empty drops them. On
+  ilf-1 the Tempo address is `core-tempo.monitoring.svc.cluster.local:4317`,
+  and the namespace needs `network.allowTracingEgress` in its
+  deploy-environment.
+
+`otelCollector.config` replaces the generated collector config wholesale.
+
+The metrics follow the OpenTelemetry semantic conventions. Examples:
+`http_server_request_duration_seconds` (with `http_route` on the backend),
+`http_client_request_duration_seconds`, `db_client_operation_duration_seconds`
+and `nodejs_eventloop_*`.
+
 ## Image Tags
 
 By default both deployments use the chart's `appVersion` as the image tag. To pin a specific tag for one image:

helm/testnet-wallet/templates/_helpers.tpl

@@ -0,0 +1,194 @@
+{{/*
+testnet-wallet.otelCollectorName is the collector's Deployment and ConfigMap
+name.
+*/}}
+{{- define "testnet-wallet.otelCollectorName" -}}
+{{- printf "%s-%s" (include "common.fullname" .) .Values.otelCollector.name -}}
+{{- end -}}
+
+{{/*
+testnet-wallet.otelCollectorServiceName is the collector Service's name, the
+host the backend and the frontend dial. common.svc names it after
+otelCollector.service.name, which need not match otelCollector.name.
+*/}}
+{{- define "testnet-wallet.otelCollectorServiceName" -}}
+{{- printf "%s-%s" (include "common.fullname" .) .Values.otelCollector.service.name -}}
+{{- end -}}
+
+{{/*
+testnet-wallet.otelCollectorContainerPort is the collector container port with
+the given name. The collector config listens on these ports, so the container
+ports are the one place to move a listener. Fails the render if the port is
+missing.
+
+Usage: include "testnet-wallet.otelCollectorContainerPort" (list . "otlp-grpc")
+*/}}
+{{- define "testnet-wallet.otelCollectorContainerPort" -}}
+{{- $top := index . 0 -}}
+{{- $name := index . 1 -}}
+{{- $port := 0 -}}
+{{- range $top.Values.otelCollector.ports -}}
+{{- if eq .name $name -}}
+{{- $port = .containerPort -}}
+{{- end -}}
+{{- end -}}
+{{- if eq (int $port) 0 -}}
+{{- fail (printf "otelCollector.ports must contain a port named %s — the collector config listens on it" $name) -}}
+{{- end -}}
+{{- $port -}}
+{{- end -}}
+
+{{/*
+testnet-wallet.otelCollectorOTLPPort is the collector Service's OTLP/gRPC port,
+read off the port named `otlp-grpc`, so that moving it moves the endpoint the
+backend and the frontend dial.
+*/}}
+{{- define "testnet-wallet.otelCollectorOTLPPort" -}}
+{{- $port := 0 -}}
+{{- range .Values.otelCollector.service.ports -}}
+{{- if eq .name "otlp-grpc" -}}
+{{- $port = .port -}}
+{{- end -}}
+{{- end -}}
+{{- if eq (int $port) 0 -}}
+{{- fail "otelCollector.service.ports must contain a port named otlp-grpc — the backend and the frontend dial the collector on it" -}}
+{{- end -}}
+{{- $port -}}
+{{- end -}}
+
+{{/*
+testnet-wallet.telemetryEndpoint is the OTLP/gRPC target both components send
+to: config.telemetry.endpoint when set, otherwise this release's collector.
+*/}}
+{{- define "testnet-wallet.telemetryEndpoint" -}}
+{{- $telemetry := .Values.config.telemetry -}}
+{{- if $telemetry.endpoint -}}
+{{- $telemetry.endpoint -}}
+{{- else -}}
+{{- if not .Values.otelCollector.enabled -}}
+{{- fail "config.telemetry.enabled is true, but otelCollector.enabled is false and no config.telemetry.endpoint is set — enable the collector, or point the endpoint at one outside this release" -}}
+{{- end -}}
+{{- printf "%s:%v" (include "testnet-wallet.otelCollectorServiceName" .) (include "testnet-wallet.otelCollectorOTLPPort" .) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+testnet-wallet.telemetryData renders the TELEMETRY_* and SERVICE_* ConfigMap
+keys for one component, "backend" or "frontend". Both read the same
+config.telemetry block, so the two halves of the wallet are always on or off
+together and send to the same collector. With telemetry off it renders
+nothing, and the applications keep their own default, which is off.
+
+SERVICE_VERSION is the component's image tag, resolved the way the common
+chart resolves it, so a trace can be read against the build that produced it.
+*/}}
+{{- define "testnet-wallet.telemetryData" -}}
+{{- $top := index . 0 -}}
+{{- $component := index . 1 -}}
+{{- $telemetry := $top.Values.config.telemetry -}}
+{{- if $telemetry.enabled -}}
+{{- $deployment := index $top.Values.deployments $component -}}
+{{- $image := $deployment.image | default dict -}}
+{{- $imageOverride := $top.Values.imageOverride | default dict -}}
+{{- $version := coalesce $telemetry.serviceVersion $image.tag $top.Values.versionOverride $imageOverride.tag $top.Chart.AppVersion -}}
+TELEMETRY_ENABLED: "true"
+TELEMETRY_ENDPOINT: {{ include "testnet-wallet.telemetryEndpoint" $top | quote }}
+TELEMETRY_INSECURE: {{ $telemetry.insecure | toString | quote }}
+TELEMETRY_TRACES_ENABLED: {{ $telemetry.traces.enabled | toString | quote }}
+TELEMETRY_TRACES_SAMPLE_RATIO: {{ $telemetry.traces.sampleRatio | toString | quote }}
+TELEMETRY_METRICS_ENABLED: {{ $telemetry.metrics.enabled | toString | quote }}
+TELEMETRY_METRICS_INTERVAL: {{ $telemetry.metrics.interval | quote }}
+SERVICE_NAME: {{ index $telemetry.serviceNames $component | quote }}
+SERVICE_VERSION: {{ $version | quote }}
+SERVICE_ENV: {{ $telemetry.serviceEnv | quote }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+testnet-wallet.otelCollectorConfig renders the collector's own config file. Set
+otelCollector.config to replace it wholesale. The generated pipeline is the one
+the merchant and issuer charts use:
+
+  OTLP in (gRPC and HTTP)
+    -> metrics out on the Prometheus exporter, which Prometheus scrapes
+       through the ServiceMonitor
+    -> traces out over OTLP to otelCollector.traces.endpoint
+
+Three details it does not show on its face:
+
+  * With no traces endpoint the pipeline exports to `nop`. A pipeline must
+    have an exporter, and dropping the pipeline would unregister the OTLP
+    trace service, so the wallet would log a failed export on every batch.
+
+  * memory_limiter runs first, shedding load before the batcher can queue
+    enough to push the heap past the container memory limit. Its percentages
+    are of that limit, so raise the limit before raising them.
+
+  * resource_to_telemetry_conversion turns resource attributes into labels:
+    service_name, service_version, service_instance_id,
+    deployment_environment_name. One collector serves the backend and the
+    frontend, and every replica of each, so without service_instance_id
+    replicas emit identical label sets and overwrite each other's samples.
+*/}}
+{{- define "testnet-wallet.otelCollectorConfig" -}}
+{{- if .Values.otelCollector.config -}}
+{{- toYaml .Values.otelCollector.config -}}
+{{- else -}}
+{{- $c := .Values.otelCollector -}}
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:{{ include "testnet-wallet.otelCollectorContainerPort" (list . "health") }}
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:{{ include "testnet-wallet.otelCollectorContainerPort" (list . "otlp-grpc") }}
+      http:
+        endpoint: 0.0.0.0:{{ include "testnet-wallet.otelCollectorContainerPort" (list . "otlp-http") }}
+
+processors:
+  memory_limiter:
+    check_interval: 1s
+    limit_percentage: {{ $c.memoryLimiter.limitPercentage }}
+    spike_limit_percentage: {{ $c.memoryLimiter.spikeLimitPercentage }}
+  batch:
+    timeout: {{ $c.batch.timeout }}
+    send_batch_size: {{ $c.batch.sendBatchSize }}
+
+exporters:
+  prometheus:
+    endpoint: 0.0.0.0:{{ include "testnet-wallet.otelCollectorContainerPort" (list . "metrics") }}
+    resource_to_telemetry_conversion:
+      enabled: true
+    metric_expiration: {{ $c.metricExpiration }}
+{{- if $c.traces.endpoint }}
+  otlp/traces:
+    endpoint: {{ $c.traces.endpoint | quote }}
+    tls:
+      insecure: {{ $c.traces.insecure }}
+{{- else }}
+  nop: {}
+{{- end }}
+
+service:
+  extensions: [health_check]
+  telemetry:
+    metrics:
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: 0.0.0.0
+                port: {{ include "testnet-wallet.otelCollectorContainerPort" (list . "telemetry") }}
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: [memory_limiter, batch]
+      exporters: [prometheus]
+    traces:
+      receivers: [otlp]
+      processors: [memory_limiter, batch]
+      exporters: [{{ if $c.traces.endpoint }}otlp/traces{{ else }}nop{{ end }}]
+{{- end -}}
+{{- end -}}

helm/testnet-wallet/templates/configMap.backend.yaml

@@ -7,20 +7,29 @@ entry would emit `ATALLA_HOST: ""` in every environment that leaves it unset and
 stop the backend from starting. Here the whole block appears only when
 config.backend.hsm.enabled is true.
 
+The override also adds the TELEMETRY_* and SERVICE_* keys when
+config.telemetry.enabled is true. See testnet-wallet.telemetryData.
+
 common.utils.merge merges maps deeply, so these keys join the contentMap's
 `data` rather than replacing it.
 */}}
 {{- define "testnet-wallet.backend.ConfigMap" -}}
 {{- $top := first . }}
 {{- $hsm := $top.Values.config.backend.hsm }}
+{{- $telemetry := include "testnet-wallet.telemetryData" (list $top "backend") }}
+{{- if or $hsm.enabled $telemetry }}
+data:
+{{- end }}
+{{- with $telemetry }}
+  {{- . | nindent 2 }}
+{{- end }}
 {{- if $hsm.enabled }}
 {{- if not $hsm.atalla.host }}
 {{- fail "config.backend.hsm.atalla.host is required when config.backend.hsm.enabled is true" }}
 {{- end }}
 {{- if not $hsm.atalla.port }}
 {{- fail "config.backend.hsm.atalla.port is required when config.backend.hsm.enabled is true" }}
 {{- end }}
-data:
   HSM_ENABLED: "true"
   HSM_SERVER_ROLE: {{ $hsm.serverRole | quote }}
   {{- /* ISSUER_GRPC_HOST and MERCHANT_GRPC_HOST are omitted: the backend falls

helm/testnet-wallet/templates/configMap.frontend.yaml

@@ -1,3 +1,12 @@
+{{/*
+The override adds the TELEMETRY_* and SERVICE_* keys when
+config.telemetry.enabled is true. See testnet-wallet.telemetryData.
+*/}}
 {{- define "testnet-wallet.frontend.ConfigMap" -}}
+{{- $top := first . }}
+{{- with include "testnet-wallet.telemetryData" (list $top "frontend") }}
+data:
+  {{- . | nindent 2 }}
+{{- end }}
 {{- end }}
 {{- include "common.configMapper" (list . .Values.configMaps.frontend "testnet-wallet.frontend.ConfigMap") }}

helm/testnet-wallet/templates/configMap.otel-collector.yaml

@@ -0,0 +1,10 @@
+{{- if .Values.otelCollector.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "testnet-wallet.otelCollectorName" . }}
+  {{- include "common.metadata" (list . .Values.otelCollector) | nindent 2 }}
+data:
+  config.yaml: |
+    {{- include "testnet-wallet.otelCollectorConfig" . | nindent 4 }}
+{{- end }}

helm/testnet-wallet/templates/deployment.otel-collector.yaml

@@ -0,0 +1,19 @@
+{{- define "testnet-wallet.otelCollector.deployment" -}}
+{{- $top := first . }}
+{{- $values := $top.Values.otelCollector }}
+{{- $configHash := include "testnet-wallet.otelCollectorConfig" $top | sha256sum }}
+{{- $builtinVolumes := list
+      (dict "name" "otel-collector-config" "configMap" (dict "name" (include "testnet-wallet.otelCollectorName" $top)))
+}}
+spec:
+  template:
+    metadata:
+      annotations:
+        ilf-otel-collector-config-sha256: {{ $configHash | quote }}
+    spec:
+      volumes:
+        {{- toYaml (concat $builtinVolumes ($values.volumes | default list)) | nindent 8 }}
+{{- end }}
+{{- if .Values.otelCollector.enabled }}
+{{- include "common.deployment" (list . .Values.otelCollector .Values.otelCollector.autoscaling .Values.otelCollector.serviceAccount "testnet-wallet.otelCollector.deployment") }}
+{{- end }}

helm/testnet-wallet/templates/service.otel-collector.yaml

@@ -0,0 +1,5 @@
+{{- define "testnet-wallet.otelCollector.service" -}}
+{{- end }}
+{{- if .Values.otelCollector.enabled }}
+{{- include "common.svc" (list . .Values.otelCollector.service .Values.otelCollector "testnet-wallet.otelCollector.service") }}
+{{- end }}

helm/testnet-wallet/templates/serviceMonitor.otel-collector.yaml

@@ -0,0 +1,28 @@
+{{- /*
+Rendered only where the Prometheus Operator CRDs exist, so the chart still
+installs on a cluster without them. Argo CD and `helm install` both pass the
+cluster's API versions; `helm template` needs `--api-versions
+monitoring.coreos.com/v1/ServiceMonitor`.
+*/}}
+{{- if and .Values.otelCollector.enabled .Values.otelCollector.serviceMonitor.enabled (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1/ServiceMonitor") }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "testnet-wallet.otelCollectorName" . }}
+  {{- include "common.metadata" (list . .Values.otelCollector) | nindent 2 }}
+spec:
+  # Selects the Service, which common.svc labels after otelCollector.service.
+  selector:
+    matchLabels:
+      {{- include "common.selectorLabels" (list . .Values.otelCollector.service) | nindent 6 }}
+  endpoints:
+    # The backend's and the frontend's metrics, converted from OTLP. Resource
+    # attributes arrive as labels, so each series names the pod it came from.
+    - port: metrics
+      path: /metrics
+      interval: {{ .Values.otelCollector.serviceMonitor.interval }}
+    # The collector's own metrics, where a dropped batch shows up.
+    - port: telemetry
+      path: /metrics
+      interval: {{ .Values.otelCollector.serviceMonitor.interval }}
+{{- end }}