From 6dfd75a777da97406fad7bffbbb26f067fe360ac Mon Sep 17 00:00:00 2001 From: Ilia Lazebnik Date: Tue, 8 Sep 2026 11:25:48 -0400 Subject: [PATCH 1/2] helm: add configurable liveness, readiness and startup probes Runner and forwarder probe blocks are rendered only when set, so the default install is byte-identical to before. Liveness is documented on /metrics rather than /healthz. /healthz reflects sink health, so a transient sink outage on a liveness probe restarts the runner while nothing is wrong with it - the reason the 2023 probes were reverted in 2b2292c5. /healthz is documented as a readiness option instead. Closes #1158 --- docs/setup-robusta/health-probes.rst | 73 +++++++++++++++++++++++++++ docs/setup-robusta/index.rst | 1 + helm/robusta/templates/forwarder.yaml | 12 +++++ helm/robusta/templates/runner.yaml | 12 +++++ helm/robusta/values.yaml | 23 +++++++++ 5 files changed, 121 insertions(+) create mode 100644 docs/setup-robusta/health-probes.rst diff --git a/docs/setup-robusta/health-probes.rst b/docs/setup-robusta/health-probes.rst new file mode 100644 index 000000000..353437dcc --- /dev/null +++ b/docs/setup-robusta/health-probes.rst @@ -0,0 +1,73 @@ +Health Probes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Robusta ships without Kubernetes health probes. If your platform requires them - for example a policy that +rejects containers with no ``livenessProbe`` - you can define them yourself on the runner and the forwarder. + +Nothing is rendered unless you set it, so the default install is unaffected. + +Available Endpoints +------------------------------------- + +The runner serves HTTP on port ``5000``: + +* ``/metrics`` - available as soon as the web server starts, independent of any external service. +* ``/healthz`` - ``200`` only when every configured sink reports healthy, ``500`` otherwise. + +The forwarder serves ``/metrics`` on port ``2112``. It has no ``/healthz``; any other path returns ``404``. +The port is bound at process start, before the watch loop syncs. + +Neither container declares named ports, so probe ports must be numeric. + +Recommended Configuration +------------------------------------- + +.. code-block:: yaml + + runner: + startupProbe: + httpGet: + path: /metrics + port: 5000 + failureThreshold: 30 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /metrics + port: 5000 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /healthz + port: 5000 + periodSeconds: 15 + + kubewatch: + livenessProbe: + httpGet: + path: /metrics + port: 2112 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /metrics + port: 2112 + periodSeconds: 15 + +Any probe field Kubernetes accepts can be used, including ``tcpSocket`` and ``exec``. + +Choosing an Endpoint +------------------------------------- + +Use ``/metrics`` for liveness. It restarts the pod only when the process is genuinely wedged. + +Avoid ``/healthz`` for liveness. Because it tracks sink health, a transient outage at Slack or another sink +would restart the runner repeatedly while nothing is wrong with the runner itself. Robusta shipped a liveness +probe on ``/healthz`` in early 2023 and removed it again for this reason. + +``/healthz`` is a reasonable readiness endpoint if you want a sink-aware signal - it takes the pod out of the +Service and blocks a rollout from proceeding, without restarting anything. + +Note that probes are not how you detect a misconfigured install. A runner that cannot reach the Kubernetes API, +or a forwarder without a usable kubeconfig, exits at startup and lands in ``CrashLoopBackOff``, which Kubernetes +already reports on its own. diff --git a/docs/setup-robusta/index.rst b/docs/setup-robusta/index.rst index 6c84d2080..b5c1abc7a 100644 --- a/docs/setup-robusta/index.rst +++ b/docs/setup-robusta/index.rst @@ -22,6 +22,7 @@ read-only-service-account rbac-namespace-scoping node-selector + health-probes proxies privacy-and-security installation-faq diff --git a/helm/robusta/templates/forwarder.yaml b/helm/robusta/templates/forwarder.yaml index 99c7d577e..e0ed21f92 100644 --- a/helm/robusta/templates/forwarder.yaml +++ b/helm/robusta/templates/forwarder.yaml @@ -67,6 +67,18 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} + {{- with .Values.kubewatch.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.kubewatch.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.kubewatch.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} resources: requests: cpu: {{ .Values.kubewatch.resources.requests.cpu }} diff --git a/helm/robusta/templates/runner.yaml b/helm/robusta/templates/runner.yaml index f7d9bd177..23c96c6af 100644 --- a/helm/robusta/templates/runner.yaml +++ b/helm/robusta/templates/runner.yaml @@ -197,6 +197,18 @@ spec: {{- with .Values.runner.extraVolumeMounts }} {{- toYaml . | nindent 10 }} {{- end }} + {{- with .Values.runner.startupProbe }} + startupProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.runner.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.runner.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 10 }} + {{- end }} lifecycle: preStop: exec: diff --git a/helm/robusta/values.yaml b/helm/robusta/values.yaml index 485b3ab33..1d2f7d010 100644 --- a/helm/robusta/values.yaml +++ b/helm/robusta/values.yaml @@ -651,6 +651,17 @@ kubewatch: memory: 512Mi limits: cpu: ~ + # health probes, rendered only when set. ports must be numeric - the container declares no named ports + # /metrics is served as soon as the process is up, before the watch loop syncs + # e.g. + # livenessProbe: + # httpGet: + # path: /metrics + # port: 2112 + # periodSeconds: 30 + startupProbe: {} + livenessProbe: {} + readinessProbe: {} additional_env_vars: - name: ADVANCED_FILTERS value: "true" @@ -741,6 +752,18 @@ runner: memory: 1024Mi limits: cpu: ~ + # health probes, rendered only when set. ports must be numeric - the container declares no named ports + # /metrics is up whenever the web server is; /healthz additionally reflects sink health, so it + # suits readiness but will restart the pod on a transient sink outage if used for liveness + # e.g. + # livenessProbe: + # httpGet: + # path: /metrics + # port: 5000 + # periodSeconds: 30 + startupProbe: {} + livenessProbe: {} + readinessProbe: {} additional_env_vars: [] additional_env_froms: [] customCRD: [] From 8b0b14984ee7b3b862cea5275076346f686b478a Mon Sep 17 00:00:00 2001 From: Ilia Lazebnik Date: Tue, 8 Sep 2026 12:14:53 -0400 Subject: [PATCH 2/2] helm: name the runner and forwarder container ports Declares http/5000 on the runner and metrics/2112 on the forwarder so probes can reference the port by name, and documents why that is preferable. Service targetPort stays numeric on purpose. A named targetPort resolves per pod, so upgrading from a chart version whose pods had no named port leaves the old pod Ready with its endpoint published with no port at all - verified on kind, the EndpointSlice comes back with ports: null - and the Service black-holes traffic until the rollout completes. --- docs/setup-robusta/health-probes.rst | 46 +++++++++++++++++++-------- helm/robusta/templates/forwarder.yaml | 4 +++ helm/robusta/templates/runner.yaml | 4 +++ helm/robusta/values.yaml | 8 ++--- 4 files changed, 44 insertions(+), 18 deletions(-) diff --git a/docs/setup-robusta/health-probes.rst b/docs/setup-robusta/health-probes.rst index 353437dcc..ef29902bc 100644 --- a/docs/setup-robusta/health-probes.rst +++ b/docs/setup-robusta/health-probes.rst @@ -9,15 +9,13 @@ Nothing is rendered unless you set it, so the default install is unaffected. Available Endpoints ------------------------------------- -The runner serves HTTP on port ``5000``: +The runner serves HTTP on port ``5000``, named ``http``: * ``/metrics`` - available as soon as the web server starts, independent of any external service. * ``/healthz`` - ``200`` only when every configured sink reports healthy, ``500`` otherwise. -The forwarder serves ``/metrics`` on port ``2112``. It has no ``/healthz``; any other path returns ``404``. -The port is bound at process start, before the watch loop syncs. - -Neither container declares named ports, so probe ports must be numeric. +The forwarder serves ``/metrics`` on port ``2112``, named ``metrics``. It has no ``/healthz``; any other path +returns ``404``. The port is bound at process start, before the watch loop syncs. Recommended Configuration ------------------------------------- @@ -28,30 +26,25 @@ Recommended Configuration startupProbe: httpGet: path: /metrics - port: 5000 + port: http failureThreshold: 30 periodSeconds: 10 livenessProbe: httpGet: path: /metrics - port: 5000 + port: http periodSeconds: 30 readinessProbe: httpGet: path: /healthz - port: 5000 + port: http periodSeconds: 15 kubewatch: livenessProbe: httpGet: path: /metrics - port: 2112 - periodSeconds: 30 - readinessProbe: - httpGet: - path: /metrics - port: 2112 + port: metrics periodSeconds: 15 Any probe field Kubernetes accepts can be used, including ``tcpSocket`` and ``exec``. @@ -71,3 +64,28 @@ Service and blocks a rollout from proceeding, without restarting anything. Note that probes are not how you detect a misconfigured install. A runner that cannot reach the Kubernetes API, or a forwarder without a usable kubeconfig, exits at startup and lands in ``CrashLoopBackOff``, which Kubernetes already reports on its own. + +Why Reference Ports By Name +------------------------------------- + +Both containers declare their port with a name, and the examples above use the name rather than ``5000`` or +``2112``. Prefer the name in anything you write yourself: + +* **The number stops being repeated.** A probe that says ``port: http`` keeps working if the listen port + changes. A probe hardcoding ``5000`` has to be found and updated, and silently probes the wrong port until + someone notices. +* **It says what the port is for.** ``port: http`` is readable in a review; ``port: 5000`` needs a lookup. +* **Other Kubernetes objects resolve names too.** NetworkPolicy rules and Prometheus Operator scrape configs + accept a container port name and resolve it per pod, so one rename does not have to be chased across objects. +* **A typo says so.** A probe naming a port no container declares reports + ``port "..." not found``. A probe with the wrong number reports a generic connection refused, which looks + the same as an application that is genuinely down. + +Port names are more constrained than Service port names: at most 15 characters, lowercase alphanumeric and +dashes, and unique within the pod. + +One place to keep using the number is a Service ``targetPort``, which is why both Services still say +``targetPort: 5000`` and ``targetPort: 2112``. A numeric ``targetPort`` resolves whether or not the pod +declares the name, while a named one resolves per pod - so during an upgrade from a chart version whose pods +had no named port, pods stay ``Ready`` but their endpoints are published with no port at all, and the Service +black-holes traffic until the rollout finishes. diff --git a/helm/robusta/templates/forwarder.yaml b/helm/robusta/templates/forwarder.yaml index e0ed21f92..d546be6a7 100644 --- a/helm/robusta/templates/forwarder.yaml +++ b/helm/robusta/templates/forwarder.yaml @@ -49,6 +49,10 @@ spec: image: {{ .Values.image.registry }}/{{ .Values.kubewatch.imageName }} {{- end }} imagePullPolicy: {{ .Values.kubewatch.imagePullPolicy }} + ports: + - name: metrics + containerPort: 2112 + protocol: TCP env: - name: KW_CONFIG value: /config diff --git a/helm/robusta/templates/runner.yaml b/helm/robusta/templates/runner.yaml index 23c96c6af..8638f380a 100644 --- a/helm/robusta/templates/runner.yaml +++ b/helm/robusta/templates/runner.yaml @@ -104,6 +104,10 @@ spec: {{- toYaml . | nindent 12 }} {{- end }} {{- end }} + ports: + - name: http + containerPort: 5000 + protocol: TCP env: - name: PLAYBOOKS_CONFIG_FILE_PATH value: /etc/robusta/config/active_playbooks.yaml diff --git a/helm/robusta/values.yaml b/helm/robusta/values.yaml index 1d2f7d010..a7d0cf176 100644 --- a/helm/robusta/values.yaml +++ b/helm/robusta/values.yaml @@ -651,13 +651,13 @@ kubewatch: memory: 512Mi limits: cpu: ~ - # health probes, rendered only when set. ports must be numeric - the container declares no named ports + # health probes, rendered only when set. reference the container port by name # /metrics is served as soon as the process is up, before the watch loop syncs # e.g. # livenessProbe: # httpGet: # path: /metrics - # port: 2112 + # port: metrics # periodSeconds: 30 startupProbe: {} livenessProbe: {} @@ -752,14 +752,14 @@ runner: memory: 1024Mi limits: cpu: ~ - # health probes, rendered only when set. ports must be numeric - the container declares no named ports + # health probes, rendered only when set. reference the container port by name # /metrics is up whenever the web server is; /healthz additionally reflects sink health, so it # suits readiness but will restart the pod on a transient sink outage if used for liveness # e.g. # livenessProbe: # httpGet: # path: /metrics - # port: 5000 + # port: http # periodSeconds: 30 startupProbe: {} livenessProbe: {}