diff --git a/Makefile b/Makefile index b817a7d9..919edeb1 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ stop: REGISTRY ?= localhost:5000 WORKERS ?= 2 ENV_FILE ?= .env -SWARM_VARS = REGISTRY CERTBOT_DOMAIN CERTBOT_EMAIL NFS_SERVER_IP NFS_BASE_PATH WORKER_CPU_LIMIT WORKER_MEMORY_LIMIT WORKER_CPU_RESERVATION WORKER_MEMORY_RESERVATION +SWARM_VARS = REGISTRY CERTBOT_DOMAIN CERTBOT_EMAIL NFS_SERVER_IP TEXTFILE_DIR NFS_BASE_PATH WORKER_CPU_LIMIT WORKER_MEMORY_LIMIT WORKER_CPU_RESERVATION WORKER_MEMORY_RESERVATION SWARM_ENV = ENV_FILE="$(ENV_FILE)" $(foreach v,$(SWARM_VARS),$(v)="$(shell grep '^$(v)=' $(ENV_FILE) | head -1 | cut -d= -f2-)") start-swarm: diff --git a/backend/apps/ifc_validation/tasks/check_programs.py b/backend/apps/ifc_validation/tasks/check_programs.py index 4ffc9048..234c4775 100644 --- a/backend/apps/ifc_validation/tasks/check_programs.py +++ b/backend/apps/ifc_validation/tasks/check_programs.py @@ -89,6 +89,12 @@ def run_subprocess_wait(*popen_args, check=False, **popen_kwargs): raise retcode = process.returncode stdout, stderr = "".join(out_chunks), "".join(err_chunks) + if retcode is not None and retcode < 0: + # killed by a signal; -9 (SIGKILL) usually means the container hit its + # memory limit and the kernel OOM-killed this subprocess. Without this + # line such deaths are indistinguishable from ordinary failures. + logger.warning(f"Subprocess was killed by signal {-retcode} (likely OOM if 9); " + f"peak RSS before death: {peak_rss_kb} kB") if check and retcode != 0: raise subprocess.CalledProcessError(retcode, popen_args[0], output=stdout, stderr=stderr) return proc_output(retcode, stdout, stderr, popen_args[0] if popen_args else [], peak_rss_kb, min_mem_available_kb) diff --git a/backend/core/settings.py b/backend/core/settings.py index b6e34ea0..b49f278c 100644 --- a/backend/core/settings.py +++ b/backend/core/settings.py @@ -67,6 +67,7 @@ "drf_spectacular", # OpenAPI/Swagger "drf_spectacular_sidecar", # required for Django collectstatic discovery "explorer", # Django SQL Explorer + "django_prometheus", # HTTP metrics for Prometheus (/metrics, internal only) "django_celery_results", # Celery result backend "django_celery_beat", # Celery scheduled tasks @@ -88,6 +89,8 @@ ) MIDDLEWARE = [ + # must be FIRST so the request timer starts before all other middleware + "django_prometheus.middleware.PrometheusBeforeMiddleware", #"django.middleware.gzip.GZipMiddleware", # WE DO THIS IN NGINX "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", @@ -97,6 +100,8 @@ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", + # must be LAST so the response is timed after all other middleware + "django_prometheus.middleware.PrometheusAfterMiddleware", ] if DEVELOPMENT or PREVIEW: diff --git a/backend/core/urls.py b/backend/core/urls.py index 0d16ce9a..95b963dd 100644 --- a/backend/core/urls.py +++ b/backend/core/urls.py @@ -45,6 +45,11 @@ def redirect_to_v1(request, resource, suffix=None): urlpatterns = [ + # Prometheus scrape endpoint (/metrics). Internal only: nginx serves the React + # app at / and never proxies this path, so it is reachable solely on the + # overlay network (backend:8000). + path('', include('django_prometheus.urls')), + # Django Admin path("admin/", admin.site.urls), diff --git a/backend/gunicorn.conf.py b/backend/gunicorn.conf.py new file mode 100644 index 00000000..cc3836b2 --- /dev/null +++ b/backend/gunicorn.conf.py @@ -0,0 +1,11 @@ +"""Gunicorn hooks for prometheus_client multiprocess mode. + +With multiple workers each process keeps its own counters in +PROMETHEUS_MULTIPROC_DIR; this hook cleans up when a worker dies, otherwise +the directory slowly fills with files of dead pids. +""" +from prometheus_client import multiprocess + + +def child_exit(server, worker): + multiprocess.mark_process_dead(worker.pid) diff --git a/backend/requirements.txt b/backend/requirements.txt index 0bb367ef..04b8bb62 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -47,6 +47,7 @@ python-ranges==1.2.2 pyproj==3.7.1 python-dateutil==2.9.0.post0 filetype==1.2.0 +django-prometheus==2.5.0 # dev django-debug-toolbar==6.0.0 diff --git a/docker-compose.swarm.nodb.yml b/docker-compose.swarm.nodb.yml index 637275a1..0f03143f 100644 --- a/docker-compose.swarm.nodb.yml +++ b/docker-compose.swarm.nodb.yml @@ -235,6 +235,62 @@ services: condition: on-failure delay: 5s + # Serves batch metrics that cron jobs on the manager write as .prom files + # (per-rule gherkin costs, harvested peak-RSS). Separate manager-only service: + # mounting the directory into the global node_exporter breaks nodes that do + # not have the path (reject-loop, seen on DEV 30 Jul). + textfile_exporter: + image: prom/node-exporter:v1.12.1 + command: + - '--collector.disable-defaults' + - '--collector.textfile' + - '--collector.textfile.directory=/textfile' + volumes: + - ${TEXTFILE_DIR}:/textfile:ro + networks: + - validate + deploy: + replicas: 1 + placement: + constraints: [node.role == manager] + restart_policy: + condition: on-failure + delay: 5s + + # Per-container memory/CPU (cAdvisor). Closes the measuring blind spot behind + # the false-quarantine incident: long-lived daemons inside containers (clamd) + # were invisible to both host-level and per-subprocess measurements. + cadvisor: + # v0.55+: v0.52 chokes on Docker 29's containerd image store (rw-layer + # lookup fails -> ALL metrics for those containers silently dropped). + image: gcr.io/cadvisor/cadvisor:v0.55.1 + command: + - '--docker_only=true' + - '--housekeeping_interval=15s' + - '--store_container_labels=false' + # keep only this label: needed to group metrics per swarm service + - '--whitelisted_container_labels=com.docker.swarm.service.name' + # disk/diskIO also disabled: the rw-layer lookup they require breaks + # on Docker 29's containerd image store (no overlayfs/layerdb path), + # which silently drops ALL metrics for those containers. + - '--disable_metrics=percpu,sched,tcp,udp,advtcp,process,hugetlb,referenced_memory,cpu_topology,resctrl,disk,diskIO' + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + - /dev/disk/:/dev/disk:ro + networks: + - validate + deploy: + mode: global + resources: + limits: + memory: 512M + restart_policy: + condition: on-failure + delay: 5s + celery_exporter: image: danihodovic/celery-exporter:0.12.2 environment: diff --git a/docker/backend/server-entrypoint.sh b/docker/backend/server-entrypoint.sh index 48a16011..31e83f94 100644 --- a/docker/backend/server-entrypoint.sh +++ b/docker/backend/server-entrypoint.sh @@ -28,4 +28,9 @@ DJANGO_GUNICORN_THREADS_PER_WORKER=${DJANGO_GUNICORN_THREADS_PER_WORKER:-4} # de echo "Number of worker processes: $DJANGO_GUNICORN_WORKERS" echo "Number of threads per worker: $DJANGO_GUNICORN_THREADS_PER_WORKER" -gunicorn core.wsgi --bind 0.0.0.0:8000 --workers $DJANGO_GUNICORN_WORKERS --threads $DJANGO_GUNICORN_THREADS_PER_WORKER --worker-class gevent --worker-tmp-dir /dev/shm --timeout 60 --keep-alive 60 +# prometheus_client multiprocess mode: one shared dir for all gunicorn workers. +# Must be wiped on boot or counters from previous runs leak into the totals. +export PROMETHEUS_MULTIPROC_DIR=/dev/shm/prometheus_metrics +rm -rf "$PROMETHEUS_MULTIPROC_DIR" && mkdir -p "$PROMETHEUS_MULTIPROC_DIR" + +gunicorn core.wsgi -c /app/backend/gunicorn.conf.py --bind 0.0.0.0:8000 --workers $DJANGO_GUNICORN_WORKERS --threads $DJANGO_GUNICORN_THREADS_PER_WORKER --worker-class gevent --worker-tmp-dir /dev/shm --timeout 60 --keep-alive 60 diff --git a/docker/grafana/dashboards/vs-platform-usage.json b/docker/grafana/dashboards/vs-platform-usage.json index 79712838..b9f24b8d 100644 --- a/docker/grafana/dashboards/vs-platform-usage.json +++ b/docker/grafana/dashboards/vs-platform-usage.json @@ -16,11 +16,11 @@ "targets": [ { "format": "time_series", - "rawSql": "SELECT created::date AS time, COUNT(*)::float AS validations FROM ifc_validation_request WHERE created > NOW() - INTERVAL '30 days' GROUP BY 1 ORDER BY 1", + "rawSql": "SELECT created::date AS time, COUNT(*)::float AS validations FROM ifc_validation_request WHERE $__timeFilter(created) GROUP BY 1 ORDER BY 1", "refId": "A" } ], - "title": "Validation requests per day (30d)", + "title": "Validation requests per day", "type": "timeseries", "description": "Number of files submitted per day (fixed 30-day window, independent of the time range above). Includes requests that were soft-deleted later." }, @@ -39,13 +39,13 @@ "targets": [ { "format": "table", - "rawSql": "SELECT type, COUNT(*) AS n, ROUND(percentile_cont(0.5) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM ended-started))::numeric,1) AS p50_s, ROUND(percentile_cont(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM ended-started))::numeric,1) AS p95_s FROM ifc_validation_task WHERE ended IS NOT NULL AND started IS NOT NULL AND created > NOW() - INTERVAL '90 days' GROUP BY type ORDER BY p95_s DESC", + "rawSql": "SELECT type, COUNT(*) AS n, ROUND(percentile_cont(0.5) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM ended-started))::numeric,1) AS p50_s, ROUND(percentile_cont(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM ended-started))::numeric,1) AS p95_s FROM ifc_validation_task WHERE ended IS NOT NULL AND started IS NOT NULL AND $__timeFilter(created) GROUP BY type ORDER BY p95_s DESC", "refId": "A" } ], - "title": "Duration per task type: p50 / p95 (90d, seconds)", + "title": "Duration per task type: p50 / p95 (seconds)", "type": "table", - "description": "Median (p50) and slow-tail (p95) duration in seconds per validation step, over 90 days. From ifc_validation_task.ended - started." + "description": "Median (p50) and slow-tail (p95) duration in seconds per validation step, over the selected time range. From ifc_validation_task.ended - started." }, { "datasource": { @@ -62,11 +62,11 @@ "targets": [ { "format": "time_series", - "rawSql": "SELECT DATE(r.created) AS time, percentile_cont(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM ft.fs - r.created)) AS queue_wait_p95_s FROM ifc_validation_request r JOIN LATERAL (SELECT MIN(t.started) AS fs FROM ifc_validation_task t WHERE t.request_id = r.id) ft ON ft.fs IS NOT NULL WHERE r.created > NOW() - INTERVAL '30 days' GROUP BY 1 ORDER BY 1", + "rawSql": "SELECT DATE(r.created) AS time, percentile_cont(0.95) WITHIN GROUP (ORDER BY EXTRACT(EPOCH FROM ft.fs - r.created)) AS queue_wait_p95_s FROM ifc_validation_request r JOIN LATERAL (SELECT MIN(t.started) AS fs FROM ifc_validation_task t WHERE t.request_id = r.id) ft ON ft.fs IS NOT NULL WHERE $__timeFilter(r.created) GROUP BY 1 ORDER BY 1", "refId": "A" } ], - "title": "Queue wait time p95 per day (s, 30d)", + "title": "Queue wait time p95 per day (s)", "type": "timeseries", "description": "Wait time between submission and the start of the first task. NOTE: on DEV, tasks are sometimes re-run manually on old requests, which inflates this to hours. Read it as a trend, not an absolute." }, @@ -92,11 +92,11 @@ "targets": [ { "format": "time_series", - "rawSql": "SELECT created::date AS time, ROUND(100.0 * SUM((status='FAILED')::int) / COUNT(*), 1) AS failure_rate FROM ifc_validation_request WHERE created > NOW() - INTERVAL '30 days' GROUP BY 1 ORDER BY 1", + "rawSql": "SELECT created::date AS time, ROUND(100.0 * SUM((status='FAILED')::int) / COUNT(*), 1) AS failure_rate FROM ifc_validation_request WHERE $__timeFilter(created) GROUP BY 1 ORDER BY 1", "refId": "A" } ], - "title": "Failure rate per day (%, 30d)", + "title": "Failure rate per day (%)", "type": "timeseries", "description": "Percentage of requests per day that ended in status FAILED." }, @@ -115,13 +115,13 @@ "targets": [ { "format": "table", - "rawSql": "SELECT EXTRACT(HOUR FROM created)::int AS hour, COUNT(*) AS requests FROM ifc_validation_request WHERE created > NOW() - INTERVAL '90 days' GROUP BY 1 ORDER BY 1", + "rawSql": "SELECT EXTRACT(HOUR FROM created)::int AS hour, COUNT(*) AS requests FROM ifc_validation_request WHERE $__timeFilter(created) GROUP BY 1 ORDER BY 1", "refId": "A" } ], - "title": "Activity by hour of day (90d)", + "title": "Activity by hour of day", "type": "barchart", - "description": "Which hour of the day the platform is used (UTC), over 90 days." + "description": "Which hour of the day the platform is used (UTC), over the selected time range." }, { "datasource": { @@ -162,7 +162,7 @@ ], "title": "Stuck requests (>1h, not finished)", "type": "stat", - "description": "Requests older than one hour that are still not finished (not COMPLETED/FAILED). Should be 0." + "description": "Requests older than one hour that are still not finished (not COMPLETED/FAILED). Should be 0. Deliberately ignores the dashboard time range: it shows the current state." }, { "datasource": { @@ -179,13 +179,13 @@ "targets": [ { "format": "table", - "rawSql": "SELECT file_name, ROUND(size/1024.0/1024.0,1) AS mb, status, EXTRACT(EPOCH FROM completed-created)::int AS duration_s, created FROM ifc_validation_request WHERE created > NOW() - INTERVAL '30 days' ORDER BY size DESC NULLS LAST LIMIT 10", + "rawSql": "SELECT file_name, ROUND(size/1024.0/1024.0,1) AS mb, status, EXTRACT(EPOCH FROM completed-created)::int AS duration_s, created FROM ifc_validation_request WHERE $__timeFilter(created) ORDER BY size DESC NULLS LAST LIMIT 10", "refId": "A" } ], - "title": "Largest files in the last 30d (top 10)", + "title": "Largest files (top 10)", "type": "table", - "description": "The ten largest files of the last 30 days. Odd-looking file names are non-Latin names exactly as stored in the database." + "description": "The ten largest files in the selected time range. Odd-looking file names are non-Latin names exactly as stored in the database." }, { "id": 8, @@ -216,14 +216,14 @@ { "refId": "A", "format": "time_series", - "rawSql": "SELECT created::date AS time, channel, COUNT(*)::float AS uploads FROM ifc_validation_request WHERE created > NOW() - INTERVAL '90 days' AND channel IS NOT NULL GROUP BY 1,2 ORDER BY 1" + "rawSql": "SELECT created::date AS time, channel, COUNT(*)::float AS uploads FROM ifc_validation_request WHERE $__timeFilter(created) AND channel IS NOT NULL GROUP BY 1,2 ORDER BY 1" } ] }, { "id": 9, "type": "table", - "title": "Top API users (90d)", + "title": "Top API users", "description": "Per account: number of uploads, total and average size. Some accounts have no email filled in, hence grouping by username.", "gridPos": { "h": 8, @@ -239,7 +239,7 @@ { "refId": "A", "format": "table", - "rawSql": "SELECT u.username, COUNT(*) AS uploads, ROUND(SUM(r.size)/1024.0/1024.0,1) AS total_mb, ROUND(AVG(r.size)/1024.0/1024.0,2) AS avg_mb, MAX(r.created)::date AS last_upload FROM ifc_validation_request r JOIN auth_user u ON u.id = r.created_by_id WHERE r.channel='API' AND r.created > NOW() - INTERVAL '90 days' GROUP BY 1 ORDER BY uploads DESC LIMIT 15" + "rawSql": "SELECT u.username, COUNT(*) AS uploads, ROUND(SUM(r.size)/1024.0/1024.0,1) AS total_mb, ROUND(AVG(r.size)/1024.0/1024.0,2) AS avg_mb, MAX(r.created)::date AS last_upload FROM ifc_validation_request r JOIN auth_user u ON u.id = r.created_by_id WHERE r.channel='API' AND $__timeFilter(r.created) GROUP BY 1 ORDER BY uploads DESC LIMIT 15" } ] }, @@ -262,9 +262,90 @@ { "refId": "A", "format": "table", - "rawSql": "SELECT CASE WHEN status_reason LIKE '%duplicate key%' THEN 'duplicate key (race on concurrent requests)' WHEN status_reason LIKE '%NUL (0x00)%' THEN 'NUL bytes in text (PostgreSQL rejects)' WHEN status_reason LIKE '%TaskContext%' THEN 'code bug: TaskContext missing proc' WHEN status_reason LIKE '%NoneType%' THEN 'code bug: NoneType has no id' WHEN status_reason IS NULL OR status_reason='' THEN '(no reason recorded)' ELSE split_part(status_reason, E'\\n', 1) END AS cause, COUNT(*) AS count, COUNT(DISTINCT type) AS task_types, MAX(created)::date AS last_seen FROM ifc_validation_task WHERE status='FAILED' GROUP BY 1 ORDER BY aantal DESC LIMIT 15" + "rawSql": "SELECT CASE WHEN status_reason LIKE '%duplicate key%' THEN 'duplicate key (race on concurrent requests)' WHEN status_reason LIKE '%NUL (0x00)%' THEN 'NUL bytes in text (PostgreSQL rejects)' WHEN status_reason LIKE '%TaskContext%' THEN 'code bug: TaskContext missing proc' WHEN status_reason LIKE '%NoneType%' THEN 'code bug: NoneType has no id' WHEN status_reason IS NULL OR status_reason='' THEN '(no reason recorded)' ELSE split_part(status_reason, E'\\n', 1) END AS cause, COUNT(*) AS count, COUNT(DISTINCT type) AS task_types, MAX(created)::date AS last_seen FROM ifc_validation_task WHERE status='FAILED' AND $__timeFilter(created) GROUP BY 1 ORDER BY 2 DESC LIMIT 15" } ] + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 11, + "targets": [ + { + "refId": "A", + "format": "time_series", + "rawSql": "SELECT date_trunc('week', created) AS \"time\", COUNT(*) AS \"uploads\", COUNT(DISTINCT created_by_id) AS \"unique users\" FROM ifc_validation_request WHERE channel='API' AND $__timeFilter(created) GROUP BY 1 ORDER BY 1", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + } + } + ], + "title": "API uploads per week", + "type": "timeseries", + "description": "Weekly API-channel uploads and unique API users. Context: the external API user programme runs on DEV; on PROD this shows only internal usage until the API is opened up. Channel field is only reliable after Jul 2025 (migration wrote everything before that as WEBUI)." + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 39 + }, + "id": 12, + "targets": [ + { + "refId": "A", + "expr": "sum by (status) (rate(django_http_responses_total_by_status_total[5m]))", + "legendFormat": "{{status}}", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "title": "HTTP responses by status (incl. 429)", + "type": "timeseries", + "description": "HTTP responses per status code straight from Django - including 429 rate-limit rejections, which never reach the database and were invisible until now." + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 39 + }, + "id": 13, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (le) (rate(django_http_requests_latency_seconds_by_view_method_bucket[5m])))", + "legendFormat": "p95", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + } + } + ], + "title": "HTTP p95 latency (django)", + "type": "timeseries", + "description": "95th percentile response time of the Django backend, measured inside the app." } ], "refresh": "5m", @@ -282,4 +363,4 @@ "schemaVersion": 39, "editable": true, "timezone": "browser" -} \ No newline at end of file +} diff --git a/docker/grafana/dashboards/vs-system-health.json b/docker/grafana/dashboards/vs-system-health.json index c67d5abb..bc614667 100644 --- a/docker/grafana/dashboards/vs-system-health.json +++ b/docker/grafana/dashboards/vs-system-health.json @@ -184,10 +184,10 @@ "id": 11, "type": "stat", "title": "Alert candidate: stuck requests (>1h)", - "description": "Requests older than 1 hour that are neither finished nor FAILED — same definition as the panel on Platform Usage, but as a Prometheus metric so it can drive an alert later.", + "description": "Requests older than 1 hour that are neither finished nor FAILED — same definition as the panel on Platform Usage, but as a Prometheus metric so it can drive an alert later. (Source: direct SQL — no longer depends on the textfile exporter, so it works on PROD from day one.)", "datasource": { - "type": "prometheus", - "uid": "prometheus" + "type": "grafana-postgresql-datasource", + "uid": "devpg" }, "gridPos": { "h": 5, @@ -231,9 +231,13 @@ }, "targets": [ { - "expr": "max(vs_requests_stuck)", "refId": "A", - "legendFormat": "__auto" + "format": "table", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devpg" + }, + "rawSql": "SELECT COUNT(*) FROM ifc_validation_request WHERE completed IS NULL AND status NOT IN ('COMPLETED','FAILED') AND created < NOW() - INTERVAL '1 hour'" } ] }, @@ -460,7 +464,7 @@ ], "title": "OTel collector: received metric points/s (empty until SDK instrumentation)", "type": "timeseries", - "description": "Stays empty until the application itself sends OpenTelemetry metrics. The collector is running but receives nothing yet — this is expected, see backlog item C1." + "description": "Stays empty until the application itself sends OpenTelemetry metrics. The collector is running but receives nothing yet — this is expected until the SDK instrumentation lands." }, { "id": 12, @@ -578,6 +582,87 @@ "refId": "A" } ] + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 14, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "swarm_service_replicas_desired - swarm_service_replicas_running", + "legendFormat": "{{service}}" + } + ], + "title": "Swarm: replica deficit per service (0 = healthy)", + "type": "timeseries", + "description": "Desired minus running replicas per Swarm service. Zero is healthy; anything above zero for more than a few minutes means Swarm is not delivering what was asked - the signature of the silent rollback of 3 Aug (CI reported success while the worker service had quietly reverted to the old image). Requires the swarm-state cron on the manager (docker/prometheus/swarm-state-textfile.sh) plus the textfile exporter. Alert candidate: deficit > 0 for 5 minutes." + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 15, + "targets": [ + { + "expr": "celery_worker_tasks_active", + "legendFormat": "{{hostname}}", + "refId": "A" + } + ], + "title": "Celery: busy slots per worker", + "type": "timeseries", + "description": "Bezette pool-slots per celery-worker (celery_worker_tasks_active uit de celery-exporter). validate_worker heeft 4 slots per replica (CELERY_CONCURRENCY); een vlakke lijn op 4 = verzadigd, nieuwe checks wachten in de queue. Dit is de live-weergave van het slots-diagram uit de architectuurdocs." } ], "refresh": "30s", @@ -595,4 +680,4 @@ "version": 2, "schemaVersion": 39, "editable": true -} \ No newline at end of file +} diff --git a/docker/grafana/dashboards/vs-validation-perf.json b/docker/grafana/dashboards/vs-validation-perf.json index db9a2107..0193d9cb 100644 --- a/docker/grafana/dashboards/vs-validation-perf.json +++ b/docker/grafana/dashboards/vs-validation-perf.json @@ -362,8 +362,8 @@ { "id": 6, "type": "table", - "title": "20 slowest requests (last 30 days)", - "description": "Fixed 30-day window, independent of the time range above. Sorted by processing time (sum of task durations). Lead time is shown alongside: a large gap between the two means waiting, not working.", + "title": "20 slowest requests", + "description": "Respects the dashboard time range. Sorted by processing time (sum of task durations). Lead time is shown alongside: a large gap between the two means waiting, not working.", "datasource": { "type": "grafana-postgresql-datasource", "uid": "devpg" @@ -465,7 +465,7 @@ "refId": "A", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n r.created AS submitted,\n r.file_name AS file,\n ROUND(r.size / 1048576.0, 1) AS mb,\n r.status,\n ROUND(v.processing_time_s::numeric, 0) AS processing_time_s,\n ROUND(EXTRACT(EPOCH FROM (r.completed - r.created))::numeric, 0) AS lead_time_s,\n v.langzaamste_taak\nFROM ifc_validation_request r\nJOIN (\n SELECT t.request_id,\n SUM(EXTRACT(EPOCH FROM (t.ended - t.started))) AS processing_time_s,\n (ARRAY_AGG(t.type ORDER BY (t.ended - t.started) DESC))[1] AS slowest_task\n FROM ifc_validation_task t\n WHERE t.started IS NOT NULL AND t.ended IS NOT NULL\n GROUP BY t.request_id\n) v ON v.request_id = r.id\nWHERE r.created > NOW() - INTERVAL '30 days'\nORDER BY v.processing_time_s DESC NULLS LAST\nLIMIT 20" + "rawSql": "SELECT\n r.created AS submitted,\n r.file_name AS file,\n ROUND(r.size / 1048576.0, 1) AS mb,\n r.status,\n ROUND(v.processing_time_s::numeric, 0) AS processing_time_s,\n ROUND(EXTRACT(EPOCH FROM (r.completed - r.created))::numeric, 0) AS lead_time_s,\n v.slowest_task\nFROM ifc_validation_request r\nJOIN (\n SELECT t.request_id,\n SUM(EXTRACT(EPOCH FROM (t.ended - t.started))) AS processing_time_s,\n (ARRAY_AGG(t.type ORDER BY (t.ended - t.started) DESC))[1] AS slowest_task\n FROM ifc_validation_task t\n WHERE t.started IS NOT NULL AND t.ended IS NOT NULL\n GROUP BY t.request_id\n) v ON v.request_id = r.id\nWHERE $__timeFilter(r.created)\nORDER BY v.processing_time_s DESC NULLS LAST\nLIMIT 20" } ] }, @@ -473,7 +473,7 @@ "id": 7, "type": "barchart", "title": "Most expensive gherkin rules (total CPU time, cumulative)", - "description": "TEMPORARILY EMPTY after the 3 Aug release: the nightly log-ingest (textfile exporter) is not part of the release stack yet — it returns via a small follow-up PR. The underlying data (201k measurements) is safe in the gherkin logs. From the gherkin logs on the NFS share (201k measurements since Dec 2025), ingested nightly by perf-metrics/gherkin_rule_timings.sh. NOTE: this is CPU time (time.process_time), not wall-clock time, so I/O wait is not included. Logged in PRODUCTION mode only.", + "description": "From the gherkin logs on the NFS share (201k measurements since Dec 2025), ingested nightly by perf-metrics/gherkin_rule_timings.sh. NOTE: this is CPU time (time.process_time), not wall-clock time, so I/O wait is not included. Logged in PRODUCTION mode only.", "gridPos": { "h": 9, "w": 12, @@ -507,7 +507,7 @@ "id": 8, "type": "table", "title": "Rule cost: total, average and longest run", - "description": "TEMPORARILY EMPTY after the 3 Aug release: the nightly log-ingest (textfile exporter) is not part of the release stack yet — it returns via a small follow-up PR. The underlying data (201k measurements) is safe in the gherkin logs. Average versus longest run shows the skew: a few large models dominate. A rule with a low average but an extreme maximum is a tail risk.", + "description": "Average versus longest run shows the skew: a few large models dominate. A rule with a low average but an extreme maximum is a tail risk.", "gridPos": { "h": 9, "w": 12, @@ -596,9 +596,9 @@ "refId": "A", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n CASE WHEN regel ~ '^[A-Z]{2,5}[0-9]{2,3}$' THEN regel\n ELSE '(SCHEMA check, not a gherkin rule)' END AS rule,\n SUM(warnings)::bigint AS warnings,\n SUM(errors)::bigint AS errors,\n SUM(totaal)::bigint AS total\nFROM (\n SELECT split_part(feature, ' ', 1) AS rule,\n SUM((severity = 3)::int) AS warnings,\n SUM((severity = 4)::int) AS errors,\n COUNT(*) AS total\n FROM ifc_validation_outcome\n WHERE severity >= 3 AND feature IS NOT NULL\n GROUP BY 1\n) sub\nGROUP BY 1\nORDER BY totaal DESC\nLIMIT 15" + "rawSql": "SELECT\n CASE WHEN rule ~ '^[A-Z]{2,5}[0-9]{2,3}$' THEN rule\n ELSE '(SCHEMA check, not a gherkin rule)' END AS rule,\n SUM(warnings)::bigint AS warnings,\n SUM(errors)::bigint AS errors,\n SUM(total)::bigint AS total\nFROM (\n SELECT split_part(feature, ' ', 1) AS rule,\n SUM((severity = 3)::int) AS warnings,\n SUM((severity = 4)::int) AS errors,\n COUNT(*) AS total\n FROM ifc_validation_outcome\n WHERE severity >= 3 AND feature IS NOT NULL\n GROUP BY 1\n) sub\nGROUP BY 1\nORDER BY total DESC\nLIMIT 15" } ] } ] -} \ No newline at end of file +} diff --git a/docker/grafana/provisioning/alerting/vs-alerts.yaml b/docker/grafana/provisioning/alerting/vs-alerts.yaml new file mode 100644 index 00000000..7ae82dbe --- /dev/null +++ b/docker/grafana/provisioning/alerting/vs-alerts.yaml @@ -0,0 +1,73 @@ +# Alert rules as code. Loaded at Grafana startup (and via the admin reload API). +# +# No notification policy / contact point is provisioned yet: until the team picks +# a channel and an owner, a firing rule is visible in Grafana under Alerting > +# Alert rules only. Routing it is a ~10 minute follow-up once that decision lands. +# +# noDataState is OK on purpose: the underlying metrics come from cAdvisor, which +# ships in the same release as this file — but on an environment where cAdvisor +# is not (yet) running, the rule must stay green instead of flapping on NoData. + +apiVersion: 1 + +groups: + - orgId: 1 + name: vs-capacity + folder: Validation Service + interval: 1m + rules: + - uid: worker-mem-80 + title: Worker memory above 80% of its limit + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: prometheus + model: + editorMode: code + expr: max(container_memory_working_set_bytes{container_label_com_docker_swarm_service_name="validate_worker"} / container_spec_memory_limit_bytes{container_label_com_docker_swarm_service_name="validate_worker"}) + instant: true + intervalMs: 1000 + legendFormat: __auto + maxDataPoints: 43200 + range: false + refId: A + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.8 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: A + refId: C + type: threshold + noDataState: OK + execErrState: OK + for: 5m + annotations: + summary: >- + A validate_worker replica is using more than 80% of its container + memory limit. The sum of concurrent checks is approaching the fence — + look at large uploads in flight before the kernel starts killing. + labels: + severity: warning + isPaused: false diff --git a/docker/prometheus/prod-crons/db_health_metrics.sh b/docker/prometheus/prod-crons/db_health_metrics.sh new file mode 100755 index 00000000..fb0fbb0f --- /dev/null +++ b/docker/prometheus/prod-crons/db_health_metrics.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# DB health gauges for the Validation Service, exposed via the node_exporter +# textfile collector. Cron: every 15 min. +# +# Route: this script -> Grafana datasource proxy (/api/ds/query, SELECT only) +# -> Postgres. The DB password stays inside Grafana. +# NOTE: datasource uid is hardcoded below (DEV: "devpg") -- adjust per env. +# Cost: one ~40 ms query; deliberately nothing on ifc_validation_outcome (~5M rows). +# +# Gauges: vs_requests_stuck (>1h old, not done, not FAILED -- same definition +# as the "Stuck requests" panel), vs_tasks_initiated_total, +# vs_requests_soft_deleted_total, vs_requests_total, vs_queue_wait_p95_1h, +# vs_db_health_scrape_success, vs_db_health_last_run_timestamp_seconds. +# On a failed run only the two self-metrics are written (success=0), so the +# content gauges go stale in Prometheus instead of repeating old values. +set -uo pipefail + +GRAFANA_URL=${GRAFANA_URL:-http://127.0.0.1:3000} +# Fallback-wachtwoord uit GRAFANA-CREDENTIALS.txt (sinds 31/7 is admin/admin uit; +# de oude default hier brak de cron stilletjes — scrape_success stond op 0, fix 11/8). +CRED_FILE=/home/geert/runbooks/observability/GRAFANA-CREDENTIALS.txt +GRAFANA_AUTH=${GRAFANA_AUTH:-admin:$(sed -n 's/^Wachtwoord: //p' "$CRED_FILE" 2>/dev/null)} +TEXTFILE_DIR=${TEXTFILE_DIR:-/home/geert/runbooks/observability/textfile} +OUT="$TEXTFILE_DIR/vs_db_health.prom" +TMP="$OUT.$$.tmp" +NOW=$(date +%s) + +mkdir -p "$TEXTFILE_DIR" + +# Eén round-trip; subselects zijn elk goedkoop (geïndexeerde status/created/ +# deleted-kolommen; request-tabel ~4k rijen, task-tabel klein). +SQL="SELECT + (SELECT COUNT(*) FROM ifc_validation_request + WHERE completed IS NULL AND status NOT IN ('COMPLETED','FAILED') + AND created < NOW() - INTERVAL '1 hour') AS requests_stuck, + (SELECT COUNT(*) FROM ifc_validation_task + WHERE status = 'INITIATED') AS tasks_initiated, + (SELECT COUNT(*) FROM ifc_validation_request WHERE deleted) AS requests_soft_deleted, + (SELECT COUNT(*) FROM ifc_validation_request) AS requests_total, + (SELECT COALESCE(percentile_cont(0.95) WITHIN GROUP + (ORDER BY EXTRACT(EPOCH FROM ft.fs - r.created)), 0) + FROM ifc_validation_request r + JOIN LATERAL (SELECT MIN(t.started) AS fs FROM ifc_validation_task t + WHERE t.request_id = r.id) ft ON ft.fs IS NOT NULL + WHERE r.created > NOW() - INTERVAL '1 hour') AS queue_wait_p95_1h" + +if BODY=$(python3 - "$GRAFANA_URL" "$GRAFANA_AUTH" "$SQL" <<'PY' +import base64, json, sys, urllib.request + +url, auth, sql = sys.argv[1], sys.argv[2], sys.argv[3] +payload = json.dumps({ + "queries": [{"refId": "A", "datasource": {"uid": "devpg"}, + "rawSql": sql, "format": "table"}], + "from": "now-5m", "to": "now", +}).encode() +req = urllib.request.Request( + url + "/api/ds/query", data=payload, + headers={"Content-Type": "application/json", + "Authorization": "Basic " + base64.b64encode(auth.encode()).decode()}) +try: + resp = json.load(urllib.request.urlopen(req, timeout=25)) +except Exception as exc: # korte melding; geen traceback in cron-mail + sys.exit(f"ds/query onbereikbaar: {exc}") +result = resp["results"]["A"] +if result.get("status") != 200 or not result.get("frames"): + sys.exit("ds/query gaf geen 200/frames: " + json.dumps(result)[:300]) +frame = result["frames"][0] +names = [f["name"] for f in frame["schema"]["fields"]] +row = dict(zip(names, (col[0] for col in frame["data"]["values"]))) + +def fmt(v): + if v is None: + sys.exit("NULL in resultaat: " + json.dumps(row)) + f = float(v) + return str(int(f)) if f.is_integer() else f"{f:.3f}" + +HELP = { + "vs_requests_stuck": "Requests ouder dan 1u die niet af zijn (completed IS NULL, status niet COMPLETED/FAILED). Hoort 0 te zijn.", + "vs_tasks_initiated_total": "Validatietaken die op status INITIATED staan (gauge, momentopname). Blijvend hoge waarde = orphans (F1).", + "vs_requests_soft_deleted_total": "Soft-deleted validatierequests (gauge, momentopname).", + "vs_requests_total": "Alle validatierequests, inclusief soft-deleted (gauge, momentopname).", + "vs_queue_wait_p95_1h": "p95 wachttijd in seconden tussen aanmelden en start eerste taak, requests uit het laatste uur; 0 als er geen waren.", +} +KEYMAP = { + "vs_requests_stuck": "requests_stuck", + "vs_tasks_initiated_total": "tasks_initiated", + "vs_requests_soft_deleted_total": "requests_soft_deleted", + "vs_requests_total": "requests_total", + "vs_queue_wait_p95_1h": "queue_wait_p95_1h", +} +out = [] +for metric, col in KEYMAP.items(): + out.append(f"# HELP {metric} {HELP[metric]}") + out.append(f"# TYPE {metric} gauge") + out.append(f"{metric} {fmt(row[col])}") +print("\n".join(out)) +PY +); then + SUCCESS=1 +else + echo "db_health_metrics: query mislukt, schrijf alleen zelf-metrics" >&2 + SUCCESS=0 + BODY="" +fi + +{ + [ -n "$BODY" ] && printf '%s\n' "$BODY" + echo "# HELP vs_db_health_scrape_success 1 als de laatste run van db_health_metrics.sh slaagde, 0 zo niet." + echo "# TYPE vs_db_health_scrape_success gauge" + echo "vs_db_health_scrape_success $SUCCESS" + echo "# HELP vs_db_health_last_run_timestamp_seconds Unixtijd van de laatste run (geslaagd of niet)." + echo "# TYPE vs_db_health_last_run_timestamp_seconds gauge" + echo "vs_db_health_last_run_timestamp_seconds $NOW" +} > "$TMP" + +# atomisch vervangen, zodat node_exporter nooit een half bestand leest +mv "$TMP" "$OUT" +chmod 644 "$OUT" diff --git a/docker/prometheus/prod-crons/gherkin_rule_timings.sh b/docker/prometheus/prod-crons/gherkin_rule_timings.sh new file mode 100755 index 00000000..3a91d147 --- /dev/null +++ b/docker/prometheus/prod-crons/gherkin_rule_timings.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# Aggregates per-rule cost from the gherkin runner logs into Prometheus metrics +# for the node_exporter textfile collector. Cron: nightly. +# +# Source: $LOG_DIR/*.log -- "Elapsed process time" lines (CPU time, not wall +# clock) with optional "Peak RSS ... (delta ...)" suffix since the B1 change. +# Output: $TEXTFILE_DIR/gherkin_rules.prom (written atomically). +# Every run rescans ALL logs and recomputes the counters -- no incremental +# state; a one-off manual run therefore fills the panel with full history. +set -uo pipefail + +LOG_DIR=${LOG_DIR:-/srv/nfs/gherkin_logs} +TEXTFILE_DIR=${TEXTFILE_DIR:-/home/geert/runbooks/observability/textfile} +OUT="$TEXTFILE_DIR/gherkin_rules.prom" +TMP="$OUT.$$.tmp" + +mkdir -p "$TEXTFILE_DIR" + +{ + echo "# HELP gherkin_rule_cpu_seconds_total Totale CPU-tijd per gherkin-regel (uit de logs, cumulatief over alle runs)." + echo "# TYPE gherkin_rule_cpu_seconds_total counter" + echo "# HELP gherkin_rule_runs_total Aantal keren dat de regel is uitgevoerd." + echo "# TYPE gherkin_rule_runs_total counter" + echo "# HELP gherkin_rule_cpu_seconds_max Langste enkele run van deze regel (CPU-seconden)." + echo "# TYPE gherkin_rule_cpu_seconds_max gauge" + echo "# HELP gherkin_rule_cpu_seconds_avg Gemiddelde CPU-tijd per run." + echo "# TYPE gherkin_rule_cpu_seconds_avg gauge" + + # B1-uitbreiding (2/8): de logregel kan sinds de per-regel-geheugenmeting eindigen op + # " Peak RSS: MB (delta <+/-n> MB)." — veld 3/4 zijn dan gevuld, anders leeg. + # find|xargs i.p.v. glob: bij ~100k logbestanden overschrijdt "$LOG_DIR"/*.log + # de ARG_MAX-limiet (~2 MB) en faalt grep met "Argument list too long". + find "$LOG_DIR" -maxdepth 1 -name '*.log' -print0 2>/dev/null \ + | xargs -0 -r grep -h "Elapsed process time" \ + | sed -E "s/.*Feature '([^']+)'.*time: ([0-9.]+) seconds\.( Peak RSS: ([0-9]+) MB \(delta ([+-][0-9]+) MB\)\.)?.*/\2\t\1\t\4\t\5/" \ + | awk -F'\t' ' + { + # regelcode = eerste woord vóór de spatie-streepje-spatie (bv. "CTX000 - ...") + split($2, parts, " "); + rule = parts[1]; + gsub(/[^A-Za-z0-9_]/, "", rule); + if (rule == "") next; + sum[rule] += $1; n[rule]++; + if ($1 > mx[rule]) mx[rule] = $1; + if ($3 != "") { memn[rule]++; memsum[rule] += $3; if ($3 > memmx[rule]) memmx[rule] = $3; + dsum[rule] += $4; if ($4 > dmx[rule]) dmx[rule] = $4; } + } + END { + for (r in sum) { + printf "gherkin_rule_cpu_seconds_total{rule=\"%s\"} %.2f\n", r, sum[r]; + printf "gherkin_rule_runs_total{rule=\"%s\"} %d\n", r, n[r]; + printf "gherkin_rule_cpu_seconds_max{rule=\"%s\"} %.2f\n", r, mx[r]; + printf "gherkin_rule_cpu_seconds_avg{rule=\"%s\"} %.3f\n", r, sum[r]/n[r]; + if (memn[r] > 0) { + printf "gherkin_rule_peak_rss_mb_max{rule=\"%s\"} %d\n", r, memmx[r]; + printf "gherkin_rule_peak_rss_mb_avg{rule=\"%s\"} %.1f\n", r, memsum[r]/memn[r]; + printf "gherkin_rule_delta_mb_max{rule=\"%s\"} %d\n", r, dmx[r]; + printf "gherkin_rule_delta_mb_avg{rule=\"%s\"} %.1f\n", r, dsum[r]/memn[r]; + printf "gherkin_rule_mem_samples_total{rule=\"%s\"} %d\n", r, memn[r]; + } + } + }' + + echo "# HELP gherkin_rule_timings_logfiles Aantal logbestanden dat is ingelezen." + echo "# TYPE gherkin_rule_timings_logfiles gauge" + echo "gherkin_rule_timings_logfiles $(find "$LOG_DIR" -maxdepth 1 -name '*.log' 2>/dev/null | wc -l)" +} > "$TMP" + +# atomisch vervangen, zodat node_exporter nooit een half bestand leest +mv "$TMP" "$OUT" +chmod 644 "$OUT" diff --git a/docker/prometheus/prod-crons/harvest_peak_rss.sh b/docker/prometheus/prod-crons/harvest_peak_rss.sh new file mode 100755 index 00000000..25d98a0b --- /dev/null +++ b/docker/prometheus/prod-crons/harvest_peak_rss.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Harvests "Peak RSS for" lines from the validate_worker service logs into a +# persistent, deduplicated file -- container logs are lost on redeploy, this +# file is not. Cron: every 10 min. Override OUT via env prefix in the cron +# line (PROD: OUT=/data/srv/perf-collected/peak_rss.log). +OUT=${OUT:-/home/geert/runbooks/observability/perf-metrics/collected/peak_rss.log} +mkdir -p "$(dirname "$OUT")" +TMP=$(mktemp) +timeout 100 docker service logs validate_worker --since 30m 2>&1 | grep "Peak RSS for" >> "$OUT" 2>/dev/null +sort -u "$OUT" > "$TMP" && mv "$TMP" "$OUT" diff --git a/docker/prometheus/prometheus.yaml b/docker/prometheus/prometheus.yaml index 950fb19a..339b3418 100644 --- a/docker/prometheus/prometheus.yaml +++ b/docker/prometheus/prometheus.yaml @@ -21,3 +21,23 @@ scrape_configs: - job_name: "celery" static_configs: - targets: ["celery_exporter:9808"] + + # Django HTTP metrics (django-prometheus): requests, latencies and status codes + # per view - including 429s, which never reach the database. + - job_name: "django" + static_configs: + - targets: ["backend:8000"] + + # Batch metrics written as .prom files by cron jobs on the manager + # (per-rule gherkin costs, harvested peak-RSS numbers). + - job_name: "textfile" + static_configs: + - targets: ["textfile_exporter:9100"] + + # Per-container memory/CPU. Closes the gap that made the clamd incident + # possible: long-lived daemons inside containers were invisible. + - job_name: "cadvisor" + dns_sd_configs: + - names: ["tasks.cadvisor"] + type: A + port: 8080 diff --git a/docker/prometheus/swarm-state-textfile.sh b/docker/prometheus/swarm-state-textfile.sh new file mode 100755 index 00000000..a320c650 --- /dev/null +++ b/docker/prometheus/swarm-state-textfile.sh @@ -0,0 +1,38 @@ +#!/bin/sh +# Writes Docker Swarm control-plane state as Prometheus textfile metrics: +# per service desired vs running replicas, per node its readiness. +# +# Why: Swarm silently self-heals (restarts, rollbacks). A rolled-back service +# looks "Running" everywhere while the wrong image serves traffic, and an +# OOM-killed subprocess leaves no failed container behind. These gauges make +# "what should run" vs "what actually runs" visible. +# +# Install as a cron on the MANAGER, e.g.: */2 * * * * /swarm-state-textfile.sh +# The output dir must be the one served by the textfile_exporter service. +set -u +OUT_DIR="${1:?usage: $0 }" +OUT="$OUT_DIR/swarm_state.prom" +TMP="$OUT.$$.tmp" + +{ + echo "# HELP swarm_service_replicas_desired Replicas the service is configured to run." + echo "# TYPE swarm_service_replicas_desired gauge" + echo "# HELP swarm_service_replicas_running Replicas actually running right now." + echo "# TYPE swarm_service_replicas_running gauge" + docker service ls --format '{{.Name}} {{.Replicas}}' | while read -r name replicas _; do + running=${replicas%%/*} + desired=${replicas##*/}; desired=${desired%% *} + echo "swarm_service_replicas_desired{service=\"$name\"} $desired" + echo "swarm_service_replicas_running{service=\"$name\"} $running" + done + + echo "# HELP swarm_node_ready 1 when the node reports status Ready, else 0." + echo "# TYPE swarm_node_ready gauge" + docker node ls --format '{{.Hostname}} {{.Status}}' | while read -r host status _; do + ready=0; [ "$status" = "Ready" ] && ready=1 + echo "swarm_node_ready{node=\"$host\"} $ready" + done +} > "$TMP" + +mv "$TMP" "$OUT" # atomic: the exporter never sees a half-written file +chmod 644 "$OUT"