diff --git a/deploy/telemetry-functions/README.md b/deploy/telemetry-functions/README.md index 0c5c226..1437e55 100644 --- a/deploy/telemetry-functions/README.md +++ b/deploy/telemetry-functions/README.md @@ -40,6 +40,11 @@ that every heavy hitter is in the exact list (about 10 routes reach 1% here); a window whose exact top 20 is all above the threshold only gets `asap_hh_incomplete`. +Each dashboard also shows every pipeline's throughput: the points per window +that reached the end of the pipeline (`asap_points`; for raw, the points +`loki-raw` acknowledged) divided by the window length, against the offered +2 x 20,000 points/s. + Every pipeline reports its window completion delay as `asap_answer_delay` (seconds from a window's end until its result is complete at the end of the pipeline): sketch and exact when their detector emits the window's answers, diff --git a/deploy/telemetry-functions/grafana/dashboards/cardinality.json b/deploy/telemetry-functions/grafana/dashboards/cardinality.json index fc60a0e..7d1191e 100644 --- a/deploy/telemetry-functions/grafana/dashboards/cardinality.json +++ b/deploy/telemetry-functions/grafana/dashboards/cardinality.json @@ -291,6 +291,88 @@ }, "id": 5 }, + { + "type": "timeseries", + "id": 10, + "title": "Throughput per pipeline: points per second each window delivered to the end of its pipeline", + "description": "Points in each 1-minute window that reached the end of the pipeline, divided by 60 s: for sketch and exact the points their detector answered for, for raw the points loki-raw acknowledged. Offered: 2 sources x 20,000 points/s. A pipeline below the offered line is dropping or falling behind.", + "datasource": { + "type": "prometheus", + "uid": "asap-prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 28 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (method) (asap_points{method=~\"sketch|exact\",attribute_key=\"http.route\"}) / 60", + "legendFormat": "{{method}}" + }, + { + "refId": "B", + "expr": "sum by (method) (asap_points{method=\"raw\"}) / 60", + "legendFormat": "{{method}}" + }, + { + "refId": "C", + "expr": "vector(40000)", + "legendFormat": "offered" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "min": 0, + "custom": { + "spanNulls": true, + "showPoints": "always", + "pointSize": 6 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "offered" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 6, + 4 + ] + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#8e8e8e" + } + }, + { + "id": "custom.showPoints", + "value": "never" + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, { "type": "timeseries", "title": "Window completion delay: seconds from a window's end until its result is complete", @@ -302,7 +384,7 @@ "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 36 }, "targets": [ { @@ -380,7 +462,7 @@ "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 36 }, "targets": [ { @@ -457,7 +539,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 44 }, "targets": [ { @@ -538,7 +620,7 @@ "h": 8, "w": 12, "x": 12, - "y": 36 + "y": 44 }, "targets": [ { diff --git a/deploy/telemetry-functions/grafana/dashboards/drift.json b/deploy/telemetry-functions/grafana/dashboards/drift.json index d561c44..c11162a 100644 --- a/deploy/telemetry-functions/grafana/dashboards/drift.json +++ b/deploy/telemetry-functions/grafana/dashboards/drift.json @@ -393,6 +393,88 @@ ] } }, + { + "type": "timeseries", + "id": 10, + "title": "Throughput per pipeline: points per second each window delivered to the end of its pipeline", + "description": "Points in each 1-minute window that reached the end of the pipeline, divided by 60 s: for sketch and exact the points their detector answered for, for raw the points loki-raw acknowledged. Offered: 2 sources x 20,000 points/s. A pipeline below the offered line is dropping or falling behind.", + "datasource": { + "type": "prometheus", + "uid": "asap-prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (method) (asap_points{method=~\"sketch|exact\",attribute_key=\"\"}) / 60", + "legendFormat": "{{method}}" + }, + { + "refId": "B", + "expr": "sum by (method) (asap_points{method=\"raw\"}) / 60", + "legendFormat": "{{method}}" + }, + { + "refId": "C", + "expr": "vector(40000)", + "legendFormat": "offered" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "min": 0, + "custom": { + "spanNulls": true, + "showPoints": "always", + "pointSize": 6 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "offered" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 6, + 4 + ] + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#8e8e8e" + } + }, + { + "id": "custom.showPoints", + "value": "never" + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, { "type": "timeseries", "title": "Window completion delay: seconds from a window's end until its result is complete", @@ -404,7 +486,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 34 }, "targets": [ { @@ -482,7 +564,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 34 }, "targets": [ { @@ -559,7 +641,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 42 }, "targets": [ { @@ -640,7 +722,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 42 }, "targets": [ { diff --git a/deploy/telemetry-functions/grafana/dashboards/latency.json b/deploy/telemetry-functions/grafana/dashboards/latency.json index ddbd86e..5995081 100644 --- a/deploy/telemetry-functions/grafana/dashboards/latency.json +++ b/deploy/telemetry-functions/grafana/dashboards/latency.json @@ -343,6 +343,88 @@ "overrides": [] } }, + { + "type": "timeseries", + "id": 9, + "title": "Throughput per pipeline: points per second each window delivered to the end of its pipeline", + "description": "Points in each 1-minute window that reached the end of the pipeline, divided by 60 s: for sketch and exact the points their detector answered for, for raw the points loki-raw acknowledged. Offered: 2 sources x 20,000 points/s. A pipeline below the offered line is dropping or falling behind.", + "datasource": { + "type": "prometheus", + "uid": "asap-prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (method) (asap_points{method=~\"sketch|exact\",attribute_key=\"\"}) / 60", + "legendFormat": "{{method}}" + }, + { + "refId": "B", + "expr": "sum by (method) (asap_points{method=\"raw\"}) / 60", + "legendFormat": "{{method}}" + }, + { + "refId": "C", + "expr": "vector(40000)", + "legendFormat": "offered" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "min": 0, + "custom": { + "spanNulls": true, + "showPoints": "always", + "pointSize": 6 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "offered" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 6, + 4 + ] + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#8e8e8e" + } + }, + { + "id": "custom.showPoints", + "value": "never" + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, { "type": "timeseries", "title": "Window completion delay: seconds from a window's end until its result is complete", @@ -354,7 +436,7 @@ "h": 8, "w": 12, "x": 0, - "y": 26 + "y": 34 }, "targets": [ { @@ -432,7 +514,7 @@ "h": 8, "w": 12, "x": 12, - "y": 26 + "y": 34 }, "targets": [ { @@ -509,7 +591,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 42 }, "targets": [ { @@ -590,7 +672,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 42 }, "targets": [ { diff --git a/deploy/telemetry-functions/grafana/dashboards/topk.json b/deploy/telemetry-functions/grafana/dashboards/topk.json index 3626c58..11e4ac2 100644 --- a/deploy/telemetry-functions/grafana/dashboards/topk.json +++ b/deploy/telemetry-functions/grafana/dashboards/topk.json @@ -384,6 +384,88 @@ "id": 11, "description": "Mean of |sketch estimate - exact count| / exact count over the heavy hitters the sketch found. Right axis: how many heavy hitters exact has and how many the sketch reported." }, + { + "type": "timeseries", + "id": 12, + "title": "Throughput per pipeline: points per second each window delivered to the end of its pipeline", + "description": "Points in each 1-minute window that reached the end of the pipeline, divided by 60 s: for sketch and exact the points their detector answered for, for raw the points loki-raw acknowledged. Offered: 2 sources x 20,000 points/s. A pipeline below the offered line is dropping or falling behind.", + "datasource": { + "type": "prometheus", + "uid": "asap-prometheus" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 36 + }, + "targets": [ + { + "refId": "A", + "expr": "sum by (method) (asap_points{method=~\"sketch|exact\",attribute_key=\"http.route\"}) / 60", + "legendFormat": "{{method}}" + }, + { + "refId": "B", + "expr": "sum by (method) (asap_points{method=\"raw\"}) / 60", + "legendFormat": "{{method}}" + }, + { + "refId": "C", + "expr": "vector(40000)", + "legendFormat": "offered" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "decimals": 0, + "min": 0, + "custom": { + "spanNulls": true, + "showPoints": "always", + "pointSize": 6 + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "offered" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 6, + 4 + ] + } + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "#8e8e8e" + } + }, + { + "id": "custom.showPoints", + "value": "never" + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + } + } + }, { "type": "timeseries", "title": "Window completion delay: seconds from a window's end until its result is complete", @@ -395,7 +477,7 @@ "h": 8, "w": 12, "x": 0, - "y": 36 + "y": 44 }, "targets": [ { @@ -473,7 +555,7 @@ "h": 8, "w": 12, "x": 12, - "y": 36 + "y": 44 }, "targets": [ { @@ -550,7 +632,7 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 52 }, "targets": [ { @@ -631,7 +713,7 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 52 }, "targets": [ {