From 223d09a2d368ec6a77351b3c764f99ec4ac40133 Mon Sep 17 00:00:00 2001 From: Jonatas Ferreira <62180037+jonatas-lima@users.noreply.github.com> Date: Mon, 5 Oct 2026 19:16:40 -0300 Subject: [PATCH] [FEATURE] ClickHouse: use non-numeric columns of a time series query as series labels Non-numeric columns used to be dropped, so a query grouped by a dimension collapsed every group into one series. They now become series labels, one series per label combination and value column. Queries without label columns return exactly what they returned before. Signed-off-by: Jonatas Ferreira <62180037+jonatas-lima@users.noreply.github.com> --- .../click-house-query-types.test.ts | 114 +++++++++++++++++- .../get-click-house-data.ts | 74 +++++++++--- docs/clickhouse/model.md | 16 +++ 3 files changed, 185 insertions(+), 19 deletions(-) diff --git a/clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts b/clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts index 963e0e957..ba48a1797 100644 --- a/clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts +++ b/clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts @@ -16,7 +16,7 @@ vi.mock('echarts/core'); import type { TimeSeriesQueryContext } from '@perses-dev/plugin-system'; -import type { DatasourceSpec } from '@perses-dev/spec'; +import type { DatasourceSpec, TimeSeriesData } from '@perses-dev/spec'; import type { Mock } from 'vitest'; import type { ClickHouseDatasourceSpec } from '../../datasources'; @@ -35,8 +35,8 @@ clickhouseStubClient.query = vi.fn(async () => { const stubResponse: ClickHouseQueryResponse = { status: 'success', data: [ - { time: '2025-09-09 05:18:00', avg_cpu: '2.5', max_memory: 277, service: 'api' }, - { time: '2025-09-09 05:19:00', avg_cpu: '3.5', max_memory: 156102, service: 'api' }, + { time: '2025-09-09 05:18:00', avg_cpu: '2.5', max_memory: 277 }, + { time: '2025-09-09 05:19:00', avg_cpu: '3.5', max_memory: 156102 }, ], }; return stubResponse as ClickHouseQueryResponse; @@ -181,4 +181,112 @@ describe('ClickHouseTimeSeriesQuery', () => { expect(response.stepMs).toBe(24 * 60 * 60 * 1000); }); + + describe('label columns', () => { + const run = async ( + data: Array>, + query = 'SELECT 1', + ): Promise => { + (clickhouseStubClient.query as Mock).mockResolvedValueOnce({ + status: 'success', + data, + }); + return ClickHouseTimeSeriesQuery.getTimeSeriesData({ query }, createStubContext()); + }; + const t1 = new Date('2026-01-01 00:00:00').getTime(); + const t2 = new Date('2026-01-01 00:01:00').getTime(); + + it('should keep one series per value column, without labels, when no column is a label', async () => { + const response = await run([ + { time: '2026-01-01 00:00:00', a: 1, b: '2' }, + { time: '2026-01-01 00:01:00', a: 3, b: null }, + ]); + expect(response.series).toStrictEqual([ + { + name: 'a', + values: [ + [t1, 1], + [t2, 3], + ], + }, + { + name: 'b', + values: [ + [t1, 2], + [t2, null], + ], + }, + ]); + }); + + it('should split value columns into one series per label tuple', async () => { + const response = await run([ + { time: '2026-01-01 00:00:00', repository: 'network-api', jobs: 4 }, + { time: '2026-01-01 00:00:00', repository: 'billing', jobs: 7 }, + { time: '2026-01-01 00:01:00', repository: 'network-api', jobs: 5 }, + { time: '2026-01-01 00:01:00', repository: 'billing', jobs: 9 }, + ]); + expect(response.series).toStrictEqual([ + { + name: 'jobs{repository="network-api"}', + labels: { repository: 'network-api' }, + values: [ + [t1, 4], + [t2, 5], + ], + }, + { + name: 'jobs{repository="billing"}', + labels: { repository: 'billing' }, + values: [ + [t1, 7], + [t2, 9], + ], + }, + ]); + }); + + it('should combine several label columns and several value columns', async () => { + const response = await run([ + { + time: '2026-01-01 00:00:00', + repository: 'api', + branch: 'main', + jobs: 1, + failures: 0, + }, + ]); + expect(response.series?.map((s) => [s.name, s.labels])).toStrictEqual([ + ['jobs{branch="main",repository="api"}', { repository: 'api', branch: 'main' }], + ['failures{branch="main",repository="api"}', { repository: 'api', branch: 'main' }], + ]); + }); + + it('should treat a column mixing numbers and text as a value column', async () => { + const response = await run([ + { time: '2026-01-01 00:00:00', repository: 'api', jobs: 1 }, + { time: '2026-01-01 00:01:00', repository: '5', jobs: 2 }, + ]); + expect(response.series?.map((s) => s.name)).toStrictEqual(['repository', 'jobs']); + }); + + it('should keep labels on rows without a time column so tables can show them', async () => { + const response = await run([ + { repository: 'network-api', coverage: 81.5 }, + { repository: 'billing', coverage: 64 }, + ]); + expect(response.series).toStrictEqual([ + { + name: 'coverage{repository="network-api"}', + labels: { repository: 'network-api' }, + values: [[NaN, 81.5]], + }, + { + name: 'coverage{repository="billing"}', + labels: { repository: 'billing' }, + values: [[NaN, 64]], + }, + ]); + }); + }); }); diff --git a/clickhouse/src/queries/click-house-time-series-query/get-click-house-data.ts b/clickhouse/src/queries/click-house-time-series-query/get-click-house-data.ts index 3b0387d92..0814a1f21 100644 --- a/clickhouse/src/queries/click-house-time-series-query/get-click-house-data.ts +++ b/clickhouse/src/queries/click-house-time-series-query/get-click-house-data.ts @@ -13,7 +13,7 @@ import type { TimeSeriesQueryPlugin } from '@perses-dev/plugin-system'; import { replaceVariables } from '@perses-dev/plugin-system'; -import type { TimeSeries } from '@perses-dev/spec'; +import type { Labels, TimeSeries } from '@perses-dev/spec'; import type { ClickHouseClient, ClickHouseQueryResponse } from '../../model/click-house-client'; import { formatClickHouseDateTime, replaceTimeRangePlaceholders } from '../../model/click-house-client'; @@ -23,27 +23,69 @@ import type { ClickHouseTimeSeriesQuerySpec, DatasourceQueryResponse } from './c const DEFAULT_STEP_MS = 30 * 1000; +function isLabelColumn(data: TimeSeriesEntry[], column: string): boolean { + let hasText = false; + for (const row of data) { + const value = row[column]; + if (value === null || value === undefined || value === '') { + continue; + } + if (toTimeSeriesValue(value) !== null) { + return false; + } + hasText = true; + } + return hasText; +} + +// Same notation as a Prometheus series: metric{label="value",...} with sorted label names. +function formatSeriesName(metricName: string, labels: Labels): string { + const labelNames = Object.keys(labels); + if (labelNames.length === 0) { + return metricName; + } + const pairs = labelNames.toSorted().map((labelName) => `${labelName}="${labels[labelName]}"`); + return `${metricName}{${pairs.join(',')}}`; +} + +// Columns holding only non-numeric text are labels; the remaining columns (except `time`) are values. function buildTimeSeries(response?: DatasourceQueryResponse): TimeSeries[] { const data = response?.data as TimeSeriesEntry[]; if (!response || !data || data.length === 0) { return []; } - const metricNames = Object.keys(data[0] ?? {}).filter((key) => key !== 'time'); - - return metricNames - .map((metricName) => { - const values: Array<[number, number | null]> = data.map((row: TimeSeriesEntry) => { - const timestamp = new Date(row.time).getTime(); - const value = toTimeSeriesValue(row[metricName]); - return [timestamp, value]; - }); - - return { - name: metricName, - values, - }; - }) + const columns = Object.keys(data[0] ?? {}).filter((key) => key !== 'time'); + const labelNames = columns.filter((column) => isLabelColumn(data, column)); + const metricNames = columns.filter((column) => !labelNames.includes(column)); + + const groups = new Map(); + for (const row of data) { + const labels: Labels = {}; + for (const labelName of labelNames) { + labels[labelName] = String(row[labelName] ?? ''); + } + const key = JSON.stringify(labelNames.map((labelName) => labels[labelName])); + const group = groups.get(key) ?? { labels, rows: [] }; + group.rows.push(row); + groups.set(key, group); + } + + const hasLabels = labelNames.length > 0; + return Array.from(groups.values()) + .flatMap(({ labels, rows }) => + metricNames.map((metricName): TimeSeries => { + const values: Array<[number, number | null]> = rows.map((row: TimeSeriesEntry) => { + const timestamp = new Date(row.time).getTime(); + const value = toTimeSeriesValue(row[metricName]); + return [timestamp, value]; + }); + + return hasLabels + ? { name: formatSeriesName(metricName, labels), labels, values } + : { name: metricName, values }; + }), + ) .filter((series) => series.values.some(([, value]) => value !== null)); } diff --git a/docs/clickhouse/model.md b/docs/clickhouse/model.md index aa09a1556..15bf91714 100644 --- a/docs/clickhouse/model.md +++ b/docs/clickhouse/model.md @@ -96,6 +96,22 @@ spec: query: "SELECT toStartOfMinute(timestamp) as time, count() as requests FROM http_logs WHERE timestamp >= now() - INTERVAL 1 HOUR GROUP BY time ORDER BY time" ``` +### Label columns + +Columns (other than `time`) whose values are all non-numeric text are used as series labels; every other column is a +value column. One series is produced per label combination and value column, named like +`requests{service="api"}` (Prometheus notation). Without label columns, one series per value column is produced, named +after the column. + +```yaml +kind: "TimeSeriesQuery" +spec: + plugin: + kind: "ClickHouseTimeSeriesQuery" + spec: + query: "SELECT toStartOfMinute(timestamp) as time, service, count() as requests FROM http_logs GROUP BY time, service ORDER BY time" +``` + ## ClickHouseLogQuery Perses supports log queries for ClickHouse: `ClickHouseLogQuery`.