diff --git a/clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts b/clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts index 963e0e957..ba48a1797 100644 --- a/clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts +++ b/clickhouse/src/queries/click-house-time-series-query/click-house-query-types.test.ts @@ -16,7 +16,7 @@ vi.mock('echarts/core'); import type { TimeSeriesQueryContext } from '@perses-dev/plugin-system'; -import type { DatasourceSpec } from '@perses-dev/spec'; +import type { DatasourceSpec, TimeSeriesData } from '@perses-dev/spec'; import type { Mock } from 'vitest'; import type { ClickHouseDatasourceSpec } from '../../datasources'; @@ -35,8 +35,8 @@ clickhouseStubClient.query = vi.fn(async () => { const stubResponse: ClickHouseQueryResponse = { status: 'success', data: [ - { time: '2025-09-09 05:18:00', avg_cpu: '2.5', max_memory: 277, service: 'api' }, - { time: '2025-09-09 05:19:00', avg_cpu: '3.5', max_memory: 156102, service: 'api' }, + { time: '2025-09-09 05:18:00', avg_cpu: '2.5', max_memory: 277 }, + { time: '2025-09-09 05:19:00', avg_cpu: '3.5', max_memory: 156102 }, ], }; return stubResponse as ClickHouseQueryResponse; @@ -181,4 +181,112 @@ describe('ClickHouseTimeSeriesQuery', () => { expect(response.stepMs).toBe(24 * 60 * 60 * 1000); }); + + describe('label columns', () => { + const run = async ( + data: Array>, + query = 'SELECT 1', + ): Promise => { + (clickhouseStubClient.query as Mock).mockResolvedValueOnce({ + status: 'success', + data, + }); + return ClickHouseTimeSeriesQuery.getTimeSeriesData({ query }, createStubContext()); + }; + const t1 = new Date('2026-01-01 00:00:00').getTime(); + const t2 = new Date('2026-01-01 00:01:00').getTime(); + + it('should keep one series per value column, without labels, when no column is a label', async () => { + const response = await run([ + { time: '2026-01-01 00:00:00', a: 1, b: '2' }, + { time: '2026-01-01 00:01:00', a: 3, b: null }, + ]); + expect(response.series).toStrictEqual([ + { + name: 'a', + values: [ + [t1, 1], + [t2, 3], + ], + }, + { + name: 'b', + values: [ + [t1, 2], + [t2, null], + ], + }, + ]); + }); + + it('should split value columns into one series per label tuple', async () => { + const response = await run([ + { time: '2026-01-01 00:00:00', repository: 'network-api', jobs: 4 }, + { time: '2026-01-01 00:00:00', repository: 'billing', jobs: 7 }, + { time: '2026-01-01 00:01:00', repository: 'network-api', jobs: 5 }, + { time: '2026-01-01 00:01:00', repository: 'billing', jobs: 9 }, + ]); + expect(response.series).toStrictEqual([ + { + name: 'jobs{repository="network-api"}', + labels: { repository: 'network-api' }, + values: [ + [t1, 4], + [t2, 5], + ], + }, + { + name: 'jobs{repository="billing"}', + labels: { repository: 'billing' }, + values: [ + [t1, 7], + [t2, 9], + ], + }, + ]); + }); + + it('should combine several label columns and several value columns', async () => { + const response = await run([ + { + time: '2026-01-01 00:00:00', + repository: 'api', + branch: 'main', + jobs: 1, + failures: 0, + }, + ]); + expect(response.series?.map((s) => [s.name, s.labels])).toStrictEqual([ + ['jobs{branch="main",repository="api"}', { repository: 'api', branch: 'main' }], + ['failures{branch="main",repository="api"}', { repository: 'api', branch: 'main' }], + ]); + }); + + it('should treat a column mixing numbers and text as a value column', async () => { + const response = await run([ + { time: '2026-01-01 00:00:00', repository: 'api', jobs: 1 }, + { time: '2026-01-01 00:01:00', repository: '5', jobs: 2 }, + ]); + expect(response.series?.map((s) => s.name)).toStrictEqual(['repository', 'jobs']); + }); + + it('should keep labels on rows without a time column so tables can show them', async () => { + const response = await run([ + { repository: 'network-api', coverage: 81.5 }, + { repository: 'billing', coverage: 64 }, + ]); + expect(response.series).toStrictEqual([ + { + name: 'coverage{repository="network-api"}', + labels: { repository: 'network-api' }, + values: [[NaN, 81.5]], + }, + { + name: 'coverage{repository="billing"}', + labels: { repository: 'billing' }, + values: [[NaN, 64]], + }, + ]); + }); + }); }); diff --git a/clickhouse/src/queries/click-house-time-series-query/get-click-house-data.ts b/clickhouse/src/queries/click-house-time-series-query/get-click-house-data.ts index 3b0387d92..0814a1f21 100644 --- a/clickhouse/src/queries/click-house-time-series-query/get-click-house-data.ts +++ b/clickhouse/src/queries/click-house-time-series-query/get-click-house-data.ts @@ -13,7 +13,7 @@ import type { TimeSeriesQueryPlugin } from '@perses-dev/plugin-system'; import { replaceVariables } from '@perses-dev/plugin-system'; -import type { TimeSeries } from '@perses-dev/spec'; +import type { Labels, TimeSeries } from '@perses-dev/spec'; import type { ClickHouseClient, ClickHouseQueryResponse } from '../../model/click-house-client'; import { formatClickHouseDateTime, replaceTimeRangePlaceholders } from '../../model/click-house-client'; @@ -23,27 +23,69 @@ import type { ClickHouseTimeSeriesQuerySpec, DatasourceQueryResponse } from './c const DEFAULT_STEP_MS = 30 * 1000; +function isLabelColumn(data: TimeSeriesEntry[], column: string): boolean { + let hasText = false; + for (const row of data) { + const value = row[column]; + if (value === null || value === undefined || value === '') { + continue; + } + if (toTimeSeriesValue(value) !== null) { + return false; + } + hasText = true; + } + return hasText; +} + +// Same notation as a Prometheus series: metric{label="value",...} with sorted label names. +function formatSeriesName(metricName: string, labels: Labels): string { + const labelNames = Object.keys(labels); + if (labelNames.length === 0) { + return metricName; + } + const pairs = labelNames.toSorted().map((labelName) => `${labelName}="${labels[labelName]}"`); + return `${metricName}{${pairs.join(',')}}`; +} + +// Columns holding only non-numeric text are labels; the remaining columns (except `time`) are values. function buildTimeSeries(response?: DatasourceQueryResponse): TimeSeries[] { const data = response?.data as TimeSeriesEntry[]; if (!response || !data || data.length === 0) { return []; } - const metricNames = Object.keys(data[0] ?? {}).filter((key) => key !== 'time'); - - return metricNames - .map((metricName) => { - const values: Array<[number, number | null]> = data.map((row: TimeSeriesEntry) => { - const timestamp = new Date(row.time).getTime(); - const value = toTimeSeriesValue(row[metricName]); - return [timestamp, value]; - }); - - return { - name: metricName, - values, - }; - }) + const columns = Object.keys(data[0] ?? {}).filter((key) => key !== 'time'); + const labelNames = columns.filter((column) => isLabelColumn(data, column)); + const metricNames = columns.filter((column) => !labelNames.includes(column)); + + const groups = new Map(); + for (const row of data) { + const labels: Labels = {}; + for (const labelName of labelNames) { + labels[labelName] = String(row[labelName] ?? ''); + } + const key = JSON.stringify(labelNames.map((labelName) => labels[labelName])); + const group = groups.get(key) ?? { labels, rows: [] }; + group.rows.push(row); + groups.set(key, group); + } + + const hasLabels = labelNames.length > 0; + return Array.from(groups.values()) + .flatMap(({ labels, rows }) => + metricNames.map((metricName): TimeSeries => { + const values: Array<[number, number | null]> = rows.map((row: TimeSeriesEntry) => { + const timestamp = new Date(row.time).getTime(); + const value = toTimeSeriesValue(row[metricName]); + return [timestamp, value]; + }); + + return hasLabels + ? { name: formatSeriesName(metricName, labels), labels, values } + : { name: metricName, values }; + }), + ) .filter((series) => series.values.some(([, value]) => value !== null)); } diff --git a/docs/clickhouse/model.md b/docs/clickhouse/model.md index aa09a1556..15bf91714 100644 --- a/docs/clickhouse/model.md +++ b/docs/clickhouse/model.md @@ -96,6 +96,22 @@ spec: query: "SELECT toStartOfMinute(timestamp) as time, count() as requests FROM http_logs WHERE timestamp >= now() - INTERVAL 1 HOUR GROUP BY time ORDER BY time" ``` +### Label columns + +Columns (other than `time`) whose values are all non-numeric text are used as series labels; every other column is a +value column. One series is produced per label combination and value column, named like +`requests{service="api"}` (Prometheus notation). Without label columns, one series per value column is produced, named +after the column. + +```yaml +kind: "TimeSeriesQuery" +spec: + plugin: + kind: "ClickHouseTimeSeriesQuery" + spec: + query: "SELECT toStartOfMinute(timestamp) as time, service, count() as requests FROM http_logs GROUP BY time, service ORDER BY time" +``` + ## ClickHouseLogQuery Perses supports log queries for ClickHouse: `ClickHouseLogQuery`.