From 097732792c4eea0c4a97eaa290b58d0f66b2e482 Mon Sep 17 00:00:00 2001 From: luweglarz Date: Fri, 31 Jul 2026 17:55:12 +0200 Subject: [PATCH] [FEATURE] pyroscope: migrate profile queries to the Connect API Signed-off-by: luweglarz --- pyroscope/src/model/api-types.ts | 122 +++-- pyroscope/src/model/pyroscope-client.ts | 58 ++- .../src/plugins/pyroscope-datasource.tsx | 7 +- .../get-profile-data.test.ts | 475 +++++++++++------- .../get-profile-data.ts | 208 +++++--- 5 files changed, 558 insertions(+), 312 deletions(-) diff --git a/pyroscope/src/model/api-types.ts b/pyroscope/src/model/api-types.ts index a8f68c286..3d792d4e6 100644 --- a/pyroscope/src/model/api-types.ts +++ b/pyroscope/src/model/api-types.ts @@ -12,51 +12,111 @@ // limitations under the License. /** - * Request parameters of Pyroscope HTTP API endpoint GET /pyroscope/render - * https://grafana.com/docs/pyroscope/latest/reference-server-api/#querying-profile-data + * Output format for SelectMergeStacktraces, serialized as the protobuf enum + * name. When omitted, the server defaults to the flame graph format. */ -export interface SearchProfilesParameters { - query: string; - /** the start time can be an absolute value (number) or a relative value (string). Ex of relative value : now-15m*/ - from: string | number; - /** end of the search window, default : now */ - until?: string | number; - /** format of the returned profiling data, default: json */ - format?: 'json' | 'dot'; - /** maximum number of nodes the resulting flame graph will contain, default: 50 */ +export type ProfileFormat = + | 'PROFILE_FORMAT_UNSPECIFIED' + | 'PROFILE_FORMAT_FLAMEGRAPH' + | 'PROFILE_FORMAT_TREE' + | 'PROFILE_FORMAT_DOT' + | 'PROFILE_FORMAT_PPROF'; + +/** + * Aggregation function applied by SelectSeries when down-sampling to the step + * resolution. Defaults to SUM when omitted. + */ +export type TimeSeriesAggregationType = 'TIME_SERIES_AGGREGATION_TYPE_SUM' | 'TIME_SERIES_AGGREGATION_TYPE_AVERAGE'; + +/** + * Request body of POST /querier.v1.QuerierService/SelectMergeStacktraces. + * Returns matching profiles aggregated into a Flamegraph. + * https://grafana.com/docs/pyroscope/latest/reference-server-api/#querierv1querierserviceselectmergestacktraces + */ +export interface SelectMergeStacktracesRequest { + /** Profile type ID: :::: */ + profileTypeID: string; + /** Label selector string, e.g. `{service_name="my_service"}` */ + labelSelector: string; + /** Start of the query window, milliseconds since epoch */ + start: number; + /** End of the query window, milliseconds since epoch */ + end: number; + /** Caps the number of nodes in the returned flame graph */ maxNodes?: number; - groupeBy?: string; + /** Output format; defaults to the flame graph format when omitted */ + format?: ProfileFormat; } /** - * Response of Pyroscope HTTP API endpoint GET /pyroscope/render - * https://grafana.com/docs/pyroscope/latest/reference-server-api/#query-output + * Response of POST /querier.v1.QuerierService/SelectMergeStacktraces. */ -export interface SearchProfilesResponse { - flamebearer: Flamebearer; - metadata: Metadata; - timeline: Timeline; +export interface SelectMergeStacktracesResponse { + flamegraph: FlameGraph; } -export interface Flamebearer { +/** + * FlameGraph in the packed level representation. Each level's `values` array + * is a flat list of 4-tuples (offset, total, self, nameIndex) + * https://github.com/grafana/pyroscope/blob/788a581f23db4af50c2a5ebc81da2d5959ace8f6/api/querier/v1/querier.proto#L157 + */ +export interface FlameGraph { names: string[]; - levels: number[][]; - numTicks: number; - maxSelf: number; + levels: Level[]; + // int64 fields are serialized as JSON strings by the Connect/protobuf encoding, so `total`, + // `maxSelf`, and every `Level.values` entry must be coerced to a number before use. + total: number | string; + maxSelf: number | string; +} + +export interface Level { + values: Array; +} + +/** + * Request body of POST /querier.v1.QuerierService/SelectSeries. + * Returns the time series for the total of the matching profiles. + * https://grafana.com/docs/pyroscope/latest/reference-server-api/#querierv1querierserviceselectseries + */ +export interface SelectSeriesRequest { + /** Profile type ID: :::: */ + profileTypeID: string; + /** Label selector string, e.g. `{service_name="my_service"}` */ + labelSelector: string; + /** Start of the query window, milliseconds since epoch */ + start: number; + /** End of the query window, milliseconds since epoch */ + end: number; + /** Query resolution step width, in seconds */ + step: number; + /** Aggregation function; defaults to SUM when omitted */ + aggregation?: TimeSeriesAggregationType; + /** Labels to group the series by */ + groupBy?: string[]; +} + +/** + * Response of POST /querier.v1.QuerierService/SelectSeries. + */ +export interface SelectSeriesResponse { + series: Series[]; +} + +export interface Series { + labels: LabelPair[]; + points: Point[]; } -export interface Metadata { - format: 'single' | 'double'; - spyName: string; - sampleRate: number; - units: string; +export interface LabelPair { name: string; + value: string; } -export interface Timeline { - startTime: number; - samples: number[]; - durationDelta: number; +export interface Point { + /** Sample value (protobuf double, encoded as a JSON number). */ + value: number; + /** Milliseconds unix timestamp (protobuf int64, serialized as a JSON string). */ + timestamp: number | string; } /** diff --git a/pyroscope/src/model/pyroscope-client.ts b/pyroscope/src/model/pyroscope-client.ts index ec4fa06bb..fe68da815 100644 --- a/pyroscope/src/model/pyroscope-client.ts +++ b/pyroscope/src/model/pyroscope-client.ts @@ -14,14 +14,16 @@ import { DatasourceClient } from '@perses-dev/plugin-system'; import { RequestHeaders } from '@perses-dev/client'; import { - SearchProfilesParameters, - SearchProfilesResponse, SearchProfileTypesParameters, SearchProfileTypesResponse, SearchLabelNamesParameters, SearchLabelNamesResponse, SearchLabelValuesParameters, SearchLabelValuesResponse, + SelectMergeStacktracesRequest, + SelectMergeStacktracesResponse, + SelectSeriesRequest, + SelectSeriesResponse, } from './api-types'; interface PyroscopeClientOptions { @@ -31,7 +33,11 @@ interface PyroscopeClientOptions { export interface PyroscopeClient extends DatasourceClient { options: PyroscopeClientOptions; - searchProfiles(params: SearchProfilesParameters, headers?: RequestHeaders): Promise; + selectMergeStacktraces( + body: SelectMergeStacktracesRequest, + headers?: RequestHeaders + ): Promise; + selectSeries(body: SelectSeriesRequest, headers?: RequestHeaders): Promise; searchProfileTypes( params: SearchProfileTypesParameters, headers: RequestHeaders, @@ -69,26 +75,11 @@ export const executeRequest = async (...args: Parameters } }; -function fetchWithGet(apiURI: string, params: T | null, queryOptions: QueryOptions): Promise { - const { datasourceUrl, headers = {} } = queryOptions; - - let url = `${datasourceUrl}${apiURI}`; - if (params) { - url += '?' + new URLSearchParams(params); - } - const init = { - method: 'GET', - headers, - }; - - return executeRequest(url, init); -} - function fetchWithPost( apiURI: string, params: T | null, queryOptions: QueryOptions, - body: Record + body: object ): Promise { const { datasourceUrl, headers = {} } = queryOptions; @@ -98,7 +89,7 @@ function fetchWithPost( } const init = { method: 'POST', - headers, + headers: { 'content-type': 'application/json', ...headers }, body: JSON.stringify(body), }; @@ -106,13 +97,30 @@ function fetchWithPost( } /** - * Returns profiling data. + * Returns the flame graph for the matching profiles. */ -export function searchProfiles( - params: SearchProfilesParameters, +export function selectMergeStacktraces( + body: SelectMergeStacktracesRequest, queryOptions: QueryOptions -): Promise { - return fetchWithGet('/pyroscope/render', params, queryOptions); +): Promise { + return fetchWithPost, SelectMergeStacktracesResponse>( + '/querier.v1.QuerierService/SelectMergeStacktraces', + null, + queryOptions, + body + ); +} + +/** + * Returns the time series (timeline) for the matching profiles. + */ +export function selectSeries(body: SelectSeriesRequest, queryOptions: QueryOptions): Promise { + return fetchWithPost, SelectSeriesResponse>( + '/querier.v1.QuerierService/SelectSeries', + null, + queryOptions, + body + ); } /** diff --git a/pyroscope/src/plugins/pyroscope-datasource.tsx b/pyroscope/src/plugins/pyroscope-datasource.tsx index 26d44cbeb..5079503cc 100644 --- a/pyroscope/src/plugins/pyroscope-datasource.tsx +++ b/pyroscope/src/plugins/pyroscope-datasource.tsx @@ -14,7 +14,8 @@ import { DatasourcePlugin } from '@perses-dev/plugin-system'; import { PyroscopeClient, - searchProfiles, + selectMergeStacktraces, + selectSeries, searchProfileTypes, searchLabelNames, searchLabelValues, @@ -42,7 +43,9 @@ const createClient: DatasourcePlugin[' options: { datasourceUrl, }, - searchProfiles: (params, headers) => searchProfiles(params, { datasourceUrl, headers: headers ?? specHeaders }), + selectMergeStacktraces: (body, headers) => + selectMergeStacktraces(body, { datasourceUrl, headers: headers ?? specHeaders }), + selectSeries: (body, headers) => selectSeries(body, { datasourceUrl, headers: headers ?? specHeaders }), searchProfileTypes: (params, headers, body) => searchProfileTypes(params, { datasourceUrl, headers: headers ?? specHeaders }, body), searchLabelNames: (params, headers, body) => diff --git a/pyroscope/src/plugins/pyroscope-profile-query/get-profile-data.test.ts b/pyroscope/src/plugins/pyroscope-profile-query/get-profile-data.test.ts index 09be0ec53..b9f8d9a90 100644 --- a/pyroscope/src/plugins/pyroscope-profile-query/get-profile-data.test.ts +++ b/pyroscope/src/plugins/pyroscope-profile-query/get-profile-data.test.ts @@ -11,11 +11,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { ProfileData, StackTrace } from '@perses-dev/spec'; -import { SearchProfilesResponse } from '../../model'; -import { transformProfileResponse } from './get-profile-data'; - -// Flamebearer levels use groups of 4 numbers: [offset, total, self, nameIndex]. +import { ProfileQueryContext } from '@perses-dev/plugin-system'; +import { StackTrace } from '@perses-dev/spec'; +import { + FlameGraph, + Series, + PyroscopeClient, + PyroscopeProfileQuerySpec, + SelectMergeStacktracesRequest, + SelectSeriesRequest, +} from '../../model'; +import { getProfileData, transformFlameGraph, transformTimeline } from './get-profile-data'; + +// Flamegraph levels use groups of 4 numbers: [offset, total, self, nameIndex]. // `offset` is the gap (in samples) since the end of the previous sibling on the same level. // // root [0,10) @@ -23,29 +31,16 @@ import { transformProfileResponse } from './get-profile-data'; // foo [0,6) bar [6,10) // / \ \ // baz [0,3) qux [3,5) quux [6,10) -const MOCK_RESPONSE: SearchProfilesResponse = { - flamebearer: { - names: ['root', 'foo', 'bar', 'baz', 'qux', 'quux'], - levels: [ - [0, 10, 2, 0], - [0, 6, 1, 1, 0, 4, 4, 2], - [0, 3, 3, 3, 0, 2, 2, 4, 1, 4, 4, 5], - ], - numTicks: 42, - maxSelf: 4, - }, - metadata: { - format: 'single', - spyName: 'gospy', - sampleRate: 100, - units: 'samples', - name: 'process_cpu', - }, - timeline: { - startTime: 1_600_000_000, - samples: [1, 2, 3], - durationDelta: 10, - }, +// int64 values arrive as JSON strings over the Connect API; the transform must coerce them. +const MOCK_FLAMEGRAPH: FlameGraph = { + names: ['root', 'foo', 'bar', 'baz', 'qux', 'quux'], + levels: [ + { values: ['0', '10', '2', '0'] }, + { values: ['0', '6', '1', '1', '0', '4', '4', '2'] }, + { values: ['0', '3', '3', '3', '0', '2', '2', '4', '1', '4', '4', '5'] }, + ], + total: '42', + maxSelf: '4', }; // A deeper, wider tree with gaps between siblings (self time) and a function ("recurse") @@ -58,45 +53,208 @@ const MOCK_RESPONSE: SearchProfilesResponse = { // recurse [2,10) recurse [12,18) // / \ // recurse [2,6) helper [7,9) -const MOCK_COMPLEX_RESPONSE: SearchProfilesResponse = { - flamebearer: { - names: ['root', 'main', 'cleanup', 'recurse', 'helper'], - levels: [ - [0, 20, 0, 0], - [0, 18, 4, 1, 0, 2, 2, 2], - [2, 8, 2, 3, 2, 6, 6, 3], - [2, 4, 4, 3, 1, 2, 2, 4], - ], - numTicks: 20, - maxSelf: 6, - }, - metadata: { - format: 'single', - spyName: 'gospy', - sampleRate: 100, - units: 'samples', - name: 'process_cpu', - }, - timeline: { - startTime: 1_600_000_000, - samples: [1, 2, 3], - durationDelta: 10, - }, +const MOCK_COMPLEX_FLAMEGRAPH: FlameGraph = { + names: ['root', 'main', 'cleanup', 'recurse', 'helper'], + levels: [ + { values: ['0', '20', '0', '0'] }, + { values: ['0', '18', '4', '1', '0', '2', '2', '2'] }, + { values: ['2', '8', '2', '3', '2', '6', '6', '3'] }, + { values: ['2', '4', '4', '3', '1', '2', '2', '4'] }, + ], + total: '20', + maxSelf: '6', +}; + +const EMPTY_STACK_TRACE: StackTrace = { id: 0, name: '', level: 0, start: 0, end: 0, total: 0, self: 0, children: [] }; + +// Builds a stub PyroscopeClient. Individual test cases override the two methods getProfileData +// actually calls; the rest are present (as no-op mocks) only to satisfy the PyroscopeClient type. +function makeClient(overrides: Partial = {}): PyroscopeClient { + return { + options: { datasourceUrl: 'http://example.com' }, + selectMergeStacktraces: jest.fn(), + selectSeries: jest.fn(), + searchProfileTypes: jest.fn(), + searchLabelNames: jest.fn(), + searchLabelValues: jest.fn(), + searchServices: jest.fn(), + ...overrides, + }; +} + +// Builds a stub ProfileQueryContext. `absoluteTimeRange` is omitted by default, matching the +// "no time range selected" case; individual tests pass one in to exercise the other branch. +function createContext( + client: PyroscopeClient, + absoluteTimeRange?: ProfileQueryContext['absoluteTimeRange'] +): ProfileQueryContext { + return { + datasourceStore: { + getDatasource: jest.fn(), + getDatasourceClient: jest.fn(() => Promise.resolve(client)), + listDatasourceSelectItems: jest.fn(async () => []), + getLocalDatasources: jest.fn(), + setLocalDatasources: jest.fn(), + getSavedDatasources: jest.fn(), + setSavedDatasources: jest.fn(), + }, + absoluteTimeRange, + } as ProfileQueryContext; +} + +const BASE_SPEC: PyroscopeProfileQuerySpec = { + profileType: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds', + service: 'my-service', + filters: [{ labelName: 'env', operator: '=', labelValue: 'prod' }], + maxNodes: 100, }; -describe('transformProfileResponse', () => { - it('returns empty profile data when there is no response', () => { - expect(transformProfileResponse(undefined as unknown as SearchProfilesResponse)).toEqual({ - profile: { stackTrace: { id: 0, name: '', level: 0, start: 0, end: 0, total: 0, self: 0, children: [] } }, +// A 1-hour window with whole-second boundaries, so the derived milliseconds and step are exact. +const TIME_RANGE = { + start: new Date('2024-06-11T10:00:00.000Z'), + end: new Date('2024-06-11T11:00:00.000Z'), +}; + +describe('getProfileData', () => { + const EMPTY_PROFILE_DATA = { + profile: { stackTrace: EMPTY_STACK_TRACE }, + numTicks: 0, + maxSelf: 0, + metadata: { spyName: '', sampleRate: 0, units: '', name: '' }, + timeline: { startTime: 0, samples: [], durationDelta: 0 }, + }; + + it('returns empty profile data without resolving a client when profileType is missing', async () => { + const client = makeClient(); + const context = createContext(client, TIME_RANGE); + + const result = await getProfileData({ ...BASE_SPEC, profileType: '' }, context); + + expect(context.datasourceStore.getDatasourceClient).not.toHaveBeenCalled(); + expect(result).toEqual(EMPTY_PROFILE_DATA); + }); + + it('returns empty profile data without resolving a client when service is missing', async () => { + const client = makeClient(); + const context = createContext(client, TIME_RANGE); + + const result = await getProfileData({ ...BASE_SPEC, service: undefined }, context); + + expect(context.datasourceStore.getDatasourceClient).not.toHaveBeenCalled(); + expect(result).toEqual(EMPTY_PROFILE_DATA); + }); + + it('builds the flame graph and timeline requests from the spec and time range', async () => { + const selectMergeStacktraces = jest.fn().mockResolvedValue({ flamegraph: MOCK_FLAMEGRAPH }); + const selectSeries = jest.fn().mockResolvedValue({ series: [] }); + const client = makeClient({ selectMergeStacktraces, selectSeries }); + + await getProfileData(BASE_SPEC, createContext(client, TIME_RANGE)); + + const expectedStacktracesRequest: SelectMergeStacktracesRequest = { + profileTypeID: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds', + labelSelector: '{service_name="my-service",env="prod"}', + start: 1_718_100_000_000, + end: 1_718_103_600_000, + maxNodes: 100, + }; + expect(selectMergeStacktraces).toHaveBeenCalledWith(expectedStacktracesRequest); + + const expectedSeriesRequest: SelectSeriesRequest = { + profileTypeID: 'process_cpu:cpu:nanoseconds:cpu:nanoseconds', + labelSelector: '{service_name="my-service",env="prod"}', + start: 1_718_100_000_000, + end: 1_718_103_600_000, + step: 10, // 3600s window / 1000 target points = 3.6, floored to 3, floored up to the 10s minimum + aggregation: 'TIME_SERIES_AGGREGATION_TYPE_SUM', + }; + expect(selectSeries).toHaveBeenCalledWith(expectedSeriesRequest); + }); + + it('omits maxNodes from the flame graph request when the spec does not set it', async () => { + const selectMergeStacktraces = jest.fn().mockResolvedValue({ flamegraph: MOCK_FLAMEGRAPH }); + const selectSeries = jest.fn().mockResolvedValue({ series: [] }); + const client = makeClient({ selectMergeStacktraces, selectSeries }); + + await getProfileData({ ...BASE_SPEC, maxNodes: undefined }, createContext(client, TIME_RANGE)); + + expect(selectMergeStacktraces).toHaveBeenCalledWith(expect.not.objectContaining({ maxNodes: expect.anything() })); + }); + + it('builds a label selector with only service_name when there are no filters', async () => { + const selectMergeStacktraces = jest.fn().mockResolvedValue({ flamegraph: MOCK_FLAMEGRAPH }); + const selectSeries = jest.fn().mockResolvedValue({ series: [] }); + const client = makeClient({ selectMergeStacktraces, selectSeries }); + + await getProfileData({ ...BASE_SPEC, filters: undefined }, createContext(client, TIME_RANGE)); + + expect(selectMergeStacktraces).toHaveBeenCalledWith( + expect.objectContaining({ labelSelector: '{service_name="my-service"}' }) + ); + }); + + it('defaults to the last hour ending now when no absolute time range is provided', async () => { + const fixedNowMs = 1_718_100_000_000; + jest.spyOn(Date, 'now').mockReturnValue(fixedNowMs); + + const selectMergeStacktraces = jest.fn().mockResolvedValue({ flamegraph: MOCK_FLAMEGRAPH }); + const selectSeries = jest.fn().mockResolvedValue({ series: [] }); + const client = makeClient({ selectMergeStacktraces, selectSeries }); + + await getProfileData(BASE_SPEC, createContext(client, undefined)); + + expect(selectMergeStacktraces).toHaveBeenCalledWith( + expect.objectContaining({ start: fixedNowMs - 3_600_000, end: fixedNowMs }) + ); + + jest.spyOn(Date, 'now').mockRestore(); + }); + + it('assembles the final ProfileData from the flame graph and timeline responses', async () => { + const seriesResponse: Series[] = [ + { + labels: [], + points: [ + { timestamp: '1600000000000', value: 1 }, + { timestamp: '1600000010000', value: 2 }, + ], + }, + ]; + const selectMergeStacktraces = jest.fn().mockResolvedValue({ flamegraph: MOCK_FLAMEGRAPH }); + const selectSeries = jest.fn().mockResolvedValue({ series: seriesResponse }); + const client = makeClient({ selectMergeStacktraces, selectSeries }); + + const result = await getProfileData(BASE_SPEC, createContext(client, TIME_RANGE)); + + expect(result.numTicks).toBe(42); + expect(result.maxSelf).toBe(4); + expect(result.profile.stackTrace.name).toBe('root'); + expect(result.metadata).toEqual({ spyName: '', sampleRate: 0, units: 'nanoseconds', name: 'process_cpu' }); + expect(result.timeline).toEqual({ startTime: 1_600_000_000, samples: [1, 2], durationDelta: 10 }); + }); + + it('propagates a rejection if either request fails', async () => { + const selectMergeStacktraces = jest.fn().mockRejectedValue(new Error('flame graph request failed')); + const selectSeries = jest.fn().mockResolvedValue({ series: [] }); + const client = makeClient({ selectMergeStacktraces, selectSeries }); + + await expect(getProfileData(BASE_SPEC, createContext(client, TIME_RANGE))).rejects.toThrow( + 'flame graph request failed' + ); + }); +}); + +describe('transformFlameGraph', () => { + it('returns an empty stack trace when there is no flame graph', () => { + expect(transformFlameGraph(undefined)).toEqual({ + stackTrace: EMPTY_STACK_TRACE, numTicks: 0, maxSelf: 0, - metadata: { spyName: '', sampleRate: 0, units: '', name: '' }, - timeline: { startTime: 0, samples: [], durationDelta: 0 }, }); }); - it('builds the stack trace tree from the flamebearer levels', () => { - const result = transformProfileResponse(MOCK_RESPONSE); + it('builds the stack trace tree from the flame graph levels', () => { + const result = transformFlameGraph(MOCK_FLAMEGRAPH); const baz: StackTrace = { id: 4, name: 'baz', level: 2, start: 0, end: 3, total: 3, self: 3, children: [] }; const qux: StackTrace = { id: 5, name: 'qux', level: 2, start: 3, end: 5, total: 2, self: 2, children: [] }; @@ -123,49 +281,24 @@ describe('transformProfileResponse', () => { children: [foo, bar], }; - expect(result.profile.stackTrace).toEqual(root); + expect(result.stackTrace).toEqual(root); }); - it('passes through numTicks, maxSelf, metadata and timeline unchanged', () => { - const result = transformProfileResponse(MOCK_RESPONSE); + it('passes through numTicks (flame graph total) and maxSelf', () => { + const result = transformFlameGraph(MOCK_FLAMEGRAPH); expect(result.numTicks).toBe(42); expect(result.maxSelf).toBe(4); - expect(result.metadata).toEqual({ - spyName: 'gospy', - sampleRate: 100, - units: 'samples', - name: 'process_cpu', - }); - expect(result.timeline).toEqual({ - startTime: 1_600_000_000, - samples: [1, 2, 3], - durationDelta: 10, - }); }); it('returns an empty stack trace when there are no levels', () => { - const response: SearchProfilesResponse = { - ...MOCK_RESPONSE, - flamebearer: { ...MOCK_RESPONSE.flamebearer, levels: [] }, - }; + const result = transformFlameGraph({ ...MOCK_FLAMEGRAPH, levels: [] }); - const result = transformProfileResponse(response); - - expect(result.profile.stackTrace).toEqual({ - id: 0, - name: '', - level: 0, - start: 0, - end: 0, - total: 0, - self: 0, - children: [], - }); + expect(result.stackTrace).toEqual(EMPTY_STACK_TRACE); }); it('builds a deeper tree with multiple siblings, gaps between children, and a name reused across call sites', () => { - const result = transformProfileResponse(MOCK_COMPLEX_RESPONSE); + const result = transformFlameGraph(MOCK_COMPLEX_FLAMEGRAPH); const recurseLeaf: StackTrace = { id: 6, @@ -229,31 +362,59 @@ describe('transformProfileResponse', () => { children: [main, cleanup], }; - expect(result.profile.stackTrace).toEqual(root); + expect(result.stackTrace).toEqual(root); }); }); -// copy of transformProfileResponse as it existed before the O(n) rewrite -function legacyTransformProfileResponse(response: SearchProfilesResponse): ProfileData { - const newResponse: ProfileData = { - profile: { stackTrace: {} as StackTrace }, - numTicks: 0, - maxSelf: 0, - metadata: { spyName: '', sampleRate: 0, units: '', name: '' }, - timeline: { startTime: 0, samples: [], durationDelta: 0 }, - }; +describe('transformTimeline', () => { + it('returns an empty timeline when there is no series', () => { + expect(transformTimeline(undefined, 10)).toEqual({ startTime: 0, samples: [], durationDelta: 0 }); + expect(transformTimeline([], 10)).toEqual({ startTime: 0, samples: [], durationDelta: 0 }); + }); - if (!response) { - return newResponse; - } + it('maps series points to samples and derives startTime (seconds) and durationDelta from the points', () => { + const series: Series[] = [ + { + labels: [], + points: [ + { timestamp: '1600000000000', value: 1 }, + { timestamp: '1600000010000', value: 2 }, + { timestamp: '1600000020000', value: 3 }, + ], + }, + ]; + + expect(transformTimeline(series, 30)).toEqual({ + startTime: 1_600_000_000, // ms -> s + samples: [1, 2, 3], + durationDelta: 10, // 10_000 ms gap -> 10 s + }); + }); + + it('falls back to the requested step when there is only a single point', () => { + const series: Series[] = [{ labels: [], points: [{ timestamp: '1600000000000', value: 5 }] }]; + expect(transformTimeline(series, 15)).toEqual({ + startTime: 1_600_000_000, + samples: [5], + durationDelta: 15, + }); + }); +}); + +// copy of transformProfileResponse as it existed before the O(n) rewrite +function legacyTransformFlameGraph(flamegraph: FlameGraph): { + stackTrace: StackTrace; + numTicks: number; + maxSelf: number; +} { const stackTraces: StackTrace[][] = []; let id = 1; - for (let i = 0; i < response.flamebearer.levels.length; i++) { + for (let i = 0; i < flamegraph.levels.length; i++) { let current = 0; const row: StackTrace[] = []; - const level = response.flamebearer.levels[i]; + const level = flamegraph.levels[i]?.values; if (!level) { continue; } @@ -264,7 +425,7 @@ function legacyTransformProfileResponse(response: SearchProfilesResponse): Profi id += 1; const indexInNamesArray = level[j + 3]; if (indexInNamesArray !== undefined) { - const name = response.flamebearer.names[indexInNamesArray]; + const name = flamegraph.names[Number(indexInNamesArray)]; if (name) { temp.name = name; } @@ -273,21 +434,21 @@ function legacyTransformProfileResponse(response: SearchProfilesResponse): Profi const total = level[j + 1]; if (total !== undefined) { - temp.total = total; + temp.total = Number(total); } const self = level[j + 2]; if (self !== undefined) { - temp.self = self; + temp.self = Number(self); } const offset = level[j]; if (offset !== undefined) { - current += offset; + current += Number(offset); } temp.start = current; if (total !== undefined) { - current += total; + current += Number(total); } temp.end = current; temp.children = []; @@ -299,25 +460,9 @@ function legacyTransformProfileResponse(response: SearchProfilesResponse): Profi } legacyAddChildren(stackTraces); - if (stackTraces[0]?.[0]) { - newResponse.profile.stackTrace = stackTraces[0][0]; - } - - newResponse.numTicks = response.flamebearer.numTicks; - newResponse.maxSelf = response.flamebearer.maxSelf; - newResponse.metadata = { - spyName: response.metadata.spyName, - sampleRate: response.metadata.sampleRate, - units: response.metadata.units, - name: response.metadata.name, - }; - newResponse.timeline = { - startTime: response.timeline.startTime, - samples: response.timeline.samples, - durationDelta: response.timeline.durationDelta, - }; - return newResponse; + const stackTrace = stackTraces[0]?.[0] ?? ({} as StackTrace); + return { stackTrace, numTicks: Number(flamegraph.total), maxSelf: Number(flamegraph.maxSelf) }; } function legacyAddChildren(stackTraces: StackTrace[][]): void { @@ -413,11 +558,11 @@ function buildRandomTree( return { start, end, self: end - cursor, name, children }; } -// Flattens the generated tree into the same flat, per-level [offset, total, self, nameIndex] encoding -// used by the real Pyroscope API (offset is cumulative across the whole level, not per-parent). -function toSearchProfilesResponse(root: GeneratedNode): SearchProfilesResponse { +// Flattens the generated tree into the same flat, per-level [offset, total, self, nameIndex] +// encoding used by the real Pyroscope API (offset is cumulative across the whole level). +function toFlameGraph(root: GeneratedNode): FlameGraph { const names: string[] = []; - const levels: number[][] = []; + const levels: FlameGraph['levels'] = []; let currentLevel: GeneratedNode[] = [root]; while (currentLevel.length > 0) { @@ -436,38 +581,14 @@ function toSearchProfilesResponse(root: GeneratedNode): SearchProfilesResponse { nextLevel.push(...node.children); } - levels.push(rawLevel); + levels.push({ values: rawLevel }); currentLevel = nextLevel; } - return { - flamebearer: { names, levels, numTicks: root.end - root.start, maxSelf: root.self }, - metadata: { format: 'single', spyName: 'gospy', sampleRate: 100, units: 'samples', name: 'process_cpu' }, - timeline: { startTime: 0, samples: [], durationDelta: 0 }, - }; + return { names, levels, total: root.end - root.start, maxSelf: root.self }; } -describe('transformProfileResponse (regression against the pre-optimization implementation)', () => { - // this test cases is the only one that create divergent output - // the legacy fallback for a missing response returned a {} - // but new rewrite of transformProfileResponse returns a fully populated stackTrace - it('changes the empty-response fallback from an incomplete object to a valid empty StackTrace', () => { - const legacy = legacyTransformProfileResponse(undefined as unknown as SearchProfilesResponse); - const current = transformProfileResponse(undefined as unknown as SearchProfilesResponse); - - expect(legacy.profile.stackTrace).toEqual({}); - expect(current.profile.stackTrace).toEqual({ - id: 0, - name: '', - level: 0, - start: 0, - end: 0, - total: 0, - self: 0, - children: [], - }); - }); - +describe('transformFlameGraph (regression against the pre-optimization implementation)', () => { it('produces identical output to the legacy implementation for many randomly generated profiles', () => { // Several independent seeds rather than one, so coverage isn't at the mercy of a single PRNG // stream happening (or failing) to hit a given structural edge case. @@ -479,9 +600,9 @@ describe('transformProfileResponse (regression against the pre-optimization impl for (let i = 0; i < 200; i++) { const totalSamples = 20 + Math.floor(rng() * 500); const maxDepth = 2 + Math.floor(rng() * 6); - const response = toSearchProfilesResponse(buildRandomTree(rng, 0, totalSamples, 0, maxDepth, [])); + const flamegraph = toFlameGraph(buildRandomTree(rng, 0, totalSamples, 0, maxDepth, [])); - expect(transformProfileResponse(response)).toEqual(legacyTransformProfileResponse(response)); + expect(transformFlameGraph(flamegraph)).toEqual(legacyTransformFlameGraph(flamegraph)); } } }); @@ -490,28 +611,26 @@ describe('transformProfileResponse (regression against the pre-optimization impl const names = ['w0', 'w1', 'w2', 'w3', 'w4', 'w5', 'w6', 'w7', 'w8', 'w9']; // 10 siblings at the same level, back-to-back with no gaps, each a leaf - stresses the O(n) // parentIndex cursor advancing across many same-level nodes under a single parent. - const levels: number[][] = [ - [0, 100, 0, 0], - [ - 0, 10, 10, 1, 0, 10, 10, 2, 0, 10, 10, 3, 0, 10, 10, 4, 0, 10, 10, 5, 0, 10, 10, 6, 0, 10, 10, 7, 0, 10, 10, 8, - 0, 10, 10, 9, 0, 10, 10, 0, - ], + const levels: FlameGraph['levels'] = [ + { values: [0, 100, 0, 0] }, + { + values: [ + 0, 10, 10, 1, 0, 10, 10, 2, 0, 10, 10, 3, 0, 10, 10, 4, 0, 10, 10, 5, 0, 10, 10, 6, 0, 10, 10, 7, 0, 10, 10, + 8, 0, 10, 10, 9, 0, 10, 10, 0, + ], + }, ]; - const response: SearchProfilesResponse = { - flamebearer: { names, levels, numTicks: 100, maxSelf: 10 }, - metadata: { format: 'single', spyName: 'gospy', sampleRate: 100, units: 'samples', name: 'process_cpu' }, - timeline: { startTime: 0, samples: [], durationDelta: 0 }, - }; + const flamegraph: FlameGraph = { names, levels, total: 100, maxSelf: 10 }; - expect(transformProfileResponse(response)).toEqual(legacyTransformProfileResponse(response)); + expect(transformFlameGraph(flamegraph)).toEqual(legacyTransformFlameGraph(flamegraph)); }); it('produces identical output to the legacy implementation for a deep single-branch chain (recursion-like)', () => { const depth = 30; const root = buildChain(depth); - const response = toSearchProfilesResponse(root); + const flamegraph = toFlameGraph(root); - expect(transformProfileResponse(response)).toEqual(legacyTransformProfileResponse(response)); + expect(transformFlameGraph(flamegraph)).toEqual(legacyTransformFlameGraph(flamegraph)); }); it('produces identical output to the legacy implementation when children exactly touch the parent boundaries', () => { @@ -527,9 +646,9 @@ describe('transformProfileResponse (regression against the pre-optimization impl { start: 5, end: 10, self: 5, name: 'right', children: [] }, ], }; - const response = toSearchProfilesResponse(root); + const flamegraph = toFlameGraph(root); - expect(transformProfileResponse(response)).toEqual(legacyTransformProfileResponse(response)); + expect(transformFlameGraph(flamegraph)).toEqual(legacyTransformFlameGraph(flamegraph)); }); }); diff --git a/pyroscope/src/plugins/pyroscope-profile-query/get-profile-data.ts b/pyroscope/src/plugins/pyroscope-profile-query/get-profile-data.ts index 9507e8191..19f4a820d 100644 --- a/pyroscope/src/plugins/pyroscope-profile-query/get-profile-data.ts +++ b/pyroscope/src/plugins/pyroscope-profile-query/get-profile-data.ts @@ -16,14 +16,23 @@ import { getUnixTime } from 'date-fns'; import { AbsoluteTimeRange, ProfileData, StackTrace } from '@perses-dev/spec'; import { PyroscopeProfileQuerySpec, - PYROSCOPE_DATASOURCE_KIND, - PyroscopeDatasourceSelector, + isProfileQueryComplete, + DEFAULT_PYROSCOPE, PyroscopeClient, - SearchProfilesParameters, - SearchProfilesResponse, + SelectMergeStacktracesRequest, + SelectSeriesRequest, + FlameGraph, + Series, } from '../../model'; import { computeFilterExpr } from '../../utils/types'; +// Pyroscope's Connect API expects timestamps in milliseconds; Perses time ranges are in seconds. +const MILLISECONDS = 1_000; + +// Timeline resolution: target at most this many points, but never a step below MIN_STEP_SECONDS +const TIMELINE_TARGET_POINTS = 1_000; +const MIN_STEP_SECONDS = 10; + export function getUnixTimeRange(timeRange: AbsoluteTimeRange): { start: number; end: number } { const { start, end } = timeRange; return { @@ -36,78 +45,116 @@ export const getProfileData: ProfileQueryPlugin['getP spec, context ) => { - const defaultPyroscopeDatasource: PyroscopeDatasourceSelector = { - kind: PYROSCOPE_DATASOURCE_KIND, - }; + if (!isProfileQueryComplete(spec)) { + return emptyProfileData(); + } + const profileTypeID = spec.profileType; const client: PyroscopeClient = await context.datasourceStore.getDatasourceClient( - spec.datasource ?? defaultPyroscopeDatasource + spec.datasource ?? DEFAULT_PYROSCOPE ); - const buildQueryString = (): string => { - let query: string = ''; - if (spec.service) { - query = `service_name="${spec.service}"`; - } - if (spec.filters && spec.filters.length > 0) { - const filterExpr = computeFilterExpr(spec.filters); - if (query === '') { - query = filterExpr; - } else { - query += ',' + filterExpr; - } - } - query = spec.profileType + (query === '' ? '' : '{' + query + '}'); - return query; - }; + let startSeconds: number; + let endSeconds: number; + if (context.absoluteTimeRange) { + ({ start: startSeconds, end: endSeconds } = getUnixTimeRange(context.absoluteTimeRange)); + } else { + endSeconds = Math.ceil(Date.now() / MILLISECONDS); + startSeconds = endSeconds - 3600; + } + const start = startSeconds * MILLISECONDS; + const end = endSeconds * MILLISECONDS; - const getParams = (): SearchProfilesParameters => { - const params: SearchProfilesParameters = { - // example of query - // query: `process_cpu:cpu:nanoseconds:cpu:nanoseconds{service_name="pyroscope"}`, - query: buildQueryString(), - // the default value is now-1h - from: 'now-1h', - }; - - // handle time range selection from UI drop down (e.g. last 5 minutes, last 1 hour ) - if (context.absoluteTimeRange) { - const { start, end } = getUnixTimeRange(context.absoluteTimeRange); - params.from = start; - params.until = end; - } + const labelSelector = buildLabelSelector(spec); - if (spec.maxNodes) { - params.maxNodes = spec.maxNodes; - } + const stacktracesRequest: SelectMergeStacktracesRequest = { + profileTypeID, + labelSelector, + start, + end, + }; + if (spec.maxNodes) { + stacktracesRequest.maxNodes = spec.maxNodes; + } - return params; + const step = Math.max(MIN_STEP_SECONDS, Math.floor((endSeconds - startSeconds) / TIMELINE_TARGET_POINTS)); + const seriesRequest: SelectSeriesRequest = { + profileTypeID, + labelSelector, + start, + end, + step, + aggregation: 'TIME_SERIES_AGGREGATION_TYPE_SUM', }; - const response = await client.searchProfiles(getParams()); + const [stacktracesResponse, seriesResponse] = await Promise.all([ + client.selectMergeStacktraces(stacktracesRequest), + client.selectSeries(seriesRequest), + ]); + + const { stackTrace, numTicks, maxSelf } = transformFlameGraph(stacktracesResponse.flamegraph); - // return a profile data - return transformProfileResponse(response); + return { + profile: { stackTrace }, + numTicks, + maxSelf, + metadata: buildMetadata(profileTypeID), + timeline: transformTimeline(seriesResponse.series, step), + }; }; -// [offset, total, self, nameIndex]. -const FLAMEBEARER_NODE_SIZE = 4; +/** + * Builds the shared label selector string, e.g. `{service_name="app",env="prod"}`. + * Returns `{}` when neither a service nor any filter is set. + */ +function buildLabelSelector(spec: PyroscopeProfileQuerySpec): string { + const selectors: string[] = []; + if (spec.service) { + selectors.push(`service_name="${spec.service}"`); + } + if (spec.filters && spec.filters.length > 0) { + const filterExpr = computeFilterExpr(spec.filters); + if (filterExpr) { + selectors.push(filterExpr); + } + } + return `{${selectors.join(',')}}`; +} -export function transformProfileResponse(response: SearchProfilesResponse): ProfileData { - if (!response) { - return emptyProfileData(); +/** + * Derives the metadata the panel needs from the profile type ID, which has the form + * `::::`. + */ +function buildMetadata(profileTypeID: string): ProfileData['metadata'] { + const parts = profileTypeID.split(':'); + return { + spyName: '', + sampleRate: 0, + units: parts[2] ?? '', + name: parts[0] ?? '', + }; +} + +// [offset, total, self, nameIndex]. +const FLAME_GRAPH_NODE_SIZE = 4; + +export function transformFlameGraph(flamegraph: FlameGraph | undefined): { + stackTrace: StackTrace; + numTicks: number; + maxSelf: number; +} { + if (!flamegraph) { + return { stackTrace: emptyStackTrace(), numTicks: 0, maxSelf: 0 }; } - const { flamebearer, metadata, timeline } = response; + const { names, levels } = flamegraph; let id = 1; - let root: StackTrace | undefined; - let parentLevel: StackTrace[] = []; - for (let depth = 0; depth < flamebearer.levels.length; depth++) { - const level = flamebearer.levels[depth]; + for (let depth = 0; depth < levels.length; depth++) { + const level = levels[depth]?.values; if (!level) { continue; } @@ -116,11 +163,11 @@ export function transformProfileResponse(response: SearchProfilesResponse): Prof let cursor = 0; let parentIndex = 0; - for (let slot = 0; slot < level.length; slot += FLAMEBEARER_NODE_SIZE) { - const offset = level[slot] ?? 0; - const total = level[slot + 1] ?? 0; - const self = level[slot + 2] ?? 0; - const nameIndex = level[slot + 3] ?? 0; + for (let slot = 0; slot < level.length; slot += FLAME_GRAPH_NODE_SIZE) { + const offset = Number(level[slot] ?? 0); + const total = Number(level[slot + 1] ?? 0); + const self = Number(level[slot + 2] ?? 0); + const nameIndex = Number(level[slot + 3] ?? 0); const start = cursor + offset; const end = start + total; @@ -128,7 +175,7 @@ export function transformProfileResponse(response: SearchProfilesResponse): Prof const node: StackTrace = { id: id++, - name: flamebearer.names[nameIndex] ?? '', + name: names[nameIndex] ?? '', level: depth, start, end, @@ -159,20 +206,29 @@ export function transformProfileResponse(response: SearchProfilesResponse): Prof } return { - profile: { stackTrace: root ?? emptyStackTrace() }, - numTicks: flamebearer.numTicks, - maxSelf: flamebearer.maxSelf, - metadata: { - spyName: metadata.spyName, - sampleRate: metadata.sampleRate, - units: metadata.units, - name: metadata.name, - }, - timeline: { - startTime: timeline.startTime, - samples: timeline.samples, - durationDelta: timeline.durationDelta, - }, + stackTrace: root ?? emptyStackTrace(), + numTicks: Number(flamegraph.total), + maxSelf: Number(flamegraph.maxSelf), + }; +} + +export function transformTimeline(series: Series[] | undefined, fallbackStepSeconds: number): ProfileData['timeline'] { + const points = series?.[0]?.points ?? []; + const first = points[0]; + if (!first) { + return { startTime: 0, samples: [], durationDelta: 0 }; + } + + const firstTimestamp = Number(first.timestamp); + const second = points[1]; + const durationDelta = second + ? Math.round((Number(second.timestamp) - firstTimestamp) / MILLISECONDS) + : fallbackStepSeconds; + + return { + startTime: Math.floor(firstTimestamp / MILLISECONDS), + samples: points.map((point) => point.value), + durationDelta, }; }