diff --git a/README.md b/README.md index 16a9e069..03d879b9 100644 --- a/README.md +++ b/README.md @@ -186,6 +186,11 @@ abs experiments list --state running --items 50 --sort created_at --desc abs experiments list --app my-app --search "checkout" --page 2 abs experiments list --created-after 2025-01-01 --tags v1,mobile +# Find experiments that use a specific metric (by ID or name), in any role +abs experiments list --metric "Revenue" +abs experiments list --metric 42 --metric-role guardrail,primary +abs experiments list --metric 42 --type test,feature --state running -o ids + # Customize columns abs experiments list --show experiment_report archived # add extra columns abs experiments list --exclude primary_metric owner # hide columns diff --git a/package.json b/package.json index 338583d5..bd973ef4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@absmartly/cli", - "version": "1.11.0", + "version": "1.12.0", "description": "ABSmartly CLI - A/B Testing and Feature Flags command-line tool for AI agents and humans", "type": "module", "main": "./dist/index.js", diff --git a/src/commands/experiments/list.ts b/src/commands/experiments/list.ts index 98316686..5f8af4a9 100644 --- a/src/commands/experiments/list.ts +++ b/src/commands/experiments/list.ts @@ -6,7 +6,7 @@ import { shouldOutputIdsOnly, withErrorHandling, } from '../../lib/utils/api-helper.js'; -import { printPaginationFooter } from '../../lib/utils/pagination.js'; +import { printPaginationFooter, printMetricFooter } from '../../lib/utils/pagination.js'; import { getDefaultType } from './default-type.js'; import { listExperiments } from '../../core/experiments/list.js'; @@ -33,6 +33,14 @@ export const listCommand = new Command('list') ) .option('--iterations ', 'filter by iteration count', (v) => parseInt(v, 10)) .option('--iterations-of ', 'show all iterations of experiment ID', (v) => parseInt(v, 10)) + .option( + '--metric ', + 'find experiments using this metric (by ID or name) in any role; scans all pages' + ) + .option( + '--metric-role ', + 'with --metric, limit to roles: primary,secondary,guardrail,exploratory (comma-separated; default: all)' + ) .option('--created-after ', 'filter experiments created after timestamp') .option('--created-before ', 'filter experiments created before timestamp') .option('--started-after ', 'filter experiments started after timestamp') @@ -82,11 +90,23 @@ export const listCommand = new Command('list') printFormatted(result.rows, globalOptions); } - printPaginationFooter( - result.data.length, - options.items, - options.page, - globalOptions.output as string - ); + const metricPagination = result.pagination?.metric; + if (metricPagination) { + printMetricFooter( + result.pagination!.total ?? result.data.length, + metricPagination.name, + metricPagination.id, + options.page, + options.items, + globalOptions.output as string + ); + } else { + printPaginationFooter( + result.data.length, + options.items, + options.page, + globalOptions.output as string + ); + } }) ); diff --git a/src/core/experiments/list.test.ts b/src/core/experiments/list.test.ts index 3f341d7c..d6637a9e 100644 --- a/src/core/experiments/list.test.ts +++ b/src/core/experiments/list.test.ts @@ -120,3 +120,78 @@ describe('list', () => { ); }); }); + +describe('list --metric', () => { + let mockClient: Record>; + + beforeEach(() => { + vi.clearAllMocks(); + mockClient = { + getMetric: vi.fn().mockResolvedValue({ id: 42, name: 'Revenue' }), + listMetrics: vi.fn(), + listExperiments: vi.fn().mockResolvedValue([ + { id: 1, name: 'Exp A', primary_metric_id: 42, secondary_metrics: [] }, + { id: 2, name: 'Exp B', primary_metric_id: 7, secondary_metrics: [] }, + { + id: 3, + name: 'Exp C', + primary_metric_id: 7, + secondary_metrics: [{ metric_id: 42, type: 'guardrail' }], + }, + ]), + }; + }); + + it('returns only experiments using the metric, tagged with metric_role', async () => { + const result = await listExperiments(mockClient as any, { items: 25, page: 1, metric: '42' }); + expect(result.data.map((e: any) => e.id)).toEqual([1, 3]); + expect((result.data as any[]).map((e) => e.metric_role)).toEqual(['primary', 'guardrail']); + }); + + it('scans all pages with a large page size rather than the display items', async () => { + await listExperiments(mockClient as any, { items: 25, page: 1, metric: '42' }); + expect(mockClient.listExperiments).toHaveBeenCalledWith( + expect.objectContaining({ page: 1, items: 200 }) + ); + }); + + it('reports the true match total and resolved metric in pagination', async () => { + const result = await listExperiments(mockClient as any, { items: 25, page: 1, metric: '42' }); + expect(result.pagination).toMatchObject({ + total: 2, + metric: { id: 42, name: 'Revenue' }, + }); + }); + + it('narrows by role via metricRole', async () => { + const result = await listExperiments(mockClient as any, { + items: 25, + page: 1, + metric: '42', + metricRole: 'guardrail', + }); + expect(result.data.map((e: any) => e.id)).toEqual([3]); + }); + + it('appends metric_role to the shown fields', async () => { + const { summarizeExperimentRow } = await import('../../api-client/experiment-summary.js'); + await listExperiments(mockClient as any, { + items: 25, + page: 1, + metric: '42', + show: ['tags'], + }); + expect(summarizeExperimentRow).toHaveBeenCalledWith( + expect.objectContaining({ id: 1 }), + ['tags', 'metric_role'], + [], + undefined + ); + }); + + it('paginates the matched set client-side', async () => { + const result = await listExperiments(mockClient as any, { items: 1, page: 2, metric: '42' }); + expect(result.data.map((e: any) => e.id)).toEqual([3]); + expect(result.pagination).toMatchObject({ page: 2, items: 1, total: 2, hasMore: false }); + }); +}); diff --git a/src/core/experiments/list.ts b/src/core/experiments/list.ts index 32206ab4..b23745b0 100644 --- a/src/core/experiments/list.ts +++ b/src/core/experiments/list.ts @@ -10,6 +10,12 @@ import { resolveApplicationIds, resolveUnitTypeIds, } from '../resolve.js'; +import { + resolveMetricId, + parseMetricRoles, + fetchAllExperiments, + filterExperimentsByMetric, +} from './metric-filter.js'; export interface ListExperimentsParams { state?: string; @@ -27,6 +33,8 @@ export interface ListExperimentsParams { significance?: string; iterations?: number; iterationsOf?: number; + metric?: string; + metricRole?: string; createdAfter?: string; createdBefore?: string; startedAfter?: string; @@ -135,10 +143,44 @@ export async function listExperiments( ...(params.significance && { significance: params.significance }), } as ListOptions; + const excludeFields = params.exclude ?? []; + + if (params.metric) { + const { id: metricId, name: metricName } = await resolveMetricId(client, params.metric); + const roles = parseMetricRoles(params.metricRole); + + // Scan all pages (ignoring the display page/items) then filter client-side, + // since the API has no server-side metric filter. + const { page: _page, items: _items, ...scanOptions } = listOptions; + const allExperiments = await fetchAllExperiments(client, scanOptions as ListOptions); + const matched = filterExperimentsByMetric(allExperiments, metricId, roles); + + const start = (params.page - 1) * params.items; + const pageItems = matched.slice(start, start + params.items); + + const metricExtraFields = [...(params.show ?? []), 'metric_role']; + const rows = params.raw + ? (pageItems as unknown[]) + : pageItems.map((e) => + summarizeExperimentRow(e, metricExtraFields, excludeFields, params.showOnly) + ); + + return { + data: pageItems as unknown[], + rows: rows as Record[], + pagination: { + page: params.page, + items: params.items, + hasMore: start + params.items < matched.length, + total: matched.length, + metric: { id: Number(metricId), name: metricName }, + }, + }; + } + const experiments = await client.listExperiments(listOptions); const extraFields = params.show ?? []; - const excludeFields = params.exclude ?? []; const rows = params.raw ? (experiments as unknown[]) diff --git a/src/core/experiments/metric-filter.test.ts b/src/core/experiments/metric-filter.test.ts new file mode 100644 index 00000000..c3d7464b --- /dev/null +++ b/src/core/experiments/metric-filter.test.ts @@ -0,0 +1,200 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { + parseMetricRoles, + experimentMetricRoles, + filterExperimentsByMetric, + fetchAllExperiments, + resolveMetricId, + ALL_METRIC_ROLES, +} from './metric-filter.js'; +import { MetricId } from '../../lib/api/branded-types.js'; + +describe('parseMetricRoles', () => { + it('defaults to all four roles when input is empty', () => { + expect(parseMetricRoles(undefined)).toEqual(ALL_METRIC_ROLES); + expect(parseMetricRoles('')).toEqual(ALL_METRIC_ROLES); + }); + + it('parses a comma-separated subset', () => { + expect(parseMetricRoles('guardrail,primary')).toEqual(['primary', 'guardrail']); + }); + + it('is case-insensitive and trims whitespace', () => { + expect(parseMetricRoles(' Primary , GUARDRAIL ')).toEqual(['primary', 'guardrail']); + }); + + it('dedupes and returns canonical order', () => { + expect(parseMetricRoles('exploratory,secondary,secondary')).toEqual([ + 'secondary', + 'exploratory', + ]); + }); + + it('throws on an invalid role, listing valid roles', () => { + expect(() => parseMetricRoles('primary,bogus')).toThrow(/bogus/); + expect(() => parseMetricRoles('primary,bogus')).toThrow( + /primary, secondary, guardrail, exploratory/ + ); + }); +}); + +describe('experimentMetricRoles', () => { + const metricId = MetricId(42); + const allRoles = [...ALL_METRIC_ROLES]; + + it('detects a primary metric', () => { + const exp = { primary_metric_id: 42, secondary_metrics: [] }; + expect(experimentMetricRoles(exp, metricId, allRoles)).toEqual(['primary']); + }); + + it('detects secondary, guardrail, and exploratory roles by type', () => { + expect( + experimentMetricRoles( + { secondary_metrics: [{ metric_id: 42, type: 'guardrail' }] }, + metricId, + allRoles + ) + ).toEqual(['guardrail']); + expect( + experimentMetricRoles( + { secondary_metrics: [{ metric_id: 42, type: 'exploratory' }] }, + metricId, + allRoles + ) + ).toEqual(['exploratory']); + }); + + it('treats a secondary_metrics entry with no type as secondary', () => { + const exp = { secondary_metrics: [{ metric_id: 42 }] }; + expect(experimentMetricRoles(exp, metricId, allRoles)).toEqual(['secondary']); + }); + + it('returns every role the metric plays, in canonical order', () => { + const exp = { + primary_metric_id: 42, + secondary_metrics: [ + { metric_id: 42, type: 'guardrail' }, + { metric_id: 42, type: 'secondary' }, + ], + }; + expect(experimentMetricRoles(exp, metricId, allRoles)).toEqual([ + 'primary', + 'secondary', + 'guardrail', + ]); + }); + + it('honors role narrowing: guardrail-only excludes a primary use', () => { + const exp = { + primary_metric_id: 42, + secondary_metrics: [{ metric_id: 42, type: 'guardrail' }], + }; + expect(experimentMetricRoles(exp, metricId, ['guardrail'])).toEqual(['guardrail']); + }); + + it('returns [] when the metric is not used', () => { + const exp = { primary_metric_id: 7, secondary_metrics: [{ metric_id: 8, type: 'guardrail' }] }; + expect(experimentMetricRoles(exp, metricId, allRoles)).toEqual([]); + }); + + it('returns [] when the metric is used only in a non-requested role', () => { + const exp = { primary_metric_id: 42 }; + expect(experimentMetricRoles(exp, metricId, ['guardrail'])).toEqual([]); + }); +}); + +describe('filterExperimentsByMetric', () => { + const metricId = MetricId(42); + + it('keeps only matching experiments and tags them with metric_role', () => { + const experiments = [ + { id: 1, primary_metric_id: 42, secondary_metrics: [] }, + { id: 2, primary_metric_id: 7, secondary_metrics: [{ metric_id: 42, type: 'guardrail' }] }, + { id: 3, primary_metric_id: 7, secondary_metrics: [{ metric_id: 8, type: 'secondary' }] }, + ]; + const result = filterExperimentsByMetric(experiments, metricId, [...ALL_METRIC_ROLES]); + expect(result.map((e) => e.id)).toEqual([1, 2]); + expect(result[0]!.metric_role).toBe('primary'); + expect(result[1]!.metric_role).toBe('guardrail'); + }); + + it('does not mutate the source experiment objects', () => { + const exp = { id: 1, primary_metric_id: 42 }; + filterExperimentsByMetric([exp], metricId, [...ALL_METRIC_ROLES]); + expect('metric_role' in exp).toBe(false); + }); +}); + +describe('fetchAllExperiments', () => { + it('aggregates across pages and stops on a short page', async () => { + const page1 = Array.from({ length: 200 }, (_, i) => ({ id: i })); + const page2 = Array.from({ length: 200 }, (_, i) => ({ id: 200 + i })); + const page3 = [{ id: 400 }]; + const listExperiments = vi + .fn() + .mockResolvedValueOnce(page1) + .mockResolvedValueOnce(page2) + .mockResolvedValueOnce(page3); + + const all = await fetchAllExperiments({ listExperiments } as any, { state: 'running' }); + + expect(all).toHaveLength(401); + expect(listExperiments).toHaveBeenCalledTimes(3); + expect(listExperiments).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ state: 'running', page: 1, items: 200 }) + ); + expect(listExperiments).toHaveBeenNthCalledWith(2, expect.objectContaining({ page: 2 })); + }); + + it('makes a single call when the first page is short', async () => { + const listExperiments = vi.fn().mockResolvedValueOnce([{ id: 1 }, { id: 2 }]); + const all = await fetchAllExperiments({ listExperiments } as any, {}); + expect(all).toHaveLength(2); + expect(listExperiments).toHaveBeenCalledTimes(1); + }); +}); + +describe('resolveMetricId', () => { + let client: Record>; + + beforeEach(() => { + vi.clearAllMocks(); + client = { + getMetric: vi.fn(), + listMetrics: vi.fn(), + }; + }); + + it('treats a numeric value as an ID and looks up the name', async () => { + client.getMetric.mockResolvedValue({ id: 42, name: 'Revenue' }); + const result = await resolveMetricId(client as any, '42'); + expect(result).toEqual({ id: 42, name: 'Revenue' }); + expect(client.getMetric).toHaveBeenCalledWith(42); + expect(client.listMetrics).not.toHaveBeenCalled(); + }); + + it('resolves a name to its ID via an exact, case-insensitive match', async () => { + client.listMetrics.mockResolvedValue([ + { id: 7, name: 'Other' }, + { id: 42, name: 'Revenue' }, + ]); + const result = await resolveMetricId(client as any, 'revenue'); + expect(result).toEqual({ id: 42, name: 'Revenue' }); + expect(client.getMetric).not.toHaveBeenCalled(); + }); + + it('throws when no metric matches the name', async () => { + client.listMetrics.mockResolvedValue([{ id: 7, name: 'Other' }]); + await expect(resolveMetricId(client as any, 'Nope')).rejects.toThrow(/No metric/); + }); + + it('throws when multiple metrics share the name, listing candidates', async () => { + client.listMetrics.mockResolvedValue([ + { id: 10, name: 'Revenue' }, + { id: 11, name: 'Revenue' }, + ]); + await expect(resolveMetricId(client as any, 'Revenue')).rejects.toThrow(/Multiple metrics/); + await expect(resolveMetricId(client as any, 'Revenue')).rejects.toThrow(/10[\s\S]*11/); + }); +}); diff --git a/src/core/experiments/metric-filter.ts b/src/core/experiments/metric-filter.ts new file mode 100644 index 00000000..bba7c5ad --- /dev/null +++ b/src/core/experiments/metric-filter.ts @@ -0,0 +1,148 @@ +import type { APIClient } from '../../api-client/api-client.js'; +import type { ListOptions } from '../../api-client/types.js'; +import { MetricId } from '../../lib/api/branded-types.js'; + +export type MetricRole = 'primary' | 'secondary' | 'guardrail' | 'exploratory'; + +export const ALL_METRIC_ROLES: MetricRole[] = ['primary', 'secondary', 'guardrail', 'exploratory']; + +/** Page size used when scanning every experiment to filter by metric. */ +const SCAN_PAGE_SIZE = 200; + +/** + * Parse the `--metric-role` flag into a canonical, deduped list of roles. + * An empty/undefined value means "all roles". Invalid values throw. + */ +export function parseMetricRoles(input?: string): MetricRole[] { + if (!input || !input.trim()) return [...ALL_METRIC_ROLES]; + + const requested = input + .split(',') + .map((s) => s.trim().toLowerCase()) + .filter(Boolean); + + const invalid = requested.filter((r) => !ALL_METRIC_ROLES.includes(r as MetricRole)); + if (invalid.length > 0) { + throw new Error( + `Invalid metric role(s): ${invalid.join(', ')}. Valid roles: ${ALL_METRIC_ROLES.join(', ')}.` + ); + } + + return ALL_METRIC_ROLES.filter((r) => requested.includes(r)); +} + +/** + * Return the roles a given metric plays in a single experiment, restricted to + * the requested roles, in canonical order. Returns [] if the metric is not used + * in any requested role. + */ +export function experimentMetricRoles( + exp: Record, + metricId: MetricId, + requestedRoles: MetricRole[] +): MetricRole[] { + const requested = new Set(requestedRoles); + const target = Number(metricId); + const found = new Set(); + + if (requested.has('primary') && Number(exp.primary_metric_id) === target) { + found.add('primary'); + } + + const secondary = exp.secondary_metrics as Array> | undefined; + if (secondary) { + for (const sm of secondary) { + if (Number(sm.metric_id) !== target) continue; + const role = ((sm.type as string) || 'secondary') as MetricRole; + if (requested.has(role)) found.add(role); + } + } + + return ALL_METRIC_ROLES.filter((r) => found.has(r)); +} + +/** + * Filter experiments to those using the metric in a requested role, returning + * shallow clones tagged with a `metric_role` field (the matched roles, joined). + * Source objects are not mutated. + */ +export function filterExperimentsByMetric( + experiments: Array>, + metricId: MetricId, + requestedRoles: MetricRole[] +): Array> { + const matched: Array> = []; + for (const exp of experiments) { + const roles = experimentMetricRoles(exp, metricId, requestedRoles); + if (roles.length === 0) continue; + matched.push({ ...exp, metric_role: roles.join(', ') }); + } + return matched; +} + +/** + * Fetch every experiment matching `baseOptions` by paging through the API at a + * fixed page size until a short page is returned. `page`/`items` in + * `baseOptions` are ignored (overridden per page). + */ +export async function fetchAllExperiments( + client: APIClient, + baseOptions: ListOptions +): Promise>> { + const all: Array> = []; + let page = 1; + + for (;;) { + const batch = (await client.listExperiments({ + ...baseOptions, + page, + items: SCAN_PAGE_SIZE, + })) as Array>; + all.push(...batch); + if (batch.length < SCAN_PAGE_SIZE) break; + page++; + } + + return all; +} + +/** + * Resolve a `--metric` argument (numeric ID or metric name) to its ID and name. + * Numeric values are looked up by ID (validating existence and fetching the + * name). Names are matched exactly and case-insensitively; archived/draft + * metrics are not in the name search and must be referenced by ID. + */ +export async function resolveMetricId( + client: APIClient, + nameOrId: string +): Promise<{ id: MetricId; name: string }> { + const asInt = parseInt(nameOrId, 10); + if (!isNaN(asInt) && String(asInt) === nameOrId.trim()) { + const id = MetricId(asInt); + const metric = await client.getMetric(id); + return { id, name: (metric.name as string) ?? String(asInt) }; + } + + const metrics = await client.listMetrics({ search: nameOrId, items: SCAN_PAGE_SIZE }); + const exact = metrics.filter( + (m) => String(m.name ?? '').toLowerCase() === nameOrId.toLowerCase() + ); + + if (exact.length === 0) { + throw new Error( + `No metric found matching name "${nameOrId}". Use a numeric metric ID, or check the name ` + + `(archived/draft metrics must be referenced by ID).` + ); + } + + const byId = new Map(exact.map((m) => [Number(m.id), m])); + if (byId.size > 1) { + const candidates = [...byId.values()].map((m) => ` ${m.id} ${m.name}`).join('\n'); + throw new Error( + `Multiple metrics match name "${nameOrId}":\n${candidates}\nUse a numeric metric ID to disambiguate.` + ); + } + + const match = exact[0]!; + return { id: MetricId(Number(match.id)), name: match.name as string }; +} diff --git a/src/core/types.ts b/src/core/types.ts index 9c30194f..60bd7917 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -3,5 +3,13 @@ export interface CommandResult { rows?: Record[] | undefined; detail?: Record | undefined; warnings?: string[] | undefined; - pagination?: { page: number; items: number; hasMore: boolean } | undefined; + pagination?: + | { + page: number; + items: number; + hasMore: boolean; + total?: number; + metric?: { id: number; name: string }; + } + | undefined; } diff --git a/src/lib/utils/pagination.test.ts b/src/lib/utils/pagination.test.ts index 92e126a2..2ee4b297 100644 --- a/src/lib/utils/pagination.test.ts +++ b/src/lib/utils/pagination.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { Command } from 'commander'; -import { addPaginationOptions, printPaginationFooter } from './pagination.js'; +import { addPaginationOptions, printPaginationFooter, printMetricFooter } from './pagination.js'; describe('addPaginationOptions', () => { it('should add --items and --page options to a Command', () => { @@ -49,3 +49,41 @@ describe('printPaginationFooter', () => { expect(output).not.toContain('Next:'); }); }); + +describe('printMetricFooter', () => { + let logSpy: ReturnType; + + beforeEach(() => { + logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); + }); + + afterEach(() => { + logSpy.mockRestore(); + }); + + it('summarizes the total and the metric when everything fits on one page', () => { + printMetricFooter(3, 'Revenue', 42, 1, 20); + const output = logSpy.mock.calls[0]![0] as string; + expect(output).toContain('3 experiments use metric "Revenue" (#42)'); + expect(output).not.toContain('page'); + }); + + it('uses the singular form for a single match', () => { + printMetricFooter(1, 'Revenue', 42, 1, 20); + const output = logSpy.mock.calls[0]![0] as string; + expect(output).toContain('1 experiment use'.replace('use', 'uses')); + expect(output).toContain('1 experiment uses metric "Revenue" (#42)'); + }); + + it('shows the page indicator when matches span multiple pages', () => { + printMetricFooter(45, 'Revenue', 42, 2, 20); + const output = logSpy.mock.calls[0]![0] as string; + expect(output).toContain('page 2/3'); + }); + + it('stays silent for json/yaml output', () => { + printMetricFooter(3, 'Revenue', 42, 1, 20, 'json'); + printMetricFooter(3, 'Revenue', 42, 1, 20, 'yaml'); + expect(logSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/utils/pagination.ts b/src/lib/utils/pagination.ts index bbbfd407..d4f56c4e 100644 --- a/src/lib/utils/pagination.ts +++ b/src/lib/utils/pagination.ts @@ -25,3 +25,21 @@ export function printFilteredFooter(count: number, outputFormat?: string): void if (outputFormat === 'json' || outputFormat === 'yaml') return; console.log(chalk.gray(`${count} result${count === 1 ? '' : 's'} (filtered).`)); } + +export function printMetricFooter( + total: number, + metricName: string, + metricId: number, + page: number, + items: number, + outputFormat?: string +): void { + if (outputFormat === 'json' || outputFormat === 'yaml') return; + const verb = total === 1 ? 'experiment uses' : 'experiments use'; + let footer = `${total} ${verb} metric "${metricName}" (#${metricId})`; + if (total > items) { + const totalPages = Math.max(1, Math.ceil(total / items)); + footer += ` — page ${page}/${totalPages}`; + } + console.log(chalk.gray(footer)); +}