Display TensorRT-LLM AgentX server metrics

packages/app/src/lib/api-route-catalog.ts

@@ -431,7 +431,7 @@ export const apiContractSourceDigests = [
   },
   {
     source: '../db/src/queries/agentic-aggregates.ts',
-    sourceSha256: 'b0f4be6c39fe440df99db687b4b6deeed58897bf20325770631680f6e41aa304',
+    sourceSha256: '1c5ef41b6a21c7f19ab3105f68407e8c5f64a0cf82a4c0f7e0aeee7d83e7cceb',
     reviewArea: {
       en: 'Agentic aggregate percentile keys, nullability, and ID-keyed response shape.',
       zh: '智能体汇总百分位字段、可空性和按 ID 索引的响应结构。',

packages/db/src/etl/compute-aggregate-stats.ts

@@ -76,6 +76,12 @@ export const AGGREGATE_SERVER_METRIC_KEYS = new Set([
   'vllm:prefix_cache_queries',
   'vllm:gpu_prefix_cache_hits',
   'vllm:gpu_prefix_cache_queries',
+  'trtllm_kv_cache_utilization',
+  'trtllm_kv_cache_hit_rate',
+  'trtllm_prompt_cached_tokens',
+  'trtllm_prompt_cached_tokens_total',
+  'trtllm_prompt_tokens',
+  'trtllm_prompt_tokens_total',
 ]);
 
 /**

packages/db/src/etl/compute-chart-series.test.ts

@@ -119,6 +119,19 @@ function buildDynamoSeries(
   };
 }
 
+function buildTrtllmSeries(
+  endpoint_url: string,
+  dynamo_component: 'prefill' | 'backend',
+  value: number,
+  field: 'rate' | 'avg',
+) {
+  return {
+    endpoint_url,
+    labels: { dynamo_component, worker_id: `${dynamo_component}-worker` },
+    timeslices: [{ start_ns: 0, end_ns: 1e9, [field]: value }],
+  };
+}
+
 describe('computeChartSeries', () => {
   it('returns null when the blob is null', async () => {
     expect(await computeChartSeries(null)).toBeNull();
@@ -338,4 +351,68 @@ describe('computeChartSeries', () => {
 
     expect(result?.metricSources).toEqual([]);
   });
+
+  it('extracts native TensorRT-LLM metrics and preserves disaggregated worker roles', async () => {
+    const prefillUrl = 'http://prefill-a.internal.test:7500/metrics';
+    const decodeUrl = 'http://decode-a.internal.test:7501/metrics';
+    const json = JSON.stringify({
+      metrics: {
+        trtllm_kv_cache_utilization: {
+          series: [
+            buildTrtllmSeries(prefillUrl, 'prefill', 0.3, 'avg'),
+            buildTrtllmSeries(decodeUrl, 'backend', 0.7, 'avg'),
+          ],
+        },
+        trtllm_kv_cache_host_utilization: {
+          series: [buildTrtllmSeries(prefillUrl, 'prefill', 0.25, 'avg')],
+        },
+        trtllm_prompt_tokens: {
+          series: [
+            buildTrtllmSeries(prefillUrl, 'prefill', 100, 'rate'),
+            buildTrtllmSeries(decodeUrl, 'backend', 200, 'rate'),
+          ],
+        },
+        trtllm_prompt_cached_tokens: {
+          series: [
+            buildTrtllmSeries(prefillUrl, 'prefill', 40, 'rate'),
+            buildTrtllmSeries(decodeUrl, 'backend', 80, 'rate'),
+          ],
+        },
+        trtllm_generation_tokens: {
+          series: [buildTrtllmSeries(decodeUrl, 'backend', 50, 'rate')],
+        },
+        trtllm_num_requests_running: {
+          series: [
+            buildTrtllmSeries(prefillUrl, 'prefill', 2, 'avg'),
+            buildTrtllmSeries(decodeUrl, 'backend', 3, 'avg'),
+          ],
+        },
+        trtllm_num_requests_waiting: {
+          series: [buildTrtllmSeries(decodeUrl, 'backend', 4, 'avg')],
+        },
+      },
+    });
+
+    const result = await computeChartSeries(gzipSync(Buffer.from(json)), {
+      framework: 'trtllm',
+      disagg: true,
+    });
+
+    expect(result?.kvCacheUsage).toEqual([{ t: 0, value: 0.5 }]);
+    expect(result?.hostKvCacheUsage).toEqual([{ t: 0, value: 0.25 }]);
+    expect(result?.prefixCacheHitRate).toEqual([{ t: 0, value: 0.4 }]);
+    expect(result?.queueDepth).toEqual([{ t: 0, running: 5, waiting: 4, total: 9 }]);
+    expect(result?.prefillTps).toEqual([{ t: 0, value: 300 }]);
+    expect(result?.decodeTps).toEqual([{ t: 0, value: 50 }]);
+    expect(result?.promptTokensBySource).toEqual({
+      'cache hit (HBM)': [{ t: 0, value: 120 }],
+      'compute (miss)': [{ t: 0, value: 180 }],
+    });
+    expect(result?.metricSources.map(({ source }) => [source.role, source.endpointUrl])).toEqual([
+      ['prefill', prefillUrl],
+      ['decode', decodeUrl],
+    ]);
+    expect(result?.metricSources[0]?.promptTps).toEqual([{ t: 0, value: 100 }]);
+    expect(result?.metricSources[1]?.generationTps).toEqual([{ t: 0, value: 50 }]);
+  });
 });

packages/db/src/etl/compute-chart-series.ts

@@ -65,8 +65,11 @@ import {
  * warmup block are unaffected. (v11 was a short-lived, since-reverted attempt to
  * carry kvCachePoolTokens in chart_series; that value now lives in
  * benchmark_results.metrics, derived from the server log — unrelated to this.)
+ *
+ * v13: extract TensorRT-LLM's native `trtllm_*` token, cache, queue, and KV
+ * metrics, including per-prefill/decode source series for Dynamo disaggregation.
  */
-export const CHART_SERIES_VERSION = 12;
+export const CHART_SERIES_VERSION = 13;
 
 export interface TimeSeriesPoint {
   /** Seconds from benchmark start. */
@@ -188,6 +191,18 @@ export const CHART_METRIC_KEYS = new Set([
   'sglang:realtime_tokens',
   'sglang:hicache_host_used_tokens',
   'sglang:hicache_host_total_tokens',
+  // TensorRT-LLM
+  'trtllm_kv_cache_utilization',
+  'trtllm_kv_cache_host_utilization',
+  'trtllm_kv_cache_hit_rate',
+  'trtllm_prompt_cached_tokens',
+  'trtllm_prompt_cached_tokens_total',
+  'trtllm_prompt_tokens',
+  'trtllm_prompt_tokens_total',
+  'trtllm_generation_tokens',
+  'trtllm_generation_tokens_total',
+  'trtllm_num_requests_running',
+  'trtllm_num_requests_waiting',
 ]);
 
 /**
@@ -346,6 +361,7 @@ function buildSeriesFromMetrics(
     'vllm:kv_cache_usage_perc',
     'vllm:gpu_cache_usage_perc',
     'sglang:token_usage',
+    'trtllm_kv_cache_utilization',
   );
   const kvCacheUsage: TimeSeriesPoint[] = sortedEntries(
     aggregateByStart(kvSeries, 'avg', 'avg'),
@@ -377,11 +393,18 @@ function buildSeriesFromMetrics(
 
   // Prefix cache hit rate per scrape: Σhits.rate / Σqueries.rate across
   // engines, joined on start_ns. SGLang names: cached_tokens / prompt_tokens.
-  const hitsSeries = pickSeries('vllm:prefix_cache_hits', 'sglang:cached_tokens');
+  const hitsSeries = pickSeries(
+    'vllm:prefix_cache_hits',
+    'sglang:cached_tokens',
+    'trtllm_prompt_cached_tokens',
+    'trtllm_prompt_cached_tokens_total',
+  );
   const qsSeries = pickSeries(
     'vllm:prefix_cache_queries',
     'vllm:prompt_tokens',
     'sglang:prompt_tokens',
+    'trtllm_prompt_tokens',
+    'trtllm_prompt_tokens_total',
   );
   const hitsByT = aggregateByStart(hitsSeries, 'rate', 'sum');
   const qsByT = aggregateByStart(qsSeries, 'rate', 'sum');
@@ -390,10 +413,25 @@ function buildSeriesFromMetrics(
     const q = qsByT.get(t);
     if (q !== undefined && q > 0) prefixCacheHitRate.push({ t: tOf(t), value: h / q });
   }
+  if (prefixCacheHitRate.length === 0) {
+    for (const [t, value] of sortedEntries(
+      aggregateByStart(metrics['trtllm_kv_cache_hit_rate']?.series, 'avg', 'avg'),
+    )) {
+      prefixCacheHitRate.push({ t: tOf(t), value });
+    }
+  }
 
   // Queue depth: sum running + waiting across engines per timeslice.
-  const runSeries = pickSeries('vllm:num_requests_running', 'sglang:num_running_reqs');
-  const waitSeries = pickSeries('vllm:num_requests_waiting', 'sglang:num_queue_reqs');
+  const runSeries = pickSeries(
+    'vllm:num_requests_running',
+    'sglang:num_running_reqs',
+    'trtllm_num_requests_running',
+  );
+  const waitSeries = pickSeries(
+    'vllm:num_requests_waiting',
+    'sglang:num_queue_reqs',
+    'trtllm_num_requests_waiting',
+  );
   const runByT = aggregateByStart(runSeries, 'avg', 'sum');
   const waitByT = aggregateByStart(waitSeries, 'avg', 'sum');
   const queueDepth: QueueDepthPoint[] = [];
@@ -415,11 +453,26 @@ function buildSeriesFromMetrics(
       value: v,
     }));
   };
-  const prefillTps = counterRate('vllm:prompt_tokens', 'sglang:prompt_tokens');
-  const decodeTps = counterRate('vllm:generation_tokens', 'sglang:generation_tokens');
+  const prefillTps = counterRate(
+    'vllm:prompt_tokens',
+    'sglang:prompt_tokens',
+    'trtllm_prompt_tokens',
+    'trtllm_prompt_tokens_total',
+  );
+  const decodeTps = counterRate(
+    'vllm:generation_tokens',
+    'sglang:generation_tokens',
+    'trtllm_generation_tokens',
+    'trtllm_generation_tokens_total',
+  );
   // Tokens served from prefix cache per scrape. Lets the frontend derive
   // "cumulative unique input tokens served" = cumsum(prefillTps) − cumsum(hits).
-  const prefixCacheHitsTps = counterRate('vllm:prefix_cache_hits', 'sglang:cached_tokens');
+  const prefixCacheHitsTps = counterRate(
+    'vllm:prefix_cache_hits',
+    'sglang:cached_tokens',
+    'trtllm_prompt_cached_tokens',
+    'trtllm_prompt_cached_tokens_total',
+  );
 
   // SGLang hicache: host-pool KV cache utilization as used/total per
   // timeslice. Both metrics are gauges in absolute tokens. Total stays
@@ -441,6 +494,13 @@ function buildSeriesFromMetrics(
       hostKvCacheUsage.push({ t: tOf(t), value: used / total });
     }
   }
+  if (hostKvCacheUsage.length === 0) {
+    for (const [t, value] of sortedEntries(
+      aggregateByStart(metrics['trtllm_kv_cache_host_utilization']?.series, 'avg', 'avg'),
+    )) {
+      hostKvCacheUsage.push({ t: tOf(t), value });
+    }
+  }
 
   // Per-source prompt tokens — sum across engines per source label.
   //   vllm: vllm:prompt_tokens_by_source has one series per source label
@@ -504,6 +564,27 @@ function buildSeriesFromMetrics(
       addSeriesRates(label, series);
     }
   }
+  if (promptBySrcByT.size === 0) {
+    const promptByT = aggregateByStart(
+      pickSeries('trtllm_prompt_tokens', 'trtllm_prompt_tokens_total'),
+      'rate',
+      'sum',
+    );
+    const cachedByT = aggregateByStart(
+      pickSeries('trtllm_prompt_cached_tokens', 'trtllm_prompt_cached_tokens_total'),
+      'rate',
+      'sum',
+    );
+    const cachedSeries: RawSeries = { timeslices: [] };
+    const computedSeries: RawSeries = { timeslices: [] };
+    for (const [t, prompt] of promptByT) {
+      const cached = Math.max(0, cachedByT.get(t) ?? 0);
+      cachedSeries.timeslices!.push({ start_ns: t, rate: cached });
+      computedSeries.timeslices!.push({ start_ns: t, rate: Math.max(0, prompt - cached) });
+    }
+    addSeriesRates('cache hit (HBM)', cachedSeries);
+    addSeriesRates('compute (miss)', computedSeries);
+  }
   const promptTokensBySource: Record<string, TimeSeriesPoint[]> = {};
   for (const [source, byT] of promptBySrcByT) {
     const arr: TimeSeriesPoint[] = [];
@@ -516,10 +597,23 @@ function buildSeriesFromMetrics(
   const metricSources: MetricSourceSeries[] = [];
   const adapter = selectServerMetricsAdapter(context);
   if (includeMetricSources && context.disagg && adapter.id !== 'generic') {
+    const endpointRoles = new Map<string, string>();
+    for (const metric of Object.values(metrics)) {
+      for (const series of metric.series ?? []) {
+        const endpointUrl = series.endpoint_url;
+        const role = series.labels?.['disaggregation_mode'] ?? series.labels?.['dynamo_component'];
+        if (endpointUrl && role) endpointRoles.set(endpointUrl, role);
+      }
+    }
     const grouped = new Map<string, { source: MetricSource; metrics: MetricsMap }>();
     for (const [metricName, metric] of Object.entries(metrics)) {
       for (const series of metric.series ?? []) {
-        const source = adapter.identifySource(series);
+        const roleHint = series.endpoint_url ? endpointRoles.get(series.endpoint_url) : undefined;
+        const identifiedSeries =
+          roleHint && !series.labels?.['disaggregation_mode']
+            ? { ...series, labels: { ...series.labels, disaggregation_mode: roleHint } }
+            : series;
+        const source = adapter.identifySource(identifiedSeries);
         let group = grouped.get(source.id);
         if (!group) {
           group = { source, metrics: {} };

packages/db/src/etl/server-metrics-adapters.ts

@@ -71,6 +71,37 @@ const dynamoAdapter: ServerMetricsAdapter = {
   },
 };
 
+const trtllmAdapter: ServerMetricsAdapter = {
+  id: 'trtllm',
+  matches: ({ framework }) => framework?.toLowerCase().includes('trt') ?? false,
+  identifySource(series) {
+    const labels = series.labels ?? {};
+    const nativeRole = labels['disaggregation_mode'] ?? labels['dynamo_component'] ?? null;
+    const role: MetricSourceRole =
+      nativeRole === 'prefill'
+        ? 'prefill'
+        : nativeRole === 'decode' || nativeRole === 'backend'
+          ? 'decode'
+          : nativeRole === 'aggregated'
+            ? 'combined'
+            : 'unknown';
+    const endpointUrl = series.endpoint_url ?? null;
+    const workerId = labels['worker_id'] ?? null;
+    const dpRank = labels['dp_rank'] ?? null;
+    const engine = labels['engine'] ?? labels['engine_idx'] ?? null;
+    return {
+      id: stableId('trtllm', [role, endpointUrl, workerId, dpRank, engine]),
+      adapter: 'trtllm',
+      role,
+      endpointUrl,
+      nativeRole,
+      workerId,
+      dpRank,
+      engine,
+    };
+  },
+};
+
 const genericAdapter: ServerMetricsAdapter = {
   id: 'generic',
   matches: () => true,
@@ -93,7 +124,7 @@ const genericAdapter: ServerMetricsAdapter = {
   },
 };
 
-const ADAPTERS: readonly ServerMetricsAdapter[] = [dynamoAdapter, genericAdapter];
+const ADAPTERS: readonly ServerMetricsAdapter[] = [trtllmAdapter, dynamoAdapter, genericAdapter];
 
 export function selectServerMetricsAdapter(context: ServerMetricsContext): ServerMetricsAdapter {
   return ADAPTERS.find((adapter) => adapter.matches(context)) ?? genericAdapter;

packages/db/src/queries/agentic-aggregates.test.ts

@@ -120,6 +120,47 @@ describe('extractServerMetricSamples', () => {
     expect(out.kvCacheUtil).toEqual([]);
     expect(out.prefixCacheHitRate).toEqual([]);
   });
+
+  it('extracts TensorRT-LLM KV utilization and prefix cache hit rate', () => {
+    const json = JSON.stringify({
+      metrics: {
+        trtllm_kv_cache_utilization: {
+          series: [
+            {
+              timeslices: [
+                { start_ns: 0, avg: 0.2 },
+                { start_ns: 1, avg: 0.6 },
+              ],
+            },
+          ],
+        },
+        trtllm_prompt_cached_tokens: {
+          series: [
+            {
+              timeslices: [
+                { start_ns: 0, rate: 70 },
+                { start_ns: 1, rate: 20 },
+              ],
+            },
+          ],
+        },
+        trtllm_prompt_tokens: {
+          series: [
+            {
+              timeslices: [
+                { start_ns: 0, rate: 100 },
+                { start_ns: 1, rate: 50 },
+              ],
+            },
+          ],
+        },
+      },
+    });
+
+    const out = extractServerMetricSamples(json);
+    expect(out.kvCacheUtil).toEqual([0.2, 0.6]);
+    expect(out.prefixCacheHitRate).toEqual([0.7, 0.4]);
+  });
 });
 
 /** The write-back payload as bound to the UPDATE (a partial aggregate_stats). */