feat: Add frontend observability probes

.env.example

@@ -8,3 +8,7 @@
 # Only create `agent-starter-react/.env.local` for standalone frontend
 # development where this repository is launched directly with `pnpm dev`.
 # In that case, define only the variables needed for that standalone run.
+#
+# `OBSERVABILITY_ENABLED=1` uses the same unified switch as the backend. When
+# enabled by the LexVoice runtime, browser-side probes publish LiveKit data
+# packets for the local observability report.

.gitignore

@@ -41,3 +41,7 @@ yarn-error.log*
 # typescript
 *.tsbuildinfo
 next-env.d.ts
+
+# generated local VAD runtime assets
+/public/onnxruntime-web/
+/public/vad-web/

.prettierignore

@@ -4,6 +4,7 @@ node_modules/
 pnpm-lock.yaml
 .next/
 .env*
-
+public/onnxruntime-web/
+public/vad-web/
 
 

app-config.ts

@@ -1,3 +1,10 @@
+import {
+  type RoleInputDeviceOptions,
+  normalizeInputSource,
+  resolveRoleInputDevices,
+  usesServerRoomInputDevice,
+} from './lib/input-device-config';
+
 export interface VideoTrackConfig {
   id: string;
   label: string;
@@ -49,6 +56,7 @@ export interface AppConfig {
   showAudioFilterDebug?: boolean;
   debugAudio?: boolean;
   debugVideo?: boolean;
+  observabilityEnabled?: boolean;
 
   // 全局调试配置
   enableGlobalDebug?: boolean; // 全局调试开关，控制所有调试信息的显示
@@ -74,16 +82,9 @@ const ROOM_INPUT_VIDEO_TRACK_NAME =
   'room_video';
 const BROWSER_VIDEO_TRACK_NAME = 'browser_video_track';
 
-const DEFAULT_ROLE_INPUT_DEVICE = 'xunfei';
-const VALID_INPUT_DEVICES = new Set(['xunfei', 'generic', 'primebot', 'browser']);
-const SERVER_ROOM_INPUT_DEVICES = new Set(['xunfei', 'generic']);
+export { normalizeInputSource };
 
-export interface InputDeviceConfigOptions {
-  inputSource?: string | null;
-  audioInputDevice?: string | null;
-  visionInputDevice?: string | null;
-  outputDevice?: string | null;
-}
+export type InputDeviceConfigOptions = RoleInputDeviceOptions;
 
 export interface InputDeviceConfig {
   inputSource: string;
@@ -98,43 +99,23 @@ export interface InputDeviceConfig {
   showDefaultCameraPreview: boolean;
 }
 
-export function normalizeInputSource(inputSource?: string | null) {
-  const normalized = (inputSource || '').trim().toLowerCase();
-  return normalized || 'browser';
-}
-
-function normalizeRoleInputDevice(inputDevice: string | null | undefined, fallback: string) {
-  const normalized = (inputDevice || '').trim().toLowerCase();
-  if (VALID_INPUT_DEVICES.has(normalized)) {
-    return normalized;
-  }
-  return fallback;
-}
-
-function usesServerRoomInputDevice(inputDevice: string) {
-  return SERVER_ROOM_INPUT_DEVICES.has(inputDevice);
-}
-
 export function resolveInputDeviceConfig({
   inputSource,
   audioInputDevice,
   visionInputDevice,
   outputDevice,
 }: InputDeviceConfigOptions = {}): InputDeviceConfig {
-  const normalizedInputSource = normalizeInputSource(inputSource);
-  const isMixedInputSource = normalizedInputSource === 'mixed';
-  const baseInputDevice = isMixedInputSource
-    ? DEFAULT_ROLE_INPUT_DEVICE
-    : normalizeRoleInputDevice(normalizedInputSource, DEFAULT_ROLE_INPUT_DEVICE);
-  const resolvedAudioInputDevice = isMixedInputSource
-    ? normalizeRoleInputDevice(audioInputDevice, baseInputDevice)
-    : baseInputDevice;
-  const resolvedVisionInputDevice = isMixedInputSource
-    ? normalizeRoleInputDevice(visionInputDevice, baseInputDevice)
-    : baseInputDevice;
-  const resolvedOutputDevice = isMixedInputSource
-    ? normalizeRoleInputDevice(outputDevice, baseInputDevice)
-    : baseInputDevice;
+  const {
+    inputSource: normalizedInputSource,
+    audioInputDevice: resolvedAudioInputDevice,
+    visionInputDevice: resolvedVisionInputDevice,
+    outputDevice: resolvedOutputDevice,
+  } = resolveRoleInputDevices({
+    inputSource,
+    audioInputDevice,
+    visionInputDevice,
+    outputDevice,
+  });
   const usesBrowserRawAudioInput = resolvedAudioInputDevice === 'browser';
   const usesBrowserRawVideoInput = resolvedVisionInputDevice === 'browser';
   const usesBrowserRawMediaInput = usesBrowserRawAudioInput || usesBrowserRawVideoInput;
@@ -259,6 +240,7 @@ export const APP_CONFIG_DEFAULTS: AppConfig = {
   showAudioFilterDebug: process.env.NEXT_PUBLIC_SHOW_AUDIO_DEBUG === 'true' || false, // 是否显示音频过滤调试组件
   debugAudio: false,
   debugVideo: false,
+  observabilityEnabled: false,
 
   // 全局调试配置
   enableGlobalDebug: process.env.NEXT_PUBLIC_ENABLE_GLOBAL_DEBUG === 'true' || false, // 全局调试开关

app/api/session/stop/route.ts

@@ -8,7 +8,10 @@ import {
   deriveSessionIdFromLiveKitRoomName,
   isValidConnectionRoomId,
 } from '@/lib/connection-room-id';
-import { resolveLiveKitHttpUrl } from '@/lib/session-stop';
+import {
+  resolveRoomInputStopUrls as resolveConfiguredRoomInputStopUrls,
+  resolveLiveKitHttpUrl,
+} from '@/lib/session-stop';
 import {
   markRoomSessionStopped,
   markRoomSessionStopping,
@@ -22,11 +25,13 @@ const AGENT_WORKER_READINESS_TIMEOUT_MS = readPositiveIntEnv(
   10_000
 );
 const AGENT_WORKER_LOG_TAIL_BYTES = readPositiveIntEnv('AGENT_WORKER_LOG_TAIL_BYTES', 256 * 1024);
+const ROOM_INPUT_STOP_TIMEOUT_MS = readPositiveIntEnv('ROOM_INPUT_STOP_TIMEOUT_MS', 3_000);
 
 type StopResult = {
   target: string;
   ok: boolean;
   skipped?: boolean;
+  fatal?: boolean;
   status?: number;
   error?: string;
   dispatch_ids?: string[];
@@ -134,6 +139,34 @@ function shouldWaitForLocalAgentWorkerReadiness(): boolean {
   return inputSource !== 'browser' && !(inputSource === 'mixed' && usesBrowserOnlyMixedInput());
 }
 
+function shouldStopRoomInput(): boolean {
+  return shouldWaitForLocalAgentWorkerReadiness();
+}
+
+function resolveRoomInputStopUrls(): string[] {
+  if (!shouldStopRoomInput()) {
+    return [];
+  }
+
+  return resolveConfiguredRoomInputStopUrls({
+    inputSource: readStopInputSource(),
+    audioInputDevice: readStopRoleDevice(
+      'ROOM_AUDIO_INPUT_DEVICE',
+      'NEXT_PUBLIC_ROOM_AUDIO_INPUT_DEVICE'
+    ),
+    visionInputDevice: readStopRoleDevice(
+      'ROOM_VISION_INPUT_DEVICE',
+      'NEXT_PUBLIC_ROOM_VISION_INPUT_DEVICE'
+    ),
+    roomAudioInputUrl: readStopEnv('ROOM_AUDIO_INPUT_URL'),
+    roomVisionInputUrl: readStopEnv('ROOM_VISION_INPUT_URL'),
+    roomInputUrl: readStopEnv('ROOM_INPUT_URL'),
+    frontdeskInputParticipantUrl: readStopEnv('FRONTDESK_INPUT_PARTICIPANT_URL'),
+    faceServiceUrl: readStopEnv('FACE_SERVICE_URL'),
+    genericCameraParticipantUrl: readStopEnv('GENERIC_CAMERA_PARTICIPANT_URL'),
+  });
+}
+
 function resolveLocalLiveKitServerLogPath(): string {
   const runLogDir = process.env.LEXVOICE_RUN_LOG_DIR?.trim();
   return runLogDir ? path.join(runLogDir, 'server.log') : '';
@@ -272,16 +305,69 @@ async function waitForPendingDispatches(roomName: string, sessionId: string): Pr
   return { target: 'agent_dispatch_barrier', ok: true };
 }
 
+async function postRoomInputStop(
+  stopUrl: string,
+  roomName: string,
+  sessionId: string
+): Promise<StopResult> {
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), ROOM_INPUT_STOP_TIMEOUT_MS);
+  try {
+    const response = await fetch(stopUrl, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ room_name: roomName, session_id: sessionId }),
+      signal: controller.signal,
+    });
+
+    if (response.ok) {
+      return { target: 'room_input', ok: true, status: response.status };
+    }
+
+    return {
+      target: 'room_input',
+      ok: false,
+      fatal: false,
+      status: response.status,
+      error: `room-input stop returned HTTP ${response.status}`,
+    };
+  } catch (error) {
+    return {
+      target: 'room_input',
+      ok: false,
+      fatal: false,
+      error: error instanceof Error ? error.message : String(error),
+    };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+async function stopRoomInput(roomName: string, sessionId: string): Promise<StopResult[]> {
+  const stopUrls = resolveRoomInputStopUrls();
+  if (stopUrls.length === 0) {
+    return [{ target: 'room_input', ok: true, skipped: true }];
+  }
+
+  return Promise.all(stopUrls.map((stopUrl) => postRoomInputStop(stopUrl, roomName, sessionId)));
+}
+
 async function runRemoteSessionCleanup(
   roomName: string,
   sessionId: string,
   dispatchResult: StopResult,
   dispatchIds: string[]
 ): Promise<{ results: StopResult[]; failures: StopResult[] }> {
   const dispatchBarrierResult = await waitForPendingDispatches(roomName, sessionId);
+  const roomInputResults = await stopRoomInput(roomName, sessionId);
   const liveKitRoomResult = await deleteLiveKitRoom(roomName);
   const agentWorkerReadinessResult = await waitForLocalAgentWorkerReadiness();
-  const cleanupResults = [dispatchBarrierResult, liveKitRoomResult, agentWorkerReadinessResult];
+  const cleanupResults = [
+    dispatchBarrierResult,
+    ...roomInputResults,
+    liveKitRoomResult,
+    agentWorkerReadinessResult,
+  ];
   const results = [
     {
       target: 'session_registry',
@@ -291,12 +377,18 @@ async function runRemoteSessionCleanup(
     dispatchResult,
     ...cleanupResults,
   ];
-  const failures = results.filter((result) => !result.ok && !result.skipped);
+  const failures = results.filter(
+    (result) => !result.ok && !result.skipped && result.fatal !== false
+  );
+  const bestEffortFailures = results.filter(
+    (result) => !result.ok && !result.skipped && result.fatal === false
+  );
   console.info('agent session remote cleanup completed', {
     roomName,
     sessionId,
     results,
     failures,
+    bestEffortFailures,
   });
   markRoomSessionStopped(roomName, sessionId);
   return { results, failures };

components/app/app.tsx

@@ -22,6 +22,7 @@ export function App({ appConfig }: AppProps) {
       <FilteredAudioRenderer
         excludeTrackNames={appConfig.excludeAudioTracks}
         debugAudio={appConfig.debugAudio}
+        observabilityEnabled={appConfig.observabilityEnabled}
       />
       <AudioFilterDebug
         excludeTrackNames={appConfig.excludeAudioTracks}

components/app/session-provider.tsx

@@ -17,21 +17,21 @@ const DEFAULT_BROWSER_SOURCE_CLIENT: BrowserSourceClient = {
   setAudioEnabled: async () => {},
   setVideoEnabled: async () => {},
   start: async () => {},
-  stop: () => {},
+  stop: async () => {},
 };
 
 const SessionContext = createContext<{
   appConfig: AppConfig;
   isSessionActive: boolean;
   startSession: () => Promise<void>;
-  endSession: () => void;
+  endSession: () => Promise<void>;
   getCurrentSessionId: () => string | null;
   browserSourceClient: BrowserSourceClient;
 }>({
   appConfig: APP_CONFIG_DEFAULTS,
   isSessionActive: false,
   startSession: async () => {},
-  endSession: () => {},
+  endSession: async () => {},
   getCurrentSessionId: () => null,
   browserSourceClient: DEFAULT_BROWSER_SOURCE_CLIENT,
 });

components/app/tile-layout.tsx

@@ -12,6 +12,7 @@ import { APP_CONFIG_DEFAULTS, type VideoTrackConfig } from '@/app-config';
 import { useSelectedVideoTrack } from '@/hooks/useSelectedVideoTrack';
 import { useSmartVoiceAssistant } from '@/hooks/useSmartVoiceAssistant';
 import { cn } from '@/lib/utils';
+import { resolveCameraPreviewTrack } from '@/lib/video-preview-selection';
 
 function debugVideoLog(enabled: boolean | undefined, ...args: unknown[]) {
   if (enabled) {
@@ -183,11 +184,20 @@ export function TileLayout({
     videoTrackConfigs,
   ]);
 
+  const selectedTrackType = useMemo(() => {
+    if (!selectedTrackId) return null;
+    return videoTrackConfigs.find((config) => config.id === selectedTrackId)?.type ?? null;
+  }, [selectedTrackId, videoTrackConfigs]);
+
   const canShowDefaultCameraPreview = showDefaultCameraPreview && !isPreviewDisabled;
-  const cameraTrack =
-    selectedTrack ||
-    (canShowDefaultCameraPreview && selectedTrackId === null ? configuredCameraTrack : undefined) ||
-    (canShowDefaultCameraPreview ? defaultCameraTrack : undefined);
+  const cameraTrack = resolveCameraPreviewTrack<TrackReference>({
+    selectedTrack,
+    selectedTrackId,
+    selectedTrackType,
+    canShowDefaultCameraPreview,
+    configuredCameraTrack,
+    defaultCameraTrack,
+  });
 
   const isCameraEnabled = Boolean(cameraTrack?.publication && !cameraTrack.publication.isMuted);
   const isScreenShareEnabled = Boolean(screenShareTrack && !screenShareTrack.publication.isMuted);

components/livekit/agent-control-bar/agent-control-bar.tsx

@@ -117,8 +117,8 @@ export function AgentControlBar({
 
   const handleDisconnect = useCallback(async () => {
     const sessionId = getCurrentSessionId() ?? getActiveAgentSession()?.sessionId;
-    const localDisconnectPromise = Promise.resolve().then(() => {
-      endSession();
+    const localDisconnectPromise = Promise.resolve().then(async () => {
+      await endSession();
       onDisconnect?.();
     });
     registerAgentSessionLocalCleanup(localDisconnectPromise);