feat: add cljs transcription restart policy

scripts/smoke-cljs-blink.mjs

@@ -810,6 +810,7 @@ tts.dispose();
 
 const transcriptionEvents = [];
 const transcriptionCommands = [];
+const transcriptionRecommendations = [];
 const transcriptionStates = [];
 
 const transcription = createTranscriptionAgency(
@@ -818,11 +819,17 @@ const transcription = createTranscriptionAgency(
     referenceRatio: 1.5,
     releaseThreshold: 0.05,
     releaseMs: 300,
+    autoRestart: true,
+    restartDelayMs: 125,
+    maxRestartCount: 2,
   },
   {
     onTranscriptionEvent(event) {
       transcriptionEvents.push(event);
     },
+    onTranscriptionRecommendation(recommendation) {
+      transcriptionRecommendations.push(recommendation);
+    },
     onAgencyCommand(target, command) {
       transcriptionCommands.push({ target, command });
     },
@@ -859,8 +866,51 @@ if (transcriptionStates.length < 5) {
   throw new Error(`Expected CLJS transcription state callbacks, received ${transcriptionStates.length}`);
 }
 
+transcription.fail('network drop');
+const restartRecommendation = transcriptionRecommendations.find((recommendation) => recommendation.type === 'RESTART');
+if (!restartRecommendation ||
+    restartRecommendation.reason !== 'error' ||
+    restartRecommendation.delayMs !== 125 ||
+    restartRecommendation.restartCount !== 1 ||
+    restartRecommendation.maxRestartCount !== 2) {
+  throw new Error(`Expected CLJS transcription restart recommendation, received ${JSON.stringify(transcriptionRecommendations)}`);
+}
+
+const failedTranscriptionState = transcription.getState();
+if (failedTranscriptionState.restartCount !== 1 ||
+    failedTranscriptionState.pendingRestart?.type !== 'RESTART' ||
+    failedTranscriptionState.error !== 'network drop') {
+  throw new Error(`Expected CLJS transcription failure state to retain restart data, received ${JSON.stringify(failedTranscriptionState)}`);
+}
+
+transcription.stop();
+const cleanupRecommendation = transcriptionRecommendations.find((recommendation) => (
+  recommendation.type === 'CLEANUP' &&
+  recommendation.reason === 'stop'
+));
+if (!cleanupRecommendation || cleanupRecommendation.cancelRestart !== true || cleanupRecommendation.releaseBrowserResources !== true) {
+  throw new Error(`Expected CLJS transcription cleanup recommendation, received ${JSON.stringify(transcriptionRecommendations)}`);
+}
+
 transcription.dispose();
 
+const restartLimitedRecommendations = [];
+const restartLimitedTranscription = createTranscriptionAgency(
+  { autoRestart: true, maxRestartCount: 0 },
+  {
+    onTranscriptionRecommendation(recommendation) {
+      restartLimitedRecommendations.push(recommendation);
+    },
+  },
+);
+restartLimitedTranscription.start();
+restartLimitedTranscription.fail('permission denied');
+const stopRecommendation = restartLimitedRecommendations.find((recommendation) => recommendation.type === 'STOP');
+if (!stopRecommendation || stopRecommendation.reason !== 'maxRestartCount') {
+  throw new Error(`Expected CLJS transcription max-restart stop recommendation, received ${JSON.stringify(restartLimitedRecommendations)}`);
+}
+restartLimitedTranscription.dispose();
+
 const conversationEvents = [];
 const conversationCommands = [];
 const conversationStates = [];

src-cljs/latticework/runtime.cljs

@@ -173,6 +173,10 @@
     (when-let [on-transcription-event (fn-prop host "onTranscriptionEvent")]
       (on-transcription-event (protocol/data->js (:event output))))
 
+    "transcriptionRecommendation"
+    (when-let [on-transcription-recommendation (fn-prop host "onTranscriptionRecommendation")]
+      (on-transcription-recommendation (protocol/data->js (:recommendation output))))
+
     "conversationEvent"
     (when-let [on-conversation-event (fn-prop host "onConversationEvent")]
       (on-conversation-event (protocol/data->js (:event output))))

src-cljs/latticework/transcription.cljs

@@ -12,7 +12,10 @@
    :interruptionThreshold 0.12
    :referenceRatio 1.8
    :releaseThreshold 0.06
-   :releaseMs 350})
+   :releaseMs 350
+   :autoRestart true
+   :restartDelayMs 250
+   :maxRestartCount 3})
 
 (def default-state
   {:status "idle"
@@ -26,6 +29,11 @@
    :lastUserLevel 0
    :lastReferenceLevel 0
    :lastInterruptionTime nil
+   :error nil
+   :restartCount 0
+   :lastRestartReason nil
+   :pendingRestart nil
+   :lastRecommendation nil
    :config default-config
    :eventCount 0
    :lastUpdatedTime nil})
@@ -51,7 +59,16 @@
   @state)
 
 (defn transcription-state [state]
-  (select-keys @state [:status :isListening :interimTranscript :finalTranscript :isInterrupted :interruptionSource]))
+  (select-keys @state [:status
+                       :isListening
+                       :interimTranscript
+                       :finalTranscript
+                       :isInterrupted
+                       :interruptionSource
+                       :error
+                       :restartCount
+                       :pendingRestart
+                       :lastRecommendation]))
 
 (defn- state-output [state]
   (protocol/emit-state agency-name @state))
@@ -67,6 +84,11 @@
    :target target
    :command command})
 
+(defn- recommendation-output [recommendation]
+  {:type "transcriptionRecommendation"
+   :agency agency-name
+   :recommendation recommendation})
+
 (defn- result [value outputs]
   {:result value :outputs (vec (remove nil? outputs))})
 
@@ -78,6 +100,48 @@
                (update :eventCount inc)
                (assoc :lastUpdatedTime (now))))))
 
+(defn- remember-recommendation! [state recommendation]
+  (swap! state assoc :lastRecommendation recommendation)
+  recommendation)
+
+(defn- cleanup-recommendation! [state reason extra]
+  (let [recommendation (remember-recommendation!
+                        state
+                        (merge {:type "CLEANUP"
+                                :reason reason
+                                :clearInterimTranscript true
+                                :clearInterruption true
+                                :cancelRestart true}
+                               extra))]
+    (recommendation-output recommendation)))
+
+(defn- restart-recommendation! [state reason]
+  (let [config (:config @state)
+        max-restarts (max 0 (number-or (:maxRestartCount config) 3))
+        next-count (inc (:restartCount @state))
+        can-restart? (and (:continuous config)
+                          (:autoRestart config)
+                          (<= next-count max-restarts))
+        recommendation (if can-restart?
+                         {:type "RESTART"
+                          :reason reason
+                          :delayMs (max 0 (number-or (:restartDelayMs config) 250))
+                          :restartCount next-count
+                          :maxRestartCount max-restarts}
+                         {:type "STOP"
+                          :reason (if (and (:continuous config) (:autoRestart config))
+                                    "maxRestartCount"
+                                    reason)
+                          :restartCount (:restartCount @state)
+                          :maxRestartCount max-restarts})]
+    (update-state!
+     state
+     #(assoc % :restartCount (if can-restart? next-count (:restartCount %))
+               :lastRestartReason reason
+               :pendingRestart (when (= "RESTART" (:type recommendation)) recommendation)
+               :lastRecommendation recommendation))
+    (recommendation-output recommendation)))
+
 (defn normalize-transcript [transcript]
   (-> (or transcript "")
       str/trim
@@ -97,16 +161,20 @@
   (update-state! state #(assoc % :status "listening"
                                 :isListening true
                                 :interimTranscript ""
-                                :finalTranscript ""))
+                                :finalTranscript ""
+                                :error nil
+                                :pendingRestart nil))
   (result true [(event-output {:type "START"})
                 (state-output state)]))
 
 (defn stop! [state]
   (update-state! state #(assoc % :status "idle"
                                 :isListening false
                                 :isInterrupted false
-                                :interruptionSource nil))
+                                :interruptionSource nil
+                                :pendingRestart nil))
   (result true [(event-output {:type "STOP"})
+                (cleanup-recommendation! state "stop" {:releaseBrowserResources true})
                 (state-output state)]))
 
 (defn reset-state! [state]
@@ -115,8 +183,13 @@
                                 :lastTranscript ""
                                 :lastConfidence nil
                                 :isInterrupted false
-                                :interruptionSource nil))
+                                :interruptionSource nil
+                                :error nil
+                                :restartCount 0
+                                :lastRestartReason nil
+                                :pendingRestart nil))
   (result true [(event-output {:type "RESET"})
+                (cleanup-recommendation! state "reset" {:releaseBrowserResources false})
                 (state-output state)]))
 
 (defn process-result! [state transcript final? confidence source]
@@ -195,8 +268,11 @@
 (defn fail! [state message]
   (update-state! state #(assoc % :status "error"
                                 :isListening false
-                                :error (or message "Transcription failed")))
+                                :error (or message "Transcription failed")
+                                :isInterrupted false
+                                :interruptionSource nil))
   (result false [(event-output {:type "ERROR" :message (:error @state)})
+                 (restart-recommendation! state "error")
                  (state-output state)]))
 
 (defn update-config! [state config]

types/cljs.d.ts

@@ -37,6 +37,7 @@ export interface WorkerAgencyHost {
     emotionEvents: TTSEmojiTimelineEvent[],
   ) => void;
   onTranscriptionEvent?: (event: TranscriptionEvent) => void;
+  onTranscriptionRecommendation?: (recommendation: TranscriptionRecommendation) => void;
   onConversationEvent?: (event: ConversationEvent) => void;
   onAgencyCommand?: (target: string, command: Record<string, unknown>) => void;
   onTTSCommand?: (command: Record<string, unknown>) => void;
@@ -728,6 +729,9 @@ export interface TranscriptionConfig {
   referenceRatio?: number;
   releaseThreshold?: number;
   releaseMs?: number;
+  autoRestart?: boolean;
+  restartDelayMs?: number;
+  maxRestartCount?: number;
 }
 
 export interface TranscriptionAgencyState {
@@ -737,6 +741,10 @@ export interface TranscriptionAgencyState {
   finalTranscript: string;
   isInterrupted: boolean;
   interruptionSource: string | null;
+  error: string | null;
+  restartCount: number;
+  pendingRestart: TranscriptionRecommendation | null;
+  lastRecommendation: TranscriptionRecommendation | null;
 }
 
 export interface TranscriptionSnapshot extends TranscriptionAgencyState {
@@ -763,6 +771,19 @@ export interface TranscriptionEvent {
   [key: string]: unknown;
 }
 
+export interface TranscriptionRecommendation {
+  type: 'RESTART' | 'STOP' | 'CLEANUP' | string;
+  reason: string;
+  delayMs?: number;
+  restartCount?: number;
+  maxRestartCount?: number;
+  clearInterimTranscript?: boolean;
+  clearInterruption?: boolean;
+  cancelRestart?: boolean;
+  releaseBrowserResources?: boolean;
+  [key: string]: unknown;
+}
+
 export interface TranscriptionAgency {
   updateConfig(config: TranscriptionConfig): boolean;
   start(): boolean;