From 42d7e7d46886db6388dfb92aaa1964d15b4aa358 Mon Sep 17 00:00:00 2001 From: Mister Nemo Date: Thu, 28 May 2026 00:15:25 -0400 Subject: [PATCH] feat: add cljs transcription restart policy --- scripts/smoke-cljs-blink.mjs | 50 ++++++++++++++ src-cljs/latticework/runtime.cljs | 4 ++ src-cljs/latticework/transcription.cljs | 88 +++++++++++++++++++++++-- types/cljs.d.ts | 21 ++++++ 4 files changed, 157 insertions(+), 6 deletions(-) diff --git a/scripts/smoke-cljs-blink.mjs b/scripts/smoke-cljs-blink.mjs index 1ce3aaf..133192b 100644 --- a/scripts/smoke-cljs-blink.mjs +++ b/scripts/smoke-cljs-blink.mjs @@ -810,6 +810,7 @@ tts.dispose(); const transcriptionEvents = []; const transcriptionCommands = []; +const transcriptionRecommendations = []; const transcriptionStates = []; const transcription = createTranscriptionAgency( @@ -818,11 +819,17 @@ const transcription = createTranscriptionAgency( referenceRatio: 1.5, releaseThreshold: 0.05, releaseMs: 300, + autoRestart: true, + restartDelayMs: 125, + maxRestartCount: 2, }, { onTranscriptionEvent(event) { transcriptionEvents.push(event); }, + onTranscriptionRecommendation(recommendation) { + transcriptionRecommendations.push(recommendation); + }, onAgencyCommand(target, command) { transcriptionCommands.push({ target, command }); }, @@ -859,8 +866,51 @@ if (transcriptionStates.length < 5) { throw new Error(`Expected CLJS transcription state callbacks, received ${transcriptionStates.length}`); } +transcription.fail('network drop'); +const restartRecommendation = transcriptionRecommendations.find((recommendation) => recommendation.type === 'RESTART'); +if (!restartRecommendation || + restartRecommendation.reason !== 'error' || + restartRecommendation.delayMs !== 125 || + restartRecommendation.restartCount !== 1 || + restartRecommendation.maxRestartCount !== 2) { + throw new Error(`Expected CLJS transcription restart recommendation, received ${JSON.stringify(transcriptionRecommendations)}`); +} + +const failedTranscriptionState = transcription.getState(); +if (failedTranscriptionState.restartCount !== 1 || + failedTranscriptionState.pendingRestart?.type !== 'RESTART' || + failedTranscriptionState.error !== 'network drop') { + throw new Error(`Expected CLJS transcription failure state to retain restart data, received ${JSON.stringify(failedTranscriptionState)}`); +} + +transcription.stop(); +const cleanupRecommendation = transcriptionRecommendations.find((recommendation) => ( + recommendation.type === 'CLEANUP' && + recommendation.reason === 'stop' +)); +if (!cleanupRecommendation || cleanupRecommendation.cancelRestart !== true || cleanupRecommendation.releaseBrowserResources !== true) { + throw new Error(`Expected CLJS transcription cleanup recommendation, received ${JSON.stringify(transcriptionRecommendations)}`); +} + transcription.dispose(); +const restartLimitedRecommendations = []; +const restartLimitedTranscription = createTranscriptionAgency( + { autoRestart: true, maxRestartCount: 0 }, + { + onTranscriptionRecommendation(recommendation) { + restartLimitedRecommendations.push(recommendation); + }, + }, +); +restartLimitedTranscription.start(); +restartLimitedTranscription.fail('permission denied'); +const stopRecommendation = restartLimitedRecommendations.find((recommendation) => recommendation.type === 'STOP'); +if (!stopRecommendation || stopRecommendation.reason !== 'maxRestartCount') { + throw new Error(`Expected CLJS transcription max-restart stop recommendation, received ${JSON.stringify(restartLimitedRecommendations)}`); +} +restartLimitedTranscription.dispose(); + const conversationEvents = []; const conversationCommands = []; const conversationStates = []; diff --git a/src-cljs/latticework/runtime.cljs b/src-cljs/latticework/runtime.cljs index 80efd47..47374dc 100644 --- a/src-cljs/latticework/runtime.cljs +++ b/src-cljs/latticework/runtime.cljs @@ -173,6 +173,10 @@ (when-let [on-transcription-event (fn-prop host "onTranscriptionEvent")] (on-transcription-event (protocol/data->js (:event output)))) + "transcriptionRecommendation" + (when-let [on-transcription-recommendation (fn-prop host "onTranscriptionRecommendation")] + (on-transcription-recommendation (protocol/data->js (:recommendation output)))) + "conversationEvent" (when-let [on-conversation-event (fn-prop host "onConversationEvent")] (on-conversation-event (protocol/data->js (:event output)))) diff --git a/src-cljs/latticework/transcription.cljs b/src-cljs/latticework/transcription.cljs index 81c98fc..508ca5c 100644 --- a/src-cljs/latticework/transcription.cljs +++ b/src-cljs/latticework/transcription.cljs @@ -12,7 +12,10 @@ :interruptionThreshold 0.12 :referenceRatio 1.8 :releaseThreshold 0.06 - :releaseMs 350}) + :releaseMs 350 + :autoRestart true + :restartDelayMs 250 + :maxRestartCount 3}) (def default-state {:status "idle" @@ -26,6 +29,11 @@ :lastUserLevel 0 :lastReferenceLevel 0 :lastInterruptionTime nil + :error nil + :restartCount 0 + :lastRestartReason nil + :pendingRestart nil + :lastRecommendation nil :config default-config :eventCount 0 :lastUpdatedTime nil}) @@ -51,7 +59,16 @@ @state) (defn transcription-state [state] - (select-keys @state [:status :isListening :interimTranscript :finalTranscript :isInterrupted :interruptionSource])) + (select-keys @state [:status + :isListening + :interimTranscript + :finalTranscript + :isInterrupted + :interruptionSource + :error + :restartCount + :pendingRestart + :lastRecommendation])) (defn- state-output [state] (protocol/emit-state agency-name @state)) @@ -67,6 +84,11 @@ :target target :command command}) +(defn- recommendation-output [recommendation] + {:type "transcriptionRecommendation" + :agency agency-name + :recommendation recommendation}) + (defn- result [value outputs] {:result value :outputs (vec (remove nil? outputs))}) @@ -78,6 +100,48 @@ (update :eventCount inc) (assoc :lastUpdatedTime (now)))))) +(defn- remember-recommendation! [state recommendation] + (swap! state assoc :lastRecommendation recommendation) + recommendation) + +(defn- cleanup-recommendation! [state reason extra] + (let [recommendation (remember-recommendation! + state + (merge {:type "CLEANUP" + :reason reason + :clearInterimTranscript true + :clearInterruption true + :cancelRestart true} + extra))] + (recommendation-output recommendation))) + +(defn- restart-recommendation! [state reason] + (let [config (:config @state) + max-restarts (max 0 (number-or (:maxRestartCount config) 3)) + next-count (inc (:restartCount @state)) + can-restart? (and (:continuous config) + (:autoRestart config) + (<= next-count max-restarts)) + recommendation (if can-restart? + {:type "RESTART" + :reason reason + :delayMs (max 0 (number-or (:restartDelayMs config) 250)) + :restartCount next-count + :maxRestartCount max-restarts} + {:type "STOP" + :reason (if (and (:continuous config) (:autoRestart config)) + "maxRestartCount" + reason) + :restartCount (:restartCount @state) + :maxRestartCount max-restarts})] + (update-state! + state + #(assoc % :restartCount (if can-restart? next-count (:restartCount %)) + :lastRestartReason reason + :pendingRestart (when (= "RESTART" (:type recommendation)) recommendation) + :lastRecommendation recommendation)) + (recommendation-output recommendation))) + (defn normalize-transcript [transcript] (-> (or transcript "") str/trim @@ -97,7 +161,9 @@ (update-state! state #(assoc % :status "listening" :isListening true :interimTranscript "" - :finalTranscript "")) + :finalTranscript "" + :error nil + :pendingRestart nil)) (result true [(event-output {:type "START"}) (state-output state)])) @@ -105,8 +171,10 @@ (update-state! state #(assoc % :status "idle" :isListening false :isInterrupted false - :interruptionSource nil)) + :interruptionSource nil + :pendingRestart nil)) (result true [(event-output {:type "STOP"}) + (cleanup-recommendation! state "stop" {:releaseBrowserResources true}) (state-output state)])) (defn reset-state! [state] @@ -115,8 +183,13 @@ :lastTranscript "" :lastConfidence nil :isInterrupted false - :interruptionSource nil)) + :interruptionSource nil + :error nil + :restartCount 0 + :lastRestartReason nil + :pendingRestart nil)) (result true [(event-output {:type "RESET"}) + (cleanup-recommendation! state "reset" {:releaseBrowserResources false}) (state-output state)])) (defn process-result! [state transcript final? confidence source] @@ -195,8 +268,11 @@ (defn fail! [state message] (update-state! state #(assoc % :status "error" :isListening false - :error (or message "Transcription failed"))) + :error (or message "Transcription failed") + :isInterrupted false + :interruptionSource nil)) (result false [(event-output {:type "ERROR" :message (:error @state)}) + (restart-recommendation! state "error") (state-output state)])) (defn update-config! [state config] diff --git a/types/cljs.d.ts b/types/cljs.d.ts index d3a6503..ef20d05 100644 --- a/types/cljs.d.ts +++ b/types/cljs.d.ts @@ -37,6 +37,7 @@ export interface WorkerAgencyHost { emotionEvents: TTSEmojiTimelineEvent[], ) => void; onTranscriptionEvent?: (event: TranscriptionEvent) => void; + onTranscriptionRecommendation?: (recommendation: TranscriptionRecommendation) => void; onConversationEvent?: (event: ConversationEvent) => void; onAgencyCommand?: (target: string, command: Record) => void; onTTSCommand?: (command: Record) => void; @@ -728,6 +729,9 @@ export interface TranscriptionConfig { referenceRatio?: number; releaseThreshold?: number; releaseMs?: number; + autoRestart?: boolean; + restartDelayMs?: number; + maxRestartCount?: number; } export interface TranscriptionAgencyState { @@ -737,6 +741,10 @@ export interface TranscriptionAgencyState { finalTranscript: string; isInterrupted: boolean; interruptionSource: string | null; + error: string | null; + restartCount: number; + pendingRestart: TranscriptionRecommendation | null; + lastRecommendation: TranscriptionRecommendation | null; } export interface TranscriptionSnapshot extends TranscriptionAgencyState { @@ -763,6 +771,19 @@ export interface TranscriptionEvent { [key: string]: unknown; } +export interface TranscriptionRecommendation { + type: 'RESTART' | 'STOP' | 'CLEANUP' | string; + reason: string; + delayMs?: number; + restartCount?: number; + maxRestartCount?: number; + clearInterimTranscript?: boolean; + clearInterruption?: boolean; + cancelRestart?: boolean; + releaseBrowserResources?: boolean; + [key: string]: unknown; +} + export interface TranscriptionAgency { updateConfig(config: TranscriptionConfig): boolean; start(): boolean;