diff --git a/livekit/livekit_agent_simulation.pb.go b/livekit/livekit_agent_simulation.pb.go index 50a2d5807..aec1da672 100644 --- a/livekit/livekit_agent_simulation.pb.go +++ b/livekit/livekit_agent_simulation.pb.go @@ -311,7 +311,7 @@ type SimulationRun struct { Concurrency int32 `protobuf:"varint,17,opt,name=concurrency,proto3" json:"concurrency,omitempty"` // Conversation mode for every job in this run; unspecified = TEXT. Mode SimulationMode `protobuf:"varint,18,opt,name=mode,proto3,enum=livekit.SimulationMode" json:"mode,omitempty"` - // Run-level metric aggregates. + // Run-level aggregates over this run's jobs. Metrics *SimulationRun_RunMetrics `protobuf:"bytes,19,opt,name=metrics,proto3" json:"metrics,omitempty"` // zstd-compressed SimulationRunSummary: decompress, then proto.Unmarshal. // Compressed by default so the blob ships as-is on every hop. @@ -890,28 +890,30 @@ func (x *SimulationRun_Job) GetMetrics() *SimulationRun_JobMetrics { type SimulationRun_JobMetrics struct { state protoimpl.MessageState `protogen:"open.v1"` - // Headline scores, 0-1, computed by the worker as weighted means over the - // members that ran (a missing member renormalizes the rest): - // - // accuracy = task_completion x3 + stt.keyterm_recall + (1 - stt.wer) + tts.enunciation_score - // experience = conversation.turn_taking_score x2 + tts.naturalness_score + llm.conciseness_score - AccuracyScore *float32 `protobuf:"fixed32,1,opt,name=accuracy_score,json=accuracyScore,proto3,oneof" json:"accuracy_score,omitempty"` // absent when the simulator spoiled the call - ExperienceScore *float32 `protobuf:"fixed32,2,opt,name=experience_score,json=experienceScore,proto3,oneof" json:"experience_score,omitempty"` // absent when the simulator spoiled the call - // The accuracy anchor: the scenario verdict as 1/0, or the fraction of the declared target state the call reached. - TaskCompletion *float32 `protobuf:"fixed32,3,opt,name=task_completion,json=taskCompletion,proto3,oneof" json:"task_completion,omitempty"` // absent when the simulator spoiled the call - Stt *SimulationRun_JobMetrics_STT `protobuf:"bytes,4,opt,name=stt,proto3" json:"stt,omitempty"` - Llm *SimulationRun_JobMetrics_LLM `protobuf:"bytes,5,opt,name=llm,proto3" json:"llm,omitempty"` - Tts *SimulationRun_JobMetrics_TTS `protobuf:"bytes,6,opt,name=tts,proto3" json:"tts,omitempty"` - Conversation *SimulationRun_JobMetrics_Conversation `protobuf:"bytes,7,opt,name=conversation,proto3" json:"conversation,omitempty"` - Simulator *SimulationRun_JobMetrics_Simulator `protobuf:"bytes,8,opt,name=simulator,proto3" json:"simulator,omitempty"` - // Conversation timeline: one entry per transcript turn, both speakers. + // Headline scores, 0-1. All absent when the simulator spoiled the call. + AccuracyScore *float32 `protobuf:"fixed32,1,opt,name=accuracy_score,json=accuracyScore,proto3,oneof" json:"accuracy_score,omitempty"` + ExperienceScore *float32 `protobuf:"fixed32,2,opt,name=experience_score,json=experienceScore,proto3,oneof" json:"experience_score,omitempty"` + TaskCompletion *float32 `protobuf:"fixed32,3,opt,name=task_completion,json=taskCompletion,proto3,oneof" json:"task_completion,omitempty"` // the scenario verdict as 1/0 + Stt *SimulationRun_JobMetrics_STT `protobuf:"bytes,4,opt,name=stt,proto3" json:"stt,omitempty"` + Llm *SimulationRun_JobMetrics_LLM `protobuf:"bytes,5,opt,name=llm,proto3" json:"llm,omitempty"` + Tts *SimulationRun_JobMetrics_TTS `protobuf:"bytes,6,opt,name=tts,proto3" json:"tts,omitempty"` + Conversation *SimulationRun_JobMetrics_Conversation `protobuf:"bytes,7,opt,name=conversation,proto3" json:"conversation,omitempty"` + Simulator *SimulationRun_JobMetrics_Simulator `protobuf:"bytes,8,opt,name=simulator,proto3" json:"simulator,omitempty"` + // One entry per turn, both speakers. Turns []*SimulationRun_JobMetrics_Turn `protobuf:"bytes,9,rep,name=turns,proto3" json:"turns,omitempty"` JudgeModel string `protobuf:"bytes,10,opt,name=judge_model,json=judgeModel,proto3" json:"judge_model,omitempty"` // text judge for judged scores; "" if none ran - AudioJudgeModel string `protobuf:"bytes,11,opt,name=audio_judge_model,json=audioJudgeModel,proto3" json:"audio_judge_model,omitempty"` // audio (LALM) judge; "" if none ran HasRemoteSession bool `protobuf:"varint,12,opt,name=has_remote_session,json=hasRemoteSession,proto3" json:"has_remote_session,omitempty"` // false = waveform-only capture (e.g. SIP) T0 *timestamppb.Timestamp `protobuf:"bytes,13,opt,name=t0,proto3" json:"t0,omitempty"` // audio only; call start (first speech edge); turn times relative to it - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Judged over the authored dialog (text and audio); needs the text judge. + Conciseness *float32 `protobuf:"fixed32,14,opt,name=conciseness,proto3,oneof" json:"conciseness,omitempty"` // judged {0, 0.5, 1} per agent turn; job = mean + // Whole-call issue flags; true = present, absent = not judged. + UnnecessaryToolCalls *bool `protobuf:"varint,15,opt,name=unnecessary_tool_calls,json=unnecessaryToolCalls,proto3,oneof" json:"unnecessary_tool_calls,omitempty"` + InformationLoss *bool `protobuf:"varint,16,opt,name=information_loss,json=informationLoss,proto3,oneof" json:"information_loss,omitempty"` + RedundantStatements *bool `protobuf:"varint,17,opt,name=redundant_statements,json=redundantStatements,proto3,oneof" json:"redundant_statements,omitempty"` + PoorQuestionQuality *bool `protobuf:"varint,18,opt,name=poor_question_quality,json=poorQuestionQuality,proto3,oneof" json:"poor_question_quality,omitempty"` + ConversationProgression *float32 `protobuf:"fixed32,19,opt,name=conversation_progression,json=conversationProgression,proto3,oneof" json:"conversation_progression,omitempty"` // 0-1, derived from the flags above + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SimulationRun_JobMetrics) Reset() { @@ -1014,13 +1016,6 @@ func (x *SimulationRun_JobMetrics) GetJudgeModel() string { return "" } -func (x *SimulationRun_JobMetrics) GetAudioJudgeModel() string { - if x != nil { - return x.AudioJudgeModel - } - return "" -} - func (x *SimulationRun_JobMetrics) GetHasRemoteSession() bool { if x != nil { return x.HasRemoteSession @@ -1035,21 +1030,61 @@ func (x *SimulationRun_JobMetrics) GetT0() *timestamppb.Timestamp { return nil } -// Aggregates over this run's jobs. Reuses the JobMetrics group shapes. +func (x *SimulationRun_JobMetrics) GetConciseness() float32 { + if x != nil && x.Conciseness != nil { + return *x.Conciseness + } + return 0 +} + +func (x *SimulationRun_JobMetrics) GetUnnecessaryToolCalls() bool { + if x != nil && x.UnnecessaryToolCalls != nil { + return *x.UnnecessaryToolCalls + } + return false +} + +func (x *SimulationRun_JobMetrics) GetInformationLoss() bool { + if x != nil && x.InformationLoss != nil { + return *x.InformationLoss + } + return false +} + +func (x *SimulationRun_JobMetrics) GetRedundantStatements() bool { + if x != nil && x.RedundantStatements != nil { + return *x.RedundantStatements + } + return false +} + +func (x *SimulationRun_JobMetrics) GetPoorQuestionQuality() bool { + if x != nil && x.PoorQuestionQuality != nil { + return *x.PoorQuestionQuality + } + return false +} + +func (x *SimulationRun_JobMetrics) GetConversationProgression() float32 { + if x != nil && x.ConversationProgression != nil { + return *x.ConversationProgression + } + return 0 +} + +// Aggregates over this run's jobs, so a run summary does not have to pull +// every job and turn. Reuses the JobMetrics group shapes. type SimulationRun_RunMetrics struct { - state protoimpl.MessageState `protogen:"open.v1"` - AccuracyScore *float32 `protobuf:"fixed32,1,opt,name=accuracy_score,json=accuracyScore,proto3,oneof" json:"accuracy_score,omitempty"` // mean over scored jobs - ExperienceScore *float32 `protobuf:"fixed32,2,opt,name=experience_score,json=experienceScore,proto3,oneof" json:"experience_score,omitempty"` - ScenarioPassRate *float32 `protobuf:"fixed32,3,opt,name=scenario_pass_rate,json=scenarioPassRate,proto3,oneof" json:"scenario_pass_rate,omitempty"` // share of jobs whose scenario verdict passed - Stt *SimulationRun_JobMetrics_STT `protobuf:"bytes,4,opt,name=stt,proto3" json:"stt,omitempty"` - Llm *SimulationRun_JobMetrics_LLM `protobuf:"bytes,5,opt,name=llm,proto3" json:"llm,omitempty"` - Tts *SimulationRun_JobMetrics_TTS `protobuf:"bytes,6,opt,name=tts,proto3" json:"tts,omitempty"` - Conversation *SimulationRun_JobMetrics_Conversation `protobuf:"bytes,7,opt,name=conversation,proto3" json:"conversation,omitempty"` - JobsTotal uint32 `protobuf:"varint,9,opt,name=jobs_total,json=jobsTotal,proto3" json:"jobs_total,omitempty"` - JobsMeasured uint32 `protobuf:"varint,10,opt,name=jobs_measured,json=jobsMeasured,proto3" json:"jobs_measured,omitempty"` // composites cover jobs_measured - jobs_simulator_fault - JobsSimulatorFault uint32 `protobuf:"varint,11,opt,name=jobs_simulator_fault,json=jobsSimulatorFault,proto3" json:"jobs_simulator_fault,omitempty"` // spoiled by the simulator: flagged, not scored - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + AccuracyScore *float32 `protobuf:"fixed32,1,opt,name=accuracy_score,json=accuracyScore,proto3,oneof" json:"accuracy_score,omitempty"` // mean over scored jobs + ExperienceScore *float32 `protobuf:"fixed32,2,opt,name=experience_score,json=experienceScore,proto3,oneof" json:"experience_score,omitempty"` + ScenarioPassRate *float32 `protobuf:"fixed32,3,opt,name=scenario_pass_rate,json=scenarioPassRate,proto3,oneof" json:"scenario_pass_rate,omitempty"` // share of jobs whose scenario verdict passed + Stt *SimulationRun_JobMetrics_STT `protobuf:"bytes,4,opt,name=stt,proto3" json:"stt,omitempty"` + Llm *SimulationRun_JobMetrics_LLM `protobuf:"bytes,5,opt,name=llm,proto3" json:"llm,omitempty"` + Tts *SimulationRun_JobMetrics_TTS `protobuf:"bytes,6,opt,name=tts,proto3" json:"tts,omitempty"` + Conversation *SimulationRun_JobMetrics_Conversation `protobuf:"bytes,7,opt,name=conversation,proto3" json:"conversation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SimulationRun_RunMetrics) Reset() { @@ -1131,27 +1166,6 @@ func (x *SimulationRun_RunMetrics) GetConversation() *SimulationRun_JobMetrics_C return nil } -func (x *SimulationRun_RunMetrics) GetJobsTotal() uint32 { - if x != nil { - return x.JobsTotal - } - return 0 -} - -func (x *SimulationRun_RunMetrics) GetJobsMeasured() uint32 { - if x != nil { - return x.JobsMeasured - } - return 0 -} - -func (x *SimulationRun_RunMetrics) GetJobsSimulatorFault() uint32 { - if x != nil { - return x.JobsSimulatorFault - } - return 0 -} - type SimulationRun_Create struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -1438,19 +1452,20 @@ func (x *SimulationRun_Job_Usage) GetAudioTurnsCount() int32 { // The agent's perception of the caller. Audio only. type SimulationRun_JobMetrics_STT struct { - state protoimpl.MessageState `protogen:"open.v1"` - Wer *float32 `protobuf:"fixed32,1,opt,name=wer,proto3,oneof" json:"wer,omitempty"` // needs the RemoteSession; word error rate, pooled: word_errors / words - Words *uint32 `protobuf:"varint,2,opt,name=words,proto3,oneof" json:"words,omitempty"` // needs the RemoteSession; pooling stats: run WER = sum errors / sum words - WordErrors *uint32 `protobuf:"varint,3,opt,name=word_errors,json=wordErrors,proto3,oneof" json:"word_errors,omitempty"` // needs the RemoteSession - Cer *float32 `protobuf:"fixed32,4,opt,name=cer,proto3,oneof" json:"cer,omitempty"` // needs the RemoteSession; character error rate, pooled - Chars *uint32 `protobuf:"varint,9,opt,name=chars,proto3,oneof" json:"chars,omitempty"` // needs the RemoteSession - CharErrors *uint32 `protobuf:"varint,10,opt,name=char_errors,json=charErrors,proto3,oneof" json:"char_errors,omitempty"` // needs the RemoteSession - KeytermRecall *float32 `protobuf:"fixed32,5,opt,name=keyterm_recall,json=keytermRecall,proto3,oneof" json:"keyterm_recall,omitempty"` // needs the RemoteSession; recall of uttered key entities (names, IDs, amounts) - KeytermsUttered *uint32 `protobuf:"varint,6,opt,name=keyterms_uttered,json=keytermsUttered,proto3,oneof" json:"keyterms_uttered,omitempty"` // needs the RemoteSession - KeytermsRecognized *uint32 `protobuf:"varint,7,opt,name=keyterms_recognized,json=keytermsRecognized,proto3,oneof" json:"keyterms_recognized,omitempty"` // needs the RemoteSession - TranscriptionLatencyMs *uint32 `protobuf:"varint,8,opt,name=transcription_latency_ms,json=transcriptionLatencyMs,proto3,oneof" json:"transcription_latency_ms,omitempty"` // needs the RemoteSession; agent-reported STT/endpointing delay, mean - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Wer *float32 `protobuf:"fixed32,1,opt,name=wer,proto3,oneof" json:"wer,omitempty"` // needs the RemoteSession; word_errors / words + Words *uint32 `protobuf:"varint,2,opt,name=words,proto3,oneof" json:"words,omitempty"` // needs the RemoteSession + WordErrors *uint32 `protobuf:"varint,3,opt,name=word_errors,json=wordErrors,proto3,oneof" json:"word_errors,omitempty"` // needs the RemoteSession + Cer *float32 `protobuf:"fixed32,4,opt,name=cer,proto3,oneof" json:"cer,omitempty"` // needs the RemoteSession; char_errors / chars + Chars *uint32 `protobuf:"varint,9,opt,name=chars,proto3,oneof" json:"chars,omitempty"` // needs the RemoteSession + CharErrors *uint32 `protobuf:"varint,10,opt,name=char_errors,json=charErrors,proto3,oneof" json:"char_errors,omitempty"` // needs the RemoteSession + SttDelayMs *uint32 `protobuf:"varint,8,opt,name=stt_delay_ms,json=sttDelayMs,proto3,oneof" json:"stt_delay_ms,omitempty"` // needs the RemoteSession; agent-reported STT/endpointing delay + // Key entities (names, IDs, amounts) the caller said. Needs the text judge. + EntityRecognition *float32 `protobuf:"fixed32,11,opt,name=entity_recognition,json=entityRecognition,proto3,oneof" json:"entity_recognition,omitempty"` // recognized / uttered + EntitiesUttered *uint32 `protobuf:"varint,12,opt,name=entities_uttered,json=entitiesUttered,proto3,oneof" json:"entities_uttered,omitempty"` + EntitiesRecognized *uint32 `protobuf:"varint,13,opt,name=entities_recognized,json=entitiesRecognized,proto3,oneof" json:"entities_recognized,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SimulationRun_JobMetrics_STT) Reset() { @@ -1525,42 +1540,41 @@ func (x *SimulationRun_JobMetrics_STT) GetCharErrors() uint32 { return 0 } -func (x *SimulationRun_JobMetrics_STT) GetKeytermRecall() float32 { - if x != nil && x.KeytermRecall != nil { - return *x.KeytermRecall +func (x *SimulationRun_JobMetrics_STT) GetSttDelayMs() uint32 { + if x != nil && x.SttDelayMs != nil { + return *x.SttDelayMs } return 0 } -func (x *SimulationRun_JobMetrics_STT) GetKeytermsUttered() uint32 { - if x != nil && x.KeytermsUttered != nil { - return *x.KeytermsUttered +func (x *SimulationRun_JobMetrics_STT) GetEntityRecognition() float32 { + if x != nil && x.EntityRecognition != nil { + return *x.EntityRecognition } return 0 } -func (x *SimulationRun_JobMetrics_STT) GetKeytermsRecognized() uint32 { - if x != nil && x.KeytermsRecognized != nil { - return *x.KeytermsRecognized +func (x *SimulationRun_JobMetrics_STT) GetEntitiesUttered() uint32 { + if x != nil && x.EntitiesUttered != nil { + return *x.EntitiesUttered } return 0 } -func (x *SimulationRun_JobMetrics_STT) GetTranscriptionLatencyMs() uint32 { - if x != nil && x.TranscriptionLatencyMs != nil { - return *x.TranscriptionLatencyMs +func (x *SimulationRun_JobMetrics_STT) GetEntitiesRecognized() uint32 { + if x != nil && x.EntitiesRecognized != nil { + return *x.EntitiesRecognized } return 0 } type SimulationRun_JobMetrics_LLM struct { - state protoimpl.MessageState `protogen:"open.v1"` - TtftMs *uint32 `protobuf:"varint,1,opt,name=ttft_ms,json=ttftMs,proto3,oneof" json:"ttft_ms,omitempty"` // needs the RemoteSession; time to first token, mean (per-turn in turns) - TtfsMs *uint32 `protobuf:"varint,2,opt,name=ttfs_ms,json=ttfsMs,proto3,oneof" json:"ttfs_ms,omitempty"` // needs the RemoteSession; time to first sentence (the smallest speakable unit) - TokensPerSecond *float32 `protobuf:"fixed32,3,opt,name=tokens_per_second,json=tokensPerSecond,proto3,oneof" json:"tokens_per_second,omitempty"` // needs the RemoteSession; decode rate, mean (per-turn in turns.llm) - ConcisenessScore *float32 `protobuf:"fixed32,4,opt,name=conciseness_score,json=concisenessScore,proto3,oneof" json:"conciseness_score,omitempty"` // audio only; needs the audio judge; judged 0-1: brief enough for voice - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + TtftMs *uint32 `protobuf:"varint,1,opt,name=ttft_ms,json=ttftMs,proto3,oneof" json:"ttft_ms,omitempty"` // needs the RemoteSession; time to first token + TtfsMs *uint32 `protobuf:"varint,2,opt,name=ttfs_ms,json=ttfsMs,proto3,oneof" json:"ttfs_ms,omitempty"` // needs the RemoteSession; time to first sentence, the smallest speakable unit + TokensPerSecond *float32 `protobuf:"fixed32,3,opt,name=tokens_per_second,json=tokensPerSecond,proto3,oneof" json:"tokens_per_second,omitempty"` // needs the RemoteSession + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SimulationRun_JobMetrics_LLM) Reset() { @@ -1614,31 +1628,22 @@ func (x *SimulationRun_JobMetrics_LLM) GetTokensPerSecond() float32 { return 0 } -func (x *SimulationRun_JobMetrics_LLM) GetConcisenessScore() float32 { - if x != nil && x.ConcisenessScore != nil { - return *x.ConcisenessScore - } - return 0 -} - -// The agent's voice. audio only. +// The agent's voice. Audio only. type SimulationRun_JobMetrics_TTS struct { - state protoimpl.MessageState `protogen:"open.v1"` - TtfaMs *uint32 `protobuf:"varint,1,opt,name=ttfa_ms,json=ttfaMs,proto3,oneof" json:"ttfa_ms,omitempty"` // needs the RemoteSession - TtfbMs *uint32 `protobuf:"varint,2,opt,name=ttfb_ms,json=ttfbMs,proto3,oneof" json:"ttfb_ms,omitempty"` // needs the RemoteSession; provider byte-level TTFB - Wer *float32 `protobuf:"fixed32,3,opt,name=wer,proto3,oneof" json:"wer,omitempty"` // needs the RemoteSession; word error rate, pooled: word_errors / words - Words *uint32 `protobuf:"varint,7,opt,name=words,proto3,oneof" json:"words,omitempty"` // needs the RemoteSession - WordErrors *uint32 `protobuf:"varint,8,opt,name=word_errors,json=wordErrors,proto3,oneof" json:"word_errors,omitempty"` // needs the RemoteSession - Cer *float32 `protobuf:"fixed32,9,opt,name=cer,proto3,oneof" json:"cer,omitempty"` // needs the RemoteSession; character error rate, pooled - Chars *uint32 `protobuf:"varint,10,opt,name=chars,proto3,oneof" json:"chars,omitempty"` // needs the RemoteSession - CharErrors *uint32 `protobuf:"varint,11,opt,name=char_errors,json=charErrors,proto3,oneof" json:"char_errors,omitempty"` // needs the RemoteSession - SpeechRateWpm *float32 `protobuf:"fixed32,4,opt,name=speech_rate_wpm,json=speechRateWpm,proto3,oneof" json:"speech_rate_wpm,omitempty"` // speaking rate; conversational English ~110-150 - HeardWords *uint32 `protobuf:"varint,12,opt,name=heard_words,json=heardWords,proto3,oneof" json:"heard_words,omitempty"` // pooling stats for speech_rate_wpm: raw heard words (not WER tokens) - SpeechMs *uint32 `protobuf:"varint,13,opt,name=speech_ms,json=speechMs,proto3,oneof" json:"speech_ms,omitempty"` // agent voiced time, sum of raw VAD segments - NaturalnessScore *float32 `protobuf:"fixed32,5,opt,name=naturalness_score,json=naturalnessScore,proto3,oneof" json:"naturalness_score,omitempty"` // needs the audio judge; judged 0-1: prosody / expressiveness - EnunciationScore *float32 `protobuf:"fixed32,6,opt,name=enunciation_score,json=enunciationScore,proto3,oneof" json:"enunciation_score,omitempty"` // needs the audio judge; judged 0-1: key entities audibly intact - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + TtfbMs *uint32 `protobuf:"varint,2,opt,name=ttfb_ms,json=ttfbMs,proto3,oneof" json:"ttfb_ms,omitempty"` // needs the RemoteSession; provider byte-level TTFB + Wer *float32 `protobuf:"fixed32,3,opt,name=wer,proto3,oneof" json:"wer,omitempty"` // needs the RemoteSession; word_errors / words + Words *uint32 `protobuf:"varint,7,opt,name=words,proto3,oneof" json:"words,omitempty"` // needs the RemoteSession + WordErrors *uint32 `protobuf:"varint,8,opt,name=word_errors,json=wordErrors,proto3,oneof" json:"word_errors,omitempty"` // needs the RemoteSession + Cer *float32 `protobuf:"fixed32,9,opt,name=cer,proto3,oneof" json:"cer,omitempty"` // needs the RemoteSession; char_errors / chars + Chars *uint32 `protobuf:"varint,10,opt,name=chars,proto3,oneof" json:"chars,omitempty"` // needs the RemoteSession + CharErrors *uint32 `protobuf:"varint,11,opt,name=char_errors,json=charErrors,proto3,oneof" json:"char_errors,omitempty"` // needs the RemoteSession + // Key entities the agent said, as the caller heard them. Needs the text judge. + EntityRecognition *float32 `protobuf:"fixed32,14,opt,name=entity_recognition,json=entityRecognition,proto3,oneof" json:"entity_recognition,omitempty"` // recognized / uttered + EntitiesUttered *uint32 `protobuf:"varint,15,opt,name=entities_uttered,json=entitiesUttered,proto3,oneof" json:"entities_uttered,omitempty"` + EntitiesRecognized *uint32 `protobuf:"varint,16,opt,name=entities_recognized,json=entitiesRecognized,proto3,oneof" json:"entities_recognized,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SimulationRun_JobMetrics_TTS) Reset() { @@ -1671,13 +1676,6 @@ func (*SimulationRun_JobMetrics_TTS) Descriptor() ([]byte, []int) { return file_livekit_agent_simulation_proto_rawDescGZIP(), []int{1, 1, 2} } -func (x *SimulationRun_JobMetrics_TTS) GetTtfaMs() uint32 { - if x != nil && x.TtfaMs != nil { - return *x.TtfaMs - } - return 0 -} - func (x *SimulationRun_JobMetrics_TTS) GetTtfbMs() uint32 { if x != nil && x.TtfbMs != nil { return *x.TtfbMs @@ -1727,62 +1725,45 @@ func (x *SimulationRun_JobMetrics_TTS) GetCharErrors() uint32 { return 0 } -func (x *SimulationRun_JobMetrics_TTS) GetSpeechRateWpm() float32 { - if x != nil && x.SpeechRateWpm != nil { - return *x.SpeechRateWpm - } - return 0 -} - -func (x *SimulationRun_JobMetrics_TTS) GetHeardWords() uint32 { - if x != nil && x.HeardWords != nil { - return *x.HeardWords - } - return 0 -} - -func (x *SimulationRun_JobMetrics_TTS) GetSpeechMs() uint32 { - if x != nil && x.SpeechMs != nil { - return *x.SpeechMs +func (x *SimulationRun_JobMetrics_TTS) GetEntityRecognition() float32 { + if x != nil && x.EntityRecognition != nil { + return *x.EntityRecognition } return 0 } -func (x *SimulationRun_JobMetrics_TTS) GetNaturalnessScore() float32 { - if x != nil && x.NaturalnessScore != nil { - return *x.NaturalnessScore +func (x *SimulationRun_JobMetrics_TTS) GetEntitiesUttered() uint32 { + if x != nil && x.EntitiesUttered != nil { + return *x.EntitiesUttered } return 0 } -func (x *SimulationRun_JobMetrics_TTS) GetEnunciationScore() float32 { - if x != nil && x.EnunciationScore != nil { - return *x.EnunciationScore +func (x *SimulationRun_JobMetrics_TTS) GetEntitiesRecognized() uint32 { + if x != nil && x.EntitiesRecognized != nil { + return *x.EntitiesRecognized } return 0 } // Audio only. type SimulationRun_JobMetrics_Conversation struct { - state protoimpl.MessageState `protogen:"open.v1"` - TurnTakingScore *float32 `protobuf:"fixed32,1,opt,name=turn_taking_score,json=turnTakingScore,proto3,oneof" json:"turn_taking_score,omitempty"` // 0-1: latency curve + cut-in/barge-in/missed-turn penalties - ResponseLatencyP50Ms *int32 `protobuf:"varint,2,opt,name=response_latency_p50_ms,json=responseLatencyP50Ms,proto3,oneof" json:"response_latency_p50_ms,omitempty"` // floor-transfer offset; a gap is > 0, an overlap < 0 - ResponseLatencyP95Ms *int32 `protobuf:"varint,3,opt,name=response_latency_p95_ms,json=responseLatencyP95Ms,proto3,oneof" json:"response_latency_p95_ms,omitempty"` - ResponseLatencyP99Ms *int32 `protobuf:"varint,4,opt,name=response_latency_p99_ms,json=responseLatencyP99Ms,proto3,oneof" json:"response_latency_p99_ms,omitempty"` - ResponseLatencyMs *int32 `protobuf:"varint,16,opt,name=response_latency_ms,json=responseLatencyMs,proto3,oneof" json:"response_latency_ms,omitempty"` // one turn's floor-transfer offset, cut-in if negative; job/run serve the percentiles - AgentYieldLatencyMs *uint32 `protobuf:"varint,5,opt,name=agent_yield_latency_ms,json=agentYieldLatencyMs,proto3,oneof" json:"agent_yield_latency_ms,omitempty"` // agent audio continuing past a barge-in — per turn: on the barging persona row (presence = barge-in); job: mean - EotMispredictionCount *uint32 `protobuf:"varint,6,opt,name=eot_misprediction_count,json=eotMispredictionCount,proto3,oneof" json:"eot_misprediction_count,omitempty"` // agent started before the caller's turn ended; 0/1 per turn - OverlapRatio *float32 `protobuf:"fixed32,7,opt,name=overlap_ratio,json=overlapRatio,proto3,oneof" json:"overlap_ratio,omitempty"` // overlapping speech / total speech - OverlapSpeechMs *uint32 `protobuf:"varint,8,opt,name=overlap_speech_ms,json=overlapSpeechMs,proto3,oneof" json:"overlap_speech_ms,omitempty"` // pooling stats for overlap_ratio - TotalSpeechMs *uint32 `protobuf:"varint,9,opt,name=total_speech_ms,json=totalSpeechMs,proto3,oneof" json:"total_speech_ms,omitempty"` - SilenceTotalMs *uint32 `protobuf:"varint,10,opt,name=silence_total_ms,json=silenceTotalMs,proto3,oneof" json:"silence_total_ms,omitempty"` // dead air within the conversation - AwkwardSilenceCount *uint32 `protobuf:"varint,11,opt,name=awkward_silence_count,json=awkwardSilenceCount,proto3,oneof" json:"awkward_silence_count,omitempty"` // gaps past the natural-pause threshold; 0/1 per turn - UnansweredPersonaTurns *uint32 `protobuf:"varint,12,opt,name=unanswered_persona_turns,json=unansweredPersonaTurns,proto3,oneof" json:"unanswered_persona_turns,omitempty"` // the simulated party spoke, the agent never responded; 0/1 per turn - FalseInterruptionCount *uint32 `protobuf:"varint,13,opt,name=false_interruption_count,json=falseInterruptionCount,proto3,oneof" json:"false_interruption_count,omitempty"` // needs the RemoteSession; agent paused for a non-interruption - FalseInterruptionUnrecoveredCount *uint32 `protobuf:"varint,14,opt,name=false_interruption_unrecovered_count,json=falseInterruptionUnrecoveredCount,proto3,oneof" json:"false_interruption_unrecovered_count,omitempty"` // needs the RemoteSession; of those, never resumed - AgentReportedE2ELatencyMs *uint32 `protobuf:"varint,15,opt,name=agent_reported_e2e_latency_ms,json=agentReportedE2eLatencyMs,proto3,oneof" json:"agent_reported_e2e_latency_ms,omitempty"` // needs the RemoteSession; the agent's own claim, mean - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + TurnTakingScore *float32 `protobuf:"fixed32,1,opt,name=turn_taking_score,json=turnTakingScore,proto3,oneof" json:"turn_taking_score,omitempty"` // 0-1 + HeardE2ELatencyP50Ms *int32 `protobuf:"varint,2,opt,name=heard_e2e_latency_p50_ms,json=heardE2eLatencyP50Ms,proto3,oneof" json:"heard_e2e_latency_p50_ms,omitempty"` // floor-transfer offset; a gap is > 0, an overlap < 0 + HeardE2ELatencyP95Ms *int32 `protobuf:"varint,3,opt,name=heard_e2e_latency_p95_ms,json=heardE2eLatencyP95Ms,proto3,oneof" json:"heard_e2e_latency_p95_ms,omitempty"` + HeardE2ELatencyP99Ms *int32 `protobuf:"varint,4,opt,name=heard_e2e_latency_p99_ms,json=heardE2eLatencyP99Ms,proto3,oneof" json:"heard_e2e_latency_p99_ms,omitempty"` + TimeToYieldMs *uint32 `protobuf:"varint,5,opt,name=time_to_yield_ms,json=timeToYieldMs,proto3,oneof" json:"time_to_yield_ms,omitempty"` // agent audio continuing past a barge-in + EotMispredictionCount *uint32 `protobuf:"varint,6,opt,name=eot_misprediction_count,json=eotMispredictionCount,proto3,oneof" json:"eot_misprediction_count,omitempty"` // agent started before the caller's turn ended + OverlapRatio *float32 `protobuf:"fixed32,7,opt,name=overlap_ratio,json=overlapRatio,proto3,oneof" json:"overlap_ratio,omitempty"` // overlapping speech / total speech + OverlapSpeechMs *uint32 `protobuf:"varint,8,opt,name=overlap_speech_ms,json=overlapSpeechMs,proto3,oneof" json:"overlap_speech_ms,omitempty"` // pooling stats for overlap_ratio + TotalSpeechMs *uint32 `protobuf:"varint,9,opt,name=total_speech_ms,json=totalSpeechMs,proto3,oneof" json:"total_speech_ms,omitempty"` + AwkwardSilenceCount *uint32 `protobuf:"varint,11,opt,name=awkward_silence_count,json=awkwardSilenceCount,proto3,oneof" json:"awkward_silence_count,omitempty"` // gaps past the natural-pause threshold + UnansweredTurns *uint32 `protobuf:"varint,12,opt,name=unanswered_turns,json=unansweredTurns,proto3,oneof" json:"unanswered_turns,omitempty"` // the simulated party spoke, the agent never responded + FalseInterruptionCount *uint32 `protobuf:"varint,13,opt,name=false_interruption_count,json=falseInterruptionCount,proto3,oneof" json:"false_interruption_count,omitempty"` // needs the RemoteSession; agent paused for a non-interruption + E2ELatencyMs *uint32 `protobuf:"varint,15,opt,name=e2e_latency_ms,json=e2eLatencyMs,proto3,oneof" json:"e2e_latency_ms,omitempty"` // needs the RemoteSession; the agent's own claim + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SimulationRun_JobMetrics_Conversation) Reset() { @@ -1822,37 +1803,30 @@ func (x *SimulationRun_JobMetrics_Conversation) GetTurnTakingScore() float32 { return 0 } -func (x *SimulationRun_JobMetrics_Conversation) GetResponseLatencyP50Ms() int32 { - if x != nil && x.ResponseLatencyP50Ms != nil { - return *x.ResponseLatencyP50Ms +func (x *SimulationRun_JobMetrics_Conversation) GetHeardE2ELatencyP50Ms() int32 { + if x != nil && x.HeardE2ELatencyP50Ms != nil { + return *x.HeardE2ELatencyP50Ms } return 0 } -func (x *SimulationRun_JobMetrics_Conversation) GetResponseLatencyP95Ms() int32 { - if x != nil && x.ResponseLatencyP95Ms != nil { - return *x.ResponseLatencyP95Ms +func (x *SimulationRun_JobMetrics_Conversation) GetHeardE2ELatencyP95Ms() int32 { + if x != nil && x.HeardE2ELatencyP95Ms != nil { + return *x.HeardE2ELatencyP95Ms } return 0 } -func (x *SimulationRun_JobMetrics_Conversation) GetResponseLatencyP99Ms() int32 { - if x != nil && x.ResponseLatencyP99Ms != nil { - return *x.ResponseLatencyP99Ms +func (x *SimulationRun_JobMetrics_Conversation) GetHeardE2ELatencyP99Ms() int32 { + if x != nil && x.HeardE2ELatencyP99Ms != nil { + return *x.HeardE2ELatencyP99Ms } return 0 } -func (x *SimulationRun_JobMetrics_Conversation) GetResponseLatencyMs() int32 { - if x != nil && x.ResponseLatencyMs != nil { - return *x.ResponseLatencyMs - } - return 0 -} - -func (x *SimulationRun_JobMetrics_Conversation) GetAgentYieldLatencyMs() uint32 { - if x != nil && x.AgentYieldLatencyMs != nil { - return *x.AgentYieldLatencyMs +func (x *SimulationRun_JobMetrics_Conversation) GetTimeToYieldMs() uint32 { + if x != nil && x.TimeToYieldMs != nil { + return *x.TimeToYieldMs } return 0 } @@ -1885,13 +1859,6 @@ func (x *SimulationRun_JobMetrics_Conversation) GetTotalSpeechMs() uint32 { return 0 } -func (x *SimulationRun_JobMetrics_Conversation) GetSilenceTotalMs() uint32 { - if x != nil && x.SilenceTotalMs != nil { - return *x.SilenceTotalMs - } - return 0 -} - func (x *SimulationRun_JobMetrics_Conversation) GetAwkwardSilenceCount() uint32 { if x != nil && x.AwkwardSilenceCount != nil { return *x.AwkwardSilenceCount @@ -1899,9 +1866,9 @@ func (x *SimulationRun_JobMetrics_Conversation) GetAwkwardSilenceCount() uint32 return 0 } -func (x *SimulationRun_JobMetrics_Conversation) GetUnansweredPersonaTurns() uint32 { - if x != nil && x.UnansweredPersonaTurns != nil { - return *x.UnansweredPersonaTurns +func (x *SimulationRun_JobMetrics_Conversation) GetUnansweredTurns() uint32 { + if x != nil && x.UnansweredTurns != nil { + return *x.UnansweredTurns } return 0 } @@ -1913,23 +1880,15 @@ func (x *SimulationRun_JobMetrics_Conversation) GetFalseInterruptionCount() uint return 0 } -func (x *SimulationRun_JobMetrics_Conversation) GetFalseInterruptionUnrecoveredCount() uint32 { - if x != nil && x.FalseInterruptionUnrecoveredCount != nil { - return *x.FalseInterruptionUnrecoveredCount - } - return 0 -} - -func (x *SimulationRun_JobMetrics_Conversation) GetAgentReportedE2ELatencyMs() uint32 { - if x != nil && x.AgentReportedE2ELatencyMs != nil { - return *x.AgentReportedE2ELatencyMs +func (x *SimulationRun_JobMetrics_Conversation) GetE2ELatencyMs() uint32 { + if x != nil && x.E2ELatencyMs != nil { + return *x.E2ELatencyMs } return 0 } -// Was the call spoiled by the simulator, not the agent? Diagnostic only — -// never folded into the scores; run aggregation excludes flagged calls -// from the two headline scores. +// Was the call spoiled by the simulator, not the agent? Diagnostic only, +// never folded into the scores. type SimulationRun_JobMetrics_Simulator struct { state protoimpl.MessageState `protogen:"open.v1"` EarlyTermination *bool `protobuf:"varint,1,opt,name=early_termination,json=earlyTermination,proto3,oneof" json:"early_termination,omitempty"` // simulator ended a still-progressing call @@ -1982,20 +1941,29 @@ func (x *SimulationRun_JobMetrics_Simulator) GetLateTermination() bool { return false } -// Rows exist for text and audio; +// Rows exist for text and audio. The groups above hold the means and pools +// over these; anything not measured for a turn stays unset. type SimulationRun_JobMetrics_Turn struct { - state protoimpl.MessageState `protogen:"open.v1"` - Index uint32 `protobuf:"varint,1,opt,name=index,proto3" json:"index,omitempty"` // 1-based, conversation order - Role agent.ChatRole `protobuf:"varint,2,opt,name=role,proto3,enum=livekit.agent.ChatRole" json:"role,omitempty"` // ASSISTANT — the agent under test, USER — simulator persona. - StartMs *uint32 `protobuf:"varint,3,opt,name=start_ms,json=startMs,proto3,oneof" json:"start_ms,omitempty"` // audio only; relative to t0; unset if the turn could not be aligned to the audio - EndMs *uint32 `protobuf:"varint,4,opt,name=end_ms,json=endMs,proto3,oneof" json:"end_ms,omitempty"` // audio only - // Job groups hold means/pools of these; a field with no per-turn measurement stays unset here. - Stt *SimulationRun_JobMetrics_STT `protobuf:"bytes,5,opt,name=stt,proto3" json:"stt,omitempty"` // transcription_latency_ms - Llm *SimulationRun_JobMetrics_LLM `protobuf:"bytes,6,opt,name=llm,proto3" json:"llm,omitempty"` // ttft_ms, ttfs_ms, tokens_per_second, conciseness_score - Tts *SimulationRun_JobMetrics_TTS `protobuf:"bytes,7,opt,name=tts,proto3" json:"tts,omitempty"` // ttfa_ms, ttfb_ms, naturalness_score, enunciation_score - Conversation *SimulationRun_JobMetrics_Conversation `protobuf:"bytes,8,opt,name=conversation,proto3" json:"conversation,omitempty"` // response_latency_ms, agent_yield_latency_ms, agent_reported_e2e_latency_ms, 0/1 counts - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Index uint32 `protobuf:"varint,1,opt,name=index,proto3" json:"index,omitempty"` // 1-based, conversation order + Role agent.ChatRole `protobuf:"varint,2,opt,name=role,proto3,enum=livekit.agent.ChatRole" json:"role,omitempty"` // ASSISTANT = the agent under test, USER = simulator persona + StartMs *uint32 `protobuf:"varint,3,opt,name=start_ms,json=startMs,proto3,oneof" json:"start_ms,omitempty"` // audio only; relative to t0; unset if the turn could not be aligned to the audio + EndMs *uint32 `protobuf:"varint,4,opt,name=end_ms,json=endMs,proto3,oneof" json:"end_ms,omitempty"` // audio only + SttDelayMs *uint32 `protobuf:"varint,5,opt,name=stt_delay_ms,json=sttDelayMs,proto3,oneof" json:"stt_delay_ms,omitempty"` // needs the RemoteSession + LlmTtftMs *uint32 `protobuf:"varint,6,opt,name=llm_ttft_ms,json=llmTtftMs,proto3,oneof" json:"llm_ttft_ms,omitempty"` // needs the RemoteSession + LlmTtfsMs *uint32 `protobuf:"varint,7,opt,name=llm_ttfs_ms,json=llmTtfsMs,proto3,oneof" json:"llm_ttfs_ms,omitempty"` // needs the RemoteSession + LlmTps *float32 `protobuf:"fixed32,8,opt,name=llm_tps,json=llmTps,proto3,oneof" json:"llm_tps,omitempty"` // needs the RemoteSession + TtsTtfbMs *uint32 `protobuf:"varint,9,opt,name=tts_ttfb_ms,json=ttsTtfbMs,proto3,oneof" json:"tts_ttfb_ms,omitempty"` // needs the RemoteSession + E2ELatencyMs *uint32 `protobuf:"varint,10,opt,name=e2e_latency_ms,json=e2eLatencyMs,proto3,oneof" json:"e2e_latency_ms,omitempty"` // needs the RemoteSession + HeardE2ELatencyMs *int32 `protobuf:"varint,11,opt,name=heard_e2e_latency_ms,json=heardE2eLatencyMs,proto3,oneof" json:"heard_e2e_latency_ms,omitempty"` + TimeToYieldMs *uint32 `protobuf:"varint,12,opt,name=time_to_yield_ms,json=timeToYieldMs,proto3,oneof" json:"time_to_yield_ms,omitempty"` + TurnTakingScore *float32 `protobuf:"fixed32,13,opt,name=turn_taking_score,json=turnTakingScore,proto3,oneof" json:"turn_taking_score,omitempty"` + Conciseness *float32 `protobuf:"fixed32,14,opt,name=conciseness,proto3,oneof" json:"conciseness,omitempty"` // needs the text judge + EotMisprediction *bool `protobuf:"varint,15,opt,name=eot_misprediction,json=eotMisprediction,proto3,oneof" json:"eot_misprediction,omitempty"` + AwkwardSilence *bool `protobuf:"varint,16,opt,name=awkward_silence,json=awkwardSilence,proto3,oneof" json:"awkward_silence,omitempty"` + Unanswered *bool `protobuf:"varint,17,opt,name=unanswered,proto3,oneof" json:"unanswered,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SimulationRun_JobMetrics_Turn) Reset() { @@ -2056,32 +2024,95 @@ func (x *SimulationRun_JobMetrics_Turn) GetEndMs() uint32 { return 0 } -func (x *SimulationRun_JobMetrics_Turn) GetStt() *SimulationRun_JobMetrics_STT { - if x != nil { - return x.Stt +func (x *SimulationRun_JobMetrics_Turn) GetSttDelayMs() uint32 { + if x != nil && x.SttDelayMs != nil { + return *x.SttDelayMs } - return nil + return 0 } -func (x *SimulationRun_JobMetrics_Turn) GetLlm() *SimulationRun_JobMetrics_LLM { - if x != nil { - return x.Llm +func (x *SimulationRun_JobMetrics_Turn) GetLlmTtftMs() uint32 { + if x != nil && x.LlmTtftMs != nil { + return *x.LlmTtftMs } - return nil + return 0 } -func (x *SimulationRun_JobMetrics_Turn) GetTts() *SimulationRun_JobMetrics_TTS { - if x != nil { - return x.Tts +func (x *SimulationRun_JobMetrics_Turn) GetLlmTtfsMs() uint32 { + if x != nil && x.LlmTtfsMs != nil { + return *x.LlmTtfsMs } - return nil + return 0 } -func (x *SimulationRun_JobMetrics_Turn) GetConversation() *SimulationRun_JobMetrics_Conversation { - if x != nil { - return x.Conversation +func (x *SimulationRun_JobMetrics_Turn) GetLlmTps() float32 { + if x != nil && x.LlmTps != nil { + return *x.LlmTps } - return nil + return 0 +} + +func (x *SimulationRun_JobMetrics_Turn) GetTtsTtfbMs() uint32 { + if x != nil && x.TtsTtfbMs != nil { + return *x.TtsTtfbMs + } + return 0 +} + +func (x *SimulationRun_JobMetrics_Turn) GetE2ELatencyMs() uint32 { + if x != nil && x.E2ELatencyMs != nil { + return *x.E2ELatencyMs + } + return 0 +} + +func (x *SimulationRun_JobMetrics_Turn) GetHeardE2ELatencyMs() int32 { + if x != nil && x.HeardE2ELatencyMs != nil { + return *x.HeardE2ELatencyMs + } + return 0 +} + +func (x *SimulationRun_JobMetrics_Turn) GetTimeToYieldMs() uint32 { + if x != nil && x.TimeToYieldMs != nil { + return *x.TimeToYieldMs + } + return 0 +} + +func (x *SimulationRun_JobMetrics_Turn) GetTurnTakingScore() float32 { + if x != nil && x.TurnTakingScore != nil { + return *x.TurnTakingScore + } + return 0 +} + +func (x *SimulationRun_JobMetrics_Turn) GetConciseness() float32 { + if x != nil && x.Conciseness != nil { + return *x.Conciseness + } + return 0 +} + +func (x *SimulationRun_JobMetrics_Turn) GetEotMisprediction() bool { + if x != nil && x.EotMisprediction != nil { + return *x.EotMisprediction + } + return false +} + +func (x *SimulationRun_JobMetrics_Turn) GetAwkwardSilence() bool { + if x != nil && x.AwkwardSilence != nil { + return *x.AwkwardSilence + } + return false +} + +func (x *SimulationRun_JobMetrics_Turn) GetUnanswered() bool { + if x != nil && x.Unanswered != nil { + return *x.Unanswered + } + return false } type SimulationRun_Create_Request struct { @@ -2787,7 +2818,7 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\n" + "suggestion\x18\x02 \x01(\tR\n" + "suggestion\x12\x14\n" + - "\x05label\x18\x03 \x01(\tR\x05label\"\xf5<\n" + + "\x05label\x18\x03 \x01(\tR\x05label\"\xf3>\n" + "\rSimulationRun\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1d\n" + "\n" + @@ -2837,7 +2868,7 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\x10STATUS_COMPLETED\x10\x02\x12\x11\n" + "\rSTATUS_FAILED\x10\x03\x12\x14\n" + "\x10STATUS_CANCELLED\x10\x04J\x04\b\t\x10\n" + - "\x1a\xad \n" + + "\x1a\xa1#\n" + "\n" + "JobMetrics\x12*\n" + "\x0eaccuracy_score\x18\x01 \x01(\x02H\x00R\raccuracyScore\x88\x01\x01\x12.\n" + @@ -2851,10 +2882,15 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\x05turns\x18\t \x03(\v2&.livekit.SimulationRun.JobMetrics.TurnR\x05turns\x12\x1f\n" + "\vjudge_model\x18\n" + " \x01(\tR\n" + - "judgeModel\x12*\n" + - "\x11audio_judge_model\x18\v \x01(\tR\x0faudioJudgeModel\x12,\n" + + "judgeModel\x12,\n" + "\x12has_remote_session\x18\f \x01(\bR\x10hasRemoteSession\x12*\n" + - "\x02t0\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\x02t0\x1a\xa7\x04\n" + + "\x02t0\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\x02t0\x12%\n" + + "\vconciseness\x18\x0e \x01(\x02H\x03R\vconciseness\x88\x01\x01\x129\n" + + "\x16unnecessary_tool_calls\x18\x0f \x01(\bH\x04R\x14unnecessaryToolCalls\x88\x01\x01\x12.\n" + + "\x10information_loss\x18\x10 \x01(\bH\x05R\x0finformationLoss\x88\x01\x01\x126\n" + + "\x14redundant_statements\x18\x11 \x01(\bH\x06R\x13redundantStatements\x88\x01\x01\x127\n" + + "\x15poor_question_quality\x18\x12 \x01(\bH\aR\x13poorQuestionQuality\x88\x01\x01\x12>\n" + + "\x18conversation_progression\x18\x13 \x01(\x02H\bR\x17conversationProgression\x88\x01\x01\x1a\x8f\x04\n" + "\x03STT\x12\x15\n" + "\x03wer\x18\x01 \x01(\x02H\x00R\x03wer\x88\x01\x01\x12\x19\n" + "\x05words\x18\x02 \x01(\rH\x01R\x05words\x88\x01\x01\x12$\n" + @@ -2864,53 +2900,45 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\x05chars\x18\t \x01(\rH\x04R\x05chars\x88\x01\x01\x12$\n" + "\vchar_errors\x18\n" + " \x01(\rH\x05R\n" + - "charErrors\x88\x01\x01\x12*\n" + - "\x0ekeyterm_recall\x18\x05 \x01(\x02H\x06R\rkeytermRecall\x88\x01\x01\x12.\n" + - "\x10keyterms_uttered\x18\x06 \x01(\rH\aR\x0fkeytermsUttered\x88\x01\x01\x124\n" + - "\x13keyterms_recognized\x18\a \x01(\rH\bR\x12keytermsRecognized\x88\x01\x01\x12=\n" + - "\x18transcription_latency_ms\x18\b \x01(\rH\tR\x16transcriptionLatencyMs\x88\x01\x01B\x06\n" + + "charErrors\x88\x01\x01\x12%\n" + + "\fstt_delay_ms\x18\b \x01(\rH\x06R\n" + + "sttDelayMs\x88\x01\x01\x122\n" + + "\x12entity_recognition\x18\v \x01(\x02H\aR\x11entityRecognition\x88\x01\x01\x12.\n" + + "\x10entities_uttered\x18\f \x01(\rH\bR\x0fentitiesUttered\x88\x01\x01\x124\n" + + "\x13entities_recognized\x18\r \x01(\rH\tR\x12entitiesRecognized\x88\x01\x01B\x06\n" + "\x04_werB\b\n" + "\x06_wordsB\x0e\n" + "\f_word_errorsB\x06\n" + "\x04_cerB\b\n" + "\x06_charsB\x0e\n" + - "\f_char_errorsB\x11\n" + - "\x0f_keyterm_recallB\x13\n" + - "\x11_keyterms_utteredB\x16\n" + - "\x14_keyterms_recognizedB\x1b\n" + - "\x19_transcription_latency_ms\x1a\xe8\x01\n" + + "\f_char_errorsB\x0f\n" + + "\r_stt_delay_msB\x15\n" + + "\x13_entity_recognitionB\x13\n" + + "\x11_entities_utteredB\x16\n" + + "\x14_entities_recognized\x1a\xa0\x01\n" + "\x03LLM\x12\x1c\n" + "\attft_ms\x18\x01 \x01(\rH\x00R\x06ttftMs\x88\x01\x01\x12\x1c\n" + "\attfs_ms\x18\x02 \x01(\rH\x01R\x06ttfsMs\x88\x01\x01\x12/\n" + - "\x11tokens_per_second\x18\x03 \x01(\x02H\x02R\x0ftokensPerSecond\x88\x01\x01\x120\n" + - "\x11conciseness_score\x18\x04 \x01(\x02H\x03R\x10concisenessScore\x88\x01\x01B\n" + + "\x11tokens_per_second\x18\x03 \x01(\x02H\x02R\x0ftokensPerSecond\x88\x01\x01B\n" + "\n" + "\b_ttft_msB\n" + "\n" + "\b_ttfs_msB\x14\n" + - "\x12_tokens_per_secondB\x14\n" + - "\x12_conciseness_score\x1a\x84\x05\n" + + "\x12_tokens_per_second\x1a\x81\x04\n" + "\x03TTS\x12\x1c\n" + - "\attfa_ms\x18\x01 \x01(\rH\x00R\x06ttfaMs\x88\x01\x01\x12\x1c\n" + - "\attfb_ms\x18\x02 \x01(\rH\x01R\x06ttfbMs\x88\x01\x01\x12\x15\n" + - "\x03wer\x18\x03 \x01(\x02H\x02R\x03wer\x88\x01\x01\x12\x19\n" + - "\x05words\x18\a \x01(\rH\x03R\x05words\x88\x01\x01\x12$\n" + - "\vword_errors\x18\b \x01(\rH\x04R\n" + + "\attfb_ms\x18\x02 \x01(\rH\x00R\x06ttfbMs\x88\x01\x01\x12\x15\n" + + "\x03wer\x18\x03 \x01(\x02H\x01R\x03wer\x88\x01\x01\x12\x19\n" + + "\x05words\x18\a \x01(\rH\x02R\x05words\x88\x01\x01\x12$\n" + + "\vword_errors\x18\b \x01(\rH\x03R\n" + "wordErrors\x88\x01\x01\x12\x15\n" + - "\x03cer\x18\t \x01(\x02H\x05R\x03cer\x88\x01\x01\x12\x19\n" + + "\x03cer\x18\t \x01(\x02H\x04R\x03cer\x88\x01\x01\x12\x19\n" + "\x05chars\x18\n" + - " \x01(\rH\x06R\x05chars\x88\x01\x01\x12$\n" + - "\vchar_errors\x18\v \x01(\rH\aR\n" + - "charErrors\x88\x01\x01\x12+\n" + - "\x0fspeech_rate_wpm\x18\x04 \x01(\x02H\bR\rspeechRateWpm\x88\x01\x01\x12$\n" + - "\vheard_words\x18\f \x01(\rH\tR\n" + - "heardWords\x88\x01\x01\x12 \n" + - "\tspeech_ms\x18\r \x01(\rH\n" + - "R\bspeechMs\x88\x01\x01\x120\n" + - "\x11naturalness_score\x18\x05 \x01(\x02H\vR\x10naturalnessScore\x88\x01\x01\x120\n" + - "\x11enunciation_score\x18\x06 \x01(\x02H\fR\x10enunciationScore\x88\x01\x01B\n" + - "\n" + - "\b_ttfa_msB\n" + + " \x01(\rH\x05R\x05chars\x88\x01\x01\x12$\n" + + "\vchar_errors\x18\v \x01(\rH\x06R\n" + + "charErrors\x88\x01\x01\x122\n" + + "\x12entity_recognition\x18\x0e \x01(\x02H\aR\x11entityRecognition\x88\x01\x01\x12.\n" + + "\x10entities_uttered\x18\x0f \x01(\rH\bR\x0fentitiesUttered\x88\x01\x01\x124\n" + + "\x13entities_recognized\x18\x10 \x01(\rH\tR\x12entitiesRecognized\x88\x01\x01B\n" + "\n" + "\b_ttfb_msB\x06\n" + "\x04_werB\b\n" + @@ -2918,68 +2946,91 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\f_word_errorsB\x06\n" + "\x04_cerB\b\n" + "\x06_charsB\x0e\n" + - "\f_char_errorsB\x12\n" + - "\x10_speech_rate_wpmB\x0e\n" + - "\f_heard_wordsB\f\n" + - "\n" + - "_speech_msB\x14\n" + - "\x12_naturalness_scoreB\x14\n" + - "\x12_enunciation_score\x1a\xd3\n" + - "\n" + + "\f_char_errorsB\x15\n" + + "\x13_entity_recognitionB\x13\n" + + "\x11_entities_utteredB\x16\n" + + "\x14_entities_recognized\x1a\xf5\a\n" + "\fConversation\x12/\n" + - "\x11turn_taking_score\x18\x01 \x01(\x02H\x00R\x0fturnTakingScore\x88\x01\x01\x12:\n" + - "\x17response_latency_p50_ms\x18\x02 \x01(\x05H\x01R\x14responseLatencyP50Ms\x88\x01\x01\x12:\n" + - "\x17response_latency_p95_ms\x18\x03 \x01(\x05H\x02R\x14responseLatencyP95Ms\x88\x01\x01\x12:\n" + - "\x17response_latency_p99_ms\x18\x04 \x01(\x05H\x03R\x14responseLatencyP99Ms\x88\x01\x01\x123\n" + - "\x13response_latency_ms\x18\x10 \x01(\x05H\x04R\x11responseLatencyMs\x88\x01\x01\x128\n" + - "\x16agent_yield_latency_ms\x18\x05 \x01(\rH\x05R\x13agentYieldLatencyMs\x88\x01\x01\x12;\n" + - "\x17eot_misprediction_count\x18\x06 \x01(\rH\x06R\x15eotMispredictionCount\x88\x01\x01\x12(\n" + - "\roverlap_ratio\x18\a \x01(\x02H\aR\foverlapRatio\x88\x01\x01\x12/\n" + - "\x11overlap_speech_ms\x18\b \x01(\rH\bR\x0foverlapSpeechMs\x88\x01\x01\x12+\n" + - "\x0ftotal_speech_ms\x18\t \x01(\rH\tR\rtotalSpeechMs\x88\x01\x01\x12-\n" + - "\x10silence_total_ms\x18\n" + - " \x01(\rH\n" + - "R\x0esilenceTotalMs\x88\x01\x01\x127\n" + - "\x15awkward_silence_count\x18\v \x01(\rH\vR\x13awkwardSilenceCount\x88\x01\x01\x12=\n" + - "\x18unanswered_persona_turns\x18\f \x01(\rH\fR\x16unansweredPersonaTurns\x88\x01\x01\x12=\n" + - "\x18false_interruption_count\x18\r \x01(\rH\rR\x16falseInterruptionCount\x88\x01\x01\x12T\n" + - "$false_interruption_unrecovered_count\x18\x0e \x01(\rH\x0eR!falseInterruptionUnrecoveredCount\x88\x01\x01\x12E\n" + - "\x1dagent_reported_e2e_latency_ms\x18\x0f \x01(\rH\x0fR\x19agentReportedE2eLatencyMs\x88\x01\x01B\x14\n" + - "\x12_turn_taking_scoreB\x1a\n" + - "\x18_response_latency_p50_msB\x1a\n" + - "\x18_response_latency_p95_msB\x1a\n" + - "\x18_response_latency_p99_msB\x16\n" + - "\x14_response_latency_msB\x19\n" + - "\x17_agent_yield_latency_msB\x1a\n" + + "\x11turn_taking_score\x18\x01 \x01(\x02H\x00R\x0fturnTakingScore\x88\x01\x01\x12;\n" + + "\x18heard_e2e_latency_p50_ms\x18\x02 \x01(\x05H\x01R\x14heardE2eLatencyP50Ms\x88\x01\x01\x12;\n" + + "\x18heard_e2e_latency_p95_ms\x18\x03 \x01(\x05H\x02R\x14heardE2eLatencyP95Ms\x88\x01\x01\x12;\n" + + "\x18heard_e2e_latency_p99_ms\x18\x04 \x01(\x05H\x03R\x14heardE2eLatencyP99Ms\x88\x01\x01\x12,\n" + + "\x10time_to_yield_ms\x18\x05 \x01(\rH\x04R\rtimeToYieldMs\x88\x01\x01\x12;\n" + + "\x17eot_misprediction_count\x18\x06 \x01(\rH\x05R\x15eotMispredictionCount\x88\x01\x01\x12(\n" + + "\roverlap_ratio\x18\a \x01(\x02H\x06R\foverlapRatio\x88\x01\x01\x12/\n" + + "\x11overlap_speech_ms\x18\b \x01(\rH\aR\x0foverlapSpeechMs\x88\x01\x01\x12+\n" + + "\x0ftotal_speech_ms\x18\t \x01(\rH\bR\rtotalSpeechMs\x88\x01\x01\x127\n" + + "\x15awkward_silence_count\x18\v \x01(\rH\tR\x13awkwardSilenceCount\x88\x01\x01\x12.\n" + + "\x10unanswered_turns\x18\f \x01(\rH\n" + + "R\x0funansweredTurns\x88\x01\x01\x12=\n" + + "\x18false_interruption_count\x18\r \x01(\rH\vR\x16falseInterruptionCount\x88\x01\x01\x12)\n" + + "\x0ee2e_latency_ms\x18\x0f \x01(\rH\fR\fe2eLatencyMs\x88\x01\x01B\x14\n" + + "\x12_turn_taking_scoreB\x1b\n" + + "\x19_heard_e2e_latency_p50_msB\x1b\n" + + "\x19_heard_e2e_latency_p95_msB\x1b\n" + + "\x19_heard_e2e_latency_p99_msB\x13\n" + + "\x11_time_to_yield_msB\x1a\n" + "\x18_eot_misprediction_countB\x10\n" + "\x0e_overlap_ratioB\x14\n" + "\x12_overlap_speech_msB\x12\n" + - "\x10_total_speech_msB\x13\n" + - "\x11_silence_total_msB\x18\n" + - "\x16_awkward_silence_countB\x1b\n" + - "\x19_unanswered_persona_turnsB\x1b\n" + - "\x19_false_interruption_countB'\n" + - "%_false_interruption_unrecovered_countB \n" + - "\x1e_agent_reported_e2e_latency_ms\x1a\x98\x01\n" + + "\x10_total_speech_msB\x18\n" + + "\x16_awkward_silence_countB\x13\n" + + "\x11_unanswered_turnsB\x1b\n" + + "\x19_false_interruption_countB\x11\n" + + "\x0f_e2e_latency_ms\x1a\x98\x01\n" + "\tSimulator\x120\n" + "\x11early_termination\x18\x01 \x01(\bH\x00R\x10earlyTermination\x88\x01\x01\x12.\n" + "\x10late_termination\x18\x02 \x01(\bH\x01R\x0flateTermination\x88\x01\x01B\x14\n" + "\x12_early_terminationB\x13\n" + - "\x11_late_termination\x1a\x9c\x03\n" + + "\x11_late_termination\x1a\xaa\a\n" + "\x04Turn\x12\x14\n" + "\x05index\x18\x01 \x01(\rR\x05index\x12+\n" + "\x04role\x18\x02 \x01(\x0e2\x17.livekit.agent.ChatRoleR\x04role\x12\x1e\n" + "\bstart_ms\x18\x03 \x01(\rH\x00R\astartMs\x88\x01\x01\x12\x1a\n" + - "\x06end_ms\x18\x04 \x01(\rH\x01R\x05endMs\x88\x01\x01\x127\n" + - "\x03stt\x18\x05 \x01(\v2%.livekit.SimulationRun.JobMetrics.STTR\x03stt\x127\n" + - "\x03llm\x18\x06 \x01(\v2%.livekit.SimulationRun.JobMetrics.LLMR\x03llm\x127\n" + - "\x03tts\x18\a \x01(\v2%.livekit.SimulationRun.JobMetrics.TTSR\x03tts\x12R\n" + - "\fconversation\x18\b \x01(\v2..livekit.SimulationRun.JobMetrics.ConversationR\fconversationB\v\n" + + "\x06end_ms\x18\x04 \x01(\rH\x01R\x05endMs\x88\x01\x01\x12%\n" + + "\fstt_delay_ms\x18\x05 \x01(\rH\x02R\n" + + "sttDelayMs\x88\x01\x01\x12#\n" + + "\vllm_ttft_ms\x18\x06 \x01(\rH\x03R\tllmTtftMs\x88\x01\x01\x12#\n" + + "\vllm_ttfs_ms\x18\a \x01(\rH\x04R\tllmTtfsMs\x88\x01\x01\x12\x1c\n" + + "\allm_tps\x18\b \x01(\x02H\x05R\x06llmTps\x88\x01\x01\x12#\n" + + "\vtts_ttfb_ms\x18\t \x01(\rH\x06R\tttsTtfbMs\x88\x01\x01\x12)\n" + + "\x0ee2e_latency_ms\x18\n" + + " \x01(\rH\aR\fe2eLatencyMs\x88\x01\x01\x124\n" + + "\x14heard_e2e_latency_ms\x18\v \x01(\x05H\bR\x11heardE2eLatencyMs\x88\x01\x01\x12,\n" + + "\x10time_to_yield_ms\x18\f \x01(\rH\tR\rtimeToYieldMs\x88\x01\x01\x12/\n" + + "\x11turn_taking_score\x18\r \x01(\x02H\n" + + "R\x0fturnTakingScore\x88\x01\x01\x12%\n" + + "\vconciseness\x18\x0e \x01(\x02H\vR\vconciseness\x88\x01\x01\x120\n" + + "\x11eot_misprediction\x18\x0f \x01(\bH\fR\x10eotMisprediction\x88\x01\x01\x12,\n" + + "\x0fawkward_silence\x18\x10 \x01(\bH\rR\x0eawkwardSilence\x88\x01\x01\x12#\n" + + "\n" + + "unanswered\x18\x11 \x01(\bH\x0eR\n" + + "unanswered\x88\x01\x01B\v\n" + "\t_start_msB\t\n" + - "\a_end_msB\x11\n" + + "\a_end_msB\x0f\n" + + "\r_stt_delay_msB\x0e\n" + + "\f_llm_ttft_msB\x0e\n" + + "\f_llm_ttfs_msB\n" + + "\n" + + "\b_llm_tpsB\x0e\n" + + "\f_tts_ttfb_msB\x11\n" + + "\x0f_e2e_latency_msB\x17\n" + + "\x15_heard_e2e_latency_msB\x13\n" + + "\x11_time_to_yield_msB\x14\n" + + "\x12_turn_taking_scoreB\x0e\n" + + "\f_concisenessB\x14\n" + + "\x12_eot_mispredictionB\x12\n" + + "\x10_awkward_silenceB\r\n" + + "\v_unansweredB\x11\n" + "\x0f_accuracy_scoreB\x13\n" + "\x11_experience_scoreB\x12\n" + - "\x10_task_completion\x1a\xcf\x04\n" + + "\x10_task_completionB\x0e\n" + + "\f_concisenessB\x19\n" + + "\x17_unnecessary_tool_callsB\x13\n" + + "\x11_information_lossB\x17\n" + + "\x15_redundant_statementsB\x18\n" + + "\x16_poor_question_qualityB\x1b\n" + + "\x19_conversation_progression\x1a\xd9\x03\n" + "\n" + "RunMetrics\x12*\n" + "\x0eaccuracy_score\x18\x01 \x01(\x02H\x00R\raccuracyScore\x88\x01\x01\x12.\n" + @@ -2988,12 +3039,7 @@ const file_livekit_agent_simulation_proto_rawDesc = "" + "\x03stt\x18\x04 \x01(\v2%.livekit.SimulationRun.JobMetrics.STTR\x03stt\x127\n" + "\x03llm\x18\x05 \x01(\v2%.livekit.SimulationRun.JobMetrics.LLMR\x03llm\x127\n" + "\x03tts\x18\x06 \x01(\v2%.livekit.SimulationRun.JobMetrics.TTSR\x03tts\x12R\n" + - "\fconversation\x18\a \x01(\v2..livekit.SimulationRun.JobMetrics.ConversationR\fconversation\x12\x1d\n" + - "\n" + - "jobs_total\x18\t \x01(\rR\tjobsTotal\x12#\n" + - "\rjobs_measured\x18\n" + - " \x01(\rR\fjobsMeasured\x120\n" + - "\x14jobs_simulator_fault\x18\v \x01(\rR\x12jobsSimulatorFaultB\x11\n" + + "\fconversation\x18\a \x01(\v2..livekit.SimulationRun.JobMetrics.ConversationR\fconversationB\x11\n" + "\x0f_accuracy_scoreB\x13\n" + "\x11_experience_scoreB\x15\n" + "\x13_scenario_pass_rate\x1a\xf5\x03\n" + @@ -3189,36 +3235,32 @@ var file_livekit_agent_simulation_proto_depIdxs = []int32{ 22, // 29: livekit.SimulationRun.RunMetrics.tts:type_name -> livekit.SimulationRun.JobMetrics.TTS 23, // 30: livekit.SimulationRun.RunMetrics.conversation:type_name -> livekit.SimulationRun.JobMetrics.Conversation 42, // 31: livekit.SimulationRun.JobMetrics.Turn.role:type_name -> livekit.agent.ChatRole - 20, // 32: livekit.SimulationRun.JobMetrics.Turn.stt:type_name -> livekit.SimulationRun.JobMetrics.STT - 21, // 33: livekit.SimulationRun.JobMetrics.Turn.llm:type_name -> livekit.SimulationRun.JobMetrics.LLM - 22, // 34: livekit.SimulationRun.JobMetrics.Turn.tts:type_name -> livekit.SimulationRun.JobMetrics.TTS - 23, // 35: livekit.SimulationRun.JobMetrics.Turn.conversation:type_name -> livekit.SimulationRun.JobMetrics.Conversation - 6, // 36: livekit.SimulationRun.Create.Request.scenario_group:type_name -> livekit.ScenarioGroup - 0, // 37: livekit.SimulationRun.Create.Request.mode:type_name -> livekit.SimulationMode - 43, // 38: livekit.SimulationRun.Create.Response.presigned_post_request:type_name -> livekit.PresignedPostRequest - 4, // 39: livekit.SimulationRun.Get.Response.run:type_name -> livekit.SimulationRun - 1, // 40: livekit.SimulationRun.List.Request.status:type_name -> livekit.SimulationRun.Status - 44, // 41: livekit.SimulationRun.List.Request.page_token:type_name -> livekit.TokenPagination - 4, // 42: livekit.SimulationRun.List.Response.runs:type_name -> livekit.SimulationRun - 44, // 43: livekit.SimulationRun.List.Response.next_page_token:type_name -> livekit.TokenPagination - 5, // 44: livekit.Scenario.CreateFromSession.Response.scenario:type_name -> livekit.Scenario - 26, // 45: livekit.AgentSimulation.CreateSimulationRun:input_type -> livekit.SimulationRun.Create.Request - 28, // 46: livekit.AgentSimulation.ConfirmSimulationSourceUpload:input_type -> livekit.SimulationRun.ConfirmSourceUpload.Request - 30, // 47: livekit.AgentSimulation.GetSimulationRun:input_type -> livekit.SimulationRun.Get.Request - 32, // 48: livekit.AgentSimulation.ListSimulationRuns:input_type -> livekit.SimulationRun.List.Request - 34, // 49: livekit.AgentSimulation.CancelSimulationRun:input_type -> livekit.SimulationRun.Cancel.Request - 38, // 50: livekit.AgentSimulation.CreateScenarioFromSession:input_type -> livekit.Scenario.CreateFromSession.Request - 27, // 51: livekit.AgentSimulation.CreateSimulationRun:output_type -> livekit.SimulationRun.Create.Response - 29, // 52: livekit.AgentSimulation.ConfirmSimulationSourceUpload:output_type -> livekit.SimulationRun.ConfirmSourceUpload.Response - 31, // 53: livekit.AgentSimulation.GetSimulationRun:output_type -> livekit.SimulationRun.Get.Response - 33, // 54: livekit.AgentSimulation.ListSimulationRuns:output_type -> livekit.SimulationRun.List.Response - 35, // 55: livekit.AgentSimulation.CancelSimulationRun:output_type -> livekit.SimulationRun.Cancel.Response - 39, // 56: livekit.AgentSimulation.CreateScenarioFromSession:output_type -> livekit.Scenario.CreateFromSession.Response - 51, // [51:57] is the sub-list for method output_type - 45, // [45:51] is the sub-list for method input_type - 45, // [45:45] is the sub-list for extension type_name - 45, // [45:45] is the sub-list for extension extendee - 0, // [0:45] is the sub-list for field type_name + 6, // 32: livekit.SimulationRun.Create.Request.scenario_group:type_name -> livekit.ScenarioGroup + 0, // 33: livekit.SimulationRun.Create.Request.mode:type_name -> livekit.SimulationMode + 43, // 34: livekit.SimulationRun.Create.Response.presigned_post_request:type_name -> livekit.PresignedPostRequest + 4, // 35: livekit.SimulationRun.Get.Response.run:type_name -> livekit.SimulationRun + 1, // 36: livekit.SimulationRun.List.Request.status:type_name -> livekit.SimulationRun.Status + 44, // 37: livekit.SimulationRun.List.Request.page_token:type_name -> livekit.TokenPagination + 4, // 38: livekit.SimulationRun.List.Response.runs:type_name -> livekit.SimulationRun + 44, // 39: livekit.SimulationRun.List.Response.next_page_token:type_name -> livekit.TokenPagination + 5, // 40: livekit.Scenario.CreateFromSession.Response.scenario:type_name -> livekit.Scenario + 26, // 41: livekit.AgentSimulation.CreateSimulationRun:input_type -> livekit.SimulationRun.Create.Request + 28, // 42: livekit.AgentSimulation.ConfirmSimulationSourceUpload:input_type -> livekit.SimulationRun.ConfirmSourceUpload.Request + 30, // 43: livekit.AgentSimulation.GetSimulationRun:input_type -> livekit.SimulationRun.Get.Request + 32, // 44: livekit.AgentSimulation.ListSimulationRuns:input_type -> livekit.SimulationRun.List.Request + 34, // 45: livekit.AgentSimulation.CancelSimulationRun:input_type -> livekit.SimulationRun.Cancel.Request + 38, // 46: livekit.AgentSimulation.CreateScenarioFromSession:input_type -> livekit.Scenario.CreateFromSession.Request + 27, // 47: livekit.AgentSimulation.CreateSimulationRun:output_type -> livekit.SimulationRun.Create.Response + 29, // 48: livekit.AgentSimulation.ConfirmSimulationSourceUpload:output_type -> livekit.SimulationRun.ConfirmSourceUpload.Response + 31, // 49: livekit.AgentSimulation.GetSimulationRun:output_type -> livekit.SimulationRun.Get.Response + 33, // 50: livekit.AgentSimulation.ListSimulationRuns:output_type -> livekit.SimulationRun.List.Response + 35, // 51: livekit.AgentSimulation.CancelSimulationRun:output_type -> livekit.SimulationRun.Cancel.Response + 39, // 52: livekit.AgentSimulation.CreateScenarioFromSession:output_type -> livekit.Scenario.CreateFromSession.Response + 47, // [47:53] is the sub-list for method output_type + 41, // [41:47] is the sub-list for method input_type + 41, // [41:41] is the sub-list for extension type_name + 41, // [41:41] is the sub-list for extension extendee + 0, // [0:41] is the sub-list for field type_name } func init() { file_livekit_agent_simulation_proto_init() } diff --git a/livekit/livekit_agent_simulation.twirp.go b/livekit/livekit_agent_simulation.twirp.go index b2d752787..4be280096 100644 --- a/livekit/livekit_agent_simulation.twirp.go +++ b/livekit/livekit_agent_simulation.twirp.go @@ -1912,216 +1912,220 @@ func (s *agentSimulationServer) PathPrefix() string { } var twirpFileDescriptor9 = []byte{ - // 3365 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcd, 0x6f, 0x1b, 0x49, - 0x76, 0x37, 0x3f, 0x45, 0x3e, 0x8a, 0x1f, 0x2a, 0xc9, 0x32, 0xdd, 0xb3, 0x9e, 0x91, 0x3d, 0x33, - 0x3b, 0x86, 0x3d, 0x23, 0x2b, 0x9a, 0x75, 0x66, 0xed, 0x99, 0x01, 0x22, 0xca, 0xb4, 0x49, 0x47, - 0x92, 0x95, 0x22, 0x85, 0xc9, 0x4e, 0x10, 0x34, 0x5a, 0xdd, 0x65, 0xaa, 0x65, 0xb2, 0x8b, 0xe9, - 0xaa, 0xb6, 0xac, 0x05, 0x72, 0xcb, 0x61, 0x81, 0x9c, 0x72, 0xcb, 0x21, 0x08, 0x72, 0x59, 0x04, - 0x09, 0x90, 0x53, 0xae, 0x01, 0xf2, 0x27, 0x04, 0x41, 0x80, 0x9c, 0xf2, 0x07, 0x04, 0xb9, 0xef, - 0x3d, 0xa8, 0x57, 0xd5, 0xcd, 0xe6, 0x97, 0xac, 0x9d, 0xcd, 0xdc, 0x72, 0x63, 0xbf, 0xf7, 0x7b, - 0xaf, 0xab, 0x5e, 0xfd, 0xde, 0x7b, 0xd5, 0x55, 0x84, 0x0f, 0x87, 0xfe, 0x5b, 0xf6, 0xc6, 0x97, - 0xb6, 0x33, 0x60, 0x81, 0xb4, 0x85, 0x3f, 0x8a, 0x86, 0x8e, 0xf4, 0x79, 0xb0, 0x3d, 0x0e, 0xb9, - 0xe4, 0x64, 0xc5, 0xe8, 0xad, 0xbb, 0x08, 0x78, 0x34, 0x03, 0x67, 0x42, 0x24, 0x58, 0xeb, 0xa3, - 0x01, 0xe7, 0x83, 0x21, 0x7b, 0x84, 0x4f, 0xa7, 0xd1, 0xeb, 0x47, 0xd2, 0x1f, 0x31, 0x21, 0x9d, - 0xd1, 0xd8, 0x00, 0x6e, 0xc7, 0xd6, 0xee, 0x90, 0x47, 0x9e, 0xf6, 0x61, 0x54, 0x1b, 0xb1, 0x6a, - 0xc4, 0x3d, 0x36, 0x14, 0x5a, 0x7a, 0xef, 0xdf, 0x73, 0xb0, 0xd1, 0x4b, 0x86, 0x44, 0xa3, 0xa0, - 0x17, 0x8d, 0x46, 0x4e, 0x78, 0x49, 0x36, 0xa1, 0x38, 0x76, 0x84, 0x60, 0x5e, 0x33, 0xb3, 0x95, - 0xb9, 0x5f, 0xa0, 0xe6, 0x49, 0xc9, 0x5f, 0x3b, 0xfe, 0x90, 0x79, 0xcd, 0xac, 0x96, 0xeb, 0x27, - 0x72, 0x07, 0x60, 0xc0, 0xfd, 0x60, 0x60, 0x5f, 0xb0, 0xe1, 0xb0, 0x99, 0xdb, 0xca, 0xdc, 0x2f, - 0xd3, 0x32, 0x4a, 0xbe, 0x63, 0xc3, 0xa1, 0x52, 0x4b, 0x6e, 0xfb, 0xa3, 0x71, 0xc8, 0xdf, 0xb2, - 0x66, 0x5e, 0xab, 0x25, 0xef, 0x6a, 0x01, 0xf9, 0x1a, 0x8a, 0xbe, 0x10, 0x11, 0x13, 0xcd, 0xc2, - 0x56, 0xee, 0x7e, 0x65, 0xf7, 0xe3, 0x6d, 0x33, 0xda, 0xed, 0x45, 0x83, 0xdb, 0xee, 0x2a, 0x2c, - 0x35, 0x26, 0xe4, 0x8f, 0x60, 0xd5, 0x3d, 0x73, 0xa4, 0x7d, 0xe6, 0x0b, 0xc9, 0xc3, 0xcb, 0x66, - 0x11, 0x5d, 0x6c, 0x5f, 0xed, 0x62, 0xff, 0xcc, 0x91, 0x1d, 0x6d, 0xd0, 0x0e, 0x64, 0x78, 0x49, - 0x2b, 0xee, 0x44, 0x62, 0x7d, 0x0f, 0x8d, 0x59, 0x00, 0x69, 0x40, 0xee, 0x0d, 0xbb, 0xc4, 0x70, - 0x94, 0xa9, 0xfa, 0x49, 0x76, 0xa0, 0xf0, 0xd6, 0x19, 0x46, 0x0c, 0x43, 0x51, 0xd9, 0xb5, 0x92, - 0x37, 0xea, 0xb8, 0x2b, 0x0f, 0xfb, 0x3c, 0x90, 0xec, 0x9d, 0xa4, 0x1a, 0xf8, 0x34, 0xfb, 0xf3, - 0x8c, 0x65, 0x43, 0x01, 0xc7, 0x4f, 0xb6, 0xa0, 0xe2, 0x31, 0xe1, 0x86, 0xfe, 0x58, 0x8d, 0xcd, - 0x38, 0x4e, 0x8b, 0xc8, 0x87, 0x00, 0x22, 0x1a, 0x0c, 0x98, 0x40, 0x40, 0x16, 0x01, 0x29, 0x09, - 0xd9, 0x80, 0xc2, 0xd0, 0x39, 0x65, 0x71, 0xbc, 0xf5, 0xc3, 0xbd, 0xdf, 0x7c, 0x03, 0xd5, 0xa9, - 0x39, 0x93, 0x1a, 0x64, 0x7d, 0xcf, 0xbc, 0x20, 0xeb, 0xe3, 0x62, 0x8d, 0x43, 0x7e, 0xce, 0x5c, - 0x69, 0xfb, 0x9e, 0xf1, 0x5b, 0x36, 0x92, 0xae, 0x47, 0x1e, 0x43, 0x51, 0x48, 0x47, 0x46, 0x02, - 0xfd, 0xd6, 0x76, 0xef, 0x2c, 0x0e, 0xe5, 0x76, 0x0f, 0x41, 0xd4, 0x80, 0xc9, 0x43, 0x58, 0xd3, - 0xa4, 0x4d, 0xcf, 0x4a, 0x2f, 0x75, 0x03, 0x15, 0xcf, 0x52, 0x53, 0xdb, 0x80, 0x02, 0x0b, 0x43, - 0x1e, 0x36, 0x0b, 0x7a, 0xe8, 0xf8, 0x40, 0x9e, 0x00, 0xb8, 0x21, 0x73, 0x24, 0xf3, 0x6c, 0x47, - 0x36, 0x8b, 0x26, 0xac, 0x9a, 0xf5, 0xdb, 0x31, 0xeb, 0xb7, 0xfb, 0x31, 0xeb, 0x69, 0xd9, 0xa0, - 0xf7, 0x24, 0xd9, 0x86, 0xfc, 0x39, 0x3f, 0x15, 0xcd, 0x15, 0x5c, 0x7d, 0x6b, 0xc9, 0x90, 0x5f, - 0xf2, 0x53, 0x8a, 0x38, 0x15, 0x03, 0x3d, 0xda, 0xc0, 0x19, 0xb1, 0x66, 0x59, 0xc7, 0x00, 0x25, - 0x47, 0xce, 0x88, 0x91, 0x6f, 0xa1, 0x26, 0x5c, 0x16, 0x38, 0xa1, 0xcf, 0xed, 0x41, 0xc8, 0xa3, - 0x71, 0x13, 0x70, 0x34, 0x9b, 0x13, 0xc7, 0x46, 0xfd, 0x42, 0x69, 0x69, 0x55, 0xa4, 0x1f, 0xc9, - 0x63, 0x28, 0xb1, 0xc0, 0xd3, 0xd3, 0xa8, 0xbc, 0x77, 0x1a, 0x2b, 0x88, 0xdd, 0x93, 0xe4, 0x03, - 0x28, 0x9f, 0xf3, 0x53, 0xdb, 0xe5, 0x51, 0x20, 0x9b, 0xab, 0x98, 0x60, 0xa5, 0x73, 0x7e, 0xba, - 0xaf, 0x9e, 0xc9, 0x5d, 0x58, 0xd5, 0x49, 0x68, 0xf4, 0x55, 0xd4, 0x57, 0xb4, 0x2c, 0x81, 0xe8, - 0x7c, 0x34, 0x90, 0x9a, 0x86, 0x68, 0x99, 0x86, 0x7c, 0x06, 0xf5, 0x20, 0x1a, 0xa5, 0xea, 0x90, - 0x68, 0xd6, 0x11, 0x55, 0x0b, 0xa2, 0xd1, 0x24, 0x58, 0x82, 0xec, 0x42, 0x21, 0x12, 0xce, 0x80, - 0x35, 0x1b, 0x38, 0xfe, 0x9f, 0x2c, 0x89, 0xe8, 0x89, 0xc2, 0x50, 0x0d, 0x55, 0x94, 0x76, 0x79, - 0xe0, 0x46, 0x61, 0xc8, 0x02, 0xf7, 0xb2, 0xb9, 0xa6, 0x5f, 0x9f, 0x12, 0x91, 0x87, 0x90, 0x57, - 0x05, 0xa8, 0x49, 0x90, 0x59, 0xb7, 0x16, 0x38, 0x3d, 0xe4, 0x1e, 0xa3, 0x08, 0x22, 0x5f, 0xc3, - 0xca, 0x88, 0xc9, 0xd0, 0x77, 0x45, 0x73, 0x1d, 0x07, 0x71, 0x77, 0xc9, 0x20, 0x68, 0x14, 0x1c, - 0x6a, 0x20, 0x8d, 0x2d, 0x54, 0x2c, 0x84, 0x4e, 0x76, 0xfb, 0x97, 0x42, 0x7a, 0xcd, 0x8d, 0xad, - 0xcc, 0xfd, 0x55, 0x5a, 0x31, 0xb2, 0xef, 0x85, 0xf4, 0xac, 0xff, 0x2c, 0x40, 0xee, 0x25, 0x3f, - 0x9d, 0xcb, 0x8f, 0x27, 0x49, 0x02, 0x64, 0x71, 0x98, 0x77, 0x97, 0xb3, 0x69, 0x36, 0x09, 0xee, - 0xc1, 0xaa, 0x1f, 0x08, 0x19, 0x46, 0xae, 0x8e, 0xad, 0xce, 0xcc, 0x29, 0xd9, 0x84, 0xfb, 0xf9, - 0x34, 0xf7, 0xbf, 0x00, 0xa2, 0x09, 0xc9, 0xde, 0x8d, 0x99, 0x2b, 0xcd, 0xda, 0xe8, 0xf4, 0xd0, - 0x89, 0xd5, 0x4e, 0x29, 0x26, 0xb9, 0x5f, 0x4c, 0xe5, 0x3e, 0x21, 0x90, 0x97, 0xce, 0x40, 0x67, - 0x41, 0x99, 0xe2, 0x6f, 0x45, 0xaa, 0x90, 0xf3, 0x91, 0x26, 0x7a, 0x09, 0xd1, 0x25, 0x25, 0x40, - 0x9e, 0x3f, 0x01, 0x10, 0xd2, 0x09, 0x4d, 0xc6, 0xc1, 0xfb, 0x33, 0xce, 0xa0, 0xf7, 0xe4, 0x0f, - 0xe5, 0xf8, 0x2d, 0x58, 0xc1, 0xe1, 0xf8, 0x1e, 0x32, 0xbc, 0x4c, 0x8b, 0xea, 0xb1, 0xeb, 0x91, - 0xdf, 0x8f, 0x09, 0x57, 0x45, 0x67, 0x5b, 0x57, 0x04, 0x7d, 0x8a, 0x74, 0x29, 0x96, 0xd4, 0xae, - 0x64, 0xc9, 0x4b, 0x7e, 0x3a, 0xcb, 0x12, 0xeb, 0x4f, 0xa1, 0x80, 0xce, 0xc8, 0x7d, 0x68, 0xa8, - 0x4a, 0x6d, 0xcb, 0x28, 0x0c, 0x84, 0x49, 0x1f, 0xdd, 0xfa, 0x6a, 0x4a, 0xde, 0x57, 0x62, 0x9d, - 0x41, 0x0f, 0x60, 0xcd, 0x89, 0x3c, 0x9f, 0x4f, 0x41, 0x75, 0x37, 0xac, 0xa3, 0x62, 0x82, 0xbd, - 0xc7, 0xa1, 0xa8, 0x09, 0x42, 0x08, 0xd4, 0x7a, 0xfd, 0xbd, 0xfe, 0x49, 0xcf, 0x3e, 0x6e, 0x1f, - 0x3d, 0xeb, 0x1e, 0xbd, 0x68, 0xdc, 0x48, 0xc9, 0xe8, 0xc9, 0xd1, 0x91, 0x92, 0x65, 0xc8, 0x06, - 0x34, 0x8c, 0x6c, 0xff, 0xd5, 0xe1, 0xf1, 0x41, 0xbb, 0xdf, 0x7e, 0xd6, 0xc8, 0x92, 0x35, 0xa8, - 0x1a, 0xe9, 0xf3, 0xbd, 0xee, 0x41, 0xfb, 0x59, 0x23, 0x97, 0x06, 0xee, 0x1d, 0xed, 0xb7, 0x0f, - 0x94, 0x34, 0xff, 0x32, 0x5f, 0x2a, 0x37, 0xc0, 0xfa, 0xa7, 0x2d, 0x80, 0xc9, 0x6c, 0xc9, 0x03, - 0xa8, 0x39, 0xae, 0x1b, 0x85, 0x8e, 0x7b, 0x69, 0x0b, 0x97, 0x87, 0x0c, 0x67, 0x96, 0xed, 0xdc, - 0xa0, 0xd5, 0x58, 0xde, 0x53, 0xe2, 0x5f, 0x65, 0x32, 0x64, 0x1b, 0x1a, 0x8a, 0x80, 0xa1, 0xcf, - 0x02, 0x97, 0x19, 0x74, 0x16, 0xd1, 0x19, 0x5a, 0x9f, 0x68, 0x12, 0xfc, 0xe7, 0x50, 0x97, 0x8e, - 0x78, 0x63, 0xbb, 0x7c, 0x34, 0x1e, 0x32, 0xac, 0xf9, 0x39, 0x84, 0x67, 0x69, 0x4d, 0x29, 0xf6, - 0x13, 0xb9, 0x42, 0x7f, 0x05, 0x39, 0x21, 0x25, 0x12, 0xbf, 0xb2, 0xfb, 0xe9, 0x7b, 0xd7, 0x69, - 0xbb, 0xd7, 0xef, 0x53, 0x65, 0xa1, 0x0c, 0x87, 0xc3, 0x11, 0xa6, 0xc3, 0xb5, 0x0c, 0x0f, 0x0e, - 0x0e, 0xa9, 0xb2, 0x50, 0x86, 0x52, 0x0a, 0xd3, 0x4b, 0xae, 0x61, 0xd8, 0xef, 0xf7, 0xa8, 0xb2, - 0x20, 0x14, 0x56, 0x5d, 0x1e, 0xbc, 0x65, 0xa1, 0x40, 0x58, 0x73, 0x05, 0x3d, 0x6c, 0xbf, 0xdf, - 0xc3, 0x7e, 0xca, 0x8a, 0x4e, 0xf9, 0x20, 0x5d, 0x28, 0x9b, 0xc2, 0xcb, 0x43, 0x4c, 0xc5, 0xca, - 0xee, 0xc3, 0x6b, 0x04, 0x21, 0x36, 0xa1, 0x13, 0x6b, 0xf2, 0x0d, 0x14, 0x90, 0x7f, 0xcd, 0x32, - 0x36, 0xbc, 0x9f, 0x5e, 0x63, 0x66, 0x51, 0x18, 0x50, 0x6d, 0x44, 0x3e, 0x82, 0xca, 0x79, 0xe4, - 0x0d, 0x98, 0xde, 0x0d, 0x62, 0xde, 0x97, 0x29, 0xa0, 0x48, 0xd5, 0xdf, 0xe1, 0x84, 0xe4, 0x69, - 0x58, 0x05, 0x61, 0x9a, 0xe4, 0x2f, 0x27, 0xd8, 0xcf, 0x81, 0x9c, 0x39, 0xc2, 0x0e, 0xd9, 0x88, - 0x4b, 0x16, 0x6f, 0x59, 0x31, 0xb9, 0x4b, 0xb4, 0x71, 0xe6, 0x08, 0x8a, 0x8a, 0x9e, 0x96, 0x93, - 0x07, 0x90, 0x95, 0x3b, 0x26, 0xc7, 0xaf, 0x2a, 0x18, 0x59, 0xb9, 0x63, 0xfd, 0x7d, 0x1e, 0x72, - 0xbd, 0x7e, 0x9f, 0xdc, 0x84, 0xdc, 0x05, 0x0b, 0x13, 0xd6, 0xaa, 0x07, 0xc5, 0xa6, 0xdb, 0x50, - 0xb8, 0xe0, 0xa1, 0xa7, 0xcb, 0x74, 0xb5, 0x93, 0xa1, 0xfa, 0x51, 0xa9, 0x3e, 0x81, 0x8a, 0xfa, - 0x6d, 0x63, 0x6d, 0xd5, 0x65, 0xb8, 0xda, 0xc9, 0x52, 0x50, 0xc2, 0x36, 0xca, 0x14, 0xea, 0x26, - 0xe4, 0x5c, 0xa6, 0xeb, 0x70, 0xb6, 0x93, 0xa3, 0xea, 0xc1, 0xf8, 0x75, 0xcf, 0x9c, 0x50, 0xe0, - 0xb6, 0xa0, 0xda, 0xc9, 0x53, 0xfd, 0x68, 0xfc, 0xaa, 0xdf, 0xb1, 0x5f, 0x40, 0x40, 0x81, 0x82, - 0x12, 0x4e, 0xfc, 0x3e, 0x80, 0xda, 0x1b, 0x76, 0x29, 0x59, 0x38, 0xb2, 0x43, 0xe6, 0x3a, 0xc3, - 0x21, 0x12, 0x37, 0xdb, 0x29, 0xd2, 0xaa, 0x91, 0x53, 0x14, 0x9b, 0x84, 0x33, 0x32, 0x61, 0x47, - 0x52, 0xb2, 0x90, 0x79, 0xc8, 0xd6, 0x6a, 0x67, 0x85, 0xd6, 0x63, 0xcd, 0x89, 0x56, 0x28, 0xfc, - 0xcf, 0x60, 0x3d, 0xc1, 0x87, 0xcc, 0xe5, 0x83, 0xc0, 0xff, 0x25, 0xf3, 0x90, 0x9e, 0xd5, 0x4e, - 0x89, 0x92, 0x58, 0x49, 0x13, 0x9d, 0xb2, 0xfa, 0x16, 0x9a, 0x32, 0x74, 0x82, 0x64, 0x03, 0x66, - 0x0f, 0x1d, 0xa9, 0x1a, 0xb2, 0x3d, 0x12, 0x48, 0xc4, 0x6a, 0xa7, 0x4c, 0x37, 0xa7, 0x10, 0x07, - 0x1a, 0x70, 0xa8, 0x26, 0xd4, 0x2a, 0x42, 0xde, 0xbe, 0x60, 0x61, 0xab, 0x04, 0x45, 0x1b, 0x63, - 0xdc, 0xaa, 0xc1, 0xaa, 0x9d, 0x8a, 0x30, 0x22, 0x5c, 0x83, 0xc0, 0x68, 0x21, 0x22, 0x15, 0xab, - 0xd6, 0x1a, 0xd4, 0xed, 0xe9, 0xa8, 0xb4, 0xd6, 0x61, 0xcd, 0x9e, 0x9d, 0x7c, 0x6b, 0x13, 0x36, - 0xec, 0x05, 0x33, 0x6c, 0x7d, 0x00, 0xb7, 0xed, 0x65, 0x73, 0xb0, 0xfe, 0x3b, 0x03, 0xb9, 0x83, - 0x83, 0x43, 0xf2, 0x13, 0x58, 0x91, 0xf2, 0xb5, 0x54, 0xd3, 0xca, 0xe0, 0xb4, 0x6e, 0xd0, 0xa2, - 0x12, 0xe0, 0x34, 0x8c, 0x56, 0x28, 0x6d, 0x4c, 0x19, 0xa5, 0x15, 0x5a, 0xfb, 0x08, 0xd6, 0x24, - 0x7f, 0xc3, 0x02, 0x61, 0x8f, 0x59, 0x68, 0x0b, 0xe6, 0xf2, 0xc0, 0x4b, 0x8a, 0x59, 0x5d, 0xab, - 0x8e, 0x59, 0xd8, 0x43, 0x85, 0x32, 0xd8, 0x81, 0x35, 0xb5, 0xb7, 0xf1, 0x05, 0x0b, 0x98, 0x10, - 0xa6, 0x58, 0xc6, 0x64, 0x6a, 0xa4, 0x54, 0x71, 0xb5, 0x6c, 0x01, 0x94, 0x6c, 0x33, 0xbe, 0xf8, - 0xb7, 0x1a, 0x4d, 0x6b, 0x03, 0x88, 0x3d, 0xf7, 0x6e, 0x94, 0xce, 0xbd, 0xc0, 0xfa, 0x8b, 0x02, - 0xe4, 0xfa, 0xfd, 0x9e, 0x99, 0x8c, 0x33, 0x3b, 0x55, 0x27, 0x3d, 0xd5, 0xd3, 0xd9, 0xa9, 0x9e, - 0x1e, 0xc6, 0xc4, 0x57, 0x09, 0x15, 0x4f, 0x6e, 0x2e, 0xa1, 0x34, 0x9b, 0x72, 0xcb, 0x13, 0xaa, - 0x64, 0x32, 0x63, 0x71, 0x42, 0x95, 0xd1, 0x6f, 0x61, 0x3e, 0xa1, 0x74, 0xbe, 0x14, 0x97, 0x27, - 0x54, 0xc5, 0x30, 0x7f, 0x26, 0xa1, 0x1e, 0x42, 0x5d, 0x8c, 0x19, 0x73, 0xcf, 0xec, 0xd0, 0x91, - 0xcc, 0xbe, 0x18, 0x8f, 0x4c, 0x9c, 0x4b, 0xb4, 0xaa, 0x15, 0xd4, 0x91, 0xec, 0xbb, 0xf1, 0xc8, - 0xb8, 0x3c, 0x63, 0x4e, 0xe8, 0x69, 0xa6, 0x62, 0x21, 0x52, 0xf4, 0x06, 0x14, 0x7e, 0x17, 0x4f, - 0x68, 0x0b, 0xca, 0xc6, 0xe5, 0x48, 0x60, 0x39, 0xaa, 0x76, 0x80, 0x96, 0xb4, 0x48, 0x07, 0x69, - 0x07, 0xd6, 0x02, 0x47, 0x46, 0xa1, 0x33, 0x4c, 0x2d, 0xaf, 0x4e, 0xe4, 0x0a, 0x6d, 0xa4, 0x54, - 0x49, 0x33, 0xdc, 0x81, 0x35, 0x16, 0x44, 0x81, 0xeb, 0x63, 0xfd, 0x35, 0x16, 0x45, 0xb4, 0x58, - 0xa5, 0x8d, 0x94, 0x6a, 0x96, 0x10, 0x4e, 0x8a, 0x10, 0x6a, 0xcd, 0xfe, 0x4f, 0x12, 0x8e, 0x40, - 0xc3, 0x9e, 0x89, 0x1a, 0x62, 0x52, 0xc1, 0x69, 0xad, 0x02, 0xd8, 0x49, 0x18, 0x90, 0x70, 0x73, - 0x53, 0x46, 0xe9, 0xdc, 0xb4, 0xac, 0xff, 0x00, 0x58, 0x4d, 0xb7, 0x3a, 0x4c, 0x9f, 0x28, 0x0c, - 0x6c, 0xe9, 0xbc, 0xf1, 0x83, 0xc1, 0xcc, 0x46, 0xa3, 0xae, 0x54, 0x7d, 0xd4, 0x24, 0xd1, 0x7a, - 0x0a, 0xb7, 0x42, 0x26, 0xc6, 0x3c, 0x10, 0x2c, 0x49, 0xe5, 0xf1, 0xe3, 0x9d, 0x98, 0xb2, 0x85, - 0x4e, 0x86, 0x6e, 0xc4, 0x00, 0x53, 0x8d, 0x8e, 0x1f, 0xef, 0xe8, 0xb5, 0x59, 0x68, 0xfb, 0xe4, - 0xb1, 0xb2, 0xcd, 0xa1, 0x6d, 0x76, 0xde, 0xf6, 0xc9, 0xe3, 0xab, 0x6c, 0x9f, 0x28, 0xdb, 0x3c, - 0xda, 0xe6, 0x16, 0xd8, 0x3e, 0xd1, 0xb6, 0x5f, 0xc2, 0xfa, 0x9c, 0xed, 0x48, 0xe0, 0x37, 0x52, - 0xa1, 0x93, 0xa7, 0x6b, 0x33, 0x76, 0xda, 0xe8, 0xe7, 0xb0, 0xa9, 0xb7, 0xf6, 0x97, 0x3e, 0x1b, - 0x7a, 0x69, 0xbb, 0x82, 0xe9, 0x1f, 0xeb, 0xa8, 0xff, 0x85, 0x52, 0x4f, 0x59, 0x7e, 0x0d, 0xb7, - 0x18, 0x97, 0xf6, 0xc8, 0x17, 0xe3, 0x90, 0x79, 0x3e, 0x7e, 0x40, 0x98, 0x1d, 0x67, 0xd1, 0xa4, - 0xd2, 0x4d, 0xc6, 0xe5, 0x61, 0x5a, 0x8f, 0x3b, 0x4f, 0x65, 0x7c, 0x1f, 0xaa, 0xfc, 0x2d, 0x0b, - 0x87, 0xce, 0x58, 0xad, 0xbf, 0xcf, 0x31, 0xab, 0xb3, 0x9d, 0x15, 0xba, 0x6a, 0xc4, 0x54, 0x49, - 0x4d, 0xe5, 0x8b, 0x91, 0x93, 0x9c, 0x28, 0x99, 0x8e, 0x52, 0x37, 0xaa, 0x5e, 0x2a, 0x35, 0x1e, - 0x42, 0x5d, 0x72, 0xe9, 0x0c, 0x53, 0xf0, 0xb2, 0x49, 0xb3, 0x2a, 0x2a, 0xd2, 0xe0, 0x2f, 0xa0, - 0x21, 0xfc, 0x21, 0xee, 0x27, 0xb5, 0xd1, 0x28, 0x2e, 0x04, 0x40, 0x6b, 0x46, 0xd3, 0x57, 0x0a, - 0x0d, 0xff, 0x0a, 0x6e, 0x3a, 0x17, 0x6f, 0x2e, 0x14, 0x47, 0x63, 0x33, 0x3d, 0x63, 0x5d, 0x1b, - 0x2a, 0x74, 0xdd, 0xa8, 0x7b, 0x5a, 0x9b, 0xcc, 0xf7, 0x5b, 0x68, 0x46, 0x81, 0x13, 0x88, 0x0b, - 0xd5, 0x46, 0x54, 0x1d, 0x15, 0x3c, 0x70, 0xf4, 0x2e, 0xdd, 0x14, 0x81, 0x55, 0xba, 0x39, 0x41, - 0x1c, 0x6b, 0x00, 0xee, 0xd6, 0x8d, 0xf9, 0x6b, 0x67, 0x28, 0x98, 0xed, 0x07, 0x92, 0x85, 0x61, - 0x34, 0x4e, 0x05, 0x5b, 0xd7, 0x87, 0x2a, 0xdd, 0x44, 0x44, 0x37, 0x05, 0x48, 0xde, 0xde, 0x87, - 0x4f, 0x16, 0x98, 0x47, 0x81, 0xea, 0x60, 0x6f, 0x71, 0x44, 0x93, 0x6f, 0xf2, 0x6a, 0xa7, 0x46, - 0xef, 0xce, 0xb9, 0x3a, 0x99, 0x60, 0x13, 0xaf, 0x6d, 0xb8, 0xa3, 0xa9, 0x13, 0xb2, 0x31, 0xc7, - 0xcf, 0x34, 0xb6, 0x3b, 0xc5, 0xbc, 0x3a, 0xba, 0xab, 0xd3, 0xdb, 0x08, 0xa3, 0x06, 0xd5, 0xde, - 0x9d, 0x62, 0xa0, 0xee, 0x2f, 0xb3, 0xc9, 0xd9, 0xb2, 0xa0, 0x69, 0x2f, 0xc9, 0xc0, 0x25, 0x3a, - 0xcc, 0xb0, 0x65, 0x3a, 0x95, 0x41, 0xd8, 0xbd, 0x17, 0x64, 0x48, 0xeb, 0x36, 0xdc, 0xb2, 0x17, - 0x27, 0x01, 0xba, 0x5b, 0xc2, 0xf2, 0x56, 0x03, 0x6a, 0xf6, 0x14, 0x89, 0x71, 0x2a, 0x73, 0x64, - 0xc5, 0x5a, 0x37, 0xc3, 0x48, 0xdc, 0x5d, 0xcc, 0x12, 0xaf, 0xd5, 0x84, 0x4d, 0x7b, 0x21, 0xbd, - 0x70, 0x7f, 0xb1, 0x8c, 0x3f, 0xa8, 0x5c, 0xc6, 0x8e, 0xd6, 0x67, 0xf0, 0xa9, 0x7d, 0x9d, 0xb5, - 0x6f, 0x6d, 0xc1, 0x87, 0xf6, 0x95, 0xcb, 0x69, 0xfd, 0x75, 0x06, 0xca, 0xc9, 0x7e, 0x1f, 0xfb, - 0x89, 0x13, 0x0e, 0x2f, 0x6d, 0xb5, 0x19, 0xf2, 0x03, 0x27, 0x39, 0x28, 0x2c, 0x75, 0x6e, 0xd0, - 0x06, 0xaa, 0xfa, 0x13, 0x8d, 0xd9, 0x4d, 0x2a, 0x6f, 0x53, 0x06, 0x59, 0x34, 0xc8, 0xd0, 0xba, - 0xd2, 0x4c, 0xe3, 0x75, 0x6d, 0x9f, 0x7d, 0x05, 0x46, 0x6e, 0xd6, 0x8d, 0xf5, 0x37, 0x39, 0xc8, - 0xab, 0x64, 0x21, 0x1b, 0x50, 0xf0, 0x03, 0x8f, 0xbd, 0xd3, 0xdb, 0x0e, 0xaa, 0x1f, 0xc8, 0x43, - 0xc8, 0x87, 0x7c, 0xc8, 0xcc, 0x91, 0xc9, 0xad, 0x05, 0x87, 0xa1, 0x94, 0x0f, 0x19, 0x45, 0x10, - 0xf9, 0x10, 0x4a, 0x78, 0x90, 0x10, 0xd7, 0x6b, 0xb5, 0x79, 0x59, 0x41, 0x89, 0xae, 0x01, 0x16, - 0x14, 0x59, 0xe0, 0xc5, 0x15, 0x19, 0xb7, 0xf6, 0x2c, 0xf0, 0xe2, 0xfa, 0x80, 0xdf, 0x90, 0x85, - 0x1f, 0xfa, 0x0d, 0x59, 0xfc, 0xa1, 0xdf, 0x90, 0x2b, 0xbf, 0xf3, 0x37, 0x64, 0xe9, 0x77, 0xff, - 0x86, 0x6c, 0x55, 0xa0, 0x6c, 0xc7, 0xb1, 0x6b, 0x95, 0x61, 0xc5, 0xd6, 0x81, 0xc2, 0xed, 0xf5, - 0xf4, 0x57, 0x3e, 0x2e, 0xe3, 0xec, 0xc7, 0xbc, 0xce, 0x94, 0xe9, 0x2f, 0x76, 0xeb, 0xdf, 0xf2, - 0x00, 0x93, 0x33, 0xb4, 0x1f, 0xf5, 0xbc, 0xe0, 0xf7, 0x80, 0x24, 0x07, 0xab, 0x63, 0x47, 0x08, - 0xdc, 0x9a, 0x24, 0x1b, 0xd1, 0x46, 0xac, 0x3b, 0x76, 0x84, 0x50, 0x5b, 0xba, 0xff, 0x3f, 0x34, - 0x20, 0x77, 0x00, 0xce, 0xf9, 0xa9, 0xd0, 0x25, 0x4c, 0xb7, 0x59, 0x5a, 0x56, 0x12, 0xec, 0x99, - 0xe4, 0x63, 0xa8, 0xa2, 0x7a, 0xc4, 0x1c, 0x11, 0xa9, 0x8f, 0x47, 0x6c, 0xad, 0x74, 0x55, 0x09, - 0x0f, 0x8d, 0x8c, 0xec, 0xc0, 0x06, 0x82, 0x92, 0xf3, 0x03, 0xfb, 0xb5, 0x13, 0x0d, 0x4d, 0x4b, - 0xa5, 0x44, 0xe9, 0x92, 0xaa, 0xf3, 0x5c, 0x69, 0xae, 0x4d, 0xa7, 0x9b, 0xb0, 0x6e, 0xcf, 0x2f, - 0xa8, 0xf5, 0x9b, 0x1c, 0x14, 0xf7, 0xf1, 0x70, 0xde, 0xfa, 0xaf, 0x2c, 0xac, 0x50, 0xf6, 0x67, - 0x11, 0x13, 0x72, 0xe6, 0xe6, 0x21, 0x33, 0x7b, 0xf3, 0x30, 0x7d, 0x28, 0x9f, 0x9d, 0x3d, 0x94, - 0x5f, 0x70, 0x76, 0x9d, 0x5f, 0x78, 0x76, 0xbd, 0x09, 0xc5, 0x90, 0x0d, 0xd4, 0x02, 0x14, 0xcd, - 0x11, 0x23, 0x3e, 0x91, 0xd6, 0xdc, 0xa9, 0xfe, 0xca, 0x55, 0xa7, 0xfa, 0x8a, 0xf0, 0x53, 0xe7, - 0xfa, 0x8a, 0x8d, 0x9f, 0x4e, 0x9f, 0x71, 0x97, 0xcc, 0x4e, 0x35, 0x2d, 0xd4, 0x3b, 0x24, 0x7d, - 0xd0, 0x5d, 0xbe, 0xc6, 0x41, 0x37, 0x06, 0x7b, 0x7a, 0x60, 0x7a, 0x33, 0x3f, 0x71, 0xf9, 0x32, - 0x5f, 0xca, 0x35, 0xf2, 0x74, 0xfe, 0x86, 0xc5, 0xfa, 0xcb, 0x0c, 0x94, 0xa8, 0x69, 0xb8, 0xe4, - 0x01, 0xac, 0x4d, 0xa2, 0x63, 0x87, 0x51, 0x30, 0x09, 0x73, 0x5d, 0xa4, 0xb9, 0xd7, 0xf5, 0x48, - 0x0f, 0x36, 0xc7, 0x21, 0x13, 0xfe, 0x20, 0x50, 0xdd, 0x8e, 0x0b, 0xd5, 0x94, 0x70, 0x95, 0xcc, - 0x7d, 0xd6, 0xe4, 0xda, 0xe7, 0x38, 0x86, 0x1d, 0x73, 0x21, 0xcd, 0x52, 0xd2, 0x8d, 0xf1, 0x02, - 0xa9, 0xf5, 0x77, 0x19, 0x58, 0xdf, 0xe7, 0xc1, 0x6b, 0x3f, 0x1c, 0xf5, 0x78, 0x14, 0xba, 0xec, - 0x64, 0x3c, 0xe4, 0x8e, 0x67, 0xfd, 0xf9, 0xb5, 0x39, 0xb0, 0x70, 0x0a, 0xd9, 0xc5, 0x53, 0xf8, - 0x0c, 0xea, 0x2e, 0xf7, 0x98, 0xcd, 0x02, 0x19, 0x5e, 0x8e, 0xb9, 0x1f, 0x48, 0x73, 0xe0, 0x5e, - 0x53, 0xe2, 0x76, 0x22, 0xb5, 0x60, 0x12, 0x23, 0xeb, 0xaf, 0x32, 0x90, 0x7b, 0xc1, 0xa4, 0xd5, - 0xff, 0x31, 0x86, 0x64, 0xfd, 0x2c, 0xb5, 0x1a, 0xf7, 0x21, 0x17, 0x46, 0xba, 0x61, 0x4f, 0x71, - 0x2c, 0x6d, 0x42, 0x15, 0xc4, 0xfa, 0xd7, 0x2c, 0xe4, 0x0f, 0x7c, 0x21, 0xad, 0x7f, 0xc9, 0x5c, - 0x7b, 0x54, 0x4f, 0x67, 0x6e, 0x29, 0xae, 0xbe, 0xa6, 0xeb, 0xdc, 0x88, 0xef, 0x28, 0x14, 0x3b, - 0xbf, 0x01, 0x18, 0x3b, 0x03, 0xa6, 0xcf, 0x1c, 0x30, 0x66, 0x95, 0xdd, 0x66, 0x62, 0xdf, 0x57, - 0xd2, 0x63, 0x67, 0x60, 0x3a, 0x7f, 0x27, 0x43, 0xcb, 0x0a, 0x8d, 0x62, 0xb5, 0x69, 0x50, 0x5d, - 0x47, 0xfb, 0x6a, 0x55, 0xa1, 0x62, 0x4f, 0x3c, 0x59, 0xef, 0xa6, 0xb8, 0x98, 0x0f, 0xa3, 0x40, - 0x34, 0x33, 0x78, 0x40, 0xb9, 0x6c, 0xfa, 0x88, 0x21, 0x7f, 0x00, 0xf5, 0x80, 0xbd, 0x93, 0x29, - 0x57, 0x86, 0x84, 0x4b, 0x07, 0x45, 0xab, 0xca, 0xe0, 0x38, 0x1e, 0x96, 0x75, 0x0e, 0xc5, 0x7d, - 0x27, 0x70, 0xd9, 0xf0, 0x47, 0x5a, 0xd7, 0x34, 0x83, 0x7e, 0xe4, 0x4b, 0x83, 0x5f, 0x67, 0x92, - 0x5b, 0x83, 0xdb, 0x70, 0x73, 0xfa, 0xd6, 0xc0, 0x3e, 0x39, 0x3e, 0x78, 0xb5, 0xf7, 0xac, 0x71, - 0x83, 0xdc, 0x84, 0x35, 0xa3, 0x7a, 0xd1, 0x3e, 0x6a, 0xd3, 0xbd, 0xbe, 0xbe, 0x3f, 0x98, 0xbf, - 0x53, 0xc8, 0x92, 0x4d, 0x20, 0x46, 0xd6, 0x3b, 0x39, 0x3c, 0xdc, 0xa3, 0xdd, 0xef, 0x95, 0x3c, - 0xb7, 0xf0, 0xae, 0x21, 0x3f, 0x7f, 0xd7, 0x50, 0x58, 0x78, 0xd7, 0x50, 0x7c, 0x99, 0x2f, 0x95, - 0x1a, 0x65, 0xba, 0x62, 0x6e, 0xd4, 0xee, 0xfd, 0x3a, 0x07, 0xa5, 0xb8, 0x7c, 0x4e, 0xae, 0xa7, - 0x32, 0xe9, 0xeb, 0xa9, 0xd9, 0xdb, 0xb1, 0xec, 0x82, 0xdb, 0xb1, 0xc5, 0xf7, 0x60, 0xb9, 0x65, - 0xf7, 0x60, 0x8f, 0xcc, 0x8d, 0x57, 0x1e, 0x59, 0xf6, 0xc1, 0x5c, 0x21, 0xdf, 0xee, 0x3b, 0x03, - 0xa1, 0xaf, 0xf8, 0xf5, 0x75, 0x98, 0x05, 0xa5, 0x48, 0xb0, 0xd0, 0x73, 0xa4, 0x63, 0x6e, 0xd7, - 0x92, 0x67, 0xeb, 0x1f, 0x32, 0xb0, 0xa6, 0xbb, 0xd6, 0xf3, 0x90, 0x8f, 0xcc, 0x89, 0xb5, 0xf5, - 0x8b, 0x6b, 0x13, 0x2a, 0x75, 0xb7, 0x95, 0x9d, 0xba, 0xdb, 0x9a, 0x34, 0xa4, 0x5c, 0xba, 0x21, - 0x59, 0x4f, 0x52, 0xf9, 0xf2, 0x05, 0x94, 0xe2, 0x1e, 0x60, 0x4a, 0xc6, 0xda, 0xdc, 0x6c, 0x68, - 0x02, 0xb1, 0xbe, 0x82, 0x72, 0x32, 0xb5, 0x05, 0x7f, 0x4e, 0xd8, 0x48, 0xff, 0x39, 0xa1, 0x9c, - 0xfa, 0x03, 0xc2, 0xbd, 0x3e, 0x54, 0xa7, 0xba, 0x1c, 0x21, 0x90, 0xc7, 0x7e, 0xab, 0xad, 0xf1, - 0x37, 0x79, 0x04, 0xe5, 0xf8, 0x4d, 0x6a, 0x99, 0x72, 0x8b, 0x47, 0x33, 0xc1, 0xdc, 0xfb, 0xe7, - 0x0c, 0x90, 0x49, 0x66, 0x3f, 0xf3, 0xc5, 0xd8, 0x91, 0xee, 0xd9, 0x6f, 0xd5, 0x90, 0x6e, 0x42, - 0xf1, 0x9c, 0x9f, 0x4e, 0x82, 0x57, 0x38, 0xe7, 0xa7, 0x5d, 0x6f, 0x2a, 0x2e, 0xb9, 0xf7, 0xc6, - 0x25, 0x69, 0xbc, 0xf9, 0x6b, 0x34, 0xde, 0x07, 0xaf, 0xa1, 0x36, 0x2d, 0x27, 0x1f, 0xc1, 0x07, - 0xbd, 0xee, 0xe1, 0xc9, 0xc1, 0x5e, 0xbf, 0xfb, 0xea, 0xc8, 0x3e, 0x7c, 0xf5, 0xac, 0x6d, 0x9f, - 0x1c, 0xf5, 0x8e, 0xdb, 0xfb, 0xdd, 0xe7, 0xdd, 0xb6, 0xca, 0xbb, 0x26, 0x6c, 0xcc, 0x02, 0xfa, - 0xed, 0x3f, 0xee, 0x37, 0x32, 0x98, 0xac, 0x33, 0x9a, 0xbd, 0x93, 0x67, 0xdd, 0x57, 0x8d, 0xec, - 0xee, 0xdf, 0x16, 0xa0, 0xbe, 0xa7, 0xb8, 0x3b, 0x79, 0x1b, 0xf1, 0x60, 0x5d, 0x73, 0x6d, 0xfa, - 0xcf, 0x1a, 0xcb, 0xb6, 0x9b, 0x1a, 0xbb, 0x1d, 0x37, 0xda, 0x9f, 0xbe, 0x0f, 0x66, 0x58, 0xf5, - 0xab, 0x0c, 0xdc, 0x89, 0x1b, 0x72, 0x02, 0x4c, 0xb7, 0x66, 0xb2, 0xbb, 0xcc, 0xd3, 0x7c, 0x1b, - 0x4f, 0xde, 0xfe, 0xe5, 0x6f, 0x65, 0x63, 0x86, 0xf2, 0x27, 0xd0, 0x78, 0xc1, 0xe4, 0xf4, 0x6c, - 0xef, 0x2d, 0x71, 0xf4, 0x82, 0xc9, 0xe4, 0x65, 0x1f, 0x5f, 0x89, 0x31, 0xce, 0x6d, 0x20, 0xaa, - 0x81, 0x4e, 0x21, 0x04, 0x59, 0x66, 0xaa, 0xa0, 0x89, 0xff, 0x4f, 0xae, 0x06, 0x99, 0x17, 0xa8, - 0xe5, 0xc2, 0x06, 0x73, 0xcd, 0xe5, 0x42, 0xec, 0xfb, 0x97, 0x2b, 0x86, 0x99, 0xb7, 0xbc, 0x85, - 0xdb, 0x86, 0x14, 0x86, 0xcf, 0xa9, 0x42, 0x44, 0x1e, 0xce, 0x57, 0xb7, 0xb9, 0x6a, 0x95, 0xbc, - 0xf1, 0xf3, 0xeb, 0x81, 0xf5, 0x7b, 0x5b, 0xcf, 0xbf, 0xff, 0x78, 0xe0, 0xcb, 0xb3, 0xe8, 0x74, - 0xdb, 0xe5, 0xa3, 0xf8, 0x4f, 0x68, 0xfa, 0x8f, 0x66, 0x2e, 0x1f, 0xc6, 0x82, 0x7f, 0xcc, 0x56, - 0x0f, 0xfc, 0xb7, 0xec, 0x0f, 0x71, 0x4f, 0xc8, 0x25, 0xff, 0x9f, 0x6c, 0xcd, 0x3c, 0x3f, 0x7d, - 0x8a, 0x82, 0xd3, 0x22, 0x9a, 0x7c, 0xf9, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x59, 0x32, 0xc0, - 0x0f, 0xf7, 0x26, 0x00, 0x00, + // 3439 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0xcb, 0x6f, 0x1b, 0x49, + 0x73, 0x37, 0xdf, 0x64, 0x51, 0x7c, 0xa8, 0xf5, 0x30, 0x3d, 0xde, 0x87, 0x2d, 0xef, 0xc3, 0xb1, + 0xbd, 0xb4, 0xa3, 0x5d, 0xef, 0x46, 0xde, 0x07, 0x22, 0x4a, 0xb4, 0x49, 0x47, 0x92, 0xb5, 0x4d, + 0x0a, 0xc9, 0x3a, 0x08, 0x06, 0xa3, 0x99, 0x36, 0x35, 0xf2, 0x70, 0x9a, 0x3b, 0xdd, 0x63, 0x5b, + 0x0b, 0xe4, 0x90, 0x9b, 0x81, 0x1c, 0x82, 0xdc, 0x72, 0x0a, 0x92, 0xc3, 0x1e, 0xb2, 0xd7, 0x5c, + 0x03, 0xe4, 0x6f, 0xc8, 0x25, 0x40, 0x80, 0xfc, 0x05, 0xdf, 0xf5, 0xc3, 0xde, 0x3f, 0xf4, 0x63, + 0x86, 0xc3, 0x97, 0x2c, 0x7f, 0x58, 0xdf, 0xbe, 0x1b, 0xa7, 0xea, 0x57, 0x3d, 0xdd, 0xd5, 0xbf, + 0xaa, 0xea, 0xe9, 0x22, 0x7c, 0xe0, 0xb9, 0x2f, 0xc8, 0x73, 0x97, 0x9b, 0xd6, 0x80, 0xf8, 0xdc, + 0x64, 0xee, 0x30, 0xf4, 0x2c, 0xee, 0x52, 0xbf, 0x39, 0x0a, 0x28, 0xa7, 0xa8, 0xa0, 0xf5, 0xc6, + 0x75, 0x09, 0xb8, 0x3b, 0x05, 0x27, 0x8c, 0xc5, 0x58, 0xe3, 0xc3, 0x01, 0xa5, 0x03, 0x8f, 0xdc, + 0x95, 0x4f, 0xc7, 0xe1, 0xb3, 0xbb, 0xdc, 0x1d, 0x12, 0xc6, 0xad, 0xe1, 0x48, 0x03, 0xae, 0x44, + 0xd6, 0xb6, 0x47, 0x43, 0x47, 0x8d, 0xa1, 0x55, 0xab, 0x91, 0x6a, 0x48, 0x1d, 0xe2, 0x31, 0x25, + 0xdd, 0xf8, 0x9f, 0x0c, 0xac, 0xf6, 0xe2, 0x29, 0xe1, 0xd0, 0xef, 0x85, 0xc3, 0xa1, 0x15, 0x9c, + 0xa1, 0x75, 0xc8, 0x8f, 0x2c, 0xc6, 0x88, 0xd3, 0x48, 0x5d, 0x4b, 0xdd, 0xcc, 0x61, 0xfd, 0x24, + 0xe4, 0xcf, 0x2c, 0xd7, 0x23, 0x4e, 0x23, 0xad, 0xe4, 0xea, 0x09, 0xbd, 0x0f, 0x30, 0xa0, 0xae, + 0x3f, 0x30, 0x5f, 0x12, 0xcf, 0x6b, 0x64, 0xae, 0xa5, 0x6e, 0x96, 0x70, 0x49, 0x4a, 0xfe, 0x9a, + 0x78, 0x9e, 0x50, 0x73, 0x6a, 0xba, 0xc3, 0x51, 0x40, 0x5f, 0x90, 0x46, 0x56, 0xa9, 0x39, 0xed, + 0x2a, 0x01, 0xfa, 0x1a, 0xf2, 0x2e, 0x63, 0x21, 0x61, 0x8d, 0xdc, 0xb5, 0xcc, 0xcd, 0xf2, 0xe6, + 0x8d, 0xa6, 0x9e, 0x6d, 0x73, 0xde, 0xe4, 0x9a, 0x5d, 0x81, 0xc5, 0xda, 0x04, 0x7d, 0x0f, 0x4b, + 0xf6, 0x89, 0xc5, 0xcd, 0x13, 0x97, 0x71, 0x1a, 0x9c, 0x35, 0xf2, 0x72, 0x88, 0xe6, 0xf9, 0x43, + 0xec, 0x9c, 0x58, 0xbc, 0xa3, 0x0c, 0xda, 0x3e, 0x0f, 0xce, 0x70, 0xd9, 0x1e, 0x4b, 0x8c, 0xa7, + 0x50, 0x9f, 0x06, 0xa0, 0x3a, 0x64, 0x9e, 0x93, 0x33, 0xe9, 0x8e, 0x12, 0x16, 0x3f, 0xd1, 0x3d, + 0xc8, 0xbd, 0xb0, 0xbc, 0x90, 0x48, 0x57, 0x94, 0x37, 0x8d, 0xf8, 0x8d, 0xca, 0xef, 0x62, 0x84, + 0x1d, 0xea, 0x73, 0xf2, 0x8a, 0x63, 0x05, 0x7c, 0x90, 0xfe, 0x8b, 0x94, 0x61, 0x42, 0x4e, 0xce, + 0x1f, 0x5d, 0x83, 0xb2, 0x43, 0x98, 0x1d, 0xb8, 0x23, 0x31, 0x37, 0x3d, 0x70, 0x52, 0x84, 0x3e, + 0x00, 0x60, 0xe1, 0x60, 0x40, 0x98, 0x04, 0xa4, 0x25, 0x20, 0x21, 0x41, 0xab, 0x90, 0xf3, 0xac, + 0x63, 0x12, 0xf9, 0x5b, 0x3d, 0x6c, 0xfc, 0xfe, 0x3b, 0xa8, 0x4c, 0xac, 0x19, 0x55, 0x21, 0xed, + 0x3a, 0xfa, 0x05, 0x69, 0x57, 0x6e, 0xd6, 0x28, 0xa0, 0xa7, 0xc4, 0xe6, 0xa6, 0xeb, 0xe8, 0x71, + 0x4b, 0x5a, 0xd2, 0x75, 0xd0, 0x7d, 0xc8, 0x33, 0x6e, 0xf1, 0x90, 0xc9, 0x71, 0xab, 0x9b, 0xef, + 0xcf, 0x77, 0x65, 0xb3, 0x27, 0x41, 0x58, 0x83, 0xd1, 0x6d, 0x58, 0x56, 0xa4, 0x4d, 0xae, 0x4a, + 0x6d, 0x75, 0x5d, 0x2a, 0x76, 0x13, 0x4b, 0x5b, 0x85, 0x1c, 0x09, 0x02, 0x1a, 0x34, 0x72, 0x6a, + 0xea, 0xf2, 0x01, 0x6d, 0x01, 0xd8, 0x01, 0xb1, 0x38, 0x71, 0x4c, 0x8b, 0x37, 0xf2, 0xda, 0xad, + 0x8a, 0xf5, 0xcd, 0x88, 0xf5, 0xcd, 0x7e, 0xc4, 0x7a, 0x5c, 0xd2, 0xe8, 0x6d, 0x8e, 0x9a, 0x90, + 0x3d, 0xa5, 0xc7, 0xac, 0x51, 0x90, 0xbb, 0x6f, 0x2c, 0x98, 0xf2, 0x63, 0x7a, 0x8c, 0x25, 0x4e, + 0xf8, 0x40, 0xcd, 0xd6, 0xb7, 0x86, 0xa4, 0x51, 0x52, 0x3e, 0x90, 0x92, 0x03, 0x6b, 0x48, 0xd0, + 0xb7, 0x50, 0x65, 0x36, 0xf1, 0xad, 0xc0, 0xa5, 0xe6, 0x20, 0xa0, 0xe1, 0xa8, 0x01, 0x72, 0x36, + 0xeb, 0xe3, 0x81, 0xb5, 0xfa, 0x91, 0xd0, 0xe2, 0x0a, 0x4b, 0x3e, 0xa2, 0xfb, 0x50, 0x24, 0xbe, + 0xa3, 0x96, 0x51, 0x7e, 0xe3, 0x32, 0x0a, 0x12, 0xbb, 0xcd, 0xd1, 0x55, 0x28, 0x9d, 0xd2, 0x63, + 0xd3, 0xa6, 0xa1, 0xcf, 0x1b, 0x4b, 0x32, 0xc0, 0x8a, 0xa7, 0xf4, 0x78, 0x47, 0x3c, 0xa3, 0xeb, + 0xb0, 0xa4, 0x82, 0x50, 0xeb, 0x2b, 0x52, 0x5f, 0x56, 0xb2, 0x18, 0xa2, 0xe2, 0x51, 0x43, 0xaa, + 0x0a, 0xa2, 0x64, 0x0a, 0xf2, 0x29, 0xd4, 0xfc, 0x70, 0x98, 0xc8, 0x43, 0xac, 0x51, 0x93, 0xa8, + 0xaa, 0x1f, 0x0e, 0xc7, 0xce, 0x62, 0x68, 0x13, 0x72, 0x21, 0xb3, 0x06, 0xa4, 0x51, 0x97, 0xf3, + 0x7f, 0x6f, 0x81, 0x47, 0x8f, 0x04, 0x06, 0x2b, 0xa8, 0xa0, 0xb4, 0x4d, 0x7d, 0x3b, 0x0c, 0x02, + 0xe2, 0xdb, 0x67, 0x8d, 0x65, 0xf5, 0xfa, 0x84, 0x08, 0xdd, 0x86, 0xac, 0x48, 0x40, 0x0d, 0x24, + 0x99, 0x75, 0x79, 0xce, 0xa0, 0xfb, 0xd4, 0x21, 0x58, 0x82, 0xd0, 0xd7, 0x50, 0x18, 0x12, 0x1e, + 0xb8, 0x36, 0x6b, 0xac, 0xc8, 0x49, 0x5c, 0x5f, 0x30, 0x09, 0x1c, 0xfa, 0xfb, 0x0a, 0x88, 0x23, + 0x0b, 0xe1, 0x0b, 0xa6, 0x82, 0xdd, 0xfc, 0x89, 0x71, 0xa7, 0xb1, 0x7a, 0x2d, 0x75, 0x73, 0x09, + 0x97, 0xb5, 0xec, 0x29, 0xe3, 0x8e, 0xf1, 0xbf, 0x39, 0xc8, 0x3c, 0xa6, 0xc7, 0x33, 0xf1, 0xb1, + 0x15, 0x07, 0x40, 0x5a, 0x4e, 0xf3, 0xfa, 0x62, 0x36, 0x4d, 0x07, 0xc1, 0x06, 0x2c, 0xb9, 0x3e, + 0xe3, 0x41, 0x68, 0x2b, 0xdf, 0xaa, 0xc8, 0x9c, 0x90, 0x8d, 0xb9, 0x9f, 0x4d, 0x72, 0xff, 0x33, + 0x40, 0x8a, 0x90, 0xe4, 0xd5, 0x88, 0xd8, 0x5c, 0xef, 0x8d, 0x0a, 0x0f, 0x15, 0x58, 0xed, 0x84, + 0x62, 0x1c, 0xfb, 0xf9, 0x44, 0xec, 0x23, 0x04, 0x59, 0x6e, 0x0d, 0x54, 0x14, 0x94, 0xb0, 0xfc, + 0x2d, 0x48, 0x15, 0x50, 0x3a, 0x54, 0x44, 0x2f, 0x4a, 0x74, 0x51, 0x08, 0x24, 0xcf, 0xb7, 0x00, + 0x18, 0xb7, 0x02, 0x1d, 0x71, 0xf0, 0xe6, 0x88, 0xd3, 0xe8, 0x6d, 0xfe, 0xc7, 0x72, 0xfc, 0x32, + 0x14, 0xe4, 0x74, 0x5c, 0x47, 0x32, 0xbc, 0x84, 0xf3, 0xe2, 0xb1, 0xeb, 0xa0, 0x2f, 0x23, 0xc2, + 0x55, 0xe4, 0x60, 0xd7, 0xce, 0x71, 0xfa, 0x04, 0xe9, 0x12, 0x2c, 0xa9, 0x9e, 0xcb, 0x92, 0xc7, + 0xf4, 0x78, 0x9a, 0x25, 0xc6, 0xdf, 0x41, 0x4e, 0x0e, 0x86, 0x6e, 0x42, 0x5d, 0x64, 0x6a, 0x93, + 0x87, 0x81, 0xcf, 0x74, 0xf8, 0xa8, 0xd2, 0x57, 0x15, 0xf2, 0xbe, 0x10, 0xab, 0x08, 0xba, 0x05, + 0xcb, 0x56, 0xe8, 0xb8, 0x74, 0x02, 0xaa, 0xaa, 0x61, 0x4d, 0x2a, 0xc6, 0xd8, 0x0d, 0x0a, 0x79, + 0x45, 0x10, 0x84, 0xa0, 0xda, 0xeb, 0x6f, 0xf7, 0x8f, 0x7a, 0xe6, 0x61, 0xfb, 0x60, 0xb7, 0x7b, + 0xf0, 0xa8, 0x7e, 0x29, 0x21, 0xc3, 0x47, 0x07, 0x07, 0x42, 0x96, 0x42, 0xab, 0x50, 0xd7, 0xb2, + 0x9d, 0x27, 0xfb, 0x87, 0x7b, 0xed, 0x7e, 0x7b, 0xb7, 0x9e, 0x46, 0xcb, 0x50, 0xd1, 0xd2, 0x87, + 0xdb, 0xdd, 0xbd, 0xf6, 0x6e, 0x3d, 0x93, 0x04, 0x6e, 0x1f, 0xec, 0xb4, 0xf7, 0x84, 0x34, 0xfb, + 0x38, 0x5b, 0x2c, 0xd5, 0xc1, 0xf8, 0xf7, 0x1b, 0x00, 0xe3, 0xd5, 0xa2, 0x5b, 0x50, 0xb5, 0x6c, + 0x3b, 0x0c, 0x2c, 0xfb, 0xcc, 0x64, 0x36, 0x0d, 0x88, 0x5c, 0x59, 0xba, 0x73, 0x09, 0x57, 0x22, + 0x79, 0x4f, 0x88, 0x5f, 0xa7, 0x52, 0xa8, 0x09, 0x75, 0x41, 0xc0, 0xc0, 0x25, 0xbe, 0x4d, 0x34, + 0x3a, 0x2d, 0xd1, 0x29, 0x5c, 0x1b, 0x6b, 0x62, 0xfc, 0x1d, 0xa8, 0x71, 0x8b, 0x3d, 0x37, 0x6d, + 0x3a, 0x1c, 0x79, 0x44, 0xe6, 0xfc, 0x8c, 0x84, 0xa7, 0x71, 0x55, 0x28, 0x76, 0x62, 0xb9, 0x40, + 0x7f, 0x05, 0x19, 0xc6, 0xb9, 0x24, 0x7e, 0x79, 0xf3, 0xe3, 0x37, 0xee, 0x53, 0xb3, 0xd7, 0xef, + 0x63, 0x61, 0x21, 0x0c, 0x3d, 0x6f, 0x28, 0xc3, 0xe1, 0x42, 0x86, 0x7b, 0x7b, 0xfb, 0x58, 0x58, + 0x08, 0x43, 0xce, 0x99, 0xae, 0x25, 0x17, 0x30, 0xec, 0xf7, 0x7b, 0x58, 0x58, 0x20, 0x0c, 0x4b, + 0x36, 0xf5, 0x5f, 0x90, 0x80, 0x49, 0x58, 0xa3, 0x20, 0x47, 0x68, 0xbe, 0x79, 0x84, 0x9d, 0x84, + 0x15, 0x9e, 0x18, 0x03, 0x75, 0xa1, 0xa4, 0x13, 0x2f, 0x0d, 0x64, 0x28, 0x96, 0x37, 0x6f, 0x5f, + 0xc0, 0x09, 0x91, 0x09, 0x1e, 0x5b, 0xa3, 0x6f, 0x20, 0x27, 0xf9, 0xd7, 0x28, 0xc9, 0x82, 0xf7, + 0xc9, 0x05, 0x56, 0x16, 0x06, 0x3e, 0x56, 0x46, 0xe8, 0x43, 0x28, 0x9f, 0x86, 0xce, 0x80, 0xa8, + 0xd3, 0xa0, 0x8c, 0xfb, 0x12, 0x06, 0x29, 0x12, 0xf9, 0xd7, 0x43, 0x77, 0x00, 0x9d, 0x58, 0xcc, + 0x0c, 0xc8, 0x90, 0x72, 0x12, 0x1d, 0x43, 0x65, 0xc0, 0x16, 0x71, 0xfd, 0xc4, 0x62, 0x58, 0x2a, + 0x7a, 0x4a, 0x8e, 0x6e, 0x41, 0x9a, 0xdf, 0xd3, 0x71, 0x7b, 0x5e, 0x12, 0x48, 0xf3, 0x7b, 0xe8, + 0x63, 0x55, 0x23, 0x5c, 0x46, 0x7c, 0xc2, 0x54, 0xc8, 0xa6, 0x3b, 0x19, 0x9c, 0x14, 0x0a, 0xa6, + 0x6c, 0xc1, 0x7a, 0xe8, 0xfb, 0xc4, 0x26, 0x8c, 0x89, 0x14, 0xce, 0x29, 0xf5, 0x4c, 0xdb, 0xf2, + 0x3c, 0x55, 0xae, 0x8a, 0x9d, 0x2c, 0x5e, 0x4d, 0xe8, 0xfb, 0x94, 0x7a, 0x3b, 0x42, 0xab, 0x29, + 0xec, 0xfa, 0xcf, 0x68, 0x30, 0x94, 0x5e, 0x30, 0x3d, 0xca, 0x98, 0x2c, 0x62, 0xc5, 0x4e, 0x0e, + 0xd7, 0x12, 0x9a, 0x3d, 0xaa, 0x5e, 0xf5, 0x25, 0xac, 0x06, 0xc4, 0x09, 0x7d, 0xc7, 0x12, 0x27, + 0x6e, 0x6e, 0x71, 0x32, 0x24, 0x3e, 0x67, 0xb2, 0x7c, 0x15, 0x3b, 0x79, 0xbc, 0x12, 0x6b, 0x7b, + 0xb1, 0x52, 0x91, 0x79, 0x6d, 0x44, 0x69, 0x60, 0xfe, 0x18, 0xaa, 0xf3, 0x98, 0xf9, 0x63, 0x68, + 0x79, 0x2e, 0x3f, 0x93, 0xc5, 0xad, 0xd8, 0x29, 0xe0, 0x15, 0xa1, 0xfe, 0x5e, 0x6b, 0xbf, 0x57, + 0x4a, 0x61, 0xf8, 0x1d, 0x34, 0x92, 0xb4, 0x30, 0x47, 0x01, 0x1d, 0x04, 0xda, 0xc5, 0x2b, 0xd2, + 0x1f, 0x45, 0x7c, 0x39, 0x89, 0x38, 0x1c, 0x03, 0x5e, 0xa7, 0x52, 0xc6, 0x3f, 0x65, 0x21, 0xd3, + 0xeb, 0xf7, 0xd1, 0x1a, 0x64, 0x5e, 0x92, 0x20, 0x0e, 0x66, 0xf1, 0x20, 0x86, 0xbf, 0x02, 0xb9, + 0x97, 0x34, 0x70, 0x54, 0xf5, 0xaa, 0x74, 0x52, 0x58, 0x3d, 0x0a, 0xd5, 0x47, 0x50, 0x16, 0xbf, + 0x4d, 0x59, 0x72, 0x54, 0x75, 0xaa, 0x74, 0xd2, 0x18, 0x84, 0xb0, 0x2d, 0x65, 0x02, 0xb5, 0x06, + 0x19, 0x9b, 0xa8, 0xf2, 0x24, 0xb6, 0x46, 0x3c, 0xe8, 0x71, 0xed, 0x13, 0x2b, 0x60, 0xf2, 0xb4, + 0x54, 0xe9, 0x64, 0xb1, 0x7a, 0xd4, 0xe3, 0x8a, 0xdf, 0xd1, 0xb8, 0x20, 0x01, 0x39, 0x0c, 0x42, + 0x38, 0x1e, 0xf7, 0x63, 0x58, 0x62, 0x5c, 0x9c, 0x0f, 0x3d, 0xeb, 0xcc, 0x1c, 0x32, 0x19, 0x01, + 0x95, 0x4e, 0x1e, 0x03, 0xe3, 0x7c, 0x57, 0x08, 0xf7, 0x25, 0x6c, 0x13, 0x10, 0xf1, 0xb9, 0xcb, + 0xcf, 0xcc, 0x80, 0xd8, 0x74, 0xe0, 0xbb, 0x32, 0xfe, 0xca, 0x72, 0x36, 0x05, 0xbc, 0xac, 0x74, + 0x78, 0xac, 0x8a, 0xd2, 0x96, 0x90, 0xbb, 0x84, 0x99, 0x21, 0xe7, 0x24, 0x20, 0xaa, 0xbc, 0x54, + 0x3a, 0x45, 0x5c, 0x8b, 0x34, 0x47, 0x4a, 0x21, 0xf0, 0x5f, 0xc0, 0x4a, 0x8c, 0xd7, 0x6f, 0xf9, + 0x89, 0x38, 0x92, 0xc2, 0x95, 0x4e, 0x09, 0xa3, 0x48, 0x89, 0x63, 0xdd, 0xeb, 0x54, 0xaa, 0x95, + 0x87, 0xac, 0xf9, 0x92, 0x04, 0xad, 0x22, 0xe4, 0x4d, 0xe9, 0xd3, 0x56, 0x15, 0x96, 0xcc, 0x84, + 0x47, 0x25, 0xc2, 0xd6, 0x08, 0xe9, 0x1d, 0x89, 0x48, 0xf8, 0xa6, 0x55, 0x83, 0x8a, 0x99, 0xf4, + 0x42, 0x6b, 0x0d, 0x56, 0xcc, 0xd9, 0xf5, 0xb6, 0x56, 0x60, 0xd9, 0x9c, 0x5e, 0x52, 0x6b, 0x1d, + 0x56, 0xcd, 0x39, 0xf3, 0x36, 0xfe, 0x2d, 0x05, 0x99, 0xbd, 0xbd, 0x7d, 0xf4, 0x1e, 0x14, 0x38, + 0x7f, 0xc6, 0x85, 0x73, 0x53, 0x72, 0x29, 0x97, 0x70, 0x5e, 0x08, 0x94, 0x63, 0x95, 0x96, 0x09, + 0x6d, 0x44, 0x0d, 0xa1, 0x65, 0x4a, 0x7b, 0x17, 0x96, 0x39, 0x7d, 0x4e, 0x7c, 0x66, 0x8e, 0x48, + 0x60, 0x32, 0x62, 0x53, 0xdf, 0x89, 0x73, 0x79, 0x4d, 0xa9, 0x0e, 0x49, 0xd0, 0x93, 0x0a, 0xe1, + 0x0d, 0x80, 0xa2, 0xa9, 0xdf, 0x16, 0xfd, 0x16, 0x63, 0xb7, 0x56, 0x01, 0x99, 0x33, 0x23, 0x19, + 0xff, 0x90, 0x85, 0x4c, 0xbf, 0xdf, 0xd3, 0x93, 0x38, 0x1e, 0x4f, 0x42, 0x4d, 0xf1, 0x78, 0x3f, + 0xa2, 0x9e, 0xa0, 0x74, 0x46, 0x57, 0x9c, 0x19, 0x4a, 0x17, 0x34, 0x63, 0x17, 0x52, 0x5a, 0x71, + 0x2a, 0xb3, 0x80, 0xd2, 0x25, 0x39, 0x6e, 0x76, 0x96, 0xd2, 0x11, 0x63, 0x17, 0x52, 0xba, 0x1c, + 0x71, 0x75, 0x92, 0xd2, 0xf3, 0xb9, 0x5a, 0x7d, 0x6b, 0xae, 0xd6, 0xde, 0x9e, 0xab, 0xf5, 0xf3, + 0xb9, 0xaa, 0x77, 0x44, 0x38, 0xfa, 0x37, 0xe1, 0xed, 0x6f, 0x41, 0xd3, 0x5f, 0x0b, 0xb0, 0x94, + 0x2c, 0x8f, 0x92, 0x73, 0x61, 0xe0, 0x9b, 0xdc, 0x7a, 0xee, 0xfa, 0x83, 0xa9, 0xc3, 0x49, 0x4d, + 0xa8, 0xfa, 0x52, 0x13, 0x1f, 0x37, 0xbe, 0x86, 0xc6, 0x09, 0xb1, 0xc4, 0xbc, 0x37, 0x89, 0xe9, + 0x59, 0x5c, 0x7c, 0x54, 0x98, 0xa3, 0xfb, 0xf7, 0x22, 0x3a, 0xe5, 0x3a, 0x29, 0xbc, 0x2a, 0x11, + 0xed, 0x4d, 0xb2, 0xa7, 0xf4, 0x87, 0xf7, 0xef, 0x29, 0x72, 0xcd, 0x37, 0xde, 0xba, 0x2f, 0x8c, + 0x33, 0xd2, 0x38, 0x3d, 0x6b, 0xbc, 0x75, 0xff, 0x5c, 0xe3, 0x2d, 0x61, 0x9c, 0x95, 0xc6, 0x99, + 0x39, 0xc6, 0x5b, 0xca, 0xf8, 0x0e, 0xd4, 0xb9, 0x3b, 0x24, 0x26, 0xa7, 0xe6, 0x99, 0x4b, 0x3c, + 0x47, 0x18, 0xe5, 0x74, 0x16, 0xad, 0x08, 0x4d, 0x9f, 0xfe, 0x20, 0xe4, 0xd1, 0xab, 0x2e, 0x13, + 0xca, 0xcd, 0xa1, 0xcb, 0x46, 0x01, 0x71, 0x5c, 0xf9, 0xd9, 0xa0, 0xcf, 0x99, 0x79, 0xcd, 0xd3, + 0x35, 0x42, 0xf9, 0x7e, 0x52, 0x2f, 0xcf, 0x9b, 0xc2, 0xf8, 0x26, 0x54, 0xe8, 0x0b, 0x12, 0x78, + 0xd6, 0xc8, 0x0c, 0x84, 0x93, 0x65, 0xc8, 0xa4, 0x3b, 0x79, 0xbc, 0xa4, 0xc5, 0x58, 0x48, 0x75, + 0xc0, 0x47, 0x48, 0x36, 0x22, 0xc4, 0x3e, 0x19, 0xe7, 0xe4, 0x02, 0xae, 0x69, 0x55, 0x4f, 0x6a, + 0xd4, 0xbc, 0x6e, 0x43, 0x8d, 0x53, 0x6e, 0x79, 0x09, 0x78, 0x49, 0xf3, 0xb6, 0x22, 0x15, 0x49, + 0xf0, 0x57, 0xb0, 0x66, 0xbd, 0x7c, 0xfe, 0x52, 0x78, 0x8c, 0xb9, 0x9e, 0x3c, 0x4d, 0xaa, 0x25, + 0x94, 0x35, 0x6f, 0x57, 0xb4, 0xba, 0xa7, 0xb4, 0xf1, 0x02, 0x9a, 0x50, 0x0f, 0x7d, 0xcb, 0x67, + 0x2f, 0x05, 0x95, 0xd4, 0x21, 0x5b, 0xa7, 0x72, 0xc0, 0xb5, 0xb1, 0x46, 0x9e, 0xb2, 0x05, 0xfe, + 0x5b, 0x68, 0x3c, 0xb3, 0x3c, 0x46, 0x4c, 0xd7, 0xe7, 0x24, 0x08, 0xc2, 0x51, 0xc2, 0x5d, 0x2a, + 0x9f, 0x97, 0xf1, 0xba, 0x44, 0x74, 0x13, 0x80, 0xf8, 0x75, 0x7f, 0x06, 0xd5, 0xe4, 0x8e, 0x0e, + 0x99, 0x8e, 0xc5, 0x25, 0xbc, 0x44, 0xe2, 0x8d, 0x94, 0x4b, 0x52, 0x89, 0x6d, 0x9a, 0xae, 0xad, + 0xab, 0x70, 0xc5, 0x5c, 0xc4, 0xc9, 0x45, 0x4a, 0xc9, 0xb9, 0x85, 0x4a, 0xc1, 0x29, 0x19, 0x58, + 0xd3, 0x9c, 0x69, 0x19, 0xd0, 0x30, 0x17, 0x50, 0xa3, 0x55, 0x87, 0xaa, 0x39, 0xb1, 0xf3, 0x72, + 0xbe, 0x33, 0x3b, 0xdc, 0x42, 0x50, 0x37, 0xa7, 0xb6, 0xb1, 0xd5, 0x80, 0x75, 0x73, 0xee, 0x6e, + 0xc9, 0x69, 0x4c, 0x6f, 0x87, 0x9c, 0xf8, 0x22, 0x9f, 0xb7, 0x96, 0xa1, 0x66, 0x4e, 0x7a, 0xd4, + 0xf8, 0x97, 0x14, 0x94, 0xe2, 0x53, 0x2c, 0xba, 0x07, 0xcb, 0xc4, 0x0a, 0xbc, 0x33, 0x93, 0x93, + 0x60, 0xe8, 0xfa, 0x56, 0x7c, 0xfd, 0x55, 0xec, 0x5c, 0xc2, 0x75, 0xa9, 0xea, 0x8f, 0x35, 0x9a, + 0x12, 0x62, 0xb4, 0x09, 0x83, 0xb4, 0x34, 0x48, 0xe1, 0x9a, 0xd0, 0x4c, 0xe2, 0xe5, 0xc2, 0x67, + 0x5e, 0x21, 0x97, 0x32, 0x3d, 0x8c, 0xf1, 0x4b, 0x01, 0xb2, 0x82, 0x4a, 0xe2, 0x83, 0xda, 0xf5, + 0x1d, 0xf2, 0x4a, 0x15, 0x4e, 0xac, 0x1e, 0xd0, 0x6d, 0xc8, 0x06, 0xd4, 0x23, 0xfa, 0x22, 0xe0, + 0xf2, 0x9c, 0x2b, 0x3e, 0x4c, 0x3d, 0x82, 0x25, 0x08, 0x7d, 0x00, 0x45, 0xf9, 0x79, 0x1c, 0xe5, + 0x13, 0x51, 0xdb, 0x0a, 0x52, 0xa2, 0x42, 0xc2, 0x80, 0x3c, 0xf1, 0x9d, 0x28, 0x61, 0xc8, 0x93, + 0x19, 0xf1, 0x75, 0xcc, 0x4f, 0x9f, 0x8d, 0x72, 0xd1, 0xd1, 0x6c, 0xf2, 0x6c, 0x74, 0x03, 0xca, + 0x9e, 0x37, 0x8c, 0xca, 0xae, 0x4e, 0x07, 0x19, 0x5c, 0xf2, 0xbc, 0x61, 0x3f, 0xae, 0xf3, 0x63, + 0x90, 0xac, 0xf5, 0x05, 0x9d, 0x68, 0x34, 0x88, 0xc5, 0x87, 0x01, 0x09, 0x1a, 0xa9, 0x98, 0x4f, + 0x77, 0x72, 0x38, 0x2f, 0x00, 0xa3, 0x68, 0x08, 0xce, 0x59, 0x54, 0x40, 0x74, 0x98, 0xe7, 0x71, + 0x89, 0x73, 0xd6, 0x8f, 0x8b, 0xf5, 0x6c, 0xe8, 0x80, 0xce, 0x1e, 0xd3, 0xa1, 0x83, 0xbe, 0x80, + 0xd5, 0x59, 0xa6, 0x0f, 0x55, 0x59, 0xcd, 0x75, 0x8a, 0x78, 0x79, 0x2a, 0x73, 0x2e, 0x4e, 0x9b, + 0x4b, 0x3a, 0x7d, 0xcc, 0xa6, 0xcd, 0xb9, 0xc5, 0xa4, 0x22, 0xd7, 0x06, 0x73, 0x8b, 0xc9, 0xdc, + 0x4f, 0x91, 0xf2, 0xcc, 0xa7, 0x88, 0xe0, 0xeb, 0x74, 0xcc, 0xe9, 0xaf, 0x90, 0x25, 0x5c, 0x9f, + 0x4e, 0xc4, 0xfa, 0xa3, 0x78, 0x2a, 0x9a, 0xf4, 0x07, 0x48, 0x05, 0x57, 0x27, 0xb3, 0x9e, 0xf2, + 0x35, 0x8c, 0x23, 0x4c, 0x7f, 0x75, 0x54, 0x71, 0x42, 0x26, 0x28, 0x5d, 0x86, 0x92, 0x19, 0x91, + 0xab, 0x55, 0x82, 0x82, 0xa9, 0x98, 0x34, 0x7b, 0x9c, 0x14, 0x75, 0x3b, 0x41, 0x91, 0xe4, 0x33, + 0x8b, 0x4e, 0x6a, 0x7a, 0xe3, 0xa5, 0x2e, 0xb1, 0xcd, 0x73, 0x42, 0xb7, 0x75, 0x19, 0xd6, 0xcc, + 0x79, 0x3b, 0x37, 0x3f, 0x3f, 0xcd, 0xcf, 0x90, 0xf2, 0x28, 0x31, 0xf6, 0xa9, 0x0a, 0xcf, 0x69, + 0x8f, 0xca, 0xbc, 0x34, 0xe5, 0xb5, 0x56, 0x05, 0xca, 0x89, 0xec, 0x23, 0xe7, 0x37, 0x79, 0x93, + 0xa1, 0xce, 0x1f, 0x53, 0x17, 0x16, 0x2a, 0xc5, 0x4d, 0xde, 0x4a, 0xcc, 0x4c, 0xe2, 0x0a, 0x5c, + 0x36, 0xe7, 0x7f, 0x61, 0xca, 0x31, 0xa7, 0xbf, 0x20, 0xa5, 0x23, 0xe6, 0x7d, 0x26, 0xca, 0xdc, + 0x39, 0xf7, 0x3b, 0x50, 0xa6, 0xc9, 0x45, 0x1f, 0x7a, 0xc6, 0xff, 0x65, 0x00, 0xc6, 0xf7, 0x96, + 0xef, 0xf4, 0x8e, 0xe6, 0xcf, 0x01, 0xc5, 0x97, 0xd9, 0x23, 0x8b, 0x31, 0x51, 0x1e, 0x48, 0x7c, + 0xb4, 0xaf, 0x47, 0xba, 0x43, 0x8b, 0x31, 0x6c, 0x71, 0xf2, 0xa7, 0x8b, 0x9a, 0x0b, 0x13, 0x4d, + 0x1c, 0x8a, 0x67, 0x5d, 0x6b, 0xfc, 0x9a, 0x81, 0xfc, 0x8e, 0x6c, 0x4d, 0x18, 0xff, 0x9f, 0x86, + 0x02, 0x26, 0x92, 0x1a, 0x53, 0x7d, 0x97, 0xd4, 0x74, 0xdf, 0x65, 0xb2, 0x25, 0x91, 0x9e, 0x6e, + 0x49, 0xcc, 0xb9, 0xb9, 0xcf, 0xce, 0xbd, 0xb9, 0x5f, 0x87, 0x7c, 0x40, 0x06, 0xc2, 0x15, 0x79, + 0x7d, 0xc1, 0x2a, 0x9f, 0x50, 0x6b, 0xa6, 0xa7, 0x51, 0x38, 0xaf, 0xa7, 0x21, 0xa8, 0x37, 0xd1, + 0xd5, 0x48, 0xa4, 0xcc, 0xe8, 0x86, 0xbf, 0xa8, 0x8f, 0xdc, 0x49, 0xa1, 0x3a, 0x29, 0xaa, 0x6b, + 0xfe, 0xd2, 0x05, 0xae, 0xf9, 0xa5, 0xb3, 0x27, 0x27, 0x16, 0x07, 0xab, 0x1e, 0xf2, 0x71, 0xb6, + 0x98, 0xa9, 0x67, 0xf1, 0x6c, 0x7f, 0xc9, 0xf8, 0xc7, 0x14, 0x14, 0x31, 0x61, 0x23, 0xea, 0x33, + 0x82, 0x6e, 0xc1, 0xf2, 0xd8, 0x3b, 0x66, 0x10, 0xfa, 0x63, 0x37, 0xd7, 0x58, 0x92, 0x05, 0x5d, + 0x07, 0xf5, 0x60, 0x7d, 0x14, 0x10, 0xe6, 0x0e, 0x7c, 0xe2, 0x98, 0x23, 0xca, 0xb8, 0x19, 0xa8, + 0x5d, 0xd2, 0xdd, 0xbc, 0x71, 0xd3, 0xeb, 0x30, 0x82, 0x1d, 0x52, 0xc6, 0xf5, 0x56, 0xe2, 0xd5, + 0xd1, 0x1c, 0xa9, 0xf8, 0x0c, 0x5f, 0xd9, 0xa1, 0xfe, 0x33, 0x37, 0x18, 0xf6, 0x68, 0x18, 0xd8, + 0xe4, 0x68, 0xe4, 0x51, 0xcb, 0x31, 0xfe, 0xfe, 0xc2, 0x1c, 0x98, 0xbb, 0x84, 0xf4, 0xfc, 0x25, + 0x7c, 0x0a, 0x35, 0x9b, 0x3a, 0x44, 0x7c, 0x69, 0x05, 0x67, 0x23, 0xea, 0xfa, 0x5c, 0xb7, 0x1b, + 0xaa, 0x42, 0xdc, 0x8e, 0xa5, 0x06, 0x8c, 0x7d, 0x64, 0xfc, 0x73, 0x0a, 0x32, 0x8f, 0x08, 0x37, + 0xfa, 0xef, 0x62, 0x4a, 0xc6, 0x17, 0x89, 0xdd, 0xb8, 0x09, 0x99, 0x20, 0x54, 0x07, 0xbb, 0x09, + 0x8e, 0x25, 0x4d, 0xb0, 0x80, 0x18, 0xff, 0x9d, 0x86, 0xec, 0x9e, 0xcb, 0xb8, 0xf1, 0x5f, 0xa9, + 0x0b, 0xcf, 0xea, 0xc1, 0x54, 0x8f, 0xe6, 0xfc, 0x26, 0x65, 0xe7, 0x52, 0xd4, 0xa1, 0x11, 0xec, + 0xfc, 0x06, 0x60, 0x64, 0x0d, 0x88, 0xba, 0xa4, 0x90, 0x3e, 0x2b, 0x6f, 0x36, 0x62, 0xfb, 0xbe, + 0x90, 0x1e, 0x5a, 0x03, 0x7d, 0x42, 0xec, 0xa4, 0x70, 0x49, 0xa0, 0xa5, 0x58, 0x54, 0x62, 0x51, + 0x7c, 0xd5, 0x58, 0xb2, 0x3c, 0x8d, 0x47, 0x32, 0x5e, 0x4d, 0x70, 0x31, 0x1b, 0x84, 0x3e, 0x6b, + 0xa4, 0xe4, 0xf5, 0xec, 0xa2, 0xe5, 0x4b, 0x0c, 0xfa, 0x4b, 0xa8, 0xf9, 0xe4, 0x15, 0x4f, 0x0c, + 0xa5, 0x49, 0xb8, 0x70, 0x52, 0xb8, 0x22, 0x0c, 0x0e, 0xa3, 0x69, 0x19, 0xa7, 0x90, 0xdf, 0xb1, + 0x7c, 0x9b, 0x78, 0xef, 0x68, 0x5f, 0x93, 0x0c, 0x7a, 0xc7, 0x2d, 0x93, 0x9f, 0x53, 0x71, 0xcf, + 0xe4, 0x0a, 0xac, 0x4d, 0xf6, 0x4c, 0xcc, 0xa3, 0xc3, 0xbd, 0x27, 0xdb, 0xbb, 0xf5, 0x4b, 0x68, + 0x0d, 0x96, 0xb5, 0xea, 0x51, 0xfb, 0xa0, 0x8d, 0xb7, 0xfb, 0xaa, 0x7b, 0x32, 0xdb, 0x51, 0x49, + 0xa3, 0x75, 0x40, 0x5a, 0xd6, 0x3b, 0xda, 0xdf, 0xdf, 0xc6, 0xdd, 0xa7, 0x42, 0x9e, 0x99, 0xdb, + 0x69, 0xc9, 0xce, 0x76, 0x5a, 0x72, 0x73, 0x3b, 0x2d, 0xf9, 0xc7, 0xd9, 0x62, 0xb1, 0x5e, 0xc2, + 0x05, 0xdd, 0x4f, 0xdc, 0xf8, 0x39, 0x03, 0xc5, 0x28, 0x7d, 0x8e, 0x9b, 0x73, 0xa9, 0x64, 0x73, + 0x6e, 0xba, 0x37, 0x98, 0x9e, 0xd3, 0x1b, 0x9c, 0xdf, 0x05, 0xcc, 0x2c, 0xea, 0x02, 0xde, 0xd5, + 0xfd, 0xbe, 0xac, 0x64, 0xd9, 0xd5, 0x99, 0x44, 0xde, 0xec, 0x5b, 0x03, 0xa6, 0xfe, 0xe0, 0xa0, + 0x9a, 0x81, 0x06, 0x14, 0x43, 0x46, 0x02, 0xc7, 0xe2, 0x96, 0xee, 0x2d, 0xc6, 0xcf, 0xc6, 0x7f, + 0xa4, 0x60, 0x59, 0x55, 0xad, 0x87, 0x01, 0x1d, 0xea, 0xbb, 0x7d, 0xe3, 0x87, 0x0b, 0x13, 0x2a, + 0xd1, 0xd9, 0x4b, 0x4f, 0x74, 0xf6, 0xc6, 0x05, 0x29, 0x93, 0x2c, 0x48, 0xc6, 0x56, 0x22, 0x5e, + 0x3e, 0x83, 0x62, 0x54, 0x03, 0x74, 0xca, 0x58, 0x9e, 0x59, 0x0d, 0x8e, 0x21, 0xc6, 0x57, 0x50, + 0x8a, 0x97, 0x36, 0xe7, 0xaf, 0x19, 0xab, 0xc9, 0xbf, 0x66, 0x94, 0x12, 0x7f, 0xbf, 0xd8, 0xe8, + 0x43, 0x65, 0xa2, 0xca, 0x21, 0x04, 0x59, 0x59, 0x6f, 0x95, 0xb5, 0xfc, 0x8d, 0xee, 0x42, 0x29, + 0x7a, 0x93, 0xd8, 0xa6, 0xcc, 0xfc, 0xd9, 0x8c, 0x31, 0x1b, 0xff, 0x99, 0x02, 0x34, 0x8e, 0xec, + 0x5d, 0x97, 0x8d, 0x2c, 0x6e, 0x9f, 0xbc, 0x55, 0x41, 0x5a, 0x83, 0xfc, 0x29, 0x3d, 0x1e, 0x3b, + 0x2f, 0x77, 0x4a, 0x8f, 0xbb, 0xce, 0x84, 0x5f, 0x32, 0x6f, 0xf4, 0x4b, 0x5c, 0x78, 0xb3, 0x17, + 0x28, 0xbc, 0xb7, 0x9e, 0x41, 0x75, 0x52, 0x8e, 0x3e, 0x84, 0xab, 0xbd, 0xee, 0xfe, 0xd1, 0xde, + 0x76, 0xbf, 0xfb, 0xe4, 0xc0, 0xdc, 0x7f, 0xb2, 0xdb, 0x36, 0x8f, 0x0e, 0x7a, 0x87, 0xed, 0x9d, + 0xee, 0xc3, 0x6e, 0x5b, 0xc4, 0x5d, 0x03, 0x56, 0xa7, 0x01, 0xfd, 0xf6, 0xdf, 0xf4, 0xeb, 0x29, + 0x19, 0xac, 0x53, 0x9a, 0xed, 0xa3, 0xdd, 0xee, 0x93, 0x7a, 0x7a, 0xf3, 0x5f, 0x73, 0x50, 0xdb, + 0x16, 0xdc, 0x1d, 0xbf, 0x0d, 0x39, 0xb0, 0xa2, 0xb8, 0x36, 0xf9, 0x57, 0x95, 0x45, 0x07, 0x3f, + 0x85, 0x6d, 0x46, 0x85, 0xf6, 0x93, 0x37, 0xc1, 0x34, 0xab, 0x5e, 0xa7, 0xe0, 0xfd, 0xa8, 0x20, + 0xc7, 0xc0, 0x64, 0x69, 0x46, 0x9b, 0x8b, 0x46, 0x9a, 0x2d, 0xe3, 0xf1, 0xdb, 0x3f, 0x7f, 0x2b, + 0x1b, 0x3d, 0x95, 0xbf, 0x85, 0xfa, 0x23, 0xc2, 0x27, 0x57, 0xbb, 0xb1, 0x60, 0xa0, 0x47, 0x84, + 0xc7, 0x2f, 0xbb, 0x71, 0x2e, 0x46, 0x0f, 0x6e, 0x02, 0x12, 0x05, 0x74, 0x02, 0xc1, 0xd0, 0x22, + 0x53, 0x01, 0x8d, 0xc7, 0xff, 0xe8, 0x7c, 0x90, 0x7e, 0x81, 0xd8, 0x2e, 0x59, 0x60, 0x2e, 0xb8, + 0x5d, 0x12, 0xfb, 0xe6, 0xed, 0x8a, 0x60, 0xfa, 0x2d, 0x2f, 0xe0, 0x8a, 0x26, 0x85, 0xe6, 0x73, + 0x22, 0x11, 0xa1, 0xdb, 0xb3, 0xd9, 0x6d, 0x26, 0x5b, 0xc5, 0x6f, 0xbc, 0x73, 0x31, 0xb0, 0x7a, + 0x6f, 0xeb, 0xe1, 0xd3, 0x1b, 0x03, 0x97, 0x9f, 0x84, 0xc7, 0x4d, 0x9b, 0x0e, 0xa3, 0xbf, 0xe0, + 0xa9, 0xbf, 0xd9, 0xd9, 0xd4, 0x8b, 0x04, 0xbf, 0xa4, 0x2b, 0x7b, 0xee, 0x0b, 0xf2, 0x57, 0xf2, + 0x4c, 0x48, 0x39, 0xfd, 0x5d, 0xba, 0xaa, 0x9f, 0x1f, 0x3c, 0x90, 0x82, 0xe3, 0xbc, 0x34, 0xf9, + 0xfc, 0x0f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x1a, 0x62, 0xb5, 0x34, 0xf5, 0x27, 0x00, 0x00, } diff --git a/protobufs/livekit_agent_simulation.proto b/protobufs/livekit_agent_simulation.proto index fdd57806b..b1b5652da 100644 --- a/protobufs/livekit_agent_simulation.proto +++ b/protobufs/livekit_agent_simulation.proto @@ -82,14 +82,10 @@ message SimulationRun { } message JobMetrics { - // Headline scores, 0-1, computed by the worker as weighted means over the - // members that ran (a missing member renormalizes the rest): - // accuracy = task_completion x3 + stt.keyterm_recall + (1 - stt.wer) + tts.enunciation_score - // experience = conversation.turn_taking_score x2 + tts.naturalness_score + llm.conciseness_score - optional float accuracy_score = 1; // absent when the simulator spoiled the call - optional float experience_score = 2; // absent when the simulator spoiled the call - // The accuracy anchor: the scenario verdict as 1/0, or the fraction of the declared target state the call reached. - optional float task_completion = 3; // absent when the simulator spoiled the call + // Headline scores, 0-1. All absent when the simulator spoiled the call. + optional float accuracy_score = 1; + optional float experience_score = 2; + optional float task_completion = 3; // the scenario verdict as 1/0 STT stt = 4; LLM llm = 5; @@ -97,96 +93,112 @@ message SimulationRun { Conversation conversation = 7; Simulator simulator = 8; - // Conversation timeline: one entry per transcript turn, both speakers. + // One entry per turn, both speakers. repeated Turn turns = 9; string judge_model = 10; // text judge for judged scores; "" if none ran - string audio_judge_model = 11; // audio (LALM) judge; "" if none ran bool has_remote_session = 12; // false = waveform-only capture (e.g. SIP) google.protobuf.Timestamp t0 = 13; // audio only; call start (first speech edge); turn times relative to it + // Judged over the authored dialog (text and audio); needs the text judge. + optional float conciseness = 14; // judged {0, 0.5, 1} per agent turn; job = mean + // Whole-call issue flags; true = present, absent = not judged. + optional bool unnecessary_tool_calls = 15; + optional bool information_loss = 16; + optional bool redundant_statements = 17; + optional bool poor_question_quality = 18; + optional float conversation_progression = 19; // 0-1, derived from the flags above + // The agent's perception of the caller. Audio only. message STT { - optional float wer = 1; // needs the RemoteSession; word error rate, pooled: word_errors / words - optional uint32 words = 2; // needs the RemoteSession; pooling stats: run WER = sum errors / sum words + optional float wer = 1; // needs the RemoteSession; word_errors / words + optional uint32 words = 2; // needs the RemoteSession optional uint32 word_errors = 3; // needs the RemoteSession - optional float cer = 4; // needs the RemoteSession; character error rate, pooled + optional float cer = 4; // needs the RemoteSession; char_errors / chars optional uint32 chars = 9; // needs the RemoteSession optional uint32 char_errors = 10; // needs the RemoteSession - optional float keyterm_recall = 5; // needs the RemoteSession; recall of uttered key entities (names, IDs, amounts) - optional uint32 keyterms_uttered = 6; // needs the RemoteSession - optional uint32 keyterms_recognized = 7; // needs the RemoteSession - optional uint32 transcription_latency_ms = 8; // needs the RemoteSession; agent-reported STT/endpointing delay, mean + optional uint32 stt_delay_ms = 8; // needs the RemoteSession; agent-reported STT/endpointing delay + + // Key entities (names, IDs, amounts) the caller said. Needs the text judge. + optional float entity_recognition = 11; // recognized / uttered + optional uint32 entities_uttered = 12; + optional uint32 entities_recognized = 13; } message LLM { - optional uint32 ttft_ms = 1; // needs the RemoteSession; time to first token, mean (per-turn in turns) - optional uint32 ttfs_ms = 2; // needs the RemoteSession; time to first sentence (the smallest speakable unit) - optional float tokens_per_second = 3; // needs the RemoteSession; decode rate, mean (per-turn in turns.llm) - optional float conciseness_score = 4; // audio only; needs the audio judge; judged 0-1: brief enough for voice + optional uint32 ttft_ms = 1; // needs the RemoteSession; time to first token + optional uint32 ttfs_ms = 2; // needs the RemoteSession; time to first sentence, the smallest speakable unit + optional float tokens_per_second = 3; // needs the RemoteSession } - // The agent's voice. audio only. + // The agent's voice. Audio only. message TTS { - optional uint32 ttfa_ms = 1; // needs the RemoteSession optional uint32 ttfb_ms = 2; // needs the RemoteSession; provider byte-level TTFB - optional float wer = 3; // needs the RemoteSession; word error rate, pooled: word_errors / words + optional float wer = 3; // needs the RemoteSession; word_errors / words optional uint32 words = 7; // needs the RemoteSession optional uint32 word_errors = 8; // needs the RemoteSession - optional float cer = 9; // needs the RemoteSession; character error rate, pooled + optional float cer = 9; // needs the RemoteSession; char_errors / chars optional uint32 chars = 10; // needs the RemoteSession optional uint32 char_errors = 11; // needs the RemoteSession - optional float speech_rate_wpm = 4; // speaking rate; conversational English ~110-150 - optional uint32 heard_words = 12; // pooling stats for speech_rate_wpm: raw heard words (not WER tokens) - optional uint32 speech_ms = 13; // agent voiced time, sum of raw VAD segments - optional float naturalness_score = 5; // needs the audio judge; judged 0-1: prosody / expressiveness - optional float enunciation_score = 6; // needs the audio judge; judged 0-1: key entities audibly intact + + // Key entities the agent said, as the caller heard them. Needs the text judge. + optional float entity_recognition = 14; // recognized / uttered + optional uint32 entities_uttered = 15; + optional uint32 entities_recognized = 16; } // Audio only. message Conversation { - optional float turn_taking_score = 1; // 0-1: latency curve + cut-in/barge-in/missed-turn penalties - optional int32 response_latency_p50_ms = 2; // floor-transfer offset; a gap is > 0, an overlap < 0 - optional int32 response_latency_p95_ms = 3; - optional int32 response_latency_p99_ms = 4; - optional int32 response_latency_ms = 16; // one turn's floor-transfer offset, cut-in if negative; job/run serve the percentiles - optional uint32 agent_yield_latency_ms = 5; // agent audio continuing past a barge-in — per turn: on the barging persona row (presence = barge-in); job: mean - optional uint32 eot_misprediction_count = 6; // agent started before the caller's turn ended; 0/1 per turn + optional float turn_taking_score = 1; // 0-1 + optional int32 heard_e2e_latency_p50_ms = 2; // floor-transfer offset; a gap is > 0, an overlap < 0 + optional int32 heard_e2e_latency_p95_ms = 3; + optional int32 heard_e2e_latency_p99_ms = 4; + optional uint32 time_to_yield_ms = 5; // agent audio continuing past a barge-in + optional uint32 eot_misprediction_count = 6; // agent started before the caller's turn ended optional float overlap_ratio = 7; // overlapping speech / total speech optional uint32 overlap_speech_ms = 8; // pooling stats for overlap_ratio optional uint32 total_speech_ms = 9; - optional uint32 silence_total_ms = 10; // dead air within the conversation - optional uint32 awkward_silence_count = 11; // gaps past the natural-pause threshold; 0/1 per turn - optional uint32 unanswered_persona_turns = 12; // the simulated party spoke, the agent never responded; 0/1 per turn - optional uint32 false_interruption_count = 13; // needs the RemoteSession; agent paused for a non-interruption - optional uint32 false_interruption_unrecovered_count = 14; // needs the RemoteSession; of those, never resumed - optional uint32 agent_reported_e2e_latency_ms = 15; // needs the RemoteSession; the agent's own claim, mean + optional uint32 awkward_silence_count = 11; // gaps past the natural-pause threshold + optional uint32 unanswered_turns = 12; // the simulated party spoke, the agent never responded + optional uint32 false_interruption_count = 13; // needs the RemoteSession; agent paused for a non-interruption + optional uint32 e2e_latency_ms = 15; // needs the RemoteSession; the agent's own claim } - // Was the call spoiled by the simulator, not the agent? Diagnostic only — - // never folded into the scores; run aggregation excludes flagged calls - // from the two headline scores. + // Was the call spoiled by the simulator, not the agent? Diagnostic only, + // never folded into the scores. message Simulator { optional bool early_termination = 1; // simulator ended a still-progressing call optional bool late_termination = 2; // simulator dragged on after the goal was met } - // Rows exist for text and audio; + // Rows exist for text and audio. The groups above hold the means and pools + // over these; anything not measured for a turn stays unset. message Turn { uint32 index = 1; // 1-based, conversation order - livekit.agent.ChatRole role = 2; // ASSISTANT — the agent under test, USER — simulator persona. + livekit.agent.ChatRole role = 2; // ASSISTANT = the agent under test, USER = simulator persona optional uint32 start_ms = 3; // audio only; relative to t0; unset if the turn could not be aligned to the audio optional uint32 end_ms = 4; // audio only - // Job groups hold means/pools of these; a field with no per-turn measurement stays unset here. - STT stt = 5; // transcription_latency_ms - LLM llm = 6; // ttft_ms, ttfs_ms, tokens_per_second, conciseness_score - TTS tts = 7; // ttfa_ms, ttfb_ms, naturalness_score, enunciation_score - Conversation conversation = 8; // response_latency_ms, agent_yield_latency_ms, agent_reported_e2e_latency_ms, 0/1 counts + optional uint32 stt_delay_ms = 5; // needs the RemoteSession + optional uint32 llm_ttft_ms = 6; // needs the RemoteSession + optional uint32 llm_ttfs_ms = 7; // needs the RemoteSession + optional float llm_tps = 8; // needs the RemoteSession + optional uint32 tts_ttfb_ms = 9; // needs the RemoteSession + optional uint32 e2e_latency_ms = 10; // needs the RemoteSession + + optional int32 heard_e2e_latency_ms = 11; + optional uint32 time_to_yield_ms = 12; + optional float turn_taking_score = 13; + optional float conciseness = 14; // needs the text judge + + optional bool eot_misprediction = 15; + optional bool awkward_silence = 16; + optional bool unanswered = 17; } } - // Aggregates over this run's jobs. Reuses the JobMetrics group shapes. + // Aggregates over this run's jobs, so a run summary does not have to pull + // every job and turn. Reuses the JobMetrics group shapes. message RunMetrics { optional float accuracy_score = 1; // mean over scored jobs optional float experience_score = 2; @@ -196,10 +208,6 @@ message SimulationRun { JobMetrics.LLM llm = 5; JobMetrics.TTS tts = 6; JobMetrics.Conversation conversation = 7; - - uint32 jobs_total = 9; - uint32 jobs_measured = 10; // composites cover jobs_measured - jobs_simulator_fault - uint32 jobs_simulator_fault = 11; // spoiled by the simulator: flagged, not scored } message Create { @@ -290,9 +298,8 @@ message SimulationRun { int32 concurrency = 17; // Conversation mode for every job in this run; unspecified = TEXT. SimulationMode mode = 18; - // Run-level metric aggregates. + // Run-level aggregates over this run's jobs. RunMetrics metrics = 19; - // zstd-compressed SimulationRunSummary: decompress, then proto.Unmarshal. // Compressed by default so the blob ships as-is on every hop. bytes summary_zstd = 20;