Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponses.EarlyAttach.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
RouteCompletions = TRUE
IsolateSubclasses = TRUE
InitialListenerBoundary <- InitialListenerOracle
AttachAfterBuild = FALSE
RequestBoundary <- RequestOracle
CleanupBoundary <- CleanupOracle
ForwardReasoning = TRUE
HonorStreaming = TRUE
MergeExtra = TRUE
CleanText = TRUE
RouteOptIn = TRUE
ForwardTools = TRUE
ReleaseListener = TRUE
INVARIANTS NoListenerDuringBuild Routing ReasoningAndTools ListenerReleased Streaming ExtraBody WellFormed
17 changes: 17 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponses.NoCleanup.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
RouteCompletions = TRUE
IsolateSubclasses = TRUE
InitialListenerBoundary <- InitialListenerOracle
AttachAfterBuild = TRUE
RequestBoundary <- RequestOracle
CleanupBoundary <- CleanupOracle
ForwardReasoning = TRUE
HonorStreaming = TRUE
MergeExtra = TRUE
CleanText = TRUE
RouteOptIn = TRUE
ForwardTools = TRUE
ReleaseListener = FALSE
INVARIANTS NoListenerDuringBuild Routing ReasoningAndTools ListenerReleased Streaming ExtraBody WellFormed
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
RouteCompletions = FALSE
IsolateSubclasses = TRUE
InitialListenerBoundary <- InitialListenerOracle
AttachAfterBuild = TRUE
RequestBoundary <- RequestOracle
CleanupBoundary <- CleanupOracle
ForwardReasoning = TRUE
HonorStreaming = TRUE
MergeExtra = TRUE
CleanText = TRUE
RouteOptIn = TRUE
ForwardTools = TRUE
ReleaseListener = TRUE
INVARIANTS NoListenerDuringBuild Routing ReasoningAndTools ListenerReleased Streaming ExtraBody WellFormed
17 changes: 17 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponses.NoExtra.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
RouteCompletions = TRUE
IsolateSubclasses = TRUE
InitialListenerBoundary <- InitialListenerOracle
AttachAfterBuild = TRUE
RequestBoundary <- RequestOracle
CleanupBoundary <- CleanupOracle
ForwardReasoning = TRUE
HonorStreaming = TRUE
MergeExtra = FALSE
CleanText = TRUE
RouteOptIn = TRUE
ForwardTools = TRUE
ReleaseListener = TRUE
INVARIANTS NoListenerDuringBuild Routing ReasoningAndTools ListenerReleased Streaming ExtraBody WellFormed
17 changes: 17 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponses.NoIsolation.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
RouteCompletions = TRUE
IsolateSubclasses = FALSE
InitialListenerBoundary <- InitialListenerOracle
AttachAfterBuild = TRUE
RequestBoundary <- RequestOracle
CleanupBoundary <- CleanupOracle
ForwardReasoning = TRUE
HonorStreaming = TRUE
MergeExtra = TRUE
CleanText = TRUE
RouteOptIn = TRUE
ForwardTools = TRUE
ReleaseListener = TRUE
INVARIANTS NoListenerDuringBuild Routing ReasoningAndTools ListenerReleased Streaming ExtraBody WellFormed
17 changes: 17 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponses.NoReasoning.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
RouteCompletions = TRUE
IsolateSubclasses = TRUE
InitialListenerBoundary <- InitialListenerOracle
AttachAfterBuild = TRUE
RequestBoundary <- RequestOracle
CleanupBoundary <- CleanupOracle
ForwardReasoning = FALSE
HonorStreaming = TRUE
MergeExtra = TRUE
CleanText = TRUE
RouteOptIn = TRUE
ForwardTools = TRUE
ReleaseListener = TRUE
INVARIANTS NoListenerDuringBuild Routing ReasoningAndTools ListenerReleased Streaming ExtraBody WellFormed
17 changes: 17 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponses.NoRoute.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
RouteCompletions = TRUE
IsolateSubclasses = TRUE
InitialListenerBoundary <- InitialListenerOracle
AttachAfterBuild = TRUE
RequestBoundary <- RequestOracle
CleanupBoundary <- CleanupOracle
ForwardReasoning = TRUE
HonorStreaming = TRUE
MergeExtra = TRUE
CleanText = TRUE
RouteOptIn = FALSE
ForwardTools = TRUE
ReleaseListener = TRUE
INVARIANTS NoListenerDuringBuild Routing ReasoningAndTools ListenerReleased Streaming ExtraBody WellFormed
17 changes: 17 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponses.NoSanitize.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
RouteCompletions = TRUE
IsolateSubclasses = TRUE
InitialListenerBoundary <- InitialListenerOracle
AttachAfterBuild = TRUE
RequestBoundary <- RequestOracle
CleanupBoundary <- CleanupOracle
ForwardReasoning = TRUE
HonorStreaming = TRUE
MergeExtra = TRUE
CleanText = FALSE
RouteOptIn = TRUE
ForwardTools = TRUE
ReleaseListener = TRUE
INVARIANTS NoListenerDuringBuild Routing ReasoningAndTools ListenerReleased Streaming ExtraBody WellFormed
17 changes: 17 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponses.NoStreaming.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
RouteCompletions = TRUE
IsolateSubclasses = TRUE
InitialListenerBoundary <- InitialListenerOracle
AttachAfterBuild = TRUE
RequestBoundary <- RequestOracle
CleanupBoundary <- CleanupOracle
ForwardReasoning = TRUE
HonorStreaming = FALSE
MergeExtra = TRUE
CleanText = TRUE
RouteOptIn = TRUE
ForwardTools = TRUE
ReleaseListener = TRUE
INVARIANTS NoListenerDuringBuild Routing ReasoningAndTools ListenerReleased Streaming ExtraBody WellFormed
17 changes: 17 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponses.NoTools.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
RouteCompletions = TRUE
IsolateSubclasses = TRUE
InitialListenerBoundary <- InitialListenerOracle
AttachAfterBuild = TRUE
RequestBoundary <- RequestOracle
CleanupBoundary <- CleanupOracle
ForwardReasoning = TRUE
HonorStreaming = TRUE
MergeExtra = TRUE
CleanText = TRUE
RouteOptIn = TRUE
ForwardTools = FALSE
ReleaseListener = TRUE
INVARIANTS NoListenerDuringBuild Routing ReasoningAndTools ListenerReleased Streaming ExtraBody WellFormed
17 changes: 17 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponses.Unfixed.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
RouteCompletions = TRUE
IsolateSubclasses = TRUE
InitialListenerBoundary <- InitialListenerOracle
AttachAfterBuild = TRUE
RequestBoundary <- RequestOracle
CleanupBoundary <- CleanupOracle
ForwardReasoning = TRUE
HonorStreaming = FALSE
MergeExtra = FALSE
CleanText = FALSE
RouteOptIn = TRUE
ForwardTools = TRUE
ReleaseListener = TRUE
INVARIANTS NoListenerDuringBuild Routing ReasoningAndTools ListenerReleased Streaming ExtraBody WellFormed
17 changes: 17 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponses.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
RouteCompletions = TRUE
IsolateSubclasses = TRUE
InitialListenerBoundary <- InitialListenerOracle
AttachAfterBuild = TRUE
RequestBoundary <- RequestOracle
CleanupBoundary <- CleanupOracle
ForwardReasoning = TRUE
HonorStreaming = TRUE
MergeExtra = TRUE
CleanText = TRUE
RouteOptIn = TRUE
ForwardTools = TRUE
ReleaseListener = TRUE
INVARIANTS NoListenerDuringBuild Routing ReasoningAndTools ListenerReleased Streaming ExtraBody WellFormed
97 changes: 97 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponses.tla
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
---- MODULE OpenAICompatibleResponses ----
\* Sources: OpenAICompatible.handleInputChange, OpenAiHandler.createMessage and completePrompt.
\* Sinks: buildResponsesRequest, client.responses.create, bridgeExternalAbortSignal.dispose.
\* Initial expectation: find routing, option loss, or listener leaks. Rules preceded the first review.
\* Failure: the saved flag selects the wrong API. Rule: Routing.
\* Failure: reasoning, tools, streaming, Extra Body, or valid text is lost.
\* Rules: ReasoningAndTools, Streaming, ExtraBody, WellFormed. The SDK is an oracle.
\* Failure: construction or request exit retains a listener.
\* Rules: NoListenerDuringBuild and ListenerReleased model attach, abort, and disposal order.
\* Only the Responses message route registers a bridge listener.
\* Parameters and protocol rules are in ResponsesRequestParameters.tla and ResponsesProtocol.tla.
\* Initial review ratchet: 0 covered, 3 added, 2 gap (translations and pixels).
\* Threads: discussion_r4616096233, discussion_r4616097051, discussion_r4616097077 on PR 2498.
\* PR: https://github.com/Zoo-Code-Org/Zoo-Code/pull/2498
\* Self-review added the output-limit override rule after 456 requested tokens became 1000.
\* Refactor ratchet: 0 covered, 1 added, 0 gap. Derived listener state hid disposal order.
\* Review e7ed99742a: 0 covered, 9 added, 2 gap (helper duplication and code/model correspondence).
\* Finding 9's blanket rejection of max was false; supported effort values depend on the model.
\* Review 6385d18512: 1 covered (subclass isolation), 2 added (null defaults and replay scope), 5 gap.
\* Gaps 4-8 concern test evidence, malformed SDK output, comments, and UI wording.
\* The routing model already covered incompatible providers, but DeepSeek lacked the code guard.
\* Current gate: 6912 routing states pass; ten sabotage configs and the unfixed config fail.
\* Across three models: 29 sabotage configs fail with invariant error 12; three fixed configs pass.
\* Separate real-code mutations fail the focused tests. See ResponsesReviewReplays.md for each mapping.
\* These are manual gates. No script or CI step proves code/model correspondence.
\* Findings: completion routing, terminal failure, subclass flags, and replay were missing before fixes.
\* The latest model found no additional defect. No real-code replay false alarm remains.
\* Cost: about 85 agent minutes across the PR; the 33-config TLC gate takes about 30 seconds.
\* Run each config with OpenAICompatibleResponsesMC.tla. Metadata must be outside the repo.
\* Unit gate: openai.spec.ts, openai-compatible-responses.spec.ts, and responses-api-contract.spec.ts.
\* The model checks the request boundary. It does not prove the code correct.
CONSTANTS RequestBoundary, CleanupBoundary, InitialListenerBoundary
VARIABLES source, phase, listener, aborted
vars == <<source, phase, listener, aborted>>
sink == RequestBoundary[source]
ExpectedResponses == source.responses /\ source.compatible

InputDomain ==
[responses : BOOLEAN, entrypoint : {"message", "completion"}, compatible : BOOLEAN,
streaming : {"unset", "on", "off"},
reasoning : BOOLEAN, tools : BOOLEAN, extra : BOOLEAN, badText : BOOLEAN,
signal : {"none", "live", "aborted"}]

Init ==
/\ source \in InputDomain
/\ (source.entrypoint = "message" \/ ~source.tools)
/\ phase = "build"
/\ aborted = (source.signal = "aborted")
/\ listener = InitialListenerBoundary[source]

\* buildResponsesRequest can return a body or throw before listener registration.
Build ==
/\ phase = "build"
/\ phase' \in {"ready", "buildFailed"}
/\ UNCHANGED <<source, listener, aborted>>

\* bridgeExternalAbortSignal skips registration for absent or pre-aborted signals.
Attach ==
/\ phase = "ready"
/\ phase' = "request"
/\ listener' = (ExpectedResponses /\ source.entrypoint = "message" /\ source.signal = "live" /\ ~aborted)
/\ UNCHANGED <<source, aborted>>

\* The once-only abort listener removes itself when the external signal fires.
Abort ==
/\ source.signal = "live"
/\ ~aborted
/\ aborted' = TRUE
/\ listener' = FALSE
/\ UNCHANGED <<source, phase>>

\* Success, SDK failure, and generator close all enter the same finally block.
Finish ==
/\ phase = "request"
/\ phase' = "cleanup"
/\ UNCHANGED <<source, listener, aborted>>

\* The finally block calls the bridge's dispose function before the generator ends.
Dispose ==
/\ phase = "cleanup"
/\ phase' = "done"
/\ listener' = CleanupBoundary[listener]
/\ UNCHANGED <<source, aborted>>

Next == Build \/ Attach \/ Abort \/ Finish \/ Dispose
Spec == Init /\ [][Next]_vars
RequestBuilt == phase \in {"ready", "request", "cleanup", "done"}
Routing == RequestBuilt => sink.responses = ExpectedResponses
ReasoningAndTools == RequestBuilt =>
/\ sink.reasoning = source.reasoning
/\ sink.tools = source.tools
NoListenerDuringBuild == phase \in {"build", "ready", "buildFailed"} => ~listener
ListenerReleased == phase \in {"done", "buildFailed"} => ~listener
Streaming == RequestBuilt => sink.streaming = (source.entrypoint = "message" /\ source.streaming # "off")
ExtraBody == RequestBuilt => sink.extra = source.extra
WellFormed == RequestBuilt => ~sink.badText
====
17 changes: 17 additions & 0 deletions docs/architecture/tla/OpenAICompatibleResponsesMC.tla
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
---- MODULE OpenAICompatibleResponsesMC ----
\* Finite request boundaries stand for the SDK call. No parser is copied.
EXTENDS OpenAICompatibleResponses
CONSTANTS HonorStreaming, MergeExtra, CleanText, RouteOptIn,
ForwardTools, ForwardReasoning, ReleaseListener, AttachAfterBuild, RouteCompletions, IsolateSubclasses
EffectiveResponses(s) == s.responses /\ (s.compatible \/ ~IsolateSubclasses)
/\ (s.entrypoint = "message" \/ RouteCompletions)
RequestOracle == [s \in InputDomain |->
[responses |-> IF RouteOptIn THEN EffectiveResponses(s) ELSE ~EffectiveResponses(s),
streaming |-> IF s.entrypoint = "completion" THEN FALSE ELSE IF HonorStreaming THEN s.streaming # "off" ELSE TRUE,
reasoning |-> IF ForwardReasoning THEN s.reasoning ELSE FALSE,
tools |-> IF ForwardTools THEN s.tools ELSE FALSE,
extra |-> IF MergeExtra THEN s.extra ELSE FALSE,
badText |-> s.badText /\ ~CleanText]]
InitialListenerOracle == [s \in InputDomain |-> ~AttachAfterBuild /\ s.responses /\ s.compatible /\ s.entrypoint = "message" /\ s.signal = "live"]
CleanupOracle == [attached \in BOOLEAN |-> IF ReleaseListener THEN FALSE ELSE attached]
====
23 changes: 23 additions & 0 deletions docs/architecture/tla/ResponsesProtocol.NoCapture.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
GuardReplayScope = TRUE
ReplayScopeBoundary <- ReplayScopeOracle
RequireTerminal = TRUE
ReasoningSamples <- Samples
AcceptBoundary <- AcceptOracle
SaveBoundary <- SaveOracle
ReplayBoundary <- ReplayOracle
ResetBoundary <- ResetOracle
PublishFailedBoundary = FALSE
ChoiceBoundary <- ChoiceOracle
ClientBoundary <- ClientOracle
RejectFailures = TRUE
PreserveOrder = TRUE
PreserveReasoning = FALSE
ReplayReasoning = TRUE
ResetReasoning = TRUE
PreserveRestriction = TRUE
UseAzureV1 = TRUE
GuardInference = TRUE
INVARIANTS NoForeignReasoning TerminalFailuresVisible SavedComplete ReplayComplete NoStaleReasoning NoFailedReasoning ToolRestrictionPreserved EndpointPolicy
23 changes: 23 additions & 0 deletions docs/architecture/tla/ResponsesProtocol.NoFailureGuard.cfg
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
SPECIFICATION Spec
CHECK_DEADLOCK FALSE
CONSTANTS
GuardReplayScope = TRUE
ReplayScopeBoundary <- ReplayScopeOracle
RequireTerminal = TRUE
ReasoningSamples <- Samples
AcceptBoundary <- AcceptOracle
SaveBoundary <- SaveOracle
ReplayBoundary <- ReplayOracle
ResetBoundary <- ResetOracle
PublishFailedBoundary = FALSE
ChoiceBoundary <- ChoiceOracle
ClientBoundary <- ClientOracle
RejectFailures = FALSE
PreserveOrder = TRUE
PreserveReasoning = TRUE
ReplayReasoning = TRUE
ResetReasoning = TRUE
PreserveRestriction = TRUE
UseAzureV1 = TRUE
GuardInference = TRUE
INVARIANTS NoForeignReasoning TerminalFailuresVisible SavedComplete ReplayComplete NoStaleReasoning NoFailedReasoning ToolRestrictionPreserved EndpointPolicy
Loading
Loading