From 3bf96da395c59e23034c9d482aa4946dd3e25ad8 Mon Sep 17 00:00:00 2001 From: deepfates Date: Wed, 23 Sep 2026 14:34:42 -0700 Subject: [PATCH 1/5] Move req_llm to 1.24.0; the multimodal runner records the ReqLLM it loads The multimodal manifests declared an exact ReqLLM build and the runner refused to dispatch under any other, so every ReqLLM release meant editing the manifest constant, both manifest files and their checksums. The run those manifests described was invalidated and its record removed, so the declaration protected nothing. The manifests no longer carry the dependency (schema_version 3). The runner reads the loaded ReqLLM version and its Hex package checksum from mix.lock, refuses to run when they disagree, and binds that dependency into the checkpoint identity, request audits and artifact, so a checkpoint cannot resume under a different ReqLLM package. ReqLLM now honours a caller's max_retries: 0. The transport guard stays, because it is where the attempt event that campaign budgets count is emitted; its comment and test describe that instead of the old reset. benchmarks/authorities.json pins 1.24.0 (tag commit fd9e079f), and its test now reads version and Hex checksums from mix.lock rather than a copy. .dialyzer_ignore.exs: the lib/req_llm.ex entry is gone because req_llm 1.24.0 no longer produces that warning, and the multimodal_runner.ex entry moves up one line with the runner change. --- .dialyzer_ignore.exs | 5 +- .../benchmark_truth/multimodal_manifest.ex | 55 ++++++++++++------- .../imp/benchmark_truth/multimodal_runner.ex | 3 +- benchmarks/authorities.json | 10 ++-- benchmarks/data/multimodal/manifest.json | 12 +--- .../multimodal/openai-responses-manifest.json | 12 +--- docs/differentials/MULTIMODAL_FIDELITY.md | 10 +++- docs/differentials/RESEARCH_LANDSCAPE.md | 2 +- lib/imp/clients/req_llm.ex | 10 ++-- mix.lock | 14 ++--- test/authority_inventory_test.exs | 13 ++++- test/campaign_budget_test.exs | 5 +- test/multimodal_quality_benchmark_test.exs | 31 +++++++++++ 13 files changed, 110 insertions(+), 72 deletions(-) diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs index 882ea68a..5f88dfa3 100644 --- a/.dialyzer_ignore.exs +++ b/.dialyzer_ignore.exs @@ -116,9 +116,6 @@ {"lib/mix/tasks/imp.benchmark.parity.ex", :pattern_match, {715, 11}}, # raise-only helper: invalid_snapshot!/2 exists to Mix.raise {"lib/mix/tasks/imp.public_api.ex", :no_return, {838, 8}}, - # dependency code: this file ships inside the req_llm package, not this - # repo; LLMDB.Model.t/0 is a Zoi-generated spec dialyzer cannot see - {"lib/req_llm.ex", :unknown_type, {109, 24}}, # --- Surfaced 2026-08-08 during ignore-file regeneration (imp-fkwy): 583 # commits landed without CI, and these warnings accumulated unpinned. @@ -221,7 +218,7 @@ # Defensive fallbacks and MapSet opacity retained at the 0.3 cut. These are # individually pinned so a changed success type makes the gate ask again. - {"bench/imp/benchmark_truth/multimodal_runner.ex", :pattern_match_cov, {341, 16}}, + {"bench/imp/benchmark_truth/multimodal_runner.ex", :pattern_match_cov, {340, 16}}, {"lib/imp/adapter/chat.ex", :pattern_match_cov, {774, 8}}, {"lib/imp/adapter/xml.ex", :pattern_match_cov, {675, 8}}, {"lib/imp/mcp.ex", :pattern_match_cov, {372, 8}}, diff --git a/bench/imp/benchmark_truth/multimodal_manifest.ex b/bench/imp/benchmark_truth/multimodal_manifest.ex index 72aa497b..aa805c97 100644 --- a/bench/imp/benchmark_truth/multimodal_manifest.ex +++ b/bench/imp/benchmark_truth/multimodal_manifest.ex @@ -2,25 +2,16 @@ defmodule Imp.BenchmarkTruth.MultimodalManifest do @moduledoc false @payload_keys ~w(assets campaign_id claim_policy created_at limitations provider samples schema_version scoring signature) - @provider_keys ~w(api capabilities credential_env endpoint generation identity_evidence model name pricing profile req_llm_dependency req_llm_model) + @provider_keys ~w(api capabilities credential_env endpoint generation identity_evidence model name pricing profile req_llm_model) @required_generation_keys ~w(max_tokens timeout_ms) @optional_generation_keys ~w(seed temperature top_p) @pricing_keys ~w(as_of cached_input_nano_usd_per_token cached_input_usd_per_1m currency input_nano_usd_per_token input_usd_per_1m output_nano_usd_per_token output_usd_per_1m pricing_basis source) - @req_llm_dependency_keys ~w(package package_sha256 repository source source_revision version) @signature_keys ~w(input output output_schema prompt_contract) @scoring_keys ~w(family_thresholds normalization scorer) @claim_keys ~w(document_family image_family required_families) @asset_keys ~w(bytes mime_type path sha256) @sample_keys ~w(asset_ids delivery expected_capability family gold id prompt) @deliveries ~w(typed_image_data_uri typed_native_file) - @req_llm_dependency %{ - "package" => "req_llm", - "package_sha256" => "266c0e06c47b4562f243dcdf41332342cbed2ec37064750edd725fb66bb6e914", - "repository" => "https://github.com/agentjido/req_llm", - "source" => "hexpm", - "source_revision" => "33840077c2f1332eb6dff2d268dff02393014da4", - "version" => "1.17.1" - } @provider_profiles %{ "google-gemini-2.5-flash-generate-content" => %{ "api" => "generateContent", @@ -43,7 +34,6 @@ defmodule Imp.BenchmarkTruth.MultimodalManifest do "pricing_basis" => "standard_text_image_video", "source" => "https://ai.google.dev/gemini-api/docs/pricing" }, - "req_llm_dependency" => @req_llm_dependency, "req_llm_model" => "google:gemini-2.5-flash" }, "openai-gpt-4.1-mini-2025-04-14-responses" => %{ @@ -66,7 +56,6 @@ defmodule Imp.BenchmarkTruth.MultimodalManifest do "pricing_basis" => "standard", "source" => "https://developers.openai.com/api/docs/models/gpt-4.1-mini" }, - "req_llm_dependency" => @req_llm_dependency, "req_llm_model" => "openai:gpt-4.1-mini-2025-04-14" } } @@ -93,7 +82,7 @@ defmodule Imp.BenchmarkTruth.MultimodalManifest do root = Keyword.get(opts, :root, File.cwd!()) |> Path.expand() exact_keys!(payload, @payload_keys, "manifest payload") - require_equal!(payload["schema_version"], 2, "schema_version") + require_equal!(payload["schema_version"], 3, "schema_version") require_string!(payload["campaign_id"], "campaign_id") require_string!(payload["created_at"], "created_at") @@ -144,17 +133,44 @@ defmodule Imp.BenchmarkTruth.MultimodalManifest do end) end - def runtime_dependency!(payload) when is_map(payload) do - dependency = payload["provider"]["req_llm_dependency"] + @doc """ + Binds the loaded ReqLLM package into the manifest's provider, so the + checkpoint identity, request audits and artifact record the dependency that + actually serializes and sends each request. A checkpoint written under one + ReqLLM package cannot resume under another. + """ + def bind_runtime_dependency!(%{payload: payload} = manifest) do + %{ + manifest + | payload: put_in(payload, ["provider", "req_llm_dependency"], runtime_dependency!()) + } + end - runtime_version = + @doc """ + The loaded ReqLLM package: its version and the Hex package checksum from the + project's lock. Raises when nothing is loaded or when the loaded version is + not the locked one. + """ + def runtime_dependency! do + loaded = case Application.spec(:req_llm, :vsn) do nil -> raise ArgumentError, "ReqLLM runtime dependency is not loaded" version -> to_string(version) end - require_equal!(runtime_version, dependency["version"], "ReqLLM runtime version") - dependency + case Mix.Dep.Lock.read()[:req_llm] do + {:hex, :req_llm, ^loaded, package_sha256, _managers, _deps, "hexpm", _outer} -> + %{ + "package" => "req_llm", + "package_sha256" => package_sha256, + "source" => "hexpm", + "version" => loaded + } + + locked -> + raise ArgumentError, + "loaded ReqLLM #{loaded} is not the Hex package locked in mix.lock: #{inspect(locked)}" + end end def profiles, do: Map.keys(@provider_profiles) |> Enum.sort() @@ -217,9 +233,6 @@ defmodule Imp.BenchmarkTruth.MultimodalManifest do require_string!(pricing[key], "pricing.#{key}") end ) - - exact_keys!(provider["req_llm_dependency"], @req_llm_dependency_keys, "req_llm_dependency") - require_equal!(provider["req_llm_dependency"], @req_llm_dependency, "req_llm_dependency") end defp validate_signature!(signature) do diff --git a/bench/imp/benchmark_truth/multimodal_runner.ex b/bench/imp/benchmark_truth/multimodal_runner.ex index 81a31ef5..9bd39f56 100644 --- a/bench/imp/benchmark_truth/multimodal_runner.ex +++ b/bench/imp/benchmark_truth/multimodal_runner.ex @@ -15,8 +15,7 @@ defmodule Imp.BenchmarkTruth.MultimodalRunner do root = Keyword.get(opts, :root, File.cwd!()) |> Path.expand() manifest_path = Keyword.get(opts, :manifest, Path.join(root, @default_manifest)) - manifest = Manifest.load!(manifest_path, root: root) - Manifest.runtime_dependency!(manifest.payload) + manifest = manifest_path |> Manifest.load!(root: root) |> Manifest.bind_runtime_dependency!() max_concurrency = validate_concurrency!(Keyword.get(opts, :max_concurrency, 2)) case mode do diff --git a/benchmarks/authorities.json b/benchmarks/authorities.json index a2f27534..dea4db41 100644 --- a/benchmarks/authorities.json +++ b/benchmarks/authorities.json @@ -149,12 +149,12 @@ "req_llm": { "role": "beam_runtime_dependency", "repository": "https://github.com/agentjido/req_llm", - "version": "1.17.1", - "git_ref": "refs/tags/v1.17.1", - "commit": "33840077c2f1332eb6dff2d268dff02393014da4", + "version": "1.24.0", + "git_ref": "refs/tags/v1.24.0", + "commit": "fd9e079fddf253e9b719b2d2c6920f4306592809", "source_hashes": { - "hex_package": "266c0e06c47b4562f243dcdf41332342cbed2ec37064750edd725fb66bb6e914", - "hex_registry": "169d77a77bac5cee7655a7a9158ebea1c2a7857670f4104b29516a25a2d24f1e" + "hex_package": "cdc5c5cd7f38c0e17cf7a949ced99b6760dcba0d860cd4d066fea1aeba422929", + "hex_registry": "8ab6bda68e28afc3adf070cc6616d51831b42b8ff144116a9f104999af4c763f" } } }, diff --git a/benchmarks/data/multimodal/manifest.json b/benchmarks/data/multimodal/manifest.json index d0af3910..0d1c8934 100644 --- a/benchmarks/data/multimodal/manifest.json +++ b/benchmarks/data/multimodal/manifest.json @@ -81,14 +81,6 @@ "source": "https://ai.google.dev/gemini-api/docs/pricing" }, "profile": "google-gemini-2.5-flash-generate-content", - "req_llm_dependency": { - "package": "req_llm", - "package_sha256": "266c0e06c47b4562f243dcdf41332342cbed2ec37064750edd725fb66bb6e914", - "repository": "https://github.com/agentjido/req_llm", - "source": "hexpm", - "source_revision": "33840077c2f1332eb6dff2d268dff02393014da4", - "version": "1.17.1" - }, "req_llm_model": "google:gemini-2.5-flash" }, "samples": [ @@ -159,7 +151,7 @@ "prompt": "Use both pages of the attached synthetic PDF. Among projects with review code R7 on page 2, which owner on page 1 has the highest Units value? Return exactly one JSON object with one field and no markdown: {\"answer\": \"owner\"}" } ], - "schema_version": 2, + "schema_version": 3, "scoring": { "family_thresholds": { "image": 0.75, @@ -180,5 +172,5 @@ "prompt_contract": "One pinned attachment followed by one pinned question; return only a single-field JSON object named answer." } }, - "payload_sha256": "4b4c80497789d74dec60b2b26236f6eee2f1f9b29046f07ef95b3fa3faf8e50f" + "payload_sha256": "54bf6a6ca6bc66c42f927387409c5bc0ab6010a8b7b1c1fc7ef67cbbae0b930e" } diff --git a/benchmarks/data/multimodal/openai-responses-manifest.json b/benchmarks/data/multimodal/openai-responses-manifest.json index 921851e9..cb5ed68c 100644 --- a/benchmarks/data/multimodal/openai-responses-manifest.json +++ b/benchmarks/data/multimodal/openai-responses-manifest.json @@ -79,14 +79,6 @@ "source": "https://developers.openai.com/api/docs/models/gpt-4.1-mini" }, "profile": "openai-gpt-4.1-mini-2025-04-14-responses", - "req_llm_dependency": { - "package": "req_llm", - "package_sha256": "266c0e06c47b4562f243dcdf41332342cbed2ec37064750edd725fb66bb6e914", - "repository": "https://github.com/agentjido/req_llm", - "source": "hexpm", - "source_revision": "33840077c2f1332eb6dff2d268dff02393014da4", - "version": "1.17.1" - }, "req_llm_model": "openai:gpt-4.1-mini-2025-04-14" }, "samples": [ @@ -157,7 +149,7 @@ "prompt": "Use both pages of the attached synthetic PDF. Among projects with review code R7 on page 2, which owner on page 1 has the highest Units value? Return exactly one JSON object with one field and no markdown: {\"answer\": \"owner\"}" } ], - "schema_version": 2, + "schema_version": 3, "scoring": { "family_thresholds": { "image": 0.75, @@ -178,5 +170,5 @@ "prompt_contract": "One pinned attachment followed by one pinned question; return only a single-field JSON object named answer." } }, - "payload_sha256": "04daa155d3f97edfff62e329dfdca1248855f22b2271f7e954228432f1ae39d8" + "payload_sha256": "7e0f46b93d76a0f1703e6edce0276436ac686fc5dd2ad57a0e7a9826776e3a09" } diff --git a/docs/differentials/MULTIMODAL_FIDELITY.md b/docs/differentials/MULTIMODAL_FIDELITY.md index 18024342..fd9eb55a 100644 --- a/docs/differentials/MULTIMODAL_FIDELITY.md +++ b/docs/differentials/MULTIMODAL_FIDELITY.md @@ -39,8 +39,12 @@ The provider manifests are: Their strict, checksummed schema pins every sample ID, prompt, expected output, family, delivery mode, asset byte count, MIME type, asset SHA-256, provider, -endpoint, API, exact model, generation options, ReqLLM dependency, and pricing. -Runtime ReqLLM version drift fails before dispatch. +endpoint, API, exact model, generation options, and pricing. The ReqLLM +dependency is not part of the manifest: the runner reads the loaded ReqLLM +version and its Hex package checksum from `mix.lock`, refuses to run when the +two disagree, and binds that dependency into the checkpoint identity, the +request audits and the artifact. A checkpoint cannot resume under a different +ReqLLM package. All assets are synthetic and repository-owned. Images cover shape counting, spatial relation, and OCR. The native-document family sends the original @@ -57,7 +61,7 @@ observed. For every request, the redacted audit contains: - sanitized endpoint, API, HTTP method, serialized model, and body SHA-256; - ordered serialized part types; - each part's MIME type, decoded byte count, and content SHA-256; -- ReqLLM package source, version, package hash, repository, and source revision; +- ReqLLM package source, version, and package hash; - ReqLLM request ID and detected transport. The audit never persists data URIs, base64 payloads, file bytes, prompt text, or diff --git a/docs/differentials/RESEARCH_LANDSCAPE.md b/docs/differentials/RESEARCH_LANDSCAPE.md index 33466037..4d08b826 100644 --- a/docs/differentials/RESEARCH_LANDSCAPE.md +++ b/docs/differentials/RESEARCH_LANDSCAPE.md @@ -221,7 +221,7 @@ semantics, or produces useful held-out improvement. | [Ax](https://github.com/ax-llm/ax) | Strongest independent implementation comparator for a typed TypeScript interpretation of DSPy-style programming. | Current product-semantic audit: npm `24.0.4`, registry `gitHead` `a366e49759bd596c8217eca91dfdc9dd8382835d`. Historical executable differential: `23.0.0` at `eb5835e54ba0c5b2fbac380daed1cb87faeefd5e`. Use for API and behavioral comparison, not as scientific authority. | | [BAML](https://github.com/BoundaryML/baml) | Comparator for compiler diagnostics, generated typed clients, and partial structured streaming. | Study its contracts; do not add a separate Imp language unless Elixir modules and macros are demonstrably insufficient. | | [AdalFlow](https://github.com/SylphAI-Inc/AdalFlow), [TextGrad](https://github.com/zou-group/textgrad), and [SAMMO](https://github.com/microsoft/sammo) | Comparators for explicit parameter graphs, textual feedback, and structure-aware prompt transformations. | Borrow mechanisms only after pinning code and paper protocols independently. | -| [ReqLLM](https://github.com/agentjido/req_llm) | Preferred BEAM provider substrate. | Release `v1.17.1` resolves to `33840077c2f1332eb6dff2d268dff02393014da4`. Integrate its provider, multimodal, tool, stream, usage, error, and telemetry contracts instead of rebuilding them. | +| [ReqLLM](https://github.com/agentjido/req_llm) | Preferred BEAM provider substrate. | Release `v1.24.0` resolves to `fd9e079fddf253e9b719b2d2c6920f4306592809`. Integrate its provider, multimodal, tool, stream, usage, error, and telemetry contracts instead of rebuilding them. | | [Jido](https://github.com/agentjido/jido) and [Jido AI](https://github.com/agentjido/jido_ai) | Optional deployment and long-running-agent complements. | Study immutable state, explicit effects, supervision, and signals without making their agent model mandatory. | | [LangGraph](https://github.com/langchain-ai/langgraph), [Pydantic AI](https://github.com/pydantic/pydantic-ai), [MLflow](https://mlflow.org/docs/latest/genai/prompt-registry/optimize-prompts), and [Promptfoo](https://github.com/promptfoo/promptfoo) | Production references for checkpoints, durable execution, registries, eval matrices, and adversarial testing. | Treat as operational comparators, not optimizer parity authorities. | diff --git a/lib/imp/clients/req_llm.ex b/lib/imp/clients/req_llm.ex index 407b38c1..34d3a2c2 100644 --- a/lib/imp/clients/req_llm.ex +++ b/lib/imp/clients/req_llm.ex @@ -1070,11 +1070,11 @@ defmodule Imp.Clients.ReqLLM do end end - # ReqLLM attaches its retry step after constructing the Req request and - # resets `max_retries` to 3, so an explicit caller no-retry policy is - # re-applied in a final request step at the adapter boundary, where nothing - # overwrites it. The attempt event fires immediately before the Req adapter - # call, so it counts transports rather than Imp calls. + # An explicit caller no-retry policy is applied again in a final request step + # at the adapter boundary, after every ReqLLM and Req step has run, so no + # later option merge can restore retries. The same step emits the attempt + # event immediately before the Req adapter call, so it counts transports + # rather than Imp calls; campaign budgets read that count. defp enforce_explicit_no_retry(opts) do http_opts = Keyword.get(opts, :req_http_options, []) diff --git a/mix.lock b/mix.lock index c978f0e2..f5d7a1ae 100644 --- a/mix.lock +++ b/mix.lock @@ -9,7 +9,7 @@ "credo": {:hex, :credo, "1.7.19", "cc52129665fc7c15143d47838fda0f9cd6dac9ceced7bf4da6f85fcbfe64b12a", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "2d8bc95d5a7bb99dd2613621d4f08c6a3575c3fd4b62e6a2b48a100352a557b8"}, "decimal": {:hex, :decimal, "3.1.1", "430d87b04011ce6cbd4fd205be758311a81f87d552d40904abd00f015935b1d0", [:mix], [], "hexpm", "c5f25f2ced74a0587d03e6023f595db8e924c9d3922c8c8ffd9edfc4498cf1f6"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, - "dotenvy": {:hex, :dotenvy, "1.1.1", "00e318f3c51de9fafc4b48598447e386f19204dc18ca69886905bb8f8b08b667", [:mix], [], "hexpm", "c8269471b5701e9e56dc86509c1199ded2b33dce088c3471afcfef7839766d8e"}, + "dotenvy": {:hex, :dotenvy, "1.2.1", "43b28d17c996d70ac09f6a6dfd2245378b346ed7290c36452b46cdca77cd96bc", [:mix], [], "hexpm", "20cf780119be89a7cae7808543ebc8b7e56f0993b6aa058260fab686afe9073d"}, "earmark_parser": {:hex, :earmark_parser, "1.4.45", "cba8369ab2a1342e419bc2760eec731b17be828941dcf494045d44766227e1d5", [:mix], [], "hexpm", "d3ec045bf122965db20c0bdb420e19ee1415843135327124918473feb4b328e8"}, "elixir_make": {:hex, :elixir_make, "0.10.0", "16577e2583a79bb79237bbff349619ef5d80afffc07eac6e4faf0d00e2ddaf7d", [:mix], [], "hexpm", "dc1f09fb7fa68866b886abd5f0f3c83553b1a19a52359a899e92af1bb3b31982"}, "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"}, @@ -26,8 +26,8 @@ "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, "jaxon": {:hex, :jaxon, "2.0.8", "00951a79d354260e28d7e36f956c3de94818124768a4b22e0fc55559d1b3bfe7", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "74532853b1126609615ea98f0ceb5009e70465ca98027afbbd8ed314d887e82d"}, "jose": {:hex, :jose, "1.11.12", "06e62b467b61d3726cbc19e9b5489f7549c37993de846dfb3ee8259f9ed208b3", [:mix, :rebar3], [], "hexpm", "31e92b653e9210b696765cdd885437457de1add2a9011d92f8cf63e4641bab7b"}, - "jsv": {:hex, :jsv, "0.21.2", "fb84c2a1bb5884e2b91ae66c5ed84a4db1da2bcfc1df5175c8b33c8791d39353", [:mix], [{:abnf_parsec, "~> 2.0", [hex: :abnf_parsec, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:idna, "~> 6.0 or ~> 7.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:texture, "~> 1.0", [hex: :texture, repo: "hexpm", optional: false]}], "hexpm", "5963f414f9fb37c541dcebfb2df96d07ae4a40d12f52d7ac5e057969e5e7d83a"}, - "llm_db": {:hex, :llm_db, "2026.7.0", "2cd32be2907e72df3d72e57ad99b3963011ceb9418b46a95c4315ce1b0ba5429", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.7", [hex: :toml, repo: "hexpm", optional: false]}, {:zoi, "~> 0.10", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "9ffff8ccf4e3c268a72cbd19e0539cab618ebd3c4bc621a43c48dd2ee8af412f"}, + "jsv": {:hex, :jsv, "0.24.0", "71b521244b51e1849ac7cae986ce391ae13f6ffb781235ad046a4f7ebbb9472b", [:mix], [{:abnf_parsec, "~> 2.0", [hex: :abnf_parsec, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:idna, "~> 6.0 or ~> 7.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:texture, ">= 1.2.1", [hex: :texture, repo: "hexpm", optional: false]}], "hexpm", "a9829510d25fe6e16a84600ba8d2f7c3da5234401b796aa87741393b1c85aa27"}, + "llm_db": {:hex, :llm_db, "2026.9.5", "357a594559c194f65616787961badde1256d5078992639b771437ef3e039d1e5", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.7", [hex: :toml, repo: "hexpm", optional: false]}, {:zoi, "~> 0.10", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "8631c05b435ebedade83421d3d8b72b051af2caee822d953a21384cad95a9714"}, "makeup": {:hex, :makeup, "1.2.2", "882d46dc0905e9ff7abf2aab61a7e6b3dcc555533977d8a23b06019e6c89ac94", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "9a1a24e5b343b8ae16abea0822c10a6f75da27af7fa802ada5251f7579bfccfa"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, @@ -45,18 +45,18 @@ "plug_crypto": {:hex, :plug_crypto, "2.2.0", "144014737daaf485407f5ed77daeaad74d651b216a28c87543f8cc7043f8efc8", [:mix], [], "hexpm", "83a95744ab1c75876542b6fab135fcc176280e0f301a111c1f757fddcec95d2c"}, "ranch": {:hex, :ranch, "2.3.0", "7de7b041a9a6a5091a3aa5898d66c0564be671d87db4f9d63b1b5ee775b097df", [:make, :rebar3], [], "hexpm", "6168ec49409d982f7cfbd83dd083144f6cbe67caa4036551d2f0a3ad67c9d023"}, "req": {:hex, :req, "0.7.4", "23e9ffec17de032a46a4b15ed65c09793893bf4a7c680f4bbf6227fce6bdf74d", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "4b192d63253e8dcc6221ef992ea9ebef7d3555166e8423aa5b553e86bc3c69a2"}, - "req_llm": {:hex, :req_llm, "1.17.1", "266c0e06c47b4562f243dcdf41332342cbed2ec37064750edd725fb66bb6e914", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:ex_aws_auth, "~> 1.4", [hex: :ex_aws_auth, repo: "hexpm", optional: true]}, {:goth, "~> 1.4", [hex: :goth, repo: "hexpm", optional: true]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:jsv, "~> 0.11", [hex: :jsv, repo: "hexpm", optional: false]}, {:llm_db, "~> 2026.7.0", [hex: :llm_db, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:server_sent_events, "~> 1.1.0", [hex: :server_sent_events, repo: "hexpm", optional: false]}, {:splode, "~> 0.3.0", [hex: :splode, repo: "hexpm", optional: false]}, {:websockex, "~> 0.5.1", [hex: :websockex, repo: "hexpm", optional: false]}, {:zoi, "~> 0.14", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "169d77a77bac5cee7655a7a9158ebea1c2a7857670f4104b29516a25a2d24f1e"}, + "req_llm": {:hex, :req_llm, "1.24.0", "cdc5c5cd7f38c0e17cf7a949ced99b6760dcba0d860cd4d066fea1aeba422929", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:ex_aws_auth, "~> 1.4", [hex: :ex_aws_auth, repo: "hexpm", optional: true]}, {:goth, "~> 1.4", [hex: :goth, repo: "hexpm", optional: true]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:jsv, "~> 0.11", [hex: :jsv, repo: "hexpm", optional: false]}, {:llm_db, ">= 2026.9.3 and < 2027.0.0", [hex: :llm_db, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:server_sent_events, "~> 1.1.0", [hex: :server_sent_events, repo: "hexpm", optional: false]}, {:splode, "~> 0.3.0", [hex: :splode, repo: "hexpm", optional: false]}, {:websockex, "~> 0.5.1", [hex: :websockex, repo: "hexpm", optional: false]}, {:zoi, "~> 0.14", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "8ab6bda68e28afc3adf070cc6616d51831b42b8ff144116a9f104999af4c763f"}, "saxy": {:hex, :saxy, "1.6.1", "742eff28f553c066d0b54e84662dbf384a1d1f38595472ed15f6e0a33038bbe1", [:mix], [], "hexpm", "8989d504424ba29460a61950f8968380651413fa05e63b6118084db057da1a6b"}, "server_sent_events": {:hex, :server_sent_events, "1.1.0", "54606238b9182ba673a10ce90bd95f81a0939c1baa91cdf86b66acc04edd1832", [:mix], [], "hexpm", "8e164db8e295a2d869a8faafbf4a1eeaa749b62fe93f66271adff6394d93ce15"}, - "splode": {:hex, :splode, "0.3.1", "9843c54f84f71b7833fec3f0be06c3cfb5be6b35960ee195ea4fad84b1c25030", [:mix], [], "hexpm", "8f2309b6ec2ecbb01435656429ed1d9ed04ba28797a3280c3b0d1217018ecfbd"}, + "splode": {:hex, :splode, "0.3.2", "7716b6b2260a98a6f018c65cc0393da2cdf17314202eb351a0956e8762190dff", [:mix], [], "hexpm", "08fd658f80da7f1cd254b149164dcff8acd44b22f032001d5416b97223d32bc9"}, "stream_data": {:hex, :stream_data, "1.4.0", "026f929db613aabea6208012ae9b8970d3fd5f88b3bdf26831bc536f98c42036", [:mix], [], "hexpm", "2b0ee3a340dcce1c8cf6302a763ee757d1e01c54d6e16d9069062509d68b1dc9"}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, - "texture": {:hex, :texture, "1.2.0", "aa9097e0709652044ba500de817d60ce8a48dc0b4ffd92232fd33459a20282f6", [:mix], [{:abnf_parsec, "~> 2.0", [hex: :abnf_parsec, repo: "hexpm", optional: false]}], "hexpm", "f250de468cfe35f404faf232429f40114f83f73430e0f7086a9cc251865642e2"}, + "texture": {:hex, :texture, "2.0.0", "26d1d2fcb45ddedef200695d4bcc3b994461ddc1f8bba7a3e0d7fa0a7dac55c5", [:mix], [{:abnf_parsec, "~> 2.0", [hex: :abnf_parsec, repo: "hexpm", optional: false]}], "hexpm", "c85cbac5f456f4c9867deb0f70c45daa8a536142a8c8c5403401bd51962660bc"}, "thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, "websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"}, "websockex": {:hex, :websockex, "0.5.1", "9de28d37bbe34f371eb46e29b79c94c94fff79f93c960d842fbf447253558eb4", [:mix], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8ef39576ed56bc3804c9cd8626f8b5d6b5721848d2726c0ccd4f05385a3c9f14"}, "yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"}, "yaml_elixir": {:hex, :yaml_elixir, "2.12.2", "9dd1330fb4cd9a36a7b0f502e5b12486eff632792ee4a5f0eba52a4d4ec32c9c", [:mix], [{:yamerl, "~> 0.10", [hex: :yamerl, repo: "hexpm", optional: false]}], "hexpm", "e7c1b10122f973e6558462d51c39026ba0e14afbc6745318e990ea82cfe9e159"}, - "zoi": {:hex, :zoi, "0.18.5", "fc23f53d5ba6c8639fb30d9a70a80ae8fed774c180c2544bfd16f53bedf95dff", [:mix], [{:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "d69c2e6468752367941d0264e21cf229c1ba8757b2973eea17c12b3cb7d2d4c4"}, + "zoi": {:hex, :zoi, "0.18.9", "3d96b342ec9f4ab9ed4eed22f935ee1027a8be09e1ff3646a0fe06af3c333d07", [:mix], [{:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "267ac4321a27a453a3868c736cc8d1c57aadf4eb415ba045367b00c1975ba254"}, } diff --git a/test/authority_inventory_test.exs b/test/authority_inventory_test.exs index 70585d6f..8acf1bdc 100644 --- a/test/authority_inventory_test.exs +++ b/test/authority_inventory_test.exs @@ -165,7 +165,18 @@ defmodule AuthorityInventoryTest do assert pins["ax_typescript"]["source_manifest"]["file_count"] == 11 assert length(pins["ax_typescript"]["source_paths"]) == 11 assert pins["req_llm"]["role"] == "beam_runtime_dependency" - assert pins["req_llm"]["commit"] == "33840077c2f1332eb6dff2d268dff02393014da4" + assert pins["req_llm"]["commit"] == "fd9e079fddf253e9b719b2d2c6920f4306592809" + + {:hex, :req_llm, locked_version, hex_package, _, _, "hexpm", hex_registry} = + Mix.Dep.Lock.read()[:req_llm] + + assert pins["req_llm"]["version"] == locked_version + assert pins["req_llm"]["git_ref"] == "refs/tags/v" <> locked_version + + assert pins["req_llm"]["source_hashes"] == %{ + "hex_package" => hex_package, + "hex_registry" => hex_registry + } end test "mmGRPO implementation authority is DSPy source and DeepSeekMath is background only" do diff --git a/test/campaign_budget_test.exs b/test/campaign_budget_test.exs index 04429f79..65769bb0 100644 --- a/test/campaign_budget_test.exs +++ b/test/campaign_budget_test.exs @@ -383,7 +383,7 @@ defmodule Imp.BenchmarkTruth.CampaignBudgetTest do 1.0e-12 end - test "ReqLLM transport guard defeats the dependency retry reset and counts one attempt" do + test "ReqLLM transport guard keeps the caller's single attempt and counts it" do parent = self() base_url = @@ -407,8 +407,7 @@ defmodule Imp.BenchmarkTruth.CampaignBudgetTest do max_retries: 0 ) - # Pinned ReqLLM currently overwrites the caller's zero with its default. - assert prepared.options.max_retries == 3 + assert prepared.options.max_retries == 0 {:ok, budget} = CampaignBudget.start_link( diff --git a/test/multimodal_quality_benchmark_test.exs b/test/multimodal_quality_benchmark_test.exs index 9f91e55f..71a3eaa9 100644 --- a/test/multimodal_quality_benchmark_test.exs +++ b/test/multimodal_quality_benchmark_test.exs @@ -189,6 +189,36 @@ defmodule MultimodalQualityBenchmarkTest do end end + test "the campaign records the loaded ReqLLM package and a checkpoint cannot resume under another" do + manifest = Manifest.load!(@openai_manifest) + refute Map.has_key?(manifest.payload["provider"], "req_llm_dependency") + + loaded = Application.spec(:req_llm, :vsn) |> to_string() + {:hex, :req_llm, ^loaded, package_sha256, _, _, "hexpm", _} = Mix.Dep.Lock.read()[:req_llm] + + assert Manifest.runtime_dependency!() == %{ + "package" => "req_llm", + "package_sha256" => package_sha256, + "source" => "hexpm", + "version" => loaded + } + + artifact = Runner.run(mode: :plan, manifest: @openai_manifest, max_concurrency: 1) + assert artifact["provider"]["req_llm_dependency"] == Manifest.runtime_dependency!() + + path = tmp_path("other-req-llm.json") + bindings = Manifest.sample_bindings(manifest.payload) + identity = checkpoint_identity(manifest) + Checkpoint.load!(path, identity, bindings) + + other_build = + put_in(identity, ["provider", "req_llm_dependency", "version"], "0.0.0-other") + + assert_raise ArgumentError, ~r/campaign identity mismatch/, fn -> + Checkpoint.load!(path, other_build, bindings) + end + end + test "duplicate provider response IDs are rejected before checkpoint completion" do {base_url, _request_pid} = start_openai_fixture(response_id: "resp_duplicated") @@ -432,6 +462,7 @@ defmodule MultimodalQualityBenchmarkTest do end defp checkpoint_identity(manifest) do + manifest = Manifest.bind_runtime_dependency!(manifest) provider = manifest.payload["provider"] %{ From d31f01b6c2824756349049a05c172bfa4f8f43cd Mon Sep 17 00:00:00 2001 From: deepfates Date: Wed, 23 Sep 2026 15:06:15 -0700 Subject: [PATCH 2/5] Update earmark_parser, erlexec, ex_doc and mox; drop unused lock entries erlexec 2.5.0 stops kill_group from killing the port program when a child cannot join its requested process group. ex_aws_auth and goth are optional ReqLLM dependencies that nothing here declares; mix deps.unlock --check-unused reported both. dialyxir stays at 1.4.7: 1.4.8 matches ignore entries on the line alone, so every {line, column} entry in .dialyzer_ignore.exs stops matching and the check fails on unused filters. --- mix.lock | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/mix.lock b/mix.lock index f5d7a1ae..e9b9a16a 100644 --- a/mix.lock +++ b/mix.lock @@ -10,17 +10,15 @@ "decimal": {:hex, :decimal, "3.1.1", "430d87b04011ce6cbd4fd205be758311a81f87d552d40904abd00f015935b1d0", [:mix], [], "hexpm", "c5f25f2ced74a0587d03e6023f595db8e924c9d3922c8c8ffd9edfc4498cf1f6"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, "dotenvy": {:hex, :dotenvy, "1.2.1", "43b28d17c996d70ac09f6a6dfd2245378b346ed7290c36452b46cdca77cd96bc", [:mix], [], "hexpm", "20cf780119be89a7cae7808543ebc8b7e56f0993b6aa058260fab686afe9073d"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.45", "cba8369ab2a1342e419bc2760eec731b17be828941dcf494045d44766227e1d5", [:mix], [], "hexpm", "d3ec045bf122965db20c0bdb420e19ee1415843135327124918473feb4b328e8"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.46", "67607a0532e810c6f630a515c548d0b24949643f168cc556303bee4cf96105c7", [:mix], [], "hexpm", "9c44636e8a1c68c62f526b2dcd85d941dbbcee7ab82cf64ba06ce28bef8e89f5"}, "elixir_make": {:hex, :elixir_make, "0.10.0", "16577e2583a79bb79237bbff349619ef5d80afffc07eac6e4faf0d00e2ddaf7d", [:mix], [], "hexpm", "dc1f09fb7fa68866b886abd5f0f3c83553b1a19a52359a899e92af1bb3b31982"}, "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"}, - "erlexec": {:hex, :erlexec, "2.4.1", "73d2c49ecf3709cd7791a0affbef466a11cc1ecf9d679c504edbc9824b35c593", [:rebar3], [], "hexpm", "c92e1776eb1f7eee357a799cf399013dfa672ab50b4f507dfdcdeee11ca89769"}, - "ex_aws_auth": {:hex, :ex_aws_auth, "1.4.1", "2f4faf59dd62933c52faf5c026f22b6b9b7d143cf30ee462204a646e3304761f", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: true]}], "hexpm", "42ea14c586689652f51906c797378a07822b65b32febd22067b4e93ff9dc7d4f"}, - "ex_doc": {:hex, :ex_doc, "0.40.3", "4a972ffe64bc07dc605af487e98fc19b72a4185f55ca031b94c0552d6071c1d9", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "2756e357742fecd9749b489b85d67c9ce99c465f2e75728d9e6dc8d704b973de"}, + "erlexec": {:hex, :erlexec, "2.5.0", "a0c603f3903421db16d6a4f49862f4cb7431d1aa17b4b3d7f3f5e2fe20905121", [:rebar3], [], "hexpm", "b24ff24c5fc084248f3498ca96f739daf820f3db02d9ba1006ab127de14b7215"}, + "ex_doc": {:hex, :ex_doc, "0.40.4", "66f2e42bf588594d5a8aab31cad87f2ddad09d0da1b1a2f379340ec2c2e497cb", [:mix], [{:earmark_parser, "~> 1.4.46", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "6222b9e423d76584ee34df2c82a5ed72c2d53dc153f7f483ad28b378694186cc"}, "ex_json_schema": {:hex, :ex_json_schema, "0.11.5", "ea45f3238be135949dbbbcc9e8eb4682d6e561b8a66374e75d85a2e1d2bd4107", [:mix], [{:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "61ed2a8f07bd115e7ab6d45c147642a8c73b962bc419fadbb248046b9d3d0f20"}, "ex_mcp": {:git, "https://github.com/deepfates/ex_mcp.git", "7285330b490476cc153dd60fb9adac9cd39d4a94", [ref: "7285330b490476cc153dd60fb9adac9cd39d4a94"]}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, "finch": {:hex, :finch, "0.23.0", "e3f9287ac25a8832f848b144c2b57346aac65b205e2e0629a52adfe6507fd837", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.8", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "80e58d3f936f57e3fdf404f83a3642897ae6d9fb642934e46da4d8fe761b99d5"}, - "goth": {:hex, :goth, "1.4.5", "ee37f96e3519bdecd603f20e7f10c758287088b6d77c0147cd5ee68cf224aade", [:mix], [{:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:jose, "~> 1.11", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "0fc2dce5bd710651ed179053d0300ce3a5d36afbdde11e500d57f05f398d5ed5"}, "hpax": {:hex, :hpax, "1.0.4", "777de5d433b0fbdc7c418159c8055910faa8047ffdb3d6b31098d2a46cd7685c", [:mix], [], "hexpm", "afc7cb142ebcc2d01ce7816190b98ce5dd49e799111b24249f3443d730f377ca"}, "idna": {:hex, :idna, "7.1.0", "1067a13043538129602d2f2ce6899d8713125c7d19734aa557ce2e3ea55bd4f1", [:rebar3], [], "hexpm", "6ae959a025bf36df61a8cab8508d9654891b5426a84c44d82deaffd6ddf8c71f"}, "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, @@ -35,7 +33,7 @@ "mint": {:hex, :mint, "1.10.1", "c53e70867cf74017716884d8d33e0742b08b32e9cdb0031cbc69a429dc5555e3", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "0ba2a904605ed8406393444fb8b3356dc58eb59ee6c7fb94ac3f015e1be129e8"}, "mint_web_socket": {:hex, :mint_web_socket, "1.0.6", "5ffcf350df5b90f2d7a04adf877165228804993714592512374218d4679e325a", [:mix], [{:mint, ">= 1.4.1 and < 2.0.0-0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "0c360e9012413f1c115a63532601eb5d63731aab7010949178769760686c1698"}, "mix_audit": {:hex, :mix_audit, "2.1.5", "c0f77cee6b4ef9d97e37772359a187a166c7a1e0e08b50edf5bf6959dfe5a016", [:make, :mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:yaml_elixir, "~> 2.11", [hex: :yaml_elixir, repo: "hexpm", optional: false]}], "hexpm", "87f9298e21da32f697af535475860dc1d3617a010e0b418d2ec6142bc8b42d69"}, - "mox": {:hex, :mox, "1.2.0", "a2cd96b4b80a3883e3100a221e8adc1b98e4c3a332a8fc434c39526babafd5b3", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "c7b92b3cc69ee24a7eeeaf944cd7be22013c52fcb580c1f33f50845ec821089a"}, + "mox": {:hex, :mox, "1.3.2", "f34ca4331b1cce3125609c6de674739fe5f3df0d68df25510d64b6a94b2246de", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "97918a185e727f3128a826f01827b5b8168e1b815bda19eb4192c3ffdb6a8054"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_ownership": {:hex, :nimble_ownership, "1.0.2", "fa8a6f2d8c592ad4d79b2ca617473c6aefd5869abfa02563a77682038bf916cf", [:mix], [], "hexpm", "098af64e1f6f8609c6672127cfe9e9590a5d3fcdd82bc17a377b8692fd81a879"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, From 9d6b2dc62f28fd26974bda761fea32f4f64d3394 Mon Sep 17 00:00:00 2001 From: deepfates Date: Wed, 23 Sep 2026 18:23:34 -0700 Subject: [PATCH 3/5] A deadline bounds the whole model call, retries included Imp.Deadline capped :receive_timeout, which bounds one attempt's wait. ReqLLM retries a timed-out attempt, and waits out a 429's retry-after, each attempt with the full receive timeout, so a call under a deadline could run several times past it. :total_timeout (ReqLLM 1.18+) bounds the call as a whole. --- lib/imp/clients/req_llm.ex | 17 ++++++--- mix.exs | 4 ++- test/req_llm_client_test.exs | 68 ++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/lib/imp/clients/req_llm.ex b/lib/imp/clients/req_llm.ex index 34d3a2c2..dc279505 100644 --- a/lib/imp/clients/req_llm.ex +++ b/lib/imp/clients/req_llm.ex @@ -1051,15 +1051,24 @@ defmodule Imp.Clients.ReqLLM do opts deadline -> - remaining = Imp.Deadline.remaining(deadline) - + # ReqLLM takes neither timeout as zero, and an expired deadline is a + # call that should end at once as a timeout rather than fail option + # validation. + remaining = max(Imp.Deadline.remaining(deadline), 1) + + # :receive_timeout bounds one attempt's wait for the next bytes, and + # ReqLLM retries a timed-out attempt, and a 429 or 529 after its + # retry-after, so that cap alone let one call run to several + # multiples of the time left. :total_timeout is ReqLLM's bound on the + # whole call, retries and their waits included. + # # Cap :connect_options only when the caller supplied it — ReqLLM's # option schema rejects the key, so fabricating it here made every # deadline-bearing call fail validation (GEPA reflection was the - # only such caller and was undrivable live). The :receive_timeout - # cap alone bounds the call end to end. + # only such caller and was undrivable live). opts |> cap_timeout(:receive_timeout, remaining) + |> cap_timeout(:total_timeout, remaining) |> then(fn capped -> if Keyword.has_key?(capped, :connect_options) do Keyword.update!(capped, :connect_options, &cap_timeout(&1, :timeout, remaining)) diff --git a/mix.exs b/mix.exs index 76abf6b4..9dd18d23 100644 --- a/mix.exs +++ b/mix.exs @@ -108,7 +108,9 @@ defmodule Imp.MixProject do {:jsv, "~> 0.21"}, {:nimble_options, "~> 1.1"}, {:req, "~> 0.6"}, - {:req_llm, "~> 1.17"}, + # 1.18 is the first release with :total_timeout, which bounds a call + # under an Imp.Deadline including ReqLLM's retries (Imp.Clients.ReqLLM). + {:req_llm, "~> 1.18"}, {:saxy, "~> 1.6"}, {:telemetry, "~> 1.3"}, {:bandit, "~> 1.0", only: :test}, diff --git a/test/req_llm_client_test.exs b/test/req_llm_client_test.exs index 8df3dc42..a3555ee2 100644 --- a/test/req_llm_client_test.exs +++ b/test/req_llm_client_test.exs @@ -595,6 +595,74 @@ defmodule ReqLLMClientTest do assert Keyword.fetch!(opts, :connect_options)[:timeout] <= 1_000 end + # ReqLLM retries a request that timed out, and a 429 or 529 after the time its + # retry-after header names, each attempt with the full receive timeout. A cap + # on the receive timeout alone therefore lets one call run to several + # multiples of the time left, or to the provider's retry-after however far + # past the deadline that is. The deadline has to bound the call as a whole. + describe "a call under a deadline against a local provider" do + setup do + {:ok, attempts} = Agent.start_link(fn -> 0 end) + %{attempts: attempts} + end + + defp deadline_lm(base_url) do + Imp.req_llm( + %{ + provider: :openai, + id: "deadline-model", + model: "deadline-model", + base_url: base_url <> "/v1" + }, + api_key: "local-test-key", + cache: false + ) + end + + defp call_under_deadline(lm, budget_ms) do + deadline = System.monotonic_time(:millisecond) + budget_ms + + :timer.tc(fn -> + Imp.Deadline.with_deadline({:deadline, deadline}, fn -> + Imp.Clients.ReqLLM.generate(lm, [%{role: :user, content: "hello"}], []) + end) + end) + end + + test "a provider that never answers is given one attempt and the time left", %{ + attempts: attempts + } do + base_url = + Imp.Test.LocalHTTP.start(fn _request -> + Agent.update(attempts, &(&1 + 1)) + Process.sleep(10_000) + {200, %{}} + end) + + {microseconds, result} = call_under_deadline(deadline_lm(base_url), 1_000) + + assert {:error, _reason} = result + assert div(microseconds, 1_000) < 2_500 + assert Agent.get(attempts, & &1) <= 1 + end + + test "a rate-limited provider's retry-after does not outlast the deadline", %{ + attempts: attempts + } do + base_url = + Imp.Test.LocalHTTP.start(fn _request -> + Agent.update(attempts, &(&1 + 1)) + {429, [{"retry-after", "3"}], %{"error" => %{"message" => "rate limited"}}} + end) + + {microseconds, result} = call_under_deadline(deadline_lm(base_url), 1_000) + + assert {:error, _reason} = result + assert div(microseconds, 1_000) < 2_500 + assert Agent.get(attempts, & &1) <= 1 + end + end + # GEPA reflection passes a deadline but no :connect_options. The deadline cap # must not synthesize the key: ReqLLM's option schema rejects one the caller # did not supply, failing validation before the request is made. From 7e46514d634bff3f279efb6115dbfbc02fdccfb8 Mon Sep 17 00:00:00 2001 From: deepfates Date: Wed, 23 Sep 2026 20:21:47 -0700 Subject: [PATCH 4/5] A call under a deadline stays the caller's call With :total_timeout set, ReqLLM runs the call in a task under its own supervisor, not linked to the caller. Three things broke there: - A caller killed mid-call (a cancelled Imp.Run) left the task running. It kept retrying against the provider until its timeouts ran out; against a silent provider, three more requests arrived after the caller was gone. - The campaign budget's usage handler keeps only events emitted by its owner, so usage ReqLLM emitted from the task was dropped: a budgeted call under a deadline recorded 0 input tokens instead of 7. Every GEPA trial runs under a deadline. - The transport attempt event lost the caller's trace, so Imp.trace/2 of a call under a deadline did not contain it. A request step now runs first in that task: it ends the task when the caller goes down, and makes the task emit as the caller (Imp.Telemetry.act_for/2). Handlers that keep only their owner's events ask Imp.Telemetry.emitted_for?/1, which accepts that task but still excludes other processes the owner started. The two ReqLLM client ignore entries in .dialyzer_ignore.exs move to the lines the code now occupies; the previous commit left them stale and dialyzer.check failed on unused filters. --- .dialyzer_ignore.exs | 4 +- .../optimize_anything/campaign.ex | 11 ++- lib/imp/clients/req_llm.ex | 48 ++++++++++++ lib/imp/optimizer/budget.ex | 12 +-- lib/imp/telemetry.ex | 19 +++++ test/model_response_cost_test.exs | 55 +++++++++++++ test/req_llm_client_test.exs | 77 +++++++++++++++++++ 7 files changed, 212 insertions(+), 14 deletions(-) diff --git a/.dialyzer_ignore.exs b/.dialyzer_ignore.exs index 5f88dfa3..ede8f606 100644 --- a/.dialyzer_ignore.exs +++ b/.dialyzer_ignore.exs @@ -57,8 +57,8 @@ # defensive clause/guard: ReqLLM.Response types `usage` as map() on the # struct, but its schema defaults the field to nil and Response.usage/1 is # `map() | nil`, so the nil clause is reachable at runtime. - {"lib/imp/clients/req_llm.ex", :guard_fail, 1357}, - {"lib/imp/clients/req_llm.ex", :pattern_match_cov, {1385, 8}}, + {"lib/imp/clients/req_llm.ex", :guard_fail, 1414}, + {"lib/imp/clients/req_llm.ex", :pattern_match_cov, {1442, 8}}, {"lib/imp/clients/req_llm.ex", :pattern_match_cov, {147, 7}}, # defensive error clause on an always-ok internal call {"lib/imp/clients/training.ex", :pattern_match, {1215, 13}}, diff --git a/bench/imp/benchmark_truth/optimize_anything/campaign.ex b/bench/imp/benchmark_truth/optimize_anything/campaign.ex index 6344b331..29a0ec30 100644 --- a/bench/imp/benchmark_truth/optimize_anything/campaign.ex +++ b/bench/imp/benchmark_truth/optimize_anything/campaign.ex @@ -166,12 +166,11 @@ defmodule Imp.BenchmarkTruth.OptimizeAnything.Campaign do do: raise(ArgumentError, "campaign options must be a keyword list, got: #{inspect(opts)}") @doc false - def handle_usage(_event, _measurements, _metadata, {_agent, owner}) - when owner != self(), - do: :ok - - def handle_usage(event, measurements, metadata, {agent, _owner}), - do: handle_usage(event, measurements, metadata, agent) + def handle_usage(event, measurements, metadata, {agent, owner}) when is_pid(owner) do + if Imp.Telemetry.emitted_for?(owner), + do: handle_usage(event, measurements, metadata, agent), + else: :ok + end def handle_usage(_event, measurements, _metadata, agent) do Agent.update(agent, fn audit -> diff --git a/lib/imp/clients/req_llm.ex b/lib/imp/clients/req_llm.ex index dc279505..f84861ca 100644 --- a/lib/imp/clients/req_llm.ex +++ b/lib/imp/clients/req_llm.ex @@ -312,6 +312,7 @@ defmodule Imp.Clients.ReqLLM do opts |> encode_openrouter_reasoning(lm.model) |> cap_transport_timeouts() + |> bind_to_caller() |> enforce_explicit_no_retry() case lm.req_module.generate_text(lm.model, to_req_messages(messages), opts) do @@ -1079,6 +1080,53 @@ defmodule Imp.Clients.ReqLLM do end end + # ReqLLM runs a call that has a :total_timeout -- every call under a deadline, + # above -- in a task under its own supervisor, not linked to the caller + # (ReqLLM.TimeoutBudget). Three things the call had in the caller's process + # are lost in that task: a caller that dies mid-call, such as a cancelled + # Imp.Run, leaves the task retrying against the provider until its timeouts + # run out; events emitted from it (the transport attempt, ReqLLM's usage) + # lose the caller's trace and span; and handlers that count only the + # caller's own events, such as a campaign budget's usage, drop them. The + # request step below runs first in that task and restores all three: it ends + # the task when the caller goes down, and makes the task emit as the caller + # (`Imp.Telemetry.act_for/2`). It is unnecessary once ReqLLM runs the call + # in the caller's process or ties the task to it. + defp bind_to_caller(opts) do + http_opts = Keyword.get(opts, :req_http_options, []) + + if Keyword.has_key?(opts, :total_timeout) and Keyword.keyword?(http_opts) and + is_list(Keyword.get(http_opts, :plugins, [])) do + step = {__MODULE__, :act_for_caller, [self(), Imp.Telemetry.context()]} + plugin = &Req.Request.prepend_request_steps(&1, imp_act_for_caller: step) + plugins = Keyword.get(http_opts, :plugins, []) ++ [plugin] + Keyword.put(opts, :req_http_options, Keyword.put(http_opts, :plugins, plugins)) + else + opts + end + end + + @doc false + def act_for_caller(%Req.Request{} = request, caller, context) do + worker = self() + + if worker != caller do + Imp.Telemetry.act_for(caller, context) + + spawn(fn -> + caller_down = Process.monitor(caller) + worker_down = Process.monitor(worker) + + receive do + {:DOWN, ^caller_down, :process, _pid, _reason} -> Process.exit(worker, :kill) + {:DOWN, ^worker_down, :process, _pid, _reason} -> :ok + end + end) + end + + request + end + # An explicit caller no-retry policy is applied again in a final request step # at the adapter boundary, after every ReqLLM and Req step has run, so no # later option merge can restore retries. The same step emits the attempt diff --git a/lib/imp/optimizer/budget.ex b/lib/imp/optimizer/budget.ex index fe7c2145..d0893fae 100644 --- a/lib/imp/optimizer/budget.ex +++ b/lib/imp/optimizer/budget.ex @@ -122,12 +122,12 @@ defmodule Imp.Optimizer.Budget do id end - def handle_req_llm_usage_event(_event, _measurements, _metadata, {_server, _id, owner}) - when is_pid(owner) and owner != self(), - do: :ok - - def handle_req_llm_usage_event(event, measurements, metadata, {server, id, _owner}), - do: handle_req_llm_usage_event(event, measurements, metadata, {server, id}) + def handle_req_llm_usage_event(event, measurements, metadata, {server, id, owner}) + when is_pid(owner) do + if Imp.Telemetry.emitted_for?(owner), + do: handle_req_llm_usage_event(event, measurements, metadata, {server, id}), + else: :ok + end @doc false def handle_req_llm_usage_event(_event, measurements, _metadata, {server, id}) diff --git a/lib/imp/telemetry.ex b/lib/imp/telemetry.ex index e97e1733..f963ac6a 100644 --- a/lib/imp/telemetry.ex +++ b/lib/imp/telemetry.ex @@ -11,6 +11,7 @@ defmodule Imp.Telemetry do """ @context_key :imp_telemetry_span_stack + @acting_for_key :imp_telemetry_acting_for @doc """ Emits a redacted telemetry event when the optional `:telemetry` dependency is available. @@ -85,6 +86,24 @@ defmodule Imp.Telemetry do end end + @doc false + # Makes the running process emit as `owner`: under `owner`'s span context, + # and recognized by `emitted_for?/1`. For a process that exists only to make + # one call for `owner`, such as the task ReqLLM runs a call with a + # :total_timeout in (`Imp.Clients.ReqLLM`). + def act_for(owner, context) when is_pid(owner) and is_list(context) do + Process.put(@context_key, context) + Process.put(@acting_for_key, owner) + :ok + end + + @doc false + # Whether the running process is `owner`, or is acting for it. A handler that + # keeps only its owner's events asks this rather than comparing `self()`: + # other processes the owner started are still excluded. + def emitted_for?(owner) when is_pid(owner), + do: owner == self() or Process.get(@acting_for_key) == owner + @doc false def context, do: Process.get(@context_key, []) diff --git a/test/model_response_cost_test.exs b/test/model_response_cost_test.exs index e3da9116..8c07e0fc 100644 --- a/test/model_response_cost_test.exs +++ b/test/model_response_cost_test.exs @@ -185,4 +185,59 @@ defmodule Imp.ModelResponseCostTest do snapshot = Imp.Optimizer.Budget.snapshot(budget) assert snapshot["usage"] == %{"input_tokens" => 3, "output_tokens" => 2, "usd" => 0.001858} end + + # A call under an `Imp.Deadline` carries ReqLLM's `:total_timeout`, and ReqLLM + # runs such a call in a task of its own, so its usage telemetry is emitted from + # that task rather than from the process that made the call. The budget has to + # count it anyway: a campaign whose calls run under a deadline (every GEPA + # trial) otherwise spends without the ceiling seeing any of it. + test "a budgeted call under a deadline records the usage the provider reports" do + assert {:ok, budget} = + Imp.start_optimizer_budget( + limits: %{requests: 2, input_tokens: 10_000, output_tokens: 20, usd: 1.0}, + pricing: %{"input_per_million" => 1.0, "output_per_million" => 2.0}, + default_max_output_tokens: 20 + ) + + base_url = + Imp.Test.LocalHTTP.start(fn _request -> + {200, + %{ + "id" => "usage", + "object" => "chat.completion", + "model" => "usage-model", + "choices" => [ + %{ + "index" => 0, + "finish_reason" => "stop", + "message" => %{"role" => "assistant", "content" => "pong"} + } + ], + "usage" => %{"prompt_tokens" => 7, "completion_tokens" => 3, "total_tokens" => 10} + }} + end) + + inner = + Imp.req_llm( + %{ + provider: :openai, + id: "usage-model", + model: "usage-model", + base_url: base_url <> "/v1" + }, + api_key: "local-test-key", + cache: false + ) + + lm = Imp.budgeted_lm(inner, budget, max_output_tokens: 20) + + assert {:ok, _response} = + Imp.Deadline.with_deadline(5_000, fn -> + Imp.LM.generate(lm, [%{role: :user, content: "ping"}], []) + end) + + usage = Imp.Optimizer.Budget.snapshot(budget)["usage"] + assert usage["input_tokens"] == 7 + assert usage["output_tokens"] == 3 + end end diff --git a/test/req_llm_client_test.exs b/test/req_llm_client_test.exs index a3555ee2..35279347 100644 --- a/test/req_llm_client_test.exs +++ b/test/req_llm_client_test.exs @@ -619,6 +619,20 @@ defmodule ReqLLMClientTest do ) end + defp wait_until(check, tries \\ 200) do + cond do + check.() -> + :ok + + tries == 0 -> + flunk("condition never held") + + true -> + Process.sleep(10) + wait_until(check, tries - 1) + end + end + defp call_under_deadline(lm, budget_ms) do deadline = System.monotonic_time(:millisecond) + budget_ms @@ -661,6 +675,69 @@ defmodule ReqLLMClientTest do assert div(microseconds, 1_000) < 2_500 assert Agent.get(attempts, & &1) <= 1 end + + # The transport attempt is emitted from inside that task too, so it has to + # carry the caller's trace with it or a trace of the call loses it. + test "a traced call keeps its transport attempt" do + base_url = + Imp.Test.LocalHTTP.start(fn _request -> + {200, + %{ + "id" => "traced", + "object" => "chat.completion", + "model" => "deadline-model", + "choices" => [ + %{ + "index" => 0, + "finish_reason" => "stop", + "message" => %{"role" => "assistant", "content" => "pong"} + } + ], + "usage" => %{"prompt_tokens" => 1, "completion_tokens" => 1, "total_tokens" => 2} + }} + end) + + lm = deadline_lm(base_url) + + trace = + Imp.Observability.trace(fn -> + Imp.Deadline.with_deadline(5_000, fn -> + Imp.Clients.ReqLLM.generate(lm, [%{role: :user, content: "ping"}], max_retries: 0) + end) + end) + + assert {:ok, _response} = trace.result + + assert [[:imp, :lm, :start], [:imp, :lm, :transport, :attempt], [:imp, :lm, :stop]] == + Enum.map(trace.events, fn {event, _measurements, _metadata} -> event end) + end + + # ReqLLM runs a call with a :total_timeout in a task it does not link to the + # caller. A caller that dies mid-call -- an `Imp.Run` that is cancelled -- + # must take the request with it, not leave it retrying against the provider + # for as long as the deadline it was built under allows. + test "a caller that dies mid-call takes its request with it", %{attempts: attempts} do + base_url = + Imp.Test.LocalHTTP.start(fn _request -> + Agent.update(attempts, &(&1 + 1)) + Process.sleep(10_000) + {200, %{}} + end) + + lm = deadline_lm(base_url) + + caller = + spawn(fn -> + call_under_deadline(lm, 600) + end) + + wait_until(fn -> Agent.get(attempts, & &1) == 1 end) + Process.exit(caller, :kill) + + # Past the first attempt's receive timeout and a retry's backoff. + Process.sleep(1_500) + assert Agent.get(attempts, & &1) == 1 + end end # GEPA reflection passes a deadline but no :connect_options. The deadline cap From 55beffbab0e82677af5f60be28c02723202d7a2a Mon Sep 17 00:00:00 2001 From: deepfates Date: Wed, 23 Sep 2026 20:21:47 -0700 Subject: [PATCH 5/5] Move the deployment example's lock to req_llm 1.24.0 Imp now requires req_llm ~> 1.18, and this lock still held 1.17.1. CI's differential job runs mix deps.get in examples/deployment, which rewrote the tracked lock, and the hover/papillon pilot tests then refused the dirty tree ("Imp candidate tracked tree is dirty"). This is the lock that deps.get produces. --- examples/deployment/mix.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/examples/deployment/mix.lock b/examples/deployment/mix.lock index 1c8f66af..e1a53342 100644 --- a/examples/deployment/mix.lock +++ b/examples/deployment/mix.lock @@ -5,7 +5,7 @@ "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.20.0", "bb525377ba634cd6d68bac7bff5d98571f738806139d335daca827717c5dc172", [:make, :rebar3], [], "hexpm", "7d41a0dd2c093041ff3779ac5fe8a1585a68ec7cb2dd1de0536bdd2452fd7ba1"}, "decimal": {:hex, :decimal, "3.1.1", "430d87b04011ce6cbd4fd205be758311a81f87d552d40904abd00f015935b1d0", [:mix], [], "hexpm", "c5f25f2ced74a0587d03e6023f595db8e924c9d3922c8c8ffd9edfc4498cf1f6"}, - "dotenvy": {:hex, :dotenvy, "1.1.1", "00e318f3c51de9fafc4b48598447e386f19204dc18ca69886905bb8f8b08b667", [:mix], [], "hexpm", "c8269471b5701e9e56dc86509c1199ded2b33dce088c3471afcfef7839766d8e"}, + "dotenvy": {:hex, :dotenvy, "1.2.1", "43b28d17c996d70ac09f6a6dfd2245378b346ed7290c36452b46cdca77cd96bc", [:mix], [], "hexpm", "20cf780119be89a7cae7808543ebc8b7e56f0993b6aa058260fab686afe9073d"}, "elixir_make": {:hex, :elixir_make, "0.10.0", "16577e2583a79bb79237bbff349619ef5d80afffc07eac6e4faf0d00e2ddaf7d", [:mix], [], "hexpm", "dc1f09fb7fa68866b886abd5f0f3c83553b1a19a52359a899e92af1bb3b31982"}, "erlexec": {:hex, :erlexec, "2.4.1", "73d2c49ecf3709cd7791a0affbef466a11cc1ecf9d679c504edbc9824b35c593", [:rebar3], [], "hexpm", "c92e1776eb1f7eee357a799cf399013dfa672ab50b4f507dfdcdeee11ca89769"}, "ex_json_schema": {:hex, :ex_json_schema, "0.11.5", "ea45f3238be135949dbbbcc9e8eb4682d6e561b8a66374e75d85a2e1d2bd4107", [:mix], [{:decimal, "~> 3.0", [hex: :decimal, repo: "hexpm", optional: false]}], "hexpm", "61ed2a8f07bd115e7ab6d45c147642a8c73b962bc419fadbb248046b9d3d0f20"}, @@ -16,10 +16,10 @@ "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, "jaxon": {:hex, :jaxon, "2.0.8", "00951a79d354260e28d7e36f956c3de94818124768a4b22e0fc55559d1b3bfe7", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "74532853b1126609615ea98f0ceb5009e70465ca98027afbbd8ed314d887e82d"}, "jose": {:hex, :jose, "1.11.12", "06e62b467b61d3726cbc19e9b5489f7549c37993de846dfb3ee8259f9ed208b3", [:mix, :rebar3], [], "hexpm", "31e92b653e9210b696765cdd885437457de1add2a9011d92f8cf63e4641bab7b"}, - "jsv": {:hex, :jsv, "0.21.2", "fb84c2a1bb5884e2b91ae66c5ed84a4db1da2bcfc1df5175c8b33c8791d39353", [:mix], [{:abnf_parsec, "~> 2.0", [hex: :abnf_parsec, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:idna, "~> 6.0 or ~> 7.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:texture, "~> 1.0", [hex: :texture, repo: "hexpm", optional: false]}], "hexpm", "5963f414f9fb37c541dcebfb2df96d07ae4a40d12f52d7ac5e057969e5e7d83a"}, - "llm_db": {:hex, :llm_db, "2026.7.3", "b5d70fcfc0595627ede684bba8acafe1734048812add3359a85fefa456eea358", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.7", [hex: :toml, repo: "hexpm", optional: false]}, {:zoi, "~> 0.10", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "ca9405756152daf36b7b946b9972f52caefd4f165b418a1b22bb907682f47f1a"}, + "jsv": {:hex, :jsv, "0.24.0", "71b521244b51e1849ac7cae986ce391ae13f6ffb781235ad046a4f7ebbb9472b", [:mix], [{:abnf_parsec, "~> 2.0", [hex: :abnf_parsec, repo: "hexpm", optional: false]}, {:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:idna, "~> 6.0 or ~> 7.0", [hex: :idna, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:texture, ">= 1.2.1", [hex: :texture, repo: "hexpm", optional: false]}], "hexpm", "a9829510d25fe6e16a84600ba8d2f7c3da5234401b796aa87741393b1c85aa27"}, + "llm_db": {:hex, :llm_db, "2026.9.5", "357a594559c194f65616787961badde1256d5078992639b771437ef3e039d1e5", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:toml, "~> 0.7", [hex: :toml, repo: "hexpm", optional: false]}, {:zoi, "~> 0.10", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "8631c05b435ebedade83421d3d8b72b051af2caee822d953a21384cad95a9714"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, - "mint": {:hex, :mint, "1.9.3", "3337184d69179695c7a9f1714d92c11e629d36c8c037a21cf490131d3d150554", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "5f7c9342480c069dbbc4eeac3490303c9e01870ff01a7f1d29b6107054fc1e74"}, + "mint": {:hex, :mint, "1.10.1", "c53e70867cf74017716884d8d33e0742b08b32e9cdb0031cbc69a429dc5555e3", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "0ba2a904605ed8406393444fb8b3356dc58eb59ee6c7fb94ac3f015e1be129e8"}, "mint_web_socket": {:hex, :mint_web_socket, "1.0.6", "5ffcf350df5b90f2d7a04adf877165228804993714592512374218d4679e325a", [:mix], [{:mint, ">= 1.4.1 and < 2.0.0-0", [hex: :mint, repo: "hexpm", optional: false]}], "hexpm", "0c360e9012413f1c115a63532601eb5d63731aab7010949178769760686c1698"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, @@ -28,14 +28,14 @@ "plug_cowboy": {:hex, :plug_cowboy, "2.9.0", "87e21e0d9054ced99c36d128f49e3ea2cd8b745fffb97de50bff99706087af4f", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2002bafba4f3a45b55a58e68d70211b153a7ed18d37edb1ceb6e96e7a92c422e"}, "plug_crypto": {:hex, :plug_crypto, "2.2.0", "144014737daaf485407f5ed77daeaad74d651b216a28c87543f8cc7043f8efc8", [:mix], [], "hexpm", "83a95744ab1c75876542b6fab135fcc176280e0f301a111c1f757fddcec95d2c"}, "ranch": {:hex, :ranch, "2.3.0", "7de7b041a9a6a5091a3aa5898d66c0564be671d87db4f9d63b1b5ee775b097df", [:make, :rebar3], [], "hexpm", "6168ec49409d982f7cfbd83dd083144f6cbe67caa4036551d2f0a3ad67c9d023"}, - "req": {:hex, :req, "0.6.3", "7fe5e68792ff0546e45d5919104fa1764a13694cfe3e48c8a0f32ad051ae77e4", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "e85b5c6c990e6c3f52bbba68e6f099118f2b8252825f96c7c3636b97a3de307d"}, - "req_llm": {:hex, :req_llm, "1.17.1", "266c0e06c47b4562f243dcdf41332342cbed2ec37064750edd725fb66bb6e914", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:ex_aws_auth, "~> 1.4", [hex: :ex_aws_auth, repo: "hexpm", optional: true]}, {:goth, "~> 1.4", [hex: :goth, repo: "hexpm", optional: true]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:jsv, "~> 0.11", [hex: :jsv, repo: "hexpm", optional: false]}, {:llm_db, "~> 2026.7.0", [hex: :llm_db, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:server_sent_events, "~> 1.1.0", [hex: :server_sent_events, repo: "hexpm", optional: false]}, {:splode, "~> 0.3.0", [hex: :splode, repo: "hexpm", optional: false]}, {:websockex, "~> 0.5.1", [hex: :websockex, repo: "hexpm", optional: false]}, {:zoi, "~> 0.14", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "169d77a77bac5cee7655a7a9158ebea1c2a7857670f4104b29516a25a2d24f1e"}, + "req": {:hex, :req, "0.7.4", "23e9ffec17de032a46a4b15ed65c09793893bf4a7c680f4bbf6227fce6bdf74d", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "4b192d63253e8dcc6221ef992ea9ebef7d3555166e8423aa5b553e86bc3c69a2"}, + "req_llm": {:hex, :req_llm, "1.24.0", "cdc5c5cd7f38c0e17cf7a949ced99b6760dcba0d860cd4d066fea1aeba422929", [:mix], [{:dotenvy, "~> 1.1", [hex: :dotenvy, repo: "hexpm", optional: false]}, {:ex_aws_auth, "~> 1.4", [hex: :ex_aws_auth, repo: "hexpm", optional: true]}, {:goth, "~> 1.4", [hex: :goth, repo: "hexpm", optional: true]}, {:igniter, "~> 0.7", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:jsv, "~> 0.11", [hex: :jsv, repo: "hexpm", optional: false]}, {:llm_db, ">= 2026.9.3 and < 2027.0.0", [hex: :llm_db, repo: "hexpm", optional: false]}, {:nimble_options, "~> 1.1", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:req, "~> 0.5", [hex: :req, repo: "hexpm", optional: false]}, {:server_sent_events, "~> 1.1.0", [hex: :server_sent_events, repo: "hexpm", optional: false]}, {:splode, "~> 0.3.0", [hex: :splode, repo: "hexpm", optional: false]}, {:websockex, "~> 0.5.1", [hex: :websockex, repo: "hexpm", optional: false]}, {:zoi, "~> 0.14", [hex: :zoi, repo: "hexpm", optional: false]}], "hexpm", "8ab6bda68e28afc3adf070cc6616d51831b42b8ff144116a9f104999af4c763f"}, "saxy": {:hex, :saxy, "1.6.1", "742eff28f553c066d0b54e84662dbf384a1d1f38595472ed15f6e0a33038bbe1", [:mix], [], "hexpm", "8989d504424ba29460a61950f8968380651413fa05e63b6118084db057da1a6b"}, "server_sent_events": {:hex, :server_sent_events, "1.1.0", "54606238b9182ba673a10ce90bd95f81a0939c1baa91cdf86b66acc04edd1832", [:mix], [], "hexpm", "8e164db8e295a2d869a8faafbf4a1eeaa749b62fe93f66271adff6394d93ce15"}, - "splode": {:hex, :splode, "0.3.1", "9843c54f84f71b7833fec3f0be06c3cfb5be6b35960ee195ea4fad84b1c25030", [:mix], [], "hexpm", "8f2309b6ec2ecbb01435656429ed1d9ed04ba28797a3280c3b0d1217018ecfbd"}, + "splode": {:hex, :splode, "0.3.2", "7716b6b2260a98a6f018c65cc0393da2cdf17314202eb351a0956e8762190dff", [:mix], [], "hexpm", "08fd658f80da7f1cd254b149164dcff8acd44b22f032001d5416b97223d32bc9"}, "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, - "texture": {:hex, :texture, "1.2.0", "aa9097e0709652044ba500de817d60ce8a48dc0b4ffd92232fd33459a20282f6", [:mix], [{:abnf_parsec, "~> 2.0", [hex: :abnf_parsec, repo: "hexpm", optional: false]}], "hexpm", "f250de468cfe35f404faf232429f40114f83f73430e0f7086a9cc251865642e2"}, + "texture": {:hex, :texture, "2.0.0", "26d1d2fcb45ddedef200695d4bcc3b994461ddc1f8bba7a3e0d7fa0a7dac55c5", [:mix], [{:abnf_parsec, "~> 2.0", [hex: :abnf_parsec, repo: "hexpm", optional: false]}], "hexpm", "c85cbac5f456f4c9867deb0f70c45daa8a536142a8c8c5403401bd51962660bc"}, "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"}, "websockex": {:hex, :websockex, "0.5.1", "9de28d37bbe34f371eb46e29b79c94c94fff79f93c960d842fbf447253558eb4", [:mix], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8ef39576ed56bc3804c9cd8626f8b5d6b5721848d2726c0ccd4f05385a3c9f14"}, - "zoi": {:hex, :zoi, "0.18.7", "0d6b09d19fd1feff4340b7c5660bab04fbc80c1642ee1e5c75f06d527ac326db", [:mix], [{:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "5fedddd755dec84a5e78b3671070a5e595026aa3479f7fa566a42b8c4e4e5ff2"}, + "zoi": {:hex, :zoi, "0.18.9", "3d96b342ec9f4ab9ed4eed22f935ee1027a8be09e1ff3646a0fe06af3c333d07", [:mix], [{:decimal, "~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "267ac4321a27a453a3868c736cc8d1c57aadf4eb415ba045367b00c1975ba254"}, }