diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2bb2d64..e415e72 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -483,11 +483,21 @@ jobs: print("agentexperience.maf needs the maf extra, as it should") else: raise AssertionError("agentexperience.maf imported without agent-framework-core") + + # Nor does the core import OpenTelemetry: the adapter needs the otel extra, and says so. + try: + import agentexperience.telemetry + except ImportError: + print("agentexperience.telemetry needs the otel extra, as it should") + else: + raise AssertionError("agentexperience.telemetry imported without opentelemetry-api") PY wheels=("$out"/dist/*.whl) test "${#wheels[@]}" -eq 1 || { echo "::error::expected exactly one wheel, found ${#wheels[@]}"; exit 1; } uv pip install --python "$out/venv/bin/python" "${wheels[0]}[maf]" "$out/venv/bin/python" -c "import agentexperience.maf as maf; print('agentexperience.maf imports with the maf extra:', maf.use_agent_experience.__name__)" + uv pip install --python "$out/venv/bin/python" "${wheels[0]}[otel]" + "$out/venv/bin/python" -c "import agentexperience.telemetry as t; print('agentexperience.telemetry imports with the otel extra:', t.enable.__name__)" - name: PostgreSQL ${{ matrix.pg-major }} conformance tests if: ${{ matrix.pg-major }} @@ -515,20 +525,21 @@ jobs: fail-fast: false matrix: include: - # Five modules, each with its own floor: the core (.) Go 1.24; the PostgreSQL store (postgres) and the + # Six modules, each with its own floor: the core (.) Go 1.24; the PostgreSQL store (postgres) and the # in-memory store (inmemory) 1.25 (the pgx and golang.org/x/text releases without known vulnerabilities - # need it); the Agent Framework integration (maf) 1.26, the Go SDK's own minimum; and the PostgreSQL test + # need it); the Agent Framework integration (maf) 1.26, the Go SDK's own minimum; the PostgreSQL test # support with the store's PostgreSQL tests (postgres/pgtest) 1.26, where its container library's - # dependencies have no known vulnerabilities. Each leg runs every module its Go version can build; the + # dependencies have no known vulnerabilities; and the optional OpenTelemetry adapter (telemetry) 1.26, + # which the OpenTelemetry API's own releases need. Each leg runs every module its Go version can build; the # PostgreSQL legs run the two modules with PostgreSQL tests (the postgres build tag). - go-version: '1.24.x' modules: '.' - go-version: '1.25.x' modules: '. postgres inmemory' - go-version: '1.26.x' - modules: '. postgres postgres/pgtest inmemory maf' + modules: '. postgres postgres/pgtest inmemory maf telemetry' - go-version: stable - modules: '. postgres postgres/pgtest inmemory maf' + modules: '. postgres postgres/pgtest inmemory maf telemetry' - go-version: stable modules: 'postgres/pgtest maf' pg-major: 16 @@ -555,6 +566,7 @@ jobs: go/postgres/pgtest/go.sum go/inmemory/go.sum go/maf/go.sum + go/telemetry/go.sum - name: The modules' dependencies are tidy and verified run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5d847ee..e94c85d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,7 +5,7 @@ name: Release # three languages at one version, from one verified commit. The verify job re-runs the release checks on the tagged # commit and uploads what it verified: the six NuGet packages, and the Python sdist and wheel. After a reviewer # approves the `nuget-release` environment, the publish job pushes exactly those NuGet files to nuget.org, then tags -# the five Go modules at the tagged commit and creates the GitHub release. After a reviewer approves the +# the six Go modules at the tagged commit and creates the GitHub release. After a reviewer approves the # `pypi-release` environment, the publish-pypi job uploads exactly the verified Python files to PyPI. # # No API key or token is stored anywhere. The publish job exchanges its GitHub OIDC token for a short-lived nuget.org @@ -100,11 +100,12 @@ jobs: go/inmemory | go/postgres) echo "go" ;; go/postgres/pgtest) echo "go go/postgres" ;; go/maf) echo "go go/inmemory go/postgres go/postgres/pgtest" ;; + go/telemetry) echo "go go/inmemory go/maf" ;; *) echo "::error::No sibling set for the Go module in $1" >&2; return 1 ;; esac } mismatched=0 - for dir in go go/inmemory go/postgres go/postgres/pgtest go/maf; do + for dir in go go/inmemory go/postgres go/postgres/pgtest go/maf go/telemetry; do mod="$dir/go.mod" test -f "$mod" || { echo "::error::$mod is missing"; exit 1; } requires="$(grep -v '=>' "$mod" | sed -nE 's#^[[:space:]]*(require[[:space:]]+)?(github\.com/fabbrik/AgentExperience\.NET/go[^[:space:]]*)[[:space:]]+([^[:space:]]+).*#\2 \3#p')" @@ -318,7 +319,7 @@ jobs: working-directory: go run: | set -euo pipefail - for module in . inmemory postgres postgres/pgtest maf; do + for module in . inmemory postgres postgres/pgtest maf telemetry; do ( cd "$module" go mod verify @@ -435,7 +436,7 @@ jobs: # tag-go-modules:begin set -euo pipefail error="$(mktemp)" - for module in go go/inmemory go/postgres go/postgres/pgtest go/maf; do + for module in go go/inmemory go/postgres go/postgres/pgtest go/maf go/telemetry; do tag="$module/$GO_VERSION" if existing="$(gh api "repos/$GH_REPO/git/ref/tags/$tag" --jq '.object.type + " " + .object.sha' 2> "$error")"; then read -r type sha <<< "$existing" @@ -458,14 +459,14 @@ jobs: # The Go module proxy must know every module at the new version before the release is announced. Asking for a # version makes the proxy fetch it; a module path is case-escaped as the proxy protocol requires (each upper-case # letter becomes ! and the letter in lower case: AgentExperience is !agent!experience). Each module gets ten - # attempts of at most 30 seconds, 30 seconds apart: at worst 9.5 minutes a module, 47.5 for all five, which the + # attempts of at most 30 seconds, 30 seconds apart: at worst 9.5 minutes a module, 57 for all six, which the # job's 75-minute timeout covers with the push to nuget.org. - name: Check the Go module proxy has every module env: GO_VERSION: ${{ needs.verify.outputs.go-version }} run: | set -euo pipefail - for module in go go/inmemory go/postgres go/postgres/pgtest go/maf; do + for module in go go/inmemory go/postgres go/postgres/pgtest go/maf go/telemetry; do path="github.com/fabbrik/AgentExperience.NET/$module" escaped="$(sed -E 's/([A-Z])/!\L\1/g' <<< "$path")" url="https://proxy.golang.org/$escaped/@v/$GO_VERSION.info" diff --git a/CHANGELOG.md b/CHANGELOG.md index 77b3bc8..e09ee55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,23 @@ AgentExperience.NET is a **preview**. It is not production ready, and public API ## Unreleased +### Port telemetry: Python and Go emit what .NET emits (story 23.5) + +- **Python and Go report the same spans and metrics as .NET** for the operations they perform: `inject`, `retrieve`, + `record_injection`, `capture.start_run`, `capture.append_attempt`, `capture.complete_run`, `verify`, `reflect`, + `finalize` and `index`, under the same two sources and meters, span names, three instruments, four dimensions and span + attributes as [the telemetry contract](docs/telemetry.md#the-python-and-go-ports), so one dashboard and one alert serve + a mixed fleet. Neither core imports OpenTelemetry: they call a small observer hook that costs a `None`/`nil` check + when nothing is registered. Python: the optional `otel` extra (`pip install "agenticexperience[otel]"`, the + OpenTelemetry API only), `agentexperience.telemetry.enable()`. Go: the new module `go/telemetry` (the OpenTelemetry API + only), `telemetry.Enable()`; it is released with the other Go modules, now six, and nothing requires it. No + OpenTelemetry SDK or exporter is a dependency, and an observer, exporter or provider that raises is swallowed. +- **A pin for the names.** A .NET test generates [`docs/telemetry-names.json`](docs/telemetry-names.json) from the .NET + definitions (`AGENTEXPERIENCE_TELEMETRY_NAMES_UPDATE=1` rewrites it) and fails on drift; the Python and Go tests load it + and fail when a name or value a port can emit is not in it. No .NET library behaviour or public API changed. +- **Go: `...Context` forms.** `StartRunContext`, `(*RunCapture).AppendAttemptContext`, `CompleteContext` and + `FinalizeContext` take the context the operation's span parents under; the existing forms are unchanged. + ### Contract 1.7: the lessons view in Python and Go (story 23.4) - **Contract 1.7** (add-only: every earlier file is unchanged; [what 1.7 adds](contract/README-1.7.md)) publishes what diff --git a/README.md b/README.md index e5f9c99..e7e8607 100644 --- a/README.md +++ b/README.md @@ -244,9 +244,9 @@ lesson was injected) is published as [a language-neutral contract](contract/READ [what 1.5 adds](contract/README-1.5.md), [what 1.6 adds](contract/README-1.6.md): retrieval by meaning, with embeddings, [what 1.7 adds](contract/README-1.7.md): the lessons view): a JSON Schema, the SQL to run, and fixtures generated from and checked against this implementation on every build. A pure-Python core that passes -contract 1.7 (including retrieval by meaning, with an embedder you register, and a read-only lessons view), with a one-call Python Agent Framework integration, is in [`python/`](python/README.md). A lesson a .NET +contract 1.7 (including retrieval by meaning, with an embedder you register, and a read-only lessons view), with a one-call Python Agent Framework integration and optional [OpenTelemetry telemetry](docs/telemetry.md#the-python-and-go-ports), is in [`python/`](python/README.md). A lesson a .NET agent learned is injected into a Python agent, and the other way round. A Go core and PostgreSQL store that pass contract 1.7 (including retrieval by meaning, with an embedder you register, and a read-only lessons view), with a one-call Go Agent -Framework integration, are in [`go/`](go/README.md). A lesson a .NET agent learned is injected into a Go agent, and +Framework integration and an optional OpenTelemetry module, are in [`go/`](go/README.md). A lesson a .NET agent learned is injected into a Go agent, and the other way round. Both are published with each release, at the same version as the NuGet packages (a Python alpha, `0.1.0aN`, for `0.1.0-preview.N`), and are previews in the same sense: diff --git a/RELEASING.md b/RELEASING.md index 0a1c93a..6ed3c2b 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -26,7 +26,7 @@ The same tag also releases the two ports, at the same version, from the same ver - **Python:** the distribution `agenticexperience` (`python/`), an sdist and a wheel, on PyPI. The first release that carries it is described under [First publish to PyPI](#first-publish-to-pypi). -- **Go:** the five modules under `go/`: `github.com/fabbrik/AgentExperience.NET/go`, `.../go/inmemory`, `.../go/postgres`, `.../go/postgres/pgtest` and `.../go/maf`. A Go module is released by a tag +- **Go:** the six modules under `go/`: `github.com/fabbrik/AgentExperience.NET/go`, `.../go/inmemory`, `.../go/postgres`, `.../go/postgres/pgtest`, `.../go/maf` and `.../go/telemetry` (the optional OpenTelemetry adapter; nothing else requires it). A Go module is released by a tag named after its directory, `go/vX`, `go/inmemory/vX`, and so on, which the Go module proxy serves from then on. Every release from this repository is a **preview** — `0.1.0-preview.N`, set once in `Directory.Build.props` — and @@ -388,11 +388,11 @@ deployments**, select `nuget-release`, and approve. The `publish` job then: 2. exchanges the job's GitHub OIDC token for a short-lived nuget.org key through `NuGet/login`; 3. runs `dotnet nuget push "artifacts/packages/*.nupkg" --source https://api.nuget.org/v3/index.json --api-key "$NUGET_API_KEY" --skip-duplicate`, with the short-lived key from step 2, which uploads each `.snupkg` alongside its `.nupkg`; -4. tags the five Go modules at the tagged commit, through the GitHub API with the job's own token: `go/vX`, - `go/inmemory/vX`, `go/postgres/vX`, `go/postgres/pgtest/vX` and `go/maf/vX`, where `vX` is the tag. A tag that +4. tags the six Go modules at the tagged commit, through the GitHub API with the job's own token: `go/vX`, + `go/inmemory/vX`, `go/postgres/vX`, `go/postgres/pgtest/vX`, `go/maf/vX` and `go/telemetry/vX`, where `vX` is the tag. A tag that already points at that commit is kept; one that points anywhere else fails the job here, before the GitHub release. It then asks `proxy.golang.org` for each module at that version, and fails if the proxy does not know one after - ten attempts, 30 seconds apart (at worst about 48 minutes for all five; the job's timeout is 75 minutes); + ten attempts, 30 seconds apart (at worst about 57 minutes for all six; the job's timeout is 75 minutes); 5. only then creates the GitHub release for the tag: a prerelease when the version has a suffix, with that version's `CHANGELOG.md` section, the Known limits table (or "None currently.") and the Documented boundaries table from `docs/known-limits.md` as its notes. A preview's release notes say what it does not promise. @@ -551,7 +551,7 @@ partial failure: `--check-url` skips the files PyPI already has, and tags alread echo "FAILED: python/dist does not hold agenticexperience $python; run the build above first"; false else uv publish --check-url https://pypi.org/simple/ "python/dist/agenticexperience-$python.tar.gz" "python/dist/agenticexperience-$python-py3-none-any.whl" \ - && for module in go go/inmemory go/postgres go/postgres/pgtest go/maf; do + && for module in go go/inmemory go/postgres go/postgres/pgtest go/maf go/telemetry; do tag="$module/$go" git rev-parse -q --verify "refs/tags/$tag" > /dev/null || git tag "$tag" "$verified" || exit 1 [ "$(git rev-parse "$tag^{commit}")" = "$verified" ] || { echo "FAILED: $tag exists at another commit"; exit 1; } diff --git a/docs/telemetry-names.json b/docs/telemetry-names.json new file mode 100644 index 0000000..068a435 --- /dev/null +++ b/docs/telemetry-names.json @@ -0,0 +1,252 @@ +{ + "generatedFrom": "tests/AgentExperience.MicrosoftAgentFramework.Tests/Diagnostics/TelemetryNamesFileTests.cs", + "sources": [ + "AgentExperience.Core", + "AgentExperience.MicrosoftAgentFramework" + ], + "spanNamePrefix": "agentexperience.", + "faultedOutcome": "Faulted", + "instruments": [ + { + "name": "agentexperience.operation.count", + "kind": "counter", + "unit": "{operation}" + }, + { + "name": "agentexperience.operation.duration", + "kind": "histogram", + "unit": "s" + }, + { + "name": "agentexperience.operation.failures", + "kind": "counter", + "unit": "{failure}" + } + ], + "dimensions": [ + "operation", + "outcome", + "error.class", + "nested" + ], + "errorClasses": [ + "Cancelled", + "Timeout", + "Infrastructure", + "Unexpected" + ], + "attributes": [ + "agentexperience.attempt_id", + "agentexperience.confidence.admission", + "agentexperience.correlation_id", + "agentexperience.error.class", + "agentexperience.event_id", + "agentexperience.experience_id", + "agentexperience.feedback_id", + "agentexperience.history_read", + "agentexperience.independence.refusal", + "agentexperience.omitted_count", + "agentexperience.operation", + "agentexperience.outcome", + "agentexperience.reflection.screening_refusal", + "agentexperience.reflection_id", + "agentexperience.retracted_count", + "agentexperience.reuse_evidence.submitted", + "agentexperience.run_id", + "agentexperience.stage", + "error.type" + ], + "operations": [ + { + "operation": "capture.start_run", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Started", + "Continued", + "Conflict", + "CapacityExceeded" + ] + }, + { + "operation": "capture.append_attempt", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Recorded", + "DuplicateNoOp", + "Conflict", + "SanitizationRejected", + "RunNotFound", + "CapacityExceeded" + ] + }, + { + "operation": "capture.complete_run", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Recorded", + "DuplicateNoOp", + "Conflict", + "RunNotFound" + ] + }, + { + "operation": "verify", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Unknown", + "Verified", + "Failed" + ] + }, + { + "operation": "reflect", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Unknown", + "Verified", + "Failed" + ] + }, + { + "operation": "finalize", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Validated", + "Quarantined", + "AlreadyFinalized", + "StorageDenied", + "NotAuthorized", + "RunNotFound", + "RunNotFinished", + "Failed" + ] + }, + { + "operation": "lifecycle.commit", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Committed", + "TransitionNotAllowed", + "ReplacementNotAllowed", + "StaleRevision", + "StatusMismatch", + "Conflict", + "NotFound", + "Denied", + "Invalid", + "Deleted" + ] + }, + { + "operation": "confidence.apply", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Applied", + "Ineligible", + "StaleRevision", + "StatusMismatch", + "Conflict", + "NotFound", + "Denied", + "Invalid", + "Deleted", + "Unverified" + ] + }, + { + "operation": "retrieve", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Completed", + "TimedOut", + "Denied", + "Failed" + ] + }, + { + "operation": "index", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Indexed", + "Skipped", + "Stale", + "Missing", + "Denied", + "Ineligible", + "ProviderFailed", + "IndexFailed" + ] + }, + { + "operation": "deindex", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Removed", + "NotIndexed", + "Denied", + "Failed" + ] + }, + { + "operation": "reindex", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Completed", + "Denied", + "Invalid", + "Failed" + ] + }, + { + "operation": "reuse_feedback", + "source": "AgentExperience.Core", + "span": true, + "outcomes": [ + "Recorded", + "AlreadyRecorded", + "Conflict", + "Denied", + "Invalid" + ] + }, + { + "operation": "inject", + "source": "AgentExperience.MicrosoftAgentFramework", + "span": true, + "outcomes": [ + "Injected", + "NothingToInject", + "Skipped", + "RetrievalTimedOut", + "RetrievalDenied", + "RetrievalFailed", + "Failed", + "Retracted", + "SessionBudgetExhausted" + ] + }, + { + "operation": "record_injection", + "source": "AgentExperience.MicrosoftAgentFramework", + "span": false, + "outcomes": [ + "Recorded", + "NotTracked", + "NotRecorded", + "OutcomeUnknown" + ] + } + ] +} diff --git a/docs/telemetry.md b/docs/telemetry.md index 7d502f2..e8f7ab4 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -17,7 +17,10 @@ re-labelling a dashboard: - `InjectionTelemetryTests`; - `ErasureTelemetryTests`; - `DiagnosticsAgreementTests` and `ErasureDiagnosticsAgreementTests` (each adapter restates Core's wire names, and - these assert that each copy agrees with Core). + these assert that each copy agrees with Core); +- `TelemetryNamesFileTests`, which generates [`telemetry-names.json`](telemetry-names.json) from the definitions above + (sources, operations with their outcome sets, instruments, dimensions, error classes and attributes) and fails on + drift. The Python and Go ports load that file in their own tests; see [The Python and Go ports](#the-python-and-go-ports). > **Superseded names.** §24 of [`AgentExperience_NET_MAF_Production_Architecture.md`](AgentExperience_NET_MAF_Production_Architecture.md) > is the pre-implementation research proposal. Its span names (`agentexperience.rank`, `agentexperience.persist`, …) @@ -333,6 +336,71 @@ Notes: (`nested=true`). A hung embedding provider inside the post-commit hook therefore reaches `failures` or a `Timeout`-classified span, instead of being absorbed into the outer operation. +## The Python and Go ports + +The Python and Go ports emit the same telemetry for the operations they actually perform, so one dashboard and one +alert rule serve a mixed fleet. Everything above holds for them: the same two source and meter names, the same span +names, the same three instruments with the same units, the same four dimensions with the same value sets, and the same +span attributes. The names and values are pinned against [`telemetry-names.json`](telemetry-names.json), which a .NET +test generates from the definitions listed at the top of this page (`AGENTEXPERIENCE_TELEMETRY_NAMES_UPDATE=1` rewrites +it; the diff is then reviewed and committed on purpose). A Python test and a Go test load it, and fail when a name or +value a port can emit is not in it. + +### Enabling it + +Neither core imports OpenTelemetry. They call a small observer hook (`agentexperience.observer` in Python, +`agentexperience.SetObserver` in Go) that costs one `None`/`nil` check when nothing is registered. An optional adapter +turns the hook into OpenTelemetry, depends on the OpenTelemetry API only, and configures no SDK and no exporter: + +```python +# pip install "agenticexperience[otel]" +from agentexperience import telemetry + +telemetry.enable() # the global tracer and meter providers your application configured +``` + +```go +// go get github.com/fabbrik/AgentExperience.NET/go/telemetry +disable := telemetry.Enable() // the global providers; telemetry.WithTracerProvider / WithMeterProvider override +defer disable() +``` + +An exporter, listener or observer that raises or panics is swallowed and never reaches the agent or changes a result. +Attributes never carry record content, task text, scopes or exception messages; a failing operation records its +exception's (or error's) type name only. `error.type` is the language's own type name (`ValueError`, `errors.errorString`) +where .NET writes the .NET type's full name. + +### What they emit + +A port emits an operation only when it performs what the .NET operation of that name does. Outcomes are verbatim members +of that operation's .NET outcome set, or `Faulted` when the operation failed with an error (an exception in Python, a +returned error in Go). + +| `operation` | Source | Python | Go | Outcomes reached | +| --- | --- | --- | --- | --- | +| `capture.start_run`, `capture.append_attempt`, `capture.complete_run` | Core | `RunCapture.start`, `.append_attempt`, `.complete` | `StartRun`, `AppendAttempt`, `Complete` (and the `...Context` forms) | `Started`; `Recorded`, `DuplicateNoOp`, `Conflict`, `SanitizationRejected`, `CapacityExceeded`; `Recorded`, `Conflict` (Python also `DuplicateNoOp`) | +| `verify`, `reflect` | Core | `aggregate`, `reflect` | `Aggregate`, `Reflect` | `Unknown`, `Verified`, `Failed` | +| `finalize` | Core | `finalize` | `Finalize` | `Validated`, `Quarantined`. It covers verification, reflection, record creation and the initial event; the store write is not a separate `lifecycle.commit` | +| `retrieve` | Core | the text and vector search and the ranking inside an injection | the same | `Completed`, `Failed` (the text search failed) | +| `index` | Core | the background embedding write after a record is stored | the same | `Indexed`, `Stale`, `Missing`, `ProviderFailed`, `IndexFailed` | +| `inject` | MAF | the context provider | the context provider | `Injected`, `NothingToInject`, `Skipped`, `RetrievalFailed`, `Failed` | +| `record_injection` | MAF | the background injection record (measurements only, no span) | the same | `Recorded`, `NotTracked`, `NotRecorded`, `OutcomeUnknown`, with the failure classes of the table above | + +The ports never emit `lifecycle.commit`, `confidence.apply`, `deindex`, `reindex`, `reuse_feedback`, `delete`, +`retention.sweep`, `grant.purge`, `grant.access.purge`, `record.seal` or `record.authorship.backfill`: they do not perform +them. Differences from .NET that a dashboard should know: + +- `nested` is `true` for `verify` and `reflect` inside `finalize`, and `false` for everything else: `inject`, + `record_injection` and `index` are never nested, and the `retrieve` inside an `inject` is on the Core meter, so `false`. + A span's parent is the span that ran it, or the host's current span. +- The ports set only these span attributes beyond the common ones: `run_id`, `attempt_id`, `experience_id`, + `reflection_id`, `stage` (on `finalize`: `Load`, `Evaluate`, `Reflect` or `CreateRecord`, the last stage reached) and + `omitted_count` (on `inject`). They do not write `correlation_id`, `event_id`, `retracted_count`, `history_read`, + `confidence.admission`, `independence.refusal`, `reflection.screening_refusal` or `reuse_evidence.submitted`, because + they have no such thing. +- An `inject` is `Faulted` only when something escapes it (the invocation's own cancellation in Python); every other + failure is a returned outcome, as in .NET. + ## What is not instrumented - **Storage-adapter internals.** Apart from the four erasure and retention operations above, the storage adapter's methods are diff --git a/go/README.md b/go/README.md index 0cfcf7c..7d8de32 100644 --- a/go/README.md +++ b/go/README.md @@ -25,6 +25,7 @@ That brings the core (`.../go`) with it; add `.../go/postgres` or `.../go/inmemo | `.../go/inmemory` (`inmemory/`) | 1.25 | `golang.org/x/text` | An in-memory store, for development | | `.../go/maf` (`maf/`) | 1.26 | [Agent Framework for Go](https://github.com/microsoft/agent-framework-go) v0.1.1 | The one-call Agent Framework integration | | `.../go/postgres/pgtest` (`postgres/pgtest/`) | 1.26 | testcontainers-go, the PostgreSQL store | Test support only: the PostgreSQL server the tests share, and the store's PostgreSQL tests | +| `.../go/telemetry` (`telemetry/`) | 1.26 | the OpenTelemetry API | Optional: reports spans and metrics to OpenTelemetry, as [`docs/telemetry.md`](../docs/telemetry.md#the-python-and-go-ports) documents. Nothing else requires it | The core's `go.mod` has no `require`, and a test checks that no file of it, tests included, imports anything but the standard library. Each other module is its own, so a program pays only for what it imports: the stores' Go 1.25 is @@ -226,6 +227,29 @@ JSON object's top-level members, else the last match in the text, with .NET's `\ the error class's matcher, not Go's RE2). A blank tool name or check ID panics, as an invalid pattern does in `regexp.MustCompile`. +### Telemetry: the same spans and metrics as .NET + +The optional module `.../go/telemetry` (the OpenTelemetry API only; no SDK, no exporter) reports the operations the port +performs to OpenTelemetry, under the names, dimensions and attributes +[`docs/telemetry.md`](../docs/telemetry.md#the-python-and-go-ports) documents for .NET, so one dashboard serves a +mixed fleet: + +```go +import "github.com/fabbrik/AgentExperience.NET/go/telemetry" + +disable := telemetry.Enable() // the global providers your application configured; WithTracerProvider and WithMeterProvider override +defer disable() +``` + +It emits `inject`, `retrieve`, `record_injection`, `capture.start_run`, `capture.append_attempt`, `capture.complete_run`, +`verify`, `reflect`, `finalize` and `index` (the background embedding write): the operations this port performs, and no +other. The core and the integration import no OpenTelemetry: they call the observer hook (`ae.SetObserver`, one atomic +load when nothing is registered), and `go/telemetry` is the adapter. A provider, exporter or observer that fails or +panics is recovered and never reaches the agent. Spans parent under the context you pass: `ae.StartRunContext`, +`(*RunCapture).AppendAttemptContext`, `CompleteContext` and `FinalizeContext` take the context (the integration passes +the invocation's); the forms without one report root spans. Attributes carry identifiers and counts only, never record +content, task text, scopes or error messages. + ### In production The identity must come from your own authentication, never from model output: set `ResolveIdentity`, a function of diff --git a/go/capture.go b/go/capture.go index ef0f5d6..c259662 100644 --- a/go/capture.go +++ b/go/capture.go @@ -2,6 +2,7 @@ package agentexperience import ( "bytes" + "context" "errors" "sort" ) @@ -82,6 +83,28 @@ type RunCapture struct { // be strictly positive. The ToolResult policy must keep the "value" field (allowlisted, not a secret name): one that // dropped it would silently store every result and error as null, so it is refused. func StartRun(run CapturedRun, sanitization Sanitization, limits CaptureLimits) (*RunCapture, error) { + return StartRunContext(context.Background(), run, sanitization, limits) +} + +// StartRunContext is StartRun reporting to the observer (SetObserver) under ctx, so the operation's span is a child of +// the span ctx carries. +func StartRunContext(ctx context.Context, run CapturedRun, sanitization Sanitization, + limits CaptureLimits) (*RunCapture, error) { + _, op := Observe(ctx, "capture.start_run", false) + defer op.End() + if op != nil { + op.Set(AttributeRunID, run.RunID.String()) + } + capture, err := startRun(run, sanitization, limits) + if err != nil { + op.Fail(err) + return nil, err + } + op.Done("Started") + return capture, nil +} + +func startRun(run CapturedRun, sanitization Sanitization, limits CaptureLimits) (*RunCapture, error) { if run.RunID.IsZero() { return nil, errors.New("a run must be named by a non-empty UUID") } @@ -168,10 +191,25 @@ func (c *RunCapture) RecordExposure(exposures ...RunExposure) (AppendOutcome, er // AppendAttempt sanitizes and records one attempt, and returns capture's answer. An attempt with an empty // AttemptID is an error: it could never be told apart from another. func (c *RunCapture) AppendAttempt(attempt RawAttempt) (AppendOutcome, error) { + return c.AppendAttemptContext(context.Background(), attempt) +} + +// AppendAttemptContext is AppendAttempt reporting to the observer (SetObserver) under ctx. +func (c *RunCapture) AppendAttemptContext(ctx context.Context, attempt RawAttempt) (AppendOutcome, error) { + _, op := Observe(ctx, "capture.append_attempt", false) + defer op.End() + if op != nil { + op.Set(AttributeRunID, c.run.RunID.String()) + op.Set(AttributeAttemptID, attempt.AttemptID.String()) + } if attempt.AttemptID.IsZero() { - return "", errors.New("an attempt must be named by a non-empty UUID") + err := errors.New("an attempt must be named by a non-empty UUID") + op.Fail(err) + return "", err } - return c.appendAttempt(attempt), nil + outcome := c.appendAttempt(attempt) + op.Done(string(outcome)) + return outcome, nil } func (c *RunCapture) appendAttempt(attempt RawAttempt) AppendOutcome { @@ -243,19 +281,36 @@ func (c *RunCapture) appendAttempt(attempt RawAttempt) AppendOutcome { // Completed, Failed or Cancelled, and the run cannot end before it started (both are errors). A run completes once: // a second call answers Conflict and changes nothing. func (c *RunCapture) Complete(status RunExecutionStatus, endedAt Instant) (AppendOutcome, error) { + return c.CompleteContext(context.Background(), status, endedAt) +} + +// CompleteContext is Complete reporting to the observer (SetObserver) under ctx. +func (c *RunCapture) CompleteContext(ctx context.Context, status RunExecutionStatus, + endedAt Instant) (AppendOutcome, error) { + _, op := Observe(ctx, "capture.complete_run", false) + defer op.End() + if op != nil { + op.Set(AttributeRunID, c.run.RunID.String()) + } switch status { case RunCompleted, RunFailed, RunCancelled: default: - return "", errors.New("a run ends Completed, Failed or Cancelled") + err := errors.New("a run ends Completed, Failed or Cancelled") + op.Fail(err) + return "", err } if endedAt.Before(c.run.StartedAt) { - return "", errors.New("a run cannot end before it started") + err := errors.New("a run cannot end before it started") + op.Fail(err) + return "", err } if c.run.ExecutionStatus != "" { + op.Done(string(OutcomeConflict)) return OutcomeConflict, nil } c.run.ExecutionStatus = status c.run.EndedAt = &endedAt + op.Done(string(OutcomeRecorded)) return OutcomeRecorded, nil } diff --git a/go/finalize.go b/go/finalize.go index 2fb9eb9..3c19ca1 100644 --- a/go/finalize.go +++ b/go/finalize.go @@ -1,6 +1,7 @@ package agentexperience import ( + "context" "errors" ) @@ -68,6 +69,33 @@ type Verification struct { // Finalize finalizes a finished run: verification, the deterministic reflection (only for a Verified run), the // stored record (Validated with a reflection, else Quarantined) and its initial lifecycle event. func Finalize(run CapturedRun, verification Verification) (FinalizedExperience, error) { + return FinalizeContext(context.Background(), run, verification) +} + +// FinalizeContext is Finalize reporting to the observer (SetObserver) under ctx. Its verification and reflection are +// reported as nested operations of its own. +func FinalizeContext(ctx context.Context, run CapturedRun, verification Verification) (FinalizedExperience, error) { + ctx, op := Observe(ctx, "finalize", false) + defer op.End() + if op != nil { + op.Set(AttributeRunID, run.RunID.String()) + op.Set(AttributeStage, "Load") + } + finalized, err := finalize(ctx, op, run, verification) + if err != nil { + op.Fail(err) + return finalized, err + } + if op != nil { + op.Set(AttributeExperienceID, finalized.Record.ExperienceID.String()) + } + op.Done(string(finalized.Record.Status)) + return finalized, nil +} + +// finalize is the finalization; the stage it has reached is on the operation, also when it fails. +func finalize(ctx context.Context, op *Operation, run CapturedRun, verification Verification) (FinalizedExperience, + error) { if run.ExecutionStatus == "" { return FinalizedExperience{}, errors.New("the run has not finished and cannot be finalized") } @@ -81,7 +109,10 @@ func Finalize(run CapturedRun, verification Verification) (FinalizedExperience, return FinalizedExperience{}, errors.New("the finalization instant must be set") } at := verification.FinalizedAt.TruncateToMicroseconds() - evaluation, err := Aggregate(verification.Evidence, verification.RequiredChecks, verification.ClosedRound, + if op != nil { + op.Set(AttributeStage, "Evaluate") + } + evaluation, err := aggregate(ctx, true, verification.Evidence, verification.RequiredChecks, verification.ClosedRound, verification.CurrentArtifactRevision, at) if err != nil { return FinalizedExperience{}, err @@ -90,7 +121,10 @@ func Finalize(run CapturedRun, verification Verification) (FinalizedExperience, var reflection *Reflection var refusal error if evaluation.Outcome.Status == VerificationVerified { - screened, err := Screen(Reflect(&run, &evaluation, ReflectionIDFor(run.RunID), at)) + if op != nil { + op.Set(AttributeStage, "Reflect") + } + screened, err := Screen(reflectObserved(ctx, true, &run, &evaluation, ReflectionIDFor(run.RunID), at)) if err != nil { refusal = err } else { @@ -98,6 +132,9 @@ func Finalize(run CapturedRun, verification Verification) (FinalizedExperience, } } + if op != nil { + op.Set(AttributeStage, "CreateRecord") + } status := StatusQuarantined confidence, validations := 0.0, int64(0) if reflection != nil { diff --git a/go/maf/capture.go b/go/maf/capture.go index 138b47a..e516658 100644 --- a/go/maf/capture.go +++ b/go/maf/capture.go @@ -70,7 +70,7 @@ func (c *captureScope) captured() bool { } // start opens the run, its task text the invocation's own input. -func (c *captureScope) start(messages []*message.Message) { +func (c *captureScope) start(ctx context.Context, messages []*message.Message) { if c.identity == nil { return } @@ -95,7 +95,7 @@ func (c *captureScope) start(messages []*message.Message) { StartedAt: now, } var capture *ae.RunCapture - capture, err = ae.StartRun(run, s.Sanitization, ae.CaptureLimits{}) + capture, err = ae.StartRunContext(ctx, run, s.Sanitization, ae.CaptureLimits{}) if err != nil { return } @@ -221,7 +221,7 @@ func (c *captureScope) finishWork(ctx context.Context, status ae.RunExecutionSta } ended := s.Clock() - outcome, err := capture.AppendAttempt(ae.RawAttempt{AttemptID: s.NewID(), StartedAt: started, + outcome, err := capture.AppendAttemptContext(ctx, ae.RawAttempt{AttemptID: s.NewID(), StartedAt: started, Duration: elapsed(started, ended), ToolCalls: calls, Result: result, Error: failure}) if err != nil { return report(StageAppendAttempt, "Appending the attempt failed; the run is not stored.", err) @@ -230,9 +230,9 @@ func (c *captureScope) finishWork(ctx context.Context, status ae.RunExecutionSta return report(StageAppendAttempt, "Capture answered "+string(outcome)+"; the run is not stored.", nil) } if !ended.Before(started) { - _, err = capture.Complete(status, ended) + _, err = capture.CompleteContext(ctx, status, ended) } else { - _, err = capture.Complete(status, started) + _, err = capture.CompleteContext(ctx, status, started) } if err != nil { return report(StageCompleteRun, "Completing the run failed; the run is not stored.", err) @@ -266,7 +266,7 @@ func (c *captureScope) finishWork(ctx context.Context, status ae.RunExecutionSta } finalizedAt := s.Clock() - finalized, err := ae.Finalize(run, ae.Verification{ + finalized, err := ae.FinalizeContext(ctx, run, ae.Verification{ ClosedRound: &ae.ClosedVerificationRound{RoundID: roundID, ArtifactRevision: verdict.ArtifactRevision}, RequiredChecks: verdict.RequiredChecks, Evidence: verdict.Evidence, @@ -332,7 +332,7 @@ func (m *captureMiddleware) Run(next agent.RunFunc, ctx context.Context, message scope.report(StageResolveIdentity, "Resolving the identity failed; the invocation runs with no memory.", err) } - scope.start(messages) + scope.start(ctx, messages) ctx = context.WithValue(ctx, scopeKey{}, scope) var response agent.Response diff --git a/go/maf/inject.go b/go/maf/inject.go index 4c87ed9..69419fd 100644 --- a/go/maf/inject.go +++ b/go/maf/inject.go @@ -55,6 +55,8 @@ func (p *ContextProvider) Invoked(context.Context, agent.InvokedContext) error { func (p *ContextProvider) inject(ctx context.Context, messages []*message.Message, options []agent.Option) []*message.Message { + ctx, op := ae.Observe(ctx, "inject", false) + defer op.End() s := p.settings scope := scopeFrom(ctx) captured := scope != nil && scope.settings == s @@ -69,19 +71,23 @@ func (p *ContextProvider) inject(ctx context.Context, messages []*message.Messag identity, err = s.resolveIdentity(ctx, IdentityContext{Agent: running, Session: session}) if err != nil { s.reportInjection(InjectionFailure{Reason: "Resolving the identity failed; no block was added.", Err: err}) + op.Done("Failed") return messages } } if identity == nil { + op.Done("Skipped") return messages } taskText, ok, err := ae.DeriveTaskText(taskMessages(messages), ae.DefaultTaskTextMaxLength, ae.FollowUpThreshold) if err != nil { s.reportInjection(InjectionFailure{Reason: "Deriving the task text failed; no block was added.", Err: err}) + op.Done("Failed") return messages } if !ok { + op.Done("NothingToInject") return messages } parameters := ae.NewSearchParameters(identity.Scope, taskText) @@ -101,11 +107,16 @@ func (p *ContextProvider) inject(ctx context.Context, messages []*message.Messag vectorDone <- vectorAnswer{vectorRows, fallback} }() } - rows, err := s.Store.Search(ctx, identity.Authorization(), parameters) + retrieveCtx, retrieval := ae.Observe(ctx, "retrieve", false) + defer retrieval.End() + rows, err := s.Store.Search(retrieveCtx, identity.Authorization(), parameters) vector := <-vectorDone s.reportVectorFallback(vector.fallback) if err != nil { + // Retrieval reports its failure as its outcome, as .NET does; it is not an error it raised. + retrieval.Done("Failed") s.reportInjection(InjectionFailure{Reason: "The search failed; no block was added.", Err: err}) + op.Done("RetrievalFailed") return messages } // The instant the block is built at: ranking's "now", and the injection's instant. @@ -118,15 +129,23 @@ func (p *ContextProvider) inject(ctx context.Context, messages []*message.Messag ranked, err = ae.Rank(rows, identity.Scope, now, s.PreferredEnvironment, s.ExcludeModelAuthored) } if err != nil { + retrieval.Fail(err) s.reportInjection(InjectionFailure{Reason: "Ranking failed; no block was added.", Err: err}) + op.Done("Failed") return messages } + retrieval.Done("Completed") block, err := ae.RenderBlock(ranked.Ranked, s.BlockOptions) if err != nil { s.reportInjection(InjectionFailure{Reason: "Rendering the block failed; no block was added.", Err: err}) + op.Done("Failed") return messages } + if op != nil { + op.Set(ae.AttributeOmittedCount, len(block.Omitted)) + } if block.IsEmpty() { + op.Done("NothingToInject") return messages } injected := &message.Message{ @@ -165,6 +184,7 @@ func (p *ContextProvider) inject(ctx context.Context, messages []*message.Messag if len(ids) > 0 { p.startRecording(ctx, identity, append([]ae.UUID(nil), ids...), now) } + op.Done("Injected") return out } diff --git a/go/maf/lifecycle_test.go b/go/maf/lifecycle_test.go index ff98bc4..5064027 100644 --- a/go/maf/lifecycle_test.go +++ b/go/maf/lifecycle_test.go @@ -296,7 +296,7 @@ func startedScope(t *testing.T, h *host) *captureScope { identity := s.Identity scope := &captureScope{settings: s, agent: agent.New(agent.ProviderConfig{Run: newScripted(0).run}, agent.Config{}), identity: &identity, reported: map[CaptureStage]bool{}} - scope.start([]*message.Message{message.NewText(ticket)}) + scope.start(context.Background(), []*message.Message{message.NewText(ticket)}) if !scope.captured() { t.Fatal("the run did not start") } diff --git a/go/maf/record.go b/go/maf/record.go index ffde01d..267ee1f 100644 --- a/go/maf/record.go +++ b/go/maf/record.go @@ -114,6 +114,8 @@ func (p *ContextProvider) startRecording(ctx context.Context, identity *ae.Ident s := p.settings recorder, ok := s.Store.(ae.InjectionRecorder) if !ok { + _, op := ae.Observe(ctx, "record_injection", false) + op.Done(string(InjectionNotTracked)) s.reportInjectionRecorded(InjectionRecordResult{Outcome: InjectionNotTracked, ExperienceIDs: ids}) return } @@ -135,10 +137,12 @@ func (p *ContextProvider) record(ctx context.Context, recorder ae.InjectionRecor ids []ae.UUID, injectedAt ae.Instant) { s := p.settings result := InjectionRecordResult{Outcome: InjectionRecorded, ExperienceIDs: ids} + _, op := ae.Observe(ctx, "record_injection", false) defer func() { if recovered := recover(); recovered != nil { result.Outcome, result.Err = InjectionNotRecorded, panicError{recovered} } + op.DoneWithClass(string(result.Outcome), recordErrorClass(result)) s.reportInjectionRecorded(result) }() ctx, cancel := context.WithTimeout(ctx, s.InjectionRecordTimeout) @@ -179,6 +183,34 @@ func (p *ContextProvider) record(ctx context.Context, recorder ae.InjectionRecor } } +// recordErrorClass is the error.class a record_injection that failed counts, as .NET's table has it: a timeout or a +// cancellation is OutcomeUnknown (Timeout, Cancelled); too many earlier writes still running is NotRecorded (Timeout); +// a store that failed is Infrastructure, or Unexpected for anything else; a refusal is an answer and counts nothing. +func recordErrorClass(result InjectionRecordResult) string { + switch result.Outcome { + case InjectionRecorded, InjectionNotTracked: + return "" + case InjectionOutcomeUnknown: + if errors.Is(result.Err, context.Canceled) { + return ae.ErrorClassCancelled + } + return ae.ErrorClassTimeout + } + switch { + case result.Err == nil || errors.Is(result.Err, ae.ErrAuthorizationDenied): + return "" + case errors.Is(result.Err, errTooManyAbandoned): + return ae.ErrorClassTimeout + } + if _, panicked := result.Err.(panicError); panicked { + return ae.ErrorClassUnexpected + } + if class := ae.ClassifyError(result.Err); class == ae.ErrorClassUnexpected { + return class + } + return ae.ErrorClassInfrastructure +} + // reportInjectionRecorded tells OnInjectionRecorded, recovering a panic in it. func (s *settings) reportInjectionRecorded(result InjectionRecordResult) { if s.OnInjectionRecorded == nil { diff --git a/go/maf/vectors.go b/go/maf/vectors.go index 9737bab..c9516e7 100644 --- a/go/maf/vectors.go +++ b/go/maf/vectors.go @@ -266,7 +266,15 @@ func (c *captureScope) writeEmbedding(ctx context.Context, index ae.EmbeddingInd }() ctx, cancel := context.WithTimeout(ctx, s.FinalizationTimeout) defer cancel() + ctx, op := ae.Observe(ctx, "index", false) + defer op.End() + if op != nil { + op.Set(ae.AttributeExperienceID, record.ExperienceID.String()) + } + // What the index reports if it stops early: the provider failed, then (past the embedding) the store did. + stopped := "ProviderFailed" fail := func(err error) { + op.Done(stopped) if errors.Is(err, context.DeadlineExceeded) { c.report(StageIndex, fmt.Sprintf("Embedding the record did not finish within %v.", s.FinalizationTimeout), err) return @@ -282,13 +290,24 @@ func (c *captureScope) writeEmbedding(ctx context.Context, index ae.EmbeddingInd fail(err) return } + stopped = "IndexFailed" write := ae.EmbeddingWrite{ExperienceID: record.ExperienceID, Scope: record.Scope, ModelID: e.modelID, Dimension: len(e.vector), ContentHash: ae.ContentHash(e.modelID, summary), SourceRevision: record.Revision, Vector: e.vector, Now: s.Clock()} - _, err = callBounded(ctx, s.vectorSlots, func(ctx context.Context) (ae.EmbeddingWriteResult, error) { + written, err := callBounded(ctx, s.vectorSlots, func(ctx context.Context) (ae.EmbeddingWriteResult, error) { return index.WriteEmbedding(ctx, c.identity.Authorization(), write) }) if err != nil { fail(err) + return + } + outcome := string(written.Outcome) + switch outcome { + case "Written": + outcome = "Indexed" + case "Stale", "Missing": + default: + outcome = "IndexFailed" // a custom index's own word never becomes a metric label } + op.Done(outcome) } diff --git a/go/observer.go b/go/observer.go new file mode 100644 index 0000000..e248541 --- /dev/null +++ b/go/observer.go @@ -0,0 +1,288 @@ +package agentexperience + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "reflect" + "sort" + "strings" + "sync/atomic" +) + +// The observer hook: the one seam through which the core and the Microsoft Agent Framework integration report what +// they do, for telemetry (docs/telemetry.md). It imports no OpenTelemetry. The module go/telemetry is an adapter that +// turns the hook into spans and metrics; any other Observer can be registered the same way. +// +// Until an Observer is registered, every call here is one atomic load and a nil check, and allocates nothing. An +// observer that panics is recovered and never changes what the caller gets. + +// The sources and meters operations are reported under. +const ( + // SourceCore is the source of the core operations. + SourceCore = "AgentExperience.Core" + // SourceMAF is the source of inject and record_injection. + SourceMAF = "AgentExperience.MicrosoftAgentFramework" +) + +// OutcomeFaulted is the outcome of an operation that failed with an error. +const OutcomeFaulted = "Faulted" + +// The four error.class values. +const ( + ErrorClassCancelled = "Cancelled" + ErrorClassTimeout = "Timeout" + ErrorClassInfrastructure = "Infrastructure" + ErrorClassUnexpected = "Unexpected" +) + +// ErrorClasses are the four error.class values. +func ErrorClasses() []string { + return []string{ErrorClassCancelled, ErrorClassTimeout, ErrorClassInfrastructure, ErrorClassUnexpected} +} + +// OperationSpec is one operation this port performs: where it is reported, whether it has a span, and the outcomes it +// can reach. +type OperationSpec struct { + Source string + Span bool + Outcomes []string +} + +var observedOperations = map[string]OperationSpec{ + "capture.start_run": {SourceCore, true, []string{"Started"}}, + "capture.append_attempt": {SourceCore, true, + []string{"Recorded", "DuplicateNoOp", "Conflict", "SanitizationRejected", "CapacityExceeded"}}, + "capture.complete_run": {SourceCore, true, []string{"Recorded", "Conflict"}}, + "verify": {SourceCore, true, []string{"Unknown", "Verified", "Failed"}}, + "reflect": {SourceCore, true, []string{"Unknown", "Verified", "Failed"}}, + "finalize": {SourceCore, true, []string{"Validated", "Quarantined"}}, + "retrieve": {SourceCore, true, []string{"Completed", "Failed"}}, + "index": {SourceCore, true, + []string{"Indexed", "Stale", "Missing", "ProviderFailed", "IndexFailed"}}, + "inject": {SourceMAF, true, + []string{"Injected", "NothingToInject", "Skipped", "RetrievalFailed", "Failed"}}, + "record_injection": {SourceMAF, false, []string{"Recorded", "NotTracked", "NotRecorded", "OutcomeUnknown"}}, +} + +// ObservedOperations is every operation this port reports, by its operation value. The port reports no other: not +// retention.sweep, delete, grants, seal, backfill, lifecycle, confidence, reuse-feedback, deindex or reindex. A +// new map is returned each time. +func ObservedOperations() map[string]OperationSpec { + result := make(map[string]OperationSpec, len(observedOperations)) + for name, spec := range observedOperations { + spec.Outcomes = append([]string(nil), spec.Outcomes...) + result[name] = spec + } + return result +} + +// ObservedOperation is one operation of ObservedOperations. +func ObservedOperation(name string) (OperationSpec, bool) { + spec, ok := observedOperations[name] + spec.Outcomes = append([]string(nil), spec.Outcomes...) + return spec, ok +} + +// The span attributes this port sets beyond the common ones (agentexperience.operation, .outcome, .error.class and +// error.type, which the adapter writes). +const ( + AttributeRunID = "agentexperience.run_id" + AttributeAttemptID = "agentexperience.attempt_id" + AttributeExperienceID = "agentexperience.experience_id" + AttributeReflectionID = "agentexperience.reflection_id" + AttributeStage = "agentexperience.stage" + AttributeOmittedCount = "agentexperience.omitted_count" +) + +// ObservedAttributes are the span attributes this port sets beyond the common ones, sorted. +func ObservedAttributes() []string { + attributes := []string{AttributeAttemptID, AttributeExperienceID, AttributeOmittedCount, AttributeReflectionID, + AttributeRunID, AttributeStage} + sort.Strings(attributes) + return attributes +} + +// Attribute is one span attribute: a string, an int or a bool. Never record content, task text, scopes or error +// messages. +type Attribute struct { + Key string + Value any +} + +// Observation is what an operation ended as. +type Observation struct { + Source string + Operation string + // Outcome is the operation's own outcome, or OutcomeFaulted when it failed with an error. + Outcome string + // ErrorClass is set when the operation failed (and for a record_injection that failed without raising). + ErrorClass string + // ErrorType is the error's type name and nothing else from it; set only for a failed operation. + ErrorType string + Attributes []Attribute + // Nested reports whether another instrumented operation of the same source called this one. + Nested bool +} + +// Observer receives what the library does. Start runs when an operation begins and returns the context the operation +// runs in (so a span can parent the operations it calls) and a token handed back to Finish. Both may be called from +// any goroutine. +type Observer interface { + Start(ctx context.Context, source, operation string, nested bool) (context.Context, any) + Finish(token any, observation Observation) +} + +type observerBox struct{ observer Observer } + +var registered atomic.Pointer[observerBox] + +// SetObserver registers the observer every operation reports to, or nil to stop reporting. +func SetObserver(observer Observer) { + if observer == nil { + registered.Store(nil) + return + } + registered.Store(&observerBox{observer}) +} + +// CurrentObserver is the registered observer, or nil. +func CurrentObserver() Observer { + if box := registered.Load(); box != nil { + return box.observer + } + return nil +} + +// Operation is one in-flight operation of a registered observer. A nil *Operation (what Observe answers when nothing +// is registered) accepts every method and does nothing. +type Operation struct { + observer Observer + token any + obs Observation + done atomic.Bool +} + +// Observe opens operation (a key of ObservedOperations) for the registered observer and returns the context it runs +// in. With no observer registered it returns ctx and nil, and allocates nothing. nested says another instrumented +// operation of the same source called this one; inject and record_injection are never nested. +func Observe(ctx context.Context, operation string, nested bool) (context.Context, *Operation) { + box := registered.Load() + if box == nil { + return ctx, nil + } + spec := observedOperations[operation] + op := &Operation{observer: box.observer, obs: Observation{Source: spec.Source, Operation: operation, Nested: nested}} + func() { + defer func() { _ = recover() }() + inner, token := box.observer.Start(ctx, spec.Source, operation, nested) + if inner != nil { + ctx = inner + } + op.token = token + }() + return ctx, op +} + +// Set records a span attribute (a string, an int or a bool). +func (o *Operation) Set(key string, value any) { + if o == nil { + return + } + for i := range o.obs.Attributes { + if o.obs.Attributes[i].Key == key { + o.obs.Attributes[i].Value = value + return + } + } + o.obs.Attributes = append(o.obs.Attributes, Attribute{key, value}) +} + +// Done reports the outcome the operation returned. Only the first report counts. +func (o *Operation) Done(outcome string) { o.DoneWithClass(outcome, "") } + +// DoneWithClass reports a returned outcome that also counts a failure (errorClass non-empty), for an operation that +// reports its failure as an outcome instead of an error (record_injection). +func (o *Operation) DoneWithClass(outcome, errorClass string) { + if o == nil || !o.done.CompareAndSwap(false, true) { + return + } + o.obs.Outcome, o.obs.ErrorClass = outcome, errorClass + o.finish() +} + +// Fail reports an operation that failed with err: outcome Faulted, the error's class and its type name only. +func (o *Operation) Fail(err error) { + if o == nil || !o.done.CompareAndSwap(false, true) { + return + } + o.obs.Outcome, o.obs.ErrorClass, o.obs.ErrorType = OutcomeFaulted, ClassifyError(err), errorTypeName(err) + o.finish() +} + +// End closes an operation that reported nothing (a panic unwinding through it, or an exit that forgot to report) as +// Faulted/Unexpected, as .NET closes an unreported injection. Defer it: it does nothing after Done or Fail. +func (o *Operation) End() { + if o == nil || !o.done.CompareAndSwap(false, true) { + return + } + o.obs.Outcome, o.obs.ErrorClass = OutcomeFaulted, ErrorClassUnexpected + o.finish() +} + +func (o *Operation) finish() { + defer func() { _ = recover() }() + o.observer.Finish(o.token, o.obs) +} + +// ClassifyError is the error.class of a failed operation, as .NET classifies: the caller's own cancellation is +// Cancelled, a deadline or timeout is Timeout, a network, file-system or database-driver failure is Infrastructure, +// and anything else Unexpected. +func ClassifyError(err error) string { + switch { + case err == nil: + return ErrorClassUnexpected + case errors.Is(err, context.Canceled): + return ErrorClassCancelled + case errors.Is(err, context.DeadlineExceeded): + return ErrorClassTimeout + } + var timeout interface{ Timeout() bool } + if errors.As(err, &timeout) && timeout.Timeout() { + return ErrorClassTimeout + } + var network net.Error + var pathError *os.PathError + var syscallError *os.SyscallError + if errors.As(err, &network) || errors.As(err, &pathError) || errors.As(err, &syscallError) { + return ErrorClassInfrastructure + } + for current := err; current != nil; current = errors.Unwrap(current) { + // The PostgreSQL driver's errors, recognised by package so the core imports no driver. + if name := reflect.TypeOf(current).String(); strings.HasPrefix(name, "*pgconn.") || + strings.HasPrefix(name, "pgconn.") || strings.HasPrefix(name, "*pgx.") { + return ErrorClassInfrastructure + } + } + return ErrorClassUnexpected +} + +// errorTypeName is an error's type as package path and name, pointers dereferenced, never its message. +func errorTypeName(err error) string { + if err == nil { + return "" + } + t := reflect.TypeOf(err) + for t.Kind() == reflect.Pointer { + t = t.Elem() + } + switch { + case t.Name() == "": + return t.String() + case t.PkgPath() == "": + return t.Name() + } + return fmt.Sprintf("%s.%s", t.PkgPath(), t.Name()) +} diff --git a/go/observer_test.go b/go/observer_test.go new file mode 100644 index 0000000..68e9eb8 --- /dev/null +++ b/go/observer_test.go @@ -0,0 +1,139 @@ +package agentexperience + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "sync" + "testing" +) + +type recordingObserver struct { + mu sync.Mutex + started []string + finished []Observation +} + +func (r *recordingObserver) Start(ctx context.Context, _, operation string, _ bool) (context.Context, any) { + r.mu.Lock() + defer r.mu.Unlock() + r.started = append(r.started, operation) + return ctx, operation +} + +func (r *recordingObserver) Finish(_ any, observation Observation) { + r.mu.Lock() + defer r.mu.Unlock() + r.finished = append(r.finished, observation) +} + +type panickingObserver struct{} + +func (panickingObserver) Start(context.Context, string, string, bool) (context.Context, any) { + panic("start") +} + +func (panickingObserver) Finish(any, Observation) { panic("finish") } + +func TestWithNoObserverTheHookAllocatesNothingAndEveryMethodIsNilSafe(t *testing.T) { + SetObserver(nil) + ctx := context.Background() + allocations := testing.AllocsPerRun(100, func() { + _, op := Observe(ctx, "verify", false) + op.Set(AttributeRunID, "x") + op.Done("Verified") + op.DoneWithClass("NotRecorded", ErrorClassUnexpected) + op.Fail(errors.New("x")) + op.End() + }) + // errors.New above allocates; the hook itself must not. + if allocations > 1 { + t.Fatalf("the hook allocated %v times with no observer", allocations) + } + if _, op := Observe(ctx, "verify", false); op != nil { + t.Fatal("an operation was opened with no observer registered") + } +} + +func TestAnObserverThatPanicsChangesNothing(t *testing.T) { + run := func() (FinalizedExperience, error) { + return Finalize(finishedRun(t), Verification{FinalizedAt: Now(), CurrentArtifactRevision: "r1"}) + } + SetObserver(nil) + want, wantErr := run() + SetObserver(panickingObserver{}) + t.Cleanup(func() { SetObserver(nil) }) + got, gotErr := run() + if fmt.Sprint(wantErr) != fmt.Sprint(gotErr) || got.Record.Status != want.Record.Status || + got.Record.ExperienceID != want.Record.ExperienceID { + t.Fatalf("a panicking observer changed the result: %v / %v", gotErr, got.Record.Status) + } +} + +func TestFinalizeReportsItsNestedVerificationAndAFailedOneAsFaulted(t *testing.T) { + observer := &recordingObserver{} + SetObserver(observer) + t.Cleanup(func() { SetObserver(nil) }) + + if _, err := Finalize(finishedRun(t), Verification{FinalizedAt: Now(), CurrentArtifactRevision: "r1"}); err != nil { + t.Fatal(err) + } + got := map[string]Observation{} + for _, item := range observer.finished { + got[item.Operation] = item + } + if got["finalize"].Outcome != "Quarantined" || got["finalize"].Nested { + t.Fatalf("finalize: %+v", got["finalize"]) + } + if got["verify"].Outcome != "Unknown" || !got["verify"].Nested { + t.Fatalf("verify: %+v", got["verify"]) + } + + observer.finished = nil + if _, err := Finalize(CapturedRun{}, Verification{}); err == nil { + t.Fatal("an unfinished run was finalized") + } + failed := observer.finished[len(observer.finished)-1] + if failed.Operation != "finalize" || failed.Outcome != OutcomeFaulted || failed.ErrorClass != ErrorClassUnexpected || + failed.ErrorType != "errors.errorString" { + t.Fatalf("a failed finalize: %+v", failed) + } +} + +func TestClassificationMirrorsNet(t *testing.T) { + tests := []struct { + err error + want string + }{ + {context.Canceled, ErrorClassCancelled}, + {fmt.Errorf("wrapped: %w", context.DeadlineExceeded), ErrorClassTimeout}, + {&net.OpError{Op: "read", Err: errors.New("reset")}, ErrorClassInfrastructure}, + {&os.PathError{Op: "open", Path: "x", Err: errors.New("gone")}, ErrorClassInfrastructure}, + {errors.New("anything else"), ErrorClassUnexpected}, + } + for _, test := range tests { + if got := ClassifyError(test.err); got != test.want { + t.Errorf("%v: %s, want %s", test.err, got, test.want) + } + } + if got := errorTypeName(fmt.Errorf("secret %d", 1)); got != "fmt.wrapError" && got != "errors.errorString" && + got != "fmt.fmtError" { + // The type name only, never the message. + t.Logf("type name %s", got) + } +} + +func TestObservedOperationsAreTheOnesThePortPerforms(t *testing.T) { + operations := ObservedOperations() + for _, forbidden := range []string{"retention.sweep", "delete", "grant.purge", "grant.access.purge", "record.seal", + "record.authorship.backfill", "lifecycle.commit", "confidence.apply", "reuse_feedback", "deindex", "reindex"} { + if _, ok := operations[forbidden]; ok { + t.Errorf("the port reports %s, which it does not perform", forbidden) + } + } + if len(operations) != 10 { + t.Errorf("%d operations", len(operations)) + } +} diff --git a/go/reflector.go b/go/reflector.go index 470aed1..b04bec8 100644 --- a/go/reflector.go +++ b/go/reflector.go @@ -1,6 +1,7 @@ package agentexperience import ( + "context" "errors" "fmt" "sort" @@ -135,6 +136,23 @@ func bound(items []string, noun string) []string { // Reflect is the deterministic reflection of a finished run. No tool name, argument or error text beyond its class // reaches the lesson. func Reflect(run *CapturedRun, evaluation *VerificationResult, reflectionID UUID, createdAt Instant) Reflection { + return reflectObserved(context.Background(), false, run, evaluation, reflectionID, createdAt) +} + +func reflectObserved(ctx context.Context, nested bool, run *CapturedRun, evaluation *VerificationResult, + reflectionID UUID, createdAt Instant) Reflection { + _, op := Observe(ctx, "reflect", nested) + defer op.End() + if op != nil { + op.Set(AttributeRunID, run.RunID.String()) + op.Set(AttributeReflectionID, reflectionID.String()) + } + reflection := reflectRun(run, evaluation, reflectionID, createdAt) + op.Done(string(evaluation.Outcome.Status)) + return reflection +} + +func reflectRun(run *CapturedRun, evaluation *VerificationResult, reflectionID UUID, createdAt Instant) Reflection { outcome := evaluation.Outcome status := outcome.Status var evidenceIDs []UUID diff --git a/go/telemetry/go.mod b/go/telemetry/go.mod new file mode 100644 index 0000000..15e1ff1 --- /dev/null +++ b/go/telemetry/go.mod @@ -0,0 +1,38 @@ +module github.com/fabbrik/AgentExperience.NET/go/telemetry + +go 1.26.0 + +require ( + github.com/fabbrik/AgentExperience.NET/go v0.1.0-preview.15 + github.com/fabbrik/AgentExperience.NET/go/inmemory v0.1.0-preview.15 + github.com/fabbrik/AgentExperience.NET/go/maf v0.1.0-preview.15 + github.com/microsoft/agent-framework-go v0.1.1 + go.opentelemetry.io/otel v1.47.0 + go.opentelemetry.io/otel/metric v1.47.0 + go.opentelemetry.io/otel/sdk v1.47.0 + go.opentelemetry.io/otel/sdk/metric v1.47.0 + go.opentelemetry.io/otel/trace v1.47.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/jsonschema-go v0.4.3 // indirect + github.com/google/uuid v1.6.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/log v1.47.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect +) + +// The modules of this repository are developed together, so a build here uses their directories. A consumer ignores +// these replaces and gets them at the versions required above: each release tags every module at the same version +// (RELEASING.md). +replace ( + github.com/fabbrik/AgentExperience.NET/go => ../ + github.com/fabbrik/AgentExperience.NET/go/inmemory => ../inmemory + github.com/fabbrik/AgentExperience.NET/go/maf => ../maf + github.com/fabbrik/AgentExperience.NET/go/postgres => ../postgres + github.com/fabbrik/AgentExperience.NET/go/postgres/pgtest => ../postgres/pgtest +) diff --git a/go/telemetry/go.sum b/go/telemetry/go.sum new file mode 100644 index 0000000..c7112e4 --- /dev/null +++ b/go/telemetry/go.sum @@ -0,0 +1,209 @@ +cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= +cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3 h1:qgnWTiWs1G7AqcKZegFvkJD1bmXJcaDU3uC0ELSgZ5U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.3/go.mod h1:XtxD48QPInqoj+GMlcRlXHEYsoOxFnP6w2JgIZ9rDS4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0 h1:Terj1CMW4+NY5CpWjSRkqLoxh2DkYDdv/FH2z0897Uc= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0/go.mod h1:7Ut2Qw9vcwpLW5Q4MBy43uSxXS7tK2yjs94OJ6j5thE= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/a2aproject/a2a-go/v2 v2.6.0 h1:QflB0gHxOP0o5quvA/h9EbYgfOsmRsSmFkeBYtKdwwk= +github.com/a2aproject/a2a-go/v2 v2.6.0/go.mod h1:rvFIb0sBJl2qmJRYU2fQxI6hhIQksmGGMEhT/uRwWww= +github.com/ag-ui-protocol/ag-ui/sdks/community/go v0.0.0-20260312103001-8e7ab1df34c8 h1:gcgM2iVSJ+5v1Li2O2xBm8jdY9bIo0coZdOlXOd0uLU= +github.com/ag-ui-protocol/ag-ui/sdks/community/go v0.0.0-20260312103001-8e7ab1df34c8/go.mod h1:ERAMOexUee4AIuoxksuuGoEcHl3aqLwaazjGwlR9ZCI= +github.com/anthropics/anthropic-sdk-go v1.79.1 h1:R+DGl2fpI8mIg7OGjEEHpr1WN7HOja76sz89Z2DMcqk= +github.com/anthropics/anthropic-sdk-go v1.79.1/go.mod h1:x+lPk/cCl48uRegeP0hlYYBN1b7bEBTveInIMgLicnY= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/buger/jsonparser v1.1.2 h1:frqHqw7otoVbk5M8LlE/L7HTnIq2v9RX6EJ48i9AxJk= +github.com/buger/jsonparser v1.1.2/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/github/copilot-sdk/go v1.0.18 h1:2b6UvLJ5Vp/2z/Jn2mpz0YzlruLY+eTbLia/ntyEhQ8= +github.com/github/copilot-sdk/go v1.0.18/go.mod h1:U0STg9Jdv9LDwtkUtcUJCzPQ4KTnKakRfP+1LhEx248= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/gofrs/flock v0.13.1 h1:jjREztyBeSKBZYAC+mgc1laB+xsgy4kYMf3FbKF2UBo= +github.com/gofrs/flock v0.13.1/go.mod h1:sf4BFiHwnvgxa25DlQoDqXQnwRMEOwqxRq37P6MzzmE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= +github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.15 h1:xolVQTEXusUcAA5UgtyRLjelpFFHWlPQ4XfWGc7MBas= +github.com/googleapis/enterprise-certificate-proxy v0.3.15/go.mod h1:vqVt9yG9480NtzREnTlmGSBmFrA+bzb0yl0TxoBQXOg= +github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= +github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/invopop/jsonschema v0.14.0 h1:MHQqLhvpNUZfw+hM3AZDYK7jxO8FZoQeQM77g8iyZjg= +github.com/invopop/jsonschema v0.14.0/go.mod h1:ygm6C2EaVNMBDPpaPlnOA2pFAxBnxGjFlMZABxm9n2I= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.18.7 h1:aUyZsS4kH3QTKurYhAOwAHxllVPnOthb3vPfnF1Ehjw= +github.com/klauspost/compress v1.18.7/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/microsoft/agent-framework-go v0.1.1 h1:E30HCNYnYliIbJwC8pXoJGFrwo7vbhKYZe+wmrfAzNQ= +github.com/microsoft/agent-framework-go v0.1.1/go.mod h1:vujAnBmMvmFpRZzMWTS9VTsnOB6iC3GBMofGJpaYHzE= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.3.3 h1:OxxR9paxsluYi+zDUEXTTaIxtkK3viymW+Ka7vRhhME= +github.com/moby/go-archive v0.3.3/go.mod h1:Npdv43fFqlhZW7Xo8fbm3ZMYFvAGNviUPqX21VERbcE= +github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc= +github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/sys/user v0.4.1 h1:RgjRlaDKi/Xmyrz4t8lyzXT6v2ooFeO/7xtchmhVWE0= +github.com/moby/sys/user v0.4.1/go.mod h1:E9QsW5WRe1kUAf7kW8hXKwu1uhsZEAdPLYHYSDudF4Y= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/modelcontextprotocol/go-sdk v1.8.0 h1:KIvahhYqwtbeniWVPs3TcXEA7b8jEtwfBpOTAI+Urx4= +github.com/modelcontextprotocol/go-sdk v1.8.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= +github.com/openai/openai-go/v3 v3.74.0 h1:MvAN/lrn8OUhA5KcFBerS86AzKpc+idrQzSvk8lDzTk= +github.com/openai/openai-go/v3 v3.74.0/go.mod h1:+dSPa+nbX+dNoXg1jecMnVpgRP+E/5IBA6Jiz9Pc8WM= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pb33f/ordered-map/v2 v2.3.1 h1:5319HDO0aw4DA4gzi+zv4FXU9UlSs3xGZ40wcP1nBjY= +github.com/pb33f/ordered-map/v2 v2.3.1/go.mod h1:qxFQgd0PkVUtOMCkTapqotNgzRhMPL7VvaHKbd1HnmQ= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 h1:1EYB5IzjZawrrnELUi78f9fPu57HuXjmddZPjrls/28= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.3/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= +github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= +github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= +github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/standard-webhooks/standard-webhooks/libraries v0.0.1 h1:uOfcYT+3QungH6tIGSVCR/Y3KJmgJiHcojJbMTPDZAI= +github.com/standard-webhooks/standard-webhooks/libraries v0.0.1/go.mod h1:L1MQhA6x4dn9r007T033lsaZMv9EmBAdXyU/+EF40fo= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI= +github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE= +github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 h1:8fdv/9y3JMxjQ+ULAcOG8RtgeNu5t9XF9LolSXDuTwM= +github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0/go.mod h1:CFr2LncGYokw+OKjXcr8ARCKG1SaC2UEnGxFBovE86g= +github.com/tidwall/gjson v1.19.0 h1:xwxm7n691Uf3u5OFjzngavjGTh55KX5q/9w9xHW88JU= +github.com/tidwall/gjson v1.19.0/go.mod h1:V37/opeE/JbLUOfH0QTXiNez2l0RUjYUhpT4szFQAfc= +github.com/tidwall/match v1.2.0 h1:0pt8FlkOwjN2fPt4bIl4BoNxb98gGHN2ObFEDkrfZnM= +github.com/tidwall/match v1.2.0/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU= +go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= +go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= +go.opentelemetry.io/otel/log v1.47.0 h1:cOTS1CcLbSQeZKanGJ+0JpF/+t4PELi3O3bbl2lqCcI= +go.opentelemetry.io/otel/log v1.47.0/go.mod h1:9byitSQ5pLC6PpqwGXjqdMKya6ZTswHRZh2vvXT33nw= +go.opentelemetry.io/otel/metric v1.47.0 h1:4PptaldXx3Eat1XjMZ68pPJEs5wrhlemctZE9a3UdWY= +go.opentelemetry.io/otel/metric v1.47.0/go.mod h1:ADGSXxRrXM6bjbvLo535EstVFlPpPYZm4LBKixjDHwU= +go.opentelemetry.io/otel/metric/x v0.69.0 h1:DjRLr15H83v+hCW7JA9NoJvOkYTtmq5YoDRbe9deYpM= +go.opentelemetry.io/otel/metric/x v0.69.0/go.mod h1:uVvsMPMFFyj/HUQfrUnH3JjnOQ1dwFDorgFLRBasM0k= +go.opentelemetry.io/otel/sdk v1.47.0 h1:zWXEr4j2lFefG87TU6Yg8a7ngfohIKFZHKp0Hf5hC6I= +go.opentelemetry.io/otel/sdk v1.47.0/go.mod h1:VUc24kiOeoGsxG8G9ULx3fWKvB7jMhnGE8Oi607lgR0= +go.opentelemetry.io/otel/sdk/metric v1.47.0 h1:lfISg2j93VT6yqdk9OfUaZmw/GfcZqCCV3jdXtsPnKw= +go.opentelemetry.io/otel/sdk/metric v1.47.0/go.mod h1:ypLp+mW1Nt2x+Szt3b5/i1syodyts49lMOwxpDI3VGw= +go.opentelemetry.io/otel/trace v1.47.0 h1:JOjX/Oci8K94QHddo+bbfya/Ai/nf6/dt9ZfrFNWSrM= +go.opentelemetry.io/otel/trace v1.47.0/go.mod h1:jNaSLa2PZEYFG6fRjJABAu+bw4FS08uDmPg28lTghu0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +go.yaml.in/yaml/v4 v4.0.0-rc.2 h1:/FrI8D64VSr4HtGIlUtlFMGsm7H7pWTbj6vOLVZcA6s= +go.yaml.in/yaml/v4 v4.0.0-rc.2/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +google.golang.org/api v0.278.0 h1:W7jiRvRi53VYFfZ/HoZjQBtJk7gOFbHD8ot1RzVZU6E= +google.golang.org/api v0.278.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ= +google.golang.org/genai v1.73.0 h1:QiB89pqDl5xm3fUcY2QTvyQ2kQzSeDuKd3E7vyod8BI= +google.golang.org/genai v1.73.0/go.mod h1:2j40fGpqPPIZNjaDKmjGGjYwCJi6Nm2ofmyP65GnqtY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800 h1:qEHAMpSaUhtD0p3NbEEI83HwNGFxEwaSJ1G9PLnCBZE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260706201446-f0a921348800/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/go/telemetry/telemetry.go b/go/telemetry/telemetry.go new file mode 100644 index 0000000..429ba0b --- /dev/null +++ b/go/telemetry/telemetry.go @@ -0,0 +1,249 @@ +// Package telemetry reports what AgentExperience does to OpenTelemetry: the spans and metrics that docs/telemetry.md +// documents for .NET, under the same source and meter names, span names, instruments, dimensions and attributes, for +// the operations the Go port performs. +// +// The core and the Agent Framework integration only call a small observer hook (agentexperience.SetObserver) and +// import no OpenTelemetry. This module is the optional adapter: it depends on the OpenTelemetry API only, configures +// no SDK and no exporter (the host does, as it does for the .NET library), and is enabled with one call: +// +// disable := telemetry.Enable() // uses the global tracer and meter providers the application configured +// defer disable() +// +// A failure in a provider, an exporter or this adapter is recovered and never reaches the agent. Attributes never +// carry record content, task text, scopes or error messages; a failing operation records its error's type name only. +package telemetry + +import ( + "context" + "runtime/debug" + "sync" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + + ae "github.com/fabbrik/AgentExperience.NET/go" +) + +// The names the adapter writes. Tests pin each against docs/telemetry-names.json, which is generated from the .NET +// definitions. +const ( + SpanNamePrefix = "agentexperience." + OperationCount = "agentexperience.operation.count" + OperationDuration = "agentexperience.operation.duration" + OperationFailures = "agentexperience.operation.failures" + + // The metric dimensions. + DimensionOperation = "operation" + DimensionOutcome = "outcome" + DimensionErrorClass = "error.class" + DimensionNested = "nested" + + // The span attributes the adapter writes itself. + AttributeOperation = "agentexperience.operation" + AttributeOutcome = "agentexperience.outcome" + AttributeErrorClass = "agentexperience.error.class" + AttributeErrorType = "error.type" +) + +// Instruments are the three instruments every meter publishes: name, kind and unit. +func Instruments() [][3]string { + return [][3]string{ + {OperationCount, "counter", "{operation}"}, + {OperationDuration, "histogram", "s"}, + {OperationFailures, "counter", "{failure}"}, + } +} + +// Dimensions are the only metric dimensions. +func Dimensions() []string { + return []string{DimensionOperation, DimensionOutcome, DimensionErrorClass, DimensionNested} +} + +// SpanAttributes is every span attribute the adapter can write. +func SpanAttributes() []string { + return append([]string{AttributeOperation, AttributeOutcome, AttributeErrorClass, AttributeErrorType}, + ae.ObservedAttributes()...) +} + +// Option configures an Observer. +type Option func(*Observer) + +// WithTracerProvider reports spans to tp instead of the global tracer provider. +func WithTracerProvider(tp trace.TracerProvider) Option { return func(o *Observer) { o.tracers = tp } } + +// WithMeterProvider reports metrics to mp instead of the global meter provider. +func WithMeterProvider(mp metric.MeterProvider) Option { return func(o *Observer) { o.meters = mp } } + +// Observer is the agentexperience.Observer that reports to OpenTelemetry. +type Observer struct { + tracers trace.TracerProvider + meters metric.MeterProvider + + mu sync.Mutex + instruments map[string]*instruments +} + +var _ ae.Observer = (*Observer)(nil) + +type instruments struct { + tracer trace.Tracer + count metric.Int64Counter + duration metric.Float64Histogram + failures metric.Int64Counter +} + +// NewObserver is an observer reporting to the global providers, or to the ones the options name. +func NewObserver(options ...Option) *Observer { + o := &Observer{instruments: map[string]*instruments{}} + for _, option := range options { + option(o) + } + return o +} + +// Enable registers an observer built from options with the library and returns the function that unregisters it. +func Enable(options ...Option) (disable func()) { + observer := NewObserver(options...) + ae.SetObserver(observer) + return func() { + if ae.CurrentObserver() == ae.Observer(observer) { + ae.SetObserver(nil) + } + } +} + +func (o *Observer) forSource(source string) (*instruments, error) { + o.mu.Lock() + defer o.mu.Unlock() + if found := o.instruments[source]; found != nil { + return found, nil + } + tracers, meters := o.tracers, o.meters + if tracers == nil { + tracers = otel.GetTracerProvider() + } + if meters == nil { + meters = otel.GetMeterProvider() + } + version := moduleVersion() + meter := meters.Meter(source, metric.WithInstrumentationVersion(version)) + made := &instruments{tracer: tracers.Tracer(source, trace.WithInstrumentationVersion(version))} + var err error + if made.count, err = meter.Int64Counter(OperationCount, metric.WithUnit("{operation}"), + metric.WithDescription("Instrumented Experience operations, by operation and outcome.")); err != nil { + return nil, err + } + if made.duration, err = meter.Float64Histogram(OperationDuration, metric.WithUnit("s"), + metric.WithDescription("How long each instrumented Experience operation took.")); err != nil { + return nil, err + } + if made.failures, err = meter.Int64Counter(OperationFailures, metric.WithUnit("{failure}"), + metric.WithDescription("Instrumented Experience operations that threw.")); err != nil { + return nil, err + } + o.instruments[source] = made + return made, nil +} + +func moduleVersion() string { + info, ok := debug.ReadBuildInfo() + if !ok { + return "" + } + for _, dependency := range info.Deps { + if dependency.Path == "github.com/fabbrik/AgentExperience.NET/go" && dependency.Version != "(devel)" { + return dependency.Version + } + } + return "" +} + +var allowedAttributes = func() map[string]bool { + allowed := map[string]bool{} + for _, name := range ae.ObservedAttributes() { + allowed[name] = true + } + return allowed +}() + +type token struct { + span trace.Span + started time.Time +} + +// Start opens the operation's span (when it has one) as a child of the span ctx carries, and returns a context +// carrying it, so the operations it runs are its children. +func (o *Observer) Start(ctx context.Context, source, operation string, _ bool) (context.Context, any) { + started := time.Now() + spec, known := ae.ObservedOperation(operation) + if !known || !spec.Span { + return ctx, &token{started: started} + } + found, err := o.forSource(source) + if err != nil { + return ctx, nil + } + ctx, span := found.tracer.Start(ctx, SpanNamePrefix+operation, trace.WithSpanKind(trace.SpanKindInternal), + trace.WithAttributes(attribute.String(AttributeOperation, operation))) + return ctx, &token{span: span, started: started} +} + +// Finish records the operation's measurements and closes its span. +func (o *Observer) Finish(value any, observation ae.Observation) { + state, ok := value.(*token) + if !ok || state == nil { + return + } + elapsed := time.Since(state.started).Seconds() + if state.span != nil { + defer state.span.End() + } + found, err := o.forSource(observation.Source) + if err == nil { + dimensions := metric.WithAttributes( + attribute.String(DimensionOperation, observation.Operation), + attribute.String(DimensionOutcome, observation.Outcome), + attribute.Bool(DimensionNested, observation.Nested)) + found.count.Add(context.Background(), 1, dimensions) + found.duration.Record(context.Background(), elapsed, dimensions) + if observation.ErrorClass != "" { + found.failures.Add(context.Background(), 1, metric.WithAttributes( + attribute.String(DimensionOperation, observation.Operation), + attribute.String(DimensionErrorClass, observation.ErrorClass), + attribute.Bool(DimensionNested, observation.Nested))) + } + } + if state.span == nil { + return + } + attributes := []attribute.KeyValue{attribute.String(AttributeOutcome, observation.Outcome)} + for _, item := range observation.Attributes { + if !allowedAttributes[item.Key] { + continue + } + switch value := item.Value.(type) { + case string: + attributes = append(attributes, attribute.String(item.Key, value)) + case int: + attributes = append(attributes, attribute.Int(item.Key, value)) + case bool: + attributes = append(attributes, attribute.Bool(item.Key, value)) + } + } + if observation.ErrorClass != "" { + attributes = append(attributes, attribute.String(AttributeErrorClass, observation.ErrorClass)) + } + if observation.ErrorType != "" { + attributes = append(attributes, attribute.String(AttributeErrorType, observation.ErrorType)) + } + state.span.SetAttributes(attributes...) + if observation.Outcome == ae.OutcomeFaulted { + state.span.SetStatus(codes.Error, "") + } else { + state.span.SetStatus(codes.Ok, "") + } +} diff --git a/go/telemetry/telemetry_test.go b/go/telemetry/telemetry_test.go new file mode 100644 index 0000000..3da1353 --- /dev/null +++ b/go/telemetry/telemetry_test.go @@ -0,0 +1,651 @@ +package telemetry_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "go/parser" + "go/token" + "io/fs" + "iter" + "net" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/microsoft/agent-framework-go/agent" + "github.com/microsoft/agent-framework-go/agent/harness/toolautocall" + "github.com/microsoft/agent-framework-go/message" + "github.com/microsoft/agent-framework-go/tool" + "github.com/microsoft/agent-framework-go/tool/functool" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + + ae "github.com/fabbrik/AgentExperience.NET/go" + "github.com/fabbrik/AgentExperience.NET/go/inmemory" + "github.com/fabbrik/AgentExperience.NET/go/maf" + "github.com/fabbrik/AgentExperience.NET/go/telemetry" +) + +const marker = "SECRET-MARKER-7f3a" + +// ---- docs/telemetry-names.json: generated from the .NET definitions ---------------------------------------------- + +type namesFile struct { + Sources []string `json:"sources"` + SpanNamePrefix string `json:"spanNamePrefix"` + FaultedOutcome string `json:"faultedOutcome"` + Instruments []struct { + Name string `json:"name"` + Kind string `json:"kind"` + Unit string `json:"unit"` + } `json:"instruments"` + Dimensions []string `json:"dimensions"` + ErrorClasses []string `json:"errorClasses"` + Attributes []string `json:"attributes"` + Operations []struct { + Operation string `json:"operation"` + Source string `json:"source"` + Span bool `json:"span"` + Outcomes []string `json:"outcomes"` + } `json:"operations"` +} + +func names(t *testing.T) namesFile { + t.Helper() + data, err := os.ReadFile(filepath.Join("..", "..", "docs", "telemetry-names.json")) + if err != nil { + t.Fatal(err) + } + var file namesFile + if err := json.Unmarshal(data, &file); err != nil { + t.Fatal(err) + } + return file +} + +func (n namesFile) operation(name string) (struct { + Operation string `json:"operation"` + Source string `json:"source"` + Span bool `json:"span"` + Outcomes []string `json:"outcomes"` +}, bool) { + for _, item := range n.Operations { + if item.Operation == name { + return item, true + } + } + return item0, false +} + +var item0 struct { + Operation string `json:"operation"` + Source string `json:"source"` + Span bool `json:"span"` + Outcomes []string `json:"outcomes"` +} + +func TestThePortsTableOfNamesIsInTheNetGeneratedFile(t *testing.T) { + file := names(t) + if !slices.Equal(slices.Sorted(slices.Values([]string{ae.SourceCore, ae.SourceMAF})), slices.Sorted(slices.Values(file.Sources))) { + t.Errorf("sources %v", file.Sources) + } + if telemetry.SpanNamePrefix != file.SpanNamePrefix || ae.OutcomeFaulted != file.FaultedOutcome { + t.Errorf("prefix %q faulted %q", file.SpanNamePrefix, file.FaultedOutcome) + } + if !slices.Equal(ae.ErrorClasses(), file.ErrorClasses) { + t.Errorf("error classes %v", file.ErrorClasses) + } + if !slices.Equal(telemetry.Dimensions(), file.Dimensions) { + t.Errorf("dimensions %v", file.Dimensions) + } + var instruments [][3]string + for _, item := range file.Instruments { + instruments = append(instruments, [3]string{item.Name, item.Kind, item.Unit}) + } + if !slices.Equal(telemetry.Instruments(), instruments) { + t.Errorf("instruments %v", file.Instruments) + } + for _, attribute := range telemetry.SpanAttributes() { + if !slices.Contains(file.Attributes, attribute) { + t.Errorf("the attribute %s is not in the file", attribute) + } + } + for name, spec := range ae.ObservedOperations() { + pinned, ok := file.operation(name) + if !ok { + t.Errorf("the operation %s is not in the file", name) + continue + } + if pinned.Source != spec.Source || pinned.Span != spec.Span { + t.Errorf("%s: source %s span %v, file says %s %v", name, spec.Source, spec.Span, pinned.Source, pinned.Span) + } + for _, outcome := range spec.Outcomes { + if !slices.Contains(pinned.Outcomes, outcome) { + t.Errorf("%s: the outcome %s is not in the file", name, outcome) + } + } + } +} + +// ---- A host: the SDK's in-memory exporters, a scripted model, an in-memory store --------------------------------------- + +type sink struct { + spans *tracetest.InMemoryExporter + reader *sdkmetric.ManualReader +} + +func enable(t *testing.T) *sink { + t.Helper() + spans := tracetest.NewInMemoryExporter() + reader := sdkmetric.NewManualReader() + disable := telemetry.Enable( + telemetry.WithTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSyncer(spans))), + telemetry.WithMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)))) + t.Cleanup(func() { disable(); ae.SetObserver(nil) }) + return &sink{spans: spans, reader: reader} +} + +type point struct { + meter string + name, unit string + attributes map[string]any + value float64 +} + +func (s *sink) points(t *testing.T) []point { + t.Helper() + var data metricdata.ResourceMetrics + if err := s.reader.Collect(context.Background(), &data); err != nil { + t.Fatal(err) + } + var found []point + for _, scope := range data.ScopeMetrics { + for _, m := range scope.Metrics { + add := func(set attribute.Set, value float64) { + attributes := map[string]any{} + for _, kv := range set.ToSlice() { + attributes[string(kv.Key)] = kv.Value.AsInterface() + } + found = append(found, point{scope.Scope.Name, m.Name, m.Unit, attributes, value}) + } + switch data := m.Data.(type) { + case metricdata.Sum[int64]: + for _, p := range data.DataPoints { + add(p.Attributes, float64(p.Value)) + } + case metricdata.Histogram[float64]: + for _, p := range data.DataPoints { + add(p.Attributes, p.Sum) + } + } + } + } + return found +} + +type scriptedModel struct { + mu sync.Mutex + counter int +} + +func (m *scriptedModel) run(_ context.Context, messages []*message.Message, + _ ...agent.Option) iter.Seq2[*agent.ResponseUpdate, error] { + return func(yield func(*agent.ResponseUpdate, error) bool) { + m.mu.Lock() + defer m.mu.Unlock() + lastUser, made := -1, 0 + for i, item := range messages { + if item.Role == message.RoleUser { + lastUser = i + } + } + for _, item := range messages[lastUser+1:] { + for _, content := range item.Contents { + if _, ok := content.(*message.FunctionCallContent); ok { + made++ + } + } + } + var contents message.Contents + if made < 2 { + m.counter++ + contents = message.Contents{&message.FunctionCallContent{CallID: fmt.Sprintf("call-%d", m.counter), + Name: "run_tests", Arguments: "{}"}} + } else { + contents = message.Contents{&message.TextContent{Text: "done"}} + } + yield(&agent.ResponseUpdate{Role: message.RoleAssistant, Contents: contents}, nil) + } +} + +type embedder struct{} + +func (embedder) ModelID() string { return "telemetry-test" } +func (embedder) Dimension() int { return 4 } +func (embedder) Embed(context.Context, string) ([]float32, error) { + return []float32{1, 0, 0, 0}, nil +} + +// brokenSearch is the in-memory store whose search fails; it keeps every optional interface of the store. +type brokenSearch struct{ *inmemory.Store } + +func (brokenSearch) Search(context.Context, ae.AuthorizationBounds, ae.SearchParameters) ([]ae.SearchRow, error) { + return nil, fmt.Errorf("the database is down %s", marker) +} + +// searchOnly is a store that does not record injections. +type searchOnly struct{ inner *inmemory.Store } + +func (s searchOnly) InsertFinalized(ctx context.Context, bounds ae.AuthorizationBounds, finalized *ae.FinalizedExperience, + actor string, recordedAt *time.Time) error { + return s.inner.InsertFinalized(ctx, bounds, finalized, actor, recordedAt) +} + +func (s searchOnly) Search(ctx context.Context, bounds ae.AuthorizationBounds, + parameters ae.SearchParameters) ([]ae.SearchRow, error) { + return s.inner.Search(ctx, bounds, parameters) +} + +// failingRecord is the in-memory store whose injection record fails with a network error. +type failingRecord struct{ *inmemory.Store } + +func (failingRecord) RecordInjection(context.Context, ae.AuthorizationBounds, ae.Scope, []ae.UUID, ae.Instant) error { + return &net.OpError{Op: "write", Err: errors.New("connection reset")} +} + +type suite struct { + Suite string `json:"suite,omitempty"` +} + +type outcomes struct { + mu sync.Mutex + injected []int + recorded []maf.InjectionRecordOutcome + finalized int +} + +func newStore(t *testing.T) *inmemory.Store { + t.Helper() + store, err := inmemory.New() + if err != nil { + t.Fatal(err) + } + return store +} + +// drive runs two tickets through an agent wired with the integration and waits for the background work. +func drive(t *testing.T, st ae.Store, options maf.Options, first, second string) *outcomes { + t.Helper() + seen := &outcomes{} + runs := 0 + runTests := functool.MustNew(functool.Config{Name: "run_tests", Description: "Runs the tests."}, + func(context.Context, suite) (string, error) { + runs++ + if runs%2 == 1 { + return "exit=3 1 test failed", nil + } + return "exit=0 all tests passed", nil + }) + identity, err := ae.DevelopmentIdentity(ae.DevelopmentOptions{Application: "telemetry", EnvironmentName: ae.Ptr("Development")}) + if err != nil { + t.Fatal(err) + } + config := agent.Config{Name: "triage-ticket", Tools: []tool.Tool{runTests}} + options.Store, options.Identity = st, identity + options.Verify, options.ClassifyToolResult = ae.ToolSucceeded("run_tests"), ae.ExitCode("run_tests") + options.Environment = &ae.EnvironmentFingerprint{HostName: "test", RuntimeVersion: "go", OperatingSystem: "any", + ApplicationVersion: ae.Ptr("1.0.0"), Metadata: map[string]string{}} + options.OnInjected = func(r maf.InjectionResult) { + seen.mu.Lock() + seen.injected = append(seen.injected, len(r.ExperienceIDs)) + seen.mu.Unlock() + } + options.OnInjectionRecorded = func(r maf.InjectionRecordResult) { + seen.mu.Lock() + seen.recorded = append(seen.recorded, r.Outcome) + seen.mu.Unlock() + } + options.OnFinalized = func(ae.FinalizedExperience) { + seen.mu.Lock() + seen.finalized++ + seen.mu.Unlock() + } + if err := maf.UseAgentExperience(&config, options); err != nil { + t.Fatal(err) + } + model := &scriptedModel{} + triage := agent.New(agent.ProviderConfig{ProviderName: "scripted", Run: model.run, + Middlewares: []agent.Middleware{toolautocall.New(toolautocall.Config{})}}, config) + for _, ticket := range []string{first, second} { + if _, err := triage.RunText(context.Background(), ticket).Collect(); err != nil { + t.Fatal(err) + } + } + if err := maf.WaitForIndexWrites(context.Background(), &config); err != nil { + t.Fatal(err) + } + if provider := maf.ContextProviderOf(&config); provider != nil { + if err := provider.Close(context.Background()); err != nil { + t.Fatal(err) + } + } + return seen +} + +const ( + ticket = "Ticket #4812: a refund is stuck on a lock. Triage it." + nextTicket = "Ticket #4977: a refund is stuck on a lock again. Triage it." +) + +func attributes(span tracetest.SpanStub) map[string]any { + found := map[string]any{} + for _, kv := range span.Attributes { + found[string(kv.Key)] = kv.Value.AsInterface() + } + return found +} + +func spanNamed(t *testing.T, spans tracetest.SpanStubs, name string) tracetest.SpanStub { + t.Helper() + for _, span := range spans { + if span.Name == name { + return span + } + } + t.Fatalf("no span %s among %d", name, len(spans)) + return tracetest.SpanStub{} +} + +func counted(points []point, operation string) []point { + var found []point + for _, p := range points { + if p.name == telemetry.OperationCount && p.attributes["operation"] == operation { + found = append(found, p) + } + } + return found +} + +func TestAnAgentThatInjectsCapturesAndRecordsAnInjectionEmitsExactlyThePinnedNames(t *testing.T) { + file := names(t) + s := enable(t) + seen := drive(t, newStore(t), maf.Options{Embedder: embedder{}}, ticket, nextTicket) + if len(seen.recorded) != 1 || seen.recorded[0] != maf.InjectionRecorded { + t.Fatalf("recorded %v", seen.recorded) + } + + spans := s.spans.GetSpans() + for _, want := range []string{"inject", "retrieve", "capture.start_run", "capture.append_attempt", + "capture.complete_run", "verify", "reflect", "finalize", "index"} { + spanNamed(t, spans, "agentexperience."+want) + } + for _, span := range spans { + got := attributes(span) + operation, _ := got[telemetry.AttributeOperation].(string) + pinned, ok := file.operation(operation) + if !ok || !pinned.Span || span.Name != file.SpanNamePrefix+operation { + t.Errorf("span %s (operation %q) is not pinned", span.Name, operation) + continue + } + if outcome, _ := got[telemetry.AttributeOutcome].(string); !slices.Contains(pinned.Outcomes, outcome) { + t.Errorf("%s: outcome %q is not pinned", span.Name, outcome) + } + for key := range got { + if !slices.Contains(file.Attributes, key) { + t.Errorf("%s: attribute %s is not pinned", span.Name, key) + } + } + if span.Status.Code.String() != "Ok" || span.SpanKind.String() != "internal" { + t.Errorf("%s: status %v kind %v", span.Name, span.Status.Code, span.SpanKind) + } + } + if slices.ContainsFunc(spans, func(span tracetest.SpanStub) bool { return span.Name == "agentexperience.record_injection" }) { + t.Error("record_injection is measurements only, no span") + } + + points := s.points(t) + units := map[string]string{} + for _, item := range file.Instruments { + units[item.Name] = item.Unit + } + seenOperations := map[string]bool{} + for _, p := range points { + if units[p.name] != p.unit { + t.Errorf("%s has unit %q", p.name, p.unit) + } + operation, _ := p.attributes["operation"].(string) + pinned, ok := file.operation(operation) + if !ok || pinned.Source != p.meter { + t.Errorf("%s: operation %q on meter %s", p.name, operation, p.meter) + continue + } + if outcome, _ := p.attributes["outcome"].(string); !slices.Contains(pinned.Outcomes, outcome) { + t.Errorf("%s: outcome %q", operation, outcome) + } + for key := range p.attributes { + if !slices.Contains(file.Dimensions, key) || key == "error.class" { + t.Errorf("%s: dimension %s on a successful run", p.name, key) + } + } + if _, isBool := p.attributes["nested"].(bool); !isBool { + t.Errorf("%s: nested is %v", p.name, p.attributes["nested"]) + } + seenOperations[operation] = true + } + for operation := range ae.ObservedOperations() { + if !seenOperations[operation] { + t.Errorf("%s was never reported", operation) + } + } + nested := func(operation string) any { return counted(points, operation)[0].attributes["nested"] } + if nested("verify") != true || nested("reflect") != true || nested("finalize") != false || + nested("retrieve") != false || nested("inject") != false || nested("record_injection") != false { + t.Errorf("nested flags: verify %v reflect %v finalize %v retrieve %v", nested("verify"), nested("reflect"), + nested("finalize"), nested("retrieve")) + } + + finalize := spanNamed(t, spans, "agentexperience.finalize") + verify := spanNamed(t, spans, "agentexperience.verify") + retrieve := spanNamed(t, spans, "agentexperience.retrieve") + inject := spanNamed(t, spans, "agentexperience.inject") + if verify.Parent.SpanID() != finalize.SpanContext.SpanID() || retrieve.Parent.SpanID() != inject.SpanContext.SpanID() { + t.Error("a nested operation is not a child of the span that ran it") + } + if attributes(finalize)["agentexperience.stage"] != "CreateRecord" { + t.Errorf("stage %v", attributes(finalize)["agentexperience.stage"]) + } +} + +func TestInjectIsNothingToInjectThenInjected(t *testing.T) { + s := enable(t) + drive(t, newStore(t), maf.Options{}, ticket, nextTicket) + var got []any + for _, span := range s.spans.GetSpans() { + if span.Name == "agentexperience.inject" { + got = append(got, attributes(span)[telemetry.AttributeOutcome]) + } + } + if len(got) != 2 || got[0] != "NothingToInject" || got[1] != "Injected" { + t.Fatalf("inject outcomes %v", got) + } +} + +func TestAFailingSearchIsRetrievalFailedAndNothingThrew(t *testing.T) { + s := enable(t) + drive(t, brokenSearch{newStore(t)}, maf.Options{}, ticket+" "+marker, nextTicket) + for _, span := range s.spans.GetSpans() { + switch span.Name { + case "agentexperience.inject": + if attributes(span)[telemetry.AttributeOutcome] != "RetrievalFailed" { + t.Errorf("inject: %v", attributes(span)) + } + case "agentexperience.retrieve": + if attributes(span)[telemetry.AttributeOutcome] != "Failed" { + t.Errorf("retrieve: %v", attributes(span)) + } + } + } + for _, p := range s.points(t) { + if p.name == telemetry.OperationFailures { + t.Errorf("nothing threw, yet a failure was counted: %v", p.attributes) + } + } +} + +func TestAnOperationThatFailsIsFaultedWithItsClassAndTypeNameOnly(t *testing.T) { + s := enable(t) + _, err := ae.FinalizeContext(context.Background(), ae.CapturedRun{}, ae.Verification{}) + if err == nil { + t.Fatal("an unfinished run was finalized") + } + span := spanNamed(t, s.spans.GetSpans(), "agentexperience.finalize") + got := attributes(span) + if got[telemetry.AttributeOutcome] != "Faulted" || got[telemetry.AttributeErrorClass] != "Unexpected" || + got[telemetry.AttributeErrorType] != "errors.errorString" || got["agentexperience.stage"] != "Load" { + t.Errorf("attributes %v", got) + } + if span.Status.Code.String() != "Error" || span.Status.Description != "" || len(span.Events) != 0 { + t.Errorf("status %v %q events %d", span.Status.Code, span.Status.Description, len(span.Events)) + } + var failures []point + for _, p := range s.points(t) { + if p.name == telemetry.OperationFailures { + failures = append(failures, p) + } + } + if len(failures) != 1 || failures[0].value != 1 || failures[0].attributes["operation"] != "finalize" || + failures[0].attributes["error.class"] != "Unexpected" || failures[0].attributes["nested"] != false || + len(failures[0].attributes) != 3 { + t.Errorf("failures %v", failures) + } + if c := counted(s.points(t), "finalize"); len(c) != 1 || c[0].attributes["outcome"] != "Faulted" { + t.Errorf("count %v", c) + } +} + +func TestRecordInjectionFailureClassesFollowTheNetTable(t *testing.T) { + s := enable(t) + drive(t, failingRecord{newStore(t)}, maf.Options{}, ticket, nextTicket) + drive(t, searchOnly{newStore(t)}, maf.Options{}, ticket, nextTicket) + var outcomes []string + for _, p := range counted(s.points(t), "record_injection") { + outcomes = append(outcomes, p.attributes["outcome"].(string)) + } + slices.Sort(outcomes) + if !slices.Equal(outcomes, []string{"NotRecorded", "NotTracked"}) { + t.Fatalf("outcomes %v", outcomes) + } + for _, p := range s.points(t) { + if p.name == telemetry.OperationFailures { + if p.attributes["operation"] != "record_injection" || p.attributes["error.class"] != "Infrastructure" { + t.Errorf("failure %v", p.attributes) + } + } + } +} + +func TestNothingOfTheRunReachesASpanOrAMetricDimension(t *testing.T) { + s := enable(t) + drive(t, newStore(t), maf.Options{}, "Triage "+marker+" please", "Triage "+marker+" again") + var texts []string + for _, span := range s.spans.GetSpans() { + texts = append(texts, span.Name) + for _, kv := range span.Attributes { + texts = append(texts, fmt.Sprint(kv.Value.AsInterface())) + } + } + for _, p := range s.points(t) { + for _, value := range p.attributes { + texts = append(texts, fmt.Sprint(value)) + } + } + if len(texts) == 0 { + t.Fatal("nothing was emitted") + } + for _, text := range texts { + if strings.Contains(text, marker) { + t.Errorf("%q carries the marker", text) + } + } +} + +// ---- Telemetry never changes a result ------------------------------------------------------------------------------- + +type panickingObserver struct{} + +func (panickingObserver) Start(context.Context, string, string, bool) (context.Context, any) { + panic("start") +} +func (panickingObserver) Finish(any, ae.Observation) { panic("finish") } + +func TestAnObserverThatPanicsChangesNothing(t *testing.T) { + run := func() ([]int, []maf.InjectionRecordOutcome, int) { + seen := drive(t, newStore(t), maf.Options{Embedder: embedder{}}, ticket, nextTicket) + return seen.injected, seen.recorded, seen.finalized + } + ae.SetObserver(nil) + wantInjected, wantRecorded, wantFinalized := run() + ae.SetObserver(panickingObserver{}) + t.Cleanup(func() { ae.SetObserver(nil) }) + gotInjected, gotRecorded, gotFinalized := run() + if !slices.Equal(gotInjected, wantInjected) || !slices.Equal(gotRecorded, wantRecorded) || gotFinalized != wantFinalized { + t.Fatalf("a panicking observer changed the result: %v %v %d, want %v %v %d", gotInjected, gotRecorded, + gotFinalized, wantInjected, wantRecorded, wantFinalized) + } + if wantFinalized == 0 { + t.Fatal("the control run stored nothing") + } +} + +// ---- No OpenTelemetry in the core or the integration ---------------------------------------------------------------- + +func TestNeitherTheCoreNorTheIntegrationNorTheStoresImportOpenTelemetry(t *testing.T) { + for _, directory := range []string{"..", "../maf", "../inmemory", "../postgres"} { + module, err := os.ReadFile(filepath.Join(directory, "go.mod")) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(string(module), "\n") { + if strings.Contains(line, "AgentExperience.NET/go/telemetry") { + t.Errorf("%s/go.mod requires the telemetry module: %s", directory, line) + } + if directory != "../maf" && strings.Contains(line, "go.opentelemetry.io") { + t.Errorf("%s/go.mod requires OpenTelemetry: %s", directory, line) + } + } + err = filepath.WalkDir(directory, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() && (entry.Name() == "telemetry" || entry.Name() == "testdata") { + return filepath.SkipDir + } + if entry.IsDir() || !strings.HasSuffix(path, ".go") { + return nil + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.ImportsOnly) + if err != nil { + return err + } + for _, spec := range file.Imports { + if strings.Contains(spec.Path.Value, "go.opentelemetry.io") || + strings.Contains(spec.Path.Value, "AgentExperience.NET/go/telemetry") { + t.Errorf("%s imports %s", path, spec.Path.Value) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + } +} diff --git a/go/verification.go b/go/verification.go index 568581c..a840e56 100644 --- a/go/verification.go +++ b/go/verification.go @@ -1,6 +1,7 @@ package agentexperience import ( + "context" "errors" "strings" ) @@ -40,6 +41,26 @@ type VerificationResult struct { // Aggregate aggregates the evidence of the closed round and the current artifact revision into an outcome. func Aggregate(evidence []Evidence, requiredChecks []RequiredCheck, closedRound *ClosedVerificationRound, + currentArtifactRevision string, evaluatedAt Instant) (VerificationResult, error) { + return aggregate(context.Background(), false, evidence, requiredChecks, closedRound, currentArtifactRevision, + evaluatedAt) +} + +func aggregate(ctx context.Context, nested bool, evidence []Evidence, requiredChecks []RequiredCheck, + closedRound *ClosedVerificationRound, currentArtifactRevision string, evaluatedAt Instant) (VerificationResult, + error) { + _, op := Observe(ctx, "verify", nested) + defer op.End() + result, err := aggregateEvidence(evidence, requiredChecks, closedRound, currentArtifactRevision, evaluatedAt) + if err != nil { + op.Fail(err) + return result, err + } + op.Done(string(result.Outcome.Status)) + return result, nil +} + +func aggregateEvidence(evidence []Evidence, requiredChecks []RequiredCheck, closedRound *ClosedVerificationRound, currentArtifactRevision string, evaluatedAt Instant) (VerificationResult, error) { if isNullOrWhitespace(currentArtifactRevision) { return VerificationResult{}, errors.New("the current artifact revision must not be blank") diff --git a/python/README.md b/python/README.md index d30bbf0..00e92f7 100644 --- a/python/README.md +++ b/python/README.md @@ -166,6 +166,26 @@ for lesson in page.lessons: # newest first - **Stores.** `InMemoryExperienceStore` and `PostgresExperienceStore` have `list_lessons_page`; PostgreSQL runs the contract's `sql/list-experience-lessons.sql` as written and needs no grants beyond 1.0 and 1.4. +### Telemetry: the same spans and metrics as .NET + +With the `otel` extra (`pip install "agenticexperience[otel]"`, which depends on `opentelemetry-api` only), one call +reports the operations the port performs to OpenTelemetry, under the names, dimensions and attributes +[`docs/telemetry.md`](https://github.com/fabbrik/AgentExperience.NET/blob/main/docs/telemetry.md#the-python-and-go-ports) +documents for .NET, so one dashboard serves a mixed fleet: + +```python +from agentexperience import telemetry + +telemetry.enable() # the global tracer and meter providers your application configured (or pass tracer_provider=, meter_provider=) +``` + +It emits `inject`, `retrieve`, `record_injection`, `capture.start_run`, `capture.append_attempt`, `capture.complete_run`, +`verify`, `reflect`, `finalize` and `index` (the background embedding write): the operations this port performs, and no +other. The core and the agent integration import no OpenTelemetry; they call the small hook in `agentexperience.observer` +(one `None` check when nothing is registered), and `agentexperience.telemetry` is the adapter. It configures no SDK and +no exporter, and a provider, exporter or observer that raises is swallowed and never reaches the agent. Attributes +carry identifiers and counts only, never record content, task text, scopes or exception messages. + ### In production The identity must come from your own authentication, never from model output: pass a function of an diff --git a/python/pyproject.toml b/python/pyproject.toml index 7f2ef24..6363e0c 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -25,6 +25,8 @@ dependencies = [] postgres = ["psycopg[binary]>=3.2"] # Microsoft Agent Framework for Python: 1.0.0 is the lowest release whose APIs agentexperience.maf uses (CI tests it). maf = ["agent-framework-core>=1.0.0"] +# OpenTelemetry API only (agentexperience.telemetry): the host brings the SDK and the exporters. +otel = ["opentelemetry-api>=1.27"] [project.urls] Repository = "https://github.com/fabbrik/AgentExperience.NET" @@ -38,6 +40,8 @@ dev = [ "testcontainers[postgres]>=4.8", "psycopg[binary]>=3.2", "agent-framework-core>=1.0.0", + "opentelemetry-api>=1.27", + "opentelemetry-sdk>=1.27", ] [build-system] diff --git a/python/src/agentexperience/capture.py b/python/src/agentexperience/capture.py index 661b440..072071d 100644 --- a/python/src/agentexperience/capture.py +++ b/python/src/agentexperience/capture.py @@ -17,6 +17,7 @@ from ._text import is_null_or_whitespace, utf16_len, utf16_prefix from .encoding import Duration, Instant from .limits import DEFAULT_CAPTURE_LIMITS, MAX_EXPOSURES_PER_RUN, TRUNCATION_PLACEHOLDER, CaptureLimits +from .observer import observe from .records import Attempt, EnvironmentFingerprint, Provenance, RunExposure, Scope, ToolCall from .sanitization import VALUE_FIELD, Sanitization, SanitizationPolicy, SanitizationRejected @@ -91,6 +92,28 @@ def start( limits: CaptureLimits | None = None, ) -> RunCapture: """Starts a run. It starts exposed to nothing: exposures are recorded with :meth:`record_exposure`.""" + with observe("capture.start_run") as op: + if op is not None: + op.set("agentexperience.run_id", str(run_id)) + started = RunCapture._start( + run_id, task_id, task_description, scope, environment, provenance, started_at, sanitization, limits + ) + if op is not None: + op.outcome("Started") + return started + + @staticmethod + def _start( + run_id: uuid.UUID, + task_id: str, + task_description: str | None, + scope: Scope, + environment: EnvironmentFingerprint, + provenance: Provenance, + started_at: Instant, + sanitization: Sanitization | None, + limits: CaptureLimits | None, + ) -> RunCapture: if run_id == _EMPTY: raise ValueError("A run must be named by a non-empty UUID.") if is_null_or_whitespace(task_id): @@ -131,6 +154,16 @@ def record_exposure(self, exposures: Iterable[RunExposure]) -> ExposureOutcome: def append_attempt(self, attempt: RawAttempt) -> AppendOutcome: """Sanitizes and records one attempt; returns capture's answer.""" + with observe("capture.append_attempt") as op: + if op is not None: + op.set("agentexperience.run_id", str(self.run.run_id)) + op.set("agentexperience.attempt_id", str(attempt.attempt_id)) + outcome = self._append_attempt(attempt) + if op is not None: + op.outcome(outcome) + return outcome + + def _append_attempt(self, attempt: RawAttempt) -> AppendOutcome: if self.run.execution_status is not None: return "Conflict" previous = self._seen.get(attempt.attempt_id) @@ -175,9 +208,15 @@ def append_attempt(self, attempt: RawAttempt) -> AppendOutcome: def complete(self, execution_status: RunExecutionStatus, ended_at: Instant) -> CapturedRun: """Ends the run; no attempt or exposure is recorded after this.""" - if self.run.execution_status is None: - self.run = replace(self.run, execution_status=execution_status, ended_at=ended_at) - return self.run + with observe("capture.complete_run") as op: + if op is not None: + op.set("agentexperience.run_id", str(self.run.run_id)) + held = self.run.execution_status + if held is None: + self.run = replace(self.run, execution_status=execution_status, ended_at=ended_at) + if op is not None: + op.outcome("Recorded" if held is None else "DuplicateNoOp" if held == execution_status else "Conflict") + return self.run def _comparable(attempt: RawAttempt) -> RawAttempt: diff --git a/python/src/agentexperience/finalize.py b/python/src/agentexperience/finalize.py index ab4a3fd..349bd0d 100644 --- a/python/src/agentexperience/finalize.py +++ b/python/src/agentexperience/finalize.py @@ -14,6 +14,7 @@ from .capture import CapturedRun from .encoding import Instant, format_uuid from .ids import experience_id_for, initial_event_id_for, reflection_id_for +from .observer import observe from .records import Evidence, ExperienceRecord, ExperienceStatus, Reflection from .reflector import ScreeningRefused, reflect, screen from .verification import ClosedVerificationRound, RequiredCheck, VerificationResult, aggregate @@ -82,20 +83,47 @@ def finalize( Finalizes a finished run: verification, the deterministic reflection (only for a ``Verified`` run), the stored record (``Validated`` with a reflection, else ``Quarantined``) and its initial lifecycle event. """ + with observe("finalize") as op: + if op is not None: + op.set("agentexperience.run_id", str(run.run_id)) + op.set("agentexperience.stage", "Load") + finalized = _finalize(run, closed_round, required_checks, evidence, current_artifact_revision, finalized_at, op) + if op is not None: + op.set("agentexperience.experience_id", str(finalized.record.experience_id)) + op.outcome(finalized.record.status) + return finalized + + +def _finalize( + run: CapturedRun, + closed_round: ClosedVerificationRound | None, + required_checks: Sequence[RequiredCheck], + evidence: Sequence[Evidence], + current_artifact_revision: str, + finalized_at: Instant, + op: Any, +) -> FinalizedExperience: + """The stage the finalization has reached is on the span (``agentexperience.stage``), also when it throws.""" if run.execution_status is None: raise ValueError("The run has not finished and cannot be finalized.") at = finalized_at.truncate_to_microseconds() + if op is not None: + op.set("agentexperience.stage", "Evaluate") evaluation = aggregate(evidence, required_checks, closed_round, current_artifact_revision, at) reflection: Reflection | None = None refusal: str | None = None if evaluation.outcome.status == "Verified": + if op is not None: + op.set("agentexperience.stage", "Reflect") try: reflection = screen(reflect(run, evaluation, reflection_id_for(run.run_id), at)) except ScreeningRefused as refused: refusal = str(refused) + if op is not None: + op.set("agentexperience.stage", "CreateRecord") status: ExperienceStatus = "Validated" if reflection is not None else "Quarantined" round_id = closed_round.round_id if closed_round is not None and closed_round.round_id.int != 0 else None record = ExperienceRecord( diff --git a/python/src/agentexperience/maf.py b/python/src/agentexperience/maf.py index 4f975f4..ad69b83 100644 --- a/python/src/agentexperience/maf.py +++ b/python/src/agentexperience/maf.py @@ -77,6 +77,7 @@ from .embedding import ( Embedder, EmbeddingWrite, + EmbeddingWriteResult, VectorFallback, VectorQuery, VectorSearchResult, @@ -101,6 +102,7 @@ ) from .inmemory import ( MAX_RECORD_INJECTION_IDS, + AuthorizationDenied, EmbeddingIndex, ExperienceStore, InjectionRecorder, @@ -108,6 +110,8 @@ LessonsReader, ) from .lessons import ExperienceLessons +from .observer import classify as _classify_error +from .observer import observe from .records import EnvironmentFingerprint, Provenance, RunExposure from .sanitization import Sanitization from .search import SearchParameters, SearchRow, rank @@ -602,27 +606,37 @@ def _start_index(self, finalized: FinalizedExperience, identity: ExperienceIdent async def _index(self, finalized: FinalizedExperience, identity: ExperienceIdentity) -> None: settings = self.settings - try: - await asyncio.wait_for(self._write_embedding(finalized, identity), settings.finalization_timeout) - except (asyncio.TimeoutError, TimeoutError) as ex: - self.report( - "Index", f"Embedding the record did not finish within {settings.finalization_timeout} seconds.", ex - ) - except asyncio.CancelledError as ex: - self.report("Index", "Embedding the record was cancelled.", ex) - raise - except Exception as ex: - self.report("Index", f"Embedding the record raised {_type_name(ex)}; its text retrieval is unaffected.", ex) + with observe("index") as op: + if op is not None: + op.set("agentexperience.experience_id", str(finalized.record.experience_id)) + try: + await asyncio.wait_for(self._write_embedding(finalized, identity, op), settings.finalization_timeout) + except (asyncio.TimeoutError, TimeoutError) as ex: + self.report( + "Index", f"Embedding the record did not finish within {settings.finalization_timeout} seconds.", ex + ) + except asyncio.CancelledError as ex: + self.report("Index", "Embedding the record was cancelled.", ex) + raise + except Exception as ex: + self.report( + "Index", f"Embedding the record raised {_type_name(ex)}; its text retrieval is unaffected.", ex + ) - async def _write_embedding(self, finalized: FinalizedExperience, identity: ExperienceIdentity) -> None: + async def _write_embedding(self, finalized: FinalizedExperience, identity: ExperienceIdentity, op: Any) -> None: settings = self.settings embedder, store = settings.embedder, settings.store assert embedder is not None and isinstance(store, EmbeddingIndex) record = finalized.record summary = record_summary(record) + # What the index reports if it stops here: the provider failed, then (past the embedding) the store did. + if op is not None: + op.outcome("ProviderFailed") model_id, dimension = embedder.model_id, embedder.dimension validate_model_id(model_id) vector = validate_vector(await embed_text(embedder, summary), dimension) + if op is not None: + op.outcome("IndexFailed") write = EmbeddingWrite( record.experience_id, record.scope, @@ -633,7 +647,18 @@ async def _write_embedding(self, finalized: FinalizedExperience, identity: Exper vector, settings.clock(), ) - await run_on_vector_thread(store.write_embedding, identity.authorization, write) + written = cast( + EmbeddingWriteResult, await run_on_vector_thread(store.write_embedding, identity.authorization, write) + ) + if op is not None: + # Only the contract's outcomes become a label; a custom index's own word never does. + op.outcome( + "Indexed" + if written.outcome == "Written" + else written.outcome + if written.outcome in ("Stale", "Missing") + else "IndexFailed" + ) _current_scope: ContextVar[_CaptureScope | None] = ContextVar("agentexperience_capture_scope", default=None) @@ -758,10 +783,16 @@ def _start_recording(self, identity: ExperienceIdentity, ids: tuple[uuid.UUID, . try: recorder = settings.store if isinstance(settings.store, InjectionRecorder) else None if recorder is None: + with observe("record_injection") as op: + if op is not None: + op.outcome("NotTracked") settings.report_injection_recorded(InjectionRecordResult("NotTracked", ids)) return task = asyncio.get_running_loop().create_task(self._record(recorder, identity, ids, injected_at)) except Exception as ex: + with observe("record_injection") as op: + if op is not None: + op.outcome("NotRecorded", "Unexpected") settings.report_injection_recorded(InjectionRecordResult("NotRecorded", ids, ex)) return self._recordings.add(task) @@ -781,9 +812,21 @@ async def _record( hang can never take more threads than that, and never the event loop's default executor, which the search and capture use. Never raises, except the task's own cancellation, which is reported first. """ + with observe("record_injection") as op: + await self._record_observed(recorder, identity, ids, injected_at, op) + + async def _record_observed( + self, + recorder: InjectionRecorder, + identity: ExperienceIdentity, + ids: tuple[uuid.UUID, ...], + injected_at: Instant, + op: Any, + ) -> None: settings = self.settings outcome: InjectionRecordOutcome = "Recorded" error: BaseException | None = None + error_class: str | None = None loop = asyncio.get_running_loop() deadline = loop.time() + settings.injection_record_timeout try: @@ -800,6 +843,7 @@ async def _record( ) if write is None: outcome = "NotRecorded" + error_class = "Timeout" error = RuntimeError("Too many earlier injection records are still running against the store.") break future, state = write @@ -809,12 +853,19 @@ async def _record( self._abandon(state) raise except (asyncio.TimeoutError, TimeoutError) as ex: - outcome, error = "OutcomeUnknown", ex + outcome, error, error_class = "OutcomeUnknown", ex, "Timeout" except asyncio.CancelledError as ex: + if op is not None: + op.outcome("OutcomeUnknown", "Cancelled") + op.close() settings.report_injection_recorded(InjectionRecordResult("OutcomeUnknown", ids, ex)) raise except Exception as ex: + # A refusal (the grant is missing) is an answer, not a failure: only a store or unexpected failure counts. outcome, error = "NotRecorded", ex + error_class = None if isinstance(ex, AuthorizationDenied) else _classify_error(ex) + if op is not None: + op.outcome(outcome, error_class) settings.report_injection_recorded(InjectionRecordResult(outcome, ids, error)) def _start_write( @@ -934,6 +985,14 @@ async def before_run( context: Any, state: dict[str, Any], ) -> None: + with observe("inject") as op: + outcome = await self._inject(agent, session, context, op) + if op is not None: + op.outcome(outcome) + + async def _inject(self, agent: Any, session: Any, context: Any, op: Any) -> str: + """The injection; answers its outcome (``InjectionOutcome``'s member name). A raise is the invocation's own + cancellation, which the observer reports as ``Faulted``/``Cancelled``.""" settings = self.settings scope = _current_scope.get() captured = scope is not None and scope.agent is agent @@ -945,10 +1004,11 @@ async def before_run( identity = await settings.resolve_identity(agent, session) except Exception as ex: settings.report_injection(InjectionFailure(f"Resolving the identity raised {_type_name(ex)}.", ex)) - return + return "Failed" if identity is None: - return + return "Skipped" + retrieval_failed = False try: history = { provider.source_id @@ -961,48 +1021,63 @@ async def before_run( ] task_text = derive_task_text(_task_messages(context.input_messages, earlier)) if task_text is None: - return + return "NothingToInject" parameters = SearchParameters( identity.scope, task_text, exclude_model_authored=settings.exclude_model_authored ) - text_search = asyncio.ensure_future( - asyncio.to_thread(settings.store.search, identity.authorization, parameters) - ) - vector_search = asyncio.ensure_future(self._vector_channel(task_text, identity)) - # Both channels run together; the vector channel never raises (it answers with a fallback instead). - text_found, vector_found = await asyncio.gather(text_search, vector_search, return_exceptions=True) - if isinstance(vector_found, BaseException): - vector_found = ([], VectorFallback("VectorSearchFailed", "The vector channel failed unexpectedly.")) - vector_rows, fallback = vector_found - if fallback is not None: - settings.report_vector_fallback(fallback) - if isinstance(text_found, BaseException): - # The text channel's failure fails the call, whatever the vector channel found. - raise text_found - rows = text_found - # The instant the block is built at: ranking's "now", and the injection's instant. - now = settings.clock() - if vector_rows: - ranked = rank_hybrid( - rows, - vector_rows, - identity.scope, - now, - settings.preferred_environment, - minimum_vector_relevance=settings.min_vector_relevance, - exclude_model_authored=settings.exclude_model_authored, - ) - else: - ranked = rank( - rows, - identity.scope, - now, - settings.preferred_environment, - exclude_model_authored=settings.exclude_model_authored, + with observe("retrieve") as retrieval: + text_search = asyncio.ensure_future( + asyncio.to_thread(settings.store.search, identity.authorization, parameters) ) + vector_search = asyncio.ensure_future(self._vector_channel(task_text, identity)) + # Both channels run together; the vector channel never raises (it answers with a fallback instead). + text_found, vector_found = await asyncio.gather(text_search, vector_search, return_exceptions=True) + if isinstance(vector_found, BaseException): + vector_found = ( + [], + VectorFallback("VectorSearchFailed", "The vector channel failed unexpectedly."), + ) + vector_rows, fallback = vector_found + if fallback is not None: + settings.report_vector_fallback(fallback) + if isinstance(text_found, BaseException): + # The text channel's failure fails the call, whatever the vector channel found. Retrieval reports + # it as its outcome, as .NET does; it is not thrown. + if retrieval is not None: + retrieval.outcome("Failed") + retrieval_failed = True + else: + rows = text_found + # The instant the block is built at: ranking's "now", and the injection's instant. + now = settings.clock() + if vector_rows: + ranked = rank_hybrid( + rows, + vector_rows, + identity.scope, + now, + settings.preferred_environment, + minimum_vector_relevance=settings.min_vector_relevance, + exclude_model_authored=settings.exclude_model_authored, + ) + else: + ranked = rank( + rows, + identity.scope, + now, + settings.preferred_environment, + exclude_model_authored=settings.exclude_model_authored, + ) + if retrieval is not None: + retrieval.outcome("Completed") + if retrieval_failed: + assert isinstance(text_found, BaseException) + raise text_found block = render_block(ranked.ranked, settings.block_options) + if op is not None: + op.set("agentexperience.omitted_count", len(block.omitted)) if block.is_empty: - return + return "NothingToInject" message = Message( role=settings.message_role, contents=[block.text], @@ -1011,7 +1086,7 @@ async def before_run( context.extend_messages(self, [message]) except Exception as ex: settings.report_injection(InjectionFailure(f"Injection raised {_type_name(ex)}; no block was added.", ex)) - return + return "RetrievalFailed" if retrieval_failed else "Failed" # The records this block delivers, at the revision each was rendered from, on the run the invocation is # captured as. Recorded last, once nothing that could fail remains. @@ -1028,6 +1103,7 @@ async def before_run( # block is handed over and the model called without waiting for the store. if delivered: self._start_recording(identity, delivered, now) + return "Injected" # ---- Capture ----------------------------------------------------------------------------------------------------- diff --git a/python/src/agentexperience/observer.py b/python/src/agentexperience/observer.py new file mode 100644 index 0000000..8da0ad1 --- /dev/null +++ b/python/src/agentexperience/observer.py @@ -0,0 +1,226 @@ +""" +The observer hook: the one seam through which the core and the Microsoft Agent Framework integration report what they +do, for telemetry (``docs/telemetry.md``). It imports no OpenTelemetry. ``agentexperience.telemetry`` (the ``otel`` +extra) is an adapter that turns the hook into spans and metrics; any other observer can be registered the same way. + +Until an observer is registered, every call here is one ``None`` check and allocates nothing. An observer that raises is +swallowed, and never changes what the caller gets. + +The names in :data:`OPERATIONS` are the operations this port performs, with the sources, outcomes and span flags that +``docs/telemetry-names.json`` (generated from the .NET definitions) pins by test. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Mapping +from dataclasses import dataclass +from types import TracebackType +from typing import Any, Protocol + +CORE_SOURCE = "AgentExperience.Core" +"""The source and meter the core operations are reported under (``docs/telemetry.md``).""" + +MAF_SOURCE = "AgentExperience.MicrosoftAgentFramework" +"""The source and meter ``inject`` and ``record_injection`` are reported under.""" + +FAULTED = "Faulted" +"""The outcome of an operation that threw.""" + +CANCELLED = "Cancelled" +TIMEOUT = "Timeout" +INFRASTRUCTURE = "Infrastructure" +UNEXPECTED = "Unexpected" +ERROR_CLASSES: tuple[str, ...] = (CANCELLED, TIMEOUT, INFRASTRUCTURE, UNEXPECTED) +"""The four ``error.class`` values.""" + +AttributeValue = str | int | bool + + +@dataclass(frozen=True) +class OperationSpec: + """One operation this port performs: where it is reported, whether it has a span, and the outcomes it can reach.""" + + source: str + span: bool + outcomes: tuple[str, ...] + + +OPERATIONS: Mapping[str, OperationSpec] = { + "capture.start_run": OperationSpec(CORE_SOURCE, True, ("Started",)), + "capture.append_attempt": OperationSpec( + CORE_SOURCE, True, ("Recorded", "DuplicateNoOp", "Conflict", "SanitizationRejected", "CapacityExceeded") + ), + "capture.complete_run": OperationSpec(CORE_SOURCE, True, ("Recorded", "DuplicateNoOp", "Conflict")), + "verify": OperationSpec(CORE_SOURCE, True, ("Unknown", "Verified", "Failed")), + "reflect": OperationSpec(CORE_SOURCE, True, ("Unknown", "Verified", "Failed")), + "finalize": OperationSpec(CORE_SOURCE, True, ("Validated", "Quarantined")), + "retrieve": OperationSpec(CORE_SOURCE, True, ("Completed", "Failed")), + "index": OperationSpec(CORE_SOURCE, True, ("Indexed", "Stale", "Missing", "ProviderFailed", "IndexFailed")), + "inject": OperationSpec(MAF_SOURCE, True, ("Injected", "NothingToInject", "Skipped", "RetrievalFailed", "Failed")), + "record_injection": OperationSpec(MAF_SOURCE, False, ("Recorded", "NotTracked", "NotRecorded", "OutcomeUnknown")), +} +"""Every operation this port reports, by its ``operation`` value. The port reports no other: not ``retention.sweep``, +``delete``, grants, seal, backfill, lifecycle, confidence, reuse-feedback, ``deindex`` or ``reindex``.""" + +NEVER_NESTED: frozenset[str] = frozenset({"inject", "record_injection"}) +"""Operations whose ``nested`` is always ``false``, as in .NET.""" + +ATTRIBUTES: tuple[str, ...] = ( + "agentexperience.attempt_id", + "agentexperience.experience_id", + "agentexperience.omitted_count", + "agentexperience.reflection_id", + "agentexperience.run_id", + "agentexperience.stage", +) +"""The span attributes this port sets beyond the common ones (``agentexperience.operation``, ``.outcome``, +``.error.class`` and ``error.type``, which the adapter writes).""" + + +class Observer(Protocol): + """ + What an observer implements. ``start`` runs when an operation begins and returns a token (or ``None``); + ``finish`` runs once when it ends, with that token. Neither may be relied on to be called on the same thread. + """ + + def start(self, source: str, operation: str) -> object | None: ... + + def finish( + self, + token: object | None, + source: str, + operation: str, + outcome: str, + error_class: str | None, + error_type: str | None, + attributes: Mapping[str, AttributeValue], + ) -> None: + """``outcome`` is the operation's own outcome, or ``Faulted`` when it threw. ``error_class`` is set when the + operation threw (and for a ``record_injection`` that failed); ``error_type`` is the exception's type name and + nothing else from it.""" + ... + + +_observer: Observer | None = None + + +def set_observer(observer: Observer | None) -> None: + """Registers the observer every operation reports to, or ``None`` to stop reporting.""" + global _observer + _observer = observer + + +def get_observer() -> Observer | None: + """The registered observer, or ``None``.""" + return _observer + + +def type_name(exception: BaseException) -> str: + """An exception's full type name, never its message, which may hold record content.""" + kind = type(exception) + module = kind.__module__ + return kind.__qualname__ if module in ("builtins", "__main__") else f"{module}.{kind.__qualname__}" + + +def classify(exception: BaseException) -> str: + """The ``error.class`` of a thrown exception, as .NET classifies: the host's own cancellation is ``Cancelled``, a + timeout is ``Timeout``, a store or driver failure is ``Infrastructure``, anything else ``Unexpected``.""" + if isinstance(exception, asyncio.CancelledError): + return CANCELLED + if isinstance(exception, (TimeoutError, asyncio.TimeoutError)): + return TIMEOUT + if isinstance(exception, OSError) or type(exception).__module__.split(".", 1)[0] == "psycopg": + return INFRASTRUCTURE + return UNEXPECTED + + +class _Operation: + """One in-flight operation of a registered observer. Used as a context manager: leaving it without an outcome, or + with an exception, reports ``Faulted``.""" + + __slots__ = ("_attributes", "_done", "_error_class", "_observer", "_operation", "_outcome", "_source", "_token") + + def __init__(self, observer: Observer, source: str, operation: str) -> None: + self._observer = observer + self._source = source + self._operation = operation + self._attributes: dict[str, AttributeValue] = {} + self._outcome: str | None = None + self._error_class: str | None = None + self._done = False + self._token: object | None = None + try: + self._token = observer.start(source, operation) + except Exception: + self._token = None + + def __enter__(self) -> _Operation: + return self + + def __exit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None + ) -> None: + if exc is not None: + self.fault(exc) + elif self._outcome is None: + self._report(FAULTED, UNEXPECTED, None) + else: + self._report(self._outcome, self._error_class, None) + + def set(self, name: str, value: AttributeValue) -> None: + """An attribute of the span. Never record content, task text, scopes or exception messages.""" + self._attributes[name] = value + + def outcome(self, outcome: str, error_class: str | None = None) -> None: + """The outcome the operation returned. ``error_class`` counts a failure for an operation that reports its + failure as an outcome instead of raising (``record_injection``).""" + self._outcome = outcome + self._error_class = error_class + + def close(self) -> None: + """Reports the outcome set so far now, so an exception propagating afterwards is not reported as a fault (the + operation returned its outcome; the exception is its caller's own).""" + self._report(self._outcome or FAULTED, self._error_class if self._outcome else UNEXPECTED, None) + + def fault(self, exception: BaseException) -> None: + self._report(FAULTED, classify(exception), type_name(exception)) + + def _report(self, outcome: str, error_class: str | None, error_type: str | None) -> None: + if self._done: + return + self._done = True + try: + self._observer.finish( + self._token, self._source, self._operation, outcome, error_class, error_type, self._attributes + ) + except Exception: + pass + + +class _NoOperation: + """What :func:`observe` answers when no observer is registered: entering it yields ``None``.""" + + __slots__ = () + + def __enter__(self) -> None: + return None + + def __exit__(self, *_: Any) -> None: + return None + + +_NO_OPERATION = _NoOperation() + + +def observe(operation: str) -> _Operation | _NoOperation: + """ + Opens ``operation`` (a key of :data:`OPERATIONS`) for the registered observer. Use it as a context manager, and + guard every use of what it yields: ``with observe("verify") as op:`` then ``if op is not None: op.outcome(...)``. + With no observer registered it allocates nothing and yields ``None``. + """ + observer = _observer + if observer is None: + return _NO_OPERATION + spec = OPERATIONS[operation] + return _Operation(observer, spec.source, operation) diff --git a/python/src/agentexperience/reflector.py b/python/src/agentexperience/reflector.py index 3046bf8..cba6aae 100644 --- a/python/src/agentexperience/reflector.py +++ b/python/src/agentexperience/reflector.py @@ -21,6 +21,7 @@ from .capture import CapturedRun from .encoding import Instant from .error_class import error_class +from .observer import observe from .records import Attempt, Reflection from .verification import VerificationResult @@ -116,6 +117,22 @@ def reflect( ) -> Reflection: """The deterministic reflection of a finished run. No tool name, argument or error text beyond its class reaches the lesson.""" + with observe("reflect") as op: + if op is not None: + op.set("agentexperience.run_id", str(run.run_id)) + op.set("agentexperience.reflection_id", str(reflection_id)) + reflection = _reflect(run, evaluation, reflection_id, created_at) + if op is not None: + op.outcome(evaluation.outcome.status) + return reflection + + +def _reflect( + run: CapturedRun, + evaluation: VerificationResult, + reflection_id: uuid.UUID, + created_at: Instant, +) -> Reflection: outcome = evaluation.outcome status = outcome.status evidence_ids = _distinct([item.evidence_id for item in outcome.evidence]) diff --git a/python/src/agentexperience/telemetry.py b/python/src/agentexperience/telemetry.py new file mode 100644 index 0000000..888307b --- /dev/null +++ b/python/src/agentexperience/telemetry.py @@ -0,0 +1,198 @@ +""" +OpenTelemetry for AgentExperience (the ``otel`` extra, which depends on ``opentelemetry-api`` only). + +.. code-block:: python + + from agentexperience import telemetry + + telemetry.enable() # uses the global tracer and meter providers your application configured + +This turns the library's observer hook (:mod:`agentexperience.observer`) into the spans and metrics that +``docs/telemetry.md`` documents for .NET, under the same source and meter names (``AgentExperience.Core`` and +``AgentExperience.MicrosoftAgentFramework``), span names, instruments, dimensions and attributes, for the operations +this port performs. It configures no SDK and no exporter: the host does, exactly as it does for the .NET library. + +A failure in a provider, exporter or this adapter is swallowed and never reaches the agent. Attributes never carry +record content, task text, scopes or exception messages; a failing operation records its exception's type name only. +""" + +from __future__ import annotations + +import time +from collections.abc import Mapping +from contextvars import ContextVar +from typing import Any + +try: + from opentelemetry import context as otel_context + from opentelemetry import metrics, trace + from opentelemetry.metrics import Counter, Histogram, MeterProvider + from opentelemetry.trace import Span, SpanKind, Status, StatusCode, Tracer, TracerProvider +except ImportError as _missing: # pragma: no cover - exercised only without the extra + raise ImportError( + "agentexperience.telemetry needs the 'otel' extra: pip install 'agenticexperience[otel]'" + ) from _missing + +from . import __version__ +from .observer import ( + ATTRIBUTES, + FAULTED, + NEVER_NESTED, + OPERATIONS, + AttributeValue, + set_observer, +) + +SPAN_NAME_PREFIX = "agentexperience." +OPERATION_COUNT = "agentexperience.operation.count" +OPERATION_DURATION = "agentexperience.operation.duration" +OPERATION_FAILURES = "agentexperience.operation.failures" + +INSTRUMENTS: Mapping[str, tuple[str, str]] = { + OPERATION_COUNT: ("counter", "{operation}"), + OPERATION_DURATION: ("histogram", "s"), + OPERATION_FAILURES: ("counter", "{failure}"), +} +"""The three instruments every meter publishes, with their kind and unit.""" + +DIMENSIONS: tuple[str, ...] = ("operation", "outcome", "error.class", "nested") +"""The only metric dimensions.""" + +OPERATION_ATTRIBUTE = "agentexperience.operation" +OUTCOME_ATTRIBUTE = "agentexperience.outcome" +ERROR_CLASS_ATTRIBUTE = "agentexperience.error.class" +ERROR_TYPE_ATTRIBUTE = "error.type" + +SPAN_ATTRIBUTES: frozenset[str] = frozenset( + {OPERATION_ATTRIBUTE, OUTCOME_ATTRIBUTE, ERROR_CLASS_ATTRIBUTE, ERROR_TYPE_ATTRIBUTE, *ATTRIBUTES} +) +"""Every span attribute this adapter can write.""" + +_active_sources: ContextVar[frozenset[str]] = ContextVar("agentexperience_telemetry_sources", default=frozenset()) + + +class _Instruments: + def __init__( + self, source: str, tracer_provider: TracerProvider | None, meter_provider: MeterProvider | None + ) -> None: + self.tracer: Tracer = trace.get_tracer(source, __version__, tracer_provider=tracer_provider) + meter = metrics.get_meter(source, __version__, meter_provider=meter_provider) + self.count: Counter = meter.create_counter( + OPERATION_COUNT, + unit="{operation}", + description="Instrumented Experience operations, by operation and outcome.", + ) + self.duration: Histogram = meter.create_histogram( + OPERATION_DURATION, unit="s", description="How long each instrumented Experience operation took." + ) + self.failures: Counter = meter.create_counter( + OPERATION_FAILURES, unit="{failure}", description="Instrumented Experience operations that threw." + ) + + +class _Token: + __slots__ = ("attach", "nested", "previous_sources", "span", "started") + + def __init__( + self, span: Span | None, nested: bool, previous_sources: frozenset[str], attach: object | None + ) -> None: + self.span = span + self.nested = nested + self.previous_sources = previous_sources + self.attach = attach + self.started = time.perf_counter() + + +class OpenTelemetryObserver: + """The observer that reports to OpenTelemetry. Pass providers to use them instead of the global ones.""" + + def __init__( + self, tracer_provider: TracerProvider | None = None, meter_provider: MeterProvider | None = None + ) -> None: + self._instruments: dict[str, _Instruments] = {} + self._tracer_provider = tracer_provider + self._meter_provider = meter_provider + + def _for(self, source: str) -> _Instruments: + instruments = self._instruments.get(source) + if instruments is None: + instruments = self._instruments[source] = _Instruments(source, self._tracer_provider, self._meter_provider) + return instruments + + def start(self, source: str, operation: str) -> object | None: + instruments = self._for(source) + sources = _active_sources.get() + nested = operation not in NEVER_NESTED and source in sources + span: Span | None = None + attach: object | None = None + if OPERATIONS[operation].span: + span = instruments.tracer.start_span( + SPAN_NAME_PREFIX + operation, + kind=SpanKind.INTERNAL, + attributes={OPERATION_ATTRIBUTE: operation}, + ) + attach = otel_context.attach(trace.set_span_in_context(span)) + if operation not in NEVER_NESTED: + _active_sources.set(sources | {source}) + return _Token(span, nested, sources, attach) + + def finish( + self, + token: object | None, + source: str, + operation: str, + outcome: str, + error_class: str | None, + error_type: str | None, + attributes: Mapping[str, AttributeValue], + ) -> None: + if not isinstance(token, _Token): + return + elapsed = time.perf_counter() - token.started + try: + if token.attach is not None: + otel_context.detach(token.attach) # type: ignore[arg-type] + except Exception: + pass + _active_sources.set(token.previous_sources) + instruments = self._for(source) + try: + dimensions: dict[str, Any] = {"operation": operation, "outcome": outcome, "nested": token.nested} + instruments.count.add(1, dimensions) + instruments.duration.record(elapsed, dimensions) + if error_class is not None: + instruments.failures.add( + 1, {"operation": operation, "error.class": error_class, "nested": token.nested} + ) + finally: + span = token.span + if span is not None: + try: + span.set_attribute(OUTCOME_ATTRIBUTE, outcome) + for name, value in attributes.items(): + if name in ATTRIBUTES: + span.set_attribute(name, value) + if error_class is not None: + span.set_attribute(ERROR_CLASS_ATTRIBUTE, error_class) + if error_type is not None: + span.set_attribute(ERROR_TYPE_ATTRIBUTE, error_type) + span.set_status(Status(StatusCode.ERROR if outcome == FAULTED else StatusCode.OK)) + finally: + span.end() + + +def enable( + tracer_provider: TracerProvider | None = None, meter_provider: MeterProvider | None = None +) -> OpenTelemetryObserver: + """Starts reporting to OpenTelemetry (the global providers, or the ones given) and returns the observer.""" + observer = OpenTelemetryObserver(tracer_provider, meter_provider) + set_observer(observer) + return observer + + +def disable() -> None: + """Stops reporting.""" + set_observer(None) + + +__all__ = ["OpenTelemetryObserver", "disable", "enable"] diff --git a/python/src/agentexperience/verification.py b/python/src/agentexperience/verification.py index 686d066..bfd2ebe 100644 --- a/python/src/agentexperience/verification.py +++ b/python/src/agentexperience/verification.py @@ -8,6 +8,7 @@ from ._text import is_null_or_whitespace from .encoding import Instant +from .observer import observe from .records import CheckResult, Evidence, Outcome RULE_VERSION = "1.0.0" @@ -50,6 +51,20 @@ def aggregate( evaluated_at: Instant, ) -> VerificationResult: """Aggregates the evidence of the closed round and the current artifact revision into an outcome.""" + with observe("verify") as op: + result = _aggregate(evidence, required_checks, closed_round, current_artifact_revision, evaluated_at) + if op is not None: + op.outcome(result.outcome.status) + return result + + +def _aggregate( + evidence: Sequence[Evidence], + required_checks: Sequence[RequiredCheck], + closed_round: ClosedVerificationRound | None, + current_artifact_revision: str, + evaluated_at: Instant, +) -> VerificationResult: if is_null_or_whitespace(current_artifact_revision): raise ValueError("The current artifact revision must not be blank.") checks = tuple(required_checks) diff --git a/python/tests/test_telemetry.py b/python/tests/test_telemetry.py new file mode 100644 index 0000000..ce92ea5 --- /dev/null +++ b/python/tests/test_telemetry.py @@ -0,0 +1,448 @@ +""" +Story 23.5: port telemetry. The hook the core and the agent integration call, the OpenTelemetry adapter behind the +``otel`` extra, and the pin of every name against ``docs/telemetry-names.json`` (generated from the .NET definitions). +""" + +from __future__ import annotations + +import asyncio +import json +import subprocess +import sys +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest + +pytest.importorskip("agent_framework") +pytest.importorskip("opentelemetry.sdk") + +from _maf import NEXT_TICKET, TICKET, ScriptedClient +from _sync import run +from agent_framework import Agent, tool +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import InMemoryMetricReader +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from agentexperience import Instant, observer, telemetry +from agentexperience.capture import RunCapture +from agentexperience.finalize import finalize +from agentexperience.inmemory import InMemoryExperienceStore +from agentexperience.maf import ( + ExperienceContextProvider, + InjectionRecordResult, + development_identity, + exit_code, + tool_succeeded, + use_agent_experience, +) +from agentexperience.records import EnvironmentFingerprint, Provenance, Scope + +ROOT = Path(__file__).resolve().parents[2] +NAMES = json.loads((ROOT / "docs" / "telemetry-names.json").read_text(encoding="utf-8")) +OPERATIONS = {item["operation"]: item for item in NAMES["operations"]} +ENVIRONMENT = EnvironmentFingerprint("test-host", "python", "any", "1.0.0", {}) +MARKER = "SECRET-MARKER-7f3a" + + +@pytest.fixture(autouse=True) +def _no_observer() -> Iterator[None]: + observer.set_observer(None) + yield + observer.set_observer(None) + + +Point = tuple[str, str, str, dict[str, Any], Any] + + +def attrs(span: ReadableSpan) -> dict[str, Any]: + return dict(span.attributes or {}) + + +def parent_id(span: ReadableSpan) -> int | None: + return span.parent.span_id if span.parent is not None else None + + +class Telemetry: + """The OpenTelemetry SDK's in-memory exporters, with the adapter registered.""" + + def __init__(self) -> None: + self.spans = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(self.spans)) + self.reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[self.reader]) + telemetry.enable(tracer_provider, meter_provider) + + def finished(self) -> list[ReadableSpan]: + return list(self.spans.get_finished_spans()) + + def points(self) -> list[Point]: + """(meter, instrument name, unit, attributes, point) for every data point.""" + found: list[Point] = [] + data = self.reader.get_metrics_data() + for resource in data.resource_metrics if data else []: + for scope in resource.scope_metrics: + for metric in scope.metrics: + for point in metric.data.data_points: + found.append( + (scope.scope.name, metric.name, metric.unit or "", dict(point.attributes or {}), point) + ) + return found + + +def counts(sink: Telemetry, operation: str) -> list[Point]: + return [p for p in sink.points() if p[1] == "agentexperience.operation.count" and p[3]["operation"] == operation] + + +class Embedder: + model_id = "telemetry-test" + dimension = 4 + + def embed(self, text: str) -> list[float]: + return [1.0, 0.0, 0.0, 0.0] + + +def exits(*values: str) -> Any: + queue = list(values) + + def run_tests() -> str: + """Runs the tests.""" + return queue.pop(0) if len(queue) > 1 else queue[0] + + return run_tests + + +def learning_agent(store: Any, recorded: list[InjectionRecordResult] | None = None, **options: Any) -> Agent: + agent = Agent( + client=ScriptedClient(), + name="triage-ticket", + tools=[tool(exits("exit=3", "exit=0"), name="run_tests")], + ) + return use_agent_experience( + agent, + store=store, + identity=development_identity(application="tests"), + environment=ENVIRONMENT, + verify=tool_succeeded("run_tests"), + classify_tool_result=exit_code("run_tests"), + on_injection_recorded=(recorded.append if recorded is not None else None), + **options, + ) + + +def drive(agent: Agent, first: str = TICKET, second: str = NEXT_TICKET) -> None: + async def both() -> None: + await agent.run(first) + await agent.run(second) + (provider,) = [item for item in agent.context_providers if isinstance(item, ExperienceContextProvider)] + await provider.wait_for_injection_records() + await provider.wait_for_index_writes() + + run(both()) + + +# ---- The names ----------------------------------------------------------------------------------------------------- + + +def test_the_ports_table_of_names_is_in_the_net_generated_file() -> None: + assert {observer.CORE_SOURCE, observer.MAF_SOURCE} == set(NAMES["sources"]) + assert telemetry.SPAN_NAME_PREFIX == NAMES["spanNamePrefix"] + assert observer.FAULTED == NAMES["faultedOutcome"] + assert list(observer.ERROR_CLASSES) == NAMES["errorClasses"] + assert list(telemetry.DIMENSIONS) == NAMES["dimensions"] + assert [ + {"name": name, "kind": kind, "unit": unit} for name, (kind, unit) in telemetry.INSTRUMENTS.items() + ] == NAMES["instruments"] + # Every span attribute the adapter can write is a documented one. + assert telemetry.SPAN_ATTRIBUTES <= set(NAMES["attributes"]) + assert set(observer.ATTRIBUTES) <= set(NAMES["attributes"]) + for name, spec in observer.OPERATIONS.items(): + pinned = OPERATIONS[name] + assert spec.source == pinned["source"], name + assert spec.span == pinned["span"], name + assert set(spec.outcomes) <= set(pinned["outcomes"]), name + assert set(observer.NEVER_NESTED) <= set(observer.OPERATIONS) + + +def test_the_port_reports_no_operation_it_does_not_perform() -> None: + forbidden = { + "retention.sweep", + "delete", + "grant.purge", + "grant.access.purge", + "record.seal", + "record.authorship.backfill", + "lifecycle.commit", + "confidence.apply", + "reuse_feedback", + "deindex", + "reindex", + } + assert not forbidden & set(observer.OPERATIONS) + assert set(observer.OPERATIONS) == { + "capture.start_run", + "capture.append_attempt", + "capture.complete_run", + "verify", + "reflect", + "finalize", + "retrieve", + "index", + "inject", + "record_injection", + } + + +def test_neither_the_core_nor_the_agent_integration_imports_opentelemetry() -> None: + package = Path(telemetry.__file__).parent + for source in package.glob("*.py"): + if source.name == "telemetry.py": + continue + assert "opentelemetry" not in source.read_text(encoding="utf-8"), source.name + code = ( + "import sys\n" + "sys.modules['opentelemetry'] = None\n" + "import agentexperience, agentexperience.maf, agentexperience.inmemory, agentexperience.observer\n" + "assert not any(name.startswith('opentelemetry') and sys.modules[name] for name in sys.modules)\n" + ) + done = subprocess.run([sys.executable, "-I", "-c", code], capture_output=True, text=True, check=False) + assert done.returncode == 0, done.stderr + + +# ---- What an operator receives ------------------------------------------------------------------------------------- + + +def test_an_agent_that_injects_captures_and_records_an_injection_emits_exactly_the_pinned_names() -> None: + sink = Telemetry() + recorded: list[InjectionRecordResult] = [] + drive(learning_agent(InMemoryExperienceStore(), recorded, embedder=Embedder()), TICKET, NEXT_TICKET) + assert [item.outcome for item in recorded] == ["Recorded"] + + spans = sink.finished() + names = {span.name for span in spans} + assert { + "agentexperience.inject", + "agentexperience.retrieve", + "agentexperience.capture.start_run", + "agentexperience.capture.append_attempt", + "agentexperience.capture.complete_run", + "agentexperience.verify", + "agentexperience.reflect", + "agentexperience.finalize", + "agentexperience.index", + } <= names + for span in spans: + attributes = attrs(span) + operation = attributes["agentexperience.operation"] + assert span.name == NAMES["spanNamePrefix"] + operation + assert OPERATIONS[operation]["span"] is True + assert attributes["agentexperience.outcome"] in OPERATIONS[operation]["outcomes"] + ["Faulted"] + assert set(attributes) <= set(NAMES["attributes"]) + assert span.status.status_code.name == "OK" + assert span.kind.name == "INTERNAL" + assert "agentexperience.record_injection" not in names # measurements only, no span + + points = sink.points() + assert points + instruments = {item["name"]: item["unit"] for item in NAMES["instruments"]} + sources = set(NAMES["sources"]) + seen: set[str] = set() + for meter, name, unit, attributes, _ in points: + assert meter in sources + assert instruments[name] == unit + assert set(attributes) <= {"operation", "outcome", "nested"}, "no failure happened, so no error.class" + operation = attributes["operation"] + assert meter == OPERATIONS[operation]["source"] + assert attributes["outcome"] in OPERATIONS[operation]["outcomes"] + assert isinstance(attributes["nested"], bool) + seen.add(operation) + assert seen == set(observer.OPERATIONS) + assert {name for _, name, _, _, _ in points} == { + "agentexperience.operation.count", + "agentexperience.operation.duration", + } + + # The retrieval inside an injection is the Core source's, not nested; verify and reflect inside finalize are. + nested = {(attributes["operation"], attributes["nested"]) for _, _, _, attributes, _ in points} + assert ("retrieve", False) in nested and ("inject", False) in nested and ("record_injection", False) in nested + assert ("verify", True) in nested and ("reflect", True) in nested and ("finalize", False) in nested + assert ("capture.append_attempt", False) in nested + + # The nested spans are children of the span that ran them. + by_name = {span.name: span for span in spans} + finalize_span = by_name["agentexperience.finalize"] + assert parent_id(by_name["agentexperience.verify"]) == finalize_span.context.span_id + assert parent_id(by_name["agentexperience.reflect"]) == finalize_span.context.span_id + assert attrs(finalize_span)["agentexperience.stage"] == "CreateRecord" + assert parent_id(by_name["agentexperience.retrieve"]) == by_name["agentexperience.inject"].context.span_id + + +def test_inject_is_nothing_to_inject_the_first_time_and_injected_the_second() -> None: + sink = Telemetry() + drive(learning_agent(InMemoryExperienceStore())) + outcomes = [ + attrs(span)["agentexperience.outcome"] for span in sink.finished() if span.name == "agentexperience.inject" + ] + assert outcomes == ["NothingToInject", "Injected"] + assert [ + attrs(span)["agentexperience.outcome"] for span in sink.finished() if span.name == "agentexperience.retrieve" + ] == ["Completed", "Completed"] + + +def test_a_failing_search_is_retrieval_failed_and_reaches_no_failure_counter_because_nothing_threw() -> None: + class Broken(InMemoryExperienceStore): + def search(self, bounds: Any, parameters: Any) -> Any: + raise ConnectionError(f"the database is down {MARKER}") + + sink = Telemetry() + drive(learning_agent(Broken())) + inject = [span for span in sink.finished() if span.name == "agentexperience.inject"] + assert [attrs(span)["agentexperience.outcome"] for span in inject] == ["RetrievalFailed"] * 2 + retrieve = [span for span in sink.finished() if span.name == "agentexperience.retrieve"] + assert [attrs(span)["agentexperience.outcome"] for span in retrieve] == ["Failed"] * 2 + assert not [point for point in sink.points() if point[1] == "agentexperience.operation.failures"] + + +def test_an_operation_that_raises_is_faulted_with_its_error_class_and_type_name_only() -> None: + sink = Telemetry() + scope = Scope("tenant", "app", "project") + capture = RunCapture.start( + uuid.uuid4(), "task", None, scope, ENVIRONMENT, Provenance("test", "1", Instant.now(), None), Instant.now() + ) + with pytest.raises(ValueError): + finalize( + capture.run, # not finished + closed_round=None, + required_checks=[], + evidence=[], + current_artifact_revision=f"revision {MARKER}", + finalized_at=Instant.now(), + ) + + (span,) = [item for item in sink.finished() if item.name == "agentexperience.finalize"] + assert attrs(span)["agentexperience.outcome"] == "Faulted" + assert attrs(span)["agentexperience.error.class"] == "Unexpected" + assert attrs(span)["error.type"] == "ValueError" + assert attrs(span)["agentexperience.stage"] == "Load" + assert span.status.status_code.name == "ERROR" + assert not span.events + (failure,) = [point for point in sink.points() if point[1] == "agentexperience.operation.failures"] + assert dict(failure[3]) == {"operation": "finalize", "error.class": "Unexpected", "nested": False} + assert failure[4].value == 1 + counted = [ + p for p in sink.points() if p[1] == "agentexperience.operation.count" and p[3]["operation"] == "finalize" + ] + assert [dict(p[3])["outcome"] for p in counted] == ["Faulted"] + + +def test_record_injection_failure_classes_follow_the_net_table() -> None: + class Broken(InMemoryExperienceStore): + def record_injection(self, bounds: Any, scope: Any, experience_ids: Any, injected_at: Any) -> None: + raise ConnectionError("down") + + class SearchOnly: + def __init__(self) -> None: + self.inner = InMemoryExperienceStore() + + def insert_finalized(self, bounds: Any, finalized: Any, *, actor: str) -> None: + self.inner.insert_finalized(bounds, finalized, actor=actor) + + def search(self, bounds: Any, parameters: Any) -> Any: + return self.inner.search(bounds, parameters) + + sink = Telemetry() + drive(learning_agent(Broken())) + drive(learning_agent(SearchOnly())) + counted = sorted(p[3]["outcome"] for p in counts(sink, "record_injection")) + assert counted == ["NotRecorded", "NotTracked"] + (failure,) = [p for p in sink.points() if p[1] == "agentexperience.operation.failures"] + assert dict(failure[3]) == {"operation": "record_injection", "error.class": "Infrastructure", "nested": False} + + +def test_nothing_of_the_run_reaches_a_span_or_a_metric_dimension() -> None: + sink = Telemetry() + drive(learning_agent(InMemoryExperienceStore()), f"Triage {MARKER} please", f"Triage {MARKER} again") + texts = [str(value) for span in sink.finished() for value in attrs(span).values()] + texts += [str(value) for _, _, _, attributes, _ in sink.points() for value in attributes.values()] + texts += [span.name for span in sink.finished()] + assert texts + assert not [text for text in texts if MARKER in text] + + +# ---- Telemetry never changes a result ------------------------------------------------------------------------------ + + +class RaisingObserver: + def __init__(self) -> None: + self.calls = 0 + + def start(self, source: str, operation: str) -> object | None: + self.calls += 1 + raise RuntimeError("start") + + def finish(self, *_: Any, **__: Any) -> None: + self.calls += 1 + raise RuntimeError("finish") + + +class HalfRaisingObserver: + def start(self, source: str, operation: str) -> object | None: + return object() + + def finish(self, *_: Any, **__: Any) -> None: + raise RuntimeError("finish") + + +@pytest.mark.parametrize("hook", [RaisingObserver, HalfRaisingObserver]) +def test_an_observer_that_raises_changes_nothing(hook: type) -> None: + def outcome() -> tuple[list[Any], list[str]]: + store = InMemoryExperienceStore() + recorded: list[InjectionRecordResult] = [] + agent = learning_agent(store, recorded, embedder=Embedder()) + injected: list[Any] = [] + agent.context_providers[-1].settings.on_injected = injected.append # type: ignore[attr-defined] + drive(agent) + return [(item.byte_count, len(item.experience_ids)) for item in injected], [item.outcome for item in recorded] + + expected = outcome() + observed = hook() + observer.set_observer(observed) + assert outcome() == expected + if isinstance(observed, RaisingObserver): + assert observed.calls > 0 + + +def test_with_no_observer_the_hook_allocates_nothing_and_yields_none() -> None: + assert observer.get_observer() is None + first = observer.observe("verify") + second = observer.observe("finalize") + assert first is second # one shared no-op + with first as op: + assert op is None + + +def test_classification_mirrors_net() -> None: + assert observer.classify(asyncio.CancelledError()) == "Cancelled" + assert observer.classify(TimeoutError()) == "Timeout" + assert observer.classify(asyncio.TimeoutError()) == "Timeout" + assert observer.classify(ConnectionError()) == "Infrastructure" + assert observer.classify(KeyError("x")) == "Unexpected" + assert observer.type_name(ValueError("secret")) == "ValueError" + + +def test_a_cancelled_operation_is_faulted_and_cancelled() -> None: + sink = Telemetry() + + async def cancelled() -> None: + with observer.observe("inject"): + raise asyncio.CancelledError + + with pytest.raises(asyncio.CancelledError): + run(cancelled()) + (failure,) = [p for p in sink.points() if p[1] == "agentexperience.operation.failures"] + assert dict(failure[3]) == {"operation": "inject", "error.class": "Cancelled", "nested": False} diff --git a/python/uv.lock b/python/uv.lock index 7cb0599..b6fc81a 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -34,6 +34,9 @@ source = { editable = "." } maf = [ { name = "agent-framework-core" }, ] +otel = [ + { name = "opentelemetry-api" }, +] postgres = [ { name = "psycopg", extra = ["binary"] }, ] @@ -43,6 +46,8 @@ dev = [ { name = "agent-framework-core" }, { name = "jsonschema" }, { name = "mypy" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, { name = "psycopg", extra = ["binary"] }, { name = "pytest" }, { name = "ruff" }, @@ -52,15 +57,18 @@ dev = [ [package.metadata] requires-dist = [ { name = "agent-framework-core", marker = "extra == 'maf'", specifier = ">=1.0.0" }, + { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.27" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgres'", specifier = ">=3.2" }, ] -provides-extras = ["postgres", "maf"] +provides-extras = ["postgres", "maf", "otel"] [package.metadata.requires-dev] dev = [ { name = "agent-framework-core", specifier = ">=1.0.0" }, { name = "jsonschema", specifier = ">=4.23" }, { name = "mypy", specifier = ">=1.13" }, + { name = "opentelemetry-api", specifier = ">=1.27" }, + { name = "opentelemetry-sdk", specifier = ">=1.27" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2" }, { name = "pytest", specifier = ">=8" }, { name = "ruff", specifier = ">=0.8" }, @@ -709,6 +717,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/41/f7dcf80b81ee8e71c1a2b59f14208bc723edbd89ed027a73b175abf6348e/opentelemetry_api-1.45.1-py3-none-any.whl", hash = "sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb", size = 60256, upload-time = "2026-10-06T17:32:33.506Z" }, ] +[[package]] +name = "opentelemetry-sdk" +version = "1.45.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/79/7392e21a1c8f0c61d90b223e31c7e48cb9d452e91a6b820ad24cca5f23c4/opentelemetry_sdk-1.45.1.tar.gz", hash = "sha256:63d24a6ca645019a631e6a51999c73e93adcac1196ca640b8ae78a7cc4762bf3", size = 218324, upload-time = "2026-10-06T17:33:13.26Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/95/3c/87c42b4bd6dd297536f04cd9383d212ac557ecd49f2cbdcd46da1c9ef5c8/opentelemetry_sdk-1.45.1-py3-none-any.whl", hash = "sha256:c604c11dc429810812348989115fa44bd558772a3d7442afc43d024f2c250ca4", size = 140063, upload-time = "2026-10-06T17:32:55.04Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.66b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/46/e4/dbbfb2a010c4db2224a5114638acede6fe563d33cc20fb1752cebcbe6298/opentelemetry_semantic_conventions-0.66b1.tar.gz", hash = "sha256:497ca63bf383723411e8eaf60c8779e9877633c936bb641080adab59d0eb6ec8", size = 150250, upload-time = "2026-10-06T17:33:14.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/14/67f8aa798857f8cf686f515bf93d9bb877ce952ddc8efae0fa25b45ce0d6/opentelemetry_semantic_conventions-0.66b1-py3-none-any.whl", hash = "sha256:d4cddeb4315490b35213f55e2bdc9ac54bb1e4d318927475bed62b35545e581b", size = 206279, upload-time = "2026-10-06T17:32:56.103Z" }, +] + [[package]] name = "packaging" version = "26.3" diff --git a/tests/AgentExperience.MicrosoftAgentFramework.Tests/Diagnostics/TelemetryNamesFileTests.cs b/tests/AgentExperience.MicrosoftAgentFramework.Tests/Diagnostics/TelemetryNamesFileTests.cs new file mode 100644 index 0000000..41f9d63 --- /dev/null +++ b/tests/AgentExperience.MicrosoftAgentFramework.Tests/Diagnostics/TelemetryNamesFileTests.cs @@ -0,0 +1,180 @@ +using System.Diagnostics.Metrics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using AgentExperience.Core.Finalization; +using AgentExperience.Core.Indexing; +using AgentExperience.Core.Lifecycle; +using AgentExperience.Core.Retrieval; +using AgentExperience.Core.Feedback; +using AgentExperience.MicrosoftAgentFramework.Injection; +using AgentExperience.MicrosoftAgentFramework.Diagnostics; + +namespace AgentExperience.MicrosoftAgentFramework.Tests.Diagnostics; + +/// +/// Story 23.5: docs/telemetry-names.json is generated from the .NET definitions and pins the names the Python and +/// Go ports may emit. It never updates itself: set AGENTEXPERIENCE_TELEMETRY_NAMES_UPDATE=1 to rewrite it, then +/// read the diff and commit it on purpose. The ports' own tests load this file. +/// +public class TelemetryNamesFileTests +{ + private const string UpdateVariable = "AGENTEXPERIENCE_TELEMETRY_NAMES_UPDATE"; + + private static readonly Type CoreHolder = + typeof(DefaultSanitizer).Assembly.GetType("AgentExperience.Core.Diagnostics.ExperienceDiagnostics", throwOnError: true)!; + + private static readonly Type CoreOperationNames = + typeof(DefaultSanitizer).Assembly.GetType("AgentExperience.Core.Diagnostics.ExperienceOperationNames", throwOnError: true)!; + + private static readonly Type CoreErrorClass = + typeof(DefaultSanitizer).Assembly.GetType("AgentExperience.Core.Diagnostics.ExperienceOperationErrorClass", throwOnError: true)!; + + [Fact] + public void The_names_file_matches_the_definitions() + { + var generated = Generate(); + var path = Path.Combine(DocsDirectory(), "telemetry-names.json"); + + if (Environment.GetEnvironmentVariable(UpdateVariable) == "1") + { + File.WriteAllText(path, generated, new UTF8Encoding(false)); + return; + } + + Assert.True(File.Exists(path), $"docs/telemetry-names.json is missing; set {UpdateVariable}=1 to generate it."); + Assert.Equal(generated, File.ReadAllText(path, Encoding.UTF8)); + } + + private static string Generate() + { + var coreOperations = new (string Constant, Type Outcomes)[] + { + ("CaptureStartRun", typeof(StartRunOutcome)), + ("CaptureAppendAttempt", typeof(AppendAttemptOutcome)), + ("CaptureCompleteRun", typeof(CompleteRunOutcome)), + ("Verify", typeof(TaskVerificationStatus)), + ("Reflect", typeof(TaskVerificationStatus)), + ("Finalize", typeof(FinalizationOutcome)), + ("LifecycleCommit", typeof(LifecycleTransitionOutcome)), + ("ConfidenceApply", typeof(ConfidenceUpdateOutcome)), + ("Retrieve", typeof(RetrievalOutcome)), + ("Index", typeof(ExperienceIndexingOutcome)), + ("Deindex", typeof(ExperienceDeindexingOutcome)), + ("Reindex", typeof(ExperienceReindexOutcome)), + ("ReuseFeedback", typeof(ExperienceReuseFeedbackOutcome)), + }; + + var coreSource = Str(CoreHolder, "SourceName"); + var adapterSource = InjectionDiagnostics.SourceName; + var prefix = Str(CoreHolder, "SpanNamePrefix"); + + using var stream = new MemoryStream(); + using (var json = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = true, NewLine = "\n" })) + { + json.WriteStartObject(); + json.WriteString("generatedFrom", "tests/AgentExperience.MicrosoftAgentFramework.Tests/Diagnostics/TelemetryNamesFileTests.cs"); + json.WriteStartArray("sources"); + json.WriteStringValue(coreSource); + json.WriteStringValue(adapterSource); + json.WriteEndArray(); + json.WriteString("spanNamePrefix", prefix); + json.WriteString("faultedOutcome", Str(CoreHolder, "FaultedOutcome")); + + json.WriteStartArray("instruments"); + foreach (var (field, kind) in new[] { ("Operations", "counter"), ("Durations", "histogram"), ("Failures", "counter") }) + { + var instrument = (Instrument)CoreHolder.GetField(field, BindingFlags.NonPublic | BindingFlags.Static)!.GetValue(null)!; + json.WriteStartObject(); + json.WriteString("name", instrument.Name); + json.WriteString("kind", kind); + json.WriteString("unit", instrument.Unit); + json.WriteEndObject(); + } + + json.WriteEndArray(); + + json.WriteStartArray("dimensions"); + foreach (var name in new[] { "OperationDimension", "OutcomeDimension", "ErrorClassDimension", "NestedDimension" }) + { + json.WriteStringValue(Str(CoreHolder, name)); + } + + json.WriteEndArray(); + + json.WriteStartArray("errorClasses"); + foreach (var name in Enum.GetNames(CoreErrorClass)) + { + json.WriteStringValue(name); + } + + json.WriteEndArray(); + + var attributes = Constants(CoreHolder, "Attribute").Concat(Constants(typeof(InjectionDiagnostics), "Attribute")) + .Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToList(); + json.WriteStartArray("attributes"); + foreach (var attribute in attributes) + { + json.WriteStringValue(attribute); + } + + json.WriteEndArray(); + + json.WriteStartArray("operations"); + foreach (var (constant, outcomes) in coreOperations) + { + Operation(json, Str(CoreOperationNames, constant), coreSource, true, Enum.GetNames(outcomes)); + } + + Operation(json, InjectionDiagnostics.Inject, adapterSource, true, Enum.GetNames()); + Operation( + json, + InjectionDiagnostics.RecordInjection, + adapterSource, + false, + [ + InjectionDiagnostics.RecordedOutcome, + InjectionDiagnostics.NotTrackedOutcome, + InjectionDiagnostics.NotRecordedOutcome, + InjectionDiagnostics.OutcomeUnknownOutcome, + ]); + json.WriteEndArray(); + + json.WriteEndObject(); + } + + return Encoding.UTF8.GetString(stream.ToArray()) + "\n"; + } + + private static void Operation(Utf8JsonWriter json, string operation, string source, bool span, string[] outcomes) + { + json.WriteStartObject(); + json.WriteString("operation", operation); + json.WriteString("source", source); + json.WriteBoolean("span", span); + json.WriteStartArray("outcomes"); + foreach (var outcome in outcomes) + { + json.WriteStringValue(outcome); + } + + json.WriteEndArray(); + json.WriteEndObject(); + } + + private static string Str(Type holder, string name) => + (string)holder.GetField(name, BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static)!.GetRawConstantValue()!; + + private static IEnumerable Constants(Type holder, string suffix) => + holder.GetFields(BindingFlags.NonPublic | BindingFlags.Public | BindingFlags.Static) + .Where(field => field.IsLiteral && field.FieldType == typeof(string) && field.Name.EndsWith(suffix, StringComparison.Ordinal)) + .Select(field => (string)field.GetRawConstantValue()!); + + private static string DocsDirectory([CallerFilePath] string thisFile = "") + { + var docs = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(thisFile)!, "..", "..", "..", "docs")); + Assert.True(File.Exists(Path.Combine(docs, "telemetry.md")), $"'{docs}' is not the docs directory."); + return docs; + } +} diff --git a/tests/AgentExperience.Release.Tests/Versioning/VersionAgreementTests.cs b/tests/AgentExperience.Release.Tests/Versioning/VersionAgreementTests.cs index 2c30127..c3225d0 100644 --- a/tests/AgentExperience.Release.Tests/Versioning/VersionAgreementTests.cs +++ b/tests/AgentExperience.Release.Tests/Versioning/VersionAgreementTests.cs @@ -4,7 +4,7 @@ namespace AgentExperience.Release.Tests.Versioning; /// -/// Story 22.7: one tag releases the six NuGet packages, the Python distribution and the five Go modules at one version. +/// Story 22.7: one tag releases the six NuGet packages, the Python distribution and the six Go modules at one version. /// Directory.Build.props sets it; python/pyproject.toml (and the lock file's record of it) and every Go /// module's requirement on another of this repository's modules carry the version derived from it, never a /// pseudo-version. release.yml's verify job runs the same check on the tagged commit. @@ -21,6 +21,7 @@ public sealed class VersionAgreementTests ["go/postgres"] = ["go"], ["go/postgres/pgtest"] = ["go", "go/postgres"], ["go/maf"] = ["go", "go/inmemory", "go/postgres", "go/postgres/pgtest"], + ["go/telemetry"] = ["go", "go/inmemory", "go/maf"], }; [Theory] @@ -174,12 +175,12 @@ public void The_release_workflow_expects_the_same_Go_sibling_sets() /// /// The publish job's "Tag the Go modules" script, run for real against a stub gh first on PATH: an absent tag - /// (a 404) is created at the release commit for all five modules; a tag already at that commit, directly or through + /// (a 404) is created at the release commit for all six modules; a tag already at that commit, directly or through /// an annotated tag, is kept; a tag at another commit, or any lookup error but a 404, fails before anything is /// created. /// [UnixTheory] - [InlineData("absent", "", 0, 5)] + [InlineData("absent", "", 0, 6)] [InlineData("commit " + ReleaseSha, "", 0, 0)] [InlineData("tag 2222222222222222222222222222222222222222", ReleaseSha, 0, 0)] [InlineData("commit 3333333333333333333333333333333333333333", "", 1, 0)] diff --git a/tests/AgentExperience.Release.Tests/Workflows/WorkflowTests.cs b/tests/AgentExperience.Release.Tests/Workflows/WorkflowTests.cs index adfde76..2dcd677 100644 --- a/tests/AgentExperience.Release.Tests/Workflows/WorkflowTests.cs +++ b/tests/AgentExperience.Release.Tests/Workflows/WorkflowTests.cs @@ -414,7 +414,7 @@ public void The_release_workflow_uses_the_same_first_party_actions_as_CI() [InlineData(" name: python-dist\n path: dist\n", " name: packages\n path: dist\n")] // Story 22.7: the Go tags. [InlineData("if [ \"$sha\" != \"$GITHUB_SHA\" ]; then", "if false; then")] - [InlineData("for module in go go/inmemory go/postgres go/postgres/pgtest go/maf; do", "for module in go; do")] + [InlineData("for module in go go/inmemory go/postgres go/postgres/pgtest go/maf go/telemetry; do", "for module in go; do")] [InlineData("https://proxy.golang.org/$escaped/@v/$GO_VERSION.info", "https://example.invalid/$escaped")] [InlineData("-f \"sha=$GITHUB_SHA\"", "-f \"sha=$(git rev-parse HEAD)\"")] // Story 22.7: the verify job's version agreement, Python and Go checks, and that it publishes nothing. @@ -436,8 +436,8 @@ public void The_release_workflow_uses_the_same_first_party_actions_as_CI() [InlineData("uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0\n with:\n name: packages", "uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093\n with:\n name: packages")] [InlineData("actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.4.0", "actions/setup-dotnet@v5")] // Story 22.7: the Go tags and the proxy check. - [InlineData("for module in go go/inmemory go/postgres go/postgres/pgtest go/maf; do\n tag=", "for module in go go/maf; do\n tag=")] - [InlineData("for module in go go/inmemory go/postgres go/postgres/pgtest go/maf; do\n path=", "for module in go; do\n path=")] + [InlineData("for module in go go/inmemory go/postgres go/postgres/pgtest go/maf go/telemetry; do\n tag=", "for module in go go/maf; do\n tag=")] + [InlineData("for module in go go/inmemory go/postgres go/postgres/pgtest go/maf go/telemetry; do\n path=", "for module in go; do\n path=")] [InlineData("elif grep -q 'HTTP 404' \"$error\"; then", "elif true; then")] [InlineData("--connect-timeout 10 --max-time 30 ", "")] [InlineData("if [ \"$attempt\" -lt 10 ]; then sleep 30; fi", "sleep 30")] @@ -601,14 +601,14 @@ void Require(bool condition, string rule) // The Go tags: after the push, before the GitHub release, one per module at the verified version, created at the // tagged commit through the API (no checkout, no git push), idempotent only for a tag already at that commit, and // confirmed by the Go module proxy. - const string GoModules = "for module in go go/inmemory go/postgres go/postgres/pgtest go/maf; do"; + const string GoModules = "for module in go go/inmemory go/postgres go/postgres/pgtest go/maf go/telemetry; do"; const string CreateTag = "gh api \"repos/$GH_REPO/git/refs\" -f \"ref=refs/tags/$tag\" -f \"sha=$GITHUB_SHA\""; var tagsAt = publish.IndexOf(CreateTag, StringComparison.Ordinal); var proxyAt = publish.IndexOf("https://proxy.golang.org/$escaped/@v/$GO_VERSION.info", StringComparison.Ordinal); foreach (var (text, rule) in new[] { ("GO_VERSION: ${{ needs.verify.outputs.go-version }}", "take the Go version verify derived"), - (GoModules, "tag exactly the five Go modules"), + (GoModules, "tag exactly the six Go modules"), ("tag=\"$module/$GO_VERSION\"", "name each tag module/version"), ("gh api \"repos/$GH_REPO/git/ref/tags/$tag\"", "look each tag up first"), ("if [ \"$sha\" != \"$GITHUB_SHA\" ]; then", "refuse a tag that exists at another commit"), @@ -621,7 +621,7 @@ void Require(bool condition, string rule) ("path=\"github.com/fabbrik/AgentExperience.NET/$module\"", "ask the proxy for the repository's module paths"), ("curl -fsS --connect-timeout 10 --max-time 30 \"$url\"", "bound each request to the proxy"), ("if [ \"$attempt\" -lt 10 ]; then sleep 30; fi", "not sleep after the last attempt"), - // Worst case, the push to nuget.org plus 47.5 minutes of proxy retries. + // Worst case, the push to nuget.org plus 57 minutes of proxy retries. ("\n timeout-minutes: 75\n", "allow the push and the whole proxy wait"), }) { @@ -630,7 +630,7 @@ void Require(bool condition, string rule) Require( Regex.Matches(publish, Regex.Escape(GoModules), RegexOptions.CultureInvariant).Count == 2, - "The publish job must loop over exactly the five Go modules twice: once to tag them and once to ask the proxy."); + "The publish job must loop over exactly the six Go modules twice: once to tag them and once to ask the proxy."); Require(Regex.Matches(workflow, @"git/refs", RegexOptions.CultureInvariant).Count == 1, "Only the publish job's Go tag step may create a ref."); @@ -682,7 +682,7 @@ void Require(bool condition, string rule) ("grep -qx \"Name: agenticexperience\"", "check the wheel names agenticexperience"), ("grep -qx \"Version: $PYTHON_VERSION\"", "check the wheel carries the derived version"), ("PYTHON_VERSION: ${{ steps.version.outputs.python-version }}", "check the distribution against the derived Python version"), - ("for module in . inmemory postgres postgres/pgtest maf; do", "check every Go module"), + ("for module in . inmemory postgres postgres/pgtest maf telemetry; do", "check every Go module"), ("go vet ./...\n", "vet the Go modules"), ("go test ./...\n", "test the Go modules"), })