diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cd1790a..a957fd2 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6,29 +6,52 @@ on: - "research/**" - "ideas/**" - ".github/scripts/validate_notes.py" + - "scripts/**" + - "generated/**" + - "tests/**" - ".github/workflows/lint.yml" + - ".github/workflows/pages.yml" push: branches: [main] paths: - "research/**" - "ideas/**" - ".github/scripts/validate_notes.py" + - "scripts/**" + - "generated/**" + - "tests/**" - ".github/workflows/lint.yml" + - ".github/workflows/pages.yml" + +permissions: + contents: read jobs: validate: runs-on: ubuntu-latest steps: - name: Check out - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" - name: Install dependencies - run: pip install "pyyaml>=6" + run: pip install "pyyaml==6.0.2" - name: Validate notes and ideas run: python .github/scripts/validate_notes.py + + - name: Check rating distributions + run: python scripts/summarize_ratings.py --check + + - name: Test map generator + run: python -m unittest discover -s tests + + - name: Generate research map + run: python scripts/generate_research_map.py + + - name: Verify generated map is checked in + run: git diff --exit-code -- generated/research-map.html diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml new file mode 100644 index 0000000..c176d02 --- /dev/null +++ b/.github/workflows/pages.yml @@ -0,0 +1,80 @@ +name: Deploy research map to Pages + +on: + push: + branches: [main] + paths: + - "research/**" + - "ideas/**" + - "scripts/**" + - "generated/**" + - "tests/**" + - ".github/workflows/pages.yml" + - ".github/workflows/lint.yml" + +permissions: + contents: read + +concurrency: + group: pages + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Check out + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install dependencies + run: pip install "pyyaml==6.0.2" + + - name: Validate notes and ideas + run: python .github/scripts/validate_notes.py + + - name: Check rating distributions + run: python scripts/summarize_ratings.py --check + + - name: Test map generator + run: python -m unittest discover -s tests + + - name: Generate research map + run: python scripts/generate_research_map.py + + - name: Verify generated map is checked in + run: git diff --exit-code -- generated/research-map.html + + - name: Create Pages site + run: mkdir _site + + - name: Copy research map to site index + run: cp generated/research-map.html _site/index.html + + - name: Configure Pages + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 + with: + path: _site + + deploy: + needs: build + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 diff --git a/docs/superpowers/plans/2026-09-02-map-overlap-clustering.md b/docs/superpowers/plans/2026-09-02-map-overlap-clustering.md new file mode 100644 index 0000000..14d1999 --- /dev/null +++ b/docs/superpowers/plans/2026-09-02-map-overlap-clustering.md @@ -0,0 +1,166 @@ +# Map Overlap Clustering Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make exact-overlap notes render as count-bearing cluster markers with a picker overlay that keeps every note accessible. + +**Architecture:** The Python generator will group each matrix's placed payload by exact `(x, y)` coordinates and emit either a singleton marker or a cluster marker carrying its note IDs. Browser JavaScript will use the marker's matrix and cluster data to render a picker, then reuse the existing detail rendering for the selected note. No scores are rounded or changed. + +**Tech Stack:** Python 3.12, PyYAML, unittest, generated vanilla HTML/CSS/JavaScript. + +--- + +## Files and Responsibilities + +- Modify `scripts/generate_research_map.py`: grouping helper, cluster marker HTML, picker data, and picker/detail interaction behavior. +- Modify `tests/test_generate_research_map.py`: exact-overlap, singleton, per-matrix, count, picker, and single-close-button tests. +- Regenerate `generated/research-map.html`: checked-in artifact containing the clustering behavior. + +### Task 1: Add testable overlap grouping + +**Files:** +- Modify: `tests/test_generate_research_map.py` +- Modify: `scripts/generate_research_map.py` + +- [ ] **Step 1: Write the failing grouping tests** + +Add tests for a helper named `group_payload`: + +```python +def test_group_payload_combines_only_exact_positions(): + payload = [ + {"id": "a", "position": {"x": 50, "y": 40}}, + {"id": "b", "position": {"x": 50, "y": 40}}, + {"id": "c", "position": {"x": 51, "y": 40}}, + ] + groups = group_payload(payload) + self.assertEqual([item["id"] for item in groups[(50, 40)]], ["a", "b"]) + self.assertEqual([item["id"] for item in groups[(51, 40)]], ["c"]) + +def test_group_payload_leaves_unplaced_items_out_of_coordinate_groups(): + self.assertEqual(group_payload([{"id": "a", "position": None}]), {}) +``` + +- [ ] **Step 2: Run the focused tests and verify the expected failure** + +Run: `devbox run -- python -m unittest tests.test_generate_research_map.ResearchMapTests.test_group_payload_combines_only_exact_positions` + +Expected: FAIL with an import or attribute error because `group_payload` does not exist. + +- [ ] **Step 3: Implement exact grouping** + +Add `group_payload(payload)` that returns a dictionary keyed by `(position["x"], position["y"])`, skips items whose position is `None`, and preserves input order within each group. Do not round, bucket, jitter, or distance-cluster coordinates. + +- [ ] **Step 4: Run grouping tests** + +Run: `devbox run -- python -m unittest tests.test_generate_research_map.ResearchMapTests.test_group_payload_combines_only_exact_positions tests.test_generate_research_map.ResearchMapTests.test_group_payload_leaves_unplaced_items_out_of_coordinate_groups` + +Expected: PASS. + +- [ ] **Step 5: Commit the grouping behavior** + +Run: `git add scripts/generate_research_map.py tests/test_generate_research_map.py && git commit -m "test: define exact map overlap grouping"` + +### Task 2: Render singleton and cluster markers + +**Files:** +- Modify: `scripts/generate_research_map.py` +- Modify: `tests/test_generate_research_map.py` + +- [ ] **Step 1: Write failing HTML assertions** + +Add a test using two notes with identical ratings and one with a nearby rating. Assert generated HTML contains a cluster marker with `data-cluster`, a visible count, and an individual marker for the nearby note. Assert the cluster label includes its count and that exact coordinate values remain unchanged in the marker style. + +- [ ] **Step 2: Implement grouped marker generation** + +Within each matrix, call `group_payload` and render one normal marker for a singleton group. Render one `
+ +
Actors, lifecycle, decisions, and evidence
Primary actor
A platform customer deploying a coding harness on Cloud Foundry.
Beneficiary
A developer delegating repository changes and code execution to the harness.
Lifecycle
Stage a reusable environment; create an isolated mutable edit/test session; submit a candidate artifact; let the broker validate target, policy, provenance, and approval; then create conventional CF package, build, deployment, and revision resources, verify, audit, and clean up.
Authority boundary
The harness gets restricted Git, model, and package-registry credentials but no CAPI deployment authority; only the narrow trusted deployment broker holds restricted CAPI credentials and may promote an approved artifact to its validated target.
Failure domain
Session failure stays isolated and recoverable; stale-base or concurrent submissions fail safely; broker failure cannot broaden authority, and deployment supports rollback and audit.

Unique capabilities

  • Reuse an immutable staged environment while retaining isolated mutable workspace state across edit/test turns.
  • Produce a provenance-bound candidate artifact without granting the harness deployment authority.
  • Validate target, policy, provenance, and approval in a narrow broker before conventional CF deployment and revision creation.

RFC decisions

  • How session identity, mutable workspace lifecycle, isolation, networking, stale-base detection, and concurrency are represented.
  • What candidate artifact, provenance, target, policy, and approval contract the trusted deployment broker validates.
  • How restricted CAPI, Git, model, and package-registry credentials are issued, audited, revoked, and kept out of the wrong trust domain.
  • How conventional CF package, build, deployment, and revision creation exposes verification, rollback, and audit outcomes.

Primitive applicability

  • Durable, Addressable Execution: Supporting
  • Attested Workload Authority and Mediated Tool Access: Core
  • Session-Scoped Isolated Execution: Core

Core

Supporting

+ +
Actors, lifecycle, decisions, and evidence
Primary actor
An application developer deploying a multi-tenant agentic application to Cloud Foundry.
Beneficiary
An authenticated application user delegating a long-running task while retaining approval and audit control.
Lifecycle
Authenticate a user, create a durable execution, invoke authorized tools, checkpoint state, pause for approval or failure, resume on replaceable compute, and complete or cancel with an audit trail.
Authority boundary
The application acts with both workload identity and explicitly delegated user authority; tool access is scoped, auditable, revocable, and unavailable as ambient process credentials.
Failure domain
One tenant's execution, tool failure, or compromised prompt must not leak authority or state across users, and replacement compute must resume without duplicating external effects.

Unique capabilities

  • Address and resume framework-neutral agent executions independently of an application instance.
  • Delegate user authority to specific tools without placing durable credentials in the agent process.
  • Pause for human approval and recover from process or provider failure without repeating committed effects.

RFC decisions

  • Whether CF owns durable execution identity, events, timers, retries, suspension, and cancellation or integrates an external engine.
  • How workload identity, user delegation, tool authorization, token exchange, audit, and revocation fit CF APIs and UAA.
  • Which state, quota, scaling, telemetry, and optional isolated-worker contracts remain framework-neutral platform responsibilities.

Primitive applicability

  • Durable, Addressable Execution: Core
  • Attested Workload Authority and Mediated Tool Access: Core
  • Session-Scoped Isolated Execution: Conditional

Core

Supporting

Candidate platform primitives

+ +
Gap, experiments, and evidence
Current CF gap
CF tasks are tied to one compute attempt and lack stable execution identity, checkpoints, suspend/resume, durable events, timers, and retry policy.
Candidate POC
Run a checkpointing task under a stable execution ID, suspend it after persisting state, then resume it on replacement compute through events, timers, and bounded retries.
Candidate RFC scope
Define execution identity, lifecycle states, checkpoint handoff, event and timer delivery, retry semantics, observability, and the boundary between CF lifecycle ownership and bound state stores.

Core

Supporting

+ +
Gap, experiments, and evidence
Current CF gap
CF issues workload identity certificates but does not exchange them for scoped tool authority, keep third-party credentials out of workloads, mediate off-platform access, or record delegation-aware audit events.
Candidate POC
Exchange a Diego instance identity certificate for a short-lived scoped token, invoke one allowed tool through a credential proxy and egress mediator, deny another, and emit attributable audit events.
Candidate RFC scope
Define workload token exchange, authority and delegation claims, credential brokering, outbound mediation and policy enforcement, audit events, revocation, and integration boundaries for UAA, routing, and service brokers.

Core

Supporting

+ +
Gap, experiments, and evidence
Current CF gap
CF can stage apps and run ephemeral tasks but cannot cheaply compose a reusable environment with per-session workspace state, select stronger isolation, constrain session networking, or resume the session lifecycle.
Candidate POC
Start two isolated sessions from one content-addressed staged environment, attach separate mutable workspaces, apply per-session egress policy, stop one session, and resume it on fresh compute.
Candidate RFC scope
Define environment and workspace references, session identity and lifecycle, isolation classes, network policy, workspace persistence and cleanup, scheduling, quotas, and compatibility with existing CF staging and task APIs.

Core

Supporting

ResearchIdea

Platform Impact x Maturity

Emerging < Maturity > EstablishedLocal concern < Platform Impact > Platform-wide concern
Unplaced notes (0)
+
 + \ No newline at end of file diff --git a/ideas/TEMPLATE.md b/ideas/TEMPLATE.md index b5140f0..28ed02e 100644 --- a/ideas/TEMPLATE.md +++ b/ideas/TEMPLATE.md @@ -3,6 +3,19 @@ title: author: (@your-github-handle) date: 2026-01-01 tags: [] +ratings: + platform-impact: + value: 50 + note: "Explain the provisional platform-impact score." + maturity: + value: 50 + note: "Explain the provisional maturity score." + novelty: + value: 50 + note: "Explain the provisional novelty score." + actionability: + value: 50 + note: "Explain the provisional actionability score." --- + +## Optional map ratings + +Ratings are provisional working-group judgments on a 0-100 scale. Each value must have a +short justification in its `note` field. The generator uses named ratings to position notes on +plots, so future plots can reuse these ratings or introduce new ones. diff --git a/ideas/agent-failure-checkpointing.md b/ideas/agent-failure-checkpointing.md index ac9ef59..9e7384d 100644 --- a/ideas/agent-failure-checkpointing.md +++ b/ideas/agent-failure-checkpointing.md @@ -3,6 +3,20 @@ title: Agent Failure Checkpointing author: Arsalan Khan (@asalan316) date: 2026-08-13 tags: [runtime-lifecycle, sandboxing-isolation] +ratings: + platform-impact: + value: 75 + note: 'CF restarts crashed apps but cannot restore agent memory, task outputs, queue position, or bound-service session state from a platform-managed checkpoint.' + maturity: + value: 50 + note: 'Framework checkpointers demonstrate credible persistence and resume mechanisms, but transparent platform restoration across agent memory, queues, outputs, and bound-service sessions has no implementation or operational evidence here.' + novelty: + value: 25 + note: 'The proposal adapts the established checkpoint/restart pattern to agent conversation state, tool outputs, and work queues rather than introducing a new durability architecture.' + actionability: + value: 50 + note: 'The manifest sketch and invoice recovery example define desired behavior, but checkpoint granularity, state boundaries, storage, and multi-tenant quotas still require substantial scoping.' + --- ## The idea diff --git a/ideas/agent-identity-and-tool-authorization.md b/ideas/agent-identity-and-tool-authorization.md index c06213a..606b156 100644 --- a/ideas/agent-identity-and-tool-authorization.md +++ b/ideas/agent-identity-and-tool-authorization.md @@ -3,6 +3,20 @@ title: Agent identity and tool authorization — the platform as the agent's ide author: Wayne E. Seguin (@wayneeseguin) date: 2026-08-12 tags: [identity, inter-agent-comms, observability-governance] +ratings: + platform-impact: + value: 50 + note: "Diego identity certificates, UAA, CredHub, and accepted RFC-0055 cover much of the substrate, but CF lacks workload token exchange, a per-user token vault, and runtime tool-authorization policy." + maturity: + value: 75 + note: "Workload identity, OAuth token exchange, mTLS client authentication, and policy engines are production-proven, while the layer-3 AOAT delegation chain remains only an individual IETF draft." + novelty: + value: 50 + note: "Exchanging CF instance certificates for scoped workload tokens combines established identity standards in a CF-specific way; user-to-agent-to-tool delegation is the newer element." + actionability: + value: 100 + note: "Accepted RFC-0055 provides a no-new-component first step, and UAA PRs #3972 and #3968 give concrete POCs for certificate exchange and JWT-SVID issuance to evaluate." + --- ## The idea diff --git a/ideas/credential-less-agent-processes.md b/ideas/credential-less-agent-processes.md index 49e1a4a..a484c78 100644 --- a/ideas/credential-less-agent-processes.md +++ b/ideas/credential-less-agent-processes.md @@ -3,6 +3,20 @@ title: Credential-less agent processes author: Rashid Rashidov (@rrashidov) date: 2026-07-08 tags: [identity, sandboxing-isolation] +ratings: + platform-impact: + value: 50 + note: 'CF can bind and store service credentials, but it exposes them to the app process; a platform-held credential and localhost request proxy are missing.' + maturity: + value: 75 + note: 'Credential vaults and outbound credential proxies are production-capable patterns used by managed agent platforms, though the note leaves CF provisioning and multi-tenant bindings unresolved.' + novelty: + value: 25 + note: 'Keeping secrets in a local proxy is an established vault-and-sidecar architecture, here applied to prompt-injection risk in agent processes.' + actionability: + value: 50 + note: 'The standard-provider API over localhost supplies a plausible prototype boundary, but credential storage, sidecar provisioning, rotation, and per-user tenancy need design first.' + --- ## The idea diff --git a/ideas/dapr-aware-gorouter.md b/ideas/dapr-aware-gorouter.md index 755fbbc..7894164 100644 --- a/ideas/dapr-aware-gorouter.md +++ b/ideas/dapr-aware-gorouter.md @@ -3,6 +3,20 @@ title: Dapr-aware GoRouter — routing to the instance where the work lives author: Ruben Koster (@rkoster) date: 2026-08-11 tags: [inter-agent-comms, runtime-lifecycle, orchestration] +ratings: + platform-impact: + value: 50 + note: 'GoRouter already has instance-addressed routing, endpoint metadata, and NATS updates, but it cannot resolve an actor ID to the instance that currently owns the work.' + maturity: + value: 50 + note: 'Dapr placement and CF routing are production-capable ingredients, but no implementation or operational evidence demonstrates actor-to-instance resolution through GoRouter or safe behavior during placement migration.' + novelty: + value: 75 + note: 'Making a PaaS edge router consume or replace a virtual-actor placement table is an emerging combination, especially with authenticated actor-addressed routing through RFC-0055.' + actionability: + value: 75 + note: 'The shallow design bounds an experiment to daprd placement lookup plus X-CF-APP-INSTANCE, with explicit checks for host-to-index mapping, header propagation, and migration correctness.' + --- ## The idea diff --git a/ideas/dapr-durable-execution-on-cf.md b/ideas/dapr-durable-execution-on-cf.md index 680cc51..9d69ff3 100644 --- a/ideas/dapr-durable-execution-on-cf.md +++ b/ideas/dapr-durable-execution-on-cf.md @@ -3,6 +3,20 @@ title: Dapr durable execution on CF, built on CF's own identity and config primi author: Ruben Koster (@rkoster) date: 2026-08-11 tags: [runtime-lifecycle, orchestration, identity, inter-agent-comms] +ratings: + platform-impact: + value: 75 + note: "CF has identity, bindings, process injection, and partial placement primitives, but it has no workflow engine, virtual actors, durable scheduler, timers, or reminders." + maturity: + value: 75 + note: "Dapr is CNCF-graduated and provides production-capable polyglot workflow, actor, scheduling, identity, and sidecar APIs; Dapr Agents v1.0 is GA, though the proposed CF integration remains unimplemented." + novelty: + value: 50 + note: "The design substantially adapts known Dapr sidecar and control-plane patterns by substituting Diego identity, service bindings, Envoy-style injection, and potentially GoRouter placement." + actionability: + value: 75 + note: "The component-by-component mapping and three adoption strategies bound investigations into external-cert trust, scheduler storage, injection cost, and placement without requiring a full platform design first." + --- ## The idea diff --git a/ideas/durable-tasks-for-cf.md b/ideas/durable-tasks-for-cf.md index e88f25d..76c1ad8 100644 --- a/ideas/durable-tasks-for-cf.md +++ b/ideas/durable-tasks-for-cf.md @@ -3,6 +3,20 @@ title: Durable execution as a CF-native primitive — what tasks are missing author: Ruben Koster (@rkoster) date: 2026-08-11 tags: [runtime-lifecycle, orchestration] +ratings: + platform-impact: + value: 100 + note: 'Between one-shot Tasks and always-running app instances, CF has no stable execution identity, suspend/resume lifecycle, durable event wakeup, timers, scheduler, or retry primitive.' + maturity: + value: 50 + note: 'Temporal, Dapr, and Azure Durable Task prove the component semantics, but the proposed CF execution resource, suspend/resume lifecycle, per-cell API, and framework adapters have no demonstrated implementation or operations.' + novelty: + value: 50 + note: 'The proposal recombines established workflow identity and ephemeral compute-slice patterns into a deliberately narrow CAPI resource plus per-cell API rather than a workflow engine.' + actionability: + value: 75 + note: 'A CAPI execution row, Diego Tasks as compute slices, bound-service state pointers, and mTLS per-cell control calls define a bounded architecture to test against framework adapters.' + --- ## The idea diff --git a/ideas/localhost-only-egress-for-agents.md b/ideas/localhost-only-egress-for-agents.md index cf4b986..e712601 100644 --- a/ideas/localhost-only-egress-for-agents.md +++ b/ideas/localhost-only-egress-for-agents.md @@ -3,6 +3,20 @@ title: Localhost-only egress for agent workloads author: Rashid Rashidov (@rrashidov) date: 2026-07-08 tags: [sandboxing-isolation, observability-governance] +ratings: + platform-impact: + value: 75 + note: 'CF lacks a platform-owned, non-bypassable outbound proxy that turns declared external bindings into enforced destinations and request-level audit logs for agent traffic.' + maturity: + value: 75 + note: 'The 75 reflects mature, widely deployed proxy, allowlist, interception, and request-logging technologies; it does not imply maturity for the proposed non-bypassable CF integration, binding-derived policy, or ownership model, which lack implementation evidence.' + novelty: + value: 25 + note: 'The architecture applies conventional mandatory-egress-proxy and allowlist controls to inference-selected destinations rather than creating a new networking mechanism.' + actionability: + value: 50 + note: 'Declared bindings, a localhost endpoint, enforced forwarding, and logging give a prototype outline, but the proxy component, bypass prevention, and policy ownership remain unspecified.' + --- ## The idea diff --git a/ideas/per-session-sandboxes.md b/ideas/per-session-sandboxes.md index 3c808f9..5f0234c 100644 --- a/ideas/per-session-sandboxes.md +++ b/ideas/per-session-sandboxes.md @@ -3,6 +3,20 @@ title: Per-session sandboxes with lifecycle states author: Ruben Koster (@rkoster) date: 2026-07-02 tags: [runtime-lifecycle, sandboxing-isolation] +ratings: + platform-impact: + value: 75 + note: 'CF has no per-session sandbox resource that can release CPU while retaining disk or serialize state to blobstore, leaving only continuously running or state-losing processes.' + maturity: + value: 50 + note: 'Kubernetes Agent Sandbox demonstrates warm pools, PVC persistence, hibernation, and gVisor snapshots, but the agent-specific control plane and graduated lifecycle remain relatively early.' + novelty: + value: 50 + note: 'Session-scoped isolated compute combines known pooling, suspend/resume, persistent-volume, and blob-checkpoint patterns into a newer agent sandbox lifecycle.' + actionability: + value: 50 + note: 'The three lifecycle states and external K8s and Anthropic comparisons identify a direction, but snapshot mechanics, storage format, and available CF volume/blobstore primitives need scoping.' + --- ## The idea diff --git a/ideas/staged-sandbox-environments.md b/ideas/staged-sandbox-environments.md index 31bcc9c..664d3e9 100644 --- a/ideas/staged-sandbox-environments.md +++ b/ideas/staged-sandbox-environments.md @@ -3,6 +3,20 @@ title: Split environment staging from workspace state for agent sandboxes author: Ruben Koster (@rkoster) date: 2026-08-24 tags: [runtime-lifecycle, sandboxing-isolation, ecosystem-survey] +ratings: + platform-impact: + value: 50 + note: 'CF already stages Packages into Droplets and runs Tasks, but cross-app content-addressed Droplet reuse and mounting a separate mutable workspace Package are meaningful missing integrations.' + maturity: + value: 50 + note: 'CF staging and Tasks prove several ingredients, but cross-app content-addressed Droplet reuse, separate mutable workspace mounting, and checkpointed diffs are an unimplemented composition without operational evidence.' + novelty: + value: 50 + note: 'Separating a globally cached environment Droplet from a per-turn workspace Package is a substantial adaptation of CF staging and Nix-style substitution to generated agent code.' + actionability: + value: 75 + note: 'Three concrete operations map to CAPI Build, Droplet, Package, resource-match, and Task resources, with focused questions around cross-tenant reuse and extra volume mounting.' + --- ## The idea diff --git a/ideas/stronger-workload-isolation-for-agents.md b/ideas/stronger-workload-isolation-for-agents.md index a5d5e94..9b71712 100644 --- a/ideas/stronger-workload-isolation-for-agents.md +++ b/ideas/stronger-workload-isolation-for-agents.md @@ -3,6 +3,20 @@ title: Stronger workload isolation for agent workloads author: Rashid Rashidov (@rrashidov) date: 2026-07-08 tags: [sandboxing-isolation, runtime-lifecycle] +ratings: + platform-impact: + value: 50 + note: 'CF supplies container isolation but offers no gVisor- or Kata-style runtime choice to put inference-selected code behind a user-space or dedicated-kernel boundary.' + maturity: + value: 75 + note: 'The 75 reflects production-capable gVisor, Kata, and Kubernetes RuntimeClass isolation technologies; it does not imply maturity for manifest-selectable isolation through Garden and Diego, which has no demonstrated implementation, lifecycle integration, or CF operational evidence.' + novelty: + value: 25 + note: 'A manifest-selectable sandbox runtime is the familiar RuntimeClass pattern adapted to CF applications that execute unreviewed agent-generated instructions.' + actionability: + value: 50 + note: 'The runtime: sandbox field and gVisor target define a plausible spike, but the Garden or Diego enforcement point, lifecycle wiring, and operator policy are still open.' + --- ## The idea diff --git a/research/TEMPLATE.md b/research/TEMPLATE.md index 3b457b9..c8b6cbe 100644 --- a/research/TEMPLATE.md +++ b/research/TEMPLATE.md @@ -5,6 +5,19 @@ date: 2026-01-01 tags: [, ] cf_areas: [] status: draft +ratings: + platform-impact: + value: 50 + note: "Explain the provisional platform-impact score." + maturity: + value: 50 + note: "Explain the provisional maturity score." + novelty: + value: 50 + note: "Explain the provisional novelty score." + actionability: + value: 50 + note: "Explain the provisional actionability score." sources: - --- @@ -31,3 +44,9 @@ sources: ## Open questions - + +## Optional map ratings + +Ratings are provisional working-group judgments on a 0-100 scale. Each value must have a +short justification in its `note` field. The generator uses named ratings to position notes on +plots, so future plots can reuse these ratings or introduce new ones. diff --git a/research/a2a-protocol.md b/research/a2a-protocol.md index 603acf0..b909adf 100644 --- a/research/a2a-protocol.md +++ b/research/a2a-protocol.md @@ -8,6 +8,20 @@ status: draft sources: - https://a2a-protocol.org/latest/ - https://github.com/a2aproject/A2A +ratings: + platform-impact: + value: 55 + note: 'CF already supplies HTTP routing and UAA trust primitives, but it lacks A2A Agent Card discovery and explicit handling for stateful, long-running delegated Tasks across org and space boundaries.' + maturity: + value: 68 + note: 'A2A is a Linux Foundation open standard with authentication, authorization, streaming, async Tasks, and an extension mechanism, although the note does not present long-term operational evidence.' + novelty: + value: 58 + note: 'Agent Cards and stateful delegation standardize an emerging agent-to-agent layer, but reuse familiar decentralized HTTP discovery, capability metadata, and asynchronous task patterns.' + actionability: + value: 72 + note: 'The note identifies a bounded CF investigation: expose Agent Cards through route metadata or a scoped registry, then test delegated Tasks and UAA authentication across spaces.' + --- ## Summary diff --git a/research/anthropic-managed-agents.md b/research/anthropic-managed-agents.md index 072b173..0b63a24 100644 --- a/research/anthropic-managed-agents.md +++ b/research/anthropic-managed-agents.md @@ -6,6 +6,20 @@ tags: [runtime-lifecycle, sandboxing-isolation, identity] status: draft sources: - https://www.anthropic.com/engineering/managed-agents +ratings: + platform-impact: + value: 75 + note: 'Stateless CF app processes map to the brain, but CF lacks on-demand sandbox hands, an external append-only agent session service, and credential proxies that keep secrets out of generated-code environments.' + maturity: + value: 62 + note: 'Anthropic reports operating the architecture and measured p50 TTFT improvements of about 60% and p95 improvements over 90%, but publishes neither a specification nor an open implementation for its session interface.' + novelty: + value: 72 + note: 'The independently scalable brain, replaceable hands, and external positional event log form an emerging decomposition, strengthened by vault-backed proxies that structurally exclude credentials from both harness and sandbox.' + actionability: + value: 78 + note: 'CF can prototype a stateless brain app against a durable event service and on-demand sandbox, then route one service credential through a proxy instead of injecting it into the sandbox.' + --- ## Summary diff --git a/research/aws-agents.md b/research/aws-agents.md index cf118cf..2e71fd6 100644 --- a/research/aws-agents.md +++ b/research/aws-agents.md @@ -19,6 +19,20 @@ sources: - https://aws.amazon.com/bedrock/agentcore/ - https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/what-is-bedrock-agentcore.html - https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/agents-tools-runtime.html +ratings: + platform-impact: + value: 85 + note: 'AgentCore exposes a broad CF gap spanning per-session microVMs, persistent resumable filesystems, agent identity, memory, MCP gateway and Cedar policy, registry, evaluations, and automated optimization.' + maturity: + value: 78 + note: 'AgentCore is a documented managed service with multiple compute modes and integrations, while Apache-licensed Strands has Python and TypeScript SDKs, about 6.9k stars, governance, releases, and many deployment targets.' + novelty: + value: 58 + note: 'MicroVM session isolation, gateways, Cedar policy, OTel, and agent loops are established ideas; their modular assembly into an observe-evaluate-optimize managed agent platform is a newer combination.' + actionability: + value: 72 + note: 'The framework-agnostic platform raises several large design choices, but Strands provides a concrete CF prototype target for MCP/A2A deployment and for testing gateway-enforced Cedar authorization on tool calls.' + --- ## Summary diff --git a/research/azure-hosted-agents.md b/research/azure-hosted-agents.md index 337d0d5..dd8d4f1 100644 --- a/research/azure-hosted-agents.md +++ b/research/azure-hosted-agents.md @@ -7,6 +7,20 @@ cf_areas: [diego, capi, uaa] status: draft sources: - https://learn.microsoft.com/en-us/azure/foundry/agents/concepts/hosted-agents +ratings: + platform-impact: + value: 82 + note: 'CF can already deploy container images, but lacks Azure Foundry''s per-session VM sandboxes, persistent stateful resume, scale-to-zero session lifecycle, automatic per-deployment identity, and injected agent telemetry.' + maturity: + value: 74 + note: 'The hosted service documents concrete quotas, a 15-minute idle timeout, 30-day session lifetime, Entra identity, persistent filesystems, and three protocols, though the note provides limited independent adoption history.' + novelty: + value: 61 + note: 'Container packaging and dedicated identity are familiar PaaS features, while VM-isolated per-session scaling with persistent resume and managed conversation protocols is a newer agent-specific synthesis.' + actionability: + value: 75 + note: 'A bounded CF spike can package one A2A agent, add OTel auto-instrumentation and identity bootstrap, and evaluate whether session persistence and isolation can be layered onto Diego processes.' + --- ## Summary diff --git a/research/cloudflare-agents.md b/research/cloudflare-agents.md index 77d1d1a..711e964 100644 --- a/research/cloudflare-agents.md +++ b/research/cloudflare-agents.md @@ -14,6 +14,20 @@ sources: - https://developers.cloudflare.com/agents/runtime/operations/observability/ - https://developers.cloudflare.com/agents/runtime/operations/observability/tracing/ - https://developers.cloudflare.com/durable-objects/ +ratings: + platform-impact: + value: 88 + note: 'CF has no equivalent to a cheap globally addressable compute unit that combines isolate execution, private colocated SQLite, hibernation-safe WebSockets, durable alarms, and automatic geographic placement.' + maturity: + value: 72 + note: 'The SDK rests on production Durable Objects that scale to millions of instances and supplies state, scheduling, MCP, workflows, and tracing, but remains Cloudflare-controlled, rejects external contributions, and lightly documents A2A.' + novelty: + value: 82 + note: 'Making every agent a single-threaded V8 isolate with its own colocated SQLite database, durable alarms, transparent hibernation, and global identity departs sharply from container, microVM, and external-state agent runtimes.' + actionability: + value: 68 + note: 'The note defines a focused comparison of isolate trust, hibernation, and bindable per-agent state, but a CF prototype requires substantial design because Durable Objects'' routing and storage substrate is not portable.' + --- ## Summary diff --git a/research/crewai.md b/research/crewai.md index 752e550..cb4b959 100644 --- a/research/crewai.md +++ b/research/crewai.md @@ -16,6 +16,20 @@ sources: - https://docs.crewai.com/en/observability/overview - https://docs.crewai.com/en/observability/tracing - https://docs-platform.crewai.com/platform/en/introduction +ratings: + platform-impact: + value: 52 + note: 'CrewAI''s OSS crews, flows, MCP/A2A support, memory, and SQLite checkpoints can run inside a normal CF app; the main gap is optional platform support for durable storage, hosted tracing, deployment, and RBAC.' + maturity: + value: 78 + note: 'The MIT-licensed package is published on PyPI with about 56.9k stars, 8.1k forks, 2,732-plus commits, broad orchestration features, and a commercial hosted platform, indicating meaningful adoption.' + novelty: + value: 46 + note: 'Role-based LLM teams and event-driven workflows adapt familiar manager, DAG, checkpoint, and pub/sub patterns; checkpoint forking and the recommended Flow-around-Crew composition add a newer agent-specific layer.' + actionability: + value: 70 + note: 'CF can deploy an OSS CrewAI Flow and test SQLite checkpoint resume and fork behavior, although deciding whether AMP-like tracing and RBAC belong in the platform needs additional scope.' + --- ## Summary diff --git a/research/dapr-agents.md b/research/dapr-agents.md index 49fb8d1..db40842 100644 --- a/research/dapr-agents.md +++ b/research/dapr-agents.md @@ -12,6 +12,20 @@ sources: - https://docs.dapr.io/developing-ai/dapr-agents/dapr-agents-patterns/ - https://docs.dapr.io/developing-ai/dapr-agents/dapr-agents-why/ - https://docs.dapr.io/developing-ai/dapr-agents/dapr-agents-integrations/ +ratings: + platform-impact: + value: 70 + note: 'CF lacks workflow-durable agent loops, virtual-actor scale-to-zero, an agent registry, and sidecar-discovered MCP tools, although service brokers and app routing provide partial state and messaging primitives.' + maturity: + value: 64 + note: 'Dapr Agents has reached v1.0 GA and builds on CNCF-graduated Dapr workflows and actors, but its own governance status is unclear and the note offers claims rather than broad production adoption evidence.' + novelty: + value: 55 + note: 'Running every LLM and tool call as a durable workflow activity combines established actors, workflows, pub/sub, and registries in an agent-specific way rather than introducing a wholly new substrate.' + actionability: + value: 78 + note: 'A focused CF experiment can run a Dapr sidecar with one DurableAgent, kill it mid-tool-call, verify workflow recovery, and measure actor resume and MCP discovery against CF instance lifecycle constraints.' + --- ## Summary diff --git a/research/dapr.md b/research/dapr.md index dec5830..e3baec7 100644 --- a/research/dapr.md +++ b/research/dapr.md @@ -13,6 +13,20 @@ sources: - https://docs.dapr.io/operations/security/mtls/ - https://docs.dapr.io/developing-applications/building-blocks/workflow/workflow-overview/ - https://docs.dapr.io/developing-ai/dapr-agents/dapr-agents-introduction/ +ratings: + platform-impact: + value: 62 + note: 'CF service bindings overlap Dapr''s pluggable state, pub/sub, secrets, and bindings, but CF lacks its standardized sidecar APIs, virtual actors, durable workflows, SPIFFE workload identity, and supporting placement and scheduler control planes.' + maturity: + value: 94 + note: 'Dapr is CNCF graduated and supplies eleven stable building blocks, pluggable production backends, Kubernetes and self-hosted modes, short-lived mTLS certificates, actors, and crash-resumable workflows.' + novelty: + value: 25 + note: 'Sidecars, service invocation, pub/sub, state stores, virtual actors, workflow engines, and workload certificates are established distributed-systems patterns assembled behind a language-neutral API.' + actionability: + value: 67 + note: 'The note supports a concrete sidecar-on-Diego compatibility investigation, but actor placement, Sentry, scheduler footprint, and overlap with service brokers make a full CF integration broader than one bounded experiment.' + --- ## Summary diff --git a/research/domyn-swarm.md b/research/domyn-swarm.md new file mode 100644 index 0000000..30e198d --- /dev/null +++ b/research/domyn-swarm.md @@ -0,0 +1,194 @@ +--- +title: "Domyn Swarm — HPC-Native Batch Inference Orchestration for vLLM" +author: Ruben Koster (@rkoster) +date: 2026-08-19 +tags: [runtime-lifecycle, orchestration, observability-governance] +status: draft +sources: + - https://github.com/igeniusai/domyn-swarm +ratings: + platform-impact: + value: 75 + note: 'CF has Diego/CAPI workload lifecycle primitives, but no integrated equivalent for provisioning GPU inference endpoints across heterogeneous schedulers, supervising vLLM replicas, and reconciling durable batch-job state with resumable checkpoints.' + maturity: + value: 50 + note: 'Version 0.29.0 has a substantial implementation with migrations, health supervision, retries, checkpointing, and rapid releases through 2026, but it is classified Alpha, is not on PyPI, has 23 stars and one fork, and shows only internal Domyn dogfooding rather than broad production evidence.' + novelty: + value: 40 + note: 'The ServingBackend/ComputeBackend split applies familiar adapter and deployment-lifecycle patterns to HPC inference; watchdog supervision, SQLite reconciliation, and sharded Parquet checkpoints are a useful combination, but each is established technology rather than a new architecture.' + actionability: + value: 80 + note: 'The two small backend protocols define a bounded CF spike: implement serving and compute adapters for CAPI/Diego plus a GPU scheduler, then exercise endpoint lifecycle, job-status reconciliation, watchdog failure recovery, and Parquet resume behavior.' + +--- + +## Summary + +Domyn Swarm (`igeniusai/domyn-swarm`, Apache-2.0) is a CLI + Python library that stands up +vLLM OpenAI-compatible serving endpoints on **Slurm** or **DGX Cloud Lepton**, then runs +high-throughput batch inference jobs (DataFrame-in/DataFrame-out, or arbitrary scripts) +against them with retries, checkpointing, and process-level health supervision. This +analysis is based on a local checkout (commit `4824560`, tagged `v0.29.0-11-g4824560`, +2026-07-17) rather than the truncated GitHub web view — the codebase is considerably more +sophisticated than the public README alone suggests: it includes a SQLite-backed state/job +store with Alembic migrations, a dedicated watchdog/collector process pair for per-replica +health monitoring and auto-restart, and pluggable Parquet checkpoint stores with resumable, +shard-based writes. It is HPC batch-inference tooling, not a general agent framework — no +tool-calling, planning, or multi-agent orchestration concepts appear anywhere in the codebase. + +## Key findings + +**Core abstraction: two-protocol backend split** +- `src/domyn_swarm/platform/protocols.py` defines two `Protocol` interfaces that everything + else implements: `ServingBackend` (`create_or_update`, `wait_ready`, `delete`, + `ensure_ready`, `status`) manages the life of an inference endpoint; `ComputeBackend` + (`submit`, `wait`, `cancel`, `probe`, plus `default_python`/`default_image`/ + `default_resources`/`default_env` via a `DefaultComputeMixin`) manages the life of a job + that targets that endpoint. Both use opaque `ServingHandle`/`JobHandle` value objects and + standardized `ServingPhase`/`JobStatus` enums so callers never touch platform-specific + types. +- `Deployment` (`deploy/deployment.py`) is a thin composition of one `ServingBackend` + one + `ComputeBackend`: `up()` creates+waits for the endpoint, `run()` submits a job against it, + `down()` tears it down. This is a clean, minimal pattern for decoupling "where inference + runs" from "how batch jobs are scheduled against it" — deliberately reusable beyond Slurm. +- Two backend pairs are implemented today: **Slurm** (`backends/serving/slurm.py` + + `backends/compute/slurm.py`, using `srun` inside a load-balanced allocation) and + **DGX Cloud Lepton** (`backends/serving/lepton.py` + `backends/compute/lepton.py`, via the + `leptonai` Python SDK, an optional extra). Adding a new platform means implementing the two + protocols, not touching the orchestration core. + +**Orchestrator: `DomynLLMSwarm`** +- `core/swarm.py` defines `DomynLLMSwarm`, a Pydantic `BaseModel` used as a context manager + (`__enter__`/`__exit__` bring the endpoint up/down). It owns: job submission + (`submit_job`, `submit_script`), job lifecycle (`wait_job`, `cancel_job`, + `refresh_job_status`), local persistence (`_persist`, `from_state` — a swarm can be + rehydrated later from a saved name), and `status()`. +- Every job submission is tracked in a local SQLite state DB: `_record_job_submission` / + `_update_job_submission` persist name, command, resources, kind, status, external ID + (Slurm job/step ID or Lepton job ID), and log paths — i.e. Domyn Swarm keeps its own + durable job audit trail independent of the underlying scheduler, and `refresh_job_status` + can re-probe the backend to reconcile it. +- `create_swarm_pool` (used in `examples/api/swarm_launch.py`) launches multiple + `DomynLLMSwarm` instances concurrently (e.g. two independent replicas/configs), and jobs + can be submitted `detach=True` to run as background child processes, with the caller + later `waitpid`-ing on the returned PIDs — a simple fan-out pattern for parallel swarms. + +**Config: auto-computed resource allocation** +- `config/swarm.py`'s `DomynLLMSwarmConfig` (Pydantic model) has a + `validate_resource_allocations` model validator that derives `nodes`, `cpus_per_task`, + and `replicas_per_node` from `replicas`, `gpus_per_replica`, and `gpus_per_node` when not + given explicitly — e.g. `nodes = ceil(replicas / replicas_per_node)` or, for multi-GPU + multi-node replicas, `ceil((replicas * gpus_per_replica) / gpus_per_node)`. This is the + concrete mechanism behind the "just write gpus_per_replica/replicas in YAML" quickstart + experience. +- `config/plan.py`'s `PlanBuilder` normalizes this into a `DeploymentPlan` + (serving+compute backend instances plus per-backend specs) and `DeploymentContext` + (normalized fields shared across serving and compute) — a single place where + backend-specific defaults (default container image, default Python interpreter, default + resources/env) get resolved before either backend is touched. + +**Jobs: `SwarmJob` abstract base + checkpointed execution** +- `jobs/api/base.py` defines `SwarmJob(abc.ABC)`. User code implements one method: + `async def transform_items(items: list[Any]) -> list[Any]` (pure, order-preserving). A + `transform_streaming` variant supports checkpoint-as-you-go without retaining all outputs + in memory. The constructor takes ~20 parameters covering input/output column naming, + concurrency (`max_concurrency`), `retries`, `timeout`, `checkpoint_interval`, an + `OutputJoinMode` (e.g. `APPEND`), and a pluggable `data_backend`. +- `jobs/base.py` (the module named in the original README) is now a **deprecated + compatibility shim** re-exporting from `jobs/api/base.py` with a `DeprecationWarning` — + the legacy `transform(df)`-based job shape has been fully replaced by the + `transform_items(items)` contract. +- Checkpointing is a separate, swappable concern (`checkpoint/store.py`): a + `CheckpointStore[T]` protocol (`prepare`, `flush`, `finalize`) with two implementations — + `ParquetShardStore` (writes monotonically-named Parquet shards to local or cloud URIs via + `fsspec`, tracks already-completed IDs to support resume, and merges shards on + `finalize()`) and `InMemoryStore` (no disk I/O, for tests/small jobs). +- Data backends are pluggable via a registry (`data/backends/registry.py`): pandas + (default), optional Polars, optional Ray — so the same `SwarmJob` can run over different + DataFrame engines depending on scale. + +**Runtime health: watchdog + collector** +- `runtime/watchdog.py` runs as a per-replica supervisor process: it spawns the actual vLLM + child process, polls its HTTP health endpoint (`_check_http`) and, for Ray-backed + multi-node replicas, probes Ray cluster health and expected worker/tensor-parallel + capacity (`_ray_cluster_ok`, `_ray_capacity_ok`). A hardcoded `RAY_FATAL_EXIT_CODE = 190` + distinguishes non-retryable Ray failures from transient ones (`_should_restart` decides + whether to respawn based on exit code). +- Replica state (`ReplicaState` enum) and failures are reported via `send_status()` to a + separate **collector** process, described in `AGENTS.md` as "single writer to + `watchdog.db`" — i.e. watchdogs never write the SQLite DB directly, avoiding + multi-writer contention; only the collector does, and `domyn-swarm status` reads from it. + `build_fail_reason` classifies failures from log tails into a human-readable reason plus + a `retryable` boolean. +- This is a real, if minimal, self-healing mechanism for long-running Slurm-allocated vLLM + replicas — not just "start it and hope," which matters on HPC clusters where a node/GPU + fault shouldn't require a human to notice and manually resubmit. + +**State & CLI** +- Two SQLite databases: a global `swarm.db` (`${DOMYN_SWARM_HOME:-~/.domyn_swarm}/swarm.db`) + for swarm/job records, and a per-swarm `watchdog.db` under + `.../swarms//` for replica health. Schema changes go through Alembic + migrations with an auto-upgrade step that runs on every CLI invocation (skippable via + `DOMYN_SWARM_SKIP_DB_UPGRADE=1`), guarded by a threading lock (per changelog) to avoid + concurrent-upgrade races. +- The CLI (`cli/main.py`, Typer) uses a `LazyGroup` that defers importing heavy subcommand + modules (job management, swarm lifecycle) until actually invoked, plus lazy proxies + (`_LazyDomynLLMSwarm`, `_LazySwarmStateManager`, `_LazyLogger`) — purely a startup-latency + optimization (per recent changelog entries: "defer heavy imports off the swarm-load + path"), notable mainly as evidence of active performance-focused maintenance. +- Commands include `up`/`down` (swarm lifecycle), `status` (table via Rich TUI or stable + JSON via `-o json`), and a `job` subcommand group (`submit`, `submit-script`, `status`, + `cancel`, `list` — added incrementally through v0.26–v0.29 per CHANGELOG.md, including + job persistence with external-ID tracking and idempotent cancellation). + +**Maturity signals** +- `pyproject.toml`: version `0.29.0`, `Development Status :: 3 - Alpha`, Python + `>=3.10,<3.14`, Apache-2.0. Two named maintainers with `@domyn.com` addresses (Federico + D'Ambrosio, Alessandro Rognoni) — a small, identifiable internal team, not a broad + open-source community project (1 fork, 23 stars at time of writing). +- CHANGELOG.md shows fast, incremental delivery: Ray backend support, Polars backend, + stable sharding strategies (`id` vs `index` mode), sharded/resumable checkpoint stores, + the watchdog/collector health system, and full job CRUD with JSON status output were all + added across versions v0.25.0–v0.29.0 (Jan–Jun 2026) — consistent with a tool under active + internal dogfooding at Domyn rather than a one-off open-source drop. +- Still not published to PyPI as of this checkout (per AGENTS.md/README); installed via + `uv`/`pip` directly from the GitHub repo at a pinned tag. + +## CF relevance + +The `ServingBackend` / `ComputeBackend` protocol split, composed by a single `Deployment` +object, is a directly reusable pattern for any platform wanting to decouple "stand up an +inference/agent endpoint" from "schedule work against it" across heterogeneous compute +(here: Slurm vs. Lepton; for CF, potentially Diego/CAPI vs. some GPU-scheduling backend). +Three other pieces are worth studying as concrete, load-bearing reference implementations +rather than abstract patterns: (1) the **watchdog/collector split** — one writer to a local +health-status store, workers only report — is a simple, robust pattern for supervising +long-running GPU workloads without database contention; (2) the **resumable, sharded +Parquet checkpoint store** with monotonic shard naming and "already-done ID" tracking is a +concrete answer to "how do you make a long batch job restart-safe"; (3) **local SQLite job +persistence independent of the underlying scheduler**, with a `refresh_job_status` +reconciliation path, is a lightweight pattern for a platform to keep its own durable +job/audit record without depending entirely on the backend's own state (relevant to +audit/observability-governance concerns raised in the broader agentic-workload research). +Note the scope limit: none of this is agent orchestration (no planning, tool use, or +multi-agent coordination) — it is HPC batch-inference plumbing, so its relevance is to the +"run the model reliably at scale" layer, not the "agent decides what to do" layer. + +## Open questions + +- How does the watchdog/collector health system behave across a full node failure (not + just process crash) — does Slurm's own requeue interact with domyn-swarm's restart + logic, or can they conflict (e.g. double-restart)? +- The `ParquetShardStore` resume logic depends on stable ID columns across runs — what + happens if the input DataFrame's row order or ID scheme changes between a failed run and + its resume attempt (partial-shard consistency wasn't verified in this pass)? +- Is there a production user of the `ray` data-backend / Ray-backed multi-node serving path + outside Domyn's own Colosseum cluster, or is Ray support Slurm-specific tooling that + wouldn't transfer to a non-HPC scheduler? +- The two named maintainers and Apache-2.0 license suggest Domyn intends this as a genuine + reusable open-source tool (vs. a marketing artifact) — is there any public roadmap or + issue tracker activity indicating outside contributions, or is it effectively + single-vendor maintained? +- How (if at all) does Domyn Swarm relate to the proprietary "Platform" product marketed on + domyn.com for building/orchestrating AI Agents — is Swarm the inference substrate + underneath that product, a separate internal tool, or unrelated? diff --git a/research/firecracker-microvm.md b/research/firecracker-microvm.md index ce52593..7d258c8 100644 --- a/research/firecracker-microvm.md +++ b/research/firecracker-microvm.md @@ -14,6 +14,20 @@ sources: - https://firecracker-microvm.github.io/ - https://fly.io/blog/sandboxing-and-workload-isolation/ - https://aws.amazon.com/blogs/aws/firecracker-lightweight-virtualization-for-serverless-computing +ratings: + platform-impact: + value: 80 + note: 'CF''s long-lived container app instances do not provide per-session hardware isolation, 125ms-class microVM startup, or snapshot pause/resume for untrusted generated code and long-idle agent sessions.' + maturity: + value: 95 + note: 'Firecracker has powered AWS Lambda and Fargate since its 2018 release, enforces boot and memory targets in CI, and is integrated by Fly.io, Kata Containers, and containerd.' + novelty: + value: 42 + note: 'Its minimal five-device VMM and copy-on-write snapshots substantially optimize familiar KVM virtualization, but hardware VMs, seccomp, namespaces, cgroups, and snapshot restoration are established techniques.' + actionability: + value: 66 + note: 'Kata-on-Firecracker offers a concrete Diego-cell spike for isolation and resume measurements, but networking, snapshot identity safety, host integration, and tenant policy require significant scoping.' + --- ## Summary diff --git a/research/google-adk.md b/research/google-adk.md index 28b1c61..8a1ba29 100644 --- a/research/google-adk.md +++ b/research/google-adk.md @@ -19,6 +19,20 @@ sources: - https://google.github.io/adk-docs/agents/workflow-agents/ - https://google.github.io/adk-docs/agents/models/ - https://google.github.io/adk-docs/evaluate/ +ratings: + platform-impact: + value: 58 + note: 'ADK agents can run as ordinary CF containers and bring their own graph, A2A, MCP, OTel, and evaluation libraries; CF gaps remain around durable replay, session affinity, identity, and managed behavioral evaluation.' + maturity: + value: 78 + note: 'Google used ADK in internal products before open sourcing it; ADK 2.0 is GA, five first-party language implementations move in lockstep, Python has about 21k stars, and kagent uses ADK as its engine.' + novelty: + value: 55 + note: 'Graph workflows, event logs, tool adapters, and evaluation harnesses are known patterns, while isolated task delegation modes and replay-oriented conformance tests are newer agent-specific adaptations.' + actionability: + value: 82 + note: 'A bounded CF deployment can exercise ADK''s A2A endpoint, OTel traces, MCP connection recovery after instance replacement, and eval conformance tests while documenting the missing durability substrate.' + --- ## Summary diff --git a/research/hatchet.md b/research/hatchet.md index b916460..ea1fb07 100644 --- a/research/hatchet.md +++ b/research/hatchet.md @@ -19,6 +19,20 @@ sources: - https://docs.hatchet.run/v1/concurrency - https://docs.hatchet.run/v1/opentelemetry - https://docs.hatchet.run/self-hosting +ratings: + platform-impact: + value: 68 + note: 'CF can host Hatchet workers and bind Postgres, but does not itself provide durable sleeps, event waits, replay, tenant-fair scheduling, rate limits, or centralized workflow observability.' + maturity: + value: 70 + note: 'Hatchet has about 7.7k stars, four SDKs, self-hosted and managed offerings, named AI customers, and claimed high daily task volume, but comes from a small 2023 startup with limited enterprise history.' + novelty: + value: 43 + note: 'Using Postgres alone for task history and observability is a useful simplification, while task queues, DAGs, durable waits, retries, rate limits, and worker slots are conventional orchestration mechanisms.' + actionability: + value: 84 + note: 'CF can directly deploy Hatchet with a bound Postgres service, interrupt and resume a worker, and measure throughput and per-tenant fairness against the proposed durable-tasks-for-CF requirements.' + --- ## Summary diff --git a/research/heroku-ai-platform.md b/research/heroku-ai-platform.md index 268a0e1..50972a9 100644 --- a/research/heroku-ai-platform.md +++ b/research/heroku-ai-platform.md @@ -13,6 +13,20 @@ sources: - https://www.heroku.com/blog/code-execution-sandbox-for-agents-on-heroku/ - https://www.heroku.com/ai/mcp-on-heroku/ - https://github.com/heroku/mcp-code-exec-python +ratings: + platform-impact: + value: 55 + note: 'CF already has the app, buildpack, service-binding, Postgres, and one-off-task primitives Heroku reuses, but lacks its managed inference add-on, MCP gateway, model lifecycle policy, and agent-oriented routing optimizations.' + maturity: + value: 72 + note: 'Heroku operates inference, agents, MCP Toolkits, pgvector, and code execution as shipped services on its long-proven dyno platform, although the Anthropic-compatible endpoint remains preview and no durable execution is offered.' + novelty: + value: 32 + note: 'The design deliberately combines established PaaS patterns: add-on bindings, compatible HTTP APIs, Postgres vector search, and disposable one-off dynos rather than a new agent runtime architecture.' + actionability: + value: 90 + note: 'The note directly supports an OSBAPI prototype that binds OpenAI-compatible inference through VCAP_SERVICES and a sandbox experiment that maps code calls to short-lived Diego tasks with offline dependency staging.' + --- ## Summary diff --git a/research/k8s-agent-sandbox.md b/research/k8s-agent-sandbox.md index 536737d..2790a97 100644 --- a/research/k8s-agent-sandbox.md +++ b/research/k8s-agent-sandbox.md @@ -8,6 +8,20 @@ status: draft sources: - https://github.com/kubernetes-sigs/agent-sandbox - https://agent-sandbox.sigs.k8s.io/docs +ratings: + platform-impact: + value: 72 + note: 'Diego and Garden provide isolated containers, but CF has no agent-sandbox API combining stable singleton identity, claims, persistent volumes, warm pools, and suspend/resume snapshots.' + maturity: + value: 40 + note: 'The Kubernetes SIG implementation has concrete Sandbox, Claim, Template, and WarmPool CRDs, but the note presents a young agent-specific control plane with limited production evidence and GKE-specific snapshot support.' + novelty: + value: 62 + note: 'Stable singleton sandboxes and millisecond claims from pre-running pools are an emerging agent-lifecycle combination, though they build on conventional pods, PVCs, RBAC, namespaces, and network policy.' + actionability: + value: 72 + note: 'A bounded CF investigation can compare SandboxClaim and WarmPool semantics with Diego tasks and instance pools, then identify the smallest API needed for stable identity, persistence, and pre-warmed allocation.' + --- ## Summary diff --git a/research/kagent.md b/research/kagent.md index e0140ff..ff523e5 100644 --- a/research/kagent.md +++ b/research/kagent.md @@ -9,6 +9,20 @@ sources: - https://github.com/kagent-dev/kagent - https://kagent.dev/docs/kagent/getting-started/quickstart - https://kagent.dev/docs/kagent/concepts/agents +ratings: + platform-impact: + value: 55 + note: 'CF exposes operator data through cf, BOSH, Diego, logs, and metrics, but lacks a packaged operations agent with declarative definitions, shared tool servers, and bundled platform-management tools.' + maturity: + value: 52 + note: 'kagent is a functioning CNCF project with Helm-managed CRDs and bundled Kubernetes, Istio, Argo, Prometheus, Grafana, and Cilium tools, while governance, roadmap, and human-approval maturity remain open questions.' + novelty: + value: 48 + note: 'Its distinctive contribution is packaging an operations-focused agent and exposing agents through both A2A and MCP; CRD reconciliation, MCP tool servers, and agents-as-tools are adaptations of known patterns.' + actionability: + value: 58 + note: 'The note identifies a CF operator-agent prototype using cf, BOSH, Diego, logs, and metrics, but first requires deciding whether to extend Kubernetes-bound kagent or build a CF-native equivalent.' + --- ## Summary diff --git a/research/keda.md b/research/keda.md index 3a61314..5406513 100644 --- a/research/keda.md +++ b/research/keda.md @@ -8,6 +8,20 @@ status: draft sources: - https://keda.sh/ - https://github.com/kedacore/keda +ratings: + platform-impact: + value: 68 + note: 'CF autoscaling is primarily metric and instance based; it lacks KEDA-style queue-depth triggers, a broad scaler catalog, and reliable event-driven scale-to-zero and scale-from-zero for workers.' + maturity: + value: 92 + note: 'KEDA is a CNCF graduated, de facto Kubernetes standard with more than 70 production-oriented scalers and an established integration with HPA rather than an experimental autoscaler.' + novelty: + value: 28 + note: 'Queue-length and external-metric autoscaling are long-established techniques; KEDA mainly standardizes and packages them for Kubernetes with a large adapter catalog.' + actionability: + value: 88 + note: 'A concrete CF experiment can bind a queue service, feed queue depth into autoscaling, and measure worker scale-from-zero latency and backlog recovery against KEDA behavior.' + --- ## Summary diff --git a/research/langgraph.md b/research/langgraph.md index 4b9df87..9fb2064 100644 --- a/research/langgraph.md +++ b/research/langgraph.md @@ -16,6 +16,20 @@ sources: - https://github.com/langchain-ai/langchain-mcp-adapters - https://docs.langchain.com/langsmith/deployment - https://docs.langchain.com/langsmith/observability +ratings: + platform-impact: + value: 58 + note: 'CF can host LangGraph processes but does not provide its checkpoint store, cross-thread memory, durable pause/resume, or Agent Server packaging as platform services.' + maturity: + value: 78 + note: 'LangGraph is a production-oriented LangChain runtime with persistence, interrupts, streaming, MCP integration, and commercial LangSmith deployment options, including a standalone Agent Server.' + novelty: + value: 55 + note: 'Applying Pregel-style supersteps, reducer-governed shared state, and boundary checkpoints to agent graphs is a substantial adaptation of established graph-processing and workflow ideas.' + actionability: + value: 72 + note: 'The standalone Agent Server provides a bounded buildpack trial using bound Postgres and Redis, with crash recovery testing for non-idempotent nodes and explicit checkpointer/store mapping questions.' + --- ## Summary diff --git a/research/letta.md b/research/letta.md index 004a5da..151ff8b 100644 --- a/research/letta.md +++ b/research/letta.md @@ -18,6 +18,20 @@ sources: - https://docs.letta.com/self-hosting - https://docs.letta.com/reference/terminology - https://docs.letta.com/agent-sdk/mcp +ratings: + platform-impact: + value: 65 + note: 'CF can host the App Server and bind storage, but has no durable, addressable agent identity or managed evolving-memory service equivalent to Letta agents, shared blocks, and MemFS.' + maturity: + value: 60 + note: 'Letta has an Apache-licensed implementation, 24k-plus stars, extensive history, self-hosting, and a hosted service, but its classic memory API is legacy while the product pivots to Letta Code and MemFS.' + novelty: + value: 78 + note: 'The MemGPT model lets an LLM page and edit its own context like virtual memory, while durable addressable agents and git-backed MemFS make memory and identity primary runtime abstractions.' + actionability: + value: 58 + note: 'A MemFS-backed agent-state binding and App Server isolation trial are plausible, but the ongoing V1-to-MemFS pivot and unrestricted filesystem and shell access leave substantial scoping work.' + --- ## Summary diff --git a/research/llamaindex.md b/research/llamaindex.md index d6654b1..ade818e 100644 --- a/research/llamaindex.md +++ b/research/llamaindex.md @@ -16,6 +16,20 @@ sources: - https://github.com/run-llama/llama_index/blob/main/docs/src/content/docs/framework/module_guides/observability/index.md - https://github.com/run-llama/llama_deploy/blob/main/README.md - https://llamatrace.com/ +ratings: + platform-impact: + value: 52 + note: 'CF can run LlamaIndex libraries, but it does not supply persistence for serializable Context state, workflow checkpoints, or the hosting layer left incomplete by the project deployment tooling.' + maturity: + value: 65 + note: 'The core project has 51k-plus stars and established RAG adoption, while Workflows is newly extracted, llama_deploy is deprecated, llama-agents remains young, and the main deployment documentation is a stub.' + novelty: + value: 62 + note: 'Inferring and validating control flow from typed events between decorated steps is an uncommon orchestration design, though event-driven workflows, worker concurrency, handoffs, and checkpointing are established concepts.' + actionability: + value: 70 + note: 'The note supports a focused buildpack trial plus a bindable Context/checkpoint store, with a direct comparison of inferred control-flow auditability against LangGraph-style explicit graphs.' + --- ## Summary diff --git a/research/mcp-protocol.md b/research/mcp-protocol.md index c7d21d6..e77a0bd 100644 --- a/research/mcp-protocol.md +++ b/research/mcp-protocol.md @@ -16,6 +16,20 @@ sources: - https://en.wikipedia.org/wiki/Model_Context_Protocol - https://arxiv.org/abs/2503.23278 - https://techcrunch.com/2025/12/09/openai-anthropic-and-block-join-new-linux-foundation-effort-to-standardize-the-ai-agent-era/ +ratings: + platform-impact: + value: 55 + note: 'CF can host HTTP services and secure them with UAA, but lacks a first-class MCP service type, registry, lifecycle management, and standardized tool authorization for agent workloads.' + maturity: + value: 78 + note: 'MCP has broad framework and vendor adoption, dated specifications, official SDKs and registry work, and Linux Foundation governance, although transport evolution and authorization guidance are still moving.' + novelty: + value: 55 + note: 'MCP newly standardizes model-facing tools, resources, prompts, capability negotiation, and sampling across vendors, while deliberately borrowing JSON-RPC, OAuth, and Language Server Protocol patterns.' + actionability: + value: 88 + note: 'CF can directly prototype a Streamable HTTP MCP server as a bound multi-instance app, validate audience-bound OAuth tokens through UAA, and document why stdio and session affinity do not fit that service model.' + --- ## Summary diff --git a/research/microsoft-agent-framework.md b/research/microsoft-agent-framework.md index 50ee306..cebe059 100644 --- a/research/microsoft-agent-framework.md +++ b/research/microsoft-agent-framework.md @@ -14,6 +14,20 @@ sources: - https://docs.diagrid.io/develop/agents/microsoft/ - https://github.com/diagridio/dotnet-ai - https://github.com/diagridio/python-ai +ratings: + platform-impact: + value: 55 + note: 'CF can host MAF applications but offers no native agent packaging, graph checkpoint service, or interchangeable durability substrate comparable to Durable Task or the Diagrid Dapr integration.' + maturity: + value: 62 + note: 'MAF consolidates mature AutoGen and Semantic Kernel lineage with Python and .NET implementations, but Go is public preview and durable execution lives in separate first- and third-party extensions.' + novelty: + value: 48 + note: 'Agents, harnesses, explicit workflow graphs, MCP, A2A, and checkpointing form a broad modern combination, but each builds on familiar SDK, workflow, and protocol patterns.' + actionability: + value: 62 + note: 'A CF trial can package a declarative MAF agent and compare Azure Durable Task with Dapr-backed durability, but the note leaves the target substrate and cross-language scope unresolved.' + --- ## Summary diff --git a/research/open-agent-auth.md b/research/open-agent-auth.md index a5af6f7..783bba1 100644 --- a/research/open-agent-auth.md +++ b/research/open-agent-auth.md @@ -8,6 +8,20 @@ status: draft sources: - https://github.com/alibaba/open-agent-auth - https://datatracker.ietf.org/doc/draft-liu-agent-operation-authorization/ +ratings: + platform-impact: + value: 70 + note: 'UAA authenticates users and applications, but CF lacks operation-specific tokens that cryptographically bind a human, agent workload, requested action, and request-level audit context.' + maturity: + value: 35 + note: 'Alibaba provides a concrete implementation, but AOAT is only an IETF draft-02 from March 2026 and the note offers little evidence of independent adoption or production operation.' + novelty: + value: 75 + note: 'Cryptographically binding user delegation, agent workload identity, and semantic operation details into one authorization token is emerging, despite its foundation in OIDC, OAuth PAR, and WIMSE.' + actionability: + value: 68 + note: 'A bounded UAA investigation can issue and validate an AOAT-like token for one destructive agent tool call, though draft churn and the required workload-identity mapping remain open.' + --- ## Summary diff --git a/research/openai-agents-sdk.md b/research/openai-agents-sdk.md index b035ce2..85b5de1 100644 --- a/research/openai-agents-sdk.md +++ b/research/openai-agents-sdk.md @@ -19,6 +19,20 @@ sources: - https://openai.github.io/openai-agents-python/models/ - https://github.com/openai/openai-agents-js - https://github.com/openai/swarm +ratings: + platform-impact: + value: 48 + note: 'CF readily hosts the library-only agent loop, while the meaningful gap is optional platform support for sessions, tracing, MCP connectivity, and external durable execution rather than a missing runtime requirement.' + maturity: + value: 80 + note: 'The MIT Python SDK is widely adopted, has a JavaScript counterpart and comprehensive tools, handoffs, guardrails, sessions, MCP, and tracing support, with Temporal and Dapr integrations for durability.' + novelty: + value: 35 + note: 'Its small set of agents, tools, handoffs, guardrails, sessions, and tracing intentionally favors conventional composable library primitives over a novel graph or hosting architecture.' + actionability: + value: 65 + note: 'The DaprSession bridge enables a focused CF session-store trial, but choosing whether CF should supply durability, tracing, or only ordinary bindings requires additional platform scoping.' + --- ## Summary diff --git a/research/opentelemetry-genai.md b/research/opentelemetry-genai.md index 9a47cb2..d879d8d 100644 --- a/research/opentelemetry-genai.md +++ b/research/opentelemetry-genai.md @@ -7,6 +7,20 @@ cf_areas: [loggregator] status: draft sources: - https://github.com/open-telemetry/semantic-conventions-genai +ratings: + platform-impact: + value: 58 + note: 'CF already transports application telemetry through Loggregator, but lacks standard platform treatment for model, token, agent, tool, and MCP spans and metrics.' + maturity: + value: 48 + note: 'The conventions cover major model vendors, agent operations, MCP, spans, events, and metrics within OpenTelemetry, but the document explicitly records Development status rather than stable standardization.' + novelty: + value: 40 + note: 'Token usage, time-to-first-chunk, planning, and tool-call semantics adapt established tracing and metrics conventions to GenAI rather than introducing a new observability architecture.' + actionability: + value: 82 + note: 'CF can instrument one model-and-tool request with the named gen_ai attributes and metrics, propagate its trace through platform routing, and test Loggregator export without designing a new protocol.' + --- ## Summary diff --git a/research/orleans.md b/research/orleans.md index 8568d8c..49710ad 100644 --- a/research/orleans.md +++ b/research/orleans.md @@ -20,6 +20,20 @@ sources: - https://www.nuget.org/packages/Microsoft.Orleans.Journaling - https://www.nuget.org/packages/Microsoft.Orleans.DurableJobs - https://github.com/managedcode/dotPilot +ratings: + platform-impact: + value: 62 + note: 'CF lacks a virtual-actor runtime providing stable logical identities, transparent activation, a distributed directory, per-actor persistence, streams, and cross-actor transactions.' + maturity: + value: 92 + note: 'Orleans is a long-running .NET Foundation project proven in Halo cloud services, remains active through v10.2.x, and offers mature clustering, persistence, streaming, transactions, and versioning.' + novelty: + value: 28 + note: 'Orleans pioneered the virtual-actor model, but transparent activation and location, actor persistence, and single-threaded grains are now established architecture inherited by systems such as Dapr.' + actionability: + value: 52 + note: 'The note suggests comparing grains with durable agent identities, but a CF experiment must first resolve .NET-only coupling, silo membership, storage bindings, and the absence of official agent-framework integration.' + --- ## Summary diff --git a/research/tanzu-platform-ai.md b/research/tanzu-platform-ai.md index 2541eb3..6af30a7 100644 --- a/research/tanzu-platform-ai.md +++ b/research/tanzu-platform-ai.md @@ -11,6 +11,20 @@ sources: - https://www.cloudfoundry.org/blog/from-idea-to-production-delivering-an-ai-ready-platform-as-a-service-with-vmware-tanzu-platform/ - https://investors.broadcom.com/news-releases/news-release-details/broadcom-announces-tanzu-platform-agent-foundations-bringing - https://blogs.vmware.com/tanzu/scalable-agentic-applications-with-model-context-protocol-mcp/ +ratings: + platform-impact: + value: 75 + note: 'Commercial Tanzu adds model brokering, journaling, and agent-specific isolation around CF, demonstrating that the open-source platform lacks a substantial integrated AI and agent operations layer.' + maturity: + value: 62 + note: 'The BOSH-managed GenAI tile and broker build on production Tanzu and CF machinery, while Agent Foundations was only announced in April 2026 and has less demonstrated operational evidence.' + novelty: + value: 45 + note: 'Model services through a tile and broker reuse established BOSH and service-binding patterns; agent journaling and secure-by-default isolation are newer additions but parallel other managed agent platforms.' + actionability: + value: 88 + note: 'Because the product is built on CF, the note directly supports separating reusable broker, binding, journaling, buildpack, secrets, and networking patterns from proprietary Agent Foundations components.' + --- ## Summary diff --git a/research/temporal.md b/research/temporal.md index bb12547..1500e3c 100644 --- a/research/temporal.md +++ b/research/temporal.md @@ -16,6 +16,20 @@ sources: - https://github.com/temporalio/sdk-python/tree/main/temporalio/contrib/openai_agents - https://temporal.io/blog/durable-flexible-multi-agent-systems - https://temporal.io/cloud +ratings: + platform-impact: + value: 72 + note: 'CF can run stateless Temporal workers but has no durable-execution service providing event histories, deterministic replay, task queues, signals, queries, and resumable agent loops.' + maturity: + value: 92 + note: 'Temporal is a production-proven successor to Cadence with 22k-plus stars, a managed cloud, multiple persistence backends, and official durable-agent integrations for OpenAI, ADK, and LangGraph.' + novelty: + value: 45 + note: 'Append-only event history and deterministic replay are distinctive relative to checkpoint and actor models, but they are established durable-workflow techniques rather than new agent architecture.' + actionability: + value: 78 + note: 'A bound Temporal service with CF workers can concretely test restart recovery, Continue-As-New, namespace isolation, and mTLS, although operating the clustered persistence tier remains a significant question.' + --- ## Summary diff --git a/research/toolhive.md b/research/toolhive.md index 6c46a36..ac42c3b 100644 --- a/research/toolhive.md +++ b/research/toolhive.md @@ -10,6 +10,20 @@ sources: - https://github.com/stacklok/toolhive - https://docs.stacklok.com/toolhive/concepts/mcp-primer - https://docs.stacklok.com/toolhive/concepts/auth-framework +ratings: + platform-impact: + value: 62 + note: 'Diego, UAA, and service brokers offer related primitives, but CF lacks an MCP-specific runtime and gateway combining per-server isolation, delegated credentials, Cedar authorization, registry governance, and audits.' + maturity: + value: 55 + note: 'ToolHive ships a CLI, desktop UI, gateway, registry, and Kubernetes operator with OIDC, audit, OTel, and Prometheus support, but the note gives limited evidence of broad production adoption.' + novelty: + value: 55 + note: 'Centralizing MCP OAuth complexity, backend credential separation, Cedar policy, and isolated server containers is a useful new package of established gateway, policy, and container techniques.' + actionability: + value: 70 + note: 'A focused comparison can deploy one MCP server in Diego, front it with UAA and deny-by-default policy, and test whether a curated service marketplace can reproduce ToolHive registry governance.' + --- ## Summary diff --git a/research/vercel-ai-sdk.md b/research/vercel-ai-sdk.md index 0820b5a..46150ec 100644 --- a/research/vercel-ai-sdk.md +++ b/research/vercel-ai-sdk.md @@ -14,6 +14,20 @@ sources: - https://ai-sdk.dev/docs/ai-sdk-core/telemetry - https://useworkflow.dev - https://www.anthropic.com/research/building-effective-agents +ratings: + platform-impact: + value: 48 + note: 'CF can already host the stateless TypeScript SDK; its main gap is an optional bindable session, memory, or workflow service for the state and durability the SDK intentionally leaves external.' + maturity: + value: 88 + note: 'The Apache-licensed SDK records more than 78 million monthly downloads and supports major JavaScript frameworks, providers, agents, MCP, telemetry, and documented external memory providers.' + novelty: + value: 38 + note: 'Provider abstraction, tool loops, stateless application code, external memory, and Anthropic-derived workflow patterns are familiar techniques assembled into an unusually popular TypeScript API.' + actionability: + value: 78 + note: 'A Node.js buildpack sample can bind an external Memory Provider and OTel exporter, then test the shipped MCP tool-drift detector as a concrete CF security recommendation.' + --- ## Summary diff --git a/research/vertex-agent-engine.md b/research/vertex-agent-engine.md index feb3d43..e4ba6df 100644 --- a/research/vertex-agent-engine.md +++ b/research/vertex-agent-engine.md @@ -14,6 +14,20 @@ sources: - https://cloud.google.com/gemini-enterprise-agent-platform/scale/runtime/tracing - https://cloud.google.com/gemini-enterprise-agent-platform/optimize/evaluation/agent-evaluation - https://cloud.google.com/gemini-enterprise-agent-platform/agents +ratings: + platform-impact: + value: 85 + note: 'CF lacks the integrated managed-agent lifecycle shown by Agent Runtime, Sessions, Memory Bank, separate untrusted Sandboxes, SPIFFE identity, policy gateways, tracing, evaluation, and optimization.' + maturity: + value: 78 + note: 'Google operates a fully managed, framework-agnostic runtime supporting ADK, LangGraph, AG2, LlamaIndex, and custom templates, while A2A deployment and some surrounding capabilities remain preview or unevenly governed.' + novelty: + value: 58 + note: 'The trusted-runtime plus separate untrusted-sandbox split and memory-to-evaluation lifecycle are a newer agent-platform combination, though containers, SPIFFE, gateways, OTel, and managed memory are known patterns.' + actionability: + value: 55 + note: 'The note supplies a strong reference architecture, but CF must still choose among broad experiments in workload identity, sandbox separation, policy gateways, memory, and evaluation rather than one defined implementation step.' + --- ## Summary diff --git a/research/wasmcloud.md b/research/wasmcloud.md index bfa9e42..5bfb73b 100644 --- a/research/wasmcloud.md +++ b/research/wasmcloud.md @@ -12,6 +12,20 @@ sources: - https://wasmcloud.com/docs/v1/concepts/ - https://www.cncf.io/projects/wasmcloud/ - https://wasmcloud.com/blog/2025-01-15-running-distributed-ml-and-ai-workloads-with-wasmcloud +ratings: + platform-impact: + value: 48 + note: 'CF lacks deny-by-default WIT capability contracts, but its general container hosting already covers far more workloads; wasmCloud requires WASI components and is not a direct replacement for buildpack applications.' + maturity: + value: 65 + note: 'wasmCloud is a CNCF Incubating project with multi-organization maintainers, but its v1 architecture is no longer maintained and the early-2026 v2 Kubernetes rearchitecture leaves compatibility and operational questions.' + novelty: + value: 70 + note: 'Language-neutral WIT imports as enforceable deny-by-default capabilities and swappable in-process providers offer an unconventional alternative to container and sidecar security models.' + actionability: + value: 35 + note: 'The document flags capability security as inspiration but finds no clear CF mapping; WASI support for Python and Node agent dependencies, v2 clustering, and a representative MCP workload all need discovery first.' + --- ## Summary diff --git a/scripts/focus_use_cases.yaml b/scripts/focus_use_cases.yaml new file mode 100644 index 0000000..aeabbd2 --- /dev/null +++ b/scripts/focus_use_cases.yaml @@ -0,0 +1,78 @@ +- id: cf-hosted-coding-harnesses + title: CF-hosted coding harnesses + workshop_outcome: Define the minimum CF contract from isolated mutable edit/test through candidate artifact submission to a narrow trusted deployment broker. + primary_actor: A platform customer deploying a coding harness on Cloud Foundry. + beneficiary: A developer delegating repository changes and code execution to the harness. + lifecycle: Stage a reusable environment; create an isolated mutable edit/test session; submit a candidate artifact; let the broker validate target, policy, provenance, and approval; then create conventional CF package, build, deployment, and revision resources, verify, audit, and clean up. + authority_boundary: The harness gets restricted Git, model, and package-registry credentials but no CAPI deployment authority; only the narrow trusted deployment broker holds restricted CAPI credentials and may promote an approved artifact to its validated target. + unique_capabilities: + - Reuse an immutable staged environment while retaining isolated mutable workspace state across edit/test turns. + - Produce a provenance-bound candidate artifact without granting the harness deployment authority. + - Validate target, policy, provenance, and approval in a narrow broker before conventional CF deployment and revision creation. + failure_domain: Session failure stays isolated and recoverable; stale-base or concurrent submissions fail safely; broker failure cannot broaden authority, and deployment supports rollback and audit. + poc: Edit and test in an isolated session, submit a provenance-bound candidate artifact, then have a narrow broker validate target, policy, provenance, and approval and create conventional CF package, build, deployment, and revision resources with rollback and audit. + rfc_decisions: + - How session identity, mutable workspace lifecycle, isolation, networking, stale-base detection, and concurrency are represented. + - What candidate artifact, provenance, target, policy, and approval contract the trusted deployment broker validates. + - How restricted CAPI, Git, model, and package-registry credentials are issued, audited, revoked, and kept out of the wrong trust domain. + - How conventional CF package, build, deployment, and revision creation exposes verification, rollback, and audit outcomes. + core: + - ideas/per-session-sandboxes.md + - ideas/staged-sandbox-environments.md + - ideas/stronger-workload-isolation-for-agents.md + - ideas/localhost-only-egress-for-agents.md + - research/k8s-agent-sandbox.md + - research/firecracker-microvm.md + supporting: + - research/anthropic-managed-agents.md + - research/azure-hosted-agents.md + - research/heroku-ai-platform.md + - research/toolhive.md + - ideas/credential-less-agent-processes.md + primitive_applicability: + durable-addressable-execution: supporting + attested-workload-authority: core + session-scoped-isolated-execution: core + +- id: user-facing-agentic-applications + title: User-facing agentic applications + workshop_outcome: Define the platform contract for routed CF applications that run durable, authorized agent workflows on behalf of end users. + primary_actor: An application developer deploying a multi-tenant agentic application to Cloud Foundry. + beneficiary: An authenticated application user delegating a long-running task while retaining approval and audit control. + lifecycle: Authenticate a user, create a durable execution, invoke authorized tools, checkpoint state, pause for approval or failure, resume on replaceable compute, and complete or cancel with an audit trail. + authority_boundary: The application acts with both workload identity and explicitly delegated user authority; tool access is scoped, auditable, revocable, and unavailable as ambient process credentials. + unique_capabilities: + - Address and resume framework-neutral agent executions independently of an application instance. + - Delegate user authority to specific tools without placing durable credentials in the agent process. + - Pause for human approval and recover from process or provider failure without repeating committed effects. + failure_domain: One tenant's execution, tool failure, or compromised prompt must not leak authority or state across users, and replacement compute must resume without duplicating external effects. + poc: Run a routed invoice agent that checkpoints progress, pauses for user approval, survives instance replacement, and calls one tool with a short-lived user-delegated token. + rfc_decisions: + - Whether CF owns durable execution identity, events, timers, retries, suspension, and cancellation or integrates an external engine. + - How workload identity, user delegation, tool authorization, token exchange, audit, and revocation fit CF APIs and UAA. + - Which state, quota, scaling, telemetry, and optional isolated-worker contracts remain framework-neutral platform responsibilities. + core: + - ideas/durable-tasks-for-cf.md + - ideas/agent-failure-checkpointing.md + - ideas/agent-identity-and-tool-authorization.md + - ideas/credential-less-agent-processes.md + - research/temporal.md + - research/langgraph.md + - research/open-agent-auth.md + supporting: + - ideas/dapr-durable-execution-on-cf.md + - research/dapr-agents.md + - research/hatchet.md + - research/letta.md + - research/llamaindex.md + - research/microsoft-agent-framework.md + - research/openai-agents-sdk.md + - research/opentelemetry-genai.md + - research/mcp-protocol.md + - research/vercel-ai-sdk.md + - ideas/per-session-sandboxes.md + - research/k8s-agent-sandbox.md + primitive_applicability: + durable-addressable-execution: core + attested-workload-authority: core + session-scoped-isolated-execution: conditional diff --git a/scripts/generate_research_map.py b/scripts/generate_research_map.py new file mode 100644 index 0000000..5e23eae --- /dev/null +++ b/scripts/generate_research_map.py @@ -0,0 +1,447 @@ +#!/usr/bin/env python3 +"""Generate the static workshop map from repository notes.""" + +from __future__ import annotations + +import argparse +import html +import json +import pathlib +import re +import sys +from dataclasses import dataclass + +import yaml + +ROOT = pathlib.Path(__file__).resolve().parent.parent +PLOTS_PATH = ROOT / "scripts" / "research_map_plots.yaml" +PRIMITIVES_PATH = ROOT / "scripts" / "platform_primitives.yaml" +FOCUS_USE_CASES_PATH = ROOT / "scripts" / "focus_use_cases.yaml" +OUTPUT_PATH = ROOT / "generated" / "research-map.html" +GITHUB_BASE = "https://github.com/cloudfoundry/agentic-runtime-notes/blob/main" +FRONTMATTER_RE = re.compile(r"^---\n(.*?)\n---\n", re.DOTALL) + + +@dataclass +class Note: + path: pathlib.Path + kind: str + title: str + summary: str + metadata: dict + + +def parse_frontmatter(text: str) -> tuple[dict, str]: + match = FRONTMATTER_RE.match(text) + if not match: + raise ValueError("missing YAML frontmatter block") + metadata = yaml.safe_load(match.group(1)) + if not isinstance(metadata, dict): + raise ValueError("frontmatter must be a YAML mapping") + return metadata, text[match.end() :] + + +def _section_body(body: str, heading: str) -> str | None: + match = re.search(rf"^## {re.escape(heading)}\s*$", body, re.MULTILINE) + if not match: + return None + section = body[match.end() :] + section = re.split(r"^##\s+", section, maxsplit=1, flags=re.MULTILINE)[0] + paragraphs = [p.strip() for p in section.split("\n\n") if p.strip()] + return paragraphs[0] if paragraphs else None + + +def _plain_text(value: str) -> str: + value = re.sub(r"", "", value, flags=re.DOTALL) + value = re.sub(r"\[([^]]+)\]\([^)]*\)", r"\1", value) + value = re.sub(r"[*_`>#]", "", value) + return re.sub(r"\s+", " ", value).strip() + + +def extract_summary(path: str | pathlib.Path, body: str) -> str: + heading = "Summary" if str(path).startswith("research/") else "The idea" + summary = _section_body(body, heading) + if summary is None: + paragraphs = [p.strip() for p in body.split("\n\n") if p.strip()] + summary = paragraphs[0] if paragraphs else "No summary provided." + return _plain_text(summary) + + +def validate_ratings(ratings: object) -> dict: + if ratings is None: + return {} + if not isinstance(ratings, dict): + raise ValueError("ratings must be a mapping") + for name, rating in ratings.items(): + if not isinstance(rating, dict): + raise ValueError(f"rating '{name}' must be a mapping") + value = rating.get("value") + if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= 100: + raise ValueError(f"rating '{name}' value must be an integer in 0..100") + if not isinstance(rating.get("note"), str) or not rating["note"].strip(): + raise ValueError(f"rating '{name}' requires a non-empty note") + return ratings + + +def parse_note(path: pathlib.Path, text: str) -> Note: + metadata, body = parse_frontmatter(text) + relative = path.as_posix() + kind = "research" if relative.startswith("research/") else "idea" + title = str(metadata.get("title") or path.stem.replace("-", " ").title()) + ratings = validate_ratings(metadata.get("ratings")) + metadata["ratings"] = ratings + return Note(path, kind, title, extract_summary(relative, body), metadata) + + +def derive_position(ratings: dict, plot: dict) -> dict | None: + x_name = plot["x"]["rating"] + y_name = plot["y"]["rating"] + if x_name not in ratings or y_name not in ratings: + return None + return {"x": ratings[x_name]["value"], "y": ratings[y_name]["value"]} + + +def group_payload(payload: list[dict]) -> dict[tuple[int, int], list[dict]]: + groups: dict[tuple[int, int], list[dict]] = {} + for item in payload: + position = item.get("position") + if position is not None: + key = (position["x"], position["y"]) + groups.setdefault(key, []).append(item) + return groups + + +def github_url(path: pathlib.Path) -> str: + return f"{GITHUB_BASE}/{path.as_posix()}" + + +def validate_plot(plot: dict) -> None: + for axis in ("x", "y"): + if not isinstance(plot.get(axis), dict) or not plot[axis].get("rating"): + raise ValueError(f"plot axis '{axis}' must name a rating") + + +def load_notes(root: pathlib.Path = ROOT) -> list[Note]: + notes = [] + for directory, kind in ((root / "research", "research"), (root / "ideas", "idea")): + for path in sorted(directory.glob("*.md")): + if path.name in {"README.md", "TEMPLATE.md"}: + continue + notes.append(parse_note(path.relative_to(root), path.read_text(encoding="utf-8"))) + return notes + + +def load_plots() -> dict: + plots = yaml.safe_load(PLOTS_PATH.read_text(encoding="utf-8")) + if not isinstance(plots, dict) or not plots: + raise ValueError("plot configuration must be a non-empty mapping") + for plot in plots.values(): + validate_plot(plot) + return plots + + +def validate_primitives(primitives: object, known_paths: set[str]) -> list[dict]: + if not isinstance(primitives, list) or not primitives: + raise ValueError("primitive configuration must be a non-empty list") + + required_fields = ( + "id", + "title", + "proposition", + "cf_gap", + "strategic_decision", + "poc", + "rfc_scope", + ) + seen_ids = set() + for primitive in primitives: + if not isinstance(primitive, dict): + raise ValueError("each primitive must be a mapping") + for field in required_fields: + if not isinstance(primitive.get(field), str) or not primitive[field].strip(): + raise ValueError(f"primitive requires a non-empty '{field}'") + + primitive_id = primitive["id"] + if primitive_id in seen_ids: + raise ValueError(f"duplicate primitive id: {primitive_id}") + seen_ids.add(primitive_id) + + core = primitive.get("core") + supporting = primitive.get("supporting") + if not isinstance(core, list) or not core: + raise ValueError(f"primitive '{primitive_id}' requires non-empty core membership") + if not isinstance(supporting, list): + raise ValueError(f"primitive '{primitive_id}' supporting membership must be a list") + + memberships = core + supporting + if any(not isinstance(path, str) or not path.strip() for path in memberships): + raise ValueError(f"primitive '{primitive_id}' note paths must be non-empty strings") + if len(memberships) != len(set(memberships)): + raise ValueError(f"primitive '{primitive_id}' has a duplicate note path") + for path in memberships: + if path not in known_paths: + raise ValueError(f"primitive '{primitive_id}' references unknown note path: {path}") + + if len(primitives) != 3: + raise ValueError("primitive configuration must contain exactly 3 initial primitives") + return primitives + + +def load_primitives() -> list[dict]: + primitives = yaml.safe_load(PRIMITIVES_PATH.read_text(encoding="utf-8")) + known_paths = {note.path.as_posix() for note in load_notes()} + return validate_primitives(primitives, known_paths) + + +def validate_focus_use_cases( + use_cases: object, known_paths: set[str], primitive_ids: set[str] +) -> list[dict]: + if not isinstance(use_cases, list) or len(use_cases) != 2: + raise ValueError("focus use case configuration must contain exactly 2 entries") + + required_ids = { + "cf-hosted-coding-harnesses", + "user-facing-agentic-applications", + } + string_fields = ( + "id", + "title", + "workshop_outcome", + "primary_actor", + "beneficiary", + "lifecycle", + "authority_boundary", + "failure_domain", + "poc", + ) + list_fields = ("unique_capabilities", "rfc_decisions") + allowed_applicability = {"core", "conditional", "supporting"} + seen_ids = set() + for use_case in use_cases: + if not isinstance(use_case, dict): + raise ValueError("each focus use case must be a mapping") + for field in string_fields: + if not isinstance(use_case.get(field), str) or not use_case[field].strip(): + raise ValueError(f"focus use case requires a non-empty '{field}'") + for field in list_fields: + values = use_case.get(field) + if ( + not isinstance(values, list) + or not values + or any(not isinstance(value, str) or not value.strip() for value in values) + ): + raise ValueError(f"focus use case requires a non-empty '{field}' list of strings") + + use_case_id = use_case["id"] + if use_case_id in seen_ids: + raise ValueError(f"duplicate focus use case id: {use_case_id}") + seen_ids.add(use_case_id) + + core = use_case.get("core") + supporting = use_case.get("supporting") + if not isinstance(core, list) or not core: + raise ValueError(f"focus use case '{use_case_id}' requires non-empty core membership") + if not isinstance(supporting, list): + raise ValueError(f"focus use case '{use_case_id}' supporting membership must be a list") + memberships = core + supporting + if any(not isinstance(path, str) or not path.strip() for path in memberships): + raise ValueError(f"focus use case '{use_case_id}' note paths must be non-empty strings") + if len(memberships) != len(set(memberships)): + raise ValueError(f"focus use case '{use_case_id}' has a duplicate note path") + for path in memberships: + if path not in known_paths: + raise ValueError(f"focus use case '{use_case_id}' references unknown note path: {path}") + + applicability = use_case.get("primitive_applicability") + if not isinstance(applicability, dict) or set(applicability) != primitive_ids: + raise ValueError( + f"focus use case '{use_case_id}' primitive applicability must contain exactly the known primitive ids" + ) + for primitive_id, value in applicability.items(): + if not isinstance(value, str): + raise ValueError( + f"focus use case '{use_case_id}' applicability for primitive '{primitive_id}' must be a string" + ) + if value not in allowed_applicability: + raise ValueError( + f"focus use case '{use_case_id}' has invalid applicability '{value}' for primitive '{primitive_id}'" + ) + + if seen_ids != required_ids: + raise ValueError("focus use case configuration must contain the approved focus use case ids") + return use_cases + + +def load_focus_use_cases() -> list[dict]: + use_cases = yaml.safe_load(FOCUS_USE_CASES_PATH.read_text(encoding="utf-8")) + known_paths = {note.path.as_posix() for note in load_notes()} + primitive_ids = {primitive["id"] for primitive in load_primitives()} + return validate_focus_use_cases(use_cases, known_paths, primitive_ids) + + +def note_payload(note: Note, plot: dict) -> dict: + ratings = note.metadata["ratings"] + return { + "id": note.path.as_posix(), + "kind": note.kind, + "title": note.title, + "summary": note.summary, + "tags": note.metadata.get("tags", []), + "author": note.metadata.get("author", ""), + "date": str(note.metadata.get("date", "")), + "ratings": ratings, + "position": derive_position(ratings, plot), + "url": github_url(note.path), + } + + +def generate_html( + notes: list[Note], plots: dict, primitives: list[dict], use_cases: list[dict] | None = None +) -> str: + if use_cases is None: + use_cases = load_focus_use_cases() + plot_payloads = {plot_id: [note_payload(note, plot) for note in notes] for plot_id, plot in plots.items()} + data = json.dumps(plot_payloads, ensure_ascii=True).replace("{html.escape(note_titles.get(path, pathlib.Path(path).stem.replace("-", " ").title()))}' + for path in use_case[relationship] + ) + evidence.append(f'

{relationship.title()}

    {links}
') + capabilities = "".join(f'
  • {html.escape(value)}
    • ' for value in use_case["unique_capabilities"]) + decisions = "".join(f'
  • {html.escape(value)}
    • ' for value in use_case["rfc_decisions"]) + applicability = "".join( + f'
  • {html.escape(primitive_titles.get(primitive_id, primitive_id))}: {html.escape(value.title())}
    • ' + for primitive_id, value in use_case["primitive_applicability"].items() + ) + use_case_cards.append(f'''
    + +
    Actors, lifecycle, decisions, and evidence
    Primary actor
    {html.escape(use_case["primary_actor"])}
    Beneficiary
    {html.escape(use_case["beneficiary"])}
    Lifecycle
    {html.escape(use_case["lifecycle"])}
    Authority boundary
    {html.escape(use_case["authority_boundary"])}
    Failure domain
    {html.escape(use_case["failure_domain"])}

    Unique capabilities

      {capabilities}

    RFC decisions

      {decisions}

    Primitive applicability

      {applicability}
    {''.join(evidence)}
    ''') + accents = ("#68d5ac", "#f1b866", "#8eb8ff") + cards = [] + for primitive, accent in zip(primitives, accents): + primitive_id = html.escape(primitive["id"]) + memberships = [] + for relationship in ("core", "supporting"): + links = "".join( + f'
  • {html.escape(note_titles.get(path, pathlib.Path(path).stem.replace("-", " ").title()))}
    • ' + for path in primitive[relationship] + ) + memberships.append(f'

    {relationship.title()}

      {links}
    ') + cards.append(f'''
    + +
    Gap, experiments, and evidence
    Current CF gap
    {html.escape(primitive["cf_gap"])}
    Candidate POC
    {html.escape(primitive["poc"])}
    Candidate RFC scope
    {html.escape(primitive["rfc_scope"])}
    {''.join(memberships)}
    ''') + matrices = [] + tabs = [] + for index, (plot_id, plot) in enumerate(plots.items()): + payload = plot_payloads[plot_id] + markers = [] + unplaced = [] + for key, group in group_payload(payload).items(): + p = group[0]["position"] + if len(group) == 1: + item = group[0] + markers.append( + f'' + ) + else: + cluster_id = f"{plot_id}-cluster-{len(markers)}" + note_ids = html.escape(json.dumps([item["id"] for item in group]), quote=True) + markers.append( + f'' + ) + for item in payload: + if item["position"] is None: + label = "Research" if item["kind"] == "research" else "Idea" + escaped_kind = html.escape(item["kind"]) + unplaced.append(f'
  • {label}{html.escape(item["title"])}
    • ') + title = html.escape(plot["title"]) + x = plot["x"] + y = plot["y"] + selected = "true" if index == 0 else "false" + tab_index = "0" if index == 0 else "-1" + hidden = "" if index == 0 else " hidden" + escaped_plot_id = html.escape(plot_id) + tabs.append(f'') + matrices.append(f'''

    {title}

    {''.join(markers)}{html.escape(x["low"])} < {html.escape(x["label"])} > {html.escape(x["high"])}{html.escape(y["low"])} < {html.escape(y["label"])} > {html.escape(y["high"])}
    Unplaced notes ({len(unplaced)})
      {''.join(unplaced) or '
    • All notes are placed.
      • '}
    ''') + return f''' + +Agentic Runtime Workshop Results

    Agentic Runtime Working Group

    Workshop results

    +

    This page summarizes the workshop results: two focus use cases, three candidate platform primitives, and the collected evidence behind them.

    +

    Focus use cases

    {''.join(use_case_cards)}

    Candidate platform primitives

    {''.join(cards)}

    ResearchIdea
    {''.join(tabs)}
    {''.join(matrices)}
    +
     +''' + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + try: + output = generate_html(load_notes(), load_plots(), load_primitives(), load_focus_use_cases()) + except (OSError, ValueError, yaml.YAMLError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + if args.check: + current = OUTPUT_PATH.read_text(encoding="utf-8") if OUTPUT_PATH.exists() else "" + if current != output: + print(f"{OUTPUT_PATH} is stale; run the generator", file=sys.stderr) + return 1 + else: + OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + OUTPUT_PATH.write_text(output, encoding="utf-8") + print(f"wrote {OUTPUT_PATH}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/platform_primitives.yaml b/scripts/platform_primitives.yaml new file mode 100644 index 0000000..1b9c75a --- /dev/null +++ b/scripts/platform_primitives.yaml @@ -0,0 +1,66 @@ +- id: durable-addressable-execution + title: Durable, Addressable Execution + proposition: Keep logical execution durable and addressable while compute remains replaceable. + cf_gap: CF tasks are tied to one compute attempt and lack stable execution identity, checkpoints, suspend/resume, durable events, timers, and retry policy. + strategic_decision: Decide whether CF should provide a durable execution lifecycle that frameworks and bound state services can build on. + poc: Run a checkpointing task under a stable execution ID, suspend it after persisting state, then resume it on replacement compute through events, timers, and bounded retries. + rfc_scope: Define execution identity, lifecycle states, checkpoint handoff, event and timer delivery, retry semantics, observability, and the boundary between CF lifecycle ownership and bound state stores. + core: + - ideas/durable-tasks-for-cf.md + - ideas/dapr-durable-execution-on-cf.md + - ideas/agent-failure-checkpointing.md + - research/temporal.md + - research/dapr-agents.md + - research/cloudflare-agents.md + supporting: + - research/hatchet.md + - research/orleans.md + - research/langgraph.md + - research/dapr.md + - research/letta.md + - research/aws-agents.md + - research/azure-hosted-agents.md + +- id: attested-workload-authority + title: Attested Workload Authority and Mediated Tool Access + proposition: Exchange platform-attested workload identity for scoped authority while credentials and outbound tool access remain mediated by the platform. + cf_gap: CF issues workload identity certificates but does not exchange them for scoped tool authority, keep third-party credentials out of workloads, mediate off-platform access, or record delegation-aware audit events. + strategic_decision: Decide whether CF should become the portable trust and policy layer between agent workloads and the tools they invoke. + poc: Exchange a Diego instance identity certificate for a short-lived scoped token, invoke one allowed tool through a credential proxy and egress mediator, deny another, and emit attributable audit events. + rfc_scope: Define workload token exchange, authority and delegation claims, credential brokering, outbound mediation and policy enforcement, audit events, revocation, and integration boundaries for UAA, routing, and service brokers. + core: + - ideas/agent-identity-and-tool-authorization.md + - ideas/credential-less-agent-processes.md + - ideas/localhost-only-egress-for-agents.md + - research/open-agent-auth.md + - research/toolhive.md + - research/mcp-protocol.md + supporting: + - research/aws-agents.md + - research/vertex-agent-engine.md + - research/anthropic-managed-agents.md + - research/azure-hosted-agents.md + - research/cloudflare-agents.md + - research/opentelemetry-genai.md + +- id: session-scoped-isolated-execution + title: Session-Scoped Isolated Execution + proposition: Create resumable, session-scoped execution from reusable staged environments, mutable workspaces, selectable isolation, and controlled networking. + cf_gap: CF can stage apps and run ephemeral tasks but cannot cheaply compose a reusable environment with per-session workspace state, select stronger isolation, constrain session networking, or resume the session lifecycle. + strategic_decision: Decide whether CF should expose session execution as a composition of existing package, build, droplet, task, volume, networking, and isolation capabilities. + poc: Start two isolated sessions from one content-addressed staged environment, attach separate mutable workspaces, apply per-session egress policy, stop one session, and resume it on fresh compute. + rfc_scope: Define environment and workspace references, session identity and lifecycle, isolation classes, network policy, workspace persistence and cleanup, scheduling, quotas, and compatibility with existing CF staging and task APIs. + core: + - ideas/per-session-sandboxes.md + - ideas/staged-sandbox-environments.md + - ideas/stronger-workload-isolation-for-agents.md + - ideas/localhost-only-egress-for-agents.md + - research/k8s-agent-sandbox.md + - research/firecracker-microvm.md + supporting: + - research/heroku-ai-platform.md + - research/azure-hosted-agents.md + - research/anthropic-managed-agents.md + - research/aws-agents.md + - research/cloudflare-agents.md + - research/toolhive.md diff --git a/scripts/research_map_plots.yaml b/scripts/research_map_plots.yaml new file mode 100644 index 0000000..bcaca56 --- /dev/null +++ b/scripts/research_map_plots.yaml @@ -0,0 +1,72 @@ +platform-impact-maturity: + title: Platform Impact x Maturity + x: + rating: maturity + label: Maturity + low: Emerging + high: Established + y: + rating: platform-impact + label: Platform Impact + low: Local concern + high: Platform-wide concern +platform-impact-novelty: + title: Platform Impact x Novelty + x: + rating: novelty + label: Novelty + low: Familiar + high: Emerging + y: + rating: platform-impact + label: Platform Impact + low: Local concern + high: Platform-wide concern +platform-impact-actionability: + title: Platform Impact x Actionability + x: + rating: actionability + label: Actionability + low: Exploratory + high: Ready to act + y: + rating: platform-impact + label: Platform Impact + low: Local concern + high: Platform-wide concern +novelty-actionability: + title: Novelty x Actionability + x: + rating: novelty + label: Novelty + low: Familiar + high: Emerging + y: + rating: actionability + label: Actionability + low: Exploratory + high: Ready to act +maturity-novelty: + title: Maturity x Novelty + x: + rating: novelty + label: Novelty + low: Familiar + high: Emerging + y: + rating: maturity + label: Maturity + low: Emerging + high: Established +maturity-actionability: + title: Maturity x Actionability + x: + rating: actionability + label: Actionability + low: Exploratory + high: Ready to act + y: + rating: maturity + label: Maturity + low: Emerging + high: Established diff --git a/scripts/summarize_ratings.py b/scripts/summarize_ratings.py new file mode 100644 index 0000000..c2d3808 --- /dev/null +++ b/scripts/summarize_ratings.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Summarize and validate rating distributions across all notes.""" + +from __future__ import annotations + +import argparse +import statistics +import sys +from itertools import combinations + +if __package__: + from scripts.generate_research_map import load_notes +else: + from generate_research_map import load_notes + + +REQUIRED_RATINGS = ("platform-impact", "maturity", "novelty", "actionability") +MATRICES = { + "platform-impact-maturity": ("platform-impact", "maturity"), + "platform-impact-novelty": ("platform-impact", "novelty"), + "platform-impact-actionability": ("platform-impact", "actionability"), + "novelty-actionability": ("actionability", "novelty"), + "maturity-novelty": ("maturity", "novelty"), + "maturity-actionability": ("maturity", "actionability"), +} + + +def summarize_ratings(notes) -> dict: + values = {name: [] for name in REQUIRED_RATINGS} + rows = [] + for note in notes: + ratings = note.metadata.get("ratings", {}) + row = { + name: rating["value"] + for name, rating in ratings.items() + if name in values + and isinstance(rating, dict) + and isinstance(rating.get("value"), (int, float)) + } + rows.append(row) + for name, value in row.items(): + values[name].append(value) + + rating_summaries = {} + for name, observed in values.items(): + if not observed: + continue + quartiles = ( + statistics.quantiles(observed, n=4, method="inclusive") + if len(observed) > 1 + else (observed[0], observed[0], observed[0]) + ) + distinct = len(set(observed)) + rating_summaries[name] = { + "count": len(observed), + "minimum": min(observed), + "maximum": max(observed), + "median": statistics.median(observed), + "first_quartile": quartiles[0], + "third_quartile": quartiles[2], + "distinct": distinct, + "duplicates": len(observed) - distinct, + } + + correlations = {} + for first, second in combinations(REQUIRED_RATINGS, 2): + paired = [(row[first], row[second]) for row in rows if first in row and second in row] + key = f"{first}:{second}" + try: + correlations[key] = statistics.correlation( + [pair[0] for pair in paired], [pair[1] for pair in paired] + ) + except statistics.StatisticsError: + correlations[key] = None + + matrices = {} + for matrix, (vertical, horizontal) in MATRICES.items(): + quadrants = {"low-low": 0, "low-high": 0, "high-low": 0, "high-high": 0} + for row in rows: + if vertical in row and horizontal in row: + vertical_side = "low" if row[vertical] <= 50 else "high" + horizontal_side = "low" if row[horizontal] <= 50 else "high" + quadrants[f"{vertical_side}-{horizontal_side}"] += 1 + matrices[matrix] = quadrants + + return { + "count": len(rows), + "ratings": rating_summaries, + "correlations": correlations, + "matrices": matrices, + } + + +def validate_summary(summary: dict) -> None: + ratings = summary["ratings"] + for name in REQUIRED_RATINGS: + if name not in ratings or ratings[name]["count"] != summary["count"]: + raise ValueError(f"missing required rating: {name}") + for name in REQUIRED_RATINGS: + rating = ratings[name] + if rating["distinct"] < 5: + raise ValueError(f"rating '{name}' has fewer than 5 distinct values") + if rating["maximum"] <= 50: + raise ValueError(f"rating '{name}' values are strictly below 50") + if rating["minimum"] >= 50: + raise ValueError(f"rating '{name}' values are strictly above 50") + + +def print_summary(summary: dict) -> None: + for name, rating in summary["ratings"].items(): + print( + f"{name}: count={rating['count']} min={rating['minimum']} " + f"max={rating['maximum']} median={rating['median']} " + f"q1={rating['first_quartile']} q3={rating['third_quartile']} " + f"distinct={rating['distinct']} duplicates={rating['duplicates']}" + ) + print("correlations:") + for pair, correlation in summary["correlations"].items(): + value = "undefined" if correlation is None else f"{correlation:.3f}" + print(f" {pair}: {value}") + print("matrix quadrants (vertical-horizontal):") + for matrix, quadrants in summary["matrices"].items(): + counts = " ".join(f"{name}={count}" for name, count in quadrants.items()) + print(f" {matrix}: {counts}") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + try: + summary = summarize_ratings(load_notes()) + print_summary(summary) + if args.check: + validate_summary(summary) + except (OSError, ValueError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_generate_research_map.py b/tests/test_generate_research_map.py new file mode 100644 index 0000000..bd83c32 --- /dev/null +++ b/tests/test_generate_research_map.py @@ -0,0 +1,984 @@ +import html as html_lib +import json +import pathlib +import re +import unittest +from types import SimpleNamespace + +from scripts.generate_research_map import ( + FOCUS_USE_CASES_PATH, + PRIMITIVES_PATH, + derive_position, + extract_summary, + github_url, + generate_html, + group_payload, + load_focus_use_cases, + load_primitives, + load_plots, + load_notes, + parse_note, + validate_focus_use_cases, + validate_primitives, + validate_ratings, +) +from scripts.summarize_ratings import summarize_ratings, validate_summary + + +PLOT = {"x": {"rating": "maturity"}, "y": {"rating": "platform-impact"}} +REQUIRED_RATINGS = ("platform-impact", "maturity", "novelty", "actionability") +PRIMITIVE = { + "id": "durable-addressable-execution", + "title": "Durable, Addressable Execution", + "proposition": "Keep execution durable while compute remains replaceable.", + "cf_gap": "CF tasks have no durable execution identity or suspend/resume lifecycle.", + "strategic_decision": "Decide whether CF should own durable execution lifecycle semantics.", + "poc": "Resume a checkpointed task on replacement compute.", + "rfc_scope": "Execution identity, lifecycle, events, timers, and retries.", + "core": ["ideas/durable-tasks-for-cf.md", "research/temporal.md"], + "supporting": ["research/dapr-agents.md"], +} +PRIMITIVE_IDS = { + "durable-addressable-execution", + "attested-workload-authority", + "session-scoped-isolated-execution", +} +FOCUS_USE_CASE = { + "id": "cf-hosted-coding-harnesses", + "title": "CF-hosted coding harnesses", + "workshop_outcome": "Determine the minimum CF platform contract for hosted coding harnesses.", + "primary_actor": "A developer deploying a coding harness to Cloud Foundry.", + "beneficiary": "A software team using the harness to change a repository.", + "lifecycle": "Stage an environment, start a session, execute tools, suspend it, and resume it.", + "authority_boundary": "The harness delegates only scoped repository and tool access.", + "unique_capabilities": ["Reusable staged environments", "Resumable isolated sessions"], + "failure_domain": "A failed sandbox must not lose the session workspace or affect another session.", + "poc": "Run and resume two isolated coding sessions from one staged environment.", + "rfc_decisions": ["Session resource and lifecycle", "Workspace and network policy"], + "core": ["ideas/per-session-sandboxes.md", "research/k8s-agent-sandbox.md"], + "supporting": ["research/firecracker-microvm.md"], + "primitive_applicability": { + "durable-addressable-execution": "supporting", + "attested-workload-authority": "core", + "session-scoped-isolated-execution": "core", + }, +} + + +def focus_use_cases(first=FOCUS_USE_CASE): + return [ + first, + {**FOCUS_USE_CASE, "id": "user-facing-agentic-applications"}, + ] + + +def rating_notes(values): + return [ + SimpleNamespace(metadata={"ratings": ratings}) + for ratings in ( + { + name: {"value": value, "note": "reason"} + for name, value in zip(REQUIRED_RATINGS, row) + } + for row in values + ) + ] + + +def mixed_cluster_fixture(): + paths = [pathlib.Path(f"research/cluster-{index}.md") for index in range(5)] + notes = [ + SimpleNamespace( + path=path, + kind="research", + title=f"Cluster note {index}", + summary=f"Summary {index}", + metadata={ + "ratings": { + "maturity": {"value": 42, "note": "reason"}, + "platform-impact": {"value": 37, "note": "reason"}, + } + }, + ) + for index, path in enumerate(paths) + ] + plot = { + "title": "Mixed cluster", + "x": {"rating": "maturity", "label": "Maturity", "low": "Low", "high": "High"}, + "y": { + "rating": "platform-impact", + "label": "Platform impact", + "low": "Low", + "high": "High", + }, + } + primitive = { + **PRIMITIVE, + "core": [paths[1].as_posix()], + "supporting": [paths[2].as_posix(), paths[4].as_posix()], + } + use_case = { + **FOCUS_USE_CASE, + "core": [paths[0].as_posix(), paths[2].as_posix()], + "supporting": [paths[4].as_posix()], + } + return notes, {"mixed:plot": plot}, [primitive], focus_use_cases(use_case) + + +class ResearchMapTests(unittest.TestCase): + def test_load_focus_use_cases_loads_the_two_approved_use_cases(self): + use_cases = load_focus_use_cases() + + self.assertEqual(FOCUS_USE_CASES_PATH.name, "focus_use_cases.yaml") + self.assertEqual( + [use_case["id"] for use_case in use_cases], + ["cf-hosted-coding-harnesses", "user-facing-agentic-applications"], + ) + + def test_coding_harness_use_case_covers_edit_to_trusted_deployment_lifecycle(self): + use_case = load_focus_use_cases()[0] + narrative = " ".join( + str(use_case[field]) + for field in ( + "workshop_outcome", + "lifecycle", + "authority_boundary", + "unique_capabilities", + "failure_domain", + "poc", + "rfc_decisions", + ) + ).lower() + + for required_phrase in ( + "mutable edit/test", + "candidate artifact", + "trusted deployment broker", + "target, policy, provenance, and approval", + "package, build, deployment, and revision", + "capi, git, model, and package-registry credentials", + "stale-base", + "concurrency", + "rollback", + "audit", + ): + with self.subTest(required_phrase=required_phrase): + self.assertIn(required_phrase, narrative) + + def test_validate_focus_use_cases_preserves_use_case_and_membership_order(self): + first = { + **FOCUS_USE_CASE, + "core": ["b.md", "a.md"], + "supporting": ["d.md", "c.md"], + } + + validated = validate_focus_use_cases( + focus_use_cases(first), + {"a.md", "b.md", "c.md", "d.md", *FOCUS_USE_CASE["core"], *FOCUS_USE_CASE["supporting"]}, + PRIMITIVE_IDS, + ) + + self.assertEqual([item["id"] for item in validated], [first["id"], "user-facing-agentic-applications"]) + self.assertEqual(validated[0]["core"], ["b.md", "a.md"]) + self.assertEqual(validated[0]["supporting"], ["d.md", "c.md"]) + + def test_validate_focus_use_cases_requires_exactly_two_approved_ids(self): + known_paths = set(FOCUS_USE_CASE["core"] + FOCUS_USE_CASE["supporting"]) + with self.assertRaisesRegex(ValueError, "exactly 2"): + validate_focus_use_cases([FOCUS_USE_CASE], known_paths, PRIMITIVE_IDS) + with self.assertRaisesRegex(ValueError, "approved focus use case ids"): + validate_focus_use_cases( + focus_use_cases({**FOCUS_USE_CASE, "id": "another-use-case"}), known_paths, PRIMITIVE_IDS + ) + + def test_validate_focus_use_cases_rejects_invalid_required_fields(self): + known_paths = set(FOCUS_USE_CASE["core"] + FOCUS_USE_CASE["supporting"]) + string_fields = ( + "id", "title", "workshop_outcome", "primary_actor", "beneficiary", "lifecycle", + "authority_boundary", "failure_domain", "poc", + ) + list_fields = ("unique_capabilities", "rfc_decisions") + for field in string_fields: + with self.subTest(field=field), self.assertRaisesRegex(ValueError, f"non-empty '{field}'"): + validate_focus_use_cases(focus_use_cases({**FOCUS_USE_CASE, field: " "}), known_paths, PRIMITIVE_IDS) + for field in list_fields: + with self.subTest(field=field), self.assertRaisesRegex(ValueError, f"non-empty '{field}'"): + validate_focus_use_cases(focus_use_cases({**FOCUS_USE_CASE, field: []}), known_paths, PRIMITIVE_IDS) + + def test_validate_focus_use_cases_rejects_duplicate_ids(self): + known_paths = set(FOCUS_USE_CASE["core"] + FOCUS_USE_CASE["supporting"]) + with self.assertRaisesRegex(ValueError, "duplicate focus use case id"): + validate_focus_use_cases([FOCUS_USE_CASE, dict(FOCUS_USE_CASE)], known_paths, PRIMITIVE_IDS) + + def test_validate_focus_use_cases_rejects_empty_core_membership(self): + with self.assertRaisesRegex(ValueError, "non-empty core"): + validate_focus_use_cases(focus_use_cases({**FOCUS_USE_CASE, "core": []}), set(FOCUS_USE_CASE["supporting"]), PRIMITIVE_IDS) + + def test_validate_focus_use_cases_rejects_duplicate_membership(self): + use_case = {**FOCUS_USE_CASE, "supporting": [FOCUS_USE_CASE["core"][0]]} + with self.assertRaisesRegex(ValueError, "duplicate note path"): + validate_focus_use_cases(focus_use_cases(use_case), set(FOCUS_USE_CASE["core"]), PRIMITIVE_IDS) + + def test_validate_focus_use_cases_rejects_unknown_note(self): + known_paths = {FOCUS_USE_CASE["core"][0], *FOCUS_USE_CASE["supporting"]} + with self.assertRaisesRegex(ValueError, "unknown note path.*research/k8s-agent-sandbox.md"): + validate_focus_use_cases(focus_use_cases(), known_paths, PRIMITIVE_IDS) + + def test_validate_focus_use_cases_rejects_unknown_or_missing_primitive_ids(self): + known_paths = set(FOCUS_USE_CASE["core"] + FOCUS_USE_CASE["supporting"]) + applicability = {**FOCUS_USE_CASE["primitive_applicability"]} + applicability.pop("attested-workload-authority") + applicability["unknown-primitive"] = "core" + with self.assertRaisesRegex(ValueError, "primitive applicability must contain exactly"): + validate_focus_use_cases( + focus_use_cases({**FOCUS_USE_CASE, "primitive_applicability": applicability}), + known_paths, + PRIMITIVE_IDS, + ) + + def test_validate_focus_use_cases_rejects_invalid_applicability(self): + known_paths = set(FOCUS_USE_CASE["core"] + FOCUS_USE_CASE["supporting"]) + applicability = {**FOCUS_USE_CASE["primitive_applicability"], "attested-workload-authority": "optional"} + with self.assertRaisesRegex(ValueError, "invalid applicability.*optional"): + validate_focus_use_cases( + focus_use_cases({**FOCUS_USE_CASE, "primitive_applicability": applicability}), + known_paths, + PRIMITIVE_IDS, + ) + + def test_validate_focus_use_cases_rejects_non_string_applicability(self): + known_paths = set(FOCUS_USE_CASE["core"] + FOCUS_USE_CASE["supporting"]) + applicability = {**FOCUS_USE_CASE["primitive_applicability"], "attested-workload-authority": ["core"]} + with self.assertRaisesRegex( + ValueError, + "applicability for primitive 'attested-workload-authority' must be a string", + ): + validate_focus_use_cases( + focus_use_cases({**FOCUS_USE_CASE, "primitive_applicability": applicability}), + known_paths, + PRIMITIVE_IDS, + ) + + def test_load_primitives_loads_the_three_approved_primitives(self): + primitives = load_primitives() + + self.assertEqual(PRIMITIVES_PATH.name, "platform_primitives.yaml") + self.assertEqual( + [primitive["id"] for primitive in primitives], + [ + "durable-addressable-execution", + "attested-workload-authority", + "session-scoped-isolated-execution", + ], + ) + + def test_validate_primitives_preserves_core_and_supporting_order(self): + primitive = {**PRIMITIVE, "core": ["b.md", "a.md"], "supporting": ["d.md", "c.md"]} + primitives = [ + primitive, + {**PRIMITIVE, "id": "second"}, + {**PRIMITIVE, "id": "third"}, + ] + + validated = validate_primitives( + primitives, + {"a.md", "b.md", "c.md", "d.md", *PRIMITIVE["core"], *PRIMITIVE["supporting"]}, + ) + + self.assertEqual(validated[0]["core"], ["b.md", "a.md"]) + self.assertEqual(validated[0]["supporting"], ["d.md", "c.md"]) + + def test_validate_primitives_rejects_non_list_configuration(self): + with self.assertRaisesRegex(ValueError, "non-empty list"): + validate_primitives({"primitives": [PRIMITIVE]}, set(PRIMITIVE["core"] + PRIMITIVE["supporting"])) + + def test_validate_primitives_requires_exactly_three_initial_primitives(self): + primitives = load_primitives()[:2] + known_paths = {path for primitive in primitives for path in primitive["core"] + primitive["supporting"]} + + with self.assertRaisesRegex(ValueError, "exactly 3"): + validate_primitives(primitives, known_paths) + + def test_validate_primitives_rejects_blank_required_fields(self): + for field in ("id", "title", "proposition", "cf_gap", "strategic_decision", "poc", "rfc_scope"): + with self.subTest(field=field): + primitive = {**PRIMITIVE, field: " "} + with self.assertRaisesRegex(ValueError, f"non-empty '{field}'"): + validate_primitives([primitive], set(PRIMITIVE["core"] + PRIMITIVE["supporting"])) + + def test_validate_primitives_rejects_duplicate_ids(self): + with self.assertRaisesRegex(ValueError, "duplicate primitive id"): + validate_primitives([PRIMITIVE, dict(PRIMITIVE)], set(PRIMITIVE["core"] + PRIMITIVE["supporting"])) + + def test_validate_primitives_rejects_empty_core_membership(self): + with self.assertRaisesRegex(ValueError, "non-empty core"): + validate_primitives([{**PRIMITIVE, "core": []}], set(PRIMITIVE["supporting"])) + + def test_validate_primitives_rejects_duplicate_membership(self): + primitive = {**PRIMITIVE, "supporting": [PRIMITIVE["core"][0]]} + with self.assertRaisesRegex(ValueError, "duplicate note path"): + validate_primitives([primitive], set(PRIMITIVE["core"])) + + def test_validate_primitives_rejects_unknown_note(self): + with self.assertRaisesRegex(ValueError, "unknown note path.*research/temporal.md"): + validate_primitives([PRIMITIVE], {"ideas/durable-tasks-for-cf.md", "research/dapr-agents.md"}) + + def test_summarize_ratings_reports_distribution_correlations_and_quadrants(self): + notes = rating_notes( + [ + (10, 90, 10, 10), + (30, 70, 30, 70), + (50, 50, 50, 50), + (70, 30, 70, 30), + (90, 10, 90, 90), + ] + ) + + summary = summarize_ratings(notes) + + self.assertEqual( + summary["ratings"]["platform-impact"], + { + "count": 5, + "minimum": 10, + "maximum": 90, + "median": 50, + "first_quartile": 30.0, + "third_quartile": 70.0, + "distinct": 5, + "duplicates": 0, + }, + ) + self.assertEqual(summary["correlations"]["platform-impact:maturity"], -1.0) + self.assertEqual(summary["correlations"]["platform-impact:novelty"], 1.0) + self.assertEqual(len(summary["correlations"]), 6) + self.assertEqual( + summary["matrices"]["platform-impact-maturity"], + {"low-low": 1, "low-high": 2, "high-low": 2, "high-high": 0}, + ) + self.assertEqual(len(summary["matrices"]), 6) + + def test_summarize_ratings_reports_duplicate_count(self): + summary = summarize_ratings( + rating_notes([(10, 10, 10, 10), (10, 10, 10, 10), (20, 20, 20, 20)]) + ) + + self.assertEqual(summary["ratings"]["maturity"]["duplicates"], 1) + + def test_validate_summary_checks_all_missing_ratings_before_sparse_distribution(self): + notes = rating_notes([(50, 50, 50, 50)]) + notes[0].metadata["ratings"].pop("novelty") + + summary = summarize_ratings(notes) + + with self.assertRaisesRegex(ValueError, "missing required rating.*novelty"): + validate_summary(summary) + + def test_validate_summary_checks_all_malformed_ratings_before_sparse_distribution(self): + notes = rating_notes([(50, 50, 50, 50), (60, 60, 60, 60)]) + notes[0].metadata["ratings"]["novelty"] = {} + + summary = summarize_ratings(notes) + + with self.assertRaisesRegex(ValueError, "missing required rating.*novelty"): + validate_summary(summary) + + def test_validate_summary_rejects_missing_required_rating(self): + notes = rating_notes([(value, value, value, value) for value in (10, 30, 50, 70, 90)]) + notes[0].metadata["ratings"].pop("novelty") + summary = summarize_ratings(notes) + with self.assertRaisesRegex(ValueError, "missing required rating.*novelty"): + validate_summary(summary) + + def test_validate_summary_rejects_fewer_than_five_distinct_values(self): + summary = summarize_ratings( + rating_notes([(10, 10, 10, 10), (20, 20, 20, 20)] * 3) + ) + with self.assertRaisesRegex(ValueError, "fewer than 5 distinct values"): + validate_summary(summary) + + def test_validate_summary_rejects_values_strictly_on_one_side_of_midpoint(self): + for values, side in ( + ((10, 20, 30, 40, 45), "below"), + ((55, 60, 70, 80, 90), "above"), + ): + with self.subTest(side=side): + summary = summarize_ratings( + rating_notes([(value, value, value, value) for value in values]) + ) + with self.assertRaisesRegex(ValueError, f"strictly {side} 50"): + validate_summary(summary) + + def test_validate_summary_rejects_boundary_without_values_on_both_sides(self): + for values, side in ( + ((10, 20, 30, 40, 50), "below"), + ((50, 60, 70, 80, 90), "above"), + ): + with self.subTest(side=side): + summary = summarize_ratings( + rating_notes([(value, value, value, value) for value in values]) + ) + with self.assertRaisesRegex(ValueError, f"strictly {side} 50"): + validate_summary(summary) + + def test_group_payload_combines_only_exact_positions(self): + payload = [ + {"id": "a", "position": {"x": 50, "y": 40}}, + {"id": "b", "position": {"x": 50, "y": 40}}, + {"id": "c", "position": {"x": 51, "y": 40}}, + ] + groups = group_payload(payload) + self.assertEqual([item["id"] for item in groups[(50, 40)]], ["a", "b"]) + self.assertEqual([item["id"] for item in groups[(51, 40)]], ["c"]) + + def test_group_payload_leaves_unplaced_items_out_of_coordinate_groups(self): + self.assertEqual(group_payload([{"id": "a", "position": None}]), {}) + + def test_parse_note_extracts_metadata_and_research_summary(self): + text = """--- +title: Example +author: A Person +date: 2026-01-01 +ratings: {} +--- + +## Summary + +A concise summary. +""" + note = parse_note(pathlib.Path("research/example.md"), text) + self.assertEqual(note.title, "Example") + self.assertEqual(note.kind, "research") + self.assertEqual(note.summary, "A concise summary.") + + def test_extract_summary_uses_idea_section(self): + self.assertEqual( + extract_summary("ideas/example.md", "## The idea\n\nA useful spark.\n"), + "A useful spark.", + ) + + def test_validate_ratings_rejects_out_of_range_value(self): + with self.assertRaisesRegex(ValueError, "0..100"): + validate_ratings({"maturity": {"value": 101, "note": "reason"}}) + + def test_validate_ratings_requires_note(self): + with self.assertRaisesRegex(ValueError, "non-empty note"): + validate_ratings({"maturity": {"value": 50, "note": ""}}) + + def test_derive_position_maps_x_and_y_from_named_ratings(self): + ratings = { + "maturity": {"value": 55, "note": "reason"}, + "platform-impact": {"value": 80, "note": "reason"}, + } + self.assertEqual(derive_position(ratings, PLOT), {"x": 55, "y": 80}) + + def test_missing_plot_rating_is_unplaced(self): + self.assertIsNone( + derive_position({"maturity": {"value": 55, "note": "reason"}}, PLOT) + ) + + def test_github_url_uses_canonical_repository_path(self): + self.assertEqual( + github_url(pathlib.Path("research/langgraph.md")), + "https://github.com/cloudfoundry/agentic-runtime-notes/blob/main/research/langgraph.md", + ) + + def test_all_current_notes_have_initial_ratings_and_justifications(self): + required = {"platform-impact", "maturity", "novelty", "actionability"} + notes = load_notes() + for note in notes: + ratings = note.metadata["ratings"] + self.assertEqual(set(ratings), required, note.path) + for name in required: + self.assertIsInstance(ratings[name]["value"], int) + self.assertIn(ratings[name]["value"], range(101)) + self.assertTrue(ratings[name]["note"].strip(), note.path) + + def test_generated_html_contains_markers_dialog_and_source_links(self): + html = generate_html(load_notes(), load_plots(), load_primitives()) + self.assertGreater(html.count('class="marker '), 0) + self.assertIn('{applicability.title()}', html) + for path in use_case["core"] + use_case["supporting"]: + self.assertIn(f'href="{github_url(pathlib.Path(path))}"', html) + + def test_generated_html_styles_use_cases_responsively_with_dark_card_language(self): + html = generate_html(load_notes(), load_plots(), load_primitives()) + + self.assertIn(".use-case-grid { display:grid", html) + self.assertIn(".use-case-card { min-width:0; background:#15211e", html) + self.assertIn(".use-case-details summary", html) + self.assertIn("@media(max-width:800px) { .use-case-grid", html) + + def test_generated_html_escapes_all_dynamic_dialog_content(self): + html = generate_html(load_notes(), load_plots(), load_primitives()) + + self.assertIn("function escapeHtml(value)", html) + for expression in ( + "escapeHtml(note.kind)", + "escapeHtml(note.title)", + "escapeHtml(note.summary)", + "escapeHtml(t)", + "escapeHtml(name)", + "escapeHtml(r.value)", + "escapeHtml(r.note)", + "escapeHtml(note.url)", + ): + self.assertIn(expression, html) + + def test_hostile_dynamic_content_is_not_emitted_as_raw_executable_markup(self): + attack = '' + note = SimpleNamespace( + path=pathlib.Path("research/hostile.md"), + kind=attack, + title=attack, + summary=attack, + metadata={ + "tags": [attack], + "ratings": { + "maturity": {"value": 50, "note": attack}, + "platform-impact": {"value": 50, "note": attack}, + }, + }, + ) + primitive = { + **PRIMITIVE, + "id": attack, + "title": attack, + "proposition": attack, + "cf_gap": attack, + "strategic_decision": attack, + "poc": attack, + "rfc_scope": attack, + "core": [note.path.as_posix()], + "supporting": [], + } + + generated = generate_html([note], {"hostile": { + "title": "Hostile", + "x": {"rating": "maturity", "label": "Maturity", "low": "Low", "high": "High"}, + "y": {"rating": "platform-impact", "label": "Impact", "low": "Low", "high": "High"}, + }}, [primitive], focus_use_cases({ + **FOCUS_USE_CASE, + "id": attack, + "title": attack, + "workshop_outcome": attack, + "primary_actor": attack, + "beneficiary": attack, + "lifecycle": attack, + "authority_boundary": attack, + "unique_capabilities": [attack], + "failure_domain": attack, + "poc": attack, + "rfc_decisions": [attack], + "core": [note.path.as_posix()], + "supporting": [], + "primitive_applicability": {attack: attack}, + })) + + static_markup, script = generated.split("