Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 19 additions & 1 deletion docs/AUTHORING-HOST-ADAPTERS.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,14 @@ Field by field:
| `driving.surfaces` | Declare `cli-subprocess`. See below. |
| `lifecycle` / `execution` | Your hooks ([section 3](#3-write-the-hooks)). Both are optional; a manifest with neither is a pure description. |

> **Capabilities describe what the adapter *delivers through `ak`*, not what your host can do in
> principle.** A real Hermes adapter's first draft declared `nativeMcpConfig: true` and
> `nativeGuidance: true` — both true of Hermes itself (`hermes mcp add`, reading `AGENTS.md` from
> cwd) but false of the adapter, which shipped no `apply`/`undo` hooks to actually *wire* either.
> If your manifest has no hooks for a capability, declare it `false` even when the underlying CLI
> supports it — the example manifest above is right (all three `false`) precisely because it has
> no `lifecycle.apply`/`lifecycle.undo` hooks yet.

### Driving surfaces

The vocabulary has three names — `cli-subprocess`, `acp`, `mcp` — but **`cli-subprocess` is the only
Expand Down Expand Up @@ -160,7 +168,10 @@ This is the one that actually drives your host as a worker under `ak run`.
- **stdin** carries the worker prompt.
- **the environment** carries `AK_WORKER_ID`, `AK_WORKER_ACTIVITY`, `AK_WORKER_ROLE`,
`AK_WORKER_MODEL`, and `AK_WORKER_CWD` (the repository being worked on — your hook does *not*
spawn there, see below).
spawn there, see below). **`AK_WORKER_CWD` is advisory, not a sandbox boundary** — `ak` tells
your hook which directory to work in, but nothing confines an auto-approving host to it once your
hook hands control to it. If your host has no permission event to intercept (many local-model
CLIs don't), it is your hook's job to make that boundary real.
- **stdout** carries either a JSON object — `{summary, observedModel, provider, usage}`, all
optional — or plain text, which is taken as the summary.
- **the exit code** is the sole authority for success.
Expand Down Expand Up @@ -277,6 +288,13 @@ prints an honest per-tier verdict. It warns first, listing every hook command it
a real subprocess. Here is a real run against the repository's conformance fixture
(an adapter shaped exactly like the manifest in section 2):

`activity-routing` and `primary-eligible` drive a genuine worker through your `execution.run` hook,
so two things are handled for you: the outer time budget honors your manifest's own declared
`execution.run.hook.timeoutMs` automatically (pass `--timeout <ms>` to override it), and the worker's
cwd is a throwaway scratch directory, never wherever you happened to run the command from — an
auto-approving agentic host has no business landing in your real working directory during a
self-test.

```text
host adapter conformance — acme (56fa107674d2)
admission passed host id 'acme', contract 1
Expand Down
2 changes: 1 addition & 1 deletion src/commands/run.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ Examples:
ak run feature "fix the flaky parser" --escalate`;

/** @param {string|undefined} value @param {string} name @param {{ ceiling?: number }} [opts] */
function positiveInt(value, name, { ceiling } = {}) {
export function positiveInt(value, name, { ceiling } = {}) {
if (value === undefined) return undefined;
if (!/^\d+$/.test(value) || Number(value) < 1) throw new TypeError(`${name} must be a positive integer`);
const n = Number(value);
Expand Down
14 changes: 12 additions & 2 deletions src/commands/x/host-adapters.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
// raw file — an invalid manifest (e.g. one claiming canBePrimary) is refused
// with its .reason and nothing is ever recorded for it.
import readline from 'node:readline/promises';
import { positiveInt } from '../run.mjs';
import { hashManifest, SUPPORTED_CONTRACT } from '../../lib/adapters/admission.mjs';
import { validateAdapterManifest } from '../../lib/adapters/manifest.mjs';
import { HOST_REGISTRY } from '../../lib/adapters/registries.mjs';
Expand Down Expand Up @@ -350,12 +351,20 @@ async function warnAboutHooks(name, entry, rawReader) {
* recording semantics (passed -> recordTierResult, upstream-gated ->
* recordTierGate, everything else persists nothing). */
async function conformance({
name, cfg, reader, runTiered, consentFile, grantsFile,
name, cfg, reader, runTiered, consentFile, grantsFile, flags = /** @type {{timeout?:string}} */ ({}),
}) {
if (typeof name !== 'string' || !name) { fail('usage: ak host adapters conformance <name>'); return 2; }
const entry = findEntry(cfg, name);
if (!entry) { fail(`no host adapter named '${name}' in kit.json hostAdapters`); return 1; }

let timeoutMs;
try {
timeoutMs = positiveInt(flags.timeout, 'timeout', { ceiling: 2_147_483_647 });
} catch (error) {
fail(error.message);
return 2;
}

const rawReader = toRawManifestReader(reader);
await warnAboutHooks(name, entry, rawReader);

Expand All @@ -367,6 +376,7 @@ async function conformance({
readManifest: rawReader,
consentFile,
grantsFile,
timeoutMs,
});
} catch (error) {
fail(`'${name}' conformance run failed: ${stripControl(error?.message ?? String(error))}`);
Expand Down Expand Up @@ -426,7 +436,7 @@ export async function run({
}
if (sub === 'conformance') {
return conformance({
name, cfg: resolvedCfg, reader, runTiered: runTieredConformance, consentFile, grantsFile,
name, cfg: resolvedCfg, reader, runTiered: runTieredConformance, consentFile, grantsFile, flags,
});
}
// F-8 (security review, ADR-0031-accurate naming): `bless` is the alias —
Expand Down
5 changes: 5 additions & 0 deletions src/commands/x/host.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ export const options = {
route: { type: 'string', multiple: true }, // repeatable: 'activity:host[:model]' per-activity routing override
activity: { type: 'string' }, // refresh: csv of activities to re-seed (default = prompt)
'expect-hash': { type: 'string' }, // adapters trust: required sha256 pin when --yes resolves a non-file source
timeout: { type: 'string' }, // adapters conformance: outer ms budget override (default: manifest's own execution.run.hook.timeoutMs, else 120000)
yes: { type: 'boolean', default: false },
json: { type: 'boolean', default: false },
};
Expand Down Expand Up @@ -88,6 +89,10 @@ Subcommands:
list show each configured adapter's trust state (default)
trust <name> [--expect-hash <sha256>] grant consent (required
with --yes against a non-file source); revoke <name>
conformance <name> [--timeout <ms>] run the tiered black-box
harness; --timeout overrides the outer per-worker
budget (default: the manifest's own declared
execution.run.hook.timeoutMs, else 120000)

Options (pick, all optional — omit for interactive):
--host <csv> the complete desired enabled-host set, e.g.
Expand Down
65 changes: 55 additions & 10 deletions src/lib/adapters/conformance.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,13 @@ export { CONFORMANCE_TIERS, TIER_GRANTS };

const nowIso = () => new Date().toISOString();

// A real adapter drives a real, often agentic, host (Wave adrianco#131: an
// open-ended probe like "conformance harness probe" reads as a TASK to an
// agentic CLI, which then goes and builds one — minutes of real work instead
// of a transport check). Directive and bounded so every tier's exercise
// tests wiring, not the model's ambition.
const CONFORMANCE_PROBE_PROMPT = 'Reply with exactly: OK';

/** admitAdapters' readManifest contract: `(source) => Promise<any>` — a real
* resolve, matching admission.mjs's own default (dynamic import so this
* module's own load stays cheap for callers that always inject a reader). */
Expand Down Expand Up @@ -201,7 +208,7 @@ function checkSessionDriving({ manifest, upstreamRef }) {
// real, so it is the one tier expected to genuinely PASS against a conforming
// fixture today.
async function checkActivityRouting({
manifest, name, baseDir, haveFn, clock,
manifest, name, baseDir, haveFn, clock, cwd, timeoutMs,
}) {
if (!manifest) {
return { status: 'skipped', checks: [{ name: 'admission prerequisite', ok: false, detail: 'admission tier did not pass — cannot evaluate' }] };
Expand Down Expand Up @@ -231,10 +238,10 @@ async function checkActivityRouting({
// subprocess, not a stub.
const plan = {
workers: [{
id: 'conformance-w1', activity: 'implementation', role: 'coder', host: name, prompt: 'conformance harness probe',
id: 'conformance-w1', activity: 'implementation', role: 'coder', host: name, prompt: CONFORMANCE_PROBE_PROMPT,
}],
};
const [result] = await executeRunPlan(plan, { clock });
const [result] = await executeRunPlan(plan, { clock, cwd, timeoutMs });
if (result.status !== 'succeeded') throw new Error(result.failure?.reason ?? `expected a succeeded WorkerResult, got '${result.status}'`);
succeeded = result;
return `worker '${result.workerId}' succeeded via host '${result.host}' (exitCategory=${result.exitCategory})`;
Expand Down Expand Up @@ -311,7 +318,7 @@ async function checkActivityRouting({
const PRIMARY_ELIGIBLE_UNROUTED_HOST_SUFFIX = 'conformance-unrouted-rung';

async function runPrimaryEligibleExercise({
manifest, name, baseDir, haveFn, clock,
manifest, name, baseDir, haveFn, clock, cwd, timeoutMs,
}) {
const unroutedHost = `${PRIMARY_ELIGIBLE_UNROUTED_HOST_SUFFIX}-${name}`;
resetAdmittedExecution();
Expand All @@ -321,16 +328,18 @@ async function runPrimaryEligibleExercise({
workers: [
{
id: 'primary-eligible-direct', activity: 'implementation', role: 'coder', host: name,
prompt: 'conformance harness probe: primary-eligible direct run (non-escalated)',
prompt: CONFORMANCE_PROBE_PROMPT,
},
{
id: 'primary-eligible-escalation', activity: 'implementation', role: 'coder', host: unroutedHost,
prompt: 'conformance harness probe: primary-eligible escalation (receives an escalation)',
prompt: CONFORMANCE_PROBE_PROMPT,
escalate: [{ host: name, model: null }],
},
],
};
const [direct, escalation] = await executeRunPlan(plan, { clock, escalate: true });
const [direct, escalation] = await executeRunPlan(plan, {
clock, escalate: true, cwd, timeoutMs,
});

if (direct?.status !== 'succeeded' || direct.host !== name) {
return { ok: false, detail: `direct (non-escalated) worker did not complete via '${name}' (status=${direct?.status})` };
Expand Down Expand Up @@ -373,11 +382,11 @@ async function runPrimaryEligibleExercise({
* lets a maintainer's later grantCapability succeed at all.
*/
async function checkPrimaryEligible({
manifest, name, baseDir, haveFn, clock,
manifest, name, baseDir, haveFn, clock, cwd, timeoutMs,
}) {
const exerciseLabel = 'leads a run and receives an escalation (ADR-0019)';
const outcome = await runPrimaryEligibleExercise({
manifest, name, baseDir, haveFn, clock,
manifest, name, baseDir, haveFn, clock, cwd, timeoutMs,
});
if (outcome.ok) {
return { status: 'passed', checks: [{ name: exerciseLabel, ok: true, detail: outcome.detail }], evidence: outcome.detail };
Expand Down Expand Up @@ -455,6 +464,18 @@ async function checkGrantGatedTier({
* conformance run to touch the real store (tests, dry runs) must pass an
* explicit path. Pass `persist: false` to skip recording altogether.
*
* activity-routing and primary-eligible drive a REAL worker through the
* adapter's execution.run hook (adrianco#131): `timeoutMs` explicitly
* overrides the outer budget; omitted, the manifest's own declared
* `execution.run.hook.timeoutMs` is honored instead of the runner's 120s
* default, so a manifest that already declares a longer budget just works
* (the hook-level `resolveTimeout` still takes the tighter of the two, so
* this can only raise the ceiling, never bypass a SHORTER hook-declared one).
* `cwd` defaults to a throwaway temp directory (cleaned up on return) rather
* than the operator's `process.cwd()` — an auto-approving agentic worker has
* no business landing in whatever directory the operator happened to run
* `ak host adapters conformance` from.
*
* primary-eligible has no caller-injectable exercise (unlike statusline,
* still a placeholder this wave): it always runs runPrimaryEligibleExercise,
* a real escalation-plus-direct-run probe driven through executeRunPlan
Expand All @@ -475,7 +496,7 @@ async function checkGrantGatedTier({
* haveFn?: (cmd: string, opts?: any) => Promise<boolean>, baseDir?: string|null,
* sessionDrivingUpstreamRef?: string,
* exerciseStatusline?: (ctx: {manifest:any,name:string,hash:string}) => Promise<{ok:boolean,detail?:string}>,
* clock?: () => string,
* clock?: () => string, timeoutMs?: number, cwd?: string,
* }} [options]
* @returns {Promise<{ name: string, hash: string|null,
* tiers: Array<{ tier: string, status: 'passed'|'failed'|'gated'|'skipped',
Expand All @@ -496,6 +517,8 @@ export async function runTieredConformance({
sessionDrivingUpstreamRef,
exerciseStatusline,
clock = nowIso,
timeoutMs,
cwd,
} = {}) {
if (typeof manifestSource !== 'string' || !manifestSource) {
throw new TypeError('runTieredConformance requires fixtureRoot or manifestSource');
Expand All @@ -508,6 +531,16 @@ export async function runTieredConformance({
consentFileUsed = path.join(tempDir, 'adapter-consent.json');
}

// adrianco#131 #2: a live, often auto-approving worker must never land in
// the operator's own $PWD by accident — a throwaway scratch directory,
// never process.cwd(), unless a caller (tests) supplies its own.
let workerCwdTempDir = null;
let workerCwd = cwd;
if (!workerCwd) {
workerCwdTempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ak-adapter-conformance-cwd-'));
workerCwd = workerCwdTempDir;
}

try {
// F1 (Wave C): baseDir is computed BEFORE checkAdmission so it can thread
// straight into registerAdmittedLifecycle for the detect-hook check —
Expand All @@ -533,6 +566,15 @@ export async function runTieredConformance({
const effectiveManifest = admissionPassed ? admission.manifest : null;
const wantTier = (tier) => tiers.includes(tier);

// adrianco#131 #1: an explicit override always wins; otherwise honor the
// manifest's own declared execution.run.hook.timeoutMs (if any) as the
// OUTER runner budget too, so a manifest that already declares a longer
// one (e.g. a local-model host) isn't silently capped at the runner's
// 120s default before the hook's own tighter-of-the-two ever applies.
const effectiveTimeoutMs = timeoutMs !== undefined
? timeoutMs
: effectiveManifest?.execution?.run?.hook?.timeoutMs;

const tierResults = [];

if (wantTier('admission')) {
Expand All @@ -557,6 +599,7 @@ export async function runTieredConformance({
if (wantTier('activity-routing') || wantTier('primary-eligible')) {
activityRoutingResult = await checkActivityRouting({
manifest: effectiveManifest, name: resolvedName, baseDir: derivedBaseDir, haveFn, clock,
cwd: workerCwd, timeoutMs: effectiveTimeoutMs,
});
if (wantTier('activity-routing')) {
tierResults.push({ tier: 'activity-routing', ...activityRoutingResult });
Expand Down Expand Up @@ -586,6 +629,7 @@ export async function runTieredConformance({
} else {
result = await checkPrimaryEligible({
manifest: effectiveManifest, name: resolvedName, baseDir: derivedBaseDir, haveFn, clock,
cwd: workerCwd, timeoutMs: effectiveTimeoutMs,
});
}
tierResults.push({ tier: 'primary-eligible', ...result });
Expand Down Expand Up @@ -648,5 +692,6 @@ export async function runTieredConformance({
resetAdmittedExecution();
resetAdmittedLifecycle();
if (tempDir) { try { fs.rmSync(tempDir, { recursive: true, force: true }); } catch { /* best-effort cleanup */ } }
if (workerCwdTempDir) { try { fs.rmSync(workerCwdTempDir, { recursive: true, force: true }); } catch { /* best-effort cleanup */ } }
}
}
Loading
Loading