Skip to content
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"adlc-skills-cli": "bin/adlc-skills-cli.mjs"
},
"scripts": {
"test": "node --test && node tests/e2e-node-gate.mjs --skip-agent",
"test": "node --test && node tests/e2e-node-gate.mjs --skip-agent && node tests/e2e-cross-session.mjs",
"test:evals": "node --test evals/opencode-session-end/tests/test_check_session_end.mjs",
"test:all": "npm test && npm run test:evals",
"prepublishOnly": "npm test",
Expand Down
30 changes: 28 additions & 2 deletions src/workflow/engine.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ import { RunStatus, StepStatus, StepContext, typeName, repr } from "./base.mjs";
import { evaluateCondition } from "./expressions.mjs";
import { readAgent } from "../utils/init-options.mjs";
import { Lease } from "./lease.mjs";
import { pushRunRef } from "./refs.mjs";
import { syncRunRef } from "./refs.mjs";
import { initTrace, appendTraceEvent, finalizeTrace } from "./trace.mjs";
import { currentHeadSha, hasContextGate } from "./gate-context.mjs";
import {
loadBundledCatalog,
Expand Down Expand Up @@ -526,6 +527,11 @@ export class RunState {
} finally {
closeSync(fd);
}
// Trace bundle v1 (ADR-490): mirror the lifecycle entry into the
// ADR-362 stream. Fail-soft — telemetry never breaks a run.
try {
appendTraceEvent(this.runsDir, entry);
} catch {}
});
}

Expand Down Expand Up @@ -750,6 +756,11 @@ export class WorkflowEngine {
state.runStartSha = currentHeadSha(String(this.projectRoot));
}
await state.save();
// Trace bundle v1 (ADR-490): identity + lifecycle stream from the start.
try {
initTrace(state.runsDir, { state, projectRoot: this.projectRoot, definition });
appendTraceEvent(state.runsDir, { event: "run_started", workflow_id: definition.id });
} catch {}
this.emit("run_started", { run_id: state.runId, workflow_id: definition.id });

const context = new StepContext({
Expand Down Expand Up @@ -803,7 +814,14 @@ export class WorkflowEngine {
this.emit("run_failed", { run_id: state.runId, error: state.error, status: state.status });
}
try {
pushRunRef(this.projectRoot, state.runId, state.runsDir);
// Trace: close the bundle (retention) before the ref is pushed, so a
// pruned successful trace never reaches the remote.
finalizeTrace(state.runsDir, state);
} catch {}
try {
// Exit path (pause/terminal): push unconditionally so another machine
// can restore the run even if this pod is torn down immediately.
syncRunRef(this.projectRoot, state.runId, state.runsDir, { minIntervalSeconds: 0 });
} catch {}
if (lease) lease.release();
}
Expand Down Expand Up @@ -871,6 +889,11 @@ export class WorkflowEngine {
state.error = null;
state.status = RunStatus.RUNNING;
await state.save();
// Trace: resume keeps the original manifest identity and appends a marker.
try {
initTrace(state.runsDir, { state, projectRoot: this.projectRoot, definition });
appendTraceEvent(state.runsDir, { event: "run_resumed", workflow_id: definition.id });
} catch {}
this.emit("run_started", { run_id: state.runId, workflow_id: definition.id, resumed: true });

// Resume from the current step — re-execute it so gates can prompt again.
Expand Down Expand Up @@ -921,6 +944,9 @@ export class WorkflowEngine {
state.currentStepIndex = stepOffset + i;
}
await state.save();
// Step-boundary durability: throttled remote sync bounds the crash
// window to ADLC_WORKFLOW_PUSH_INTERVAL (ADR-475).
syncRunRef(this.projectRoot, state.runId, state.runsDir);
if (lease) lease.renew();
this.emit("step_started", { run_id: state.runId, step_id: stepId, step_type: stepType });

Expand Down
35 changes: 35 additions & 0 deletions src/workflow/redact.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
// Shared secret redaction for text bound for logs, notifiers, persisted run
// state, or the pushed run ref (ADR-478). Extracted from steps/shell.mjs so
// the refs transport can redact without importing a step implementation.

// Secret-shaped env var names whose values never appear in outbound text.
const SECRET_ENV_NAME = /TOKEN|SECRET|PASSWORD|PASSWD|API_?KEY|PRIVATE_?KEY|CREDENTIAL|JWT/i;
// Well-known token formats (GitLab, GitHub, Slack, Anthropic/OpenAI-style keys, JWTs).
const TOKEN_PATTERNS = [
/\bglpat-[A-Za-z0-9_-]{10,}/g,
/\bgl(?:dt|rt|cbt|ptt|ft|ffct|imt|oas|agent|soat)-[A-Za-z0-9_-]{10,}/g,
/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}/g,
/\bgithub_pat_[A-Za-z0-9_]{20,}/g,
/\bxox[abprs]-[A-Za-z0-9-]{10,}/g,
/\bsk-[A-Za-z0-9_-]{20,}/g,
/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g,
];

/** Mask credentials in text bound for logs, notifiers or persisted run state. */
export function redactSecrets(text, env = process.env) {
let out = text
// userinfo in URLs: https://user:token@host → https://***@host
.replace(/([a-z][a-z0-9+.-]*:\/\/)[^\s/@]+@/gi, "$1***@")
// Authorization-style headers and key=value pairs: everything after the
// separator to the end of the line is masked, keeping only a known auth
// scheme. Multi-parameter schemes (AWS SigV4, Digest) carry credentials
// past the first word, and over-masking an error message costs nothing.
// A value on the next line (YAML, folded header) is masked too.
.replace(/\b(authorization|private-token|x-api-key)([ \t]*[:=](?:[ \t]*\r?\n)?[ \t]*)((?:bearer|basic|token|bot|digest|negotiate|apikey|key)[ \t]+)?[^\s][^\r\n]*/gi, "$1$2$3***");
for (const re of TOKEN_PATTERNS) out = out.replace(re, "***");
for (const [name, value] of Object.entries(env)) {
if (!value || value.length < 8 || !SECRET_ENV_NAME.test(name)) continue;
out = out.split(value).join("***");
}
return out;
}
209 changes: 204 additions & 5 deletions src/workflow/refs.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,49 @@
// main history stays untouched. Called at session exit points (pause and
// terminal states); best-effort: silently skipped outside a git repo.

import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync, copyFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { dirname, join, resolve, sep } from "node:path";
import { spawnSync } from "node:child_process";
import { err } from "../ui/index.mjs";
import { redactSecrets } from "./redact.mjs";

// Machine-local run-dir files never cross machines (ADR-477):
// lease.json — liveness claim with hostname/pid/heartbeat
// .push-ts — local throttle stamp for remote run-ref sync
const EXCLUDED_RUN_FILES = new Set(["lease.json", ".push-ts"]);

// Push throttle for state transitions (ADR-475 crash-window bound). The
// stamp lives inside the run dir and is excluded from the pushed tree.
const PUSH_STAMP = ".push-ts";
const DEFAULT_PUSH_INTERVAL = 60;

// ADR-490 retention is decided only at finalize. While the run is still
// executing (state.json status "running") its event stream must not reach
// the remote — a run that later completes would otherwise have already
// uploaded the very events.jsonl blobs that finalize prunes (the guarantee
// would be tip-only). Mid-run syncs stage the run dir without this file.
// Paused runs ship it (cross-session restore/resume needs the stream to
// continue where it left off — park, AC-T6) and terminal statuses ship the
// tree after finalizeTrace applied the retention class.
const MIDRUN_EXCLUDED_FILES = ["trace/events.jsonl"];

function pushIntervalSeconds() {
const raw = process.env.ADLC_WORKFLOW_PUSH_INTERVAL;
if (raw === undefined || raw === "") return DEFAULT_PUSH_INTERVAL;
const n = Number(raw);
return Number.isFinite(n) && n >= 0 ? n : DEFAULT_PUSH_INTERVAL;
}

// Status recorded in the run dir's state.json, or null when unreadable —
// never throws, the sync stays fail-soft.
function runStatusOf(runDir) {
try {
return JSON.parse(readFileSync(join(runDir, "state.json"), "utf-8"))?.status ?? null;
} catch {
return null;
}
}

export function pushRunRef(projectRoot, runId, runDir) {
const git = (args, input) =>
Expand All @@ -31,8 +70,14 @@ export function pushRunRef(projectRoot, runId, runDir) {
const hashDir = (absDir) => {
const entries = [];
for (const name of readdirSync(absDir).sort()) {
if (EXCLUDED_RUN_FILES.has(name)) continue;
const abs = join(absDir, name);
if (statSync(abs).isDirectory()) {
// lstat, never stat: a symlink (file or directory) is skipped, not
// followed — a run dir must never ship or hash out-of-tree content,
// and a dangling symlink must not fail the walk (ADR-478).
const st = lstatSync(abs);
if (st.isSymbolicLink()) continue;
if (st.isDirectory()) {
entries.push(`040000 tree ${hashDir(abs)}\t${name}`);
} else {
const h = git(["hash-object", "-w", abs]);
Expand All @@ -59,16 +104,165 @@ export function pushRunRef(projectRoot, runId, runDir) {
}
}

// Redact secrets in every UTF-8 text file of the run dir, in place, before it
// is hashed into the pushed ref (ADR-478 redact-on-park). Binary/non-UTF-8
// files are left untouched. Returns the number of files rewritten.
export function redactRunDir(runDir, env = process.env) {
const decoder = new TextDecoder("utf-8", { fatal: true });
let rewritten = 0;
const walk = (absDir) => {
for (const name of readdirSync(absDir)) {
const abs = join(absDir, name);
// lstat, never stat: redaction must not write through a symlink into
// files outside the run dir, nor recurse into symlinked directories;
// a dangling symlink must not fail the walk (ADR-478).
const st = lstatSync(abs);
if (st.isSymbolicLink()) continue;
if (st.isDirectory()) {
walk(abs);
continue;
}
if (EXCLUDED_RUN_FILES.has(name)) continue;
const buf = readFileSync(abs);
let text;
try {
text = decoder.decode(buf);
} catch {
continue; // not UTF-8 text
}
const redacted = redactSecrets(text, env);
if (redacted !== text) {
writeFileSync(abs, redacted, "utf-8");
rewritten += 1;
}
}
};
walk(runDir);
return rewritten;
}

// Stage a throwaway copy of the run dir for redaction: the pushed tree is
// built from the copy, never from the live run dir. Machine-local files
// (EXCLUDED_RUN_FILES, ADR-477) and symlinks (ADR-478, no traversal) are
// left out of the copy entirely. Returns { dir, cleanup } or null when the
// copy itself fails — callers treat null as "skip this push" (fail-closed,
// fail-soft).
function stageRunDir(runDir) {
let staged;
try {
staged = mkdtempSync(join(tmpdir(), "adlc-runref-"));
const copy = (absSrc, absDst) => {
for (const name of readdirSync(absSrc)) {
if (EXCLUDED_RUN_FILES.has(name)) continue;
const src = join(absSrc, name);
const dst = join(absDst, name);
// lstat, never stat: symlinks are skipped, not copied or followed.
const st = lstatSync(src);
if (st.isSymbolicLink()) continue;
if (st.isDirectory()) {
mkdirSync(dst, { recursive: true });
copy(src, dst);
} else {
copyFileSync(src, dst);
}
}
};
copy(runDir, staged);
} catch {
if (staged) {
try { rmSync(staged, { recursive: true, force: true }); } catch {}
}
return null;
}
return {
dir: staged,
cleanup() {
try { rmSync(staged, { recursive: true, force: true }); } catch {}
},
};
}

// syncRunRef — record the run dir as refs/factory-runs/<runId> locally, then
// force-push that ref to the remote so another machine can restore it
// (ADR-475). The remote push is throttled by ADLC_WORKFLOW_PUSH_INTERVAL
// seconds (default 60; 0 = every transition) and is fail-soft: an offline
// pod never fails the run, and the local ref stays valid for the next sync.
export function syncRunRef(projectRoot, runId, runDir, { remote = "origin", minIntervalSeconds = pushIntervalSeconds() } = {}) {
const ref = `refs/factory-runs/${runId}`;
const stamp = join(runDir, PUSH_STAMP);
if (minIntervalSeconds > 0 && existsSync(stamp)) {
// Unreadable stamp (EACCES, TOCTOU removal) counts as absent: proceed
// to push, never fail the transition (ADR-497 fail-soft).
let last = NaN;
try {
last = Number(readFileSync(stamp, "utf-8").trim());
} catch {
last = NaN;
}
if (Number.isFinite(last) && Date.now() - last < minIntervalSeconds * 1000) {
return true;
}
}
// Every sync is redacted, but never in place: the live run dir must stay
// byte-identical across a transition, because resume() reads inputs.json
// (and later steps read their original values) from it — masking there
// would silently substitute "***" into a resumed run (ADR-497: "no run
// file edited in place except redaction at park"). The pushed tree is
// built from a throwaway staging copy instead: copy, redact the copy,
// hash+push the copy, discard it. Machine-local files (ADR-477) never
// enter the staging copy at all. Fail-closed: if redaction of the staged
// copy fails, the push is skipped entirely — the wire never receives
// unredacted content — but the transition itself stays fail-soft.
const staged = stageRunDir(runDir);
if (staged === null) return false;
try {
// ADR-490: while the run executes (status "running"), the retention
// class is still pending — the event stream stays off the wire until
// finalize decides it (paused/terminal pushes below ship the tree per
// retention class). Only the staged copy is trimmed; the live run dir
// keeps recording events.
if (runStatusOf(runDir) === "running") {
for (const rel of MIDRUN_EXCLUDED_FILES) {
rmSync(join(staged.dir, ...rel.split("/")), { force: true });
}
}
try {
redactRunDir(staged.dir);
} catch (exc) {
err(`[workflow] run-ref redaction failed for ${runId}; push skipped: ${exc?.message ?? exc}`);
return true;
}
if (!pushRunRef(projectRoot, runId, staged.dir)) return false;
} finally {
staged.cleanup();
}
const pushed = spawnSync("git", ["push", "--quiet", remote, `+${ref}:${ref}`], {
cwd: projectRoot,
encoding: "utf-8",
maxBuffer: 16 * 1024 * 1024,
timeout: 30_000,
});
if (pushed.error || pushed.status !== 0) {
const detail = pushed.error ? pushed.error.message : String(pushed.stderr ?? "").trim();
err(`[workflow] run-ref remote sync skipped: ${detail}`);
return true;
}
try {
writeFileSync(stamp, String(Date.now()), "utf-8");
} catch {}
return true;
}

// Park and restore (ADR-391-amendment-3 §2–3): a run paused at a gate past
// its in-pod wait is saved — with the gate's Slack thread handle — to
// refs/factory-runs/<run_id> on a git remote, so a later pod in another clone
// can restore runs/<run_id>/ and resume it. Both throw on any failure: unlike
// the best-effort pushRunRef, a park or restore that silently did nothing
// would lose the approver's answer or resume the wrong run.
//
// pushRunRef hashes the whole runDir, inputs.json included: whatever was
// passed via --input lands on the remote verbatim. A secret-valued input
// must never be parked to a broadly-readable remote.
// pushRunRef hashes the whole runDir, inputs.json included, after
// redactRunDir masks secret-shaped values: whatever was passed via --input
// still lands on the remote, but a secret-valued input is masked first.

const RUN_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;

Expand All @@ -84,6 +278,11 @@ export function parkRun(projectRoot, runId, { handleFile, remote = "origin" } =
throw new Error("park: the handle file must hold a JSON object");
}
writeFileSync(join(runDir, "gate-handle.json"), JSON.stringify(handle, null, 2) + "\n", "utf-8");
// ADR-478: redact every text file in place before it is hashed into the
// ref, so a secret that leaked into state/logs/inputs never reaches the
// remote. Machine-local files (lease.json, .push-ts) are excluded from the
// tree by pushRunRef (ADR-477).
redactRunDir(runDir);
if (!pushRunRef(projectRoot, runId, runDir)) throw new Error(`park: could not record refs/factory-runs/${runId}`);
const ref = `refs/factory-runs/${runId}`;
// The run ref is rebuilt from scratch at every save, so it never descends
Expand Down
Loading
Loading