From 19f43d0b52e6f2aab099f6fb65c400e7b302d832 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Jul 2026 07:11:34 +0000 Subject: [PATCH 1/4] Initial plan From 48b6fcafe8a19cc507da17f04c629ac8ff13eaa4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Jul 2026 08:58:45 +0000 Subject: [PATCH 2/4] feat(harness): distinguish long-run vs startup Copilot CLI failures via token-count classification Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/copilot_harness.cjs | 33 +++++++- actions/setup/js/copilot_harness.test.cjs | 97 +++++++++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index ff555713a3a..484bfbc7952 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -81,6 +81,10 @@ const OUTPUT_TAIL_MAX_CHARS = 600; const OUTPUT_TAIL_MAX_LINES = 12; const MIN_POST_RESULT_WATCHDOG_TIMEOUT_MS = 50; const DEFAULT_POST_RESULT_WATCHDOG_IDLE_TIMEOUT_MS = 20 * 1000; +// Token count threshold above which a 0-turn failure is classified as "long_run_exit" +// rather than the generic "partial_execution". Corresponds to ~30+ minutes of Copilot +// CLI work where the wrapper exits non-zero after the agent has completed substantial work. +const LONG_RUN_TOKEN_THRESHOLD = 10000; function resolvePostResultWatchdogIdleTimeoutMs(env = process.env) { const configuredTimeoutMs = Number(env.GH_AW_HARNESS_WATCHDOG_TIMEOUT_MS); if (!Number.isFinite(configuredTimeoutMs) || configuredTimeoutMs <= 0) { @@ -476,6 +480,24 @@ function extractOutputTail(output, options) { return tail; } +/** + * Sum all `"total_tokens"` values emitted in debug JSON response blocks in CLI output. + * The Copilot CLI emits JSON blocks containing token usage data in its stdout/stderr. + * Returns 0 when no token data is present (e.g., startup failures that produced no output). + * @param {string} output - Combined stdout/stderr from the Copilot CLI + * @returns {number} + */ +function extractTokenCountFromOutput(output) { + if (!output) return 0; + let total = 0; + const pattern = /"total_tokens"\s*:\s*(\d+)/g; + let match; + while ((match = pattern.exec(output)) !== null) { + total += parseInt(match[1], 10); + } + return total; +} + /** * Classify a failed Copilot attempt into a short, named failure class. * @param {{ @@ -492,6 +514,7 @@ function extractOutputTail(output, options) { * isQuotaExceeded?: boolean, * isSDKSessionIdleTimeout?: boolean, * hasNumerousPermissionDenied?: boolean, + * tokenCount?: number, * }} detection * @returns {string} */ @@ -508,6 +531,7 @@ function classifyCopilotFailure(detection) { if (detection.isMCPGatewayShutdown) return "mcp_gateway_shutdown"; if (detection.hasNumerousPermissionDenied) return "permission_denied"; if (detection.isTransientCAPIError) return "capi_error_400"; + if (detection.hasOutput && (detection.tokenCount ?? 0) > LONG_RUN_TOKEN_THRESHOLD) return "long_run_exit"; return detection.hasOutput ? "partial_execution" : "no_output"; } @@ -1071,6 +1095,7 @@ async function main() { // Redact --prompt / -p value from logs to avoid leaking prompt content const safeArgs = currentArgs.map((arg, i) => (currentArgs[i - 1] === "--prompt" || currentArgs[i - 1] === "-p" ? "" : arg)); + const attemptStartTime = Date.now(); // Driver mode: run copilot_sdk_driver.cjs as a normal subprocess. The harness has // already started the sidecar; the driver only opens an SDK client connection. const result = await runProcess({ @@ -1130,6 +1155,8 @@ async function main() { const hasNumerousPermissionDenied = hasNumerousPermissionDeniedIssues(result.output); const nonRetryableGuard = detectNonRetryableHarnessGuard(result.output); const isInvocationCapExceeded = nonRetryableGuard.maxRunsExceeded; + const tokenCount = extractTokenCountFromOutput(result.output); + const attemptDurationMs = Date.now() - attemptStartTime; const failureClass = classifyCopilotFailure({ hasOutput: result.hasOutput, isAuthErr, @@ -1144,6 +1171,7 @@ async function main() { isQuotaExceeded, isSDKSessionIdleTimeout, hasNumerousPermissionDenied, + tokenCount, }); const outputTail = extractOutputTail(result.output); log( @@ -1164,6 +1192,8 @@ async function main() { ` permissionDeniedCount=${permissionDeniedCount}` + ` hasNumerousPermissionDenied=${hasNumerousPermissionDenied}` + ` hasOutput=${result.hasOutput}` + + ` tokenCount=${tokenCount}` + + ` attemptDurationMs=${attemptDurationMs}` + ` retriesRemaining=${maxRetries - attempt}` ); if (outputTail) { @@ -1186,7 +1216,7 @@ async function main() { // (watchdogFired=true), as well as any other partial_execution failure that occurs // after the primary task output was already produced. Retrying would reproduce the // same pattern and exhaust the retry budget without ever posting a final safe-output. - if (failureClass === "partial_execution" && safeOutputsPath && hasTerminalSafeOutput(safeOutputsPath)) { + if ((failureClass === "partial_execution" || failureClass === "long_run_exit") && safeOutputsPath && hasTerminalSafeOutput(safeOutputsPath)) { const reason = result.watchdogFired ? "post-result watchdog fired after terminal safe-output was emitted" : "partial execution after terminal safe-output was already produced"; log(`attempt ${attempt + 1}: ${reason} — treating as success (late-activity exit suppressed)`); lastExitCode = 0; @@ -1397,6 +1427,7 @@ if (typeof module !== "undefined" && module.exports) { countPermissionDeniedIssues, detectCopilotErrors, classifyCopilotFailure, + extractTokenCountFromOutput, shouldRetryFailedExecution, extractOutputTail, isRetryableProxyAuthenticationFailure, diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index 4a89c7fe71c..57f857bd6a8 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -14,6 +14,7 @@ const { appendSafeOutputLine, buildMissingToolPermissionIssuePayload, classifyCopilotFailure, + extractTokenCountFromOutput, buildMissingToolAlternatives, buildInfrastructureIncompletePayload, buildCopilotProxyAuthFailureDiagnostic, @@ -500,6 +501,102 @@ describe("copilot_harness.cjs", () => { const tail = extractOutputTail(`prefix\n${"x".repeat(40)}`, { maxLines: 5, maxChars: 16 }); expect(tail).toBe(`…${"x".repeat(15)}`); }); + + describe("extractTokenCountFromOutput", () => { + it("returns 0 for empty string", () => { + expect(extractTokenCountFromOutput("")).toBe(0); + }); + + it("returns 0 for null/undefined", () => { + expect(extractTokenCountFromOutput(null)).toBe(0); + expect(extractTokenCountFromOutput(undefined)).toBe(0); + }); + + it("returns 0 when no total_tokens field is present", () => { + expect(extractTokenCountFromOutput("some output with no token data")).toBe(0); + }); + + it("extracts a single total_tokens value", () => { + const output = '{"total_tokens": 5000, "prompt_tokens": 3000}'; + expect(extractTokenCountFromOutput(output)).toBe(5000); + }); + + it("sums multiple total_tokens fields", () => { + const output = '{"total_tokens": 5000}\n{"total_tokens": 7500}'; + expect(extractTokenCountFromOutput(output)).toBe(12500); + }); + + it("handles total_tokens without spaces around colon", () => { + const output = '{"total_tokens":4321}'; + expect(extractTokenCountFromOutput(output)).toBe(4321); + }); + + it("handles total_tokens with extra spaces", () => { + const output = '{"total_tokens" : 9999}'; + expect(extractTokenCountFromOutput(output)).toBe(9999); + }); + + it("ignores non-total_tokens token fields like prompt_tokens and completion_tokens", () => { + const output = '{"prompt_tokens": 1000, "completion_tokens": 500}'; + expect(extractTokenCountFromOutput(output)).toBe(0); + }); + + it("returns correct sum across multiple JSON blocks in CLI output", () => { + const block1 = '{"model":"gpt-4","usage":{"prompt_tokens":3000,"completion_tokens":2000,"total_tokens":5000}}'; + const block2 = '{"model":"gpt-4","usage":{"prompt_tokens":4000,"completion_tokens":3000,"total_tokens":7000}}'; + expect(extractTokenCountFromOutput(`${block1}\n${block2}`)).toBe(12000); + }); + }); + + describe("long_run_exit classification", () => { + it("classifies as long_run_exit when hasOutput and tokenCount exceeds threshold", () => { + expect(classifyCopilotFailure({ hasOutput: true, tokenCount: 10001 })).toBe("long_run_exit"); + }); + + it("classifies as partial_execution when tokenCount is exactly at threshold", () => { + expect(classifyCopilotFailure({ hasOutput: true, tokenCount: 10000 })).toBe("partial_execution"); + }); + + it("classifies as partial_execution when tokenCount is below threshold", () => { + expect(classifyCopilotFailure({ hasOutput: true, tokenCount: 9999 })).toBe("partial_execution"); + }); + + it("classifies as no_output when hasOutput is false even with high tokenCount", () => { + expect(classifyCopilotFailure({ hasOutput: false, tokenCount: 50000 })).toBe("no_output"); + }); + + it("classifies as partial_execution when tokenCount is 0", () => { + expect(classifyCopilotFailure({ hasOutput: true, tokenCount: 0 })).toBe("partial_execution"); + }); + + it("classifies as partial_execution when tokenCount is absent", () => { + expect(classifyCopilotFailure({ hasOutput: true })).toBe("partial_execution"); + }); + + it("named error classes outrank long_run_exit: auth error", () => { + expect(classifyCopilotFailure({ hasOutput: true, isAuthErr: true, tokenCount: 50000 })).toBe("no_auth_info"); + }); + + it("named error classes outrank long_run_exit: quota exceeded", () => { + expect(classifyCopilotFailure({ hasOutput: true, isQuotaExceeded: true, tokenCount: 50000 })).toBe("capi_quota_exceeded"); + }); + + it("named error classes outrank long_run_exit: invocation cap exceeded", () => { + expect(classifyCopilotFailure({ hasOutput: true, isInvocationCapExceeded: true, tokenCount: 50000 })).toBe("invocation_cap_exceeded"); + }); + + it("named error classes outrank long_run_exit: MCP policy", () => { + expect(classifyCopilotFailure({ hasOutput: true, isMCPPolicy: true, tokenCount: 50000 })).toBe("mcp_policy_blocked"); + }); + + it("named error classes outrank long_run_exit: capi_error_400", () => { + expect(classifyCopilotFailure({ hasOutput: true, isTransientCAPIError: true, tokenCount: 50000 })).toBe("capi_error_400"); + }); + + it("named error classes outrank long_run_exit: sdk_session_idle_timeout", () => { + expect(classifyCopilotFailure({ hasOutput: true, isSDKSessionIdleTimeout: true, tokenCount: 50000 })).toBe("sdk_session_idle_timeout"); + }); + }); }); it("does not claim a retry when already at max retry attempt", () => { From b76349fd8a029db8edd99541aedb21fca214518f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Jul 2026 08:59:33 +0000 Subject: [PATCH 3/4] fix: rename attemptStartTime to attemptStart for naming consistency Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- actions/setup/js/copilot_harness.cjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index 484bfbc7952..45a4eb8b818 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -1095,7 +1095,7 @@ async function main() { // Redact --prompt / -p value from logs to avoid leaking prompt content const safeArgs = currentArgs.map((arg, i) => (currentArgs[i - 1] === "--prompt" || currentArgs[i - 1] === "-p" ? "" : arg)); - const attemptStartTime = Date.now(); + const attemptStart = Date.now(); // Driver mode: run copilot_sdk_driver.cjs as a normal subprocess. The harness has // already started the sidecar; the driver only opens an SDK client connection. const result = await runProcess({ @@ -1156,7 +1156,7 @@ async function main() { const nonRetryableGuard = detectNonRetryableHarnessGuard(result.output); const isInvocationCapExceeded = nonRetryableGuard.maxRunsExceeded; const tokenCount = extractTokenCountFromOutput(result.output); - const attemptDurationMs = Date.now() - attemptStartTime; + const attemptDurationMs = Date.now() - attemptStart; const failureClass = classifyCopilotFailure({ hasOutput: result.hasOutput, isAuthErr, From 7d498875a54af659bb0ef8ac6ffd39934101d923 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 19 Jul 2026 10:44:01 +0000 Subject: [PATCH 4/4] feat(harness): make LONG_RUN_TOKEN_THRESHOLD configurable via GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD env var Co-authored-by: gh-aw-bot <259018956+gh-aw-bot@users.noreply.github.com> --- actions/setup/js/copilot_harness.cjs | 14 ++++++++++++-- actions/setup/js/copilot_harness.test.cjs | 23 +++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/actions/setup/js/copilot_harness.cjs b/actions/setup/js/copilot_harness.cjs index 45a4eb8b818..4ed31e07d16 100644 --- a/actions/setup/js/copilot_harness.cjs +++ b/actions/setup/js/copilot_harness.cjs @@ -81,10 +81,19 @@ const OUTPUT_TAIL_MAX_CHARS = 600; const OUTPUT_TAIL_MAX_LINES = 12; const MIN_POST_RESULT_WATCHDOG_TIMEOUT_MS = 50; const DEFAULT_POST_RESULT_WATCHDOG_IDLE_TIMEOUT_MS = 20 * 1000; -// Token count threshold above which a 0-turn failure is classified as "long_run_exit" +// Default token count threshold above which a 0-turn failure is classified as "long_run_exit" // rather than the generic "partial_execution". Corresponds to ~30+ minutes of Copilot // CLI work where the wrapper exits non-zero after the agent has completed substantial work. -const LONG_RUN_TOKEN_THRESHOLD = 10000; +// Override via GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD env var. +const DEFAULT_LONG_RUN_TOKEN_THRESHOLD = 10000; +function resolveLongRunTokenThreshold(env = process.env) { + const configured = Number(env.GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD); + if (!Number.isFinite(configured) || configured < 0) { + return DEFAULT_LONG_RUN_TOKEN_THRESHOLD; + } + return configured; +} +const LONG_RUN_TOKEN_THRESHOLD = resolveLongRunTokenThreshold(); function resolvePostResultWatchdogIdleTimeoutMs(env = process.env) { const configuredTimeoutMs = Number(env.GH_AW_HARNESS_WATCHDOG_TIMEOUT_MS); if (!Number.isFinite(configuredTimeoutMs) || configuredTimeoutMs <= 0) { @@ -1450,6 +1459,7 @@ if (typeof module !== "undefined" && module.exports) { applyCopilotModelAliasResolution, applyCopilotWireAPI, loadAwfConfigData, + resolveLongRunTokenThreshold, }; } diff --git a/actions/setup/js/copilot_harness.test.cjs b/actions/setup/js/copilot_harness.test.cjs index 57f857bd6a8..7bad0a88bab 100644 --- a/actions/setup/js/copilot_harness.test.cjs +++ b/actions/setup/js/copilot_harness.test.cjs @@ -56,6 +56,7 @@ const { writeCopilotOutputs, parseCopilotSDKServerArgsFromEnv, applyCopilotWireAPI, + resolveLongRunTokenThreshold, } = require("./copilot_harness.cjs"); const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard } = require("./harness_retry_guard.cjs"); @@ -597,6 +598,28 @@ describe("copilot_harness.cjs", () => { expect(classifyCopilotFailure({ hasOutput: true, isSDKSessionIdleTimeout: true, tokenCount: 50000 })).toBe("sdk_session_idle_timeout"); }); }); + + describe("resolveLongRunTokenThreshold", () => { + it("returns default 10000 when env var is unset", () => { + expect(resolveLongRunTokenThreshold({})).toBe(10000); + }); + + it("returns the configured value when GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD is set", () => { + expect(resolveLongRunTokenThreshold({ GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD: "5000" })).toBe(5000); + }); + + it("returns default when env var is not a number", () => { + expect(resolveLongRunTokenThreshold({ GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD: "abc" })).toBe(10000); + }); + + it("returns default when env var is negative", () => { + expect(resolveLongRunTokenThreshold({ GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD: "-1" })).toBe(10000); + }); + + it("returns 0 when env var is '0' (disables long_run_exit classification)", () => { + expect(resolveLongRunTokenThreshold({ GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD: "0" })).toBe(0); + }); + }); }); it("does not claim a retry when already at max retry attempt", () => {