feat(harness): distinguish long-run vs startup Copilot CLI failures via token-count classification

actions/setup/js/copilot_harness.cjs

@@ -81,6 +81,19 @@ const OUTPUT_TAIL_MAX_CHARS = 600;
 const OUTPUT_TAIL_MAX_LINES = 12;
 const MIN_POST_RESULT_WATCHDOG_TIMEOUT_MS = 50;
 const DEFAULT_POST_RESULT_WATCHDOG_IDLE_TIMEOUT_MS = 20 * 1000;
+// Default token count threshold above which a 0-turn failure is classified as "long_run_exit"
+// rather than the generic "partial_execution". Corresponds to ~30+ minutes of Copilot
+// CLI work where the wrapper exits non-zero after the agent has completed substantial work.
+// Override via GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD env var.
+const DEFAULT_LONG_RUN_TOKEN_THRESHOLD = 10000;
+function resolveLongRunTokenThreshold(env = process.env) {
+  const configured = Number(env.GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD);
+  if (!Number.isFinite(configured) || configured < 0) {
+    return DEFAULT_LONG_RUN_TOKEN_THRESHOLD;
+  }
+  return configured;
+}
+const LONG_RUN_TOKEN_THRESHOLD = resolveLongRunTokenThreshold();
 function resolvePostResultWatchdogIdleTimeoutMs(env = process.env) {
   const configuredTimeoutMs = Number(env.GH_AW_HARNESS_WATCHDOG_TIMEOUT_MS);
   if (!Number.isFinite(configuredTimeoutMs) || configuredTimeoutMs <= 0) {
@@ -476,6 +489,24 @@ function extractOutputTail(output, options) {
   return tail;
 }
 
+/**
+ * Sum all `"total_tokens"` values emitted in debug JSON response blocks in CLI output.
+ * The Copilot CLI emits JSON blocks containing token usage data in its stdout/stderr.
+ * Returns 0 when no token data is present (e.g., startup failures that produced no output).
+ * @param {string} output - Combined stdout/stderr from the Copilot CLI
+ * @returns {number}
+ */
+function extractTokenCountFromOutput(output) {
+  if (!output) return 0;
+  let total = 0;
+  const pattern = /"total_tokens"\s*:\s*(\d+)/g;
+  let match;
+  while ((match = pattern.exec(output)) !== null) {
+    total += parseInt(match[1], 10);
+  }
+  return total;
+}
+
 /**
  * Classify a failed Copilot attempt into a short, named failure class.
  * @param {{
@@ -492,6 +523,7 @@ function extractOutputTail(output, options) {
  *   isQuotaExceeded?: boolean,
  *   isSDKSessionIdleTimeout?: boolean,
  *   hasNumerousPermissionDenied?: boolean,
+ *   tokenCount?: number,
  * }} detection
  * @returns {string}
  */
@@ -508,6 +540,7 @@ function classifyCopilotFailure(detection) {
   if (detection.isMCPGatewayShutdown) return "mcp_gateway_shutdown";
   if (detection.hasNumerousPermissionDenied) return "permission_denied";
   if (detection.isTransientCAPIError) return "capi_error_400";
+  if (detection.hasOutput && (detection.tokenCount ?? 0) > LONG_RUN_TOKEN_THRESHOLD) return "long_run_exit";
   return detection.hasOutput ? "partial_execution" : "no_output";
 }
 
@@ -1071,6 +1104,7 @@ async function main() {
 
         // Redact --prompt / -p value from logs to avoid leaking prompt content
         const safeArgs = currentArgs.map((arg, i) => (currentArgs[i - 1] === "--prompt" || currentArgs[i - 1] === "-p" ? "<redacted>" : arg));
+        const attemptStart = Date.now();
         // Driver mode: run copilot_sdk_driver.cjs as a normal subprocess. The harness has
         // already started the sidecar; the driver only opens an SDK client connection.
         const result = await runProcess({
@@ -1130,6 +1164,8 @@ async function main() {
         const hasNumerousPermissionDenied = hasNumerousPermissionDeniedIssues(result.output);
         const nonRetryableGuard = detectNonRetryableHarnessGuard(result.output);
         const isInvocationCapExceeded = nonRetryableGuard.maxRunsExceeded;
+        const tokenCount = extractTokenCountFromOutput(result.output);
+        const attemptDurationMs = Date.now() - attemptStart;
         const failureClass = classifyCopilotFailure({
           hasOutput: result.hasOutput,
           isAuthErr,
@@ -1144,6 +1180,7 @@ async function main() {
           isQuotaExceeded,
           isSDKSessionIdleTimeout,
           hasNumerousPermissionDenied,
+          tokenCount,
         });
         const outputTail = extractOutputTail(result.output);
         log(
@@ -1164,6 +1201,8 @@ async function main() {
             ` permissionDeniedCount=${permissionDeniedCount}` +
             ` hasNumerousPermissionDenied=${hasNumerousPermissionDenied}` +
             ` hasOutput=${result.hasOutput}` +
+            ` tokenCount=${tokenCount}` +
+            ` attemptDurationMs=${attemptDurationMs}` +
             ` retriesRemaining=${maxRetries - attempt}`
         );
         if (outputTail) {
@@ -1186,7 +1225,7 @@ async function main() {
         // (watchdogFired=true), as well as any other partial_execution failure that occurs
         // after the primary task output was already produced.  Retrying would reproduce the
         // same pattern and exhaust the retry budget without ever posting a final safe-output.
-        if (failureClass === "partial_execution" && safeOutputsPath && hasTerminalSafeOutput(safeOutputsPath)) {
+        if ((failureClass === "partial_execution" || failureClass === "long_run_exit") && safeOutputsPath && hasTerminalSafeOutput(safeOutputsPath)) {
           const reason = result.watchdogFired ? "post-result watchdog fired after terminal safe-output was emitted" : "partial execution after terminal safe-output was already produced";
           log(`attempt ${attempt + 1}: ${reason} — treating as success (late-activity exit suppressed)`);
           lastExitCode = 0;
@@ -1397,6 +1436,7 @@ if (typeof module !== "undefined" && module.exports) {
     countPermissionDeniedIssues,
     detectCopilotErrors,
     classifyCopilotFailure,
+    extractTokenCountFromOutput,
     shouldRetryFailedExecution,
     extractOutputTail,
     isRetryableProxyAuthenticationFailure,
@@ -1419,6 +1459,7 @@ if (typeof module !== "undefined" && module.exports) {
     applyCopilotModelAliasResolution,
     applyCopilotWireAPI,
     loadAwfConfigData,
+    resolveLongRunTokenThreshold,
   };
 }
 

actions/setup/js/copilot_harness.test.cjs

@@ -14,6 +14,7 @@ const {
   appendSafeOutputLine,
   buildMissingToolPermissionIssuePayload,
   classifyCopilotFailure,
+  extractTokenCountFromOutput,
   buildMissingToolAlternatives,
   buildInfrastructureIncompletePayload,
   buildCopilotProxyAuthFailureDiagnostic,
@@ -55,6 +56,7 @@ const {
   writeCopilotOutputs,
   parseCopilotSDKServerArgsFromEnv,
   applyCopilotWireAPI,
+  resolveLongRunTokenThreshold,
 } = require("./copilot_harness.cjs");
 
 const { detectNonRetryableHarnessGuard, buildSoftTimeoutGuard } = require("./harness_retry_guard.cjs");
@@ -500,6 +502,124 @@ describe("copilot_harness.cjs", () => {
         const tail = extractOutputTail(`prefix\n${"x".repeat(40)}`, { maxLines: 5, maxChars: 16 });
         expect(tail).toBe(`…${"x".repeat(15)}`);
       });
+
+      describe("extractTokenCountFromOutput", () => {
+        it("returns 0 for empty string", () => {
+          expect(extractTokenCountFromOutput("")).toBe(0);
+        });
+
+        it("returns 0 for null/undefined", () => {
+          expect(extractTokenCountFromOutput(null)).toBe(0);
+          expect(extractTokenCountFromOutput(undefined)).toBe(0);
+        });
+
+        it("returns 0 when no total_tokens field is present", () => {
+          expect(extractTokenCountFromOutput("some output with no token data")).toBe(0);
+        });
+
+        it("extracts a single total_tokens value", () => {
+          const output = '{"total_tokens": 5000, "prompt_tokens": 3000}';
+          expect(extractTokenCountFromOutput(output)).toBe(5000);
+        });
+
+        it("sums multiple total_tokens fields", () => {
+          const output = '{"total_tokens": 5000}\n{"total_tokens": 7500}';
+          expect(extractTokenCountFromOutput(output)).toBe(12500);
+        });
+
+        it("handles total_tokens without spaces around colon", () => {
+          const output = '{"total_tokens":4321}';
+          expect(extractTokenCountFromOutput(output)).toBe(4321);
+        });
+
+        it("handles total_tokens with extra spaces", () => {
+          const output = '{"total_tokens"  :  9999}';
+          expect(extractTokenCountFromOutput(output)).toBe(9999);
+        });
+
+        it("ignores non-total_tokens token fields like prompt_tokens and completion_tokens", () => {
+          const output = '{"prompt_tokens": 1000, "completion_tokens": 500}';
+          expect(extractTokenCountFromOutput(output)).toBe(0);
+        });
+
+        it("returns correct sum across multiple JSON blocks in CLI output", () => {
+          const block1 = '{"model":"gpt-4","usage":{"prompt_tokens":3000,"completion_tokens":2000,"total_tokens":5000}}';
+          const block2 = '{"model":"gpt-4","usage":{"prompt_tokens":4000,"completion_tokens":3000,"total_tokens":7000}}';
+          expect(extractTokenCountFromOutput(`${block1}\n${block2}`)).toBe(12000);
+        });
+      });
+
+      describe("long_run_exit classification", () => {
+        it("classifies as long_run_exit when hasOutput and tokenCount exceeds threshold", () => {
+          expect(classifyCopilotFailure({ hasOutput: true, tokenCount: 10001 })).toBe("long_run_exit");
+        });
+
+        it("classifies as partial_execution when tokenCount is exactly at threshold", () => {
+          expect(classifyCopilotFailure({ hasOutput: true, tokenCount: 10000 })).toBe("partial_execution");
+        });
+
+        it("classifies as partial_execution when tokenCount is below threshold", () => {
+          expect(classifyCopilotFailure({ hasOutput: true, tokenCount: 9999 })).toBe("partial_execution");
+        });
+
+        it("classifies as no_output when hasOutput is false even with high tokenCount", () => {
+          expect(classifyCopilotFailure({ hasOutput: false, tokenCount: 50000 })).toBe("no_output");
+        });
+
+        it("classifies as partial_execution when tokenCount is 0", () => {
+          expect(classifyCopilotFailure({ hasOutput: true, tokenCount: 0 })).toBe("partial_execution");
+        });
+
+        it("classifies as partial_execution when tokenCount is absent", () => {
+          expect(classifyCopilotFailure({ hasOutput: true })).toBe("partial_execution");
+        });
+
+        it("named error classes outrank long_run_exit: auth error", () => {
+          expect(classifyCopilotFailure({ hasOutput: true, isAuthErr: true, tokenCount: 50000 })).toBe("no_auth_info");
+        });
+
+        it("named error classes outrank long_run_exit: quota exceeded", () => {
+          expect(classifyCopilotFailure({ hasOutput: true, isQuotaExceeded: true, tokenCount: 50000 })).toBe("capi_quota_exceeded");
+        });
+
+        it("named error classes outrank long_run_exit: invocation cap exceeded", () => {
+          expect(classifyCopilotFailure({ hasOutput: true, isInvocationCapExceeded: true, tokenCount: 50000 })).toBe("invocation_cap_exceeded");
+        });
+
+        it("named error classes outrank long_run_exit: MCP policy", () => {
+          expect(classifyCopilotFailure({ hasOutput: true, isMCPPolicy: true, tokenCount: 50000 })).toBe("mcp_policy_blocked");
+        });
+
+        it("named error classes outrank long_run_exit: capi_error_400", () => {
+          expect(classifyCopilotFailure({ hasOutput: true, isTransientCAPIError: true, tokenCount: 50000 })).toBe("capi_error_400");
+        });
+
+        it("named error classes outrank long_run_exit: sdk_session_idle_timeout", () => {
+          expect(classifyCopilotFailure({ hasOutput: true, isSDKSessionIdleTimeout: true, tokenCount: 50000 })).toBe("sdk_session_idle_timeout");
+        });
+      });
+
+      describe("resolveLongRunTokenThreshold", () => {
+        it("returns default 10000 when env var is unset", () => {
+          expect(resolveLongRunTokenThreshold({})).toBe(10000);
+        });
+
+        it("returns the configured value when GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD is set", () => {
+          expect(resolveLongRunTokenThreshold({ GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD: "5000" })).toBe(5000);
+        });
+
+        it("returns default when env var is not a number", () => {
+          expect(resolveLongRunTokenThreshold({ GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD: "abc" })).toBe(10000);
+        });
+
+        it("returns default when env var is negative", () => {
+          expect(resolveLongRunTokenThreshold({ GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD: "-1" })).toBe(10000);
+        });
+
+        it("returns 0 when env var is '0' (disables long_run_exit classification)", () => {
+          expect(resolveLongRunTokenThreshold({ GH_AW_HARNESS_LONG_RUN_TOKEN_THRESHOLD: "0" })).toBe(0);
+        });
+      });
     });
 
     it("does not claim a retry when already at max retry attempt", () => {