Classify unsupported Codex custom tools as invalid-model failures

actions/setup/js/detect_agent_errors.cjs

@@ -157,6 +157,41 @@ function isAgenticEngineTimeout(logContent) {
 const MODEL_NOT_SUPPORTED_PATTERN =
   /(?:The requested model is not supported|invalid model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|unknown model\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?\s+(?:is\s+)?(?:not found|does not exist|not supported|not available|unavailable)|404\b[^\n]*\bModel\s+not\s+found|No model available\b[^\n]*policy enablement)/i;
 
+/**
+ * Determines if Codex emitted a `turn.failed` event for a model that does not
+ * support its custom tool schema.
+ * @param {string} output - Collected stdout+stderr from the process
+ * @returns {boolean}
+ */
+function isUnsupportedModelToolsError(output) {
+  return output.split(/\r?\n/).some(line => {
+    try {
+      const event = JSON.parse(line);
+      if (event?.type !== "turn.failed" || !event.error) return false;
+
+      const candidates = [event.error];
+      for (let visited = 0; visited < 8 && candidates.length > 0; visited++) {
+        const current = candidates.shift();
+        if (!current || typeof current !== "object") continue;
+        if (current.message === "Invalid value: 'custom'" && current.param === "tools") return true;
+
+        if (current.error && typeof current.error === "object") candidates.push(current.error);
+        for (const value of [current.message, current.metadata?.raw]) {
+          if (typeof value !== "string") continue;
+          try {
+            candidates.push(JSON.parse(value));
+          } catch {
+            // Ignore non-JSON strings.
+          }
+        }
+      }
+      return false;
+    } catch {
+      return false;
+    }
+  });
+}
+
 // Pattern: Generic HTTP 400 Bad Request responses emitted by engine / SDK wrappers.
 // NOTE: keep in sync with HTTP_400_RESPONSE_ERROR_PATTERN in copilot_harness.cjs.
 // Also matches "400 400 400 no model endpoints available given user constraints" which is emitted
@@ -359,7 +394,7 @@ function detectErrors(logContent) {
     inferenceAccessError: INFERENCE_ACCESS_ERROR_PATTERN.test(logContent),
     mcpPolicyError: MCP_POLICY_BLOCKED_PATTERN.test(logContent),
     agenticEngineTimeout: isAgenticEngineTimeout(logContent),
-    modelNotSupportedError: MODEL_NOT_SUPPORTED_PATTERN.test(logContent),
+    modelNotSupportedError: MODEL_NOT_SUPPORTED_PATTERN.test(logContent) || isUnsupportedModelToolsError(logContent),
     http400ResponseError: HTTP_400_RESPONSE_ERROR_PATTERN.test(logContent),
     capiQuotaExceededError: isCAPIQuotaExceededError(logContent),
     invocationCapExceeded: isInvocationCapExceededError(logContent),
@@ -590,6 +625,7 @@ module.exports = {
   isInvocationCapExceededError,
   isMaxCacheMissesExceededError,
   isAgenticEngineTimeout,
+  isUnsupportedModelToolsError,
   isStepTimeout,
   detectStepTimeoutFromEnvironment,
   INFERENCE_ACCESS_ERROR_PATTERN,

actions/setup/js/detect_agent_errors.test.cjs

@@ -18,6 +18,7 @@ const {
   isInvocationCapExceededError,
   isMaxCacheMissesExceededError,
   isAgenticEngineTimeout,
+  isUnsupportedModelToolsError,
   isStepTimeout,
   INFERENCE_ACCESS_ERROR_PATTERN,
   MCP_POLICY_BLOCKED_PATTERN,
@@ -160,6 +161,24 @@ describe("detect_agent_errors.cjs", () => {
       expect(MODEL_NOT_SUPPORTED_PATTERN.test(errorOutput)).toBe(true);
     });
 
+    it("classifies the raw Codex custom-tools rejection from unsupported models", () => {
+      const errorOutput = String.raw`{"type":"turn.failed","error":{"message":"{\n  \"error\": {\n    \"message\": \"Invalid value: 'custom'\",\n    \"type\": \"invalid_request_error\",\n    \"param\": \"tools\",\n    \"code\": \"unknown_parameter\"\n  }\n}"}}`;
+      expect(isUnsupportedModelToolsError(errorOutput)).toBe(true);
+      expect(detectErrors(errorOutput).modelNotSupportedError).toBe(true);
+    });
+
+    it("classifies the raw Codex rejection when the error fields are reordered", () => {
+      const errorOutput = String.raw`{"type":"turn.failed","error":{"message":"{\"error\":{\"param\":\"tools\",\"message\":\"Invalid value: 'custom'\"}}"}}`;
+      expect(isUnsupportedModelToolsError(errorOutput)).toBe(true);
+      expect(detectErrors(errorOutput).modelNotSupportedError).toBe(true);
+    });
+
+    it("does not combine custom-value and tools fields from separate error objects", () => {
+      const errorOutput = String.raw`{"type":"turn.failed","error":{"message":"{\"first\":{\"message\":\"Invalid value: 'custom'\"},\"second\":{\"param\":\"tools\"}}"}}`;
+      expect(isUnsupportedModelToolsError(errorOutput)).toBe(false);
+      expect(detectErrors(errorOutput).modelNotSupportedError).toBe(false);
+    });
+
     it("does not match 'No model available' without the policy-enablement hint", () => {
       expect(MODEL_NOT_SUPPORTED_PATTERN.test("No model available. Retrying shortly.")).toBe(false);
       expect(MODEL_NOT_SUPPORTED_PATTERN.test("No model available\nCheck policy enablement under GitHub Settings > Copilot")).toBe(false);