diff --git a/actions/setup/js/detect_agent_errors.cjs b/actions/setup/js/detect_agent_errors.cjs index b7793d6e339..ce511ac0515 100644 --- a/actions/setup/js/detect_agent_errors.cjs +++ b/actions/setup/js/detect_agent_errors.cjs @@ -157,6 +157,41 @@ function isAgenticEngineTimeout(logContent) { const MODEL_NOT_SUPPORTED_PATTERN = /(?:The requested model is not supported|invalid model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|unknown model\s+['"`]?[a-z0-9._:/@-]+['"`]?(?=(?:\s*$|\s*[\n\r.,;:!?)]))|model(?:\s+name)?\s+['"`]?[a-z0-9._:/@-]+['"`]?\s+(?:is\s+)?(?:not found|does not exist|not supported|not available|unavailable)|404\b[^\n]*\bModel\s+not\s+found|No model available\b[^\n]*policy enablement)/i; +/** + * Determines if Codex emitted a `turn.failed` event for a model that does not + * support its custom tool schema. + * @param {string} output - Collected stdout+stderr from the process + * @returns {boolean} + */ +function isUnsupportedModelToolsError(output) { + return output.split(/\r?\n/).some(line => { + try { + const event = JSON.parse(line); + if (event?.type !== "turn.failed" || !event.error) return false; + + const candidates = [event.error]; + for (let visited = 0; visited < 8 && candidates.length > 0; visited++) { + const current = candidates.shift(); + if (!current || typeof current !== "object") continue; + if (current.message === "Invalid value: 'custom'" && current.param === "tools") return true; + + if (current.error && typeof current.error === "object") candidates.push(current.error); + for (const value of [current.message, current.metadata?.raw]) { + if (typeof value !== "string") continue; + try { + candidates.push(JSON.parse(value)); + } catch { + // Ignore non-JSON strings. + } + } + } + return false; + } catch { + return false; + } + }); +} + // Pattern: Generic HTTP 400 Bad Request responses emitted by engine / SDK wrappers. // NOTE: keep in sync with HTTP_400_RESPONSE_ERROR_PATTERN in copilot_harness.cjs. // Also matches "400 400 400 no model endpoints available given user constraints" which is emitted @@ -359,7 +394,7 @@ function detectErrors(logContent) { inferenceAccessError: INFERENCE_ACCESS_ERROR_PATTERN.test(logContent), mcpPolicyError: MCP_POLICY_BLOCKED_PATTERN.test(logContent), agenticEngineTimeout: isAgenticEngineTimeout(logContent), - modelNotSupportedError: MODEL_NOT_SUPPORTED_PATTERN.test(logContent), + modelNotSupportedError: MODEL_NOT_SUPPORTED_PATTERN.test(logContent) || isUnsupportedModelToolsError(logContent), http400ResponseError: HTTP_400_RESPONSE_ERROR_PATTERN.test(logContent), capiQuotaExceededError: isCAPIQuotaExceededError(logContent), invocationCapExceeded: isInvocationCapExceededError(logContent), @@ -590,6 +625,7 @@ module.exports = { isInvocationCapExceededError, isMaxCacheMissesExceededError, isAgenticEngineTimeout, + isUnsupportedModelToolsError, isStepTimeout, detectStepTimeoutFromEnvironment, INFERENCE_ACCESS_ERROR_PATTERN, diff --git a/actions/setup/js/detect_agent_errors.test.cjs b/actions/setup/js/detect_agent_errors.test.cjs index 99b3503ce36..ca952f0be55 100644 --- a/actions/setup/js/detect_agent_errors.test.cjs +++ b/actions/setup/js/detect_agent_errors.test.cjs @@ -18,6 +18,7 @@ const { isInvocationCapExceededError, isMaxCacheMissesExceededError, isAgenticEngineTimeout, + isUnsupportedModelToolsError, isStepTimeout, INFERENCE_ACCESS_ERROR_PATTERN, MCP_POLICY_BLOCKED_PATTERN, @@ -160,6 +161,24 @@ describe("detect_agent_errors.cjs", () => { expect(MODEL_NOT_SUPPORTED_PATTERN.test(errorOutput)).toBe(true); }); + it("classifies the raw Codex custom-tools rejection from unsupported models", () => { + const errorOutput = String.raw`{"type":"turn.failed","error":{"message":"{\n \"error\": {\n \"message\": \"Invalid value: 'custom'\",\n \"type\": \"invalid_request_error\",\n \"param\": \"tools\",\n \"code\": \"unknown_parameter\"\n }\n}"}}`; + expect(isUnsupportedModelToolsError(errorOutput)).toBe(true); + expect(detectErrors(errorOutput).modelNotSupportedError).toBe(true); + }); + + it("classifies the raw Codex rejection when the error fields are reordered", () => { + const errorOutput = String.raw`{"type":"turn.failed","error":{"message":"{\"error\":{\"param\":\"tools\",\"message\":\"Invalid value: 'custom'\"}}"}}`; + expect(isUnsupportedModelToolsError(errorOutput)).toBe(true); + expect(detectErrors(errorOutput).modelNotSupportedError).toBe(true); + }); + + it("does not combine custom-value and tools fields from separate error objects", () => { + const errorOutput = String.raw`{"type":"turn.failed","error":{"message":"{\"first\":{\"message\":\"Invalid value: 'custom'\"},\"second\":{\"param\":\"tools\"}}"}}`; + expect(isUnsupportedModelToolsError(errorOutput)).toBe(false); + expect(detectErrors(errorOutput).modelNotSupportedError).toBe(false); + }); + it("does not match 'No model available' without the policy-enablement hint", () => { expect(MODEL_NOT_SUPPORTED_PATTERN.test("No model available. Retrying shortly.")).toBe(false); expect(MODEL_NOT_SUPPORTED_PATTERN.test("No model available\nCheck policy enablement under GitHub Settings > Copilot")).toBe(false);