From 6c5e41f94e70edf1ce10005d0283d3be69e413a7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:38:22 +0000 Subject: [PATCH 1/3] Initial plan From e99f307107b5f640d08820820b65ca2f499dc370 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:45:23 +0000 Subject: [PATCH 2/3] Implement exploration-error trajectory grader Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/shared/graders/README.md | 2 +- .../shared/graders/exploration-error.md | 84 +++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/shared/graders/exploration-error.md diff --git a/.github/workflows/shared/graders/README.md b/.github/workflows/shared/graders/README.md index 0f43e8abc4b..cd9f94856ee 100644 --- a/.github/workflows/shared/graders/README.md +++ b/.github/workflows/shared/graders/README.md @@ -43,7 +43,7 @@ to `Implemented` in the same PR that adds `shared/graders/.md`. |---|---|---|---| | 1 | `policy-near-miss` | Policy/guard predicates | Implemented | | 2 | `skill-constraint-coverage` | Precompiled constraints | Implemented | -| 3 | `exploration-error` | State/task model | Not started | +| 3 | `exploration-error` | State/task model | Implemented | | 4 | `exploitation-error` | State/task model | Not started | | 12 | `tool-output-consumption-rate` | Provenance/reference IDs | Not started | | 13 | `end-to-end-lineage-completeness` | Provenance graph | Not started | diff --git a/.github/workflows/shared/graders/exploration-error.md b/.github/workflows/shared/graders/exploration-error.md new file mode 100644 index 00000000000..08ce3fbe0ce --- /dev/null +++ b/.github/workflows/shared/graders/exploration-error.md @@ -0,0 +1,84 @@ +--- +graders: + # For runs that left one or more declared objectives unsatisfied, measures + # whether the failure was due to insufficient search: 1 - (observations / + # distinctStatesVisited), clamped to [0, 1]. distinctStatesVisited comes + # from distinct state_change event refs, falling back to the declared + # states[] count when no state_change events are recorded. Runs with all + # objectives satisfied score 0 (no exploration error to attribute). This is + # the complement of exploitation-error (not yet implemented), which covers + # runs that had enough evidence but failed anyway. Lower is better: fewer + # unmet objectives attributable to insufficient search. + exploration-error: + name: Exploration Error + unit: ratio + direction: lower_is_better + min: 0.0 + max: 1.0 + script: | + const isRecord = value => value !== null && typeof value === "object" && !Array.isArray(value); + const candidates = [ + trace.trajectoryIR, + trace.trajectoryIr, + trace.ir, + isRecord(trace.agentOutput) ? trace.agentOutput.trajectoryIR : null, + isRecord(trace.agentOutput) ? trace.agentOutput.trajectoryIr : null, + isRecord(trace.agentOutput) ? trace.agentOutput.trajectory : null, + isRecord(trace.agentOutput) ? trace.agentOutput : null, + ].filter(isRecord); + + const objectives = (candidates.find(value => Array.isArray(value.objectives) && value.objectives.some(isRecord))?.objectives ?? []).filter(isRecord); + if (objectives.length === 0) { + return { value: null, unit: "ratio", passed: null, message: "not applicable: no declared objectives in the trace" }; + } + + const unmet = objectives.filter(objective => objective.satisfiedAtEventIndex === null || objective.satisfiedAtEventIndex === undefined); + if (unmet.length === 0) { + return { value: 0, unit: "ratio", details: `objectives=${objectives.length} unmet=0; all objectives satisfied` }; + } + + const events = (candidates.find(value => Array.isArray(value.events) && value.events.some(isRecord))?.events ?? []).filter(isRecord); + const states = (candidates.find(value => Array.isArray(value.states) && value.states.some(isRecord))?.states ?? []).filter(isRecord); + const observations = (candidates.find(value => Array.isArray(value.observations) && value.observations.some(isRecord))?.observations ?? []).filter(isRecord); + + const stateChangeEvents = events.filter(event => event.kind === "state_change"); + let distinctStatesVisited = 0; + let source = ""; + if (stateChangeEvents.length > 0) { + const visited = new Set(stateChangeEvents.map(event => (typeof event.ref === "string" ? event.ref : JSON.stringify(event.ref)))); + distinctStatesVisited = visited.size; + source = "state_change events"; + } else if (states.length > 0) { + distinctStatesVisited = states.length; + source = "declared states[]"; + } else { + return { value: null, unit: "ratio", passed: null, message: "not applicable: no state_change events or declared states in the trace" }; + } + + const value = helpers.clamp(1 - observations.length / distinctStatesVisited, 0, 1); + const unmetDescriptions = unmet.slice(0, 5).map(objective => (typeof objective.id === "string" && objective.id !== "" ? objective.id : objective.description)); + + return { + value, + unit: "ratio", + details: `objectives=${objectives.length} unmet=${unmet.length} observations=${observations.length} distinctStatesVisited=${distinctStatesVisited} (from ${source})${unmetDescriptions.length === 0 ? "" : `; unmet objectives: ${unmetDescriptions.join(", ")}`}`, + }; +--- + + From 4daf3828addc3b7bf922e61c6e5e98f649d25f91 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 30 Aug 2026 06:23:12 +0000 Subject: [PATCH 3/3] Fix exploration-error grader candidate selection Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../shared/graders/exploration-error.md | 17 ++- actions/setup/js/trace_graders.test.cjs | 109 ++++++++++++++++++ 2 files changed, 122 insertions(+), 4 deletions(-) diff --git a/.github/workflows/shared/graders/exploration-error.md b/.github/workflows/shared/graders/exploration-error.md index 08ce3fbe0ce..4c8bb8eba7a 100644 --- a/.github/workflows/shared/graders/exploration-error.md +++ b/.github/workflows/shared/graders/exploration-error.md @@ -27,7 +27,16 @@ graders: isRecord(trace.agentOutput) ? trace.agentOutput : null, ].filter(isRecord); - const objectives = (candidates.find(value => Array.isArray(value.objectives) && value.objectives.some(isRecord))?.objectives ?? []).filter(isRecord); + const candidate = + candidates.find(value => Array.isArray(value.objectives) && value.objectives.some(isRecord)) ?? + candidates.find(value => + Array.isArray(value.objectives) || + Array.isArray(value.events) || + Array.isArray(value.states) || + Array.isArray(value.observations) + ) ?? + null; + const objectives = (candidate && Array.isArray(candidate.objectives) ? candidate.objectives : []).filter(isRecord); if (objectives.length === 0) { return { value: null, unit: "ratio", passed: null, message: "not applicable: no declared objectives in the trace" }; } @@ -37,9 +46,9 @@ graders: return { value: 0, unit: "ratio", details: `objectives=${objectives.length} unmet=0; all objectives satisfied` }; } - const events = (candidates.find(value => Array.isArray(value.events) && value.events.some(isRecord))?.events ?? []).filter(isRecord); - const states = (candidates.find(value => Array.isArray(value.states) && value.states.some(isRecord))?.states ?? []).filter(isRecord); - const observations = (candidates.find(value => Array.isArray(value.observations) && value.observations.some(isRecord))?.observations ?? []).filter(isRecord); + const events = (candidate && Array.isArray(candidate.events) ? candidate.events : []).filter(isRecord); + const states = (candidate && Array.isArray(candidate.states) ? candidate.states : []).filter(isRecord); + const observations = (candidate && Array.isArray(candidate.observations) ? candidate.observations : []).filter(isRecord); const stateChangeEvents = events.filter(event => event.kind === "state_change"); let distinctStatesVisited = 0; diff --git a/actions/setup/js/trace_graders.test.cjs b/actions/setup/js/trace_graders.test.cjs index f327442ab88..720a775dfb3 100644 --- a/actions/setup/js/trace_graders.test.cjs +++ b/actions/setup/js/trace_graders.test.cjs @@ -86,6 +86,24 @@ function runPolicyNearMiss(trace) { }); } +const explorationErrorScriptMatch = fs.readFileSync(path.join(__dirname, "../../../.github/workflows/shared/graders/exploration-error.md"), "utf8").match(/script: \|\n([\s\S]*?)\n^---\s*$/m); +if (!explorationErrorScriptMatch?.[1]) { + throw new Error("unable to extract exploration-error grader script"); +} +const explorationErrorScript = explorationErrorScriptMatch[1] + .split("\n") + .map(line => line.slice(6)) + .join("\n"); + +function runExplorationError(trace) { + return runCustomGrader("exploration-error", explorationErrorScript, makeTrace(trace), { + name: "Exploration Error", + unit: "ratio", + direction: "lower_is_better", + source: "inline", + }); +} + const skillConstraintCoverageScriptMatch = fs.readFileSync(path.join(__dirname, "../../../.github/workflows/shared/graders/skill-constraint-coverage.md"), "utf8").match(/script: \|\n([\s\S]*?)\n^---\s*$/m); if (!skillConstraintCoverageScriptMatch?.[1]) { throw new Error("unable to extract skill-constraint-coverage grader script"); @@ -648,6 +666,97 @@ describe("trace_graders", () => { }); }); + describe("exploration-error custom grader", () => { + it("reports unavailable when no objectives are declared", () => { + const result = runExplorationError({ trajectoryIR: { events: [{ kind: "state_change", ref: "a" }] } }); + + expect(result.value).toBeNull(); + expect(result.passed).toBeNull(); + expect(result.message).toContain("no declared objectives"); + }); + + it("returns zero when all objectives are already satisfied", () => { + const result = runExplorationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Read all files", satisfiedAtEventIndex: 0 }], + }, + }); + + expect(result.value).toBe(0); + expect(result.details).toContain("all objectives satisfied"); + }); + + it("prefers the objective-bearing IR candidate over unrelated agentOutput observations", () => { + const result = runExplorationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }], + events: [{ kind: "state_change", ref: "repo-root" }], + }, + agentOutput: { + observations: [{ id: "obs-1" }], + }, + }); + + expect(result.value).toBe(1); + expect(result.details).toContain("observations=0"); + }); + + it("counts distinct state refs from state_change events", () => { + const result = runExplorationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }], + events: [ + { kind: "state_change", ref: "repo-root" }, + { kind: "state_change", ref: "repo-readme" }, + { kind: "state_change", ref: "repo-root" }, + ], + observations: [{ id: "obs-1" }], + }, + }); + + expect(result.value).toBeCloseTo(0.5); + expect(result.details).toContain("distinctStatesVisited=2"); + }); + + it("falls back to declared states[] when no state_change events exist", () => { + const result = runExplorationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }], + states: [{ id: "a" }, { id: "b" }], + observations: [{ id: "obs-1" }], + }, + }); + + expect(result.value).toBeCloseTo(0.5); + expect(result.details).toContain("from declared states[]"); + }); + + it("clamps the ratio at zero when observations exceed the visited-state count", () => { + const result = runExplorationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }], + states: [{ id: "a" }], + observations: [{ id: "obs-1" }, { id: "obs-2" }, { id: "obs-3" }], + }, + }); + + expect(result.value).toBe(0); + }); + + it("is unavailable without state_change events or declared states", () => { + const result = runExplorationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }], + observations: [{ id: "obs-1" }], + }, + }); + + expect(result.value).toBeNull(); + expect(result.passed).toBeNull(); + expect(result.message).toContain("no state_change events or declared states"); + }); + }); + describe("skill-constraint-coverage custom grader", () => { it("reports full coverage when every constraint is exercised and succeeds", () => { const result = runSkillConstraintCoverage(