From cca6b0b8faf7f05721077edba885bdbc772c5a69 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:32:47 +0000 Subject: [PATCH 1/3] Initial plan From ebb5a984237d902884ca8ad5c4990f64b6a95c9a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:37:42 +0000 Subject: [PATCH 2/3] Initial plan for exploitation-error grader Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/schemas/github-workflow.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/workflow/schemas/github-workflow.json b/pkg/workflow/schemas/github-workflow.json index d155681f698..c0130091534 100644 --- a/pkg/workflow/schemas/github-workflow.json +++ b/pkg/workflow/schemas/github-workflow.json @@ -288,16 +288,16 @@ "statuses": { "$ref": "#/definitions/permissions-level" }, + "vulnerability-alerts": { + "type": "string", + "enum": ["read", "none"] + }, "copilot-requests": { "type": "string", "enum": ["write", "none"] }, "drives": { "$ref": "#/definitions/permissions-level" - }, - "vulnerability-alerts": { - "type": "string", - "enum": ["read", "none"] } } }, From 5a3fd9c3b0b4b3ab64f2325f9a76e8098a313f79 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 30 Aug 2026 12:40:35 +0000 Subject: [PATCH 3/3] Implement exploitation-error trajectory grader Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .github/workflows/shared/graders/README.md | 2 +- .../shared/graders/exploitation-error.md | 113 +++++++++++++++ .../shared/graders/exploration-error.md | 8 +- actions/setup/js/trace_graders.test.cjs | 133 ++++++++++++++++++ 4 files changed, 251 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/shared/graders/exploitation-error.md diff --git a/.github/workflows/shared/graders/README.md b/.github/workflows/shared/graders/README.md index cd9f94856ee..17d68511a6d 100644 --- a/.github/workflows/shared/graders/README.md +++ b/.github/workflows/shared/graders/README.md @@ -44,7 +44,7 @@ to `Implemented` in the same PR that adds `shared/graders/.md`. | 1 | `policy-near-miss` | Policy/guard predicates | Implemented | | 2 | `skill-constraint-coverage` | Precompiled constraints | Implemented | | 3 | `exploration-error` | State/task model | Implemented | -| 4 | `exploitation-error` | State/task model | Not started | +| 4 | `exploitation-error` | State/task model | Implemented | | 12 | `tool-output-consumption-rate` | Provenance/reference IDs | Not started | | 13 | `end-to-end-lineage-completeness` | Provenance graph | Not started | | 14 | `action-provenance-coverage` | Provenance graph | Not started | diff --git a/.github/workflows/shared/graders/exploitation-error.md b/.github/workflows/shared/graders/exploitation-error.md new file mode 100644 index 00000000000..4645a84c60f --- /dev/null +++ b/.github/workflows/shared/graders/exploitation-error.md @@ -0,0 +1,113 @@ +--- +graders: + # For runs that left one or more declared objectives unsatisfied but had + # gathered enough evidence to succeed (observations >= distinctStatesVisited), + # measures how much of that evidence was never used: unusedObservations / + # observations, clamped to [0, 1]. An observation is used when it declares a + # non-empty consumedByActionIds. distinctStatesVisited comes from distinct + # state_change event refs, falling back to the declared states[] count when + # no state_change events are recorded. Runs with all objectives satisfied + # score 0 (no exploitation error to attribute); runs whose exploration was + # insufficient are not applicable and defer to exploration-error. Lower is + # better: fewer unmet objectives attributable to unused evidence. + exploitation-error: + name: Exploitation Error + unit: ratio + direction: lower_is_better + min: 0.0 + max: 1.0 + script: | + const isRecord = value => value !== null && typeof value === "object" && !Array.isArray(value); + const candidates = [ + trace.trajectoryIR, + trace.trajectoryIr, + trace.ir, + isRecord(trace.agentOutput) ? trace.agentOutput.trajectoryIR : null, + isRecord(trace.agentOutput) ? trace.agentOutput.trajectoryIr : null, + isRecord(trace.agentOutput) ? trace.agentOutput.trajectory : null, + isRecord(trace.agentOutput) ? trace.agentOutput : null, + ].filter(isRecord); + + const candidate = + candidates.find(value => Array.isArray(value.objectives) && value.objectives.some(isRecord)) ?? + candidates.find(value => + Array.isArray(value.objectives) || + Array.isArray(value.events) || + Array.isArray(value.states) || + Array.isArray(value.observations) + ) ?? + null; + const objectives = (candidate && Array.isArray(candidate.objectives) ? candidate.objectives : []).filter(isRecord); + if (objectives.length === 0) { + return { value: null, unit: "ratio", passed: null, message: "not applicable: no declared objectives in the trace" }; + } + + const unmet = objectives.filter(objective => objective.satisfiedAtEventIndex === null || objective.satisfiedAtEventIndex === undefined); + if (unmet.length === 0) { + return { value: 0, unit: "ratio", details: `objectives=${objectives.length} unmet=0; all objectives satisfied` }; + } + + const events = (candidate && Array.isArray(candidate.events) ? candidate.events : []).filter(isRecord); + const states = (candidate && Array.isArray(candidate.states) ? candidate.states : []).filter(isRecord); + const observations = (candidate && Array.isArray(candidate.observations) ? candidate.observations : []).filter(isRecord); + + const stateChangeEvents = events.filter(event => event.kind === "state_change"); + let distinctStatesVisited = 0; + let source = ""; + if (stateChangeEvents.length > 0) { + const visited = new Set(stateChangeEvents.map(event => (typeof event.ref === "string" ? event.ref : JSON.stringify(event.ref)))); + distinctStatesVisited = visited.size; + source = "state_change events"; + } else if (states.length > 0) { + distinctStatesVisited = states.length; + source = "declared states[]"; + } else { + return { value: null, unit: "ratio", passed: null, message: "not applicable: no state_change events or declared states in the trace" }; + } + + if (observations.length === 0) { + return { value: null, unit: "ratio", passed: null, message: "not applicable: no observations in the trace" }; + } + + if (observations.length < distinctStatesVisited) { + return { + value: null, + unit: "ratio", + passed: null, + message: `not applicable: exploration was insufficient (observations=${observations.length} < distinctStatesVisited=${distinctStatesVisited}); see exploration-error`, + }; + } + + const unused = observations.filter( + observation => !Array.isArray(observation.consumedByActionIds) || observation.consumedByActionIds.length === 0 + ); + const value = helpers.clamp(unused.length / observations.length, 0, 1); + const unmetDescriptions = unmet.slice(0, 5).map(objective => (typeof objective.id === "string" && objective.id !== "" ? objective.id : objective.description)); + + return { + value, + unit: "ratio", + details: `objectives=${objectives.length} unmet=${unmet.length} observations=${observations.length} unused=${unused.length} distinctStatesVisited=${distinctStatesVisited} (from ${source})${unmetDescriptions.length === 0 ? "" : `; unmet objectives: ${unmetDescriptions.join(", ")}`}`, + }; +--- + + diff --git a/.github/workflows/shared/graders/exploration-error.md b/.github/workflows/shared/graders/exploration-error.md index 4c8bb8eba7a..7c3942135b2 100644 --- a/.github/workflows/shared/graders/exploration-error.md +++ b/.github/workflows/shared/graders/exploration-error.md @@ -6,8 +6,8 @@ graders: # from distinct state_change event refs, falling back to the declared # states[] count when no state_change events are recorded. Runs with all # objectives satisfied score 0 (no exploration error to attribute). This is - # the complement of exploitation-error (not yet implemented), which covers - # runs that had enough evidence but failed anyway. Lower is better: fewer + # the complement of exploitation-error, which covers runs that had + # enough evidence but failed anyway. Lower is better: fewer # unmet objectives attributable to insufficient search. exploration-error: name: Exploration Error @@ -86,8 +86,8 @@ refs across events[] of kind "state_change"; when no such events are recorded it falls back to the declared states[] count. Runs with all objectives satisfied score 0 -- there is no exploration error to attribute, since exploration failures only apply to failed runs. This is the -complement of exploitation-error (not yet implemented), which will cover -runs that had enough evidence but misused it. Reports not-applicable +complement of exploitation-error, which covers runs that had enough +evidence but misused it. Reports not-applicable (passed: null) when no objectives are declared, or when neither state_change events nor declared states are present in the trace. --> diff --git a/actions/setup/js/trace_graders.test.cjs b/actions/setup/js/trace_graders.test.cjs index 720a775dfb3..357472ba551 100644 --- a/actions/setup/js/trace_graders.test.cjs +++ b/actions/setup/js/trace_graders.test.cjs @@ -104,6 +104,24 @@ function runExplorationError(trace) { }); } +const exploitationErrorScriptMatch = fs.readFileSync(path.join(__dirname, "../../../.github/workflows/shared/graders/exploitation-error.md"), "utf8").match(/script: \|\n([\s\S]*?)\n^---\s*$/m); +if (!exploitationErrorScriptMatch?.[1]) { + throw new Error("unable to extract exploitation-error grader script"); +} +const exploitationErrorScript = exploitationErrorScriptMatch[1] + .split("\n") + .map(line => line.slice(6)) + .join("\n"); + +function runExploitationError(trace) { + return runCustomGrader("exploitation-error", exploitationErrorScript, makeTrace(trace), { + name: "Exploitation Error", + unit: "ratio", + direction: "lower_is_better", + source: "inline", + }); +} + const skillConstraintCoverageScriptMatch = fs.readFileSync(path.join(__dirname, "../../../.github/workflows/shared/graders/skill-constraint-coverage.md"), "utf8").match(/script: \|\n([\s\S]*?)\n^---\s*$/m); if (!skillConstraintCoverageScriptMatch?.[1]) { throw new Error("unable to extract skill-constraint-coverage grader script"); @@ -757,6 +775,121 @@ describe("trace_graders", () => { }); }); + describe("exploitation-error custom grader", () => { + it("reports unavailable when no objectives are declared", () => { + const result = runExploitationError({ trajectoryIR: { observations: [{ id: "obs-1" }] } }); + + expect(result.value).toBeNull(); + expect(result.passed).toBeNull(); + expect(result.message).toContain("no declared objectives"); + }); + + it("returns zero when all objectives are already satisfied", () => { + const result = runExploitationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Read all files", satisfiedAtEventIndex: 0 }], + }, + }); + + expect(result.value).toBe(0); + expect(result.details).toContain("all objectives satisfied"); + }); + + it("is unavailable without state_change events or declared states", () => { + const result = runExploitationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }], + observations: [{ id: "obs-1" }], + }, + }); + + expect(result.value).toBeNull(); + expect(result.passed).toBeNull(); + expect(result.message).toContain("no state_change events or declared states"); + }); + + it("is unavailable when the trace records no observations", () => { + const result = runExploitationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }], + states: [{ id: "a" }], + }, + }); + + expect(result.value).toBeNull(); + expect(result.passed).toBeNull(); + expect(result.message).toContain("no observations"); + }); + + it("defers to exploration-error when exploration was insufficient", () => { + const result = runExploitationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }], + events: [ + { kind: "state_change", ref: "repo-root" }, + { kind: "state_change", ref: "repo-readme" }, + ], + observations: [{ id: "obs-1", consumedByActionIds: ["act-1"] }], + }, + }); + + expect(result.value).toBeNull(); + expect(result.passed).toBeNull(); + expect(result.message).toContain("exploration was insufficient"); + expect(result.message).toContain("exploration-error"); + }); + + it("scores the unused fraction of observations when exploration was sufficient", () => { + const result = runExploitationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }], + events: [ + { kind: "state_change", ref: "repo-root" }, + { kind: "state_change", ref: "repo-root" }, + ], + observations: [ + { id: "obs-1", consumedByActionIds: ["act-1"] }, + { id: "obs-2", consumedByActionIds: [] }, + ], + }, + }); + + expect(result.value).toBeCloseTo(0.5); + expect(result.details).toContain("observations=2 unused=1"); + expect(result.details).toContain("distinctStatesVisited=1"); + expect(result.details).toContain("unmet objectives: goal"); + }); + + it("falls back to declared states[] when no state_change events exist", () => { + const result = runExploitationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }], + states: [{ id: "a" }, { id: "b" }], + observations: [{ id: "obs-1" }, { id: "obs-2" }], + }, + }); + + expect(result.value).toBe(1); + expect(result.details).toContain("from declared states[]"); + }); + + it("prefers the objective-bearing IR candidate over unrelated agentOutput observations", () => { + const result = runExploitationError({ + trajectoryIR: { + objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }], + states: [{ id: "a" }], + observations: [{ id: "obs-1", consumedByActionIds: ["act-1"] }], + }, + agentOutput: { + observations: [{ id: "obs-x" }], + }, + }); + + expect(result.value).toBe(0); + expect(result.details).toContain("observations=1 unused=0"); + }); + }); + describe("skill-constraint-coverage custom grader", () => { it("reports full coverage when every constraint is exercised and succeeds", () => { const result = runSkillConstraintCoverage(