Implement exploration-error trajectory grader

.github/workflows/shared/graders/README.md

@@ -43,7 +43,7 @@ to `Implemented` in the same PR that adds `shared/graders/<id>.md`.
 |---|---|---|---|
 | 1 | `policy-near-miss` | Policy/guard predicates | Implemented |
 | 2 | `skill-constraint-coverage` | Precompiled constraints | Implemented |
-| 3 | `exploration-error` | State/task model | Not started |
+| 3 | `exploration-error` | State/task model | Implemented |
 | 4 | `exploitation-error` | State/task model | Not started |
 | 12 | `tool-output-consumption-rate` | Provenance/reference IDs | Not started |
 | 13 | `end-to-end-lineage-completeness` | Provenance graph | Not started |

.github/workflows/shared/graders/exploration-error.md

@@ -0,0 +1,93 @@
+---
+graders:
+  # For runs that left one or more declared objectives unsatisfied, measures
+  # whether the failure was due to insufficient search: 1 - (observations /
+  # distinctStatesVisited), clamped to [0, 1]. distinctStatesVisited comes
+  # from distinct state_change event refs, falling back to the declared
+  # states[] count when no state_change events are recorded. Runs with all
+  # objectives satisfied score 0 (no exploration error to attribute). This is
+  # the complement of exploitation-error (not yet implemented), which covers
+  # runs that had enough evidence but failed anyway. Lower is better: fewer
+  # unmet objectives attributable to insufficient search.
+  exploration-error:
+    name: Exploration Error
+    unit: ratio
+    direction: lower_is_better
+    min: 0.0
+    max: 1.0
+    script: |
+      const isRecord = value => value !== null && typeof value === "object" && !Array.isArray(value);
+      const candidates = [
+        trace.trajectoryIR,
+        trace.trajectoryIr,
+        trace.ir,
+        isRecord(trace.agentOutput) ? trace.agentOutput.trajectoryIR : null,
+        isRecord(trace.agentOutput) ? trace.agentOutput.trajectoryIr : null,
+        isRecord(trace.agentOutput) ? trace.agentOutput.trajectory : null,
+        isRecord(trace.agentOutput) ? trace.agentOutput : null,
+      ].filter(isRecord);
+
+      const candidate =
+        candidates.find(value => Array.isArray(value.objectives) && value.objectives.some(isRecord)) ??
+        candidates.find(value =>
+          Array.isArray(value.objectives) ||
+          Array.isArray(value.events) ||
+          Array.isArray(value.states) ||
+          Array.isArray(value.observations)
+        ) ??
+        null;
+      const objectives = (candidate && Array.isArray(candidate.objectives) ? candidate.objectives : []).filter(isRecord);
+      if (objectives.length === 0) {
+        return { value: null, unit: "ratio", passed: null, message: "not applicable: no declared objectives in the trace" };
+      }
+
+      const unmet = objectives.filter(objective => objective.satisfiedAtEventIndex === null || objective.satisfiedAtEventIndex === undefined);
+      if (unmet.length === 0) {
+        return { value: 0, unit: "ratio", details: `objectives=${objectives.length} unmet=0; all objectives satisfied` };
+      }
+
+      const events = (candidate && Array.isArray(candidate.events) ? candidate.events : []).filter(isRecord);
+      const states = (candidate && Array.isArray(candidate.states) ? candidate.states : []).filter(isRecord);
+      const observations = (candidate && Array.isArray(candidate.observations) ? candidate.observations : []).filter(isRecord);
+
+      const stateChangeEvents = events.filter(event => event.kind === "state_change");
+      let distinctStatesVisited = 0;
+      let source = "";
+      if (stateChangeEvents.length > 0) {
+        const visited = new Set(stateChangeEvents.map(event => (typeof event.ref === "string" ? event.ref : JSON.stringify(event.ref))));
+        distinctStatesVisited = visited.size;
+        source = "state_change events";
+      } else if (states.length > 0) {
+        distinctStatesVisited = states.length;
+        source = "declared states[]";
+      } else {
+        return { value: null, unit: "ratio", passed: null, message: "not applicable: no state_change events or declared states in the trace" };
+      }
+
+      const value = helpers.clamp(1 - observations.length / distinctStatesVisited, 0, 1);
+      const unmetDescriptions = unmet.slice(0, 5).map(objective => (typeof objective.id === "string" && objective.id !== "" ? objective.id : objective.description));
+
+      return {
+        value,
+        unit: "ratio",
+        details: `objectives=${objectives.length} unmet=${unmet.length} observations=${observations.length} distinctStatesVisited=${distinctStatesVisited} (from ${source})${unmetDescriptions.length === 0 ? "" : `; unmet objectives: ${unmetDescriptions.join(", ")}`}`,
+      };
+---
+
+<!--
+exploration-error attributes objective failure to insufficient search: for
+runs that left one or more declared objectives unsatisfied
+(satisfiedAtEventIndex null/undefined), it computes
+1 - (observations / distinctStatesVisited), clamped to [0, 1]. A low
+observation-to-state ratio (few observations gathered across the states the
+run visited) yields a score near 1 -- the run likely failed because it never
+gathered enough evidence. distinctStatesVisited is the count of distinct
+refs across events[] of kind "state_change"; when no such events are
+recorded it falls back to the declared states[] count. Runs with all
+objectives satisfied score 0 -- there is no exploration error to attribute,
+since exploration failures only apply to failed runs. This is the
+complement of exploitation-error (not yet implemented), which will cover
+runs that had enough evidence but misused it. Reports not-applicable
+(passed: null) when no objectives are declared, or when neither
+state_change events nor declared states are present in the trace.
+-->

actions/setup/js/trace_graders.test.cjs

@@ -86,6 +86,24 @@ function runPolicyNearMiss(trace) {
   });
 }
 
+const explorationErrorScriptMatch = fs.readFileSync(path.join(__dirname, "../../../.github/workflows/shared/graders/exploration-error.md"), "utf8").match(/script: \|\n([\s\S]*?)\n^---\s*$/m);
+if (!explorationErrorScriptMatch?.[1]) {
+  throw new Error("unable to extract exploration-error grader script");
+}
+const explorationErrorScript = explorationErrorScriptMatch[1]
+  .split("\n")
+  .map(line => line.slice(6))
+  .join("\n");
+
+function runExplorationError(trace) {
+  return runCustomGrader("exploration-error", explorationErrorScript, makeTrace(trace), {
+    name: "Exploration Error",
+    unit: "ratio",
+    direction: "lower_is_better",
+    source: "inline",
+  });
+}
+
 const skillConstraintCoverageScriptMatch = fs.readFileSync(path.join(__dirname, "../../../.github/workflows/shared/graders/skill-constraint-coverage.md"), "utf8").match(/script: \|\n([\s\S]*?)\n^---\s*$/m);
 if (!skillConstraintCoverageScriptMatch?.[1]) {
   throw new Error("unable to extract skill-constraint-coverage grader script");
@@ -648,6 +666,97 @@ describe("trace_graders", () => {
     });
   });
 
+  describe("exploration-error custom grader", () => {
+    it("reports unavailable when no objectives are declared", () => {
+      const result = runExplorationError({ trajectoryIR: { events: [{ kind: "state_change", ref: "a" }] } });
+
+      expect(result.value).toBeNull();
+      expect(result.passed).toBeNull();
+      expect(result.message).toContain("no declared objectives");
+    });
+
+    it("returns zero when all objectives are already satisfied", () => {
+      const result = runExplorationError({
+        trajectoryIR: {
+          objectives: [{ id: "goal", description: "Read all files", satisfiedAtEventIndex: 0 }],
+        },
+      });
+
+      expect(result.value).toBe(0);
+      expect(result.details).toContain("all objectives satisfied");
+    });
+
+    it("prefers the objective-bearing IR candidate over unrelated agentOutput observations", () => {
+      const result = runExplorationError({
+        trajectoryIR: {
+          objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }],
+          events: [{ kind: "state_change", ref: "repo-root" }],
+        },
+        agentOutput: {
+          observations: [{ id: "obs-1" }],
+        },
+      });
+
+      expect(result.value).toBe(1);
+      expect(result.details).toContain("observations=0");
+    });
+
+    it("counts distinct state refs from state_change events", () => {
+      const result = runExplorationError({
+        trajectoryIR: {
+          objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }],
+          events: [
+            { kind: "state_change", ref: "repo-root" },
+            { kind: "state_change", ref: "repo-readme" },
+            { kind: "state_change", ref: "repo-root" },
+          ],
+          observations: [{ id: "obs-1" }],
+        },
+      });
+
+      expect(result.value).toBeCloseTo(0.5);
+      expect(result.details).toContain("distinctStatesVisited=2");
+    });
+
+    it("falls back to declared states[] when no state_change events exist", () => {
+      const result = runExplorationError({
+        trajectoryIR: {
+          objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }],
+          states: [{ id: "a" }, { id: "b" }],
+          observations: [{ id: "obs-1" }],
+        },
+      });
+
+      expect(result.value).toBeCloseTo(0.5);
+      expect(result.details).toContain("from declared states[]");
+    });
+
+    it("clamps the ratio at zero when observations exceed the visited-state count", () => {
+      const result = runExplorationError({
+        trajectoryIR: {
+          objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }],
+          states: [{ id: "a" }],
+          observations: [{ id: "obs-1" }, { id: "obs-2" }, { id: "obs-3" }],
+        },
+      });
+
+      expect(result.value).toBe(0);
+    });
+
+    it("is unavailable without state_change events or declared states", () => {
+      const result = runExplorationError({
+        trajectoryIR: {
+          objectives: [{ id: "goal", description: "Inspect repo", satisfiedAtEventIndex: null }],
+          observations: [{ id: "obs-1" }],
+        },
+      });
+
+      expect(result.value).toBeNull();
+      expect(result.passed).toBeNull();
+      expect(result.message).toContain("no state_change events or declared states");
+    });
+  });
+
   describe("skill-constraint-coverage custom grader", () => {
     it("reports full coverage when every constraint is exercised and succeeds", () => {
       const result = runSkillConstraintCoverage(