Implement policy-near-miss trajectory grader (Tier 2, Rank 1)

.github/workflows/shared/graders/README.md

@@ -41,7 +41,7 @@ to `Implemented` in the same PR that adds `shared/graders/<id>.md`.
 
 | Rank | Grader ID | Runtime requirement | Status |
 |---|---|---|---|
-| 1 | `policy-near-miss` | Policy/guard predicates | Not started |
+| 1 | `policy-near-miss` | Policy/guard predicates | Implemented |
 | 2 | `skill-constraint-coverage` | Precompiled constraints | Not started |
 | 3 | `exploration-error` | State/task model | Not started |
 | 4 | `exploitation-error` | State/task model | Not started |

.github/workflows/shared/graders/policy-near-miss.md

@@ -0,0 +1,73 @@
+---
+graders:
+  # Detects "successful" traces (traces that emitted at least one safe_output
+  # event) which nonetheless left one or more guard/policy-shaped objectives
+  # unsatisfied -- i.e. traces that reached the correct outcome without
+  # performing required checks. Guard-shaped objectives are matched by
+  # keyword against objectives[].description. Lower is better: fewer
+  # near-misses.
+  policy-near-miss:
+    name: Policy Near-Miss Rate
+    unit: ratio
+    direction: lower_is_better
+    min: 0.0
+    max: 1.0
+    script: |
+      const isRecord = value => value !== null && typeof value === "object" && !Array.isArray(value);
+      const candidates = [
+        trace.trajectoryIR,
+        trace.trajectoryIr,
+        trace.ir,
+        isRecord(trace.agentOutput) ? trace.agentOutput.trajectoryIR : null,
+        isRecord(trace.agentOutput) ? trace.agentOutput.trajectoryIr : null,
+        isRecord(trace.agentOutput) ? trace.agentOutput.trajectory : null,
+        isRecord(trace.agentOutput) ? trace.agentOutput : null,
+      ].filter(isRecord);
+
+      const candidate = candidates.find(value => Array.isArray(value.events) && Array.isArray(value.objectives));
+      const events = candidate?.events ?? candidates.find(value => Array.isArray(value.events))?.events ?? [];
+      const objectives = candidate?.objectives ?? candidates.find(value => Array.isArray(value.objectives))?.objectives ?? [];
+
+      if (objectives.length === 0) {
+        return { value: null, unit: "ratio", passed: null, message: "not applicable: no declared objectives in the trace" };
+      }
+
+      const reachedOutcome = events.some(event => isRecord(event) && event.kind === "safe_output");
+      if (!reachedOutcome) {
+        return { value: null, unit: "ratio", passed: null, message: "not applicable: no safe_output event; run did not reach an outcome" };
+      }
+
+      // Guard/policy-shaped objectives: matched by keyword against the
+      // objective's description, not by an explicit "guard" flag, since the
+      // IR does not distinguish guard objectives from other objectives.
+      const guardKeywords = ["check", "verify", "verification", "policy", "approval", "approve", "guard", "confirm", "authorize", "authorization"];
+      const guardPattern = new RegExp(`\\b(${guardKeywords.join("|")})\\b`, "i");
+      const guardObjectives = objectives.filter(objective => isRecord(objective) && typeof objective.description === "string" && guardPattern.test(objective.description));
+
+      if (guardObjectives.length === 0) {
+        return { value: null, unit: "ratio", passed: null, message: "not applicable: no guard/policy-shaped objectives in the trace" };
+      }
+
+      const unmet = guardObjectives.filter(objective => objective.satisfiedAtEventIndex === null || objective.satisfiedAtEventIndex === undefined);
+      const value = helpers.ratio(unmet.length, guardObjectives.length);
+      const unmetDescriptions = unmet.slice(0, 5).map(objective => (typeof objective.id === "string" && objective.id !== "" ? objective.id : objective.description));
+
+      return {
+        value,
+        unit: "ratio",
+        details: `guardObjectives=${guardObjectives.length} unmet=${unmet.length}${unmetDescriptions.length === 0 ? "" : `; unmet guards: ${unmetDescriptions.join(", ")}`}`,
+      };
+---
+
+<!--
+policy-near-miss flags "successful" traces (at least one safe_output event
+emitted) that nonetheless left one or more guard/policy-shaped objectives
+unsatisfied -- runs that reached the correct outcome without performing a
+required check. Guard-shaped objectives are identified by keyword match
+against objectives[].description (e.g. "check", "verify", "policy",
+"approval"); it does not evaluate objectives that aren't guard-shaped
+(see objective-coverage, not yet implemented, for that). Reports
+not-applicable (passed: null) rather than a fabricated value when no
+objectives are declared, no outcome was reached, or no guard-shaped
+objectives exist in the trace.
+-->

actions/setup/js/trace_graders.test.cjs

@@ -68,6 +68,24 @@ function makeTrace(overrides = {}) {
   };
 }
 
+const policyNearMissScriptMatch = fs.readFileSync(path.join(__dirname, "../../../.github/workflows/shared/graders/policy-near-miss.md"), "utf8").match(/script: \|\n([\s\S]*?)\n^---\s*$/m);
+if (!policyNearMissScriptMatch?.[1]) {
+  throw new Error("unable to extract policy-near-miss grader script");
+}
+const policyNearMissScript = policyNearMissScriptMatch[1]
+  .split("\n")
+  .map(line => line.slice(6))
+  .join("\n");
+
+function runPolicyNearMiss(trace) {
+  return runCustomGrader("policy-near-miss", policyNearMissScript, makeTrace(trace), {
+    name: "Policy Near-Miss Rate",
+    unit: "ratio",
+    direction: "lower_is_better",
+    source: "inline",
+  });
+}
+
 describe("trace_graders", () => {
   describe("buildGradersSummaryBody", () => {
     it("renders all computed grader values without emojis", () => {
@@ -566,6 +584,46 @@ describe("trace_graders", () => {
     });
   });
 
+  describe("policy-near-miss custom grader", () => {
+    it("discovers objectives after an events-only candidate", () => {
+      const result = runPolicyNearMiss({
+        trajectoryIR: { events: [{ kind: "safe_output" }] },
+        ir: { objectives: [{ id: "guard", description: "Verify approval", satisfiedAtEventIndex: null }] },
+      });
+
+      expect(result.value).toBe(1);
+      expect(result.details).toContain("guardObjectives=1 unmet=1");
+    });
+
+    it("identifies guard objectives and treats event index zero as satisfied", () => {
+      const result = runPolicyNearMiss({
+        trajectoryIR: {
+          events: [{ kind: "safe_output" }],
+          objectives: [
+            { id: "met", description: "Check authorization", satisfiedAtEventIndex: 0 },
+            { id: "unmet", description: "Verification required", satisfiedAtEventIndex: null },
+            { id: "not-a-guard", description: "Complete checkbox", satisfiedAtEventIndex: null },
+          ],
+        },
+      });
+
+      expect(result.value).toBeCloseTo(0.5);
+      expect(result.details).toContain("guardObjectives=2 unmet=1");
+    });
+
+    it.each([
+      ["no objectives", { trajectoryIR: { events: [{ kind: "safe_output" }], objectives: [] } }],
+      ["no outcome", { trajectoryIR: { events: [], objectives: [{ description: "Check approval" }] } }],
+      ["no guard objective", { trajectoryIR: { events: [{ kind: "safe_output" }], objectives: [{ description: "Write report" }] } }],
+    ])("normalizes %s as unavailable", (_name, trace) => {
+      const result = runPolicyNearMiss(trace);
+
+      expect(result.value).toBeNull();
+      expect(result.passed).toBeNull();
+      expect(result.status).toBe("unavailable");
+    });
+  });
+
   // --- Hostile data ---
   describe("hostile data handling", () => {
     it("handles hostile strings in tool call names", () => {