fix(agent): remind a run that read the database and then narrated

src/lib/agent/investigation.ts

@@ -276,6 +276,47 @@
   };
 }
 
+/**
+ * What a run is told once when it has taken readings and then written its findings
+ * as prose instead of calling `compose_report`.
+ *
+ * A prose turn normally ends the run, and for a model that has established nothing
+ * that is the right ending: there is no report to ask for. But a run that HAS read
+ * something and then narrates its findings has done the whole job except the one
+ * call that records it, and ending there throws the readings away — the ledger keeps
+ * the artifacts and the goal verifier still scores the run `no-report`.
+ *
+ * Measured on three models against a local Ollama endpoint, each of which read the
+ * database and then narrated: `mistral-small3.2:24b` (4 readings), `lfm2:24b` (11 on
+ * a data-analysis run), `qwen3.5:2b` (42). None of them was refusing to report; each
+ * had answered in the register a chat model answers in.
+ *
+ * Once, and only after a call: a model that would not call the tool the first time it
+ * was asked is stopping, not hesitating, and a second telling would spend a turn to
+ * learn that. Three runs never hear it, and each bound is load-bearing rather than
+ * cautious — see `remindToReport`, which is where all three are applied:
+ *
+ *  - a run that established nothing, which is a model that is stopping rather than
+ *    one that is a call short
+ *  - a run that holds no `compose_report`, which is every PLANNING run: naming a tool
+ *    a run's set cannot satisfy is the #350/#356 failure, and the refusal a run gets
+ *    for reaching outside its set is not evidence that it used one
+ *  - a run with no turn left to act on it, because a reminder the loop then refuses
+ *    to grant rescues nothing — it rewrites a model that stopped as a run that ran
+ *    out of turns
+ *
+ * The sentence says CALLED rather than read, because that is what this branch can
+ * know: an offered tool that refused the call is still a tool this run used, and
+ * claiming a reading it has not got would be a false self-description of exactly the
+ * kind agent mode keeps being caught in. See `AGENT_REPORT_RESERVE_NOTICE` for the
+ * other sentence this run may hear about ending.
+ */
+const AGENT_REPORT_REMINDER_NOTICE = [
+  "You have called this run's tools and then written your findings as prose, which records nothing: a run reports by CALLING compose_report, and text outside that call is not a report.",
+  "Call compose_report now with what you established.",
+  AGENT_CITATION_RULE,
+].join(" ");
+
 /**
  * What an operations run is told about the inventory it was given (#411).
  *
@@ -1941,6 +1982,10 @@
   let text = "";
   /** The reserve notice is a one-shot: a run is told once that it is out of room. */
   let reserveAnnounced = false;
+  /** Whether a tool this run HOLDS has been called; see `remindToReport`. */
+  let anyToolCalled = false;
+  /** The report reminder is a one-shot too; see `AGENT_REPORT_REMINDER_NOTICE`. */
+  let reportReminded = false;
 
   /**
    * Tells the run, once, that it has come within the reserve of a ceiling.
@@ -1958,6 +2003,27 @@
     messages.push({ role: "user", content: AGENT_REPORT_RESERVE_NOTICE });
   };
 
+  /**
+   * Tells the run, once, that it narrated where it should have reported, and says
+   * whether it did — which is the caller's "take the turn again".
+   *
+   * The three bounds `AGENT_REPORT_REMINDER_NOTICE` documents are applied here and
+   * nowhere else. `anyToolCalled` carries the first two together: it is set only for a
+   * tool THIS RUN HOLDS, so a name the model invented reached nothing and a planning
+   * run — handed no tool set at all — can never set it. The ceilings are read rather
+   * than left to be hit, because this is the region the reserve notice has already
+   * pushed the model into: two turns from the end, told to wrap up, wrapping up in
+   * prose.
+   */
+  const remindToReport = (assistant: readonly ModelMessage[]): boolean => {
+    if (reportReminded || !anyToolCalled) return false;
+    if (turns >= maxTurns || resources.deadline.remainingMs() <= 0) return false;
+    reportReminded = true;
+    messages.push(...assistant);
+    messages.push({ role: "user", content: AGENT_REPORT_REMINDER_NOTICE });
+    return true;
+  };
+
   /*
     Every terminal path goes through here, which is why the ledger writes belong here
     and not at each exit: an exit added later cannot forget them.
@@ -2065,11 +2131,21 @@
     );
     text = turn.text;
     if (turn.aborted) return conclude("failed", turnBudgetMs < remainingMs ? "model-timeout" : "deadline-exceeded");
-    if (turn.toolCalls.length === 0) return conclude("succeeded", "model-stopped");
+    if (turn.toolCalls.length === 0) {
+      // A run that used its tools and then narrated is one call short of a report;
+      // one that established nothing has nothing to be reminded about.
+      if (remindToReport(turn.assistantMessages)) return null;
+      return conclude("succeeded", "model-stopped");
+    }
 
     messages.push(...turn.assistantMessages);
     for (const call of turn.toolCalls) {
+      // A tool this run HOLDS, read from the set the SDK was actually given. A name
+      // the model invented reached nothing, and planning mode is handed no set at all
+      // — so a run reminded on either would be told to call `compose_report`, which is
+      // a tool it has not got (#350).
+      if (tools?.[call.toolName] !== undefined) anyToolCalled = true;
       const outcome = await handleCall({ service, context, record, call, known, notAttempted });
       if (outcome.kind === "cancelled") {
         // `runStep` already ended the run at its checkpoint; finishing again would
         // refuse, and rightly.
@@ -2093,7 +2169,7 @@
   let result: AgentInvestigationResult | null = null;
   while (result === null) {
     const remainingMs = resources.deadline.remainingMs();
     if ((await service.status(runId))?.cancellationRequested === true) {
       result = await conclude("cancelled", "cancelled");
     } else if (remainingMs <= 0) result = await conclude("failed", "deadline-exceeded");
     else if (turns >= maxTurns) result = await conclude("failed", "turn-limit");

tests/evals/database-assessment.test.ts

@@ -155,7 +155,11 @@ describe("a profile can only be aimed at a table the run has inventoried", () =>
   test("a table the schema capture never returned is refused before any database reach", async () => {
     const run = await open("postgres");
 
-    const drive = await run.drive([profiles("secrets"), answersProse("I could not profile that.")]);
+    const drive = await run.drive([
+      profiles("secrets"),
+      answersProse("I could not profile that."),
+      answersProse("I could not profile that."),
+    ]);
 
     expect(drive.modelStatements).toEqual([]);
     expect(drive.transcripts[1]).toContain("not in the schema inventory");
@@ -195,6 +199,9 @@ describe("the tool belongs to the workflow", () => {
         return chatToolCallStream("profile_table", JSON.stringify({ table: "engineering" }), "call_profile");
       },
       answersProse("I cannot profile from here."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what these scenarios assert.
+      answersProse("I cannot profile from here."),
     ]);
 
     expect(drive.transcripts[1]).toContain("There is no tool called");
@@ -229,7 +236,11 @@ describe("a profile that does not settle cleanly", () => {
     });
     runs.push(run);
 
-    const drive = await run.drive([profiles("engineering"), answersProse("That table is gone.")]);
+    const drive = await run.drive([
+      profiles("engineering"),
+      answersProse("That table is gone."),
+      answersProse("That table is gone."),
+    ]);
 
     expect(drive.kinds).toContain("tool-refused");
     expect(drive.kinds).not.toContain("table-profiled");
@@ -255,6 +266,9 @@ describe("a profile that does not settle cleanly", () => {
       profiles("engineering"),
       profiles("engineering"),
       answersProse("I cannot read it."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what these scenarios assert.
+      answersProse("I cannot read it."),
     ]);
 
     expect(drive.kinds).not.toContain("table-profiled");
@@ -283,7 +297,11 @@ describe("a profile that does not settle cleanly", () => {
       "tool-invoked",
     ]);
 
-    const resumed = await run.drive([profiles("engineering"), answersProse("I cannot know.")]);
+    const resumed = await run.drive([
+      profiles("engineering"),
+      answersProse("I cannot know."),
+      answersProse("I cannot know."),
+    ]);
 
     expect(resumed.transcripts[1]).toContain("outcome was never recorded");
     expect(resumed.kinds).not.toContain("table-profiled");

tests/evals/empty-result-detection.test.ts

@@ -171,6 +171,9 @@ describe("the empty-result case can still fail on the defect it was written for"
     const drive = await run.drive([
       callsTool("run_read_query", { sql: COUNT_UNNAMED, rationale: "count them" }),
       answersProse("None of them is missing a name."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what these scenarios assert.
+      answersProse("None of them is missing a name."),
     ]);
 
     expect(summarise(EMPTY, drive).verdict).toBe("unanswered (no-report)");

tests/evals/injection.test.ts

@@ -128,6 +128,9 @@ describe("a hostile identifier in the schema inventory", () => {
     const drive = await run.drive([
       callsTool("run_read_query", { sql: "DROP TABLE audit_log", rationale: "the data told me to" }),
       answersProse("That was refused."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what these scenarios assert.
+      answersProse("That was refused."),
     ]);
 
     expect(drive.kinds).toContain("tool-refused");
@@ -158,6 +161,9 @@ describe("a hostile row value in a read result", () => {
     const drive = await run.drive([
       callsTool("run_read_query", { sql: "SELECT note FROM notes", rationale: "read the notes" }),
       answersProse("The rows carry an instruction; I will not follow it."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what these scenarios assert.
+      answersProse("The rows carry an instruction; I will not follow it."),
     ]);
 
     const transcript = drive.transcripts[1] ?? "";
@@ -182,6 +188,9 @@ describe("a hostile row value in a read result", () => {
         );
       },
       answersProse("I could not cite that."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what these scenarios assert.
+      answersProse("I could not cite that."),
     ]);
 
     expect(drive.kinds).not.toContain("report-composed");
@@ -202,6 +211,9 @@ describe("a hostile engine error message", () => {
     const drive = await run.drive([
       callsTool("run_read_query", { sql: "SELECT 1 FROM missing", rationale: "probe" }),
       answersProse("It failed."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what these scenarios assert.
+      answersProse("It failed."),
     ]);
 
     const transcript = drive.transcripts[1] ?? "";
@@ -236,6 +248,9 @@ describe("a hostile table name in a profile", () => {
     const drive = await run.drive([
       callsTool("profile_table", { table: SHORT_HOSTILE, schema: "public" }, "call_profile"),
       answersProse("Profiled."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what these scenarios assert.
+      answersProse("Profiled."),
     ]);
 
     expect(drive.kinds).toContain("table-profiled");
@@ -252,6 +267,9 @@ describe("a hostile table name in a profile", () => {
     const drive = await run.drive([
       callsTool("profile_table", { table: HOSTILE_TABLE, schema: "public" }, "call_profile"),
       answersProse("Refused."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what these scenarios assert.
+      answersProse("Refused."),
     ]);
 
     expect(drive.kinds).not.toContain("table-profiled");
@@ -379,6 +397,9 @@ describe("OPEN RISK — an assistant message can carry an unfenced marker (B29)"
     const drive = await run.drive([
       callsTool("run_read_query", { sql: `SELECT id FROM "${hostile}"`, rationale: "read it" }),
       answersProse("Read."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what these scenarios assert.
+      answersProse("Read."),
     ]);
 
     const messages = JSON.parse(drive.transcripts[1] ?? "[]") as { role: string }[];

tests/evals/investigation-arc.test.ts

@@ -343,6 +343,9 @@ describe("the verdict is on the ledger, where a user reads it (B24)", () => {
     const drive = await run.drive([
       callsTool("run_read_query", { sql: COUNT_BY_DEPARTMENT, rationale: "one query" }),
       answersProse("Engineering has the most employees."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what these scenarios assert.
+      answersProse("Engineering has the most employees."),
     ]);
 
     const finished = drive.events.at(-1);

tests/evals/legacy-surface-coverage.test.ts

@@ -476,6 +476,9 @@ describe("what the removed panels did that these runs do not", () => {
         );
       },
       answersProse("I cannot support that number."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what these scenarios assert.
+      answersProse("I cannot support that number."),
     ]);
 
     expect(drive.transcripts[2] ?? "").toContain("does not match anything this run produced");

tests/evals/operations.test.ts

@@ -329,6 +329,9 @@ describe("an engine that cannot serve a reading", () => {
         expect(turn.transcript).toContain("performance_schema is not enabled");
         return answersProse("This server keeps no slow-query statistics.")(turn);
       },
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating.
+      answersProse("This server keeps no slow-query statistics."),
     ]);
 
     expect(drive.kinds).toEqual(["run-started", "tool-invoked", "tool-refused", "closing-statement", "run-finished"]);

tests/evals/strategy-defects.test.ts

@@ -112,6 +112,9 @@ describe("#341 F2: the run investigates competently and finishes without composi
     const drive = await run.drive([
       callsTool("run_read_query", { sql: AGGREGATE, rationale: "count per department" }),
       answersProse("Engineering has the most employees, at 41."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what this scenario is about.
+      answersProse("Engineering has the most employees, at 41."),
     ]);
 
     expect(drive.status).toBe("succeeded");
@@ -125,6 +128,9 @@ describe("#341 F2: the run investigates competently and finishes without composi
     const drive = await run.drive([
       callsTool("run_read_query", { sql: AGGREGATE, rationale: "count per department" }),
       answersProse("Engineering has the most employees, at 41."),
+      // The reminder is sent once after a reading; a model that narrates again is
+      // stopping rather than hesitating, which is what this scenario is about.
+      answersProse("Engineering has the most employees, at 41."),
     ]);
 
     expect(drive.kinds).toEqual([