Emit eval outcomes and post V4 results

README.md

@@ -499,11 +499,13 @@ kcap eval --skip efficiency <sessionId>    # run everything except efficiency
 kcap eval --list-questions                 # print the question taxonomy
 ```
 
-Output is a per-category + overall score (1-5, with `pass`/`warn`/`fail` verdicts), with a specific finding and supporting evidence per question. The aggregate is also persisted back to the session's stream as a `SessionEvalCompleted` event, so past evaluations can be queried from the dashboard or used to track quality trends across sessions.
+Each question resolves to one of three outcomes: `assessed` (scored 1-5, with a `pass`/`warn`/`fail` verdict, marked `✓`/`!`/`✗`), `insufficient_evidence` (the judge lacked what it needed to answer — marked `?`, no score), or `not_applicable` (nothing the question asks about could have occurred in this session — marked `–`, no score). Category and overall scores are computed from assessed questions only; a category or run with no assessed question prints `Overall: not scored` rather than a fabricated number. Each row shows its finding and evidence, plus — when the judge's retrieval was measurably incomplete — an indented `coverage:` line naming what was truncated, unavailable, or cut short, and a `cites:` line for any evidence reference the judge grounded its finding in.
+
+The aggregate is persisted back to the session's stream as a `SessionEvalCompleted` event, so past evaluations can be queried from the dashboard or used to track quality trends across sessions.
 
 Expect ~1-3 minutes total depending on the model and session size — judges run sequentially.
 
-> **Server requirement:** `kcap eval` fetches its question catalog from the server (`GET /api/eval/catalog`) and posts results to `POST /api/sessions/{id}/evals/v3`. It fails fast with a clear error against a server that doesn't expose the catalog endpoint, so the server must be running a build that includes the eval catalog (Capacitor AI-9). Upgrade the server if `kcap eval` reports the catalog endpoint is unavailable.
+> **Server requirement:** `kcap eval` fetches its question catalog from the server (`GET /api/eval/catalog`) and posts results to `POST /api/sessions/{id}/evals/v4`. It fails fast with a clear error against a server that doesn't expose the catalog or v4 endpoint, so the server must be running a build that includes them. Upgrade the server if `kcap eval` reports the endpoint is unavailable. An older `kcap` build (predating outcomes and coverage) posts to `/evals/v3` instead — every question scored, no outcome or coverage — and the server accepts both.
 
 ### PR review with full context
 

src/Capacitor.Cli.Core/Eval/Contracts/EvalCategoryAssessment.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval.Contracts;
+
+/// <summary>Mirrors the server's <c>JudgeCategoryAssessment</c> field for field. <c>Score</c> is
+/// null when the category has no assessed question.</summary>
+public record EvalCategoryAssessment {
+    [JsonPropertyName("name")]      public required string Name { get; init; }
+    [JsonPropertyName("score")]     public int?    Score   { get; init; }
+    [JsonPropertyName("verdict")]   public string? Verdict { get; init; }
+    [JsonPropertyName("questions")] public List<EvalQuestionAssessment> Questions { get; init; } = [];
+}

src/Capacitor.Cli.Core/Eval/Contracts/EvalEvidenceCitation.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval.Contracts;
+
+/// <summary>Mirrors the server's <c>JudgeEvidenceCitation</c> field for field.</summary>
+public record EvalEvidenceCitation {
+    [JsonPropertyName("ref")] public required string Ref { get; init; }
+
+    [JsonPropertyName("digest")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Digest { get; init; }
+}

src/Capacitor.Cli.Core/Eval/Contracts/EvalEvidenceCoverage.cs

@@ -0,0 +1,26 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval.Contracts;
+
+/// <summary>Mirrors the server's <c>JudgeEvidenceCoverage</c> field for field. Null on a question
+/// means retrieval was not observed for it (the tools path, or a degraded server-side discovery);
+/// non-null is a strict account of what was consulted, cited and omitted.</summary>
+public record EvalEvidenceCoverage {
+    [JsonPropertyName("policy_version")] public required string PolicyVersion { get; init; }
+
+    [JsonPropertyName("scope_version")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? ScopeVersion { get; init; }
+
+    [JsonPropertyName("sources_consulted")]   public List<string> SourcesConsulted   { get; init; } = [];
+    [JsonPropertyName("sources_unavailable")] public List<string> SourcesUnavailable { get; init; } = [];
+    [JsonPropertyName("citations")]           public List<EvalEvidenceCitation> Citations { get; init; } = [];
+    [JsonPropertyName("omissions")]           public List<EvalEvidenceOmission> Omissions { get; init; } = [];
+
+    [JsonPropertyName("stop_reason")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? StopReason { get; init; }
+
+    [JsonIgnore]
+    public bool IsComplete => Omissions.Count == 0 && SourcesUnavailable.Count == 0 && StopReason is null;
+}

src/Capacitor.Cli.Core/Eval/Contracts/EvalEvidenceOmission.cs

@@ -0,0 +1,20 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval.Contracts;
+
+/// <summary>Mirrors the server's <c>JudgeEvidenceOmission</c> field for field.</summary>
+public record EvalEvidenceOmission {
+    [JsonPropertyName("kind")] public required string Kind { get; init; }
+
+    [JsonPropertyName("count")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? Count { get; init; }
+
+    [JsonPropertyName("ref")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Ref { get; init; }
+
+    [JsonPropertyName("detail")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? Detail { get; init; }
+}

src/Capacitor.Cli.Core/Eval/Contracts/EvalFailureCodes.cs

@@ -0,0 +1,9 @@
+namespace Capacitor.Cli.Core.Eval.Contracts;
+
+/// <summary>Coded failure reasons the CLI can attach to a judge invocation that produced no
+/// verdict — mirrors the codes the server's <c>EvalFailureCodes</c> accepts.</summary>
+static class EvalFailureCodes {
+    public const string JudgeTimeout       = "judge_timeout";
+    public const string ChatError          = "chat_error";
+    public const string VerdictParseFailed = "verdict_parse_failed";
+}

src/Capacitor.Cli.Core/Eval/Contracts/EvalOutcomes.cs

@@ -0,0 +1,13 @@
+namespace Capacitor.Cli.Core.Eval.Contracts;
+
+/// <summary>The three outcomes a judge can report for a question — mirrors the server's
+/// <c>Capacitor.Events.EvalOutcomes</c>. Anything else is a parse failure.</summary>
+static class EvalOutcomes {
+    public const string Assessed             = "assessed";
+    public const string InsufficientEvidence = "insufficient_evidence";
+    public const string NotApplicable        = "not_applicable";
+
+    public static readonly IReadOnlySet<string> All = new HashSet<string>(StringComparer.Ordinal) {
+        Assessed, InsufficientEvidence, NotApplicable
+    };
+}

src/Capacitor.Cli.Core/Eval/Contracts/EvalQuestionAssessment.cs

@@ -0,0 +1,30 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval.Contracts;
+
+/// <summary>The V4 per-question judge result. Mirrors the server's <c>JudgeQuestionAssessment</c>
+/// field for field, minus <c>trace_coverage</c> — a turn-navigating-only field the CLI never
+/// produces, so omitting it is wire-identical to always sending it null.
+///
+/// <para><c>Outcome</c> is nullable rather than defaulted: System.Text.Json constructs a type with
+/// `required` members through an uninitialized object, which skips property initializers, so a
+/// judge response that omits the field would silently bind an empty string instead of the intended
+/// default. <see cref="Capacitor.Cli.Core.Eval.EvalService.ParseVerdict"/> coalesces a null/missing
+/// outcome to <see cref="EvalOutcomes.Assessed"/> after deserializing.</para>
+/// </summary>
+public record EvalQuestionAssessment {
+    [JsonPropertyName("category")]       public required string Category   { get; init; }
+    [JsonPropertyName("question_id")]    public required string QuestionId { get; init; }
+    [JsonPropertyName("outcome")]        public string? Outcome { get; init; }
+    [JsonPropertyName("score")]          public int?    Score          { get; init; }
+    [JsonPropertyName("verdict")]        public string? Verdict        { get; init; }
+    [JsonPropertyName("finding")]        public required string Finding { get; init; }
+    [JsonPropertyName("evidence")]       public string? Evidence       { get; init; }
+    [JsonPropertyName("recommendation")] public string? Recommendation { get; init; }
+    [JsonPropertyName("tools_used")]     public int?    ToolsUsed      { get; init; }
+    [JsonPropertyName("prompt_version")] public string? PromptVersion  { get; init; }
+
+    [JsonPropertyName("evidence_coverage")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public EvalEvidenceCoverage? EvidenceCoverage { get; init; }
+}

src/Capacitor.Cli.Core/Eval/Contracts/EvalQuestionFailure.cs

@@ -0,0 +1,27 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval.Contracts;
+
+/// <summary>One question that produced no verdict. Mirrors the server's
+/// <c>JudgeQuestionFailure</c> field for field. Coded, never free-text.</summary>
+public record EvalQuestionFailure {
+    [JsonPropertyName("category")]    public required string Category   { get; init; }
+    [JsonPropertyName("question_id")] public required string QuestionId { get; init; }
+    [JsonPropertyName("code")]        public required string Code       { get; init; }
+
+    [JsonPropertyName("max_iterations")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? MaxIterations { get; init; }
+
+    [JsonPropertyName("turns_fetched")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? TurnsFetched { get; init; }
+
+    [JsonPropertyName("turns_total")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? TurnsTotal { get; init; }
+
+    [JsonPropertyName("http_status")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public int? HttpStatus { get; init; }
+}

src/Capacitor.Cli.Core/Eval/Contracts/FinalizeEvalV2Command.cs

@@ -0,0 +1,11 @@
+namespace Capacitor.Cli.Core.Eval.Contracts;
+
+/// <summary>Server → daemon: finalize an eval protocol 2 run with the assessments and coded
+/// failures the orchestrator collected, rather than a single verdict list. Mirrors the server's
+/// <c>FinalizeEvalV2Command</c>.</summary>
+public readonly record struct FinalizeEvalV2Command(
+        string                                 EvalRunId,
+        IReadOnlyList<EvalQuestionAssessment>  Assessments,
+        IReadOnlyList<EvalQuestionFailure>     Failures,
+        string                                 Model
+    );

src/Capacitor.Cli.Core/Eval/Contracts/QuestionResultV2.cs

@@ -0,0 +1,14 @@
+namespace Capacitor.Cli.Core.Eval.Contracts;
+
+/// <summary>Daemon → server: per-question judge result on eval protocol 2. Mirrors the server's
+/// <c>QuestionResultV2</c> — plain PascalCase, no <c>[JsonPropertyName]</c>, round-tripping through
+/// SignalR's JSON hub protocol the same way <c>QuestionResult</c> does. Exactly one of
+/// <see cref="Assessment"/> and <see cref="Failure"/> is set; <see cref="Error"/> is diagnostic text
+/// that may accompany a failure.</summary>
+public readonly record struct QuestionResultV2(
+        EvalQuestionAssessment? Assessment,
+        EvalQuestionFailure?    Failure,
+        string?                 Error,
+        long                    InputTokens,
+        long                    OutputTokens
+    );

src/Capacitor.Cli.Core/Eval/Contracts/QuestionRunResult.cs

@@ -0,0 +1,6 @@
+namespace Capacitor.Cli.Core.Eval.Contracts;
+
+/// <summary>Result of one <c>EvalService.RunQuestionAsync</c> invocation. Exactly one of
+/// <see cref="Assessment"/> and <see cref="Failure"/> is set — never used over the wire, purely an
+/// in-process return shape.</summary>
+public readonly record struct QuestionRunResult(EvalQuestionAssessment? Assessment, EvalQuestionFailure? Failure);

src/Capacitor.Cli.Core/Eval/Contracts/SessionEvalCompletedPayloadV4.cs

@@ -0,0 +1,26 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval.Contracts;
+
+/// <summary>Posted to <c>POST /api/sessions/{id}/evals/v4</c>. Mirrors the server's
+/// <c>JudgeRunReportV4</c> field for field (the server fills <c>evaluated_at</c>).</summary>
+public record SessionEvalCompletedPayloadV4 {
+    [JsonPropertyName("eval_run_id")]   public required string EvalRunId  { get; init; }
+    [JsonPropertyName("judge_model")]   public required string JudgeModel { get; init; }
+    [JsonPropertyName("categories")]    public List<EvalCategoryAssessment> Categories { get; init; } = [];
+    [JsonPropertyName("overall_score")] public int?    OverallScore { get; init; }
+    [JsonPropertyName("summary")]       public required string Summary { get; init; }
+
+    [JsonPropertyName("retrospective")]                public EvalRetrospectiveV2? Retrospective { get; init; }
+    [JsonPropertyName("retrospective_prompt_version")] public string? RetrospectivePromptVersion { get; init; }
+    [JsonPropertyName("facts_used")]                   public List<EvalFactSnapshotPayload> FactsUsed { get; init; } = [];
+
+    [JsonPropertyName("assessed_questions")]   public required int AssessedQuestions   { get; init; }
+    [JsonPropertyName("unassessed_questions")] public required int UnassessedQuestions { get; init; }
+    [JsonPropertyName("judged_questions")]     public required int JudgedQuestions     { get; init; }
+    [JsonPropertyName("total_questions")]      public required int TotalQuestions      { get; init; }
+    [JsonPropertyName("failed_questions")]     public List<EvalQuestionFailure> FailedQuestions { get; init; } = [];
+
+    [JsonPropertyName("coverage_policy_version")] public required string CoveragePolicyVersion { get; init; }
+    [JsonPropertyName("evidence_scope_version")]  public string? EvidenceScopeVersion { get; init; }
+}