[AI-2748] Expose redaction members and kcap eval --baseline-out

README.md

@@ -515,6 +515,7 @@ kcap eval --chain <sessionId>              # include the full continuation chain
 kcap eval --threshold 5000 <sessionId>     # keep more of each tool output before truncation
 kcap eval --questions safety <sessionId>   # run only the 4 safety judges
 kcap eval --skip efficiency <sessionId>    # run everything except efficiency
+kcap eval --baseline-out out.json <sessionId>  # also write per-question usage/route/timing
 kcap eval --list-questions                 # print the question taxonomy
 ```
 

src/Capacitor.Cli.Core/Eval/BaselineObserver.cs

@@ -0,0 +1,110 @@
+using Capacitor.Cli.Core.Eval.Contracts;
+
+namespace Capacitor.Cli.Core.Eval;
+
+/// <summary>Decorates an <see cref="IEvalObserver"/> to collect each question's and the
+/// retrospective's judge usage, then writes a <see cref="BaselineOutput"/> to <paramref name="path"/>
+/// on the run's one terminal callback (<see cref="OnFinished"/> or <see cref="OnFailed"/>), so a
+/// baseline is still written for a partially-completed run.</summary>
+public sealed class BaselineObserver(
+        IEvalObserver inner,
+        string        path,
+        string        sessionId,
+        string        model,
+        bool          chain,
+        TimeProvider  time
+    ) : IEvalObserver {
+    readonly List<BaselineQuestionOutput>  _questions = [];
+    readonly List<BaselineQuestionFailure> _failures  = [];
+    readonly long                          _started   = time.GetTimestamp();
+    BaselineRetrospectiveOutput?           _retrospective;
+    string                                 _evalRunId = "";
+
+    /// <summary>True once a terminal callback tried and failed to write the baseline file, so the
+    /// caller can exit non-zero instead of reporting a success that produced no baseline.</summary>
+    public bool WriteFailed { get; private set; }
+
+    public void OnInfo(string message) => inner.OnInfo(message);
+
+    public void OnStarted(string evalRunId, string judgeModel, int totalQuestions) {
+        _evalRunId = evalRunId;
+        inner.OnStarted(evalRunId, judgeModel, totalQuestions);
+    }
+
+    public void OnContextFetched(int traceEntries, int traceChars, int toolResultsTotal, int toolResultsTruncated, long bytesSaved) =>
+        inner.OnContextFetched(traceEntries, traceChars, toolResultsTotal, toolResultsTruncated, bytesSaved);
+
+    public void OnQuestionStarted(int index, int total, string category, string questionId) =>
+        inner.OnQuestionStarted(index, total, category, questionId);
+
+    public void OnQuestionCompleted(int index, int total, EvalQuestionAssessment assessment, EvalUsage usage, string route, TimeSpan elapsed, int runnerInvocations) {
+        _questions.Add(new BaselineQuestionOutput {
+            QuestionId = assessment.QuestionId,
+            Route      = route,
+            Usage      = usage,
+            Calls      = runnerInvocations,
+            ElapsedMs  = (long)elapsed.TotalMilliseconds
+        });
+        inner.OnQuestionCompleted(index, total, assessment, usage, route, elapsed, runnerInvocations);
+    }
+
+    public void OnQuestionFailed(int index, int total, string category, string questionId, string reason) {
+        _failures.Add(new BaselineQuestionFailure {
+            QuestionId = questionId,
+            Category   = category,
+            Reason     = reason
+        });
+        inner.OnQuestionFailed(index, total, category, questionId, reason);
+    }
+
+    public void OnFactRetained(string category, string fact) => inner.OnFactRetained(category, fact);
+
+    public void OnRetrospectiveStarted() => inner.OnRetrospectiveStarted();
+
+    public void OnRetrospectiveCompleted(EvalRetrospectiveV2 retrospective, EvalUsage usage, TimeSpan elapsed) {
+        _retrospective = new BaselineRetrospectiveOutput {
+            Usage     = usage,
+            Calls     = 1,
+            ElapsedMs = (long)elapsed.TotalMilliseconds
+        };
+        inner.OnRetrospectiveCompleted(retrospective, usage, elapsed);
+    }
+
+    public void OnRetrospectiveFailed(string reason) => inner.OnRetrospectiveFailed(reason);
+
+    public void OnFinished(SessionEvalCompletedPayloadV4 aggregate) {
+        WriteBaseline();
+        inner.OnFinished(aggregate);
+    }
+
+    public void OnFailed(string reason) {
+        WriteBaseline();
+        inner.OnFailed(reason);
+    }
+
+    void WriteBaseline() {
+        var usages = new List<EvalUsage>(_questions.Select(q => q.Usage));
+        if (_retrospective is not null) usages.Add(_retrospective.Usage);
+
+        var output = new BaselineOutput {
+            SessionId      = sessionId,
+            EvalRunId      = _evalRunId,
+            Model          = model,
+            Chain          = chain,
+            Questions      = _questions,
+            Failures       = _failures,
+            Retrospective  = _retrospective,
+            Totals         = EvalUsage.Sum(usages),
+            TotalElapsedMs = (long)time.GetElapsedTime(_started).TotalMilliseconds
+        };
+
+        try {
+            BaselineOutputWriter.Write(path, output);
+        } catch (Exception ex) {
+            // Throwing here would rob the inner observer of its one terminal callback, and swallowing
+            // the failure would report a baseline that was never written — flag it so the run fails.
+            WriteFailed = true;
+            inner.OnInfo($"failed to write baseline output to '{path}': {ex.Message}");
+        }
+    }
+}

src/Capacitor.Cli.Core/Eval/BaselineOutput.cs

@@ -0,0 +1,17 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval;
+
+/// <summary>The full record <c>kcap eval --baseline-out</c> writes for one run. Snake_case keys are
+/// a cross-repo contract with the server's parsing mirror — do not rename without updating both.</summary>
+public sealed record BaselineOutput {
+    [JsonPropertyName("session_id")]       public string                                 SessionId      { get; init; } = "";
+    [JsonPropertyName("eval_run_id")]      public string                                 EvalRunId      { get; init; } = "";
+    [JsonPropertyName("model")]            public string                                 Model          { get; init; } = "";
+    [JsonPropertyName("chain")]            public bool                                   Chain          { get; init; }
+    [JsonPropertyName("questions")]        public IReadOnlyList<BaselineQuestionOutput>  Questions      { get; init; } = [];
+    [JsonPropertyName("failures")]         public IReadOnlyList<BaselineQuestionFailure> Failures       { get; init; } = [];
+    [JsonPropertyName("retrospective")]    public BaselineRetrospectiveOutput?           Retrospective  { get; init; }
+    [JsonPropertyName("totals")]           public EvalUsage                              Totals         { get; init; } = new();
+    [JsonPropertyName("total_elapsed_ms")] public long                                   TotalElapsedMs { get; init; }
+}

src/Capacitor.Cli.Core/Eval/BaselineOutputWriter.cs

@@ -0,0 +1,12 @@
+using System.Text.Json;
+
+namespace Capacitor.Cli.Core.Eval;
+
+/// <summary>Writes a completed <see cref="BaselineOutput"/> as JSON via the source-generated
+/// <see cref="CapacitorJsonContext"/> — the AOT-compiled CLI has no reflection-based fallback.</summary>
+public static class BaselineOutputWriter {
+    public static void Write(string path, BaselineOutput output) {
+        var json = JsonSerializer.Serialize(output, CapacitorJsonContext.Default.BaselineOutput);
+        File.WriteAllText(path, json);
+    }
+}

src/Capacitor.Cli.Core/Eval/BaselineQuestionFailure.cs

@@ -0,0 +1,13 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval;
+
+/// <summary>A question that failed before producing a verdict, recorded in a <c>--baseline-out</c>
+/// file so a failed judge is not silently absent from the run's record. It carries no route, usage
+/// or timing — only which question failed and why. Snake_case keys are a cross-repo contract with
+/// the server's parsing mirror — do not rename without updating both.</summary>
+public sealed record BaselineQuestionFailure {
+    [JsonPropertyName("question_id")] public string QuestionId { get; init; } = "";
+    [JsonPropertyName("category")]    public string Category   { get; init; } = "";
+    [JsonPropertyName("reason")]      public string Reason     { get; init; } = "";
+}

src/Capacitor.Cli.Core/Eval/BaselineQuestionOutput.cs

@@ -0,0 +1,14 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval;
+
+/// <summary>One question's record in a <c>--baseline-out</c> file: how <c>kcap eval</c> routed it,
+/// its judge usage, how many times the runner was called, and wall time. Snake_case keys are a
+/// cross-repo contract with the server's parsing mirror — do not rename without updating both.</summary>
+public sealed record BaselineQuestionOutput {
+    [JsonPropertyName("question_id")] public string    QuestionId { get; init; } = "";
+    [JsonPropertyName("route")]       public string    Route      { get; init; } = "";
+    [JsonPropertyName("usage")]       public EvalUsage Usage      { get; init; } = new();
+    [JsonPropertyName("calls")]       public int       Calls      { get; init; }
+    [JsonPropertyName("elapsed_ms")]  public long      ElapsedMs  { get; init; }
+}

src/Capacitor.Cli.Core/Eval/BaselineRetrospectiveOutput.cs

@@ -0,0 +1,12 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval;
+
+/// <summary>The retrospective synthesis record in a <c>--baseline-out</c> file — its judge usage,
+/// runner invocation count, and wall time. Snake_case keys are a cross-repo contract with the
+/// server's parsing mirror — do not rename without updating both.</summary>
+public sealed record BaselineRetrospectiveOutput {
+    [JsonPropertyName("usage")]      public EvalUsage Usage     { get; init; } = new();
+    [JsonPropertyName("calls")]      public int       Calls     { get; init; }
+    [JsonPropertyName("elapsed_ms")] public long      ElapsedMs { get; init; }
+}

src/Capacitor.Cli.Core/Eval/EvalService.cs

@@ -510,6 +510,8 @@ CancellationToken  ct
         // observers log OnInfo at Debug level where they vanish.
         var               diagnostics = new List<string>();
         ClaudeCliOutcome  outcome;
+        var               route       = question.NeedsTools || ctx.ForceTools ? "tools" : "text";
+        var               started     = time.GetTimestamp();
 
         if (question.NeedsTools || ctx.ForceTools) {
             // DEV-1486 tools-enabled path. Session-scoped MCP tool surface
@@ -615,7 +617,7 @@ CancellationToken  ct
             EvidenceCoverage = ReconcileEvidenceCoverage(assessment.Outcome, CoverageForTextPath(ctx, question))
         };
 
-        observer.OnQuestionCompleted(index, total, assessment, result.InputTokens, result.OutputTokens);
+        observer.OnQuestionCompleted(index, total, assessment, EvalUsage.FromResult(result), route, time.GetElapsedTime(started), runnerInvocations: 1);
 
         // If the judge emitted a retain_fact, persist it for future evals.
         if (ExtractRetainFact(result.Result) is { } retainedFact) {
@@ -1562,6 +1564,8 @@ CancellationToken                  ct
 
         observer.OnRetrospectiveStarted();
 
+        var started = time.GetTimestamp();
+
         var sessionMeta  = $"session-id: {sessionId}\nrun-id: {evalRunId}\nmodel: {model}\noverall-score: {aggregate.OverallScore}/5";
         var verdictsJson = JsonSerializer.Serialize(verdicts, CapacitorJsonContext.Default.IReadOnlyListEvalQuestionVerdict);
 
@@ -1615,7 +1619,7 @@ CancellationToken                  ct
                 return null;
             }
 
-            observer.OnRetrospectiveCompleted(retrospective);
+            observer.OnRetrospectiveCompleted(retrospective, EvalUsage.FromResult(result), time.GetElapsedTime(started));
 
             return retrospective;
         } catch (OperationCanceledException) {
@@ -1651,6 +1655,8 @@ CancellationToken                      ct
 
         observer.OnRetrospectiveStarted();
 
+        var started = time.GetTimestamp();
+
         var overallText  = aggregate.OverallScore is { } score ? $"{score}/5" : "not scored";
         var sessionMeta  = $"session-id: {sessionId}\nrun-id: {evalRunId}\nmodel: {model}\noverall-score: {overallText}";
         var verdictsJson = JsonSerializer.Serialize(assessments, CapacitorJsonContext.Default.ListEvalQuestionAssessment);
@@ -1691,7 +1697,7 @@ CancellationToken                      ct
                 return null;
             }
 
-            observer.OnRetrospectiveCompleted(retrospective);
+            observer.OnRetrospectiveCompleted(retrospective, EvalUsage.FromResult(result), time.GetElapsedTime(started));
 
             return retrospective;
         } catch (OperationCanceledException) {
@@ -1766,8 +1772,8 @@ public void OnContextFetched(int traceEntries, int traceChars, int toolResultsTo
         public void OnQuestionStarted(int index, int total, string category, string questionId) =>
             Safe(() => inner.OnQuestionStarted(index, total, category, questionId), nameof(OnQuestionStarted));
 
-        public void OnQuestionCompleted(int index, int total, EvalQuestionAssessment assessment, long inputTokens, long outputTokens) =>
-            Safe(() => inner.OnQuestionCompleted(index, total, assessment, inputTokens, outputTokens), nameof(OnQuestionCompleted));
+        public void OnQuestionCompleted(int index, int total, EvalQuestionAssessment assessment, EvalUsage usage, string route, TimeSpan elapsed, int runnerInvocations) =>
+            Safe(() => inner.OnQuestionCompleted(index, total, assessment, usage, route, elapsed, runnerInvocations), nameof(OnQuestionCompleted));
 
         public void OnQuestionFailed(int index, int total, string category, string questionId, string reason) =>
             Safe(() => inner.OnQuestionFailed(index, total, category, questionId, reason), nameof(OnQuestionFailed));
@@ -1778,8 +1784,8 @@ public void OnFactRetained(string category, string fact) =>
         public void OnRetrospectiveStarted() =>
             Safe(inner.OnRetrospectiveStarted, nameof(OnRetrospectiveStarted));
 
-        public void OnRetrospectiveCompleted(EvalRetrospectiveV2 retrospective) =>
-            Safe(() => inner.OnRetrospectiveCompleted(retrospective), nameof(OnRetrospectiveCompleted));
+        public void OnRetrospectiveCompleted(EvalRetrospectiveV2 retrospective, EvalUsage usage, TimeSpan elapsed) =>
+            Safe(() => inner.OnRetrospectiveCompleted(retrospective, usage, elapsed), nameof(OnRetrospectiveCompleted));
 
         public void OnRetrospectiveFailed(string reason) =>
             Safe(() => inner.OnRetrospectiveFailed(reason), nameof(OnRetrospectiveFailed));

src/Capacitor.Cli.Core/Eval/EvalUsage.cs

@@ -0,0 +1,46 @@
+using System.Text.Json.Serialization;
+using Capacitor.Cli.Core.Harness.Claude;
+
+namespace Capacitor.Cli.Core.Eval;
+
+/// <summary>Token usage and self-reported cost for one judge invocation, or a run's total, as
+/// written to <c>--baseline-out</c>. Snake_case keys are a cross-repo contract with the server's
+/// parsing mirror — do not rename without updating both.</summary>
+public sealed record EvalUsage {
+    [JsonPropertyName("input_tokens")]       public long    InputTokens      { get; init; }
+    [JsonPropertyName("output_tokens")]      public long    OutputTokens     { get; init; }
+    [JsonPropertyName("cache_read_tokens")]  public long    CacheReadTokens  { get; init; }
+    [JsonPropertyName("cache_write_tokens")] public long    CacheWriteTokens { get; init; }
+    [JsonPropertyName("reported_cost_usd")]  public double? ReportedCostUsd  { get; init; }
+
+    internal static EvalUsage FromResult(ClaudeCliResult result) => new() {
+        InputTokens      = result.InputTokens,
+        OutputTokens     = result.OutputTokens,
+        CacheReadTokens  = result.CacheReadTokens,
+        CacheWriteTokens = result.CacheWriteTokens,
+        ReportedCostUsd  = result.CostUsd
+    };
+
+    /// <summary>Sums token counts across every entry; cost sums only the entries that reported one,
+    /// staying null when none did.</summary>
+    public static EvalUsage Sum(IEnumerable<EvalUsage> usages) {
+        long    inputTokens = 0, outputTokens = 0, cacheReadTokens = 0, cacheWriteTokens = 0;
+        double? cost        = null;
+
+        foreach (var u in usages) {
+            inputTokens      += u.InputTokens;
+            outputTokens     += u.OutputTokens;
+            cacheReadTokens  += u.CacheReadTokens;
+            cacheWriteTokens += u.CacheWriteTokens;
+            if (u.ReportedCostUsd is { } c) cost = (cost ?? 0) + c;
+        }
+
+        return new EvalUsage {
+            InputTokens      = inputTokens,
+            OutputTokens     = outputTokens,
+            CacheReadTokens  = cacheReadTokens,
+            CacheWriteTokens = cacheWriteTokens,
+            ReportedCostUsd  = cost
+        };
+    }
+}

src/Capacitor.Cli.Core/Eval/IEvalObserver.cs

@@ -40,8 +40,10 @@ public interface IEvalObserver {
     void OnQuestionStarted(int index, int total, string category, string questionId);
 
     /// <summary>Fired after a judge question completed and its verdict was parsed — including an
-    /// unassessed outcome, which carries no score.</summary>
-    void OnQuestionCompleted(int index, int total, EvalQuestionAssessment assessment, long inputTokens, long outputTokens);
+    /// unassessed outcome, which carries no score. <paramref name="route"/> is <c>text</c> or
+    /// <c>tools</c>; <paramref name="runnerInvocations"/> is how many times the runner was called to
+    /// produce this result.</summary>
+    void OnQuestionCompleted(int index, int total, EvalQuestionAssessment assessment, EvalUsage usage, string route, TimeSpan elapsed, int runnerInvocations);
 
     /// <summary>Fired when a judge question fails (null Claude result, unparseable JSON, etc.); the eval continues.</summary>
     void OnQuestionFailed(int index, int total, string category, string questionId, string reason);
@@ -53,7 +55,7 @@ public interface IEvalObserver {
     void OnRetrospectiveStarted();
 
     /// <summary>Fired after the retrospective completed successfully and its payload was parsed.</summary>
-    void OnRetrospectiveCompleted(EvalRetrospectiveV2 retrospective);
+    void OnRetrospectiveCompleted(EvalRetrospectiveV2 retrospective, EvalUsage usage, TimeSpan elapsed);
 
     /// <summary>Fired when retrospective synthesis failed (null Claude result, unparseable JSON, etc.); the eval still completes.</summary>
     void OnRetrospectiveFailed(string reason);

src/Capacitor.Cli.Core/Models.cs

@@ -1022,6 +1022,11 @@ public sealed record CurationApplyResponse {
 [JsonSerializable(typeof(EvalQuestionFailure))]
 [JsonSerializable(typeof(List<EvalQuestionFailure>))]
 [JsonSerializable(typeof(SessionEvalCompletedPayloadV4))]
+[JsonSerializable(typeof(Capacitor.Cli.Core.Eval.EvalUsage))]
+[JsonSerializable(typeof(Capacitor.Cli.Core.Eval.BaselineQuestionOutput))]
+[JsonSerializable(typeof(Capacitor.Cli.Core.Eval.BaselineQuestionFailure))]
+[JsonSerializable(typeof(Capacitor.Cli.Core.Eval.BaselineRetrospectiveOutput))]
+[JsonSerializable(typeof(Capacitor.Cli.Core.Eval.BaselineOutput))]
 [JsonSerializable(typeof(QuestionResultV2))]
 [JsonSerializable(typeof(FinalizeEvalV2Command))]
 [JsonSerializable(typeof(List<ErrorEntry>))]

src/Capacitor.Cli.Core/Resources/help-eval.txt

@@ -24,6 +24,10 @@ Options:
                       exclusive with --skip.
   --skip <csv>        Run everything except the listed questions/categories.
                       Mutually exclusive with --questions.
+  --baseline-out <path>
+                      Write one JSON record per question (route, judge token
+                      usage/cost, wall time) plus the retrospective and run
+                      totals to <path>.
   --list-questions    Print the question taxonomy (grouped by category) and exit.
   --help, -h          Show this help.