Hold the evidence scope for the whole run when the server offers it

src/Capacitor.Cli.Core/Eval/EvalService.Evidence.cs

@@ -69,10 +69,11 @@ public static partial class EvalService {
         var root = tempRoot ?? Path.GetTempPath();
         EvidenceRunContext.SweepStale(root, time, observer.OnInfo);
         var context  = EvidenceRunContext.Create(evalRunId, root);
+        var scope    = new EvidenceScopeClient(httpClient, baseUrl, sessionId, time, holds: ad.ScopeHolds == true);
         var prepared = false;
         try {
-            var scope  = new EvidenceScopeClient(httpClient, baseUrl, sessionId, time);
             var status = await scope.ResolveAsync(ct);
+            if (scope.HoldRefused) observer.OnInfo("the server could not hold this evidence scope; a session that grows during the run can end it");
             if (status != EvidenceScopeStatus.Ok) {
                 observer.OnFailed(status switch {
                     EvidenceScopeStatus.NotVisible => "session not found or not visible",
@@ -113,6 +114,8 @@ public static partial class EvalService {
             };
         } finally {
             if (!prepared) {
+                await scope.DisposeAsync();
+                if (scope.ReleaseFailure is { } failure) observer.OnInfo(failure);
                 try { await context.DisposeAsync(); }
                 catch (Exception e) when (e is IOException or UnauthorizedAccessException) { observer.OnInfo($"could not remove {context.RunDirectory}: {e.Message}"); }
             }
@@ -346,10 +349,11 @@ await setup.Context.DrainRetainedFactsAsync((category, fact, token) => PostJudge
         return aggregate;
     }
 
-    /// <summary>Deletes the run directory, logging rather than throwing when it cannot be removed.</summary>
+    /// <summary>Releases the scope's hold and deletes the run directory, reporting rather than throwing when either fails.</summary>
     public static async Task DisposeSetupAsync(EvidenceRunSetup setup, IEvalObserver observer) {
         try { await setup.DisposeAsync(); }
         catch (Exception e) when (e is IOException or UnauthorizedAccessException) { observer.OnInfo($"could not remove {setup.Context.RunDirectory}: {e.Message}"); }
+        if (setup.Scope.ReleaseFailure is { } failure) observer.OnInfo(failure);
     }
 
     static SessionEvalCompletedPayloadV4 EvidenceAggregate(EvidenceRunSetup setup, IReadOnlyList<EvalQuestionAssessment> assessments,

src/Capacitor.Cli.Core/Eval/Evidence/EvalEvidenceAdvertisementDto.cs

@@ -10,4 +10,6 @@ public sealed record EvalEvidenceAdvertisementDto {
     [JsonPropertyName("one_shot_limit_chars")]         public required int    OneShotLimitChars          { get; init; }
     [JsonPropertyName("retrospective_evidence_bytes")] public required int    RetrospectiveEvidenceBytes { get; init; }
     [JsonPropertyName("coverage_policy_version")]      public required string CoveragePolicyVersion      { get; init; }
+    // Absent from a server that cannot hold a scope; the run then renews by resolving again.
+    [JsonPropertyName("scope_holds")]                  public          bool?  ScopeHolds                 { get; init; }
 }

src/Capacitor.Cli.Core/Eval/Evidence/EvidenceRunSetup.cs

@@ -4,7 +4,8 @@
 namespace Capacitor.Cli.Core.Eval.Evidence;
 
 /// <summary>One evidence-route run as prepared: its private directory, the bound scope and its clients, the route, the fitting
-/// one-shot trace (empty on retrieval), the orientation and the reconciled questions. Disposing it deletes the directory.</summary>
+/// one-shot trace (empty on retrieval), the orientation and the reconciled questions. Disposing it releases the scope's
+/// hold, if the server holds it, and deletes the directory.</summary>
 public sealed class EvidenceRunSetup : IAsyncDisposable {
     public required EvidenceRunContext             Context                    { get; init; }
     public required EvidenceScopeClient            Scope                      { get; init; }
@@ -39,5 +40,8 @@ public sealed class EvidenceRunSetup : IAsyncDisposable {
     public EvidenceRunBudgets Budgets  => new(Advertisement.MaxToolCalls, Advertisement.JudgeByteBudgetBytes, Advertisement.PageBudgetBytes);
     public int                MaxTurns => EvidenceBudgets.MaxTurns(Advertisement.MaxToolCalls);
 
-    public ValueTask DisposeAsync() => Context.DisposeAsync();
+    public async ValueTask DisposeAsync() {
+        try { await Scope.DisposeAsync(); }
+        finally { await Context.DisposeAsync(); }
+    }
 }

src/Capacitor.Cli.Core/Eval/Evidence/EvidenceScopeClient.cs

@@ -1,10 +1,13 @@
+using System.Text;
 using System.Text.Json;
 
 namespace Capacitor.Cli.Core.Eval.Evidence;
 
 /// <summary>Resolves and renews one run's viewer scope. The deadline is the client's clock at the request plus the
-/// server's stated lifetime, so a skewed clock cannot make it later than the server's own expiry.</summary>
-public sealed class EvidenceScopeClient(HttpClient http, string baseUrl, string sessionId, TimeProvider time) {
+/// server's stated lifetime, so a skewed clock cannot make it later than the server's own expiry. With
+/// <paramref name="holds"/> the server holds the scope for the run: renewal extends that hold instead of resolving
+/// again, so the session's growth cannot move the version, and disposing the client releases it.</summary>
+public sealed class EvidenceScopeClient(HttpClient http, string baseUrl, string sessionId, TimeProvider time, bool holds = false) : IAsyncDisposable {
     public const int    MaxManifestPages = 10;
     public const string SingleQuery      = "continuations=false&delegates=true&adopted_children=false";
 
@@ -17,6 +20,7 @@ public sealed class EvidenceScopeClient(HttpClient http, string baseUrl, string
     public static readonly TimeSpan PreDrainHeadroom           = TimeSpan.FromSeconds(60);
     public static readonly TimeSpan CertificationHeadroom      = EvidenceCitationClient.CertificationBudget + RpcMargin;
     public static readonly TimeSpan RetrospectiveHeadroom      = RetrospectiveExcerptBudget + EvalService.RetrospectiveTimeout + DrainPersistBudget;
+    public static readonly TimeSpan ReleaseTimeout             = TimeSpan.FromSeconds(10);
 
     /// <summary>Everything a question runs before the next renewal: setup, the harness, parsing and coverage, and certification.</summary>
     public static TimeSpan QuestionHeadroom(EvidenceRoute route) =>
@@ -28,8 +32,18 @@ public static TimeSpan QuestionHeadroom(EvidenceRoute route) =>
     public int                 Reopens   { get; private set; }
     public string?             LastError { get; private set; }
 
+    /// <summary>Why releasing the hold failed, when it did; the hold then expires on the server.</summary>
+    public string? ReleaseFailure { get; private set; }
+
+    /// <summary>True when the server was asked to hold the scope and refused, as it does for a scope too large to hold.</summary>
+    public bool HoldRefused { get; private set; }
+
+    // The latest token of a hold this client took, owned from the first page that reported it, so a hold is released
+    // even when a later page of its manifest failed.
+    string? _heldToken;
+
     public async Task<EvidenceScopeStatus> ResolveAsync(CancellationToken ct) {
-        var (status, fresh) = await FetchAllPagesAsync(ct);
+        var (status, fresh) = holds ? await FetchAllPagesAsync(TakeHoldAsync, ct) : await FetchAllPagesAsync(ct);
         if (status == EvidenceScopeStatus.Ok) State = fresh;
         return status;
     }
@@ -55,6 +69,7 @@ public async Task<EvidenceScopeStatus> EnsureScopeAsync(TimeSpan headroom, Cance
         if (state.Deadline - time.GetUtcNow() > headroom) return EvidenceScopeStatus.Ok;
 
         Refreshes++;
+        if (state.Held) return await RenewAsync(state, ct);
         var (status, fresh) = await FetchAllPagesAsync(ct);
         if (status is EvidenceScopeStatus.NotVisible or EvidenceScopeStatus.Moved) return EvidenceScopeStatus.Moved;
         if (status != EvidenceScopeStatus.Ok) return status;
@@ -63,12 +78,57 @@ public async Task<EvidenceScopeStatus> EnsureScopeAsync(TimeSpan headroom, Cance
         return EvidenceScopeStatus.Ok;
     }
 
+    /// <summary>Extends the hold without resolving again; a renewal answering another version is a moved scope.</summary>
+    async Task<EvidenceScopeStatus> RenewAsync(EvidenceScopeState state, CancellationToken ct) {
+        var sentAt = time.GetUtcNow();
+        var (status, page) = await SendPageAsync(HttpMethod.Post, Route("renewal"), HoldBody(state.Token), ct);
+        if (status is EvidenceScopeStatus.NotVisible or EvidenceScopeStatus.Moved) return EvidenceScopeStatus.Moved;
+        if (status != EvidenceScopeStatus.Ok) return status;
+        _heldToken = page!.Token;
+        if (page.ScopeVersion != state.ScopeVersion) return EvidenceScopeStatus.Moved;
+        State = state with { Token = page.Token, Deadline = sentAt + Lifetime(page), ServerExpiresAt = page.ExpiresAt };
+        return EvidenceScopeStatus.Ok;
+    }
+
+    /// <summary>Releases the hold this client took, if any, within <see cref="ReleaseTimeout"/>. Never throws: a release
+    /// that fails for any reason leaves the hold to expire on the server, and the run's own cleanup still runs.</summary>
+    public async ValueTask DisposeAsync() {
+        if (Interlocked.Exchange(ref _heldToken, null) is not { } token) return;
+        using var timeout = new CancellationTokenSource(ReleaseTimeout, time);
+        try {
+            using var request = new HttpRequestMessage(HttpMethod.Delete, Route("hold")) { Content = HoldBody(token) };
+            using var resp    = await http.SendAsync(request, timeout.Token);
+            if (!resp.IsSuccessStatusCode) ReleaseFailure = $"could not release the evidence scope hold: HTTP {(int)resp.StatusCode}";
+        } catch (Exception e) {
+            ReleaseFailure = $"could not release the evidence scope hold: {e.Message}";
+        }
+    }
+
     string Url(string query) => $"{baseUrl.TrimEnd('/')}/api/sessions/{Uri.EscapeDataString(sessionId)}/evidence-scope?{query}";
 
-    async Task<(EvidenceScopeStatus, EvidenceScopeState?)> FetchAllPagesAsync(CancellationToken ct) {
+    string Route(string leaf) => $"{baseUrl.TrimEnd('/')}/api/sessions/{Uri.EscapeDataString(sessionId)}/evidence-scope/{leaf}";
+
+    static StringContent HoldBody(string token) =>
+        new(JsonSerializer.Serialize(new EvidenceScopeHoldRequestDto { Token = token }, CapacitorJsonContext.Default.EvidenceScopeHoldRequestDto), Encoding.UTF8, "application/json");
+
+    // One request key per resolution, so a redelivery of this request can never take a second hold.
+    Task<(EvidenceScopeStatus, EvidenceScopeManifestDto?)> TakeHoldAsync(CancellationToken ct) {
+        var body = JsonSerializer.Serialize(new EvidenceScopeHoldCreateRequestDto {
+            RequestId = Guid.NewGuid().ToString("N"), Continuations = false, Delegates = true, AdoptedChildren = false
+        }, CapacitorJsonContext.Default.EvidenceScopeHoldCreateRequestDto);
+        return SendPageAsync(HttpMethod.Post, Route("holds"), new StringContent(body, Encoding.UTF8, "application/json"), ct);
+    }
+
+    static TimeSpan Lifetime(EvidenceScopeManifestDto page) => page.IssuedAt is { } issued && page.ExpiresAt is { } expires ? expires - issued : ArtifactLifetime;
+
+    Task<(EvidenceScopeStatus, EvidenceScopeState?)> FetchAllPagesAsync(CancellationToken ct) => FetchAllPagesAsync(t => GetPageAsync(Url(SingleQuery), t), ct);
+
+    async Task<(EvidenceScopeStatus, EvidenceScopeState?)> FetchAllPagesAsync(Func<CancellationToken, Task<(EvidenceScopeStatus, EvidenceScopeManifestDto?)>> firstPage, CancellationToken ct) {
         var sentAt = time.GetUtcNow();
-        var (status, first) = await GetPageAsync(Url(SingleQuery), ct);
+        var (status, first) = await firstPage(ct);
         if (status != EvidenceScopeStatus.Ok) return (status, null);
+        if (first!.Held == true) _heldToken = first.Token;
+        else if (first.Held == false) HoldRefused = true;
 
         var sources = new List<EvidenceSourceDto>(first!.Sources);
         var next    = first.NextCursor;
@@ -81,13 +141,16 @@ public async Task<EvidenceScopeStatus> EnsureScopeAsync(TimeSpan headroom, Cance
             next = page.NextCursor;
         }
 
-        var lifetime = first.IssuedAt is { } issued && first.ExpiresAt is { } expires ? expires - issued : ArtifactLifetime;
-        return (EvidenceScopeStatus.Ok, new(first.ScopeVersion, first.RootSessionId, first.Complete, first.IncompleteReasons, sources, first.Token, sentAt + lifetime, first.ExpiresAt));
+        return (EvidenceScopeStatus.Ok, new(first.ScopeVersion, first.RootSessionId, first.Complete, first.IncompleteReasons, sources, first.Token, sentAt + Lifetime(first),
+            first.ExpiresAt, first.Held == true));
     }
 
-    async Task<(EvidenceScopeStatus, EvidenceScopeManifestDto?)> GetPageAsync(string url, CancellationToken ct) {
+    Task<(EvidenceScopeStatus, EvidenceScopeManifestDto?)> GetPageAsync(string url, CancellationToken ct) => SendPageAsync(HttpMethod.Get, url, null, ct);
+
+    async Task<(EvidenceScopeStatus, EvidenceScopeManifestDto?)> SendPageAsync(HttpMethod method, string url, HttpContent? content, CancellationToken ct) {
         try {
-            using var resp = await http.GetAsync(url, ct);
+            using var request = new HttpRequestMessage(method, url) { Content = content };
+            using var resp    = await http.SendAsync(request, ct);
             switch ((int)resp.StatusCode) {
                 case 401: LastError = "authentication failed — run 'kcap login' to re-authenticate"; return (EvidenceScopeStatus.Unauthorized, null);
                 case 404: return (EvidenceScopeStatus.NotVisible, null);

src/Capacitor.Cli.Core/Eval/Evidence/EvidenceScopeHoldCreateRequestDto.cs

@@ -0,0 +1,11 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval.Evidence;
+
+/// <summary>Posted to <c>/evidence-scope/holds</c>; mirrors the server's <c>EvidenceScopeHoldCreateRequest</c>.</summary>
+public sealed record EvidenceScopeHoldCreateRequestDto {
+    [JsonPropertyName("request_id")]       public required string RequestId       { get; init; }
+    [JsonPropertyName("continuations")]    public required bool   Continuations   { get; init; }
+    [JsonPropertyName("delegates")]        public required bool   Delegates       { get; init; }
+    [JsonPropertyName("adopted_children")] public required bool   AdoptedChildren { get; init; }
+}

src/Capacitor.Cli.Core/Eval/Evidence/EvidenceScopeHoldRequestDto.cs

@@ -0,0 +1,8 @@
+using System.Text.Json.Serialization;
+
+namespace Capacitor.Cli.Core.Eval.Evidence;
+
+/// <summary>Names a held scope by its token, to renew or release it; mirrors the server's <c>EvidenceScopeHoldRequest</c>.</summary>
+public sealed record EvidenceScopeHoldRequestDto {
+    [JsonPropertyName("token")] public required string Token { get; init; }
+}

src/Capacitor.Cli.Core/Eval/Evidence/EvidenceScopeManifestDto.cs

@@ -13,4 +13,6 @@ public sealed record EvidenceScopeManifestDto {
     // Absent from a server that predates the wire lifetime; the scope client then assumes the 30-minute artifact.
     [JsonPropertyName("issued_at")]          public          DateTimeOffset?         IssuedAt          { get; init; }
     [JsonPropertyName("expires_at")]         public          DateTimeOffset?         ExpiresAt         { get; init; }
+    // True for a held scope, false when the server refused a requested hold, absent otherwise.
+    [JsonPropertyName("held")]               public          bool?                   Held              { get; init; }
 }

src/Capacitor.Cli.Core/Eval/Evidence/EvidenceScopeState.cs

@@ -2,4 +2,4 @@ namespace Capacitor.Cli.Core.Eval.Evidence;
 
 /// <summary>The bound scope: the whole manifest, the current token and the client-clock deadline it stays valid until.</summary>
 public sealed record EvidenceScopeState(string ScopeVersion, string RootSessionId, bool Complete, IReadOnlyList<string> IncompleteReasons,
-    IReadOnlyList<EvidenceSourceDto> Sources, string Token, DateTimeOffset Deadline, DateTimeOffset? ServerExpiresAt);
+    IReadOnlyList<EvidenceSourceDto> Sources, string Token, DateTimeOffset Deadline, DateTimeOffset? ServerExpiresAt, bool Held = false);

src/Capacitor.Cli.Core/Models.cs

@@ -1080,6 +1080,8 @@ public sealed record CurationApplyResponse {
 [JsonSerializable(typeof(Capacitor.Cli.Core.Eval.Evidence.EvidenceScopeErrorDto))]
 [JsonSerializable(typeof(Capacitor.Cli.Core.Eval.Evidence.EvidenceReadErrorDto))]
 [JsonSerializable(typeof(Capacitor.Cli.Core.Eval.Evidence.EvidenceCitationsRequestDto))]
+[JsonSerializable(typeof(Capacitor.Cli.Core.Eval.Evidence.EvidenceScopeHoldCreateRequestDto))]
+[JsonSerializable(typeof(Capacitor.Cli.Core.Eval.Evidence.EvidenceScopeHoldRequestDto))]
 [JsonSerializable(typeof(Capacitor.Cli.Core.Eval.Evidence.EvidenceCitationsResponseDto))]
 [JsonSerializable(typeof(Capacitor.Cli.Core.Eval.Evidence.EvidenceEventPageDto))]
 [JsonSerializable(typeof(Capacitor.Cli.Core.Eval.Evidence.EvidenceTurnPageDto))]

src/Capacitor.Cli.Daemon/Services/EvalContextCache.cs

@@ -158,17 +158,31 @@ bool ShouldRelease() {
             return true;
         }
 
-        // Disposal is synchronous underneath, so it is waited on here; a failure is logged and never stops the rest.
+        // Releasing a held scope is a network call, so it runs in the background, never on the thread retiring the entry,
+        // and the owned client it uses is disposed after it; one the process does not finish expires on the server.
+        // Removing the directory is synchronous underneath, so it is waited on here; a failure of either is logged and
+        // never stops the rest.
         void Release() {
-            if (context is EvidenceRunSetup setup) {
-                try {
-                    var disposal = setup.DisposeAsync();
-                    if (!disposal.IsCompletedSuccessfully) disposal.AsTask().GetAwaiter().GetResult();
-                } catch (Exception e) {
-                    logger.LogWarning(e, "Could not remove the run directory of eval {RunId}", evalRunId);
-                }
+            if (context is not EvidenceRunSetup setup) {
+                owned?.Dispose();
+                return;
+            }
+            _ = ReleaseHoldAsync(setup.Scope);
+            try {
+                var disposal = setup.Context.DisposeAsync();
+                if (!disposal.IsCompletedSuccessfully) disposal.AsTask().GetAwaiter().GetResult();
+            } catch (Exception e) {
+                logger.LogWarning(e, "Could not remove the run directory of eval {RunId}", evalRunId);
+            }
+        }
+
+        async Task ReleaseHoldAsync(EvidenceScopeClient scope) {
+            try {
+                await scope.DisposeAsync();
+                if (scope.ReleaseFailure is { } failure) logger.LogWarning("Eval {RunId}: {Failure}", evalRunId, failure);
+            } finally {
+                owned?.Dispose();
             }
-            owned?.Dispose();
         }
 
         public void Dispose() => _retired.Dispose();