fix(#1778): a floor between automatic rolls, since a roll is a pod restart

deploy/aks/values.aks.yaml

@@ -244,23 +244,31 @@ config:
     # cadence and your rollback headroom.
     AssemblyCache__Retention__Delete: "false"
     # ---- Platform self-update cadence --------------------------------------
-    # The poller's built-in default is 6h, which is why a freshly published image
-    # could sit unpicked for hours and look like "self-update is broken" (memex,
-    # 2026-08-03: running ci.1672 while ci.1674 had been in ACR since 07:34).
+    # The check is EVENT-DRIVEN (#1773): one pass at startup, then a check per
+    # published build of the platform or of any module this environment deploys.
+    # There is no polling interval any more — SelfUpdate__PollInterval was renamed
+    # and this key sat here inert, reading like a live daily throttle while binding
+    # to nothing (#1778).
     #
-    # DAILY, deliberately (2026-08-10, measured): every roll invalidates the whole
-    # NodeType compile cache — the framework MVID changes on every CI image — so a
-    # roll costs a full sequential rebuild (~2.4 s/type; ~10 min on the largest
-    # mesh) plus a degraded warm-up window while the 3 s enrichment probe is
-    # starved. CI publishes ~44 images/day; polling faster than the rebuild is
-    # worth just multiplies that cost. One roll per day keeps fixes flowing daily
-    # at a quarter of the compile cost and a quarter of the degraded windows.
-    # For an URGENT fix, don't shorten this — `kubectl rollout restart` the
-    # portal: the poll fires on startup (StartWith(-1L)) and picks the newest
-    # image immediately. Binding only works from the image that added
-    # SelfUpdateOptions.SectionName — before that this key is inert, because
-    # AddSelfUpdate() constructed its options directly.
-    SelfUpdate__PollInterval: "1.00:00:00"
+    # What this number now sets is the floor between two AUTOMATIC rolls. It is a
+    # RESTART budget, not a delivery speed: a roll restarts the pod and drops every
+    # live circuit on it. The trade, so this can be changed by editing one number:
+    #
+    #     24h  ->  <=1 restart/day,     a merged fix reaches prod the NEXT DAY
+    #      6h  ->  <=4 restarts/day,    same working day
+    #      1h  ->  <=24 restarts/day,   ~1.5 h after merge   <-- current
+    #
+    # 1h matches CD's reconcile tick, so hourly publication actually means hourly
+    # delivery. The former 24h is deliberately NOT restored: it predates
+    # adopt-before-compile (a prod roll went from 80 compiles / 64.8 s to 0
+    # compiles / 32.1 s), so the compile-cost half of its 2026-08-10 rationale is
+    # gone, and "tomorrow" is the answer that prompted the faster tick in the first
+    # place. What remains true is the restart cost, which is what this bounds.
+    #
+    # Neither extreme delays an URGENT fix: `kubectl rollout restart` takes the
+    # newest image immediately (the startup pass), and `gh workflow run
+    # main-cd.yml --ref main` bypasses the batch window.
+    SelfUpdate__MinRollInterval: "01:00:00"
     Mcp__BaseUrl: "http://memex-portal-service:8080"
     # ---- AI model providers (templated by the chart) ------------------------
     # These seed the system model catalog on deploy (BuiltInLanguageModelProvider

memex/Memex.Portal.Shared/SelfUpdate/SelfUpdateHostedService.cs

@@ -426,10 +426,44 @@ private IObservable<Unit> Apply(string target) =>
                 return Observable.Return(Unit.Default);
             }
 
-            _logger?.LogInformation(
-                "[SelfUpdate] applying update {Tag} (was {Current}).",
-                target, ShippedReleaseSeed.InstalledPlatformVersion);
-            return _http.Invoke(ct => _updater.PatchToVersionAsync(target, ct));
+            // 🚨 The pacing floor. A roll is a POD RESTART, so without it publication frequency is
+            // restart frequency and every restart drops the live circuits of everyone using the
+            // portal. The floor bounds the AUTOMATIC cadence only: `kubectl rollout restart` still
+            // takes the newest image immediately (the startup pass), and a main-cd dispatch still
+            // bypasses the batch window, so nothing here delays an urgent fix.
+            //
+            // Deferring is safe WITHOUT a timer for the same reason an availability hold is: the
+            // next publication event re-decides it. The floor never schedules anything.
+            // A disabled floor must not pay for the stamp read: LastRolledAtAsync is a Kubernetes GET
+            // in the AKS implementation, and its answer cannot change the decision when the floor is
+            // zero. Skipping it removes an API call and a failure point from the happy path.
+            if (_options.MinRollInterval <= TimeSpan.Zero)
+            {
+                _logger?.LogInformation(
+                    "[SelfUpdate] applying update {Tag} (was {Current}; no roll floor configured).",
+                    target, ShippedReleaseSeed.InstalledPlatformVersion);
+                return _http.Invoke(ct => _updater.PatchToVersionAsync(target, ct));
+            }
+
+            return _http.Invoke(ct => _updater.LastRolledAtAsync(ct)).SelectMany(lastRolledAt =>
+            {
+                var since = lastRolledAt is null ? (TimeSpan?)null : DateTimeOffset.UtcNow - lastRolledAt.Value;
+                if (since is { } elapsed && elapsed < _options.MinRollInterval)
+                {
+                    _logger?.LogInformation(
+                        "[SelfUpdate] {Tag} is available but this install rolled {Elapsed} ago, inside the "
+                        + "{Floor} floor — deferring. The next publication re-decides it; "
+                        + "`kubectl rollout restart` applies it now.",
+                        target, elapsed, _options.MinRollInterval);
+                    return Observable.Return(Unit.Default);
+                }
+
+                _logger?.LogInformation(
+                    "[SelfUpdate] applying update {Tag} (was {Current}; last rolled {Last}).",
+                    target, ShippedReleaseSeed.InstalledPlatformVersion,
+                    lastRolledAt?.ToString("O") ?? "never");
+                return _http.Invoke(ct => _updater.PatchToVersionAsync(target, ct));
+            });
         });
 
     /// <summary>Record the newest available tag on the policy node (as System). Drives the admin tab

memex/Memex.Portal.Shared/SelfUpdate/UnavailableUpdateMechanics.cs

@@ -51,6 +51,11 @@ internal sealed class DetectOnly : IDeploymentUpdater
     {
         public bool CanPatch => false;
 
+        /// <summary>Never — an install with no patcher has never rolled and never will, so the
+        /// floor can only ever pass. Correct rather than a hole: it cannot roll anyway.</summary>
+        public Task<DateTimeOffset?> LastRolledAtAsync(CancellationToken ct) =>
+            Task.FromResult<DateTimeOffset?>(null);
+
         public Task PatchToVersionAsync(string versionTag, CancellationToken ct) =>
             throw new InvalidOperationException(
                 $"Cannot roll this install to '{versionTag}': no deployment patcher is registered. "

src/MeshWeaver.Hosting/SelfUpdate/IDeploymentUpdater.cs

@@ -19,6 +19,21 @@ public interface IDeploymentUpdater
     /// rolling update. Patching the migration alongside the portal is how the database schema /
     /// <c>db_version</c> stays in step — the meaningful, safe "auto-update Postgres".</summary>
     Task PatchToVersionAsync(string versionTag, CancellationToken ct);
+
+    /// <summary>
+    /// When self-update last rolled THIS install, or null when it never has (or cannot tell).
+    ///
+    /// <para>🚨 This must be state that SURVIVES A RESTART, because a successful roll restarts the
+    /// process: an in-memory "last rolled at" is always empty exactly when it is needed, so a floor
+    /// built on it would never hold. The Kubernetes implementation stamps an annotation on the
+    /// Deployment it patches and reads that back.</para>
+    ///
+    /// <para>🚨 And it must NOT be process uptime, which is the tempting third option and is wrong
+    /// for crash recovery: a pod that comes back on an OLD image has a young process but an old
+    /// deployment, and uptime would make it wait out a floor it has long since satisfied. The
+    /// annotation gives the right answer there — old stamp, floor elapsed, roll immediately.</para>
+    /// </summary>
+    Task<DateTimeOffset?> LastRolledAtAsync(CancellationToken ct);
 }
 
 /// <summary>Probes the deployment target so the k8s-patch path only arms where it can actually

src/MeshWeaver.Hosting/SelfUpdate/SelfUpdateOptions.cs

@@ -44,6 +44,26 @@ public record SelfUpdateOptions
     /// </summary>
     public TimeSpan RetryInterval { get; init; } = TimeSpan.FromHours(6);
 
+    /// <summary>
+    /// The minimum time between two automatic rolls of this install.
+    ///
+    /// <para>Since the check became event-driven, a publication is a roll and a roll is a pod
+    /// restart — so without a floor, publication frequency IS restart frequency, and every restart
+    /// drops the live circuits of everyone using the portal. This paces the AUTOMATIC cadence only:
+    /// an operator who wants a fix now still has <c>kubectl rollout restart</c> (which picks the
+    /// newest image immediately via the startup pass) and a <c>main-cd.yml</c> dispatch.</para>
+    ///
+    /// <para>The default is one hour, matching CD's reconcile tick — the value that makes hourly
+    /// publication actually mean hourly delivery. It is deliberately NOT the 24 h the AKS chart
+    /// used to impose: that predates adopt-before-compile (which took a prod roll from 80 compiles
+    /// / 64.8 s to 0 compiles / 32.1 s) and it answers "how fast can a fix reach prod" with
+    /// "tomorrow", which is the complaint that prompted the faster tick in the first place.</para>
+    ///
+    /// <para>A roll deferred by this floor is re-decided by the next publication event, exactly
+    /// like one deferred for a missing artifact — the floor introduces no timer.</para>
+    /// </summary>
+    public TimeSpan MinRollInterval { get; init; } = TimeSpan.FromHours(1);
+
     /// <summary>
     /// How long a burst of build-completion events is coalesced before one check runs.
     ///

src/MeshWeaver.SelfUpdate.Aks/KubernetesDeploymentUpdater.cs

@@ -1,3 +1,5 @@
+using System.Collections.Generic;
+using System.Net.Http.Headers;
 using System.Runtime.Versioning;
 using System.Security.Cryptography.X509Certificates;
 using System.Text;
@@ -73,6 +75,42 @@ public KubernetesDeploymentUpdater(SelfUpdateOptions options, ILogger<Kubernetes
 
     public bool CanPatch => _http is not null && _apiBase is not null && !string.IsNullOrEmpty(_namespace);
 
+    /// <summary>Annotation self-update stamps on the deployments it rolls; the floor reads it back.</summary>
+    internal const string LastRolledAnnotation = "meshweaver.io/self-update-rolled-at";
+
+    /// <inheritdoc />
+    public async Task<DateTimeOffset?> LastRolledAtAsync(CancellationToken ct)
+    {
+        if (!CanPatch)
+            return null;
+        var token = SafeRead(TokenFile)?.Trim();
+        if (string.IsNullOrEmpty(token) || _http is null)
+            return null;
+        try
+        {
+            using var req = new HttpRequestMessage(HttpMethod.Get,
+                $"{_apiBase}/apis/apps/v1/namespaces/{_namespace}/deployments/{_options.PortalDeployment}");
+            req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);
+            using var res = await _http.SendAsync(req, ct).ConfigureAwait(false);
+            if (!res.IsSuccessStatusCode)
+                return null;   // cannot tell ⇒ do not hold the roll on evidence we could not gather
+            using var doc = JsonDocument.Parse(await res.Content.ReadAsStringAsync(ct).ConfigureAwait(false));
+            return doc.RootElement.TryGetProperty("metadata", out var meta)
+                   && meta.TryGetProperty("annotations", out var annotations)
+                   && annotations.TryGetProperty(LastRolledAnnotation, out var stamp)
+                   && DateTimeOffset.TryParse(stamp.GetString(), out var rolledAt)
+                ? rolledAt
+                : null;        // never rolled by self-update (or a stamp we cannot parse)
+        }
+        catch (Exception ex)
+        {
+            // Never let the floor's own read block a roll: an unreadable deployment reports "no
+            // stamp", which lets the roll proceed rather than freezing this install silently.
+            _logger?.LogWarning(ex, "[SelfUpdate] could not read the last-rolled annotation; treating as never.");
+            return null;
+        }
+    }
+
     public async Task PatchToVersionAsync(string versionTag, CancellationToken ct)
     {
         if (!CanPatch)
@@ -95,8 +133,21 @@ private async Task PatchDeploymentImageAsync(string deployment, string container
 
         // Strategic-merge patch: matches the container by name and sets ONLY its image. Built with
         // JsonSerializer so the (deeply-nested) braces are never a string-literal hazard.
+        // Strategic-merge patch: matches the container by name and sets ONLY its image, and stamps
+        // WHEN self-update rolled this deployment. The stamp is the floor's restart-surviving state
+        // (MinRollInterval): a successful roll restarts this process, so anything held in memory is
+        // gone exactly when the floor needs it, while an annotation on the object being patched
+        // outlives the pod — and a pod that crash-restarts on an OLD image reads the OLD stamp and
+        // is correctly free to roll at once.
         var body = JsonSerializer.Serialize(new
         {
+            metadata = new
+            {
+                annotations = new Dictionary<string, string>
+                {
+                    [LastRolledAnnotation] = DateTimeOffset.UtcNow.ToString("O"),
+                },
+            },
             spec = new { template = new { spec = new { containers = new[] { new { name = container, image } } } } }
         });
         using var req = new HttpRequestMessage(HttpMethod.Patch,

test/MeshWeaver.Hosting.Monolith.Test/SelfUpdateAvailabilityGateTest.cs

@@ -64,6 +64,13 @@ private sealed class RecordingUpdater : IDeploymentUpdater
         public ImmutableList<string> Tags => tags;
         public bool CanPatch => true;
 
+        /// <summary>No stamp: these fakes exercise the roll decision, not the floor —
+        /// the floor's own behaviour is pinned in SelfUpdateRollFloorTest.</summary>
+        public Task<DateTimeOffset?> LastRolledAtAsync(CancellationToken ct) =>
+
+            Task.FromResult<DateTimeOffset?>(null);
+
+
         public Task PatchToVersionAsync(string versionTag, CancellationToken ct)
         {
             ImmutableInterlocked.Update(ref tags, current => current.Add(versionTag));

test/MeshWeaver.Hosting.Monolith.Test/SelfUpdatePollerResilienceTest.cs

@@ -63,6 +63,13 @@ private sealed class RecordingUpdater : IDeploymentUpdater
         public ImmutableList<string> Tags => _tags;
         public bool CanPatch => true;
 
+        /// <summary>No stamp: these fakes exercise the roll decision, not the floor —
+        /// the floor's own behaviour is pinned in SelfUpdateRollFloorTest.</summary>
+        public Task<DateTimeOffset?> LastRolledAtAsync(CancellationToken ct) =>
+
+            Task.FromResult<DateTimeOffset?>(null);
+
+
         public Task PatchToVersionAsync(string versionTag, CancellationToken ct)
         {
             ImmutableInterlocked.Update(ref _tags, tags => tags.Add(versionTag));