fix(self-update): establishing the policy is not the same as re-establishing it

memex/Memex.Portal.Shared/SelfUpdate/SelfUpdateHostedService.cs

@@ -55,6 +55,15 @@ public class SelfUpdateHostedService : IHostedService
     /// </summary>
     private int _buildEventsSeen;
 
+    /// <summary>
+    /// 1 once the live policy read has produced a value — i.e. once the gate every trigger waits on
+    /// has opened. Chooses the retry cadence for a faulted watch (see
+    /// <see cref="SelfUpdateOptions.PolicyRetryDelay"/>): before it, the poller is inert and a fault
+    /// must be retried quickly; after it, the long interval paces re-establishment. An instance
+    /// field — its lifetime is this service's — never static.
+    /// </summary>
+    private int _policyEstablished;
+
     public SelfUpdateHostedService(
         IMessageHub hub,
         IAcrTagLister acr,
@@ -307,6 +316,8 @@ private IObservable<UpdatePolicyContent> CreatePolicySource()
             .Take(1)
             .SelectMany(_ => Observable
                 .Defer(ReadPolicyStream)
+                // The gate has opened: from here a fault costs freshness, not the feature.
+                .Do(_ => Interlocked.Exchange(ref _policyEstablished, 1))
                 .RetryWhen(ResubscribeAfterRetryInterval("policy stream")))
             .DistinctUntilChanged(c => (c.Policy, c.RequireCiGreen)); // <-- re-switch only on a REAL policy change
     }
@@ -414,9 +425,22 @@ public static IObservable<Unit> NewBuildEventsAcross(IObservable<IEnumerable<Mes
     private Func<IObservable<Exception>, IObservable<long>> ResubscribeAfterRetryInterval(string stage) =>
         faults => faults.SelectMany(ex =>
         {
+            // 🚨 The cadence depends on whether a policy has EVER been read, because the two states
+            // cost completely different things. Before the first read the poller is INERT — every
+            // trigger, the safety net included, is gated on that first emission — so a fault there
+            // takes the whole feature out; after it, the last policy is retained and checks keep
+            // running, so a fault costs only freshness and the long interval is right. Pacing both
+            // at RetryInterval is what left memex-cloud ~400 builds behind (see
+            // SelfUpdateOptions.PolicyEstablishRetryInterval for the measurement).
+            var established = Volatile.Read(ref _policyEstablished) == 1;
+            var delay = _options.PolicyRetryDelay(established);
             _logger?.LogWarning(ex,
-                "[SelfUpdate] {Stage} faulted; re-establishing in {Interval}.", stage, _options.RetryInterval);
-            return Observable.Timer(_options.RetryInterval);
+                "[SelfUpdate] {Stage} faulted; re-establishing in {Interval} ({State}).",
+                stage, delay,
+                established
+                    ? "a policy has been read — checks continue against it meanwhile"
+                    : "NO policy read yet — every check is gated on it, so this install is inert until it succeeds");
+            return Observable.Timer(delay);
         });
 
     /// <summary>One evaluation: list tags → pick target per policy → gate target &gt; current →

src/MeshWeaver.Documentation/Data/WhatsNew/2026-09-01-an-install-that-cannot-read-its-update-policy-no-longer-stops-updating.md

@@ -0,0 +1,37 @@
+---
+Name: An install that cannot read its update policy no longer stops updating for six hours
+Category: Fix
+Description: A slow first read of Admin/UpdatePolicy used to leave a portal with no update checks at all until the six-hour retry came round; it now retries in seconds, because until that read lands the install is not slow, it is inert.
+Icon: ArrowSync
+Order: -20260901
+---
+
+# An install that cannot read its update policy no longer stops updating for six hours
+
+Every self-update trigger — the startup pass, a build completion, an admin changing the policy, and
+the safety net that exists to bound a dead event channel — waits for the install to have actually
+**read** its `Admin/UpdatePolicy` node. That gate is deliberate: a portal must never decide to roll
+itself under a policy it only guessed at, which is how a pinned install once rolled on every pod
+start.
+
+The gate was right; the cadence behind it was not. When that first read faulted — on a busy portal
+the boot-time `SubscribeRequest` can exceed its minute — the retry was paced by `RetryInterval`,
+the six-hour value meant for re-establishing a watch that has already produced a value. But before
+the first read there is nothing to re-establish and nothing retained: the install performs **no
+checks at all**, and the safety net cannot help, because it is behind the same gate. One slow read
+therefore bought six hours of an install that looked perfectly healthy and had quietly stopped
+asking whether a newer release existed.
+
+That is not a hypothesis. On 1 September a portal was found serving `rc8.ci.6829` while the
+registry had reached `rc9.ci.7231` — roughly four hundred builds — with its pinned module set
+advanced past its image, so modules reported themselves as "contributing nothing", memory sat at
+89%, and other repositories' CI gates failed against its bundle endpoints. Two pods of the *same*
+build settled it: the one that logged `policy stream faulted; re-establishing in 06:00:00` had run
+zero checks, while its sibling, whose read succeeded, had run two.
+
+Establishing the policy now has its own cadence (`SelfUpdate__PolicyEstablishRetryInterval`,
+30 seconds by default). Once the policy **has** been read, a later fault keeps the six-hour
+interval exactly as before — the last value is retained and checks keep running, so that fault
+costs freshness rather than the feature. The log line now says which of the two states it is in,
+so the difference between "this install is pacing itself" and "this install is inert" is readable
+rather than inferred.

src/MeshWeaver.Hosting/SelfUpdate/SelfUpdateOptions.cs

@@ -44,6 +44,43 @@ public record SelfUpdateOptions
     /// </summary>
     public TimeSpan RetryInterval { get; init; } = TimeSpan.FromHours(6);
 
+    /// <summary>
+    /// 🚨 The retry cadence while the update policy has NEVER been read — the ESTABLISHING case,
+    /// which <see cref="RetryInterval"/> is the wrong value for.
+    ///
+    /// <para>Every trigger — startup, build completion, policy change, AND the
+    /// <see cref="SafetyNetCheckInterval"/> safety net — is gated on the first policy emission,
+    /// deliberately: the poller must never decide under a policy it has not read (#2731/#2797).
+    /// So until that first read succeeds self-update is not slow, it is entirely INERT, and the
+    /// safety net that exists to bound a dead channel sits BEHIND that same gate and cannot bound
+    /// this one. Pacing the first attempt at the 6 h <see cref="RetryInterval"/> therefore turns a
+    /// single slow boot-time <c>SubscribeRequest</c> into six hours with no checks at all.</para>
+    ///
+    /// <para>Measured, not hypothetical: on 2026-09-01 memex-cloud served
+    /// <c>memex.meshweaver.cloud</c> from <c>rc8.ci.6829</c> while ACR had reached
+    /// <c>rc9.ci.7231</c> — roughly 400 builds behind, its pinned module set advanced past the
+    /// image, modules "contributing nothing", and satellite CI gates failing on the bundle
+    /// endpoints. Two pods of the SAME build told the whole story: the one whose policy read
+    /// faulted (<c>policy stream faulted; re-establishing in 06:00:00</c>) had run ZERO checks,
+    /// while its sibling, which read the policy cleanly, had run two.</para>
+    ///
+    /// <para>Once the policy HAS been read a later fault keeps the long
+    /// <see cref="RetryInterval"/>: the last value is retained, checks keep running, and the
+    /// pacing intent is untouched. <see cref="PolicyRetryDelay"/> is that decision, and it is the
+    /// only place the two cadences are chosen between.</para>
+    /// </summary>
+    public TimeSpan PolicyEstablishRetryInterval { get; init; } = TimeSpan.FromSeconds(30);
+
+    /// <summary>
+    /// How long a faulted watch waits before re-establishing: <see cref="PolicyEstablishRetryInterval"/>
+    /// while the poller has never read a policy (it is inert until it has, so a fault in that state
+    /// costs the whole feature), <see cref="RetryInterval"/> once it has (the value is retained and
+    /// checks continue, so the fault costs only freshness). Pure — pinned by
+    /// <c>SelfUpdatePollerResilienceTest</c> in both directions.
+    /// </summary>
+    public TimeSpan PolicyRetryDelay(bool policyEstablished) =>
+        policyEstablished ? RetryInterval : PolicyEstablishRetryInterval;
+
     /// <summary>
     /// The minimum time between two automatic rolls of this install.
     ///

test/MeshWeaver.Hosting.Monolith.Test/SelfUpdatePollerResilienceTest.cs

@@ -196,9 +196,10 @@ public async Task Poller_survives_faulted_first_read_and_processes_a_policy_chan
         var updater = new RecordingUpdater();
         var options = new SelfUpdateOptions
         {
-            // The poll interval doubles as the fault-resubscribe delay — short so the test observes
-            // the recovery promptly.
-            RetryInterval = TimeSpan.FromMilliseconds(500),
+            // A first-read fault is the ESTABLISHING case, so this is the interval that paces the
+            // recovery — short here so the test observes it promptly. RetryInterval is deliberately
+            // left at its production default: it must not be what gets this install moving again.
+            PolicyEstablishRetryInterval = TimeSpan.FromMilliseconds(500),
             DefaultPolicy = UpdatePolicyKind.Continuous,
         };
         var service = new FaultingFirstReadService(
@@ -209,9 +210,11 @@ public async Task Poller_survives_faulted_first_read_and_processes_a_policy_chan
         try
         {
             // 1. The first live read FAULTS (TimeoutException — the prod shape) right after the seed.
-            //    The startup default (Continuous) still drives the first poll, and the faulted read
-            //    must retry silently in the background. Pre-fix the fault terminated the whole
-            //    subscription permanently.
+            //    Nothing can proceed on a guessed policy (#2731/#2797 removed the synthetic default
+            //    emission), so EVERY trigger waits on the re-read: the roll below is proof the
+            //    faulted read came back on its own. Pre-fix the fault terminated the whole
+            //    subscription permanently; pre-#PolicyEstablishRetryInterval it came back only after
+            //    the 6 h RetryInterval, which is the same outage with a longer name.
             var firstTag = await updater.Patched
                 .FirstAsync()
                 .Timeout(TimeSpan.FromSeconds(30))
@@ -344,4 +347,76 @@ await updater.Patched
             await service.StopAsync(CancellationToken.None);
         }
     }
+
+    /// <summary>
+    /// 🚨 The memex-cloud defect, at production cadence (2026-09-01). A first policy read that
+    /// faults must NOT park this install on <see cref="SelfUpdateOptions.RetryInterval"/>: every
+    /// trigger — startup, build completion, policy change and the safety net — is gated on that
+    /// first emission, so until it lands the install performs no checks AT ALL and the safety net
+    /// cannot bound it, because the safety net is behind the same gate.
+    ///
+    /// <para>What it cost: memex.meshweaver.cloud served <c>rc8.ci.6829</c> while ACR had reached
+    /// <c>rc9.ci.7231</c> — about 400 builds — with its pinned module set advanced past the image
+    /// (modules "contributing nothing"), memory at 89% and satellite CI gates failing on its bundle
+    /// endpoints. Two pods of the same build proved the mechanism: the one that logged <c>policy
+    /// stream faulted; re-establishing in 06:00:00</c> had run ZERO checks, its sibling two.</para>
+    ///
+    /// <para>🚨 The pre-existing first-read test cannot see this, and that is the point of adding a
+    /// second one: it sets <c>RetryInterval</c> itself to 500 ms, so it proves recovery HAPPENS
+    /// while saying nothing about recovery happening usefully. Here RetryInterval keeps its
+    /// production default, so the only thing that can make this test pass is the establishing
+    /// cadence.</para>
+    /// </summary>
+    [Fact(Timeout = 60000)]
+    public async Task A_faulted_FIRST_policy_read_recovers_at_the_establish_cadence_not_the_six_hour_one()
+    {
+        var ct = TestContext.Current.CancellationToken;
+        var acr = new FakeAcrTagLister();
+        var updater = new RecordingUpdater();
+        var options = new SelfUpdateOptions
+        {
+            // PRODUCTION value, deliberately: if the fix regresses, this is the wait, and the test
+            // fails by timeout rather than by passing for the wrong reason.
+            RetryInterval = TimeSpan.FromHours(6),
+            PolicyEstablishRetryInterval = TimeSpan.FromMilliseconds(200),
+            DefaultPolicy = UpdatePolicyKind.Continuous,
+        };
+        var service = new FaultingFirstReadService(
+            Mesh, acr, updater, options,
+            Mesh.ServiceProvider.GetService<ILogger<SelfUpdateHostedService>>());
+
+        await service.StartAsync(CancellationToken.None);
+        try
+        {
+            // The positive signal is a ROLL: it can only happen after a policy has actually been
+            // read, which can only happen after the faulted first read was re-established.
+            var applied = await updater.Patched
+                .FirstAsync()
+                .Timeout(TimeSpan.FromSeconds(30))
+                .Await(ct);
+            applied.Should().Be(FakeAcrTagLister.CiTag);
+            service.ReadSubscriptions.Should().BeGreaterThanOrEqualTo(2,
+                "the recovery must have gone through the injected fault and a re-subscription");
+        }
+        finally
+        {
+            await service.StopAsync(CancellationToken.None);
+        }
+    }
+
+    /// <summary>
+    /// The decision itself, both directions and in the right order: fast while no policy has been
+    /// read (the install is inert), paced once one has (the value is retained and checks continue).
+    /// Pure, so it pins the intent without waiting on a mesh.
+    /// </summary>
+    [Fact]
+    public void PolicyRetryDelay_is_fast_while_establishing_and_paced_once_established()
+    {
+        var options = new SelfUpdateOptions();
+
+        options.PolicyRetryDelay(policyEstablished: false).Should().Be(options.PolicyEstablishRetryInterval);
+        options.PolicyRetryDelay(policyEstablished: true).Should().Be(options.RetryInterval);
+        options.PolicyEstablishRetryInterval.Should().BeLessThan(options.RetryInterval,
+            "an install with no policy yet is INERT — it must not wait a re-establishment interval");
+    }
 }