fix(#1979): answer the restart-as-activation question per PROCESS, and never reassuringly

memex/aspire/Memex.Portal.Distributed/PendingModuleActivationHealthCheck.cs

@@ -4,9 +4,8 @@
 namespace Memex.Portal.Distributed;
 
 /// <summary>
-/// Reports modules that are LANDED but not yet LOADED — the restart-as-activation signal
-/// (<see cref="ModuleActivationList.PendingRestart"/>) that #1979 found written by two sites,
-/// consumed at boot, and read by nothing else.
+/// Reports modules that are LANDED but not yet LOADED — the restart-as-activation signal that
+/// #1979 found written by two sites, consumed at boot, and read by nothing else.
 ///
 /// <para><b>Why a health check rather than a log line.</b> Loading is restart-as-activation by
 /// design, which makes the restart part of the install — and an install whose last step is
@@ -15,46 +14,54 @@ namespace Memex.Portal.Distributed;
 /// paid-and-never-delivered: a success message that is true about the half that ran. An operator
 /// surface makes a fleet with pending activations visible without opening a portal.</para>
 ///
+/// <para>🚨 <b>Per PROCESS, not per deployment.</b> It asks
+/// <see cref="PendingModuleActivations"/> — the persisted sidecar compared against the assemblies
+/// THIS pod actually loaded — rather than reading <c>PendingRestart</c>, which is one
+/// deployment-wide boolean that the next boot clears. On a multi-replica deployment the pod that
+/// clears it is not the pod that is missing the module: replica A lands one and sets the flag,
+/// replica B restarts for an unrelated reason and resets it, and A keeps serving without the
+/// module while every surface reads "nothing pending". The comparison also names only what is
+/// actually missing here, instead of every enabled entry.</para>
+///
 /// <para>🚨 <b>DEGRADED, never Unhealthy.</b> A pending activation is a to-do, not a fault: the pod
 /// is serving correctly with the modules it loaded, and failing readiness here would stall a
 /// rollout over work the rollout itself performs. Degraded is reported in the payload and by
 /// <c>/health</c>'s aggregate status without taking the instance out of service.</para>
 ///
-/// <para>🚨 Reads the PERSISTED sidecar, not an in-memory flag. The process that landed the module
-/// is not necessarily the process a viewer or an operator is talking to — the state outliving the
-/// request that created it is the entire point.</para>
+/// <para>🚨 <b>An UNREADABLE sidecar is Degraded too, never Healthy.</b> A file that cannot be
+/// parsed is the absence of an answer, and this check reported it as "no pending activation" —
+/// because <see cref="ModuleActivationSidecar.Read"/> swallows corruption into the empty list. A
+/// check that cannot reach its evidence must say so, not default to the reassuring
+/// answer.</para>
 /// </summary>
-public sealed class PendingModuleActivationHealthCheck(string moduleRoot) : IHealthCheck
+public sealed class PendingModuleActivationHealthCheck(PendingModuleActivations activations) : IHealthCheck
 {
+    /// <summary>Constructs the reader from a module root — the shape the portal's boot uses.</summary>
+    public PendingModuleActivationHealthCheck(string moduleRoot)
+        : this(new PendingModuleActivations(moduleRoot)) { }
+
     /// <inheritdoc />
     public Task<HealthCheckResult> CheckHealthAsync(
         HealthCheckContext context, CancellationToken cancellationToken = default)
     {
-        ModuleActivationList activation;
+        ModuleActivationReport report;
         try
         {
-            activation = ModuleActivationSidecar.Read(moduleRoot);
+            report = activations.Read();
         }
         catch (Exception exception)
         {
-            // An unreadable sidecar is not a pending activation, and must not read as one.
-            return Task.FromResult(HealthCheckResult.Healthy(
-                $"module activation state unreadable ({exception.GetType().Name}) — "
-                + "reporting no pending activation rather than inventing one"));
+            return Task.FromResult(HealthCheckResult.Degraded(
+                "module activation state could not be determined "
+                + $"({exception.GetType().Name}: {exception.Message}) — this pod cannot say whether "
+                + "a landed module is waiting on a restart"));
         }
 
-        if (!activation.PendingRestart)
-            return Task.FromResult(HealthCheckResult.Healthy("no module activation pending"));
-
-        var names = activation.Entries
-            .Where(e => e.Enabled)
-            .Select(e => e.Name)
-            .OrderBy(n => n, StringComparer.OrdinalIgnoreCase)
-            .ToArray();
+        if (report.IsUndetermined)
+            return Task.FromResult(HealthCheckResult.Degraded(report.Describe()));
 
-        return Task.FromResult(HealthCheckResult.Degraded(
-            $"{names.Length} module(s) are landed but not yet loaded — a restart activates them: "
-            + string.Join(", ", names.Take(10))
-            + (names.Length > 10 ? $", …(+{names.Length - 10})" : string.Empty)));
+        return Task.FromResult(report.HasPending
+            ? HealthCheckResult.Degraded(report.Describe())
+            : HealthCheckResult.Healthy(report.Describe()));
     }
 }

src/MeshWeaver.PluginCatalog/ModuleActivationStatus.cs

@@ -0,0 +1,116 @@
+using System.Collections.Immutable;
+using System.Reflection;
+
+namespace MeshWeaver.PluginCatalog;
+
+/// <summary>
+/// One module that has LANDED on the deployment's volume but is not LOADED in this process — the
+/// unit of the restart-as-activation signal (#1979), carrying enough to point a viewer back at the
+/// install that produced it.
+/// </summary>
+/// <param name="Name">The module's assembly simple name, as the activation entry records it.</param>
+/// <param name="PackagePath">The mesh path of the install record that landed it, when the store
+/// lane wrote one — the back-pointer a package card matches on. Null for an entry with no
+/// recorded origin.</param>
+/// <param name="Version">The package version the landed bundle was served at, when recorded.</param>
+public sealed record PendingModuleActivation(string Name, string? PackagePath, string? Version);
+
+/// <summary>
+/// Derives, per PROCESS, which activated modules are not running here yet.
+///
+/// <para>🚨 <b>Why not just read <see cref="ModuleActivationList.PendingRestart"/>.</b> That flag is
+/// a single deployment-wide boolean that the NEXT boot clears — and on a multi-replica deployment
+/// the pod that clears it is not the pod that is missing the module. Replica A lands a module and
+/// sets the flag; replica B restarts for an unrelated reason, applies the list and resets it; A is
+/// still serving WITHOUT the module while every surface reads "nothing pending". The flag answers
+/// "did something change since some boot", which is not the question a buyer or an operator is
+/// asking. Comparing the persisted list against what THIS process actually loaded answers it
+/// exactly, per pod, and needs no extra state to stay true.</para>
+///
+/// <para>Pure and total: the caller supplies both the list and the loaded set, so the rule is
+/// testable with no filesystem, no reflection and no host.</para>
+/// </summary>
+public static class ModuleActivationStatus
+{
+    /// <summary>
+    /// The enabled entries in <paramref name="activation"/> whose assembly is not among
+    /// <paramref name="loadedAssemblyNames"/>.
+    ///
+    /// <para>A DISABLED entry is never pending: it is the record of an uninstall, and its module
+    /// being absent from this process is the outcome, not a to-do. (An uninstall that has not taken
+    /// effect yet — the module still loaded — is deliberately not reported either: nothing is
+    /// missing from the user's point of view, and reporting it would put an alarming "restart
+    /// required" on a package that has just been removed.)</para>
+    /// </summary>
+    /// <param name="activation">The persisted activation list.</param>
+    /// <param name="loadedAssemblyNames">Assembly SIMPLE names loaded in this process.</param>
+    public static ImmutableList<PendingModuleActivation> NotYetLoaded(
+        ModuleActivationList activation,
+        IReadOnlySet<string> loadedAssemblyNames)
+    {
+        ArgumentNullException.ThrowIfNull(activation);
+        ArgumentNullException.ThrowIfNull(loadedAssemblyNames);
+
+        return activation.Entries
+            .Where(entry => entry.Enabled
+                && !string.IsNullOrWhiteSpace(entry.Name)
+                && !loadedAssemblyNames.Contains(entry.Name))
+            .Select(entry => new PendingModuleActivation(entry.Name, entry.PackagePath, entry.Version))
+            .ToImmutableList();
+    }
+
+    /// <summary>
+    /// Whether the install record at <paramref name="packagePath"/> landed a module that is not
+    /// loaded here — the per-package question the Store's install step asks so it can say
+    /// "restart required to finish activating this" instead of a bare "installed".
+    ///
+    /// <para>A blank path matches nothing. It is not a wildcard: a package whose card has no path
+    /// to match on must not inherit some other package's pending restart.</para>
+    /// </summary>
+    public static bool IsPendingForPackage(
+        IEnumerable<PendingModuleActivation> pending, string? packagePath) =>
+        !string.IsNullOrWhiteSpace(packagePath)
+        && pending.Any(p => string.Equals(
+            p.PackagePath?.Trim('/'), packagePath.Trim('/'), StringComparison.OrdinalIgnoreCase));
+
+    /// <summary>
+    /// The simple names of every assembly loaded into <paramref name="domain"/>. The set the
+    /// derivation above is asked against on a live host.
+    /// </summary>
+    public static IReadOnlySet<string> LoadedAssemblyNames(AppDomain domain)
+    {
+        ArgumentNullException.ThrowIfNull(domain);
+        return domain.GetAssemblies()
+            .Select(a => a.GetName().Name)
+            .Where(n => !string.IsNullOrEmpty(n))
+            .Select(n => n!)
+            .ToHashSet(StringComparer.OrdinalIgnoreCase);
+    }
+
+    /// <summary>Convenience for the live process.</summary>
+    public static IReadOnlySet<string> LoadedAssemblyNames() =>
+        LoadedAssemblyNames(AppDomain.CurrentDomain);
+
+    /// <summary>
+    /// One human-readable line naming what is pending — shared by every surface so an operator and
+    /// a buyer are never told different numbers.
+    /// </summary>
+    /// <param name="pending">The pending activations.</param>
+    /// <param name="maxNamed">How many are named before the line truncates.</param>
+    public static string Describe(IReadOnlyCollection<PendingModuleActivation> pending, int maxNamed = 10)
+    {
+        ArgumentNullException.ThrowIfNull(pending);
+        if (pending.Count == 0)
+            return "no module activation pending";
+
+        var names = pending
+            .Select(p => p.Name)
+            .OrderBy(n => n, StringComparer.OrdinalIgnoreCase)
+            .ToArray();
+
+        return $"{names.Length} module(s) are landed but not yet loaded in this process — "
+            + "a restart activates them: "
+            + string.Join(", ", names.Take(Math.Max(1, maxNamed)))
+            + (names.Length > maxNamed ? $", …(+{names.Length - maxNamed})" : string.Empty);
+    }
+}

src/MeshWeaver.PluginCatalog/PendingModuleActivations.cs

@@ -0,0 +1,106 @@
+using System.Collections.Immutable;
+using Microsoft.Extensions.Configuration;
+
+namespace MeshWeaver.PluginCatalog;
+
+/// <summary>
+/// What this process can say about landed-but-unloaded modules — either an ANSWER, or an explicit
+/// admission that it could not determine one.
+///
+/// <para>🚨 The two are not the same and must never render the same. "Nothing pending" is evidence;
+/// "I could not read the activation sidecar" is the absence of evidence, and a surface that shows
+/// the second as the first is exactly the gate that cannot run but looks like a gate that
+/// passed.</para>
+/// </summary>
+/// <param name="Pending">The modules landed on the volume but not loaded in this process.</param>
+/// <param name="UndeterminedReason">Why the answer is unknown, or <c>null</c> when it is known.
+/// When set, <paramref name="Pending"/> is empty and means nothing.</param>
+public sealed record ModuleActivationReport(
+    ImmutableList<PendingModuleActivation> Pending,
+    string? UndeterminedReason = null)
+{
+    /// <summary>True when this process could not establish the activation state at all.</summary>
+    public bool IsUndetermined => UndeterminedReason is not null;
+
+    /// <summary>True when the state is KNOWN and something is waiting on a restart.</summary>
+    public bool HasPending => !IsUndetermined && !Pending.IsEmpty;
+
+    /// <summary>The one line every surface renders, so nobody is told two different stories.</summary>
+    public string Describe() =>
+        UndeterminedReason is { } reason
+            ? "module activation state could not be determined — " + reason
+            : ModuleActivationStatus.Describe(Pending);
+}
+
+/// <summary>
+/// Reads the restart-as-activation state for THIS process: the persisted activation sidecar
+/// compared against the assemblies actually loaded here (#1979).
+///
+/// <para>This is the seam every surface consults — the operator health check, and (through
+/// <c>hub.ServiceProvider</c>) a package card that needs to say "restart required to finish
+/// activating this" instead of a bare "installed". One reader, so the numbers cannot
+/// disagree.</para>
+///
+/// <para>A pull-on-demand READER: it starts nothing, subscribes to nothing and writes nothing, so
+/// an instance that never asks pays nothing. The read is a single small file, which is why it is
+/// plain and synchronous — the same reason <see cref="ModuleActivationSidecar"/> is.</para>
+/// </summary>
+public sealed class PendingModuleActivations(string moduleRoot)
+{
+    /// <summary>Constructs from configuration, resolving the writable module root once.</summary>
+    public PendingModuleActivations(IConfiguration? configuration)
+        : this(ModuleRoot.Resolve(configuration)) { }
+
+    /// <summary>The deployment root whose <c>modules/</c> sidecar is read.</summary>
+    public string ModuleRootPath { get; } = moduleRoot;
+
+    /// <summary>
+    /// The current report. Recomputed per call — the state changes underneath a running process
+    /// (that is the whole point), so a cached answer would be wrong exactly when it matters.
+    /// </summary>
+    public ModuleActivationReport Read() => Read(ModuleActivationStatus.LoadedAssemblyNames());
+
+    /// <summary>Testable form: the caller supplies what counts as loaded.</summary>
+    public ModuleActivationReport Read(IReadOnlySet<string> loadedAssemblyNames)
+    {
+        string? corrupt = null;
+        ModuleActivationList activation;
+        try
+        {
+            // 🚨 The corruption callback is not optional here. ModuleActivationSidecar.Read
+            // swallows an unparseable file into the EMPTY list, so a surface that ignores the
+            // callback reports a corrupt sidecar as "nothing pending" — cheerfully, forever. That
+            // is the shape this whole cluster of defects has in common.
+            activation = ModuleActivationSidecar.Read(ModuleRootPath, reason => corrupt = reason);
+        }
+        catch (Exception exception)
+        {
+            return new ModuleActivationReport(
+                [], $"the activation sidecar under '{ModuleRootPath}' could not be opened "
+                    + $"({exception.GetType().Name}: {exception.Message})");
+        }
+
+        if (corrupt is not null)
+            return new ModuleActivationReport([], corrupt);
+
+        return new ModuleActivationReport(
+            ModuleActivationStatus.NotYetLoaded(activation, loadedAssemblyNames));
+    }
+
+    /// <summary>
+    /// Whether the install record at <paramref name="packagePath"/> landed a module that has not
+    /// loaded here.
+    ///
+    /// <para>🚨 Returns <c>false</c> when the state is UNDETERMINED, and that is deliberate: this
+    /// answers a per-package question whose only honest fallback is "I have nothing to say about
+    /// this package". The undetermined case is reported by the operator surface, which is where an
+    /// unreadable sidecar is actionable — putting "restart required" on every package card because
+    /// a file could not be parsed would be noise a buyer cannot act on.</para>
+    /// </summary>
+    public bool IsPendingForPackage(string? packagePath)
+    {
+        var report = Read();
+        return !report.IsUndetermined
+            && ModuleActivationStatus.IsPendingForPackage(report.Pending, packagePath);
+    }
+}

src/MeshWeaver.PluginCatalog/PluginCatalogConfigurationExtensions.cs

@@ -126,6 +126,16 @@ public static TBuilder AddPluginCatalog<TBuilder>(this TBuilder builder) where T
                 // fails with a denied-path error the publisher sees as HTTP 409. See ModuleRoot.
                 .AddSingleton(sp => new ModuleLandingService(
                     sp.GetService<ILogger<ModuleLandingService>>(),
+                    ModuleRoot.Resolve(sp.GetService<IConfiguration>())))
+                // The restart-as-activation READER (#1979): which landed modules are not loaded in
+                // THIS process. Registered beside the writer and rooted at the same resolved
+                // module root — a reader looking at a different directory than the writer is how
+                // "installed, and nothing happened" becomes unexplainable. A plain singleton: it
+                // starts nothing and writes nothing, so an instance that never asks pays nothing.
+                // 🚨 It is registered rather than merely constructible so a NodeType's layout area
+                // can resolve it from hub.ServiceProvider — the Store's install step is the
+                // surface where the missing last step is actually met.
+                .AddSingleton(sp => new PendingModuleActivations(
                     ModuleRoot.Resolve(sp.GetService<IConfiguration>()))))
             .ConfigureHub(config =>
             {