feat(hosting): instance keys rotate through the operator — AdoptKeyHash + hosting-kv-rotate (#2802)

deploy/aks/operator/bin/hosting-kv-rotate

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# hosting-kv-rotate --vault <name> --prefix <prefix> --namespace <ns> --synced-secret <k8s secret> [--key <config key>]
+#
+# Rotate this instance's plugin-registry key: mint a fresh `mwi_` key, put it in Key Vault, and
+# report ONLY ITS HASH back to the mesh. The registry adopts the hash (the control plane reacts to
+# the `::hosting:: key_hash=` line below); the restart that follows in the plan puts the new value
+# into the pods. The raw key exists in exactly two places, ever: this process's memory and the vault.
+#
+# 🚨 IT NEVER PRINTS THE KEY. Not on success, not on failure, not in a `set -x` (deliberately not
+# enabled), not in a `::hosting::` fact. The 2026-08-30 exposure this verb exists to repair came
+# from a CHECK that selected `.value` while asking "is this stored in the clear?" — the audit was
+# the leak. Every read below compares hashes and names.
+#
+# 🚨 IT DOES OVERWRITE — the one operator script that does. `hosting-kv-ensure` refuses to touch
+# an existing secret because the master key seals stored credentials; a registry key seals
+# nothing and is meant to be replaced. That is why this is a separate verb with its own name, and
+# why `kv-ensure` must never grow a --force.
+# shellcheck source=_common.sh
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/_common.sh"
+# shellcheck disable=SC2034  # read by hosting::die in _common.sh, which shellcheck does not follow here
+HOSTING_CMD="hosting-kv-rotate"
+vault="" prefix="" namespace="" synced="" key="PluginCatalog__RegistryToken"
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --vault)         vault="${2:-}";     shift 2 ;;
+    --prefix)        prefix="${2:-}";    shift 2 ;;
+    --namespace)     namespace="${2:-}"; shift 2 ;;
+    --synced-secret) synced="${2:-}";    shift 2 ;;
+    --key)           key="${2:-}";       shift 2 ;;
+    *) hosting::die "unknown argument '$1'" ;;
+  esac
+done
+hosting::need_flag vault "$vault"
+hosting::need_flag namespace "$namespace"
+hosting::need_flag synced-secret "$synced"
+hosting::safe_name vault "$vault"
+hosting::safe_name namespace "$namespace"
+hosting::safe_name synced-secret "$synced"
+[ -z "$prefix" ] || hosting::safe_name prefix "$prefix"
+
+# The vault object name follows the SecretProviderClass convention: <prefix><Section>-<Key>, every
+# `__` becoming `-` (HelmValues.VaultSecretName). For PluginCatalog__RegistryToken that is
+# <prefix>PluginCatalog-RegistryToken — the exact name hosting-kv-ensure lists as REQUIRED.
+object="${prefix}${key//__/-}"
+
+hosting::step "Mint a new instance key"
+# 32 random bytes, URL-safe base64, no padding, `mwi_` prefix — InstanceKeys.Generate, exactly.
+raw="mwi_$(head -c 32 /dev/urandom | base64 | tr '+/' '-_' | tr -d '=\n')"
+hash="$(printf '%s' "$raw" | sha256sum | cut -c1-64)"
+hosting::log "new key hash ${hash:0:12}… (the key itself is not shown, here or anywhere)"
+
+hosting::step "Store the new key in Key Vault"
+if hosting::dry; then
+  hosting::log "(dry run) would set ${object} in vault ${vault}"
+else
+  az keyvault secret set --vault-name "$vault" --name "$object" --value "$raw" --output none \
+    || { unset raw; hosting::die "could not set ${object} in vault ${vault}"; }
+fi
+unset raw
+hosting::log "stored  ${object} in vault ${vault}"
+
+# 🚨 The fact the mesh reacts to. The control plane adopts this hash on the registry BEFORE the
+# restart below lands the new value in the pods, so the pods never come up against a registry
+# that still expects the old key. A hash is not a secret: the registry stores it in the clear.
+hosting::say key_hash "$hash"
+
+hosting::step "Wait for the synced Secret to carry the new key"
+# The Key Vault CSI driver re-syncs on its rotation interval; a restart that reads the OLD synced
+# Secret would put the OLD key into the new pods (DeploymentAKS → "KeyVault CSI env timing").
+# Compared by HASH of the decoded value — the value itself is never printed.
+if hosting::dry; then
+  hosting::log "(dry run) would wait for ${synced}/${key} in ${namespace} to hash to ${hash:0:12}…"
+else
+  attempts="${HOSTING_KV_SYNC_ATTEMPTS:-40}"
+  synced_hash=""
+  for i in $(seq 1 "$attempts"); do
+    synced_hash="$(kubectl -n "$namespace" get secret "$synced" -o "jsonpath={.data.${key}}" 2>/dev/null \
+      | base64 -d 2>/dev/null | sha256sum | cut -c1-64)"
+    if [ "$synced_hash" = "$hash" ]; then
+      hosting::log "synced Secret ${synced} carries the new key (attempt ${i})"
+      break
+    fi
+    sleep 15
+  done
+  [ "$synced_hash" = "$hash" ] \
+    || hosting::die "synced Secret ${synced} in ${namespace} did not pick up ${object} within $((attempts * 15))s — the pods would restart onto the OLD key. The SecretProviderClass must list ${object} and the driver's rotation must be enabled."
+fi
+hosting::say kv_rotated 1

memex/Memex.Portal.Shared/Authentication/MeshWeaverInstanceService.cs

@@ -29,7 +29,7 @@ namespace Memex.Portal.Shared.Authentication;
 /// </summary>
 public sealed class MeshWeaverInstanceService(
     IMeshService nodeFactory, IMessageHub hub, ILogger<MeshWeaverInstanceService> logger,
-    IConfiguration? configuration = null)
+    IConfiguration? configuration = null) : IInstanceKeyRegistry
 {
     /// <summary>Config key listing the <c>Source/Package</c> entries every new registration is
     /// seeded with (<c>PluginCatalog:DefaultGrants:0</c>, …). Absent/empty → registration grants
@@ -388,6 +388,123 @@ public IObservable<string> ReissueKey(string instancePath)
             });
     }
 
+    /// <summary>
+    /// Adopts a key that was minted ELSEWHERE — the Hosting operator's <c>hosting-kv-rotate</c>
+    /// job generates the raw <c>mwi_</c> key, puts it straight into Key Vault, and reports only its
+    /// SHA-256 through the <c>::hosting:: key_hash=</c> marker line. This is the registry-side half
+    /// of that rotation: the instance node takes the new hash, a fresh index entry points at it,
+    /// and the PREVIOUS index entry is deleted so the old key stops authenticating the moment this
+    /// completes (<see cref="ReissueKey"/> left that entry "to be cleaned up separately"; a rotation
+    /// driven by a lifecycle verb must not).
+    ///
+    /// <para>🚨 The raw key never reaches this process. That is the point of the split: the
+    /// registry stores hashes, the operator holds the vault write, and nothing in between logs,
+    /// displays or persists the secret. A caller that already has the raw key and wants a hash for
+    /// it uses <see cref="InstanceKeys.Hash"/> — never a log line.</para>
+    /// </summary>
+    /// <param name="instancePath">The <c>MeshWeaverInstance</c> node to re-key.</param>
+    /// <param name="keyHash">The lowercase SHA-256 hex of the new raw key (64 chars).</param>
+    public IObservable<Unit> AdoptKeyHash(string instancePath, string keyHash)
+    {
+        if (keyHash is not { Length: 64 } || !keyHash.All(Uri.IsHexDigit) || keyHash != keyHash.ToLowerInvariant())
+            return Observable.Throw<Unit>(new ArgumentException(
+                "keyHash must be the lowercase SHA-256 hex of the raw key (64 hex chars) — the registry "
+                + "stores hashes only, and this one arrived in the wrong shape", nameof(keyHash)));
+
+        var workspace = hub.GetWorkspace();
+        return workspace.GetMeshNodeStream(instancePath)
+            .Where(node => node is not null)
+            .Take(1)
+            .Timeout(TimeSpan.FromSeconds(10))
+            .SelectMany(node =>
+            {
+                var instance = node!.ContentAs<MeshWeaverInstance>(hub.JsonSerializerOptions)
+                    ?? throw new InvalidOperationException($"Node {instancePath} is not a MeshWeaverInstance.");
+                if (string.Equals(instance.KeyHash, keyHash, StringComparison.Ordinal))
+                {
+                    logger.LogInformation("Instance {InstanceId} already carries key hash {Prefix}… — nothing to adopt",
+                        instance.InstanceId, InstanceKeys.HashPrefix(keyHash));
+                    return Observable.Return(Unit.Default);
+                }
+                var previousHash = instance.KeyHash;
+                return WriteIndex(keyHash, instancePath, instance.InstanceId)
+                    .SelectMany(_ => workspace.GetMeshNodeStream(instancePath)
+                        .Update(current => current with
+                        {
+                            Content = (current.ContentAs<MeshWeaverInstance>(hub.JsonSerializerOptions)
+                                       ?? instance) with
+                            {
+                                KeyHash = keyHash,
+                                KeyIssuedAt = DateTimeOffset.UtcNow,
+                            },
+                        }))
+                    .SelectMany(_ => DeleteIndex(previousHash))
+                    .Select(_ =>
+                    {
+                        logger.LogInformation("Adopted rotated key for instance {InstanceId} (hash prefix {Prefix}); previous index entry removed",
+                            instance.InstanceId, InstanceKeys.HashPrefix(keyHash));
+                        return Unit.Default;
+                    });
+            });
+    }
+
+    /// <inheritdoc cref="IInstanceKeyRegistry.AdoptKeyHash"/>
+    /// <remarks>Resolves the instance NODE by id first — the operator knows the deployment's
+    /// instance id, never the owner's node path — then adopts on that path.</remarks>
+    IObservable<Unit> IInstanceKeyRegistry.AdoptKeyHash(string instanceId, string keyHash)
+    {
+        if (string.IsNullOrWhiteSpace(instanceId))
+            return Observable.Throw<Unit>(new ArgumentException("instanceId is required", nameof(instanceId)));
+        var meshService = hub.ServiceProvider.GetRequiredService<IMeshService>();
+        var accessService = hub.ServiceProvider.GetRequiredService<AccessService>();
+        var request = new MeshQueryRequest
+        {
+            Query = $"nodeType:{MeshWeaverInstanceNodeType.NodeType} id:{instanceId}",
+        };
+        return Observable.Using(
+                () => accessService.ImpersonateAsSystem(),
+                _ => meshService.Query(request))
+            .Take(1)
+            .Timeout(TimeSpan.FromSeconds(10))
+            .SelectMany(results =>
+            {
+                // The index nodes share the NodeType and use the hash prefix as id, so an instance
+                // record is the hit whose id IS the instance id and whose namespace is not the index.
+                var node = results.FirstOrDefault(n =>
+                    string.Equals(n.Id, instanceId, StringComparison.Ordinal)
+                    && !string.Equals(n.Namespace, MeshWeaverInstanceNodeType.IndexNamespace, StringComparison.Ordinal));
+                return node?.Path is null
+                    ? Observable.Throw<Unit>(new InvalidOperationException(
+                        $"no MeshWeaverInstance is registered as '{instanceId}' — nothing to rotate"))
+                    : AdoptKeyHash(node.Path, keyHash);
+            });
+    }
+
+    /// <summary>Removes the index entry for a superseded hash. Absent is fine — a first-time
+    /// adoption or an already-cleaned entry both leave nothing to delete.</summary>
+    private IObservable<Unit> DeleteIndex(string? previousHash)
+    {
+        if (string.IsNullOrWhiteSpace(previousHash) || previousHash.Length < InstanceKeys.HashPrefixLength)
+            return Observable.Return(Unit.Default);
+        var accessService = hub.ServiceProvider.GetRequiredService<AccessService>();
+        var indexPath = $"{MeshWeaverInstanceNodeType.IndexNamespace}/{InstanceKeys.HashPrefix(previousHash)}";
+        return Observable.Defer(() =>
+        {
+            var disposable = accessService.SwitchAccessContext(
+                new AccessContext { ObjectId = WellKnownUsers.System, Name = "system-security" });
+            return nodeFactory.DeleteNode(indexPath)
+                .Select(_ => Unit.Default)
+                .Catch<Unit, Exception>(ex =>
+                {
+                    // Absent already: not a failure. Anything else IS one — a stale entry that
+                    // still authenticates is the hole this method exists to close.
+                    logger.LogDebug(ex, "No index entry to remove at {Path}", indexPath);
+                    return Observable.Return(Unit.Default);
+                })
+                .Finally(() => disposable.Dispose());
+        });
+    }
+
     /// <summary>Enables or disables an instance. A disabled instance fails authentication while its
     /// grants stay intact, so access can be cut without losing the grant history.</summary>
     public IObservable<MeshNode> SetDisabled(string instancePath, bool disabled) =>

src/MeshWeaver.Documentation/Data/WhatsNew/2026-08-30-instance-keys-rotate-through-the-hosting-operator.md

@@ -0,0 +1,36 @@
+---
+Name: Instance keys rotate through the Hosting operator — never through a terminal
+Category: Feature
+Description: A registry instance key can now be rotated as a lifecycle verb — the operator job mints the key straight into Key Vault and reports only its hash; the registry adopts the hash and retires the old one. Nothing prints, logs or stores the secret.
+Icon: Key
+Order: -20260830
+---
+
+# Instance keys rotate through the Hosting operator — never through a terminal
+
+A `mwi_` instance key authenticates a deployment to the plugin registry. Until now it could be
+*issued* (at registration) but not cleanly *rotated*: `ReissueKey` existed with nothing calling it,
+and the only way to move a new value into a running portal was by hand — through a terminal that
+would have to see the key.
+
+This change ships the platform half of a rotation that no human, log or node ever sees in the clear:
+
+- **`hosting-kv-rotate`** — a new operator script, the one deliberately allowed to overwrite a vault
+  secret (`hosting-kv-ensure` never will, because the master key it guards seals stored
+  credentials). It mints a fresh key, writes it to Key Vault, discards it, reports the key's
+  **hash** through the job's marker channel, and waits for the synced Kubernetes Secret to carry the
+  new value — compared by hash, never by value — before the portal is restarted.
+- **`IInstanceKeyRegistry.AdoptKeyHash`** — the registry-side half, on the contract assembly so the
+  Hosting plugin can resolve it from inside the portal: the instance takes the new hash and issued-at,
+  a fresh index entry points at it, and the previous index entry is **deleted**, so the old key stops
+  authenticating the moment the adoption lands.
+
+The Hosting plugin's `RotateRegistryKey` verb, which drives both halves, follows in
+`MeshWeaver.Plugins`.
+
+## Why the split matters
+
+The exposure this repairs came from an *audit* — a check that selected a secret's `.value` while
+asking whether it was stored in the clear. The design here makes that mistake structurally
+impossible: the only process that holds the raw key is the operator job, and the only thing it
+hands back is a hash.

src/MeshWeaver.Mesh.Contract/Security/IInstanceKeyRegistry.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Reactive;
+
+namespace MeshWeaver.Mesh.Security;
+
+/// <summary>
+/// The registry-side half of an instance-key ROTATION driven from outside the registry process —
+/// by the Hosting operator's <c>hosting-kv-rotate</c> job, which mints the raw <c>mwi_</c> key, puts
+/// it straight into Key Vault, and reports ONLY its SHA-256 back through the job log.
+///
+/// <para>🚨 This contract carries hashes, never keys. The registry stores hashes
+/// (<see cref="MeshWeaverInstance.KeyHash"/>), the operator holds the vault write, and nothing in
+/// between logs, displays or persists the secret. It lives in the contract assembly so in-mesh
+/// code (the Hosting plugin's control plane, compiled at runtime inside the registry portal) can
+/// resolve it from the hub's service provider without referencing the portal host.</para>
+/// </summary>
+public interface IInstanceKeyRegistry
+{
+    /// <summary>
+    /// Points the instance registered as <paramref name="instanceId"/> at a new key by its hash:
+    /// the instance node takes <paramref name="keyHash"/> and a fresh issued-at, a fresh index
+    /// entry resolves the hash to the instance, and the PREVIOUS index entry is deleted so the old
+    /// key stops authenticating the moment this completes. Idempotent for a hash already adopted.
+    /// </summary>
+    /// <param name="instanceId">The instance's id — the value <c>PluginCatalog__InstanceId</c> carries.</param>
+    /// <param name="keyHash">Lowercase SHA-256 hex (64 chars) of the new raw key.</param>
+    IObservable<Unit> AdoptKeyHash(string instanceId, string keyHash);
+}