hosting-kv-copy: materialise a fleet-shared vault object under an instance's prefix (Plugins#1723)

.github/workflows/hosting-operator.yml

@@ -65,8 +65,8 @@ jobs:
         run: |
           set -euo pipefail
           tracked=$(git ls-files deploy/aks/operator/bin/ | wc -l | tr -d ' ')
-          if [ "$tracked" -lt 25 ]; then
-            echo "::error::only ${tracked} file(s) tracked under deploy/aks/operator/bin/ — expected 25 (22 commands + run.sh + _common.sh + _audit.jq). The floor is the REAL count — raise it with every command added, or a lost file passes."
+          if [ "$tracked" -lt 26 ]; then
+            echo "::error::only ${tracked} file(s) tracked under deploy/aks/operator/bin/ — expected 26 (23 commands + run.sh + _common.sh + _audit.jq). The floor is the REAL count — raise it with every command added, or a lost file passes."
             echo "::error::Almost certainly .gitignore matching this bin/ again. `git add` on an ignored path exits 0 and does NOTHING, so the loss is silent until a fresh checkout builds an image with no scripts in it."
             git check-ignore -v deploy/aks/operator/bin/run.sh || true
             exit 1
@@ -107,6 +107,7 @@ jobs:
           hosting-export
           hosting-federate
           hosting-inline-env-retire
+          hosting-kv-copy
           hosting-kv-ensure
           hosting-kv-purge
           hosting-kv-rotate
@@ -193,7 +194,7 @@ jobs:
         # through to a REAL az/kubectl on a runner that has one.
         run: |
           set -euo pipefail
-          for f in deploy/aks/operator/test/stubs/pull-secret/az deploy/aks/operator/test/stubs/pull-secret/kubectl deploy/aks/operator/test/stubs/pv-resize/kubectl deploy/aks/operator/test/stubs/inline-env/kubectl deploy/aks/operator/test/stubs/kv-rotate/az deploy/aks/operator/test/stubs/kv-ensure/az deploy/aks/operator/test/stubs/signin-app/az; do
+          for f in deploy/aks/operator/test/stubs/pull-secret/az deploy/aks/operator/test/stubs/pull-secret/kubectl deploy/aks/operator/test/stubs/pv-resize/kubectl deploy/aks/operator/test/stubs/inline-env/kubectl deploy/aks/operator/test/stubs/kv-rotate/az deploy/aks/operator/test/stubs/kv-ensure/az deploy/aks/operator/test/stubs/kv-copy/az deploy/aks/operator/test/stubs/signin-app/az; do
             git ls-files --error-unmatch "$f" >/dev/null 2>&1 || { echo "::error::${f} is not tracked"; git check-ignore -v "$f" || true; exit 1; }
             [ "$(git ls-files -s "$f" | awk '{print $1}')" = "100755" ] || { echo "::error::${f} is not committed executable (mode 100755)"; exit 1; }
           done

deploy/aks/INSTANCE-LIFECYCLE.md

@@ -124,6 +124,13 @@ rule is in the mesh's pure, unit-tested plan. What the scripts themselves guaran
   `operator.environment`) — through `--file`, never an argument, never printed. An existing object
   is kept as written.
   stored `enc:` value in that instance's database permanently unreadable.
+- **`hosting-kv-copy` materialises a fleet-shared object under the instance's prefix and never
+  rewrites one that exists** (MeshWeaver.Plugins#1723). The record's
+  `keyVaultSecrets.secrets[].copyFrom` names the source; an absent target is copied through
+  `--file`, an existing one is kept and compared with its source BY HASH — a difference is a
+  reported fact (`kv_copy_drift`), not a failure — and no value is ever printed or put on argv.
+  It exists because `hosting-kv-purge` deletes by prefix: a cross-prefix mapping would let the
+  first teardown take the shared credential with it.
 - **`hosting-signin-app` never rotates the sign-in app's secret and never shows one**
   (MeshWeaver.Plugins#1719). A present `<prefix>Authentication-Microsoft-ClientSecret` is kept and
   the app the record names is VERIFIED to redirect to `https://<host>/signin-microsoft` (an identity

deploy/aks/operator/bin/hosting-kv-copy

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# hosting-kv-copy --vault <name> --copy <target>=<source> [--copy <target>=<source> …]
+#
+# Materialise a FLEET-SHARED vault object under an instance's own prefix (MeshWeaver.Plugins#1723).
+#
+# The fleet convention is one vault object per prefix, and `hosting-kv-purge` deletes an instance's
+# objects by that prefix on teardown — so a credential several instances hold (the fleet GitHub
+# App's private key: `memexsystemorph-GitHub-App-PrivateKey`, needed by `build` as
+# `build-GitHub-App-PrivateKey`) has to be COPIED under each holder's prefix rather than mapped
+# across prefixes, or the first teardown would take the shared PEM with it. Until now that copy was
+# a hand step (`az keyvault secret show … | az keyvault secret set --file /dev/stdin`). The record
+# now states it (`keyVaultSecrets.secrets[].copyFrom`) and the Provision runs this.
+#
+#   target exists   → KEPT, never rewritten (the master-key rule: hosting-kv-ensure). Its value is
+#                     compared with the source's BY HASH and a difference is REPORTED as a fact
+#                     (`kv_copy_drift`), not treated as a failure — a rotated source is the
+#                     operator's call to propagate, by RotateRegistryKey-style action or by hand.
+#   target absent   → the source is read and written under the target name, through --file, then
+#                     read back by id.
+#
+# 🚨 NEVER PRINTS A VALUE. Both values live in shell variables, are compared by sha256, reach az
+# through a mode-600 temporary file (never argv, which every process in the pod can read), and are
+# unset. Every refusal names the one hand command that still works.
+# shellcheck source=_common.sh
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/_common.sh"
+# shellcheck disable=SC2034  # read by hosting::die in _common.sh, which shellcheck does not follow here
+HOSTING_CMD="hosting-kv-copy"
+
+vault="" copies=()
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --vault) vault="${2:-}"; shift 2 ;;
+    --copy)  copies+=("${2:-}"); shift 2 ;;
+    *) hosting::die "unknown argument '$1'" ;;
+  esac
+done
+hosting::need_flag vault "$vault"
+hosting::safe_name vault "$vault"
+[ ${#copies[@]} -gt 0 ] || hosting::die "missing required flag --copy — nothing to copy is a bug in the caller, not a no-op"
+
+# Validate EVERY pair before the vault is touched: a pair that is not <target>=<source> of two
+# plain names refuses the whole run, so a partial copy set never reports as a run.
+for pair in "${copies[@]}"; do
+  case "$pair" in
+    *=*) ;;
+    *) hosting::die "--copy '${pair}' is not <target>=<source>" ;;
+  esac
+  target="${pair%%=*}" source_name="${pair#*=}"
+  [ -n "$target" ] && [ -n "$source_name" ] || hosting::die "--copy '${pair}' names an empty target or source"
+  hosting::safe_name copy-target "$target"
+  hosting::safe_name copy-source "$source_name"
+  [ "$target" != "$source_name" ] || hosting::die "--copy '${pair}' would copy ${target} onto itself — refusing"
+done
+
+kv_exists() {
+  az keyvault secret show --vault-name "$vault" --name "$1" --query id -o tsv >/dev/null 2>&1
+}
+
+created=0 kept=0 drift=0
+for pair in "${copies[@]}"; do
+  target="${pair%%=*}" source_name="${pair#*=}"
+  manual="az keyvault secret show --vault-name ${vault} --name ${source_name} --query value -o json | jq -j . | az keyvault secret set --vault-name ${vault} --name ${target} --file /dev/stdin --output none"
+  if hosting::dry; then
+    if kv_exists "$target"; then
+      hosting::log "keep    ${target} (exists — never rewritten; a real run compares it with ${source_name} by hash)"
+      kept=$((kept + 1))
+    else
+      hosting::log "copy    ${source_name} → ${target} (dry run: the value is not read)"
+      created=$((created + 1))
+    fi
+    continue
+  fi
+  # The source is read first in BOTH branches: an unreadable source is a refusal even when the
+  # target exists, because a target nobody can compare against the fleet's copy is a drift
+  # nobody can see.
+  source_value="$(az keyvault secret show --vault-name "$vault" --name "$source_name" --query value -o tsv 2>/dev/null)" \
+    || hosting::die "could not read ${source_name} from vault ${vault} — the fleet-shared object ${target} is a copy of. Nothing was written. Grant the operator identity secret GET on it, or copy by hand: ${manual}"
+  [ -n "$source_value" ] || { unset source_value; hosting::die "vault ${vault} object ${source_name} is EMPTY — refusing to copy nothing under ${target}"; }
+  source_hash="$(printf '%s' "$source_value" | hosting::sha256)"
+  if kv_exists "$target"; then
+    target_value="$(az keyvault secret show --vault-name "$vault" --name "$target" --query value -o tsv 2>/dev/null)" || target_value=""
+    target_hash="$(printf '%s' "$target_value" | hosting::sha256)"
+    unset source_value target_value
+    if [ "$target_hash" = "$source_hash" ]; then
+      hosting::log "keep    ${target} (exists and matches ${source_name}, sha256 ${source_hash:0:12}…)"
+    else
+      hosting::log "keep    ${target} (exists — NEVER rewritten — but DIFFERS from ${source_name}: ${target_hash:0:12}… vs ${source_hash:0:12}…; propagate deliberately if the source was rotated)"
+      drift=$((drift + 1))
+    fi
+    kept=$((kept + 1))
+    continue
+  fi
+  umask 077
+  copy_file="$(mktemp)"
+  trap 'rm -f "$copy_file"' EXIT
+  printf '%s' "$source_value" > "$copy_file"
+  unset source_value
+  az keyvault secret set --vault-name "$vault" --name "$target" --file "$copy_file" --output none \
+    || hosting::die "could not create ${target} in vault ${vault}. Nothing was written; the hand command is: ${manual}"
+  rm -f "$copy_file"
+  kv_exists "$target" \
+    || hosting::die "${target} was written to vault ${vault} but cannot be read back — refusing to report a secret the CSI driver will not find"
+  hosting::log "copied  ${source_name} → ${target} (sha256 ${source_hash:0:12}…, read back by id)"
+  created=$((created + 1))
+done
+
+hosting::log "vault ${vault}: ${created} copied, ${kept} already present, ${drift} differing from their source"
+hosting::say kv_copy_created "$created"
+hosting::say kv_copy_kept "$kept"
+hosting::say kv_copy_drift "$drift"

deploy/aks/operator/test/run-tests.sh

@@ -406,6 +406,74 @@ case "$_kve_log" in *"memex-postgres-password"*|*"secret set"*) bad "a dry run n
 case "$_kve_out" in *"::hosting:: kv_db_connection=would-create"*) ok "…and reports would-create, never created" ;; *) bad "dry run reports would-create" "said: ${_kve_out}" ;; esac
 rm -rf "$_kve_state"
 unset _kve_out _kve_rc _kve_log _kve_state _kve_written
+echo "── hosting-kv-copy: a fleet-shared object materialised under the prefix, never shown ──"
+# MeshWeaver.Plugins#1723 — a credential several instances hold (the fleet GitHub App's PEM) has to
+# exist under EACH holder's prefix, because hosting-kv-purge deletes by prefix on teardown and a
+# cross-prefix mapping would let the first teardown take the shared object with it. The copy was a
+# hand step; now the record states `copyFrom` and the Provision runs this. The stub records every
+# argv and answers the vault from a state, so the decisions — copy only when ABSENT, keep when
+# present (drift reported, never rewritten), never print — are asserted here.
+KVC_STUBS="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/stubs/kv-copy" && pwd)"
+kvc() {  # kvc [env…] -- <args…>
+  local envs=()
+  while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
+  _kvc_state="$(mktemp -d)"
+  _kvc_out="$(env "${envs[@]}" PATH="$KVC_STUBS:$PATH" HOSTING_KVC_STATE="$_kvc_state" hosting-kv-copy "$@" 2>&1)"; _kvc_rc=$?
+  _kvc_log="$(cat "$_kvc_state/az.log" 2>/dev/null || true)"
+}
+KVC_PEM="-----BEGIN-FAKE-PEM-NEVER-PRINTED-----"
+
+kvc HOSTING_KVC_VALUES="memexsystemorph-GitHub-App-PrivateKey=${KVC_PEM}" -- --vault Systemorph --copy build-GitHub-App-PrivateKey=memexsystemorph-GitHub-App-PrivateKey
+[ "$_kvc_rc" -eq 0 ] && ok "kv-copy materialises an ABSENT target from its source" || bad "kv-copy copies an absent target" "exited ${_kvc_rc}: ${_kvc_out}"
+[ "$(cat "$_kvc_state/set.build-GitHub-App-PrivateKey" 2>/dev/null)" = "$KVC_PEM" ] && ok "…byte-for-byte" || bad "the copy is byte-identical" "wrote: $(cat "$_kvc_state/set.build-GitHub-App-PrivateKey" 2>/dev/null)"
+case "$_kvc_out" in *NEVER-PRINTED*) bad "kv-copy never prints the value" "it did: ${_kvc_out}" ;; *) ok "kv-copy never prints the value" ;; esac
+case "$_kvc_log" in *NEVER-PRINTED*) bad "…and never puts it on an az command line" "az saw: ${_kvc_log}" ;; *) ok "…and never puts it on an az command line" ;; esac
+case "$_kvc_out" in *"::hosting:: kv_copy_created=1"*"::hosting:: kv_copy_kept=0"*"::hosting:: kv_copy_drift=0"*) ok "the run reports created=1 kept=0 drift=0" ;; *) bad "kv-copy facts" "said: ${_kvc_out}" ;; esac
+rm -rf "$_kvc_state"
+
+kvc HOSTING_KVC_VALUES="memexsystemorph-GitHub-App-PrivateKey=${KVC_PEM} build-GitHub-App-PrivateKey=${KVC_PEM}" -- --vault Systemorph --copy build-GitHub-App-PrivateKey=memexsystemorph-GitHub-App-PrivateKey
+[ "$_kvc_rc" -eq 0 ] && ok "an EXISTING, matching target is kept (a re-provision is idempotent)" || bad "existing target kept" "exited ${_kvc_rc}: ${_kvc_out}"
+case "$_kvc_log" in *"secret set"*) bad "a kept target is never rewritten" "az saw: ${_kvc_log}" ;; *) ok "a kept target is never rewritten" ;; esac
+case "$_kvc_out" in *"kv_copy_created=0"*"kv_copy_kept=1"*"kv_copy_drift=0"*) ok "…reported as kept, no drift" ;; *) bad "kept facts" "said: ${_kvc_out}" ;; esac
+rm -rf "$_kvc_state"
+
+kvc HOSTING_KVC_VALUES="memexsystemorph-GitHub-App-PrivateKey=${KVC_PEM} build-GitHub-App-PrivateKey=rotated-elsewhere-NEVER-PRINTED" -- --vault Systemorph --copy build-GitHub-App-PrivateKey=memexsystemorph-GitHub-App-PrivateKey
+[ "$_kvc_rc" -eq 0 ] && ok "a target that DIFFERS from its source is still kept — drift is a fact, not a failure" || bad "differing target kept" "exited ${_kvc_rc}: ${_kvc_out}"
+case "$_kvc_log" in *"secret set"*) bad "…and is not rewritten" "az saw: ${_kvc_log}" ;; *) ok "…and is not rewritten" ;; esac
+case "$_kvc_out" in *"kv_copy_drift=1"*) ok "…and the drift is reported (kv_copy_drift=1)" ;; *) bad "drift reported" "said: ${_kvc_out}" ;; esac
+case "$_kvc_out" in *NEVER-PRINTED*) bad "…without printing either value" "it did: ${_kvc_out}" ;; *) ok "…without printing either value" ;; esac
+rm -rf "$_kvc_state"
+
+kvc HOSTING_KVC_VALUES="a=1 b=2" -- --vault Systemorph --copy x=a --copy y=b
+[ "$_kvc_rc" -eq 0 ] && ok "several --copy pairs run in one step" || bad "several pairs" "exited ${_kvc_rc}: ${_kvc_out}"
+case "$_kvc_out" in *"kv_copy_created=2"*) ok "…each counted" ;; *) bad "count of two" "said: ${_kvc_out}" ;; esac
+rm -rf "$_kvc_state"
+
+kvc HOSTING_KVC_VALUES="" -- --vault Systemorph --copy build-GitHub-App-PrivateKey=memexsystemorph-GitHub-App-PrivateKey
+[ "$_kvc_rc" -ne 0 ] && ok "an unreadable source refuses" || bad "unreadable source refuses" "exited 0: ${_kvc_out}"
+case "$_kvc_out" in *"could not read memexsystemorph-GitHub-App-PrivateKey"*"az keyvault secret show --vault-name Systemorph --name memexsystemorph-GitHub-App-PrivateKey --query value -o json | jq -j . | az keyvault secret set --vault-name Systemorph --name build-GitHub-App-PrivateKey --file /dev/stdin"*) ok "…naming the source and the exact hand command" ;; *) bad "names the hand command" "said: ${_kvc_out}" ;; esac
+rm -rf "$_kvc_state"
+
+kvc HOSTING_KVC_VALUES="memexsystemorph-GitHub-App-PrivateKey=${KVC_PEM}" HOSTING_KVC_SET_FAIL=1 -- --vault Systemorph --copy build-GitHub-App-PrivateKey=memexsystemorph-GitHub-App-PrivateKey
+[ "$_kvc_rc" -ne 0 ] && ok "a vault that refuses the write fails the step" || bad "refused write fails" "exited 0: ${_kvc_out}"
+case "$_kvc_out" in *NEVER-PRINTED*) bad "…without printing the value on the failure path" "it did: ${_kvc_out}" ;; *) ok "…without printing the value on the failure path" ;; esac
+rm -rf "$_kvc_state"
+
+refuses_hard "kv-copy needs --vault"                    "missing required flag --vault" hosting-kv-copy --copy a=b
+refuses_hard "kv-copy needs at least one --copy"        "missing required flag --copy"  hosting-kv-copy --vault V
+refuses_hard "kv-copy refuses a pair without '='"       "is not <target>=<source>"      hosting-kv-copy --vault V --copy ab
+refuses_hard "kv-copy refuses copying an object onto itself" "onto itself"              hosting-kv-copy --vault V --copy a=a
+refuses_hard "kv-copy refuses a target with a metacharacter" "is not a plain name"      hosting-kv-copy --vault V --copy 'a;id=b'
+refuses_hard "kv-copy refuses a source with a backtick"      "is not a plain name"      hosting-kv-copy --vault V --copy 'a=b`id`'
+refuses_hard "kv-copy rejects unknown flags"            "unknown argument"              hosting-kv-copy --vault V --copy a=b --nope 1
+
+kvc HOSTING_DRY_RUN=true HOSTING_KVC_VALUES="memexsystemorph-GitHub-App-PrivateKey=${KVC_PEM}" -- --vault Systemorph --copy build-GitHub-App-PrivateKey=memexsystemorph-GitHub-App-PrivateKey
+[ "$_kvc_rc" -eq 0 ] && ok "a dry-run kv-copy succeeds" || bad "dry-run kv-copy" "exited ${_kvc_rc}: ${_kvc_out}"
+case "$_kvc_log" in *"--query value"*|*"secret set"*) bad "a dry run reads no value and writes nothing" "az saw: ${_kvc_log}" ;; *) ok "a dry run reads no value and writes nothing" ;; esac
+case "$_kvc_out" in *"kv_copy_created=1"*) ok "…and says what it would copy" ;; *) bad "dry run would-copy" "said: ${_kvc_out}" ;; esac
+rm -rf "$_kvc_state"
+unset _kvc_out _kvc_rc _kvc_log _kvc_state
+
 echo "── hosting-signin-app: the client secret is kept, added never rotated, never shown ──"
 # MeshWeaver.Plugins#1719 — runbook step 1 registered the Entra app and minted its secret by hand.
 # The stub answers Graph and the vault from a state and — like the real az — prints a minted

deploy/aks/operator/test/stubs/kv-copy/az

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# A stand-in `az` for hosting-kv-copy's behaviour tests (MeshWeaver.Plugins#1723). Records every
+# invocation VERBATIM (argv is how the test proves a copied value never travels on a command line)
+# and answers the vault from a small state:
+#
+#   HOSTING_KVC_STATE    directory: az.log (every argv), set.<name> (what a `set --file` wrote)
+#   HOSTING_KVC_VALUES   space-separated <name>=<value> pairs the vault holds; `show --query id`
+#                        answers for any of them, `show --query value` answers the value
+#   HOSTING_KVC_SET_FAIL=1   every `keyvault secret set` refuses
+set -uo pipefail
+state="${HOSTING_KVC_STATE:?the az stub needs HOSTING_KVC_STATE}"
+printf 'az %s\n' "$*" >> "$state/az.log"
+lookup() {  # lookup <name> → prints the value, returns 1 when absent
+  local pair
+  for pair in ${HOSTING_KVC_VALUES:-}; do
+    if [ "${pair%%=*}" = "$1" ]; then printf '%s' "${pair#*=}"; return 0; fi
+  done
+  [ -f "$state/set.$1" ] && { cat "$state/set.$1"; return 0; }
+  return 1
+}
+case "${1:-} ${2:-} ${3:-}" in
+  "keyvault secret show")
+    name="" query=""
+    while [ $# -gt 0 ]; do case "$1" in --name) name="${2:-}"; shift 2 ;; --query) query="${2:-}"; shift 2 ;; *) shift ;; esac; done
+    if value="$(lookup "$name")"; then
+      if [ "$query" = "value" ]; then printf '%s\n' "$value"; else printf 'https://vault.test/secrets/%s/0\n' "$name"; fi
+      exit 0
+    fi
+    echo "ERROR: (SecretNotFound) A secret with (name/id) ${name} was not found in this key vault." >&2; exit 3 ;;
+  "keyvault secret set")
+    [ "${HOSTING_KVC_SET_FAIL:-0}" = "1" ] && { echo "ERROR: (Forbidden) secrets set permission" >&2; exit 1; }
+    name="" file=""
+    while [ $# -gt 0 ]; do case "$1" in --name) name="${2:-}"; shift 2 ;; --file) file="${2:-}"; shift 2 ;; *) shift ;; esac; done
+    [ -n "$file" ] || { echo "az stub: hosting-kv-copy must write through --file, never --value" >&2; exit 1; }
+    [ -f "$file" ] || { echo "az stub: --file ${file} does not exist" >&2; exit 1; }
+    cp "$file" "$state/set.$name"; exit 0 ;;
+esac
+echo "az stub: unexpected invocation: $*" >&2
+exit 1