hosting-image-mirror: copy the pinned platform images from ACR into the fleet registry when it lacks them (Plugins#1722)

.github/workflows/hosting-operator.yml

@@ -65,8 +65,8 @@ jobs:
         run: |
           set -euo pipefail
           tracked=$(git ls-files deploy/aks/operator/bin/ | wc -l | tr -d ' ')
-          if [ "$tracked" -lt 27 ]; then
-            echo "::error::only ${tracked} file(s) tracked under deploy/aks/operator/bin/ — expected 27 (24 commands + run.sh + _common.sh + _audit.jq). The floor is the REAL count — raise it with every command added, or a lost file passes."
+          if [ "$tracked" -lt 28 ]; then
+            echo "::error::only ${tracked} file(s) tracked under deploy/aks/operator/bin/ — expected 28 (25 commands + run.sh + _common.sh + _audit.jq). The floor is the REAL count — raise it with every command added, or a lost file passes."
             echo "::error::Almost certainly .gitignore matching this bin/ again. `git add` on an ignored path exits 0 and does NOTHING, so the loss is silent until a fresh checkout builds an image with no scripts in it."
             git check-ignore -v deploy/aks/operator/bin/run.sh || true
             exit 1
@@ -106,6 +106,7 @@ jobs:
           hosting-dns
           hosting-export
           hosting-federate
+          hosting-image-mirror
           hosting-inline-env-retire
           hosting-kv-copy
           hosting-kv-ensure
@@ -195,7 +196,7 @@ jobs:
         # through to a REAL az/kubectl on a runner that has one.
         run: |
           set -euo pipefail
-          for f in deploy/aks/operator/test/stubs/pull-secret/az deploy/aks/operator/test/stubs/pull-secret/kubectl deploy/aks/operator/test/stubs/pv-resize/kubectl deploy/aks/operator/test/stubs/inline-env/kubectl deploy/aks/operator/test/stubs/kv-rotate/az deploy/aks/operator/test/stubs/kv-ensure/az deploy/aks/operator/test/stubs/kv-copy/az deploy/aks/operator/test/stubs/registry-register/az deploy/aks/operator/test/stubs/signin-app/az; do
+          for f in deploy/aks/operator/test/stubs/pull-secret/az deploy/aks/operator/test/stubs/pull-secret/kubectl deploy/aks/operator/test/stubs/pv-resize/kubectl deploy/aks/operator/test/stubs/inline-env/kubectl deploy/aks/operator/test/stubs/kv-rotate/az deploy/aks/operator/test/stubs/kv-ensure/az deploy/aks/operator/test/stubs/kv-copy/az deploy/aks/operator/test/stubs/registry-register/az deploy/aks/operator/test/stubs/signin-app/az deploy/aks/operator/test/stubs/image-mirror/az deploy/aks/operator/test/stubs/image-mirror/crane; do
             git ls-files --error-unmatch "$f" >/dev/null 2>&1 || { echo "::error::${f} is not tracked"; git check-ignore -v "$f" || true; exit 1; }
             [ "$(git ls-files -s "$f" | awk '{print $1}')" = "100755" ] || { echo "::error::${f} is not committed executable (mode 100755)"; exit 1; }
           done
@@ -245,7 +246,7 @@ jobs:
             test -x /opt/hosting/bin/run.sh          || { echo "run.sh missing"; exit 1; }
             test -f /opt/hosting/chart/Chart.yaml    || { echo "chart missing — build context must be deploy/"; exit 1; }
             command -v hosting-deploy >/dev/null     || { echo "hosting-* not on PATH"; exit 1; }
-            command -v helm kubectl az pg_dump jq >/dev/null || { echo "a required tool is missing"; exit 1; }
+            command -v helm kubectl az pg_dump jq crane >/dev/null || { echo "a required tool is missing"; exit 1; }
             test -f /opt/hosting/bin/_audit.jq       || { echo "_audit.jq missing — hosting-audit has no detection"; exit 1; }
             echo "image carries: $(ls /opt/hosting/bin/hosting-* | wc -l) commands + chart $(grep ^version /opt/hosting/chart/Chart.yaml)"
           '

deploy/aks/INSTANCE-LIFECYCLE.md

@@ -147,6 +147,14 @@ rule is in the mesh's pure, unit-tested plan. What the scripts themselves guaran
   needs Graph `Application.ReadWrite.OwnedBy` on an app the identity owns, else RED with the exact
   hand commands. Registering the app itself (no client id on the record) stays a hand step, and the
   step says so with the runbook's commands.
+- **`hosting-image-mirror` puts the pinned platform images into the fleet registry only when it
+  lacks them, and never re-pushes over what is there** (MeshWeaver.Plugins#1722). Portal AND
+  migration ride the same tag (Memex#141): a present tag is kept (its digest compared with ACR's, a
+  difference reported as `image_drift`), an absent one is `crane copy`-ed from ACR as the registry's
+  publisher and both digests must match before it reports; a tag absent from ACR too refuses
+  naming the sealed set to pin. The publisher password (vault) and the ACR token
+  (`az acr login --expose-token`) reach crane on stdin into a per-process `DOCKER_CONFIG` — never
+  argv, never printed. Needs `AcrPull` on the source registry for the operator identity.
 - **`hosting-export` never prints the URL it mints.** A user-delegation SAS is a bearer credential;
   it goes into a one-shot Secret the mesh reads once and deletes.
 - **`hosting-verify-catalog` proves the plugin mounts took.** An instance with green pods and an

deploy/aks/operator/Dockerfile

@@ -17,6 +17,10 @@ FROM mcr.microsoft.com/azure-cli:2.77.0
 # configured: HostingOperator refuses with "no operator image configured" either way.
 ARG KUBECTL_VERSION=v1.31.4
 ARG HELM_VERSION=v3.16.3
+# crane (go-containerregistry): hosting-image-mirror's registry-to-registry copy of the pinned
+# platform images from ACR into the fleet registry (MeshWeaver.Plugins#1722). A manifest+blob copy
+# needs a registry client; az, kubectl and helm are not one, and docker is not in this image.
+ARG CRANE_VERSION=v0.20.3
 # Supplied by BuildKit. The binaries below are per-architecture, and hard-coding amd64 produces an
 # image that builds fine on an arm64 laptop and then cannot exec kubectl.
 ARG TARGETARCH=amd64
@@ -35,7 +39,11 @@ RUN tdnf install -y --refresh postgresql tar shadow-utils ca-certificates jq \
  && curl -fsSL "https://get.helm.sh/helm-${HELM_VERSION}-linux-${TARGETARCH}.tar.gz" \
       | tar -xz -C /tmp "linux-${TARGETARCH}/helm" \
  && mv "/tmp/linux-${TARGETARCH}/helm" /usr/local/bin/helm \
- && rm -rf "/tmp/linux-${TARGETARCH}"
+ && rm -rf "/tmp/linux-${TARGETARCH}" \
+ && case "${TARGETARCH}" in amd64) crane_arch=x86_64 ;; arm64) crane_arch=arm64 ;; *) echo "no crane build for ${TARGETARCH}"; exit 1 ;; esac \
+ && curl -fsSL "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_${crane_arch}.tar.gz" \
+      | tar -xz -C /usr/local/bin crane \
+ && chmod +x /usr/local/bin/crane
 
 COPY aks/operator/bin/ /opt/hosting/bin/
 RUN chmod +x /opt/hosting/bin/* && ln -s /opt/hosting/bin/hosting-* /usr/local/bin/

deploy/aks/operator/bin/hosting-image-mirror

@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+# hosting-image-mirror --registry <fleet registry host> --repository <path> --tag <tag>
+#                      --vault <vault> --publisher-secret <vault object>
+#                      [--also <path>]… [--source <acr host>] [--publisher <user>]
+#
+# Put the pinned platform images INTO the fleet registry when it lacks them (MeshWeaver.Plugins#1722).
+#
+# CD publishes every sealed set to ACR (meshweaver.azurecr.io); the fleet's pods pull from the
+# registry the public instance hosts (cr.meshweaver.cloud, MeshWeaver#3353), which serves only what
+# has been pushed to it. A record pinning a tag the fleet registry does not hold used to send a
+# human to `crane copy` as the registry's publisher (runbook step 4) — or, before #1546, straight
+# into ImagePullBackOff at step 11. This step is that copy, for the PORTAL image and, with --also,
+# the MIGRATION image (Memex#141: the two must ride the same tag).
+#
+#   per image: tag present in the fleet registry → KEPT (never re-pushed); its digest is compared
+#              with the source's and a difference is reported (`image_drift`) — a re-tagged build
+#              upstream is the operator's call, not this step's.
+#              absent → the source is asked (a tag absent THERE refuses, naming the sealed set to
+#              pin), `crane copy` runs, and BOTH digests are read back and must match.
+#
+# Credentials: the fleet registry's publisher password is read from the vault (--publisher-secret)
+# and the ACR token comes from `az acr login --expose-token` under the operator identity (which
+# needs AcrPull on the source registry). Both reach crane on STDIN (`--password-stdin`) into a
+# private DOCKER_CONFIG that lives for this process — never argv, never printed, never a fact.
+# shellcheck source=_common.sh
+source "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/_common.sh"
+# shellcheck disable=SC2034  # read by hosting::die in _common.sh, which shellcheck does not follow here
+HOSTING_CMD="hosting-image-mirror"
+
+registry="" repository="" tag="" vault="" publisher_secret="" source_host="meshweaver.azurecr.io" publisher="publisher"
+also=()
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --registry)         registry="${2:-}";         shift 2 ;;
+    --repository)       repository="${2:-}";       shift 2 ;;
+    --tag)              tag="${2:-}";              shift 2 ;;
+    --also)             also+=("${2:-}");          shift 2 ;;
+    --source)           source_host="${2:-}";      shift 2 ;;
+    --vault)            vault="${2:-}";            shift 2 ;;
+    --publisher-secret) publisher_secret="${2:-}"; shift 2 ;;
+    --publisher)        publisher="${2:-}";        shift 2 ;;
+    *) hosting::die "unknown argument '$1'" ;;
+  esac
+done
+hosting::need_flag registry "$registry"
+hosting::need_flag repository "$repository"
+hosting::need_flag tag "$tag"
+hosting::need_flag vault "$vault"
+hosting::need_flag publisher-secret "$publisher_secret"
+hosting::need_flag source "$source_host"
+hosting::need_flag publisher "$publisher"
+hosting::safe_host registry "$registry"
+hosting::safe_host source "$source_host"
+hosting::safe_name vault "$vault"
+hosting::safe_name publisher-secret "$publisher_secret"
+hosting::safe_name publisher "$publisher"
+hosting::safe_name tag "$tag"
+# A repository PATH: one or more plain segments joined by '/'.
+safe_path() {
+  [[ "$2" =~ ^[a-z0-9]([a-z0-9._-]*[a-z0-9])?(/[a-z0-9]([a-z0-9._-]*[a-z0-9])?)*$ ]] \
+    || hosting::die "${1} '${2}' is not a repository path (lowercase segments joined by '/') — refusing to interpolate it into a command"
+}
+safe_path repository "$repository"
+for extra in "${also[@]}"; do safe_path also "$extra"; done
+# Only an Azure Container Registry is a source today: the ACR token comes from az, and the
+# registry's short name is the host's first label. Another upstream is a design change, not a flag.
+case "$source_host" in
+  *.azurecr.io) acr="${source_host%%.*}" ;;
+  *) hosting::die "source '${source_host}' is not an Azure Container Registry host (<name>.azurecr.io) — the fleet's upstream is ACR, and its token comes from az" ;;
+esac
+[ "$registry" != "$source_host" ] || hosting::die "registry and source are both ${registry} — nothing to mirror"
+
+images=("$repository" "${also[@]}")
+manual=""
+for repo in "${images[@]}"; do manual="${manual}crane copy ${source_host}/${repo}:${tag} ${registry}/${repo}:${tag}; "; done
+manual="az acr login --name ${acr} --expose-token (then crane auth login) and, as the registry's ${publisher} (password in vault ${vault} object ${publisher_secret}): ${manual}"
+
+if hosting::dry; then
+  for repo in "${images[@]}"; do
+    hosting::log "(dry run) would ensure ${registry}/${repo}:${tag} — read its digest as ${publisher}; if absent, copy it from ${source_host}/${repo}:${tag} and compare both digests"
+  done
+  hosting::say image_mirror dry-run
+  exit 0
+fi
+
+command -v crane >/dev/null 2>&1 || hosting::die "crane is not installed in this image — the copy is a registry-to-registry manifest+blob copy, and nothing else here can do it. Rebuild the operator image (deploy/aks/operator/Dockerfile installs it), or copy by hand: ${manual}"
+
+# A private docker config for this process: crane reads credentials from it, nothing else does,
+# and it goes with the process.
+umask 077
+DOCKER_CONFIG="$(mktemp -d)"; export DOCKER_CONFIG
+trap 'rm -rf "$DOCKER_CONFIG"' EXIT
+
+# ── the fleet registry, as its publisher ─────────────────────────────────────────────────────
+hosting::step "Sign in to the fleet registry as its publisher"
+password="$(az keyvault secret show --vault-name "$vault" --name "$publisher_secret" --query value -o tsv 2>/dev/null)" \
+  || hosting::die "could not read the publisher password ${publisher_secret} from vault ${vault} — nothing was copied. Grant the operator identity secret GET on it, or copy by hand: ${manual}"
+[ -n "$password" ] || hosting::die "vault ${vault} object ${publisher_secret} is EMPTY — the registry accepts no push without it"
+if ! printf '%s' "$password" | crane auth login "$registry" -u "$publisher" --password-stdin >/dev/null 2>&1; then
+  unset password
+  hosting::die "crane could not sign in to ${registry} as ${publisher} — the password in ${publisher_secret} does not match the registry's publisher hash (RegistrySpec.publisherPasswordBcrypt). Nothing was copied."
+fi
+unset password
+hosting::log "signed in to ${registry} as ${publisher}"
+
+# digest_of <ref> — the manifest digest on STDOUT, or nothing (absent or unreadable). Never dies:
+# callers decide what absence means at each side.
+digest_of() { crane digest "$1" 2>/dev/null; }
+
+# ── what the fleet registry already holds ────────────────────────────────────────────────────
+hosting::step "Read the fleet registry"
+present=() absent=()
+declare -A have
+for repo in "${images[@]}"; do
+  d="$(digest_of "${registry}/${repo}:${tag}")"
+  if [ -n "$d" ]; then have["$repo"]="$d"; present+=("$repo"); hosting::log "present  ${registry}/${repo}:${tag} @ ${d:0:19}…"
+  else absent+=("$repo"); hosting::log "absent   ${registry}/${repo}:${tag}"; fi
+done
+
+# ── the source, only when something has to come from it ──────────────────────────────────────
+mirrored=0 kept=${#present[@]} drift=0
+if [ ${#absent[@]} -gt 0 ]; then
+  hosting::step "Sign in to the source registry"
+  token="$(az acr login --name "$acr" --expose-token --query accessToken -o tsv 2>/dev/null)" \
+    || hosting::die "az acr login --name ${acr} --expose-token failed — the operator identity needs AcrPull on ${source_host} (and to see it: Reader on the registry resource), which it does not hold today. Nothing was copied. Copy by hand: ${manual}"
+  [ -n "$token" ] || hosting::die "az acr login returned no token for ${acr} — nothing was copied"
+  if ! printf '%s' "$token" | crane auth login "$source_host" -u 00000000-0000-0000-0000-000000000000 --password-stdin >/dev/null 2>&1; then
+    unset token
+    hosting::die "crane could not sign in to ${source_host} with the ACR token — nothing was copied"
+  fi
+  unset token
+  hosting::log "signed in to ${source_host} as the operator identity"
+
+  for repo in "${absent[@]}"; do
+    hosting::step "Mirror ${repo}:${tag}"
+    src="${source_host}/${repo}:${tag}" dst="${registry}/${repo}:${tag}"
+    sd="$(digest_of "$src")"
+    [ -n "$sd" ] || hosting::die "${src} does not exist — the pinned tag is not in the source registry either, so no copy can put it in ${registry}. Pin a tag of a SEALED set (the newest sealed main-cd set), or publish ${tag}. Nothing was copied for ${repo}$( [ "$mirrored" -gt 0 ] && echo "; ${mirrored} image(s) earlier in this run were")."
+    hosting::do crane copy "$src" "$dst" >/dev/null \
+      || hosting::die "crane copy ${src} → ${dst} failed — nothing to undo (a partial copy is not a tag); re-request, or copy by hand: ${manual}"
+    dd="$(digest_of "$dst")"
+    [ "$dd" = "$sd" ] \
+      || hosting::die "${dst} reads back as '${dd:-absent}' but the source is ${sd} — the copy did not land as the same manifest; refusing to report it mirrored"
+    hosting::log "mirrored ${dst} @ ${dd:0:19}… (same digest as ${src})"
+    mirrored=$((mirrored + 1))
+  done
+
+  # Present images are compared with the source too, now that the source is readable — a
+  # difference is a FACT (a re-tagged build upstream), never a re-push over what the pods run.
+  for repo in "${present[@]}"; do
+    sd="$(digest_of "${source_host}/${repo}:${tag}")"
+    if [ -n "$sd" ] && [ "$sd" != "${have[$repo]}" ]; then
+      hosting::log "drift    ${registry}/${repo}:${tag} @ ${have[$repo]:0:19}… differs from ${source_host} @ ${sd:0:19}… — KEPT (never re-pushed); re-tag deliberately if the upstream build moved"
+      drift=$((drift + 1))
+    fi
+  done
+fi
+
+hosting::log "${registry}: ${mirrored} mirrored, ${kept} already present, ${drift} differing from ${source_host}"
+hosting::say image_mirrored "$mirrored"
+hosting::say image_present "$kept"
+hosting::say image_drift "$drift"
+hosting::say image_tag "$tag"