diff --git a/deploy/helm/templates/registry/_registry.tpl b/deploy/helm/templates/registry/_registry.tpl index baa7882089..633c1399a4 100644 --- a/deploy/helm/templates/registry/_registry.tpl +++ b/deploy/helm/templates/registry/_registry.tpl @@ -36,7 +36,7 @@ start. The one exception is deliberate and safe: `publisher.passwordBcrypt` is a {{- $ingress := ($r.ingress | default dict) -}} {{- $resources := ($r.resources | default dict) -}} {{- $host := required "registry.host is required when registry.enabled — the public hostname the registry answers on (e.g. cr.meshweaver.cloud)" $r.host -}} -{{- $image := required "registry.image is required when registry.enabled — the distribution image, pinned by digest (ghcr.io/distribution/distribution:3.0.0@sha256:…)" $r.image -}} +{{- $image := required "registry.image is required when registry.enabled — the distribution image, pinned by digest (ghcr.io/distribution/distribution:3.1.1@sha256:…)" $r.image -}} {{- $authImage := required "registry.authImage is required when registry.enabled — the cesanta/docker_auth image, pinned by digest" $r.authImage -}} {{- $accountName := required "registry.storage.accountName is required when registry.enabled — the Azure Storage account holding the registry's blobs" $storage.accountName -}} {{- $vaultName := required "registry.keyVault.name is required when registry.enabled — the Key Vault holding the token certificate, key and HTTP secret" $kv.name -}} diff --git a/deploy/helm/values.registry.example.yaml b/deploy/helm/values.registry.example.yaml index e521d2a4b8..d6b88511cd 100644 --- a/deploy/helm/values.registry.example.yaml +++ b/deploy/helm/values.registry.example.yaml @@ -10,7 +10,7 @@ registry: enabled: true host: "cr.meshweaver.cloud" - image: "ghcr.io/distribution/distribution:3.0.0@sha256:4ba3adf47f5c866e9a29288c758c5328ef03396cb8f5f6454463655fa8bc83e2" + image: "ghcr.io/distribution/distribution:3.1.1@sha256:bca24727f4002e51f959c18c42e816e4d1078198081a9837e16b8b7d7e43ebf8" authImage: "cesanta/docker_auth:1.14.0@sha256:98e0307e0d2dcbafe10098814a97d248143b78ca478693f78c3c0d67fe776ab4" issuer: "memex-registry" replicas: 1 diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 1d861377d4..696dae5b2f 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -709,7 +709,7 @@ registry: host: "" # (required) e.g. cr.meshweaver.cloud — NOT an Azure name # Both images pinned BY DIGEST, so what runs is what was reviewed. Resolved 2026-09-08 with # `docker buildx imagetools inspect` (multi-arch index digests): - # ghcr.io/distribution/distribution:3.0.0@sha256:4ba3adf47f5c866e9a29288c758c5328ef03396cb8f5f6454463655fa8bc83e2 + # ghcr.io/distribution/distribution:3.1.1@sha256:bca24727f4002e51f959c18c42e816e4d1078198081a9837e16b8b7d7e43ebf8 # cesanta/docker_auth:1.14.0@sha256:98e0307e0d2dcbafe10098814a97d248143b78ca478693f78c3c0d67fe776ab4 # 🚨 docker_auth must be ≥ 1.14.0: distribution 3 looks tokens up by RFC 7638 thumbprint and # 1.14.0 is the first release that can emit one (token.disable_legacy_key_id, set in the diff --git a/src/MeshWeaver.Documentation/Data/Architecture/ContainerRegistryInMemex.md b/src/MeshWeaver.Documentation/Data/Architecture/ContainerRegistryInMemex.md index b64109cb2e..f9e99b602c 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/ContainerRegistryInMemex.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/ContainerRegistryInMemex.md @@ -95,7 +95,7 @@ own host, writing to its own blob container — and none of it is MeshWeaver cod | piece | image | what it does | |---|---|---| -| the registry | `ghcr.io/distribution/distribution:3.0.0` (CNCF distribution, the reference implementation) | serves `/v2/…`; storage driver `azure`, authenticating as the namespace's **workload identity** (`credentials.type: default_credentials`) — no storage key anywhere; token auth pointed at the auth server | +| the registry | `ghcr.io/distribution/distribution:3.1.1` (CNCF distribution, the reference implementation) | serves `/v2/…`; storage driver `azure`, authenticating as the namespace's **workload identity** (`credentials.type: default_credentials`) — no storage key anywhere; token auth pointed at the auth server | | the token server | `cesanta/docker_auth:1.14.0` | answers `https://cr.meshweaver.cloud/auth`; authenticates a caller and signs a bearer token the registry verifies against the same certificate | One Ingress on the host routes `/auth` to docker_auth and everything else to distribution @@ -105,6 +105,13 @@ notification bearer are Key Vault objects, mounted through one SecretProviderCla pods. Values: `registry.*` in `deploy/helm/values.yaml`; a complete example the chart gate renders on every pull request: `deploy/helm/values.registry.example.yaml`. +**🚨 The floor is distribution ≥ 3.1.0.** `credentials.type: default_credentials` reaches +`DefaultAzureCredential` — the workload identity — only from 3.1.0 on; 3.0.0 dispatches it to the +shared-key client and signs every storage request with an empty account key, whatever identity the +pod carries. The symptom is a registry pod that never becomes ready, its storage health check +logging `403 AuthenticationFailed … including the signature` (the shared-key error) while the +config and the identity are both correct. The pin is 3.1.1. + **Who may do what.** Two ways in, in the order docker_auth tries them: 1. **One static account, the publisher** (CI). Its bcrypt hash is the only credential in values — diff --git a/src/MeshWeaver.Documentation/Data/WhatsNew/2026-09-08-registry-pins-distribution-3-1-1.md b/src/MeshWeaver.Documentation/Data/WhatsNew/2026-09-08-registry-pins-distribution-3-1-1.md new file mode 100644 index 0000000000..ef4a0bdcf1 --- /dev/null +++ b/src/MeshWeaver.Documentation/Data/WhatsNew/2026-09-08-registry-pins-distribution-3-1-1.md @@ -0,0 +1,15 @@ +--- +Name: The fleet registry pins distribution 3.1.1 +Category: Fix +Description: The container registry now runs distribution 3.1.1 — on 3.0.0 the workload-identity storage credential was signed as a shared key and the registry pods never became ready. +Icon: Box +Order: -20260908 +--- + +# The fleet registry pins distribution 3.1.1 + +The container registry the deployment chart can stand up now pins CNCF distribution 3.1.1 +instead of 3.0.0. On 3.0.0 the registry authenticated to blob storage with an empty shared key +even when configured to use the pod's workload identity, so its storage health check failed and the +registry pods never became ready. From 3.1.0 on the identity is used as configured; the example +values, the design page and the chart's own hint name the new pin.