From 2bd87068b42b609c69ac82493011145c9576756e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= <6334612+rbuergi@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:17:50 +0200 Subject: [PATCH] fix(chart): more than one portal replica on a pod-local /data is refused at render (#6052) /data holds the NodeType assembly cache (FileSystemAssemblyStore, collection "local") and every compiled NodeType's SHARED record names a file in it. On per-pod emptyDir a type compiled on pod A leaves a record naming bytes pod B never holds; B's instances render "its compiled assembly could not be loaded on this node", and a recompile does not help because it runs where the NodeType's own hub lives. The client estate in #6052 ran two replicas this way (/health bake-report bytesmissing=93 on both pods). chart-gate invariant 3 asserts RWX only over the values files tracked here; a client's are not. So deployment.yaml now fails the render when keda.enabled or replicas.portal > 1 and neither persistence.data.claimName nor persistDataVolume gives a shared /data. - check-chart-invariants.sh: refusal control with a fixture that is the valid two-replica shape minus the data claim. Neutralising the fail reds the gate (6 of 7 refusals held). - values.yaml: the replicas comment names the refusal. Co-Authored-By: Claude Opus 5.5 (1M context) --- deploy/aks/scripts/check-chart-invariants.sh | 3 +++ .../values.two-replicas-pod-local-data.yaml | 22 +++++++++++++++++++ .../templates/memex-portal/deployment.yaml | 17 ++++++++++++++ deploy/helm/values.yaml | 5 ++++- 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 deploy/aks/scripts/testdata/values.two-replicas-pod-local-data.yaml diff --git a/deploy/aks/scripts/check-chart-invariants.sh b/deploy/aks/scripts/check-chart-invariants.sh index 013a6552fb..f8036befb7 100755 --- a/deploy/aks/scripts/check-chart-invariants.sh +++ b/deploy/aks/scripts/check-chart-invariants.sh @@ -326,6 +326,9 @@ REFUSALS=( "a database release AND the bundled Postgres (two answers to which database)|deploy/helm/values.yaml:deploy/aks/scripts/testdata/values.db-release-with-bundled-postgres.yaml|exclusive with postgres.enabled" # Plugins#1738: an executor the portal would silently read as Job must fail the render. "a misspelled operator executor|deploy/helm/values.yaml:deploy/aks/scripts/testdata/values.operator-executor-misspelled.yaml|must be Job or Actions" + # MeshWeaver#6052: two replicas on per-pod emptyDir /data — each pod's assembly cache is its own, + # and the shared NodeType records name bytes only the compiling pod holds. + "two replicas with a pod-local /data (the #6052 estate)|deploy/helm/values.yaml:deploy/aks/scripts/testdata/values.two-replicas-pod-local-data.yaml|needs a SHARED /data" ) refused=0 for entry in "${REFUSALS[@]}"; do diff --git a/deploy/aks/scripts/testdata/values.two-replicas-pod-local-data.yaml b/deploy/aks/scripts/testdata/values.two-replicas-pod-local-data.yaml new file mode 100644 index 0000000000..bc492a9362 --- /dev/null +++ b/deploy/aks/scripts/testdata/values.two-replicas-pod-local-data.yaml @@ -0,0 +1,22 @@ +# FIXTURE for check-chart-invariants.sh — a REFUSAL (MeshWeaver#6052). +# +# Two plain replicas with NO persistence.data claim, so each pod's /data would be its own emptyDir. +# /data holds the NodeType assembly cache, and every compiled NodeType's shared record names a file +# in it, so a pod that did not compile a type cannot load it. The client estate in #6052 ran this +# shape: two replicas, `bake-report bytesmissing=93` on both. Everything else is the valid +# two-replica shape (values.two-replicas-no-keda.yaml), so the ONLY thing the render can refuse is +# the pod-local /data. Placeholders, no credential. +replicas: + portal: 2 +keda: + enabled: false +config: + memex_portal: + Deployment__Orleans__Clustering: "AdoNet" +secrets: + memex_portal: + ConnectionStrings__memex: "Host=pg.example.test;Database=memex;Username=memex;Password=placeholder" + ConnectionStrings__orleans: "Host=pg.example.test;Database=orleans;Username=memex;Password=placeholder" + memex_migration: + ConnectionStrings__memex: "Host=pg.example.test;Database=memex;Username=memex;Password=placeholder" + ConnectionStrings__orleans: "Host=pg.example.test;Database=orleans;Username=memex;Password=placeholder" diff --git a/deploy/helm/templates/memex-portal/deployment.yaml b/deploy/helm/templates/memex-portal/deployment.yaml index 362939eccf..85e2ad53f2 100644 --- a/deploy/helm/templates/memex-portal/deployment.yaml +++ b/deploy/helm/templates/memex-portal/deployment.yaml @@ -679,6 +679,23 @@ spec: # self-host (the default), higher for a fixed-size install that wants HA without an autoscaler. # ⚠️ Above 1 the same prerequisites apply as for KEDA — AdoNet clustering and RWX on every claim # (values.yaml → persistence). helm-template asserts both; see .github/workflows/chart-gate.yml. +{{- /* + 🚨 MORE THAN ONE PORTAL POD NEEDS A SHARED /data — refused at render time (MeshWeaver#6052). + /data holds the NodeType assembly cache (FileSystemAssemblyStore, collection "local"), and every + compiled NodeType's record — which is in the SHARED database — names a file under it. On emptyDir + each pod has its own cache, so a type compiled on pod A leaves a record naming bytes pod B never + holds; pod B's instances then render "its compiled assembly could not be loaded on this node", and + a recompile does not help, because it runs where the NodeType's own hub lives, which may be pod A + again. Measured on a client estate on 2026-09-28: two replicas, no shared /data, /health + `bake-report bytesmissing=93` on both pods. chart-gate's invariant 3 asserts RWX only over the + values files tracked in this repository; a client's values are not, so the render itself refuses. + persistDataVolume (a single node's hostPath) counts as shared: every pod lands on that one node. +*/}} +{{- $multiReplica := or (.Values.keda).enabled (gt (int ((.Values.replicas).portal | default 1)) 1) }} +{{- $sharedData := or ((.Values.persistence).data).claimName .Values.persistDataVolume }} +{{- if and $multiReplica (not $sharedData) }} +{{- fail "more than one portal replica (keda.enabled, or replicas.portal > 1) needs a SHARED /data, and persistence.data.claimName is empty, so every pod would get its own emptyDir. /data holds the NodeType assembly cache; each compiled NodeType's shared record names a file in it, and a pod that did not compile that type cannot load it (MeshWeaver#6052). Set persistence.data.claimName to a ReadWriteMany claim, or run one replica." }} +{{- end }} {{- if not (.Values.keda).enabled }} replicas: {{ (.Values.replicas).portal | default 1 }} {{- end }} diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 6f3f4c303c..31f6ed2f2f 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -341,7 +341,10 @@ hostingOperator: # ring, the NodeType assembly cache, the NuGet cache); a ReadWriteOnce claim either pins both # pods to one node or leaves the second pod unschedulable. # Both are asserted at render time by .github/workflows/chart-gate.yml, so a values file that -# raises this without them fails the build instead of the deployment. +# raises this without them fails the build instead of the deployment. A floor above one with NO +# persistence.data.claimName (per-pod emptyDir /data) is refused by the chart itself, for every +# values file including a client's untracked ones: the NodeType assembly cache lives in /data and +# the shared NodeType records name files in it (MeshWeaver#6052). replicas: portal: 1 # ---- Key Vault secrets, DECLARED (renders the SecretProviderClass) ---------