Skip to content

fix(self-managed): render pdb-value-wiring against the whole stack #956

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mikeyrcamp merged 2 commits into from
Aug 18, 2026
Merged

fix(self-managed): render pdb-value-wiring against the whole stack #956

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8ca6d2c
fix(self-managed): render pdb-value-wiring against the whole stack
mikeyrcamp Aug 18, 2026
a72eaca
test(self-managed): report pdb-value-wiring render failures as themse…
mikeyrcamp Aug 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
241 changes: 155 additions & 86 deletions deploy/stacks/self-managed/tests/pdb-value-wiring.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@
set -euo pipefail

stack_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
stacks_dir="$(cd "$stack_dir/.." && pwd)"
helm_dir="$(cd "$stack_dir/../../helm" && pwd)"
work_dir="$(mktemp -d)"
test_stack_dir="$work_dir/self-managed"
test_stacks_dir="$work_dir/stacks"
test_stack_dir="$test_stacks_dir/self-managed"
environment_name="pdb-value-wiring-test"
environment_file="$test_stack_dir/environments/$environment_name.yaml"
secrets_file="$test_stack_dir/secrets/$environment_name-secrets.yaml"
Expand All @@ -17,71 +20,134 @@ fail() {
exit 1
}

mkdir -p "$test_stack_dir"
cp -R "$stack_dir"/. "$test_stack_dir"
# Copy the whole stacks directory, not just self-managed. The self-managed
# helmfile.d pulls in a sibling stack through a relative path, so a copy of
# self-managed alone is not a working stack.
mkdir -p "$test_stacks_dir"
cp -R "$stacks_dir"/. "$test_stacks_dir"
mkdir -p "$(dirname "$secrets_file")"
printf '{}\n' >"$secrets_file"
Comment thread
coderabbitai[bot] marked this conversation as resolved.

# Every stack the copy reaches resolves ../environments/$HELMFILE_ENV.yaml
# against its own directory, so each one needs a file under this test's
# environment name.
for stack_environments in "$test_stacks_dir"/*/environments; do
test -d "$stack_environments" || continue
test -e "$stack_environments/$environment_name.yaml" ||
printf '{}\n' >"$stack_environments/$environment_name.yaml"
done

# Run against the whole helmfile.d directory, the way the stack is actually
# applied. Selecting a release out of one hand-picked state file only works
# until that release moves, and the failure then looks like the release does
# not exist rather than like the state file is wrong.
#
# global.yaml.gotmpl marks the shared gateway values as required, so every
# invocation has to supply them, including the ones that never render a route.
helmfile_common=(
--file "$test_stack_dir/helmfile.d"
--environment default
--state-values-set ingress.gatewayApi.controllerNamespace=envoy-gateway-system
--state-values-set ingress.gatewayApi.gateways.shared.name=shared-gw
--state-values-set ingress.gatewayApi.gateways.shared.namespace=envoy-gateway-system
--state-values-set ingress.gatewayApi.gateways.grpc.name=grpc-gw
--state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy-gateway-system
)

run_helmfile() {
HELMFILE_ENV="$environment_name" \
HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \
helmfile "${helmfile_common[@]}" "$@"
}

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

render_chart_values() {
# Guards the defect this harness previously hid: a selector that names a
# release the selected state does not declare. One match is the only correct
# answer. Zero means the release was renamed, removed, or looked for in the
# wrong place. More than one means two states declare it, and a later apply
# silently reinstalls whatever the earlier one decided.
assert_single_release() {
local release="$1"
local output_file="$2"
shift 2

HELMFILE_ENV="$environment_name" \
HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \
helmfile \
--file "$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" \
--environment default \
--selector "name=$release" \
"$@" \
write-values \
--output-file-template "$output_file" 2>/dev/null ||
HELMFILE_ENV="$environment_name" \
HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \
helmfile \
--file "$test_stack_dir/helmfile.d/02-core.yaml.gotmpl" \
--environment default \
--state-values-set ingress.gatewayApi.controllerNamespace=envoy-gateway-system \
--state-values-set ingress.gatewayApi.gateways.shared.name=shared-gw \
--state-values-set ingress.gatewayApi.gateways.shared.namespace=envoy-gateway-system \
--state-values-set ingress.gatewayApi.gateways.grpc.name=grpc-gw \
--state-values-set ingress.gatewayApi.gateways.grpc.namespace=envoy-gateway-system \
--selector "name=$release" \
"$@" \
write-values \
--output-file-template "$output_file"
}

pdb_enabled_in_values() {
local values_file="$1"
local key_prefix="$2" # e.g. "cassandra" or "" for root-level
grep -q "enabled: true" "$values_file"
# helmfile exits non-zero both when a selector matches nothing and when the
# stack fails to render. Only the first is this check's business. Reporting a
# render failure as a missing release is the same misdirection this test was
# fixed for, so let it surface as itself.
local list_log="$work_dir/$release-list.log"
local listed status=0
listed="$(run_helmfile --selector "name=$release" list --skip-charts --output json 2>"$list_log")" ||
status=$?
if test "$status" -ne 0 && ! grep -Fq "no releases found" "$list_log"; then
cat "$list_log" >&2
fail "$release: helmfile could not render the stack"
fi

local matches
matches="$(printf '%s' "${listed:-[]}" |
jq -r --arg name "$release" '[.[] | select(.name == $name)] | length' 2>/dev/null || true)"
test "${matches:-0}" = "1" ||
fail "expected exactly one release named $release in the stack, found ${matches:-0}"
}

pdb_key_present() {
local values_file="$1"
grep -q "podDisruptionBudget:" "$values_file"
}
render_chart_values() {
local release="$1"
local output_file="$2"
shift 2

pdb_min_available() {
local values_file="$1"
grep -q "minAvailable:" "$values_file"
local render_log="$work_dir/$release-write-values.log"
if ! run_helmfile \
--selector "name=$release" \
"$@" \
write-values \
--output-file-template "$output_file" 2>"$render_log"; then
cat "$render_log" >&2
fail "$release: helmfile could not render the stack"
fi

test -s "$output_file" ||
fail "$release: helmfile wrote no values to $output_file"
}

pdb_max_unavailable() {
local values_file="$1"
grep -q "maxUnavailable:" "$values_file"
# Renders a release against its in-repo chart so the chart's own validation is
# what fails. Without --chart the run fails on a chart pull instead, which
# looks the same to a test that only checks for a non-zero exit.
expect_chart_error() {
local release="$1"
local chart_dir="$2"
local log_file="$3"
local expected_error="$4"
local forbidden_error="${5:-}"

if run_helmfile \
--selector "name=$release" \
--chart "$chart_dir" \
--skip-deps \
template >"$log_file" 2>&1; then
fail "$release: expected the chart to reject this configuration"
fi

grep -Fq "$expected_error" "$log_file" ||
fail "$release: error did not contain the expected message: $expected_error"

# The both-fields and neither-field diagnostics share a prefix, so each case
# has to rule out the other one. Without this, a chart that collapsed the two
# branches into one message would still pass both cases.
if test -n "$forbidden_error"; then
if grep -Fq "$forbidden_error" "$log_file"; then
fail "$release: matched the wrong diagnostic: $forbidden_error"
fi
fi
}

write_env() {
cat >"$environment_file"
}

# ---------------------------------------------------------------------------
# 1. Cassandra PDB — omitted (default off)
# 0. Stack topology
# ---------------------------------------------------------------------------
write_env <<'EOF'
global:
Expand All @@ -90,14 +156,21 @@ global:
repository: test/nvcf
EOF

for release in cassandra nats openbao-server ratelimiter ess-api; do
assert_single_release "$release"
done

# ---------------------------------------------------------------------------
# 1. Cassandra PDB - omitted (default off)
# ---------------------------------------------------------------------------
render_chart_values cassandra "$work_dir/cassandra-off-values.yaml" >/dev/null
if grep -q "podDisruptionBudget:" "$work_dir/cassandra-off-values.yaml" &&
grep -A2 "podDisruptionBudget:" "$work_dir/cassandra-off-values.yaml" | grep -q "enabled: true"; then
if yq -e '.cassandra.podDisruptionBudget.enabled == true' \
"$work_dir/cassandra-off-values.yaml" >/dev/null 2>&1; then
fail "cassandra: PDB should be disabled by default but rendered enabled: true"
fi

# ---------------------------------------------------------------------------
# 2. Cassandra PDB — minAvailable passthrough
# 2. Cassandra PDB - minAvailable passthrough
# ---------------------------------------------------------------------------
write_env <<'EOF'
global:
Expand All @@ -111,13 +184,15 @@ cassandra:
EOF

render_chart_values cassandra "$work_dir/cassandra-min-values.yaml" >/dev/null
grep -A2 "podDisruptionBudget:" "$work_dir/cassandra-min-values.yaml" | grep -q "enabled: true" ||
yq -e '.cassandra.podDisruptionBudget.enabled == true' \
"$work_dir/cassandra-min-values.yaml" >/dev/null ||
fail "cassandra: minAvailable PDB did not set enabled: true"
grep -q "minAvailable: 2" "$work_dir/cassandra-min-values.yaml" ||
yq -e '.cassandra.podDisruptionBudget.minAvailable == 2' \
"$work_dir/cassandra-min-values.yaml" >/dev/null ||
fail "cassandra: minAvailable: 2 did not reach the chart values"

# ---------------------------------------------------------------------------
# 3. Cassandra PDB — maxUnavailable passthrough
# 3. Cassandra PDB - maxUnavailable passthrough
# ---------------------------------------------------------------------------
write_env <<'EOF'
global:
Expand All @@ -131,11 +206,12 @@ cassandra:
EOF

render_chart_values cassandra "$work_dir/cassandra-max-values.yaml" >/dev/null
grep -q "maxUnavailable: 1" "$work_dir/cassandra-max-values.yaml" ||
yq -e '.cassandra.podDisruptionBudget.maxUnavailable == 1' \
"$work_dir/cassandra-max-values.yaml" >/dev/null ||
fail "cassandra: maxUnavailable: 1 did not reach the chart values"

# ---------------------------------------------------------------------------
# 4. Cassandra PDB — chart fail: both fields set
# 4. Cassandra PDB - chart fail: both fields set
# ---------------------------------------------------------------------------
write_env <<'EOF'
global:
Expand All @@ -149,21 +225,16 @@ cassandra:
maxUnavailable: 1
EOF

# helm template should fail when both fields are set
if HELMFILE_ENV="$environment_name" \
HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \
helmfile \
--file "$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" \
--environment default \
--selector name=cassandra \
template >"$work_dir/cassandra-both.log" 2>&1; then
fail "cassandra: setting both minAvailable and maxUnavailable should have failed"
fi
grep -qi "exactly one" "$work_dir/cassandra-both.log" ||
fail "cassandra: both-fields error did not contain expected message"
expect_chart_error cassandra "$helm_dir/cassandra/helm" \
"$work_dir/cassandra-both.log" \
"podDisruptionBudget: set exactly one of minAvailable or maxUnavailable, not both"

# ---------------------------------------------------------------------------
# 4b. Cassandra PDB — chart fail: neither field set
# 4b. Cassandra PDB - chart fail: neither field set
#
# base.yaml ships minAvailable: 2, so an environment that only sets enabled
# still inherits an availability field. Clear it with the chart's own unset
# sentinel to reach the neither-field state the chart is supposed to reject.
# ---------------------------------------------------------------------------
write_env <<'EOF'
global:
Expand All @@ -173,23 +244,17 @@ global:
cassandra:
podDisruptionBudget:
enabled: true
minAvailable: ""
EOF

# helm template should fail when enabled but neither field is set
if HELMFILE_ENV="$environment_name" \
HELMFILE_CACHE_HOME="$work_dir/helmfile-cache" \
helmfile \
--file "$test_stack_dir/helmfile.d/01-dependencies.yaml.gotmpl" \
--environment default \
--selector name=cassandra \
template >"$work_dir/cassandra-neither.log" 2>&1; then
fail "cassandra: enabled with neither minAvailable nor maxUnavailable should have failed"
fi
grep -qi "exactly one" "$work_dir/cassandra-neither.log" ||
fail "cassandra: neither-field error did not contain expected message"
expect_chart_error cassandra "$helm_dir/cassandra/helm" \
"$work_dir/cassandra-neither.log" \
"podDisruptionBudget: set exactly one of minAvailable or maxUnavailable" \
"not both"


# ---------------------------------------------------------------------------
# 5. NATS PDB — passthrough (upstream chart; enabled by default)
# 5. NATS PDB - passthrough (upstream chart; enabled by default)
# ---------------------------------------------------------------------------
write_env <<'EOF'
global:
Expand All @@ -202,8 +267,8 @@ nats:
EOF

render_chart_values nats "$work_dir/nats-disabled-values.yaml" >/dev/null
# The nats wrapper values should reflect the override
grep -A2 "podDisruptionBudget:" "$work_dir/nats-disabled-values.yaml" | grep -q "enabled: false" ||
yq -e '.nats.podDisruptionBudget.enabled == false' \
"$work_dir/nats-disabled-values.yaml" >/dev/null ||
fail "nats: podDisruptionBudget.enabled: false did not reach chart values"

# ---------------------------------------------------------------------------
Expand All @@ -223,7 +288,8 @@ openbao:
EOF

render_chart_values openbao-server "$work_dir/openbao-ha-values.yaml" >/dev/null
grep -q "maxUnavailable: 1" "$work_dir/openbao-ha-values.yaml" ||
yq -e '.openbao.server.ha.disruptionBudget.maxUnavailable == 1' \
"$work_dir/openbao-ha-values.yaml" >/dev/null ||
fail "openbao: server.ha.disruptionBudget.maxUnavailable: 1 did not reach chart values"

# ---------------------------------------------------------------------------
Expand All @@ -241,7 +307,8 @@ openbao:
EOF

render_chart_values openbao-server "$work_dir/openbao-injector-values.yaml" >/dev/null
grep -q "minAvailable: 2" "$work_dir/openbao-injector-values.yaml" ||
yq -e '.openbao.injector.podDisruptionBudget.minAvailable == 2' \
"$work_dir/openbao-injector-values.yaml" >/dev/null ||
fail "openbao: injector.podDisruptionBudget.minAvailable: 2 did not reach chart values"

# ---------------------------------------------------------------------------
Expand All @@ -259,7 +326,8 @@ rateLimiter:
EOF

render_chart_values ratelimiter "$work_dir/ratelimiter-values.yaml" >/dev/null
grep -A2 "podDisruptionBudget:" "$work_dir/ratelimiter-values.yaml" | grep -q "enabled: true" ||
yq -e '.rateLimiter.podDisruptionBudget.enabled == true' \
"$work_dir/ratelimiter-values.yaml" >/dev/null ||
fail "rateLimiter: podDisruptionBudget.enabled: true did not reach chart values"

# ---------------------------------------------------------------------------
Expand All @@ -277,7 +345,8 @@ ess:
EOF

render_chart_values ess-api "$work_dir/ess-values.yaml" >/dev/null
grep -A2 "podDisruptionBudget:" "$work_dir/ess-values.yaml" | grep -q "enabled: true" ||
yq -e '.ess.podDisruptionBudget.enabled == true' \
"$work_dir/ess-values.yaml" >/dev/null ||
fail "ess: podDisruptionBudget.enabled: true did not reach chart values"

echo "pdb-value-wiring: all checks passed"
Loading