diff --git a/.claude/skills/validate-pr-override-images/SKILL.md b/.claude/skills/validate-pr-override-images/SKILL.md new file mode 100644 index 000000000000..41b0f60cfc5b --- /dev/null +++ b/.claude/skills/validate-pr-override-images/SKILL.md @@ -0,0 +1,38 @@ +--- +description: Validates that CPO override images in a PR actually contain the PRs they claim to include +argument-hint: "" +--- + +## Name +validate-pr-override-images + +## Synopsis +```text +/validate-pr-override-images +``` + +## Description +Validates that CPO override images in a PR actually contain the claimed fix PRs. + +The PR description must include a structured contract: +``` +branch: 4.20 wants: https://github.com/openshift/hypershift/pull/8593 +branch: 4.21 wants: https://github.com/openshift/hypershift/pull/8593, https://github.com/openshift/hypershift/pull/8565 +``` + +Prerequisites: +- `skopeo` must be installed (`brew install skopeo` on macOS) +- The local git repo must have the relevant release branches fetched +- Images must be accessible from quay.io + +## Implementation + +Extract the PR number from the argument, then run: +```bash +.claude/skills/validate-pr-override-images/validate-overrides.sh +``` + +Report the output to the user. + +## Arguments +- `$1`: PR URL (e.g., `https://github.com/openshift/hypershift/pull/8610`) or PR number (e.g., `8610`) diff --git a/.claude/skills/validate-pr-override-images/validate-overrides.sh b/.claude/skills/validate-pr-override-images/validate-overrides.sh new file mode 100755 index 000000000000..7a76ac2c84f4 --- /dev/null +++ b/.claude/skills/validate-pr-override-images/validate-overrides.sh @@ -0,0 +1,179 @@ +#!/bin/bash +# validate-overrides.sh +# Parses a PR description for the override contract (branch: X.Y wants: PR-links), +# extracts override images from the diff, and validates each image contains the claimed PRs. +# Usage: ./validate-overrides.sh [repo] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" + +if [[ $# -lt 1 || $# -gt 2 ]]; then + echo "Usage: $0 [repo]" >&2 + exit 2 +fi + +PR="$1" +GH_REPO="${2:-openshift/hypershift}" + +echo "=== Validating CPO override images for PR #${PR} ===" +echo "" + +# Step 1: Parse PR description for the contract +echo "--- Step 1: Parsing PR description ---" +BODY=$(gh pr view "$PR" --repo "$GH_REPO" --json body -q .body | tr -d '\r') + +BRANCH_LIST="" +FOUND_LINES=0 +in_code_block=false + +while IFS= read -r line; do + if [[ "$line" == '```'* ]]; then + if $in_code_block; then + in_code_block=false + else + in_code_block=true + fi + continue + fi + if $in_code_block; then + continue + fi + + lower_line=$(echo "$line" | tr '[:upper:]' '[:lower:]') + if [[ ! "$lower_line" == *branch:*wants:* ]]; then + continue + fi + + branch=$(echo "$line" | sed -n 's/^[[:space:]]*[bB][rR][aA][nN][cC][hH]:[[:space:]]*\([0-9]*\.[0-9]*\)[[:space:]]*[wW][aA][nN][tT][sS]:[[:space:]]*\(.*\)$/\1/p') + wants=$(echo "$line" | sed -n 's/^[[:space:]]*[bB][rR][aA][nN][cC][hH]:[[:space:]]*[0-9]*\.[0-9]*[[:space:]]*[wW][aA][nN][tT][sS]:[[:space:]]*\(.*\)$/\1/p') + + if [[ -n "$branch" && -n "$wants" ]]; then + FOUND_LINES=$((FOUND_LINES + 1)) + pr_numbers="" + for url in $(echo "$wants" | tr ',' ' '); do + url=$(echo "$url" | xargs) + num=$(echo "$url" | grep -oE '[0-9]+$' || true) + if [[ -n "$num" ]]; then + if [[ -n "$pr_numbers" ]]; then + pr_numbers="$pr_numbers $num" + else + pr_numbers="$num" + fi + fi + done + if [[ -z "$pr_numbers" ]]; then + echo "ERROR: branch $branch has 'wants:' but no valid PR numbers could be parsed" + exit 1 + fi + BRANCH_LIST="${BRANCH_LIST}${branch}=${pr_numbers} +" + echo " branch $branch wants PRs: $pr_numbers" + fi +done <<< "$BODY" + +if [[ $FOUND_LINES -eq 0 ]]; then + echo "" + echo "ERROR: No 'branch: X.Y wants: ' lines found in PR description." + echo "" + echo "The PR description must include lines like:" + echo " branch: 4.19 wants: https://github.com/openshift/hypershift/pull/1234" + echo " branch: 4.20 wants: https://github.com/openshift/hypershift/pull/1234, https://github.com/openshift/hypershift/pull/5678" + exit 1 +fi + +echo "" + +# Step 2: Extract images per branch from the diff +echo "--- Step 2: Extracting override images from diff ---" +DIFF=$(gh pr diff "$PR" --repo "$GH_REPO") + +IMAGE_LIST="" +current_version="" + +while IFS= read -r line; do + version_match=$(echo "$line" | sed -n 's/^[+ ].*version:[[:space:]]*\([0-9]*\.[0-9]*\)\.[0-9]*.*/\1/p') + if [[ -n "$version_match" ]]; then + current_version="$version_match" + fi + + image_match=$(echo "$line" | sed -n 's/^+.*cpoImage:[[:space:]]*\(.*\)/\1/p') + image_match="${image_match#"${image_match%%[![:space:]]*}"}" + image_match="${image_match%"${image_match##*[![:space:]]}"}" + if [[ -n "$image_match" && -n "$current_version" ]]; then + entry="${current_version}=${image_match}" + if [[ "$IMAGE_LIST" != *"$entry"* ]]; then + IMAGE_LIST="${IMAGE_LIST}${entry} +" + fi + fi +done <<< "$DIFF" + +echo "$IMAGE_LIST" | while IFS= read -r entry; do + if [[ -n "$entry" ]]; then + branch="${entry%%=*}" + image="${entry#*=}" + echo " branch $branch image: $image" + fi +done + +echo "" + +# Step 3: Validate each (branch, image, PR) tuple +echo "--- Step 3: Validating images contain claimed PRs ---" +echo "" + +echo "$BRANCH_LIST" | while IFS= read -r branch_entry; do + if [[ -z "$branch_entry" ]]; then + continue + fi + branch="${branch_entry%%=*}" + prs="${branch_entry#*=}" + + branch_images=$(echo "$IMAGE_LIST" | grep "^${branch}=" | sed "s/^${branch}=//" | sort -u) + + if [[ -z "$branch_images" ]]; then + echo "WARNING: branch $branch declared in description but no override images found in diff" + echo "FAILURE_COUNT:1" + continue + fi + + echo "$branch_images" | while IFS= read -r image; do + if [[ -z "$image" ]]; then + continue + fi + echo "Image: $image (branch $branch)" + for pr_num in $prs; do + verify_output=$("$SCRIPT_DIR/verify-pr-in-image.sh" "$image" "$pr_num" "$REPO_ROOT" 2>&1) && verify_rc=0 || verify_rc=$? + echo "$verify_output" | sed 's/^/ /' + if echo "$verify_output" | tail -1 | grep -q "PASS"; then + echo " PR #${pr_num}: PASS" + echo "PASS_COUNT:1" + else + echo " PR #${pr_num}: FAIL" + echo "FAILURE_COUNT:1" + fi + done + echo "" + done +done > /tmp/validate-overrides-output.$$ + +grep -v "COUNT:" /tmp/validate-overrides-output.$$ +PASSES=$(grep -c "PASS_COUNT:" /tmp/validate-overrides-output.$$ || true) +FAILURES=$(grep -c "FAILURE_COUNT:" /tmp/validate-overrides-output.$$ || true) +rm -f /tmp/validate-overrides-output.$$ + +# Summary +echo "=== Summary ===" +echo "Passed: $PASSES" +echo "Failed: $FAILURES" + +if [[ $FAILURES -gt 0 ]]; then + echo "" + echo "OVERALL: FAIL" + exit 1 +else + echo "" + echo "OVERALL: PASS" +fi diff --git a/.claude/skills/validate-pr-override-images/verify-pr-in-image.sh b/.claude/skills/validate-pr-override-images/verify-pr-in-image.sh new file mode 100755 index 000000000000..f173a8e66f1a --- /dev/null +++ b/.claude/skills/validate-pr-override-images/verify-pr-in-image.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# verify-pr-in-image.sh +# Verifies that a container image contains a specific PR in its git history. +# Usage: ./verify-pr-in-image.sh [repo-path] + +set -euo pipefail + +if [[ $# -lt 2 || $# -gt 3 ]]; then + echo "Usage: $0 [repo-path]" >&2 + exit 2 +fi + +IMAGE="$1" +PR="$2" +REPO="${3:-.}" + +if ! command -v skopeo &>/dev/null; then + echo "ERROR: skopeo is not installed. Install it with: brew install skopeo (macOS) or dnf install skopeo (RHEL/Fedora)" + exit 1 +fi + +echo "Inspecting image..." +INSPECT=$(skopeo inspect --override-os linux --override-arch amd64 "docker://$IMAGE") || { + echo "ERROR: Could not inspect image $IMAGE" + exit 1 +} + +COMMIT=$(echo "$INSPECT" | grep -o '"vcs-ref"[[:space:]]*:[[:space:]]*"[^"]*"' | head -1 | sed 's/.*"vcs-ref"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/') + +if [[ -z "$COMMIT" ]]; then + echo "ERROR: Could not find vcs-ref label in image $IMAGE" + exit 1 +fi + +echo "Image commit: $COMMIT" + +if ! git -C "$REPO" cat-file -e "$COMMIT" 2>/dev/null; then + echo "Commit not found locally, fetching..." + git -C "$REPO" fetch --all --quiet + if ! git -C "$REPO" cat-file -e "$COMMIT" 2>/dev/null; then + echo "ERROR: Commit $COMMIT not found in any remote" + exit 1 + fi +fi + +PR_MERGE_COMMIT=$(gh pr view "$PR" --repo openshift/hypershift --json mergeCommit --jq '.mergeCommit.oid // empty') + +if [[ -z "$PR_MERGE_COMMIT" ]]; then + echo "FAIL: PR #${PR} has no merge commit (not merged yet?)" + exit 1 +fi + +echo "PR #${PR} merge commit: $PR_MERGE_COMMIT" + +if ! git -C "$REPO" cat-file -e "$PR_MERGE_COMMIT" 2>/dev/null; then + echo "Merge commit not found locally, fetching..." + git -C "$REPO" fetch --all --quiet + if ! git -C "$REPO" cat-file -e "$PR_MERGE_COMMIT" 2>/dev/null; then + echo "ERROR: PR #${PR} merge commit $PR_MERGE_COMMIT not found in any remote" + exit 1 + fi +fi + +if git -C "$REPO" merge-base --is-ancestor "$PR_MERGE_COMMIT" "$COMMIT" 2>/dev/null; then + echo "PASS: PR #${PR} is included in image $IMAGE" +else + echo "FAIL: PR #${PR} is NOT included in image $IMAGE" + exit 1 +fi diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 7302caa789c7..7f5a12c25aca 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -3,6 +3,16 @@ language: en-US reviews: auto_review: drafts: true + pre_merge_checks: + docstrings: + mode: "off" + custom_checks: + - name: "MicroShift Test Compatibility" + mode: "off" + - name: "Single Node OpenShift (SNO) Test Compatibility" + mode: "off" + - name: "OTE Binary Stdout Contract" + mode: "off" profile: chill high_level_summary: true collapse_walkthrough: true diff --git a/.dockerignore b/.dockerignore index a78f0089c51c..0b19c9760285 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ bin/ hack/tools/bin/ contrib/ +!contrib/ci/gocacheprog/ .github/ .tekton/ .ci-operator.yaml diff --git a/.github/actions/warm-go-cache/action.yaml b/.github/actions/warm-go-cache/action.yaml deleted file mode 100644 index 993dca6e84fc..000000000000 --- a/.github/actions/warm-go-cache/action.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: 'Warm Go build cache' -description: 'Set GOCACHE via fuse-overlayfs over the EFS-backed PV or a writable fallback' -runs: - using: composite - steps: - - shell: bash - run: | - mkdir -p /tmp/go-build-cache - mounted=false - if [ -d /cache/go-build ] && command -v fuse-overlayfs >/dev/null 2>&1 && [ -e /dev/fuse ]; then - mkdir -p /tmp/go-cache-upper /tmp/go-cache-work - if fuse-overlayfs -o lowerdir=/cache/go-build,upperdir=/tmp/go-cache-upper,workdir=/tmp/go-cache-work /tmp/go-build-cache; then - mounted=true - else - echo "::warning::fuse-overlayfs mount failed, falling back to copy" - fi - fi - if [ "$mounted" = "false" ] && [ -d /cache/go-build ]; then - timeout 120 cp -a /cache/go-build/. /tmp/go-build-cache/ || \ - echo "::warning::Failed to copy EFS cache, proceeding without cache" - fi - echo "GOCACHE=/tmp/go-build-cache" >> "$GITHUB_ENV" diff --git a/.github/workflows/address-review-comments.yaml b/.github/workflows/address-review-comments.yaml new file mode 100644 index 000000000000..e6200098373b --- /dev/null +++ b/.github/workflows/address-review-comments.yaml @@ -0,0 +1,79 @@ +name: Address Review Comments + +on: + issue_comment: + types: [created] + +permissions: + id-token: write + contents: write + pull-requests: write + +jobs: + address-review-comments: + concurrency: + group: address-review-comments-${{ github.event.issue.number }} + cancel-in-progress: true + if: >- + github.event.issue.pull_request && + startsWith(github.event.comment.body, '/address-review-comments') && + (github.event.comment.author_association == 'MEMBER' || + github.event.comment.author_association == 'OWNER' || + github.event.comment.author_association == 'COLLABORATOR') + runs-on: arc-runner-set + timeout-minutes: 30 + env: + HOME: /tmp + steps: + - name: Link to run + env: + GH_TOKEN: ${{ github.token }} + run: | + gh pr comment "${{ github.event.issue.number }}" \ + --repo "${{ github.repository }}" \ + --body "🤖 Addressing review comments: [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})" + + - name: Get PR ref + id: pr + run: | + curl -fsSL -H "Authorization: token ${{ github.token }}" \ + "https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.issue.number }}" > /tmp/pr.json + echo "branch=$(jq -r '.head.ref' /tmp/pr.json)" >> "$GITHUB_OUTPUT" + echo "repo=$(jq -r '.head.repo.full_name' /tmp/pr.json)" >> "$GITHUB_OUTPUT" + + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ steps.pr.outputs.branch }} + repository: ${{ steps.pr.outputs.repo }} + persist-credentials: true + fetch-depth: 0 + + - name: Authenticate to GCP via WIF + uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0 + with: + project_id: hosted-control-planes + service_account: claude-gha@hosted-control-planes.iam.gserviceaccount.com + workload_identity_provider: projects/21066242673/locations/global/workloadIdentityPools/itpc-identity-pool/providers/github-com + + - name: Install Claude Code + run: | + curl -fsSL https://claude.ai/install.sh | bash + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Set up ai-helpers plugins + run: | + git clone --depth 1 https://github.com/openshift-eng/ai-helpers.git "$GITHUB_WORKSPACE/ai-helpers" + mkdir -p "$HOME/.claude/plugins" + printf '%s\n' '{"enabledPlugins":{"utils@ai-helpers":true}}' > "$HOME/.claude/settings.json" + printf '%s\n' "{\"ai-helpers\":{\"source\":{\"source\":\"directory\",\"path\":\"$GITHUB_WORKSPACE/ai-helpers\"},\"installLocation\":\"$GITHUB_WORKSPACE/ai-helpers\",\"lastUpdated\":\"2025-10-27T12:00:00.000Z\"}}" > "$HOME/.claude/plugins/known_marketplaces.json" + + - name: Address PR review comments + env: + CLAUDE_CODE_USE_VERTEX: "1" + CLOUD_ML_REGION: global + ANTHROPIC_VERTEX_PROJECT_ID: hosted-control-planes + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.issue.number }} + run: | + claude --version + claude -p "/utils:address-reviews $PR_NUMBER" --model claude-opus-4-6 --max-turns 100 --allowedTools "Bash Read Write Edit Grep Glob WebFetch" diff --git a/.github/workflows/envtest-kube-reusable.yaml b/.github/workflows/envtest-kube-reusable.yaml index 7e19f51213a3..e38fe093c06d 100644 --- a/.github/workflows/envtest-kube-reusable.yaml +++ b/.github/workflows/envtest-kube-reusable.yaml @@ -57,7 +57,6 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: ./.github/actions/warm-go-cache - run: make test-envtest-kube ENVTEST_KUBE_VERSIONS="${{ matrix.version }}" conclusion: diff --git a/.github/workflows/envtest-ocp-reusable.yaml b/.github/workflows/envtest-ocp-reusable.yaml index b44ac05c40e1..2f001758c436 100644 --- a/.github/workflows/envtest-ocp-reusable.yaml +++ b/.github/workflows/envtest-ocp-reusable.yaml @@ -58,7 +58,6 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: ./.github/actions/warm-go-cache - run: make test-envtest-ocp ENVTEST_OCP_K8S_VERSIONS="${{ matrix.version }}" conclusion: diff --git a/.github/workflows/gocacheprog-test-reusable.yaml b/.github/workflows/gocacheprog-test-reusable.yaml new file mode 100644 index 000000000000..f8a0d80091e5 --- /dev/null +++ b/.github/workflows/gocacheprog-test-reusable.yaml @@ -0,0 +1,25 @@ +name: gocacheprog Tests (Reusable) + +on: + workflow_call: + +permissions: + contents: read + +jobs: + test: + name: gocacheprog Unit Tests + runs-on: arc-runner-set + timeout-minutes: 10 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + env: + HOME: /tmp + with: + go-version-file: contrib/ci/gocacheprog/go.mod + cache: false + - name: Run tests + run: cd contrib/ci/gocacheprog && go test -race -count=1 ./... diff --git a/.github/workflows/gocacheprog-test.yaml b/.github/workflows/gocacheprog-test.yaml new file mode 100644 index 000000000000..53cfad779a02 --- /dev/null +++ b/.github/workflows/gocacheprog-test.yaml @@ -0,0 +1,15 @@ +name: gocacheprog Tests + +on: + pull_request: + branches: + - main + - release-4.22 + paths: + - contrib/ci/gocacheprog/** + +jobs: + test: + uses: openshift/hypershift/.github/workflows/gocacheprog-test-reusable.yaml@main + permissions: + contents: read diff --git a/.github/workflows/lint-reusable.yaml b/.github/workflows/lint-reusable.yaml index 3efe807ff570..e969b494fa8d 100644 --- a/.github/workflows/lint-reusable.yaml +++ b/.github/workflows/lint-reusable.yaml @@ -20,7 +20,6 @@ jobs: if [ -n "${{ github.base_ref }}" ]; then git fetch origin "${{ github.base_ref }}:${{ github.base_ref }}" fi - - uses: ./.github/actions/warm-go-cache - name: Use pre-built lint tools run: | if [ -d /opt/lint-tools ]; then diff --git a/.github/workflows/test-reusable.yaml b/.github/workflows/test-reusable.yaml index c5af0a57b53c..76f0595fca17 100644 --- a/.github/workflows/test-reusable.yaml +++ b/.github/workflows/test-reusable.yaml @@ -85,11 +85,10 @@ jobs: with: go-version-file: go.mod cache: false - - uses: ./.github/actions/warm-go-cache - name: Run tests run: make test-shard TEST_PACKAGES="${{ matrix.packages }}" COVER_PROFILE="cover-${{ matrix.shard }}.out" - name: Upload to Codecov - uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 env: HOME: /tmp CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/validate-cpo-overrides.yaml b/.github/workflows/validate-cpo-overrides.yaml new file mode 100644 index 000000000000..4ea62a28eb06 --- /dev/null +++ b/.github/workflows/validate-cpo-overrides.yaml @@ -0,0 +1,27 @@ +name: Validate CPO Overrides + +on: + pull_request: + branches: + - main + paths: + - 'hypershift-operator/controlplaneoperator-overrides/assets/overrides.yaml' + +permissions: + contents: read + pull-requests: read + +jobs: + validate-cpo-overrides: + name: Validate CPO Override Images + runs-on: arc-runner-set + timeout-minutes: 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + persist-credentials: false + - name: Validate override images + env: + GH_TOKEN: ${{ github.token }} + run: .claude/skills/validate-pr-override-images/validate-overrides.sh "${{ github.event.pull_request.number }}" diff --git a/.github/workflows/verify-reusable.yaml b/.github/workflows/verify-reusable.yaml index a9972031bfbd..f543d537bff0 100644 --- a/.github/workflows/verify-reusable.yaml +++ b/.github/workflows/verify-reusable.yaml @@ -15,7 +15,6 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: ./.github/actions/warm-go-cache - run: make generate update - run: make staticcheck - run: make fmt diff --git a/.golangci.yml b/.golangci.yml index f7322988a683..ea42306b0280 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -3,9 +3,16 @@ run: allow-parallel-runners: true linters: enable: + - dupword + - durationcheck + - errorlint + - fatcontext - gocyclo - misspell + - nilerr + - noctx - unparam + - usestdlibvars settings: gocyclo: min-complexity: 30 diff --git a/.tekton/hypershift-gh-actions-runner-pull-request.yaml b/.tekton/hypershift-gh-actions-runner-pull-request.yaml index 73bba6275a52..c208ffb24c1b 100644 --- a/.tekton/hypershift-gh-actions-runner-pull-request.yaml +++ b/.tekton/hypershift-gh-actions-runner-pull-request.yaml @@ -9,7 +9,8 @@ metadata: pipelinesascode.tekton.dev/cancel-in-progress: "true" pipelinesascode.tekton.dev/max-keep-runs: "3" pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch - == "main" && "Dockerfile.github-actions-runner".pathChanged() + == "main" && ("Dockerfile.github-actions-runner".pathChanged() || "contrib/ci/gocacheprog".pathChanged() + || ".tekton/hypershift-gh-actions-runner-pull-request.yaml".pathChanged()) pipelinesascode.tekton.dev/pipeline: ".tekton/pipelines/common-operator-build.yaml" creationTimestamp: null labels: diff --git a/.tekton/hypershift-gh-actions-runner-push.yaml b/.tekton/hypershift-gh-actions-runner-push.yaml index 1bfb8b6c69df..c288a6a2fb08 100644 --- a/.tekton/hypershift-gh-actions-runner-push.yaml +++ b/.tekton/hypershift-gh-actions-runner-push.yaml @@ -8,7 +8,8 @@ metadata: pipelinesascode.tekton.dev/cancel-in-progress: "false" pipelinesascode.tekton.dev/max-keep-runs: "3" pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch - == "main" && "Dockerfile.github-actions-runner".pathChanged() + == "main" && ("Dockerfile.github-actions-runner".pathChanged() || "contrib/ci/gocacheprog".pathChanged() + || ".tekton/hypershift-gh-actions-runner-push.yaml".pathChanged()) pipelinesascode.tekton.dev/pipeline: ".tekton/pipelines/common-operator-build.yaml" creationTimestamp: null labels: diff --git a/AGENTS.md b/AGENTS.md index 0e59539acb9e..fcd98509cac3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -102,7 +102,7 @@ See .claude/skills/dev - Test cases are YAML-driven following the openshift/api convention - Each YAML file defines `onCreate` and `onUpdate` test cases with expected errors - Run with `make test-envtest-ocp` (OpenShift k8s versions) or `make test-envtest-kube` (vanilla k8s versions), or `make test-envtest-api-all` for both -- Tests run across multiple Kubernetes versions (1.31–1.35) to verify validation ratcheting and compatibility +- Tests run across multiple Kubernetes versions (1.30–1.35) to verify validation ratcheting and compatibility - Feature gate filtering: test suites can target stable, tech-preview, or feature-gated CRD variants See test/envtest/README.md for details diff --git a/Dockerfile.e2e b/Dockerfile.e2e index 68086ed8bae4..f163a1c773d9 100644 --- a/Dockerfile.e2e +++ b/Dockerfile.e2e @@ -30,5 +30,5 @@ COPY --from=builder /hypershift/hack/run-reqserving-e2e.sh /hypershift/hack/run- RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc && \ dnf install -y https://packages.microsoft.com/config/rhel/9/packages-microsoft-prod.rpm && \ mv /etc/yum.repos.d/microsoft-prod.repo /etc/yum.repos.art/ci/ && \ - dnf install -y azure-cli && \ + dnf install -y azure-cli-2.72.0 && \ dnf clean all diff --git a/Dockerfile.github-actions-runner b/Dockerfile.github-actions-runner index edbf622f822f..4d3f73b81482 100644 --- a/Dockerfile.github-actions-runner +++ b/Dockerfile.github-actions-runner @@ -1,4 +1,4 @@ -FROM ghcr.io/actions/actions-runner@sha256:1ad983536759ceec39ed75a2c8f007ca8c37b66eee35ed86f13623b29a4db97d +FROM ghcr.io/actions/actions-runner@sha256:b6614fce332517f74d0a76e7c762fb08e4f2ff13dcf333183397c8a5725b6e8e USER root @@ -11,7 +11,11 @@ RUN apt-get update && \ curl \ ca-certificates \ python3-pip \ - fuse-overlayfs \ + skopeo \ + && curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg -o /usr/share/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list \ + && apt-get update \ + && apt-get install -y --no-install-recommends gh \ && rm -rf /var/lib/apt/lists/* ARG TARGETARCH @@ -35,4 +39,7 @@ RUN cd /tmp/tools && \ -o /opt/lint-tools/kube-api-linter.so sigs.k8s.io/kube-api-linter/pkg/plugin && \ rm -rf /tmp/tools +COPY contrib/ci/gocacheprog/ /tmp/gocacheprog/ +RUN cd /tmp/gocacheprog && go build -o /usr/local/bin/gocacheprog . && rm -rf /tmp/gocacheprog + USER runner diff --git a/api/hypershift/v1beta1/hostedcluster_types.go b/api/hypershift/v1beta1/hostedcluster_types.go index 7a499c864226..74b50f24ad38 100644 --- a/api/hypershift/v1beta1/hostedcluster_types.go +++ b/api/hypershift/v1beta1/hostedcluster_types.go @@ -527,7 +527,6 @@ type Capabilities struct { // +kubebuilder:validation:XValidation:rule=`!self.services.exists(s, s.service == 'APIServer' && has(s.servicePublishingStrategy.loadBalancer) && s.servicePublishingStrategy.loadBalancer.hostname != "" && has(self.configuration) && has(self.configuration.apiServer) && has(self.configuration.apiServer.servingCerts) && has(self.configuration.apiServer.servingCerts.namedCertificates) && self.configuration.apiServer.servingCerts.namedCertificates.exists(cert, has(cert.names) && cert.names.exists(n, n == s.servicePublishingStrategy.loadBalancer.hostname)))`, message="APIServer loadBalancer hostname cannot be in ClusterConfiguration.apiserver.servingCerts.namedCertificates[]" // +kubebuilder:validation:XValidation:rule="!has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.disableMultiNetwork) || !self.operatorConfiguration.clusterNetworkOperator.disableMultiNetwork || self.networking.networkType == 'Other'",message="disableMultiNetwork can only be set to true when networkType is 'Other'" // +kubebuilder:validation:XValidation:rule="self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig)", message="ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes" -// +kubebuilder:validation:XValidation:rule=`self.platform.type != "Azure" || self.dns.baseDomain == "" || !self.services.exists(s, (has(s.servicePublishingStrategy.route) && has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') + 1))))`,message="Azure service hostname domain must not overlap with the cluster base domain. An Azure Private DNS zone matching or containing the base domain would shadow *.apps DNS resolution." type HostedClusterSpec struct { // release specifies the desired OCP release payload for all the hosted cluster components. // This includes those components running management side like the Kube API Server and the CVO but also the operands which land in the hosted cluster data plane like the ingress controller, ovn agents, etc. diff --git a/api/hypershift/v1beta1/operator.go b/api/hypershift/v1beta1/operator.go index ee74790f5ba0..c14858d3fa14 100644 --- a/api/hypershift/v1beta1/operator.go +++ b/api/hypershift/v1beta1/operator.go @@ -4,6 +4,20 @@ import ( operatorv1 "github.com/openshift/api/operator/v1" ) +const ( + // KubevirtDefaultV6InternalJoinSubnet is the default IPv6 OVN join subnet + // for KubeVirt hosted clusters. The upstream OVN-Kubernetes default is fd98::/64, + // but KubeVirt guests use fd99::/64 to avoid collisions with the management + // cluster's join subnet when both run OVN-Kubernetes. + KubevirtDefaultV6InternalJoinSubnet = "fd99::/64" + + // KubevirtDefaultV4InternalSubnet is the default IPv4 OVN internal subnet + // for KubeVirt hosted clusters. The upstream OVN-Kubernetes default gateway + // router LRP CIDR is 100.64.0.0/16 and the default UDNs is 100.65.0.0/16. + // KubeVirt guests use 100.66.0.0/16 to avoid collisions with the management cluster. + KubevirtDefaultV4InternalSubnet = "100.66.0.0/16" +) + // +kubebuilder:validation:Enum="";Normal;Debug;Trace;TraceAll type LogLevel string @@ -38,6 +52,7 @@ type ClusterVersionOperatorSpec struct { OperatorLogLevel LogLevel `json:"operatorLogLevel,omitempty"` } +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)", message="ovnKubernetesConfig is immutable once set and cannot be removed" type ClusterNetworkOperatorSpec struct { // disableMultiNetwork when set to true disables the Multus CNI plugin and related components // in the hosted cluster. This prevents the installation of multus daemon sets in the @@ -62,7 +77,11 @@ type ClusterNetworkOperatorSpec struct { // OVNKubernetesConfig contains OVN-Kubernetes specific configuration options. // https://github.com/openshift/api/blob/6d3c4e25a8d3aeb57ad61649d80c38cbd27d1cc8/operator/v1/types_network.go#L400-L471 // +kubebuilder:validation:XValidation:rule="!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet", message="internalJoinSubnet and internalTransitSwitchSubnet must not be the same" +// +kubebuilder:validation:XValidation:rule="!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet != self.ipv6.internalTransitSwitchSubnet", message="ipv6 internalJoinSubnet and internalTransitSwitchSubnet must not be the same" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.mtu) || has(self.mtu)",message="mtu is immutable once set and cannot be removed" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipv6) || has(self.ipv6)", message="ipv6 is immutable once set and cannot be removed" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))", message="ipv6.internalJoinSubnet cannot be removed once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))", message="ipv6.internalTransitSwitchSubnet cannot be removed once set" // +kubebuilder:validation:MinProperties=1 type OVNKubernetesConfig struct { // ipv4 allows users to configure IP settings for IPv4 connections. When omitted, @@ -71,6 +90,15 @@ type OVNKubernetesConfig struct { // +optional IPv4 *OVNIPv4Config `json:"ipv4,omitempty"` + // ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + // this means no opinions and the default configuration is used. Check individual + // fields within ipv6 for details of default values. + // For KubeVirt hosted clusters using dual-stack networking, it is recommended to + // set ipv6.internalJoinSubnet to a value different from the management cluster's + // join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + // +optional + IPv6 OVNIPv6Config `json:"ipv6,omitzero,omitempty"` + // mtu is the MTU to use for the tunnel interface on hosted cluster nodes. // This must be 100 bytes smaller than the uplink MTU. // When unset, the cluster-network-operator will determine the MTU automatically @@ -126,6 +154,52 @@ type OVNIPv4Config struct { InternalJoinSubnet string `json:"internalJoinSubnet,omitempty"` } +// OVNIPv6Config contains IPv6-specific configuration options for OVN-Kubernetes. +// https://github.com/openshift/api/blob/6d3c4e25a8d3aeb57ad61649d80c38cbd27d1cc8/operator/v1/types_network.go#L541-L570 +// +kubebuilder:validation:MinProperties=1 +type OVNIPv6Config struct { + // internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + // by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + // architecture that connects the cluster routers on each node together to enable + // east west traffic. The subnet chosen should not overlap with other networks + // specified for OVN-Kubernetes as well as other networks used on the host. + // When omitted, this means no opinion and the platform is left to choose a reasonable + // default which is subject to change over time. + // The current default subnet is fd97::/64. + // The subnet must be large enough to accommodate one IP per node in your cluster. + // The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + // IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + // The prefix length must be in the range /0 to /125 inclusive. + // This field is immutable once set. + // +kubebuilder:validation:MaxLength=48 + // +kubebuilder:validation:MinLength=3 + // +kubebuilder:validation:XValidation:rule="isCIDR(self) && cidr(self).ip().family() == 6", message="Subnet must be in valid IPv6 CIDR format (e.g., fd97::/64)" + // +kubebuilder:validation:XValidation:rule="isCIDR(self) && cidr(self).prefixLength() <= 125", message="subnet must be in the range /0 to /125 inclusive" + // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="internalTransitSwitchSubnet is immutable" + // +optional + InternalTransitSwitchSubnet string `json:"internalTransitSwitchSubnet,omitempty"` + // internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + // default one is being already used by something else. It must not overlap with + // any other subnet being used by OpenShift or by the node network. The size of the + // subnet must be larger than the number of nodes. + // The current default value is fd98::/64. + // For KubeVirt hosted clusters, if this field is not set, HyperShift will + // automatically use fd99::/64 to avoid collisions with the management cluster's + // default join subnet (fd98::/64). + // The subnet must be large enough to accommodate one IP per node in your cluster. + // The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + // IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + // The prefix length must be in the range /0 to /125 inclusive. + // This field is immutable once set. + // +kubebuilder:validation:MaxLength=48 + // +kubebuilder:validation:MinLength=3 + // +kubebuilder:validation:XValidation:rule="isCIDR(self) && cidr(self).ip().family() == 6", message="Subnet must be in valid IPv6 CIDR format (e.g., fd98::/64)" + // +kubebuilder:validation:XValidation:rule="isCIDR(self) && cidr(self).prefixLength() <= 125", message="subnet must be in the range /0 to /125 inclusive" + // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="internalJoinSubnet is immutable" + // +optional + InternalJoinSubnet string `json:"internalJoinSubnet,omitempty"` +} + // IngressOperatorSpec is the specification of the desired behavior of the Ingress Operator. type IngressOperatorSpec struct { // endpointPublishingStrategy is used to publish the default ingress controller endpoints. diff --git a/api/hypershift/v1beta1/zz_generated.deepcopy.go b/api/hypershift/v1beta1/zz_generated.deepcopy.go index 5954bdd73ad3..a3d882374d25 100644 --- a/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -3818,6 +3818,21 @@ func (in *OVNIPv4Config) DeepCopy() *OVNIPv4Config { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OVNIPv6Config) DeepCopyInto(out *OVNIPv6Config) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OVNIPv6Config. +func (in *OVNIPv6Config) DeepCopy() *OVNIPv6Config { + if in == nil { + return nil + } + out := new(OVNIPv6Config) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OVNKubernetesConfig) DeepCopyInto(out *OVNKubernetesConfig) { *out = *in @@ -3826,6 +3841,7 @@ func (in *OVNKubernetesConfig) DeepCopyInto(out *OVNKubernetesConfig) { *out = new(OVNIPv4Config) **out = **in } + out.IPv6 = in.IPv6 } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OVNKubernetesConfig. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AAA_ungated.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AAA_ungated.yaml index d4f89dc7f67a..1c41742b57a1 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AAA_ungated.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/AAA_ungated.yaml @@ -3193,6 +3193,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3220,9 +3289,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -6533,17 +6621,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml index 182195cebaf4..039f5cf6393d 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml @@ -3184,6 +3184,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3211,9 +3280,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -6516,17 +6604,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml index dee9aaf22e92..3acebef0f14e 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml @@ -3184,6 +3184,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3211,9 +3280,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' clusterVersionOperator: description: clusterVersionOperator specifies the configuration for the Cluster Version Operator in the hosted cluster. @@ -6536,17 +6624,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDC.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDC.yaml index 712a324f90cb..4daf2de45fe9 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDC.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDC.yaml @@ -3516,6 +3516,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3543,9 +3612,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -6848,17 +6936,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml index 708eef6074fd..a76def974bbf 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml @@ -3656,6 +3656,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3683,9 +3752,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -6988,17 +7076,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml index 9e6779997079..a9b7aa0f7d70 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml @@ -3647,6 +3647,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3674,9 +3743,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -6979,17 +7067,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml index 664ded88d91a..a4b7483426ff 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/GCPPlatform.yaml @@ -3184,6 +3184,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3211,9 +3280,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -6962,17 +7050,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HCPEtcdBackup.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HCPEtcdBackup.yaml index 96efc699101b..6ae5927c5422 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HCPEtcdBackup.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HCPEtcdBackup.yaml @@ -3249,6 +3249,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3276,9 +3345,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -6581,17 +6669,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml index 3d80cd009681..deca5394369c 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml @@ -3206,6 +3206,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3233,9 +3302,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -6538,17 +6626,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ImageStreamImportMode.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ImageStreamImportMode.yaml index 541f071eec45..5e039ba658b3 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ImageStreamImportMode.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/ImageStreamImportMode.yaml @@ -3202,6 +3202,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3229,9 +3298,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -6534,17 +6622,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/KMSEncryptionProvider.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/KMSEncryptionProvider.yaml index da86225a28ba..0658922050be 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/KMSEncryptionProvider.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/KMSEncryptionProvider.yaml @@ -3260,6 +3260,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3287,9 +3356,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -6592,17 +6680,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/OpenStack.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/OpenStack.yaml index 3da4a9e3002c..2d3ea4fb777e 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/OpenStack.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/OpenStack.yaml @@ -3184,6 +3184,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3211,9 +3280,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -7067,17 +7155,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/TLSAdherence.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/TLSAdherence.yaml index 3b6458eb9e33..778d189b3305 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/TLSAdherence.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedclusters.hypershift.openshift.io/TLSAdherence.yaml @@ -3224,6 +3224,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3251,9 +3320,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -6556,17 +6644,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AAA_ungated.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AAA_ungated.yaml index d1af5de7c6f0..8915bf2fb64c 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AAA_ungated.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/AAA_ungated.yaml @@ -3081,6 +3081,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3108,9 +3177,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml index 849aee00c07b..1540ec8d3d4e 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterUpdateAcceptRisks.yaml @@ -3072,6 +3072,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3099,9 +3168,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml index 4f6507aeb1e7..4b5cea37472e 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ClusterVersionOperatorConfiguration.yaml @@ -3072,6 +3072,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3099,9 +3168,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' clusterVersionOperator: description: clusterVersionOperator specifies the configuration for the Cluster Version Operator in the hosted cluster. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDC.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDC.yaml index 1de483a48ece..8a5926d9a09c 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDC.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDC.yaml @@ -3404,6 +3404,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3431,9 +3500,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml index 40bad65502c5..e0eb8d100ecb 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUIDAndExtraClaimMappings.yaml @@ -3544,6 +3544,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3571,9 +3640,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml index 945cd0cc0095..040e4977c09d 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ExternalOIDCWithUpstreamParity.yaml @@ -3535,6 +3535,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3562,9 +3631,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml index d11902b2fac5..bcc6e44ccc1b 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/GCPPlatform.yaml @@ -3072,6 +3072,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3099,9 +3168,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HCPEtcdBackup.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HCPEtcdBackup.yaml index 9d7f764d263a..db74c118d765 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HCPEtcdBackup.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HCPEtcdBackup.yaml @@ -3137,6 +3137,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3164,9 +3233,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml index b8c3890b21b9..1a0efedcce26 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/HyperShiftOnlyDynamicResourceAllocation.yaml @@ -3094,6 +3094,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3121,9 +3190,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ImageStreamImportMode.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ImageStreamImportMode.yaml index f6b4ae7f3a21..1f8d79dfc0a9 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ImageStreamImportMode.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/ImageStreamImportMode.yaml @@ -3090,6 +3090,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3117,9 +3186,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/KMSEncryptionProvider.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/KMSEncryptionProvider.yaml index 426e52a7dca1..a73adb342347 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/KMSEncryptionProvider.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/KMSEncryptionProvider.yaml @@ -3148,6 +3148,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3175,9 +3244,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/OpenStack.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/OpenStack.yaml index 675c1d6f721f..e2cff3fb6517 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/OpenStack.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/OpenStack.yaml @@ -3072,6 +3072,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3099,9 +3168,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/TLSAdherence.yaml b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/TLSAdherence.yaml index bcb4a21223bd..6595c34e17cb 100644 --- a/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/TLSAdherence.yaml +++ b/api/hypershift/v1beta1/zz_generated.featuregated-crd-manifests/hostedcontrolplanes.hypershift.openshift.io/TLSAdherence.yaml @@ -3112,6 +3112,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3139,9 +3208,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/availability-prober/availability_prober.go b/availability-prober/availability_prober.go index d68d0df137e5..0020bb8b8521 100644 --- a/availability-prober/availability_prober.go +++ b/availability-prober/availability_prober.go @@ -103,13 +103,13 @@ func NewStartCommand() *cobra.Command { } } - check(log, url, time.Second, time.Second, opts.requiredAPIsParsed, opts.waitForInfrastructureResource, opts.waitForClusterRolebinding, opts.waitForLabeledPodsGone, discoveryClient, kubeClient) + check(cmd.Context(), log, url, time.Second, time.Second, opts.requiredAPIsParsed, opts.waitForInfrastructureResource, opts.waitForClusterRolebinding, opts.waitForLabeledPodsGone, discoveryClient, kubeClient) } return cmd } -func check(log logr.Logger, target *url.URL, requestTimeout time.Duration, sleepTime time.Duration, requiredAPIs []schema.GroupVersionKind, waitForInfrastructureResource bool, waitForClusterRolebinding, waitForLabeledPodsGone string, discoveryClient discovery.DiscoveryInterface, kubeClient crclient.Client) { +func check(ctx context.Context, log logr.Logger, target *url.URL, requestTimeout time.Duration, sleepTime time.Duration, requiredAPIs []schema.GroupVersionKind, waitForInfrastructureResource bool, waitForClusterRolebinding, waitForLabeledPodsGone string, discoveryClient discovery.DiscoveryInterface, kubeClient crclient.Client) { log = log.WithValues("sleepTime", sleepTime.String()) client := &http.Client{ Timeout: requestTimeout, @@ -118,7 +118,12 @@ func check(log logr.Logger, target *url.URL, requestTimeout time.Duration, sleep }, } for ; ; time.Sleep(sleepTime) { - response, err := client.Get(target.String()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil) + if err != nil { + log.Error(err, "Failed to create request, retrying...") + continue + } + response, err := client.Do(req) if err != nil { log.Error(err, "Request failed, retrying...") continue diff --git a/client/applyconfiguration/hypershift/v1beta1/ovnipv6config.go b/client/applyconfiguration/hypershift/v1beta1/ovnipv6config.go new file mode 100644 index 000000000000..2205ac30a97e --- /dev/null +++ b/client/applyconfiguration/hypershift/v1beta1/ovnipv6config.go @@ -0,0 +1,47 @@ +/* + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +// OVNIPv6ConfigApplyConfiguration represents a declarative configuration of the OVNIPv6Config type for use +// with apply. +type OVNIPv6ConfigApplyConfiguration struct { + InternalTransitSwitchSubnet *string `json:"internalTransitSwitchSubnet,omitempty"` + InternalJoinSubnet *string `json:"internalJoinSubnet,omitempty"` +} + +// OVNIPv6ConfigApplyConfiguration constructs a declarative configuration of the OVNIPv6Config type for use with +// apply. +func OVNIPv6Config() *OVNIPv6ConfigApplyConfiguration { + return &OVNIPv6ConfigApplyConfiguration{} +} + +// WithInternalTransitSwitchSubnet sets the InternalTransitSwitchSubnet field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the InternalTransitSwitchSubnet field is set to the value of the last call. +func (b *OVNIPv6ConfigApplyConfiguration) WithInternalTransitSwitchSubnet(value string) *OVNIPv6ConfigApplyConfiguration { + b.InternalTransitSwitchSubnet = &value + return b +} + +// WithInternalJoinSubnet sets the InternalJoinSubnet field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the InternalJoinSubnet field is set to the value of the last call. +func (b *OVNIPv6ConfigApplyConfiguration) WithInternalJoinSubnet(value string) *OVNIPv6ConfigApplyConfiguration { + b.InternalJoinSubnet = &value + return b +} diff --git a/client/applyconfiguration/hypershift/v1beta1/ovnkubernetesconfig.go b/client/applyconfiguration/hypershift/v1beta1/ovnkubernetesconfig.go index 6d3627c54206..ec04a89fa382 100644 --- a/client/applyconfiguration/hypershift/v1beta1/ovnkubernetesconfig.go +++ b/client/applyconfiguration/hypershift/v1beta1/ovnkubernetesconfig.go @@ -21,6 +21,7 @@ package v1beta1 // with apply. type OVNKubernetesConfigApplyConfiguration struct { IPv4 *OVNIPv4ConfigApplyConfiguration `json:"ipv4,omitempty"` + IPv6 *OVNIPv6ConfigApplyConfiguration `json:"ipv6,omitempty"` MTU *int32 `json:"mtu,omitempty"` } @@ -38,6 +39,14 @@ func (b *OVNKubernetesConfigApplyConfiguration) WithIPv4(value *OVNIPv4ConfigApp return b } +// WithIPv6 sets the IPv6 field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the IPv6 field is set to the value of the last call. +func (b *OVNKubernetesConfigApplyConfiguration) WithIPv6(value *OVNIPv6ConfigApplyConfiguration) *OVNKubernetesConfigApplyConfiguration { + b.IPv6 = value + return b +} + // WithMTU sets the MTU field in the declarative configuration to the given value // and returns the receiver, so that objects can be built by chaining "With" function invocations. // If called multiple times, the MTU field is set to the value of the last call. diff --git a/client/applyconfiguration/utils.go b/client/applyconfiguration/utils.go index 77e8b9c056b6..6c7a8f85bb03 100644 --- a/client/applyconfiguration/utils.go +++ b/client/applyconfiguration/utils.go @@ -347,6 +347,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &hypershiftv1beta1.OperatorConfigurationApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("OVNIPv4Config"): return &hypershiftv1beta1.OVNIPv4ConfigApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("OVNIPv6Config"): + return &hypershiftv1beta1.OVNIPv6ConfigApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("OVNKubernetesConfig"): return &hypershiftv1beta1.OVNKubernetesConfigApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("PersistentVolumeEtcdStorageSpec"): diff --git a/cmd/bastion/aws/create.go b/cmd/bastion/aws/create.go index 4d8cb5c244e9..f5abf0965f39 100644 --- a/cmd/bastion/aws/create.go +++ b/cmd/bastion/aws/create.go @@ -146,7 +146,7 @@ func (o *CreateBastionOpts) Run(ctx context.Context, logger logr.Logger) (string var err error sshPublicKey, err = os.ReadFile(o.SSHKeyFile) if err != nil { - return "", "", fmt.Errorf("cannot read SSH public key from %s: %v", o.SSHKeyFile, err) + return "", "", fmt.Errorf("cannot read SSH public key from %s: %w", o.SSHKeyFile, err) } } diff --git a/cmd/cluster/azure/create.go b/cmd/cluster/azure/create.go index 3a3cde58b9fd..f1b210cba903 100644 --- a/cmd/cluster/azure/create.go +++ b/cmd/cluster/azure/create.go @@ -161,7 +161,7 @@ func BindProductCoreFlags(opts *core.RawCreateOptions, flags *pflag.FlagSet) { } // Validate validates the Azure create cluster command options -func (o *RawCreateOptions) Validate(ctx context.Context, opts *core.CreateOptions) (core.PlatformCompleter, error) { +func (o *RawCreateOptions) Validate(ctx context.Context, _ *core.CreateOptions) (core.PlatformCompleter, error) { var err error // Check if the network security group is set and the resource group is not @@ -222,12 +222,6 @@ func (o *RawCreateOptions) Validate(ctx context.Context, opts *core.CreateOption } } - if opts != nil { - if err := validateExternalDNSDomain(opts.ExternalDNSDomain, opts.Name, opts.BaseDomain); err != nil { - return nil, err - } - } - validOpts := &ValidatedCreateOptions{ validatedCreateOptions: &validatedCreateOptions{ RawCreateOptions: o, @@ -258,36 +252,6 @@ func (o *RawCreateOptions) Validate(ctx context.Context, opts *core.CreateOption return validOpts, nil } -// validateExternalDNSDomain checks that the external DNS domain does not conflict with the cluster -// domain. When a private Azure HostedCluster uses an externalDNSDomain that matches or is a parent -// of the cluster domain (clusterName.baseDomain), the PLS controller creates an Azure Private DNS -// zone that shadows *.apps DNS resolution. -func validateExternalDNSDomain(externalDNSDomain, clusterName, baseDomain string) error { - if externalDNSDomain == "" { - return nil - } - - if clusterName == "" || baseDomain == "" { - return nil - } - - clusterDomain := clusterName + "." + baseDomain - - extLower := strings.ToLower(strings.TrimSuffix(externalDNSDomain, ".")) - clusterLower := strings.ToLower(strings.TrimSuffix(clusterDomain, ".")) - - // Check if the externalDNSDomain matches or is a parent of the cluster domain. - // An exact match means the Private DNS zone would directly shadow *.apps. - // A suffix match (with dot boundary) means the zone is a parent that would also shadow. - if extLower == clusterLower || strings.HasSuffix(clusterLower, "."+extLower) { - return fmt.Errorf("external DNS domain %q conflicts with cluster domain %q: "+ - "this would create an Azure Private DNS zone that shadows *.apps DNS resolution. "+ - "Use a different --external-dns-domain value", externalDNSDomain, clusterDomain) - } - - return nil -} - // Complete completes the Azure create cluster command options func (o *ValidatedCreateOptions) Complete(ctx context.Context, opts *core.CreateOptions) (core.Platform, error) { output := &CreateOptions{ diff --git a/cmd/cluster/azure/create_test.go b/cmd/cluster/azure/create_test.go index bd62093d3192..4f73b8d7646b 100644 --- a/cmd/cluster/azure/create_test.go +++ b/cmd/cluster/azure/create_test.go @@ -65,7 +65,7 @@ func TestValidateEndpointAccess(t *testing.T) { opts.EndpointAccessPrivateNATSubnetID = test.endpointAccessPrivateNATSubnetID opts.EndpointAccessPrivateAdditionalAllowedSubscriptions = test.endpointAccessPrivateAdditionalAllowedSubscriptions - _, err := opts.Validate(context.Background(), nil) + _, err := opts.Validate(context.Background(), &core.CreateOptions{}) if test.expectError { if err == nil { t.Fatalf("expected error but got nil") @@ -335,87 +335,6 @@ func TestCreateCluster(t *testing.T) { } } -func TestValidateExternalDNSDomain(t *testing.T) { - t.Parallel() - tests := map[string]struct { - externalDNSDomain string - name string - baseDomain string - expectError bool - }{ - "When externalDNSDomain matches cluster domain, it should return an error": { - externalDNSDomain: "test-cluster.example.com", - name: "test-cluster", - baseDomain: "example.com", - expectError: true, - }, - "When externalDNSDomain is parent of cluster domain, it should return an error": { - externalDNSDomain: "example.com", - name: "test-cluster", - baseDomain: "example.com", - expectError: true, - }, - "When externalDNSDomain differs from cluster domain, it should return nil": { - externalDNSDomain: "external.different.com", - name: "test-cluster", - baseDomain: "example.com", - expectError: false, - }, - "When externalDNSDomain is empty, it should return nil": { - externalDNSDomain: "", - name: "test-cluster", - baseDomain: "example.com", - expectError: false, - }, - "When externalDNSDomain matches cluster domain case-insensitively, it should return an error": { - externalDNSDomain: "Test-Cluster.Example.COM", - name: "test-cluster", - baseDomain: "example.com", - expectError: true, - }, - "When cluster name is empty, it should return nil": { - externalDNSDomain: "test-cluster.example.com", - name: "", - baseDomain: "example.com", - expectError: false, - }, - "When base domain is empty, it should return nil": { - externalDNSDomain: "test-cluster.example.com", - name: "test-cluster", - baseDomain: "", - expectError: false, - }, - "When externalDNSDomain shares a suffix but not on dot boundary, it should return nil": { - externalDNSDomain: "ample.com", - name: "test-cluster", - baseDomain: "example.com", - expectError: false, - }, - "When externalDNSDomain has trailing dot, it should still detect shadowing": { - externalDNSDomain: "test-cluster.example.com.", - name: "test-cluster", - baseDomain: "example.com", - expectError: true, - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - t.Parallel() - g := NewGomegaWithT(t) - - err := validateExternalDNSDomain(test.externalDNSDomain, test.name, test.baseDomain) - if test.expectError { - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("conflicts with cluster domain")) - g.Expect(err.Error()).To(ContainSubstring("shadows *.apps DNS resolution")) - } else { - g.Expect(err).NotTo(HaveOccurred()) - } - }) - } -} - func TestValidateOAuthPublishingStrategy(t *testing.T) { tests := map[string]struct { oauthPublishingStrategy string @@ -456,7 +375,7 @@ func TestValidateOAuthPublishingStrategy(t *testing.T) { opts.ManagedIdentitiesFile = test.managedIdentitiesFile opts.DataPlaneIdentitiesFile = test.dataPlaneIdentitiesFile - _, err := opts.Validate(context.Background(), nil) + _, err := opts.Validate(context.Background(), &core.CreateOptions{}) if test.expectError { g.Expect(err).To(HaveOccurred()) g.Expect(err).To(MatchError(test.expectedErrorMsg)) diff --git a/cmd/cluster/core/create.go b/cmd/cluster/core/create.go index 4bed9956c4ce..0e1fb107a2f2 100644 --- a/cmd/cluster/core/create.go +++ b/cmd/cluster/core/create.go @@ -1235,7 +1235,7 @@ func validateMgmtClusterAndNodePoolCPUArchitectures(ctx context.Context, opts *R if !validMultiArchImage { mgmtClusterCPUArch, err := hyperutil.GetMgmtClusterCPUArch(kc) if err != nil { - return fmt.Errorf("failed to check mgmt cluster CPU arch: %v", err) + return fmt.Errorf("failed to check mgmt cluster CPU arch: %w", err) } if !strings.EqualFold(mgmtClusterCPUArch, opts.Arch) { @@ -1253,7 +1253,7 @@ func validateMgmtClusterAndNodePoolCPUArchitectures(ctx context.Context, opts *R func validateVersion(ctx context.Context, versionCLI string, client crclient.Client) error { _, operatorVersion, err := supportedversion.GetSupportedOCPVersions(ctx, "hypershift", client, nil) if err != nil { - return fmt.Errorf("failed to get supported OCP versions: %v", err) + return fmt.Errorf("failed to get supported OCP versions: %w", err) } if operatorVersion != versionCLI { return fmt.Errorf("version mismatch detected, CLI: %s, Operator: %s", versionCLI, operatorVersion) diff --git a/cmd/cluster/powervs/create.go b/cmd/cluster/powervs/create.go index a3e25ae07994..1c93184d3c04 100644 --- a/cmd/cluster/powervs/create.go +++ b/cmd/cluster/powervs/create.go @@ -243,7 +243,7 @@ var _ core.Platform = (*CreateOptions)(nil) func NewCreateCommand(opts *core.RawCreateOptions) *cobra.Command { cmd := &cobra.Command{ Use: "powervs", - Short: "Creates basic functional HostedCluster resources on PowerVS PowerVS", + Short: "Creates basic functional HostedCluster resources on PowerVS", SilenceUsage: true, } diff --git a/cmd/infra/aws/iam.go b/cmd/infra/aws/iam.go index 2100dfa0f4bb..73928e74a65c 100644 --- a/cmd/infra/aws/iam.go +++ b/cmd/infra/aws/iam.go @@ -891,7 +891,7 @@ func (o *CreateIAMOptions) CreateOIDCResources(ctx context.Context, iamClient aw // Create a single shared role with all policies sharedRoleARN, err := o.CreateSharedOIDCRole(ctx, iamClient, bindings, providerARN, providerName, logger) if err != nil { - return nil, fmt.Errorf("failed to create shared OIDC role: %v", err) + return nil, fmt.Errorf("failed to create shared OIDC role: %w", err) } // Set all role ARNs to the shared role ARN for into := range bindings { @@ -903,7 +903,7 @@ func (o *CreateIAMOptions) CreateOIDCResources(ctx context.Context, iamClient aw trustPolicy := oidcTrustPolicy(providerARN, providerName, binding.serviceAccounts...) arn, err := o.CreateOIDCRole(ctx, iamClient, binding, trustPolicy, logger) if err != nil { - return nil, fmt.Errorf("failed to create OIDC Role %q: with trust policy %s and permission policy %s: %v", binding.name, trustPolicy, binding.policy, err) + return nil, fmt.Errorf("failed to create OIDC Role %q: with trust policy %s and permission policy %s: %w", binding.name, trustPolicy, binding.policy, err) } *into = arn } @@ -966,7 +966,7 @@ func (o *CreateIAMOptions) CreateOIDCResources(ctx context.Context, iamClient aw ] }`, ingressPolicyStatement, ccmPolicyStatement)), }); err != nil { - return nil, fmt.Errorf("failed to create role policy %q: with permission policy %s: %v", ingressRoleName, ingressPolicyStatement, err) + return nil, fmt.Errorf("failed to create role policy %q: with permission policy %s: %w", ingressRoleName, ingressPolicyStatement, err) } logger.Info("Added inline shared policy to ROSA Managed Role", "role", ingressRoleName) } else { @@ -979,7 +979,7 @@ func (o *CreateIAMOptions) CreateOIDCResources(ctx context.Context, iamClient aw "Statement": [%s] }`, ingressPolicyStatement)), }); err != nil { - return nil, fmt.Errorf("failed to create role policy %q: with permission policy %s: %v", ingressRoleName, ingressPolicyStatement, err) + return nil, fmt.Errorf("failed to create role policy %q: with permission policy %s: %w", ingressRoleName, ingressPolicyStatement, err) } logger.Info("Added inline policy to ROSA Ingress Managed Role", "role", ingressRoleName) @@ -992,7 +992,7 @@ func (o *CreateIAMOptions) CreateOIDCResources(ctx context.Context, iamClient aw "Statement": [%s] }`, ccmPolicyStatement)), }); err != nil { - return nil, fmt.Errorf("failed to create role policy %q: with permission policy %s: %v", ccmRoleName, ccmPolicyStatement, err) + return nil, fmt.Errorf("failed to create role policy %q: with permission policy %s: %w", ccmRoleName, ccmPolicyStatement, err) } logger.Info("Added inline policy to ROSA Cloud Controller Manager Managed Role", "role", ccmRoleName) } diff --git a/cmd/infra/aws/iam_policies.go b/cmd/infra/aws/iam_policies.go index d99594b3ca26..6f569fdae55f 100644 --- a/cmd/infra/aws/iam_policies.go +++ b/cmd/infra/aws/iam_policies.go @@ -36,7 +36,7 @@ func APIsByDelegatedServices() (ServicesByDelegate, error) { for _, binding := range bindings { p := policy{} if err := json.Unmarshal([]byte(binding.policy), &p); err != nil { - return nil, fmt.Errorf("error unmarshalling delegate policy for %q: %v", binding.name, err) + return nil, fmt.Errorf("error unmarshalling delegate policy for %q: %w", binding.name, err) } delegate := EndpointsByService{} for i, statement := range p.Statement { diff --git a/cmd/infra/aws/route53.go b/cmd/infra/aws/route53.go index bb6034aadaa8..048055579ca5 100644 --- a/cmd/infra/aws/route53.go +++ b/cmd/infra/aws/route53.go @@ -188,7 +188,10 @@ func (o *DestroyInfraOptions) CleanupPublicZone(ctx context.Context, client awsa name := o.BaseDomain id, err := LookupZone(ctx, client, name, false) if err != nil { - return nil + if strings.Contains(err.Error(), "not found") { + return nil + } + return fmt.Errorf("failed to lookup public hosted zone %s: %w", name, err) } recordName := fmt.Sprintf("*.apps.%s.%s", o.Name, o.BaseDomain) err = deleteRecord(ctx, client, id, recordName) @@ -239,12 +242,12 @@ func setSOAMinimum(ctx context.Context, client awsapi.ROUTE53API, id, name strin func deleteZone(ctx context.Context, id string, client awsapi.ROUTE53API, logger logr.Logger) error { err := deleteRecords(ctx, client, id, logger) if err != nil { - return fmt.Errorf("failed to delete hosted zone records: %v", err) + return fmt.Errorf("failed to delete hosted zone records: %w", err) } if _, err = client.DeleteHostedZone(ctx, &route53.DeleteHostedZoneInput{ Id: aws.String(id), }); err != nil { - return fmt.Errorf("failed to delete hosted zone: %v", err) + return fmt.Errorf("failed to delete hosted zone: %w", err) } return nil } diff --git a/cmd/infra/aws/route53_test.go b/cmd/infra/aws/route53_test.go index 7c65d850d6e2..01c1913a7261 100644 --- a/cmd/infra/aws/route53_test.go +++ b/cmd/infra/aws/route53_test.go @@ -317,10 +317,14 @@ func TestCreatePrivateZone(t *testing.T) { func TestCleanupPublicZone(t *testing.T) { tests := []struct { - name string - setupMock func(*awsapi.MockROUTE53API) - expectError bool - errorContains string + name string + redact bool + setupMock func(*awsapi.MockROUTE53API) + expectError bool + errorContains string + useCtx func() context.Context + wantLogRedacted bool + wantLogSubstring string }{ { name: "When zone and wildcard record exist it should delete the record and return nil", @@ -340,6 +344,27 @@ func TestCleanupPublicZone(t *testing.T) { Return(&route53.ChangeResourceRecordSetsOutput{}, nil) }, }, + { + name: "When zone and wildcard record exist with redact enabled it should delete the record and redact the domain in logs", + redact: true, + setupMock: func(m *awsapi.MockROUTE53API) { + m.EXPECT().ListHostedZones(gomock.Any(), gomock.Any(), gomock.Any()). + Return(publicZonePage("PUBZONE", testBaseDomain), nil) + m.EXPECT().ListResourceRecordSets(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&route53.ListResourceRecordSetsOutput{ + ResourceRecordSets: []route53types.ResourceRecordSet{ + { + Name: aws.String("*.apps." + testCluster + "." + testBaseDomain + "."), + Type: route53types.RRTypeA, + }, + }, + }, nil) + m.EXPECT().ChangeResourceRecordSets(gomock.Any(), gomock.Any(), gomock.Any()). + Return(&route53.ChangeResourceRecordSetsOutput{}, nil) + }, + wantLogRedacted: true, + wantLogSubstring: "[REDACTED]", + }, { name: "When the zone is not found it should return nil as a no-op", setupMock: func(m *awsapi.MockROUTE53API) { @@ -358,6 +383,16 @@ func TestCleanupPublicZone(t *testing.T) { }, nil) }, }, + { + name: "When LookupZone fails with a non-not-found error it should return a wrapped error", + useCtx: cancelledCtx, + setupMock: func(m *awsapi.MockROUTE53API) { + m.EXPECT().ListHostedZones(gomock.Any(), gomock.Any(), gomock.Any()). + Return(nil, errors.New("throttling exception")) + }, + expectError: true, + errorContains: "failed to lookup public hosted zone", + }, { name: "When ChangeResourceRecordSets fails with a non-404 error it should return the error", setupMock: func(m *awsapi.MockROUTE53API) { @@ -382,28 +417,44 @@ func TestCleanupPublicZone(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) ctrl := gomock.NewController(t) mockR53 := awsapi.NewMockROUTE53API(ctrl) tt.setupMock(mockR53) + ctx := t.Context() + if tt.useCtx != nil { + ctx = tt.useCtx() + } + + var logOutput strings.Builder + logger := logr.Discard() + if tt.wantLogRedacted { + logger = funcr.New(func(prefix, args string) { + logOutput.WriteString(args) + }, funcr.Options{}) + } + o := &DestroyInfraOptions{ - BaseDomain: testBaseDomain, - Name: testCluster, - Log: logr.Discard(), + BaseDomain: testBaseDomain, + Name: testCluster, + RedactBaseDomain: tt.redact, + Log: logger, } - err := o.CleanupPublicZone(context.Background(), mockR53) + err := o.CleanupPublicZone(ctx, mockR53) if tt.expectError { - if err == nil { - t.Fatal("expected error, got nil") - } - if tt.errorContains != "" && !strings.Contains(err.Error(), tt.errorContains) { - t.Errorf("expected error containing %q, got: %v", tt.errorContains, err) + g.Expect(err).To(HaveOccurred()) + if tt.errorContains != "" { + g.Expect(err).To(MatchError(ContainSubstring(tt.errorContains))) } } else { - if err != nil { - t.Errorf("expected no error, got: %v", err) - } + g.Expect(err).ToNot(HaveOccurred()) + } + + if tt.wantLogRedacted { + g.Expect(logOutput.String()).To(ContainSubstring(tt.wantLogSubstring)) + g.Expect(logOutput.String()).ToNot(ContainSubstring(testBaseDomain)) } }) } diff --git a/cmd/infra/aws/util/errors.go b/cmd/infra/aws/util/errors.go index 074cd062e742..e25c8926e41b 100644 --- a/cmd/infra/aws/util/errors.go +++ b/cmd/infra/aws/util/errors.go @@ -9,7 +9,8 @@ import ( ) func IsErrorRetryable(err error) bool { - if aggregate, isAggregate := err.(utilerrors.Aggregate); isAggregate { + var aggregate utilerrors.Aggregate + if errors.As(err, &aggregate) { if len(aggregate.Errors()) == 1 { err = aggregate.Errors()[0] } else { diff --git a/cmd/infra/aws/util/errors_test.go b/cmd/infra/aws/util/errors_test.go new file mode 100644 index 000000000000..2e2ed1fb5364 --- /dev/null +++ b/cmd/infra/aws/util/errors_test.go @@ -0,0 +1,76 @@ +package util + +import ( + "errors" + "fmt" + "testing" + + . "github.com/onsi/gomega" + + "github.com/aws/aws-sdk-go-v2/config" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" +) + +func TestIsErrorRetryable(t *testing.T) { + t.Parallel() + tests := []struct { + name string + err error + expected bool + }{ + { + name: "When error is a generic error it should be retryable", + err: errors.New("some transient error"), + expected: true, + }, + { + name: "When error is a credential load error it should not be retryable", + err: config.SharedConfigLoadError{}, + expected: false, + }, + { + name: "When error is a wrapped credential load error it should not be retryable", + err: fmt.Errorf("loading config: %w", config.SharedConfigLoadError{}), + expected: false, + }, + { + name: "When aggregate has single generic error it should be retryable", + err: utilerrors.NewAggregate([]error{errors.New("transient")}), + expected: true, + }, + { + name: "When aggregate has single credential load error it should not be retryable", + err: utilerrors.NewAggregate([]error{config.SharedConfigLoadError{}}), + expected: false, + }, + { + name: "When aggregate has only credential load errors it should not be retryable", + err: utilerrors.NewAggregate([]error{ + config.SharedConfigLoadError{}, + config.SharedConfigLoadError{}, + }), + expected: false, + }, + { + name: "When aggregate has mixed errors it should be retryable", + err: utilerrors.NewAggregate([]error{ + config.SharedConfigLoadError{}, + errors.New("some other error"), + }), + expected: true, + }, + { + name: "When wrapped aggregate has single generic error it should be retryable", + err: fmt.Errorf("operation failed: %w", utilerrors.NewAggregate([]error{errors.New("transient")})), + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + g.Expect(IsErrorRetryable(tt.err)).To(Equal(tt.expected)) + }) + } +} diff --git a/cmd/infra/azure/create_iam.go b/cmd/infra/azure/create_iam.go index 81b8ae2342c9..4864236006a1 100644 --- a/cmd/infra/azure/create_iam.go +++ b/cmd/infra/azure/create_iam.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "os" "github.com/openshift/hypershift/cmd/log" @@ -198,7 +199,7 @@ func (o *CreateIAMOptions) ensureResourceGroup(ctx context.Context, l logr.Logge } var respErr *azcore.ResponseError - if !errors.As(err, &respErr) || respErr.StatusCode != 404 { + if !errors.As(err, &respErr) || respErr.StatusCode != http.StatusNotFound { return fmt.Errorf("failed to check resource group %q: %w", o.ResourceGroupName, err) } diff --git a/cmd/infra/azure/rbac.go b/cmd/infra/azure/rbac.go index 2f72d35d412a..f330c4dbdc22 100644 --- a/cmd/infra/azure/rbac.go +++ b/cmd/infra/azure/rbac.go @@ -117,7 +117,7 @@ func (r *RBACManager) assignRolesForComponents(ctx context.Context, opts *Create } for component, clientID := range components { - objectID, err := r.getObjectIDFromClientID(string(clientID), token) + objectID, err := r.getObjectIDFromClientID(ctx, string(clientID), token) if err != nil { return err } @@ -150,7 +150,7 @@ func (r *RBACManager) AssignDataPlaneRoles(ctx context.Context, opts *CreateInfr } // Setup Data Plane MI role assignments - objectID, err := r.getObjectIDFromClientID(dataPlaneIdentities.ImageRegistryMSIClientID, token) + objectID, err := r.getObjectIDFromClientID(ctx, dataPlaneIdentities.ImageRegistryMSIClientID, token) if err != nil { return err } @@ -159,7 +159,7 @@ func (r *RBACManager) AssignDataPlaneRoles(ctx context.Context, opts *CreateInfr return err } - objectID, err = r.getObjectIDFromClientID(dataPlaneIdentities.DiskMSIClientID, token) + objectID, err = r.getObjectIDFromClientID(ctx, dataPlaneIdentities.DiskMSIClientID, token) if err != nil { return err } @@ -168,7 +168,7 @@ func (r *RBACManager) AssignDataPlaneRoles(ctx context.Context, opts *CreateInfr return err } - objectID, err = r.getObjectIDFromClientID(dataPlaneIdentities.FileMSIClientID, token) + objectID, err = r.getObjectIDFromClientID(ctx, dataPlaneIdentities.FileMSIClientID, token) if err != nil { return err } @@ -366,7 +366,7 @@ func (r *RBACManager) getAzureToken() (azcore.AccessToken, error) { return token, nil } -func (r *RBACManager) getObjectIDFromClientID(clientID string, token azcore.AccessToken) (string, error) { +func (r *RBACManager) getObjectIDFromClientID(ctx context.Context, clientID string, token azcore.AccessToken) (string, error) { // Validate clientID is a UUID to prevent OData injection uuidPattern := regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) if !uuidPattern.MatchString(clientID) { @@ -377,7 +377,7 @@ func (r *RBACManager) getObjectIDFromClientID(clientID string, token azcore.Acce url := graphAPIEndpoint + "?" + strings.ReplaceAll(filterQuery, " ", "%20") // Make the API request - req, err := http.NewRequest("GET", url, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return "", fmt.Errorf("failed to create request: %w", err) } diff --git a/cmd/infra/powervs/create.go b/cmd/infra/powervs/create.go index 68a915e252a7..b787bdd01efd 100644 --- a/cmd/infra/powervs/create.go +++ b/cmd/infra/powervs/create.go @@ -963,7 +963,7 @@ func (infra *Infra) createVpc(ctx context.Context, logger logr.Logger, options * }) if err != nil { - return nil, fmt.Errorf("error attaching inbound security group rule to allow %d to vpc %v", port, err) + return nil, fmt.Errorf("error attaching inbound security group rule to allow %d to vpc %w", port, err) } } diff --git a/cmd/infra/powervs/destroy.go b/cmd/infra/powervs/destroy.go index 1c15c2c5895d..7ba387596f68 100644 --- a/cmd/infra/powervs/destroy.go +++ b/cmd/infra/powervs/destroy.go @@ -3,6 +3,7 @@ package powervs import ( "context" "encoding/json" + coreerrors "errors" "fmt" "net" "net/http" @@ -768,7 +769,8 @@ func deleteCOSBucket(ctx context.Context, bucketName string, cosClient *s3.S3) e if _, err := cosClient.DeleteBucketWithContext(ctx, &s3.DeleteBucketInput{ Bucket: aws.String(bucketName), }); err != nil { - if aerr, ok := err.(awserr.Error); ok { + var aerr awserr.Error + if coreerrors.As(err, &aerr) { if aerr.Code() != s3.ErrCodeNoSuchBucket { return err } @@ -886,7 +888,7 @@ func destroyTransitGateway(ctx context.Context, logger logr.Logger, options *Des TransitGatewayID: tg.ID, }) if err != nil { - return fmt.Errorf("error retrieving transit gateway connection list: %v", err) + return fmt.Errorf("error retrieving transit gateway connection list: %w", err) } for _, tgConn := range tgConnList.Connections { @@ -895,7 +897,7 @@ func destroyTransitGateway(ctx context.Context, logger logr.Logger, options *Des TransitGatewayID: tg.ID, ID: tgConn.ID, }); err != nil { - return fmt.Errorf("error deleting transit gateway connection %s, %v", *tgConn.Name, err) + return fmt.Errorf("error deleting transit gateway connection %s, %w", *tgConn.Name, err) } } @@ -904,7 +906,7 @@ func destroyTransitGateway(ctx context.Context, logger logr.Logger, options *Des TransitGatewayID: tg.ID, }) if err != nil { - return false, fmt.Errorf("error retrieving transit gateway connection list: %v", err) + return false, fmt.Errorf("error retrieving transit gateway connection list: %w", err) } if len(tgConnList.Connections) > 0 { return false, nil @@ -915,7 +917,7 @@ func destroyTransitGateway(ctx context.Context, logger logr.Logger, options *Des logger.Info("Waiting for transit gateway connections to get deleted") if err = wait.PollUntilContextTimeout(ctx, time.Minute*1, time.Minute*10, true, f); err != nil { - return fmt.Errorf("error waiting for tranist gateway connections to get deleted: %v", err) + return fmt.Errorf("error waiting for tranist gateway connections to get deleted: %w", err) } logger.Info("Deleting transit gateway", "name", *tg.Name) diff --git a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.azure.testsuite.yaml b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.azure.testsuite.yaml index 0b35dae88ed4..ed9896396e96 100644 --- a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.azure.testsuite.yaml +++ b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.azure.testsuite.yaml @@ -813,271 +813,3 @@ tests: type: Route route: {} expectedError: "workloadIdentities.controlPlaneOperator is required when Private Link is configured with WorkloadIdentities authentication" - - # --- Azure DNS shadowing validation --- - - name: When Azure route hostname domain overlaps with baseDomain it should fail - initial: | - apiVersion: hypershift.openshift.io/v1beta1 - kind: HostedCluster - spec: - dns: - baseDomain: example.com - platform: - type: Azure - azure: - location: eastus - resourceGroupName: test-rg - vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" - subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" - subscriptionID: "12345678-1234-5678-9012-123456789012" - securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" - tenantID: "87654321-4321-8765-2109-876543210987" - azureAuthenticationConfig: - azureAuthenticationConfigType: ManagedIdentities - managedIdentities: - controlPlane: - managedIdentitiesKeyVault: - name: test-kv - tenantID: "87654321-4321-8765-2109-876543210987" - cloudProvider: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: cp-secret - nodePoolManagement: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: npm-secret - controlPlaneOperator: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: cpo-secret - imageRegistry: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: ir-secret - ingress: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: ingress-secret - network: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: network-secret - disk: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: disk-secret - file: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: file-secret - dataPlane: - imageRegistryMSIClientID: "12345678-1234-5678-9012-123456789012" - diskMSIClientID: "12345678-1234-5678-9012-123456789012" - fileMSIClientID: "12345678-1234-5678-9012-123456789012" - pullSecret: - name: secret - release: - image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 - secretEncryption: - aescbc: - activeKey: - name: key - type: aescbc - services: - - service: APIServer - servicePublishingStrategy: - type: Route - route: - hostname: api-mycluster.example.com - - service: OAuthServer - servicePublishingStrategy: - type: Route - route: {} - - service: Konnectivity - servicePublishingStrategy: - type: Route - route: {} - - service: Ignition - servicePublishingStrategy: - type: Route - route: {} - expectedError: "Azure service hostname domain must not overlap with the cluster base domain" - - - name: When Azure loadBalancer hostname domain overlaps with baseDomain it should fail - initial: | - apiVersion: hypershift.openshift.io/v1beta1 - kind: HostedCluster - spec: - dns: - baseDomain: example.com - platform: - type: Azure - azure: - location: eastus - resourceGroupName: test-rg - vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" - subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" - subscriptionID: "12345678-1234-5678-9012-123456789012" - securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" - tenantID: "87654321-4321-8765-2109-876543210987" - azureAuthenticationConfig: - azureAuthenticationConfigType: ManagedIdentities - managedIdentities: - controlPlane: - managedIdentitiesKeyVault: - name: test-kv - tenantID: "87654321-4321-8765-2109-876543210987" - cloudProvider: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: cp-secret - nodePoolManagement: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: npm-secret - controlPlaneOperator: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: cpo-secret - imageRegistry: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: ir-secret - ingress: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: ingress-secret - network: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: network-secret - disk: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: disk-secret - file: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: file-secret - dataPlane: - imageRegistryMSIClientID: "12345678-1234-5678-9012-123456789012" - diskMSIClientID: "12345678-1234-5678-9012-123456789012" - fileMSIClientID: "12345678-1234-5678-9012-123456789012" - pullSecret: - name: secret - release: - image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 - secretEncryption: - aescbc: - activeKey: - name: key - type: aescbc - services: - - service: APIServer - servicePublishingStrategy: - type: LoadBalancer - loadBalancer: - hostname: api-mycluster.example.com - - service: OAuthServer - servicePublishingStrategy: - type: Route - route: {} - - service: Konnectivity - servicePublishingStrategy: - type: Route - route: {} - - service: Ignition - servicePublishingStrategy: - type: Route - route: {} - expectedError: "Azure service hostname domain must not overlap with the cluster base domain" - - - name: When Azure route hostname domain differs from baseDomain it should pass - initial: | - apiVersion: hypershift.openshift.io/v1beta1 - kind: HostedCluster - spec: - dns: - baseDomain: example.com - platform: - type: Azure - azure: - location: eastus - resourceGroupName: test-rg - vnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet" - subnetID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/virtualNetworks/test-vnet/subnets/test-subnet" - subscriptionID: "12345678-1234-5678-9012-123456789012" - securityGroupID: "/subscriptions/12345678-1234-5678-9012-123456789012/resourceGroups/test-rg/providers/Microsoft.Network/networkSecurityGroups/test-nsg" - tenantID: "87654321-4321-8765-2109-876543210987" - azureAuthenticationConfig: - azureAuthenticationConfigType: ManagedIdentities - managedIdentities: - controlPlane: - managedIdentitiesKeyVault: - name: test-kv - tenantID: "87654321-4321-8765-2109-876543210987" - cloudProvider: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: cp-secret - nodePoolManagement: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: npm-secret - controlPlaneOperator: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: cpo-secret - imageRegistry: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: ir-secret - ingress: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: ingress-secret - network: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: network-secret - disk: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: disk-secret - file: - clientID: "12345678-1234-5678-9012-123456789012" - objectEncoding: utf-8 - credentialsSecretName: file-secret - dataPlane: - imageRegistryMSIClientID: "12345678-1234-5678-9012-123456789012" - diskMSIClientID: "12345678-1234-5678-9012-123456789012" - fileMSIClientID: "12345678-1234-5678-9012-123456789012" - pullSecret: - name: secret - release: - image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 - secretEncryption: - aescbc: - activeKey: - name: key - type: aescbc - services: - - service: APIServer - servicePublishingStrategy: - type: Route - route: - hostname: api-mycluster.external.different.com - - service: OAuthServer - servicePublishingStrategy: - type: Route - route: {} - - service: Konnectivity - servicePublishingStrategy: - type: Route - route: {} - - service: Ignition - servicePublishingStrategy: - type: Route - route: {} - diff --git a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.capabilities.testsuite.yaml b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.capabilities.testsuite.yaml index e6b2466713d3..85e45294654c 100644 --- a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.capabilities.testsuite.yaml +++ b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.capabilities.testsuite.yaml @@ -535,3 +535,232 @@ tests: servicePublishingStrategy: type: Route route: {} + + onUpdate: + - name: When capabilities enabled list is changed it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + capabilities: + enabled: + - baremetal + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + capabilities: + enabled: + - Insights + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "Enabled is immutable" + - name: When capabilities disabled list is changed it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + capabilities: + disabled: + - ImageRegistry + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + capabilities: + disabled: + - Insights + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "Capabilities is immutable" + - name: When capabilities is unchanged but release image is updated it should pass + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + capabilities: + disabled: + - ImageRegistry + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + capabilities: + disabled: + - ImageRegistry + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.16.0-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} diff --git a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.networking.testsuite.yaml b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.networking.testsuite.yaml index 66b1e94a8130..8c0073612e1f 100644 --- a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.networking.testsuite.yaml +++ b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.networking.testsuite.yaml @@ -619,3 +619,439 @@ tests: servicePublishingStrategy: type: Route route: {} + + - name: When ovnKubernetesConfig ipv6 is set and networkType is OVNKubernetes it should pass + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + networking: + networkType: OVNKubernetes + operatorConfiguration: + clusterNetworkOperator: + ovnKubernetesConfig: + ipv6: + internalJoinSubnet: "fd99::/64" + internalTransitSwitchSubnet: "fd97:1::/64" + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + + - name: When ovnKubernetesConfig has both ipv4 and ipv6 it should pass + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + networking: + networkType: OVNKubernetes + operatorConfiguration: + clusterNetworkOperator: + ovnKubernetesConfig: + ipv4: + internalJoinSubnet: "10.10.0.0/16" + ipv6: + internalJoinSubnet: "fd99::/64" + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + + - name: When ovnKubernetesConfig ipv6 has same internalJoinSubnet and internalTransitSwitchSubnet it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + networking: + networkType: OVNKubernetes + operatorConfiguration: + clusterNetworkOperator: + ovnKubernetesConfig: + ipv6: + internalJoinSubnet: "fd99::/64" + internalTransitSwitchSubnet: "fd99::/64" + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "ipv6 internalJoinSubnet and internalTransitSwitchSubnet must not be the same" + + - name: When ovnKubernetesConfig ipv6 internalJoinSubnet has invalid CIDR it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + networking: + networkType: OVNKubernetes + operatorConfiguration: + clusterNetworkOperator: + ovnKubernetesConfig: + ipv6: + internalJoinSubnet: "not-a-cidr" + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "Subnet must be in valid IPv6 CIDR format" + + - name: When ovnKubernetesConfig ipv6 internalTransitSwitchSubnet has invalid CIDR it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + networking: + networkType: OVNKubernetes + operatorConfiguration: + clusterNetworkOperator: + ovnKubernetesConfig: + ipv6: + internalTransitSwitchSubnet: "not-a-cidr" + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "Subnet must be in valid IPv6 CIDR format" + + - name: When ovnKubernetesConfig ipv6 internalJoinSubnet has prefix length greater than 125 it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + networking: + networkType: OVNKubernetes + operatorConfiguration: + clusterNetworkOperator: + ovnKubernetesConfig: + ipv6: + internalJoinSubnet: "fd99::/126" + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "subnet must be in the range /0 to /125 inclusive" + + - name: When ovnKubernetesConfig ipv6 internalTransitSwitchSubnet has prefix length greater than 125 it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + networking: + networkType: OVNKubernetes + operatorConfiguration: + clusterNetworkOperator: + ovnKubernetesConfig: + ipv6: + internalTransitSwitchSubnet: "fd97::/126" + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "subnet must be in the range /0 to /125 inclusive" + + - name: When ovnKubernetesConfig ipv6 is empty object it should fail MinProperties validation + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + networking: + networkType: OVNKubernetes + operatorConfiguration: + clusterNetworkOperator: + ovnKubernetesConfig: + ipv6: {} + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "should have at least 1 properties" + + onUpdate: + - name: When ovnKubernetesConfig is removed after being set it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + networking: + networkType: OVNKubernetes + operatorConfiguration: + clusterNetworkOperator: + ovnKubernetesConfig: + ipv4: + internalJoinSubnet: "10.10.0.0/16" + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + networking: + networkType: OVNKubernetes + operatorConfiguration: + clusterNetworkOperator: {} + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "ovnKubernetesConfig is immutable once set and cannot be removed" diff --git a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.services.testsuite.yaml b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.services.testsuite.yaml index 3e03aae164cc..5566d83ca392 100644 --- a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.services.testsuite.yaml +++ b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.services.testsuite.yaml @@ -378,3 +378,145 @@ tests: namedCertificates: - servingCertificate: name: my-cert + + onUpdate: + - name: When services publishing strategy type is changed it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: NodePort + nodePort: + address: "127.0.0.1" + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "Services is immutable" + - name: When services is unchanged but release image is updated it should pass + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.16.0-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} diff --git a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.validation.testsuite.yaml b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.validation.testsuite.yaml index 4c5a26f035c6..3d777e151c13 100644 --- a/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.validation.testsuite.yaml +++ b/cmd/install/assets/crds/hypershift-operator/tests/hostedclusters.hypershift.openshift.io/stable.hostedclusters.validation.testsuite.yaml @@ -703,3 +703,148 @@ tests: servicePublishingStrategy: type: Route route: {} + + onUpdate: + - name: When controllerAvailabilityPolicy is changed it should fail + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + controllerAvailabilityPolicy: HighlyAvailable + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + controllerAvailabilityPolicy: SingleReplica + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + expectedError: "ControllerAvailabilityPolicy is immutable" + - name: When controllerAvailabilityPolicy is unchanged but release image is updated it should pass + initial: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + controllerAvailabilityPolicy: HighlyAvailable + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.15.11-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} + updated: | + apiVersion: hypershift.openshift.io/v1beta1 + kind: HostedCluster + spec: + controllerAvailabilityPolicy: HighlyAvailable + dns: + baseDomain: example.com + platform: + type: AWS + pullSecret: + name: secret + release: + image: quay.io/openshift-release-dev/ocp-release:4.16.0-x86_64 + secretEncryption: + aescbc: + activeKey: + name: key + type: aescbc + services: + - service: APIServer + servicePublishingStrategy: + type: Route + route: {} + - service: OAuthServer + servicePublishingStrategy: + type: Route + route: {} + - service: Konnectivity + servicePublishingStrategy: + type: Route + route: {} + - service: Ignition + servicePublishingStrategy: + type: Route + route: {} diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml index d9d85e84b651..31fe863e109b 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-CustomNoUpgrade.crd.yaml @@ -4015,6 +4015,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -4042,9 +4111,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' clusterVersionOperator: description: clusterVersionOperator specifies the configuration for the Cluster Version Operator in the hosted cluster. @@ -8354,17 +8442,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-Default.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-Default.crd.yaml index 617e99b9a777..45bd439af93d 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-Default.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-Default.crd.yaml @@ -3685,6 +3685,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3712,9 +3781,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. @@ -7025,17 +7113,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml index 59441394383d..85a9425723dc 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedclusters-Hypershift-TechPreviewNoUpgrade.crd.yaml @@ -3886,6 +3886,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3913,9 +3982,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' clusterVersionOperator: description: clusterVersionOperator specifies the configuration for the Cluster Version Operator in the hosted cluster. @@ -8225,17 +8313,6 @@ spec: - message: ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes rule: self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig) - - message: Azure service hostname domain must not overlap with the cluster - base domain. An Azure Private DNS zone matching or containing the - base domain would shadow *.apps DNS resolution. - rule: self.platform.type != "Azure" || self.dns.baseDomain == "" || - !self.services.exists(s, (has(s.servicePublishingStrategy.route) && - has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') - + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) - && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') - && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') - + 1)))) status: description: status is the latest observed status of the HostedCluster. properties: diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml index 861af7174b2b..e267097b2bc2 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-CustomNoUpgrade.crd.yaml @@ -3903,6 +3903,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3930,9 +3999,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' clusterVersionOperator: description: clusterVersionOperator specifies the configuration for the Cluster Version Operator in the hosted cluster. diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-Default.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-Default.crd.yaml index 14b2906319bb..3dbb3485271c 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-Default.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-Default.crd.yaml @@ -3573,6 +3573,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3600,9 +3669,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' ingressOperator: description: |- ingressOperator specifies the configuration for the Ingress Operator in the hosted cluster. diff --git a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml index 435b260f05f5..8b61ee5d3862 100644 --- a/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml +++ b/cmd/install/assets/crds/hypershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-Hypershift-TechPreviewNoUpgrade.crd.yaml @@ -3774,6 +3774,75 @@ spec: rule: self.matches('^[0-9]{1,3}\\..*') && int(self.split('/')[0].split('.')[0]) > 0 type: object + ipv6: + description: |- + ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + this means no opinions and the default configuration is used. Check individual + fields within ipv6 for details of default values. + For KubeVirt hosted clusters using dual-stack networking, it is recommended to + set ipv6.internalJoinSubnet to a value different from the management cluster's + join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + minProperties: 1 + properties: + internalJoinSubnet: + description: |- + internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + default one is being already used by something else. It must not overlap with + any other subnet being used by OpenShift or by the node network. The size of the + subnet must be larger than the number of nodes. + The current default value is fd98::/64. + For KubeVirt hosted clusters, if this field is not set, HyperShift will + automatically use fd99::/64 to avoid collisions with the management cluster's + default join subnet (fd98::/64). + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd98::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalJoinSubnet is immutable + rule: self == oldSelf + internalTransitSwitchSubnet: + description: |- + internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + architecture that connects the cluster routers on each node together to enable + east west traffic. The subnet chosen should not overlap with other networks + specified for OVN-Kubernetes as well as other networks used on the host. + When omitted, this means no opinion and the platform is left to choose a reasonable + default which is subject to change over time. + The current default subnet is fd97::/64. + The subnet must be large enough to accommodate one IP per node in your cluster. + The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + The prefix length must be in the range /0 to /125 inclusive. + This field is immutable once set. + maxLength: 48 + minLength: 3 + type: string + x-kubernetes-validations: + - message: Subnet must be in valid IPv6 CIDR format + (e.g., fd97::/64) + rule: isCIDR(self) && cidr(self).ip().family() == + 6 + - message: subnet must be in the range /0 to /125 + inclusive + rule: isCIDR(self) && cidr(self).prefixLength() + <= 125 + - message: internalTransitSwitchSubnet is immutable + rule: self == oldSelf + type: object mtu: description: |- mtu is the MTU to use for the tunnel interface on hosted cluster nodes. @@ -3801,9 +3870,28 @@ spec: rule: '!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet' + - message: ipv6 internalJoinSubnet and internalTransitSwitchSubnet + must not be the same + rule: '!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) + || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet + != self.ipv6.internalTransitSwitchSubnet' - message: mtu is immutable once set and cannot be removed rule: '!has(oldSelf.mtu) || has(self.mtu)' + - message: ipv6 is immutable once set and cannot be removed + rule: '!has(oldSelf.ipv6) || has(self.ipv6)' + - message: ipv6.internalJoinSubnet cannot be removed once + set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) + || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))' + - message: ipv6.internalTransitSwitchSubnet cannot be removed + once set + rule: '!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) + || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))' type: object + x-kubernetes-validations: + - message: ovnKubernetesConfig is immutable once set and cannot + be removed + rule: '!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)' clusterVersionOperator: description: clusterVersionOperator specifies the configuration for the Cluster Version Operator in the hosted cluster. diff --git a/cmd/install/assets/hypershift_operator.go b/cmd/install/assets/hypershift_operator.go index d2af98020306..3163fdcbaed1 100644 --- a/cmd/install/assets/hypershift_operator.go +++ b/cmd/install/assets/hypershift_operator.go @@ -858,6 +858,7 @@ func (o HyperShiftOperatorDeployment) addWebhookResources(args *[]string, volume VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: "manager-serving-cert", + Optional: ptr.To(true), }, }, }) @@ -1163,10 +1164,11 @@ func (o ExternalDNSPodMonitor) Build() *prometheusoperatorv1.PodMonitor { "name": ExternalDNSDeploymentName, }, }, - PodMetricsEndpoints: []prometheusoperatorv1.PodMetricsEndpoint{{ - Port: ptr.To("metrics"), - Interval: "30s", - }, + PodMetricsEndpoints: []prometheusoperatorv1.PodMetricsEndpoint{ + { + Port: ptr.To("metrics"), + Interval: "30s", + }, }, }, } @@ -2050,7 +2052,6 @@ func (o HyperShiftReaderClusterRoleBinding) Build() *rbacv1.ClusterRoleBinding { type HyperShiftMutatingWebhookConfiguration struct { Namespace *corev1.Namespace EnableAuditLogPersistence bool - CABundle []byte } const ( @@ -2089,7 +2090,7 @@ func (o HyperShiftMutatingWebhookConfiguration) Build() *admissionregistrationv1 }, }, ClientConfig: admissionregistrationv1.WebhookClientConfig{ - CABundle: o.CABundle, + CABundle: nil, Service: &admissionregistrationv1.ServiceReference{ Namespace: "hypershift", Name: "operator", @@ -2116,7 +2117,7 @@ func (o HyperShiftMutatingWebhookConfiguration) Build() *admissionregistrationv1 }, }, ClientConfig: admissionregistrationv1.WebhookClientConfig{ - CABundle: o.CABundle, + CABundle: nil, Service: &admissionregistrationv1.ServiceReference{ Namespace: "hypershift", Name: "operator", @@ -2157,7 +2158,7 @@ func (o HyperShiftMutatingWebhookConfiguration) Build() *admissionregistrationv1 }, }, ClientConfig: admissionregistrationv1.WebhookClientConfig{ - CABundle: o.CABundle, + CABundle: nil, Service: &admissionregistrationv1.ServiceReference{ Namespace: "hypershift", Name: "operator", @@ -2187,7 +2188,7 @@ func (o HyperShiftMutatingWebhookConfiguration) Build() *admissionregistrationv1 }, }, ClientConfig: admissionregistrationv1.WebhookClientConfig{ - CABundle: o.CABundle, + CABundle: nil, Service: &admissionregistrationv1.ServiceReference{ Namespace: "hypershift", Name: "operator", @@ -2208,7 +2209,6 @@ func (o HyperShiftMutatingWebhookConfiguration) Build() *admissionregistrationv1 type HyperShiftValidatingWebhookConfiguration struct { Namespace string - CABundle []byte } func (o HyperShiftValidatingWebhookConfiguration) Build() *admissionregistrationv1.ValidatingWebhookConfiguration { @@ -2245,7 +2245,7 @@ func (o HyperShiftValidatingWebhookConfiguration) Build() *admissionregistration }, }, ClientConfig: admissionregistrationv1.WebhookClientConfig{ - CABundle: o.CABundle, + CABundle: nil, Service: &admissionregistrationv1.ServiceReference{ Namespace: "hypershift", Name: "operator", @@ -2274,7 +2274,7 @@ func (o HyperShiftValidatingWebhookConfiguration) Build() *admissionregistration }, }, ClientConfig: admissionregistrationv1.WebhookClientConfig{ - CABundle: o.CABundle, + CABundle: nil, Service: &admissionregistrationv1.ServiceReference{ Namespace: "hypershift", Name: "operator", diff --git a/cmd/install/install.go b/cmd/install/install.go index 2afbf6609000..6d9589a2aefc 100644 --- a/cmd/install/install.go +++ b/cmd/install/install.go @@ -30,7 +30,6 @@ import ( crdassets "github.com/openshift/hypershift/cmd/install/assets/crds" "github.com/openshift/hypershift/cmd/util" "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress" - "github.com/openshift/hypershift/hypershift-operator/controllers/webhookcerts" hyperapi "github.com/openshift/hypershift/support/api" "github.com/openshift/hypershift/support/config" "github.com/openshift/hypershift/support/metrics" @@ -756,22 +755,10 @@ func hyperShiftOperatorManifests(ctx context.Context, client crclient.Client, op operatorServiceAccount, rbacObjs := setupRBAC(opts, operatorNamespace) objects = append(objects, rbacObjs...) - // Generate self-managed webhook CA and serving cert when any webhook is enabled. - var webhookCABundle []byte - if opts.EnableDefaultingWebhook || opts.EnableConversionWebhook || opts.EnableValidatingWebhook || opts.EnableAuditLogPersistence { - caSecret, servingSecret, caBundle, err := webhookcerts.GenerateInitialWebhookCerts(operatorNamespace.Name, assets.HypershiftOperatorName) - if err != nil { - return nil, nil, fmt.Errorf("failed to generate webhook certs: %w", err) - } - objects = append(objects, caSecret, servingSecret) - webhookCABundle = caBundle - } - if opts.EnableDefaultingWebhook || opts.EnableAuditLogPersistence { mutatingWebhookConfiguration := assets.HyperShiftMutatingWebhookConfiguration{ Namespace: operatorNamespace, EnableAuditLogPersistence: opts.EnableAuditLogPersistence, - CABundle: webhookCABundle, }.Build() objects = append(objects, mutatingWebhookConfiguration) } @@ -779,7 +766,6 @@ func hyperShiftOperatorManifests(ctx context.Context, client crclient.Client, op if opts.EnableValidatingWebhook { validatingWebhookConfiguration := assets.HyperShiftValidatingWebhookConfiguration{ Namespace: operatorNamespace.Name, - CABundle: webhookCABundle, }.Build() objects = append(objects, validatingWebhookConfiguration) } @@ -831,7 +817,7 @@ func hyperShiftOperatorManifests(ctx context.Context, client crclient.Client, op objects = append(objects, sharedIngressObjs...) } - crds, err = setupCRDs(ctx, client, opts, operatorNamespace, operatorService, webhookCABundle) + crds, err = setupCRDs(ctx, client, opts, operatorNamespace, operatorService) if err != nil { return nil, nil, err } @@ -874,7 +860,7 @@ var ipamCRDNames = set.New( // related to etcd are excluded from the list. If the option EnableConversionWebhook is set to true, the CRDs related // to hypershift.openshift.io group are annotated with the necessary annotations to enable the conversion webhook. // If a client is provided, IPAM CRDs that already exist in the cluster are skipped to avoid conflicts. -func setupCRDs(ctx context.Context, client crclient.Client, opts Options, operatorNamespace *corev1.Namespace, operatorService *corev1.Service, webhookCABundle []byte) ([]crclient.Object, error) { +func setupCRDs(ctx context.Context, client crclient.Client, opts Options, operatorNamespace *corev1.Namespace, operatorService *corev1.Service) ([]crclient.Object, error) { // Build a set of existing IPAM CRDs if a client is available existingIPAMCRDs := set.New[string]() if client != nil { @@ -967,7 +953,7 @@ func setupCRDs(ctx context.Context, client crclient.Client, opts Options, operat Port: ptr.To[int32](443), Path: ptr.To("/convert"), }, - CABundle: webhookCABundle, + CABundle: nil, }, ConversionReviewVersions: []string{"v1beta1", "v1alpha1"}, }, diff --git a/cmd/install/install_test.go b/cmd/install/install_test.go index 333755441043..61052faf3a49 100644 --- a/cmd/install/install_test.go +++ b/cmd/install/install_test.go @@ -5,6 +5,7 @@ import ( "context" "io" "io/fs" + "os" "path/filepath" "strings" "testing" @@ -412,7 +413,7 @@ func TestSetupCRDs(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { g := NewGomegaWithT(t) - crds, err := setupCRDs(t.Context(), nil, tc.inputOptions, &corev1.Namespace{}, nil, nil) + crds, err := setupCRDs(t.Context(), nil, tc.inputOptions, &corev1.Namespace{}, nil) g.Expect(err).ToNot(HaveOccurred()) nodePoolCRDS := make([]crclient.Object, 0) var machineDeploymentCRD crclient.Object @@ -497,6 +498,10 @@ func TestSetupCRDs(t *testing.T) { } func TestRenderHyperShiftOperator_RenderSensitive(t *testing.T) { + g := NewGomegaWithT(t) + pullSecretFile := filepath.Join(t.TempDir(), "pull-secret.json") + g.Expect(os.WriteFile(pullSecretFile, []byte(`{"auths":{}}`), 0o600)).To(Succeed()) + tests := []struct { name string renderSensitive bool @@ -524,10 +529,12 @@ func TestRenderHyperShiftOperator_RenderSensitive(t *testing.T) { EnableDefaultingWebhook: true, EnableValidatingWebhook: true, EnableConversionWebhook: true, + PullSecretFile: pullSecretFile, RenderSensitive: tc.renderSensitive, Format: RenderFormatYaml, OutputTypes: string(OutputAll), } + err := RenderHyperShiftOperator(t.Context(), &buf, opts) g.Expect(err).ToNot(HaveOccurred()) @@ -552,6 +559,39 @@ func TestRenderHyperShiftOperator_RenderSensitive(t *testing.T) { } }) } + + t.Run("When webhooks are enabled it should not render webhook cert secrets", func(t *testing.T) { + g := NewGomegaWithT(t) + + var buf bytes.Buffer + opts := &Options{ + PrivatePlatform: string(hyperv1.NonePlatform), + PullSecretFile: pullSecretFile, + EnableDefaultingWebhook: true, + EnableValidatingWebhook: true, + EnableConversionWebhook: true, + RenderSensitive: true, + Format: RenderFormatYaml, + OutputTypes: string(OutputAll), + } + err := RenderHyperShiftOperator(t.Context(), &buf, opts) + g.Expect(err).ToNot(HaveOccurred()) + + var nonWebhookSecretCount int + for doc := range strings.SplitSeq(buf.String(), "\n---\n") { + if strings.TrimSpace(doc) == "" { + continue + } + obj, _, err := hyperapi.YamlSerializer.Decode([]byte(doc), nil, nil) + g.Expect(err).ToNot(HaveOccurred(), "failed to decode rendered manifest") + if secret, ok := obj.(*corev1.Secret); ok { + g.Expect(secret.Name).ToNot(Equal("webhook-serving-ca"), "webhook CA secret should not be rendered") + g.Expect(secret.Name).ToNot(Equal("manager-serving-cert"), "webhook serving cert should not be rendered") + nonWebhookSecretCount++ + } + } + g.Expect(nonWebhookSecretCount).To(BeNumerically(">", 0), "expected at least one non-webhook secret to be rendered") + }) } func TestHyperShiftOperatorManifests_SharedIngress(t *testing.T) { @@ -973,6 +1013,7 @@ func TestWaitForCAPIOperatorSync(t *testing.T) { }) } } + func TestApplyDefaults(t *testing.T) { tests := []struct { name string diff --git a/cmd/kubeconfig/create.go b/cmd/kubeconfig/create.go index 7fb8e6d89eb4..2c7973c71cc7 100644 --- a/cmd/kubeconfig/create.go +++ b/cmd/kubeconfig/create.go @@ -114,7 +114,7 @@ func Render(ctx context.Context, namespace string, name string, portForward bool }, } if err := c.Get(ctx, client.ObjectKeyFromObject(&kubeConfigSecret), &kubeConfigSecret); err != nil { - return fmt.Errorf("failed to get kubeconfig secret %s: %s", client.ObjectKeyFromObject(&kubeConfigSecret), err) + return fmt.Errorf("failed to get kubeconfig secret %s: %w", client.ObjectKeyFromObject(&kubeConfigSecret), err) } data, hasData := kubeConfigSecret.Data["kubeconfig"] if !hasData || len(data) == 0 { diff --git a/cmd/nodepool/core/create.go b/cmd/nodepool/core/create.go index 199c32c86fac..a623359bb7f3 100644 --- a/cmd/nodepool/core/create.go +++ b/cmd/nodepool/core/create.go @@ -195,12 +195,13 @@ func validateHostedClusterPayloadSupportsNodePoolCPUArch(ctx context.Context, cl logger := ctrl.LoggerFrom(ctx) hc := &hyperv1.HostedCluster{} - err := client.Get(context.Background(), types.NamespacedName{Namespace: namespace, Name: name}, hc) + err := client.Get(ctx, types.NamespacedName{Namespace: namespace, Name: name}, hc) if err != nil { - // This is expected to happen when we create a cluster since there is no created HostedCluster CR to check the - // payload from. - logger.Info("WARNING: failed to get HostedCluster to check payload type") - return nil + if apierrors.IsNotFound(err) { + logger.Info("WARNING: failed to get HostedCluster to check payload type") + return nil + } + return fmt.Errorf("failed to get HostedCluster to check payload type: %w", err) } if hc.Status.PayloadArch == "" { diff --git a/cmd/nodepool/core/create_test.go b/cmd/nodepool/core/create_test.go index 75c048d72577..9dc25ea84ac6 100644 --- a/cmd/nodepool/core/create_test.go +++ b/cmd/nodepool/core/create_test.go @@ -1,6 +1,8 @@ package core import ( + "context" + "fmt" "testing" . "github.com/onsi/gomega" @@ -16,6 +18,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" ) func TestValidateHostedClusterPayloadSupportsNodePoolCPUArch(t *testing.T) { @@ -104,6 +107,23 @@ func TestValidateHostedClusterPayloadSupportsNodePoolCPUArch(t *testing.T) { } }) } + + t.Run("When client.Get fails with a non-NotFound error it should return the error", func(t *testing.T) { + g := NewWithT(t) + + c := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, _ client.Object, _ ...client.GetOption) error { + return fmt.Errorf("API server unavailable") + }, + }). + Build() + + err := validateHostedClusterPayloadSupportsNodePoolCPUArch(t.Context(), c, "hc", "clusters", hyperv1.ArchitectureAMD64) + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(ContainSubstring("failed to get HostedCluster to check payload type"))) + }) } func TestValidMinorVersionCompatibility(t *testing.T) { diff --git a/codecov.yml b/codecov.yml index 31d7ec7e77f5..d0b375cb8630 100644 --- a/codecov.yml +++ b/codecov.yml @@ -3,6 +3,18 @@ codecov: notify: wait_for_ci: false +flags: + cpo-hostedcontrolplane: + carryforward: true + cpo-other: + carryforward: true + hypershift-operator: + carryforward: true + cmd-support: + carryforward: true + other: + carryforward: true + ignore: - "test/**" - "hack/**" @@ -15,6 +27,7 @@ ignore: - "**/*.md" - "**/*.yaml" - "**/*.yml" + - "**/Dockerfile*" - "*.mod" - "*.sum" # Generated mock files diff --git a/contrib/ci/gocacheprog/go.mod b/contrib/ci/gocacheprog/go.mod new file mode 100644 index 000000000000..d24d8951481a --- /dev/null +++ b/contrib/ci/gocacheprog/go.mod @@ -0,0 +1,3 @@ +module github.com/openshift/hypershift/contrib/ci/gocacheprog + +go 1.25 diff --git a/contrib/ci/gocacheprog/main.go b/contrib/ci/gocacheprog/main.go new file mode 100644 index 000000000000..74711d271497 --- /dev/null +++ b/contrib/ci/gocacheprog/main.go @@ -0,0 +1,217 @@ +// gocacheprog implements the GOCACHEPROG protocol (Go 1.24+) to serve +// a read-only Go build cache with a writable overlay. GET requests are +// served from a writable local directory first, then from a read-only +// shared directory (e.g. an EFS-backed PVC). PUT requests always write +// to the local directory. This eliminates the need to copy the shared +// cache at job start. +package main + +import ( + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +type request struct { + ID int64 `json:"ID"` + Command string `json:"Command"` + ActionID []byte `json:"ActionID,omitempty"` + OutputID []byte `json:"OutputID,omitempty"` + Body []byte `json:"-"` + BodySize int64 `json:"BodySize,omitempty"` +} + +type response struct { + ID int64 `json:"ID"` + Err string `json:"Err,omitempty"` + KnownCommands []string `json:"KnownCommands,omitempty"` + Miss bool `json:"Miss,omitempty"` + OutputID []byte `json:"OutputID,omitempty"` + Size int64 `json:"Size,omitempty"` + Time *time.Time `json:"Time,omitempty"` + DiskPath string `json:"DiskPath,omitempty"` +} + +func main() { + roDir := flag.String("ro", "", "read-only cache directory (e.g. EFS mount)") + rwDir := flag.String("rw", "", "writable cache directory (e.g. /tmp/go-build-cache)") + flag.Parse() + + if *rwDir == "" { + fmt.Fprintln(os.Stderr, "gocacheprog: --rw is required") + os.Exit(1) + } + + jd := json.NewDecoder(os.Stdin) + je := json.NewEncoder(os.Stdout) + var mu sync.Mutex + var wg sync.WaitGroup + + je.Encode(response{KnownCommands: []string{"get", "put", "close"}}) + + for { + var req request + if err := jd.Decode(&req); err != nil { + if err == io.EOF { + break + } + log.Fatalf("gocacheprog: decode request: %v", err) + } + + if req.Command == "put" && req.BodySize > 0 { + if err := jd.Decode(&req.Body); err != nil { + log.Fatalf("gocacheprog: decode body: %v", err) + } + } + + wg.Add(1) + go func() { + defer wg.Done() + res := handleRequest(&req, *roDir, *rwDir) + mu.Lock() + je.Encode(res) + mu.Unlock() + }() + } + wg.Wait() +} + +func handleRequest(req *request, roDir, rwDir string) response { + switch req.Command { + case "get": + return handleGet(req, roDir, rwDir) + case "put": + return handlePut(req, rwDir) + case "close": + return response{ID: req.ID} + default: + return response{ID: req.ID, Err: "unknown command"} + } +} + +// actionFile returns the path to a Go cache action entry. +// Format: //-a +func actionFile(dir string, id []byte) string { + if len(id) == 0 { + return "" + } + h := hex.EncodeToString(id) + return filepath.Join(dir, h[:2], h+"-a") +} + +// outputFile returns the path to a Go cache data file. +// Format: //-d +func outputFile(dir string, id []byte) string { + if len(id) == 0 { + return "" + } + h := hex.EncodeToString(id) + return filepath.Join(dir, h[:2], h+"-d") +} + +// lookup reads a Go cache action entry and verifies the data file exists. +// The action entry format is: v1 +func lookup(dir string, actionID []byte) (resp response, ok bool) { + data, err := os.ReadFile(actionFile(dir, actionID)) + if err != nil { + return + } + fields := strings.Fields(strings.TrimSpace(string(data))) + if len(fields) != 5 || fields[0] != "v1" { + return + } + if fields[1] != hex.EncodeToString(actionID) { + return + } + outputID, err := hex.DecodeString(fields[2]) + if err != nil { + return + } + nanos, err := strconv.ParseInt(fields[4], 10, 64) + if err != nil { + return + } + dPath := outputFile(dir, outputID) + fi, err := os.Stat(dPath) + if err != nil { + return + } + t := time.Unix(0, nanos) + return response{ + OutputID: outputID, + Size: fi.Size(), + Time: &t, + DiskPath: dPath, + }, true +} + +func handleGet(req *request, roDir, rwDir string) response { + if resp, ok := lookup(rwDir, req.ActionID); ok { + resp.ID = req.ID + return resp + } + if roDir != "" { + if resp, ok := lookup(roDir, req.ActionID); ok { + resp.ID = req.ID + return resp + } + } + return response{ID: req.ID, Miss: true} +} + +func handlePut(req *request, rwDir string) response { + dPath := outputFile(rwDir, req.OutputID) + if err := os.MkdirAll(filepath.Dir(dPath), 0o777); err != nil { + return response{ID: req.ID, Err: err.Error()} + } + if err := writeFileAtomic(dPath, req.Body); err != nil { + return response{ID: req.ID, Err: err.Error()} + } + + aPath := actionFile(rwDir, req.ActionID) + if err := os.MkdirAll(filepath.Dir(aPath), 0o777); err != nil { + return response{ID: req.ID, Err: err.Error()} + } + entry := fmt.Sprintf("v1 %s %s %d %d\n", + hex.EncodeToString(req.ActionID), + hex.EncodeToString(req.OutputID), + req.BodySize, + time.Now().UnixNano(), + ) + if err := writeFileAtomic(aPath, []byte(entry)); err != nil { + return response{ID: req.ID, Err: err.Error()} + } + + return response{ID: req.ID, DiskPath: dPath} +} + +// writeFileAtomic writes data to a temporary file in the same directory +// then renames it to the target path. This prevents concurrent readers +// from seeing a truncated file. +func writeFileAtomic(path string, data []byte) error { + tmp, err := os.CreateTemp(filepath.Dir(path), ".tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + if _, err := tmp.Write(data); err != nil { + tmp.Close() + os.Remove(tmpName) + return err + } + if err := tmp.Close(); err != nil { + os.Remove(tmpName) + return err + } + return os.Rename(tmpName, path) +} + diff --git a/contrib/ci/gocacheprog/main_test.go b/contrib/ci/gocacheprog/main_test.go new file mode 100644 index 000000000000..a879b04b2795 --- /dev/null +++ b/contrib/ci/gocacheprog/main_test.go @@ -0,0 +1,482 @@ +package main + +import ( + "encoding/hex" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "testing" + "time" +) + +func mustDecodeHex(t *testing.T, s string) []byte { + t.Helper() + b, err := hex.DecodeString(s) + if err != nil { + t.Fatalf("bad hex %q: %v", s, err) + } + return b +} + +func TestActionFile(t *testing.T) { + t.Parallel() + tests := []struct { + name string + id []byte + want string + }{ + { + name: "When id is nil it should return empty", + id: nil, + want: "", + }, + { + name: "When id is empty it should return empty", + id: []byte{}, + want: "", + }, + { + name: "When id is valid it should return the correct path", + id: mustDecodeHex(t, "abcdef0123456789"), + want: filepath.Join("/cache", "ab", "abcdef0123456789-a"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := actionFile("/cache", tt.id) + if got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } +} + +func TestOutputFile(t *testing.T) { + t.Parallel() + tests := []struct { + name string + id []byte + want string + }{ + { + name: "When id is nil it should return empty", + id: nil, + want: "", + }, + { + name: "When id is valid it should return the correct path", + id: mustDecodeHex(t, "abcdef0123456789"), + want: filepath.Join("/cache", "ab", "abcdef0123456789-d"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := outputFile("/cache", tt.id) + if got != tt.want { + t.Errorf("got %q, want %q", got, tt.want) + } + }) + } +} + +func writeCacheEntry(t *testing.T, dir string, actionID, outputID []byte, body []byte) { + t.Helper() + aPath := actionFile(dir, actionID) + dPath := outputFile(dir, outputID) + if err := os.MkdirAll(filepath.Dir(aPath), 0o777); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Dir(dPath), 0o777); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dPath, body, 0o666); err != nil { + t.Fatal(err) + } + entry := fmt.Sprintf("v1 %s %s %d %d\n", + hex.EncodeToString(actionID), + hex.EncodeToString(outputID), + len(body), + time.Now().UnixNano(), + ) + if err := os.WriteFile(aPath, []byte(entry), 0o666); err != nil { + t.Fatal(err) + } +} + +func TestLookup(t *testing.T) { + t.Parallel() + actionID := mustDecodeHex(t, "aaaaaaaaaaaaaaaa") + outputID := mustDecodeHex(t, "bbbbbbbbbbbbbbbb") + body := []byte("hello world") + + t.Run("When entry exists it should return hit", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + writeCacheEntry(t, dir, actionID, outputID, body) + + resp, ok := lookup(dir, actionID) + if !ok { + t.Fatal("expected hit, got miss") + } + if hex.EncodeToString(resp.OutputID) != hex.EncodeToString(outputID) { + t.Errorf("outputID = %x, want %x", resp.OutputID, outputID) + } + if resp.Size != int64(len(body)) { + t.Errorf("size = %d, want %d", resp.Size, len(body)) + } + if resp.Time == nil { + t.Error("expected non-nil Time") + } + if resp.DiskPath == "" { + t.Error("expected non-empty DiskPath") + } + }) + + t.Run("When entry does not exist it should return miss", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + _, ok := lookup(dir, actionID) + if ok { + t.Fatal("expected miss, got hit") + } + }) + + t.Run("When action file has wrong format it should return miss", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + aPath := actionFile(dir, actionID) + os.MkdirAll(filepath.Dir(aPath), 0o777) + os.WriteFile(aPath, []byte("garbage"), 0o666) + + _, ok := lookup(dir, actionID) + if ok { + t.Fatal("expected miss for malformed entry") + } + }) + + t.Run("When data file is missing it should return miss", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + aPath := actionFile(dir, actionID) + os.MkdirAll(filepath.Dir(aPath), 0o777) + entry := fmt.Sprintf("v1 %s %s 11 %d\n", + hex.EncodeToString(actionID), + hex.EncodeToString(outputID), + time.Now().UnixNano(), + ) + os.WriteFile(aPath, []byte(entry), 0o666) + + _, ok := lookup(dir, actionID) + if ok { + t.Fatal("expected miss when data file is absent") + } + }) +} + +func TestHandleGet(t *testing.T) { + t.Parallel() + actionID := mustDecodeHex(t, "1111111111111111") + outputID := mustDecodeHex(t, "2222222222222222") + body := []byte("cached output") + + t.Run("When entry is in rw dir it should return hit from rw", func(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + roDir := t.TempDir() + writeCacheEntry(t, rwDir, actionID, outputID, body) + + req := &request{ID: 42, Command: "get", ActionID: actionID} + resp := handleGet(req, roDir, rwDir) + if resp.Miss { + t.Fatal("expected hit") + } + if resp.ID != 42 { + t.Errorf("ID = %d, want 42", resp.ID) + } + if resp.Size != int64(len(body)) { + t.Errorf("size = %d, want %d", resp.Size, len(body)) + } + }) + + t.Run("When entry is only in ro dir it should return hit from ro", func(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + roDir := t.TempDir() + writeCacheEntry(t, roDir, actionID, outputID, body) + + req := &request{ID: 7, Command: "get", ActionID: actionID} + resp := handleGet(req, roDir, rwDir) + if resp.Miss { + t.Fatal("expected hit from ro") + } + if resp.ID != 7 { + t.Errorf("ID = %d, want 7", resp.ID) + } + }) + + t.Run("When entry is in neither dir it should return miss", func(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + roDir := t.TempDir() + + req := &request{ID: 3, Command: "get", ActionID: actionID} + resp := handleGet(req, roDir, rwDir) + if !resp.Miss { + t.Fatal("expected miss") + } + }) + + t.Run("When roDir is empty string it should skip ro lookup", func(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + + req := &request{ID: 5, Command: "get", ActionID: actionID} + resp := handleGet(req, "", rwDir) + if !resp.Miss { + t.Fatal("expected miss") + } + }) + + t.Run("When rw shadows ro it should prefer rw", func(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + roDir := t.TempDir() + rwBody := []byte("rw content") + roBody := []byte("ro content, different size") + writeCacheEntry(t, rwDir, actionID, outputID, rwBody) + writeCacheEntry(t, roDir, actionID, outputID, roBody) + + req := &request{ID: 1, Command: "get", ActionID: actionID} + resp := handleGet(req, roDir, rwDir) + if resp.Miss { + t.Fatal("expected hit") + } + if resp.Size != int64(len(rwBody)) { + t.Errorf("size = %d, want %d (rw should shadow ro)", resp.Size, len(rwBody)) + } + }) +} + +func TestHandlePut(t *testing.T) { + t.Parallel() + actionID := mustDecodeHex(t, "3333333333333333") + outputID := mustDecodeHex(t, "4444444444444444") + body := []byte("new output data") + + t.Run("When writing succeeds it should create action and data files", func(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + req := &request{ + ID: 10, + Command: "put", + ActionID: actionID, + OutputID: outputID, + Body: body, + BodySize: int64(len(body)), + } + resp := handlePut(req, rwDir) + if resp.Err != "" { + t.Fatalf("unexpected error: %s", resp.Err) + } + if resp.ID != 10 { + t.Errorf("ID = %d, want 10", resp.ID) + } + if resp.DiskPath == "" { + t.Error("expected non-empty DiskPath") + } + + data, err := os.ReadFile(resp.DiskPath) + if err != nil { + t.Fatalf("reading data file: %v", err) + } + if string(data) != string(body) { + t.Errorf("data = %q, want %q", data, body) + } + + lookupResp, ok := lookup(rwDir, actionID) + if !ok { + t.Fatal("expected lookup to succeed after put") + } + if lookupResp.Size != int64(len(body)) { + t.Errorf("lookup size = %d, want %d", lookupResp.Size, len(body)) + } + }) + + t.Run("When rwDir is unwritable it should return error", func(t *testing.T) { + t.Parallel() + req := &request{ + ID: 11, + Command: "put", + ActionID: actionID, + OutputID: outputID, + Body: body, + BodySize: int64(len(body)), + } + resp := handlePut(req, "/proc/nonexistent-gocacheprog-test") + if resp.Err == "" { + t.Fatal("expected error for unwritable dir") + } + if resp.ID != 11 { + t.Errorf("ID = %d, want 11", resp.ID) + } + }) +} + +func TestHandleRequest(t *testing.T) { + t.Parallel() + + t.Run("When command is close it should return empty response", func(t *testing.T) { + t.Parallel() + req := &request{ID: 99, Command: "close"} + resp := handleRequest(req, "", t.TempDir()) + if resp.ID != 99 { + t.Errorf("ID = %d, want 99", resp.ID) + } + if resp.Err != "" { + t.Errorf("unexpected error: %s", resp.Err) + } + }) + + t.Run("When command is unknown it should return error", func(t *testing.T) { + t.Parallel() + req := &request{ID: 100, Command: "delete"} + resp := handleRequest(req, "", t.TempDir()) + if resp.Err != "unknown command" { + t.Errorf("Err = %q, want %q", resp.Err, "unknown command") + } + }) +} + +func TestPutThenGet(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + actionID := mustDecodeHex(t, "5555555555555555") + outputID := mustDecodeHex(t, "6666666666666666") + body := []byte("round trip data") + + putReq := &request{ + ID: 1, + Command: "put", + ActionID: actionID, + OutputID: outputID, + Body: body, + BodySize: int64(len(body)), + } + putResp := handlePut(putReq, rwDir) + if putResp.Err != "" { + t.Fatalf("put error: %s", putResp.Err) + } + + getReq := &request{ID: 2, Command: "get", ActionID: actionID} + getResp := handleGet(getReq, "", rwDir) + if getResp.Miss { + t.Fatal("expected hit after put") + } + if getResp.Size != int64(len(body)) { + t.Errorf("size = %d, want %d", getResp.Size, len(body)) + } + if hex.EncodeToString(getResp.OutputID) != hex.EncodeToString(outputID) { + t.Errorf("outputID = %x, want %x", getResp.OutputID, outputID) + } +} + +func TestConcurrentPutAndGet(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + var wg sync.WaitGroup + + for i := range 50 { + wg.Add(1) + go func() { + defer wg.Done() + actionID := mustDecodeHex(t, fmt.Sprintf("%016x", i)) + outputID := mustDecodeHex(t, fmt.Sprintf("%016x", i+1000)) + body := []byte(fmt.Sprintf("body-%d", i)) + + putReq := &request{ + ID: int64(i), + Command: "put", + ActionID: actionID, + OutputID: outputID, + Body: body, + BodySize: int64(len(body)), + } + resp := handlePut(putReq, rwDir) + if resp.Err != "" { + t.Errorf("put %d error: %s", i, resp.Err) + return + } + + getReq := &request{ID: int64(i + 1000), Command: "get", ActionID: actionID} + getResp := handleGet(getReq, "", rwDir) + if getResp.Miss { + t.Errorf("get %d: expected hit after put", i) + } + }() + } + wg.Wait() +} + +func TestConcurrentPutSameOutputID(t *testing.T) { + t.Parallel() + rwDir := t.TempDir() + outputID := mustDecodeHex(t, "7777777777777777") + body := []byte("shared output content that all writers agree on") + + // Seed one entry so GETs can find it while PUTs overwrite the data file. + seedAction := mustDecodeHex(t, fmt.Sprintf("%016x", 2000)) + seedReq := &request{ + ID: 0, Command: "put", + ActionID: seedAction, OutputID: outputID, + Body: body, BodySize: int64(len(body)), + } + if resp := handlePut(seedReq, rwDir); resp.Err != "" { + t.Fatalf("seed put: %s", resp.Err) + } + + // Interleave PUTs (different ActionIDs, same OutputID) with GETs that + // read the data file via DiskPath. Without atomic writes, a PUT's + // O_TRUNC would momentarily zero the file, causing a reader to see + // truncated/empty content. + var wg sync.WaitGroup + var badReads atomic.Int64 + for i := range 100 { + wg.Add(2) + go func() { + defer wg.Done() + actionID := mustDecodeHex(t, fmt.Sprintf("%016x", i+2000)) + putReq := &request{ + ID: int64(i), Command: "put", + ActionID: actionID, OutputID: outputID, + Body: body, BodySize: int64(len(body)), + } + if resp := handlePut(putReq, rwDir); resp.Err != "" { + t.Errorf("put %d error: %s", i, resp.Err) + } + }() + go func() { + defer wg.Done() + getReq := &request{ID: int64(i + 5000), Command: "get", ActionID: seedAction} + resp := handleGet(getReq, "", rwDir) + if resp.Miss { + return + } + data, err := os.ReadFile(resp.DiskPath) + if err != nil { + return + } + if string(data) != string(body) { + badReads.Add(1) + } + }() + } + wg.Wait() + + if n := badReads.Load(); n > 0 { + t.Errorf("got %d reads with corrupted data from concurrent PUT/GET on same OutputID", n) + } +} diff --git a/contrib/konflux/cpo_4_21_stream.yaml b/contrib/konflux/cpo_4_21_stream.yaml new file mode 100644 index 000000000000..84bc5bf76b59 --- /dev/null +++ b/contrib/konflux/cpo_4_21_stream.yaml @@ -0,0 +1,11 @@ +apiVersion: projctl.konflux.dev/v1beta1 +kind: ProjectDevelopmentStream +metadata: + name: control-plane-operator-v4-21 +spec: + project: crt-redhat-acm-tenant + template: + name: hypershift-cpo-template + values: + - name: version + value: "4.21" diff --git a/contrib/konflux/cpo_4_22_stream.yaml b/contrib/konflux/cpo_4_22_stream.yaml new file mode 100644 index 000000000000..bf2ae45ab066 --- /dev/null +++ b/contrib/konflux/cpo_4_22_stream.yaml @@ -0,0 +1,11 @@ +apiVersion: projctl.konflux.dev/v1beta1 +kind: ProjectDevelopmentStream +metadata: + name: control-plane-operator-v4-22 +spec: + project: crt-redhat-acm-tenant + template: + name: hypershift-cpo-template + values: + - name: version + value: "4.22" diff --git a/control-plane-operator/AGENTS.md b/control-plane-operator/AGENTS.md index 63f671d62e16..b26f1ed8403b 100644 --- a/control-plane-operator/AGENTS.md +++ b/control-plane-operator/AGENTS.md @@ -21,7 +21,7 @@ See `support/controlplane-component/AGENTS.md` and `support/controlplane-compone ## HCCO (Hosted Cluster Config Operator) -HCCO is a **separate binary in the same image**, invoked as `control-plane-operator hosted-cluster-config-operator`. It runs as a Deployment **in the guest cluster** (not the management cluster), reconciling guest-side resources: `openshift-config/pull-secret`, node configuration, in-place upgrades, global pull secret, draining, etc. +HCCO is a **separate binary in the same image**, invoked as `control-plane-operator hosted-cluster-config-operator`. It runs as a Deployment in the control plane namespace, mainly reconciling guest-side resources: `openshift-config/pull-secret`, node configuration, in-place upgrades, global pull secret, draining, etc. HCCO controllers are in `hostedclusterconfigoperator/controllers/`. They do **not** use the v2 component framework today — this is a migration opportunity. diff --git a/control-plane-operator/controllers/azureprivatelinkservice/controller.go b/control-plane-operator/controllers/azureprivatelinkservice/controller.go index 79afffd3e9f6..1c75c9a2529b 100644 --- a/control-plane-operator/controllers/azureprivatelinkservice/controller.go +++ b/control-plane-operator/controllers/azureprivatelinkservice/controller.go @@ -334,7 +334,7 @@ func (r *AzurePrivateLinkServiceReconciler) Reconcile(ctx context.Context, req c // Workers need to resolve api-. and oauth-. // to the PE IP so that the console, OAuth, and other services work on private clusters. if azPLS.Spec.BaseDomain != "" { - if result, err := r.reconcileBaseDomainDNS(ctx, azPLS, hcp.Name, hcp.Spec.DNS.BaseDomain, log); err != nil || !result.IsZero() { + if result, err := r.reconcileBaseDomainDNS(ctx, azPLS, hcp.Name, log); err != nil || !result.IsZero() { return result, err } } @@ -744,19 +744,6 @@ func (r *AzurePrivateLinkServiceReconciler) reconcileDNS(ctx context.Context, az }, log) } -func baseDomainShadowsClusterDomain(baseDomain, clusterName, hcpBaseDomain string) bool { - if baseDomain == "" || clusterName == "" || hcpBaseDomain == "" { - return false - } - clusterDomain := strings.ToLower(clusterName + "." + hcpBaseDomain) - // A Private DNS zone named baseDomain is authoritative for all queries - // under *.baseDomain. Prepending a dot to both sides ensures we match at - // domain label boundaries only (e.g. "ample.com" does NOT match - // "example.com") while also catching baseDomain == hcpBaseDomain - // (e.g. baseDomain "example.com" shadows "my-cluster.example.com"). - return strings.HasSuffix("."+clusterDomain, "."+strings.ToLower(baseDomain)) -} - // reconcileBaseDomainDNS creates a Private DNS Zone for the cluster's base domain, // links it to the guest VNet, and creates A records for the API and/or OAuth hostnames. // This enables worker VMs to resolve api-. and oauth-. @@ -769,44 +756,7 @@ func baseDomainShadowsClusterDomain(baseDomain, clusterName, hcpBaseDomain strin // (backward compatibility for clusters without a separate OAuth PLS). // - Any other CR (e.g., oauth-openshift): Creates only oauth- record, pointing // to this CR's own PE IP. -// -// When the base domain would shadow the cluster's apps domain (baseDomain == -// clusterName.hcpBaseDomain or is a parent domain of it), a degraded condition -// is set on the CR and zone creation is skipped entirely. Creating a Private DNS -// zone for the base domain in this case would make it authoritative for *.apps -// queries, but the Private Endpoint routes to the management-plane private-router -// (HAProxy) whose SNI ACLs only match *.hypershift.local hostnames. Base domain -// *.apps hostnames would fall through to KAS, which presents a TLS cert that does -// not cover them, breaking ingress. The correct fix is to recreate the cluster -// with a different --external-dns-domain value. -func (r *AzurePrivateLinkServiceReconciler) reconcileBaseDomainDNS(ctx context.Context, azPLS *hyperv1.AzurePrivateLinkService, clusterName, hcpBaseDomain string, log logr.Logger) (ctrl.Result, error) { - // Skip zone creation when shadowing is detected. We cannot add *.apps → PE IP - // because the PE routes to private-router (HAProxy) which only serves - // .hypershift.local hostnames — apps traffic gets KAS certs. The data-plane - // router-default IP is not discoverable from this controller. - if baseDomainShadowsClusterDomain(azPLS.Spec.BaseDomain, clusterName, hcpBaseDomain) { - log.Info("Base domain zone shadows cluster apps domain, DNS resolution for *.apps will be affected", - "baseDomain", azPLS.Spec.BaseDomain, - "clusterDomain", clusterName+"."+hcpBaseDomain) - - patch := client.MergeFrom(azPLS.DeepCopy()) - meta.SetStatusCondition(&azPLS.Status.Conditions, metav1.Condition{ - Type: string(hyperv1.AzurePrivateDNSAvailable), - Status: metav1.ConditionFalse, - Reason: "BaseDomainShadowsClusterDomain", - Message: fmt.Sprintf("Base domain %q shadows the cluster domain %q. "+ - "The Private DNS zone will intercept *.apps queries, breaking ingress. "+ - "Recreate the cluster with a different --external-dns-domain value.", - azPLS.Spec.BaseDomain, clusterName+"."+hcpBaseDomain), - ObservedGeneration: azPLS.Generation, - }) - if err := r.Status().Patch(ctx, azPLS, patch); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to update degraded condition for base domain shadowing: %w", err) - } - - return ctrl.Result{RequeueAfter: azureutil.DriftDetectionRequeueInterval}, nil - } - +func (r *AzurePrivateLinkServiceReconciler) reconcileBaseDomainDNS(ctx context.Context, azPLS *hyperv1.AzurePrivateLinkService, clusterName string, log logr.Logger) (ctrl.Result, error) { recordNames, err := r.recordNamesForCR(ctx, azPLS, clusterName, log) if err != nil { return ctrl.Result{}, err diff --git a/control-plane-operator/controllers/azureprivatelinkservice/controller_test.go b/control-plane-operator/controllers/azureprivatelinkservice/controller_test.go index ced1429be6d8..3e63b043a3e8 100644 --- a/control-plane-operator/controllers/azureprivatelinkservice/controller_test.go +++ b/control-plane-operator/controllers/azureprivatelinkservice/controller_test.go @@ -182,29 +182,24 @@ func (m *mockVirtualNetworkLinks) BeginDelete(_ context.Context, _ string, _ str } type mockRecordSets struct { - createErr error - deleteErr error - deleteErrZone string // if set, deleteErr only applies to this zone - createCalled bool - deleteCalled bool - createCallCount int - deleteCallCount int - createdRecordNames []string - createdRecordsByZone map[string][]string - deletedRecordNames []string - lastRecordSetName string - lastRecordType armprivatedns.RecordType - lastRecordParams armprivatedns.RecordSet -} - -func (m *mockRecordSets) CreateOrUpdate(_ context.Context, _ string, privateDnsZoneName string, recordType armprivatedns.RecordType, relativeRecordSetName string, parameters armprivatedns.RecordSet, _ *armprivatedns.RecordSetsClientCreateOrUpdateOptions) (armprivatedns.RecordSetsClientCreateOrUpdateResponse, error) { + createErr error + deleteErr error + deleteErrZone string // if set, deleteErr only applies to this zone + createCalled bool + deleteCalled bool + createCallCount int + deleteCallCount int + createdRecordNames []string + deletedRecordNames []string + lastRecordSetName string + lastRecordType armprivatedns.RecordType + lastRecordParams armprivatedns.RecordSet +} + +func (m *mockRecordSets) CreateOrUpdate(_ context.Context, _ string, _ string, recordType armprivatedns.RecordType, relativeRecordSetName string, parameters armprivatedns.RecordSet, _ *armprivatedns.RecordSetsClientCreateOrUpdateOptions) (armprivatedns.RecordSetsClientCreateOrUpdateResponse, error) { m.createCalled = true m.createCallCount++ m.createdRecordNames = append(m.createdRecordNames, relativeRecordSetName) - if m.createdRecordsByZone == nil { - m.createdRecordsByZone = make(map[string][]string) - } - m.createdRecordsByZone[privateDnsZoneName] = append(m.createdRecordsByZone[privateDnsZoneName], relativeRecordSetName) m.lastRecordSetName = relativeRecordSetName m.lastRecordType = recordType m.lastRecordParams = parameters @@ -1531,7 +1526,7 @@ func TestReconcileBaseDomainDNS_WhenPrivateRouterWithNoSibling_ItShouldCreateBot RecordSets: mockRecords, } - result, err := r.reconcileBaseDomainDNS(t.Context(), azPLS, "test-hcp", "", testr.New(t)) + result, err := r.reconcileBaseDomainDNS(t.Context(), azPLS, "test-hcp", testr.New(t)) g.Expect(err).ToNot(HaveOccurred()) g.Expect(result.IsZero()).To(BeTrue()) @@ -1570,7 +1565,7 @@ func TestReconcileBaseDomainDNS_WhenPrivateRouterWithSiblingOAuth_ItShouldOnlyCr RecordSets: mockRecords, } - result, err := r.reconcileBaseDomainDNS(t.Context(), azPLS, "test-hcp", "", testr.New(t)) + result, err := r.reconcileBaseDomainDNS(t.Context(), azPLS, "test-hcp", testr.New(t)) g.Expect(err).ToNot(HaveOccurred()) g.Expect(result.IsZero()).To(BeTrue()) @@ -1605,7 +1600,7 @@ func TestReconcileBaseDomainDNS_WhenOAuthCR_ItShouldOnlyCreateOAuthRecord(t *tes RecordSets: mockRecords, } - result, err := r.reconcileBaseDomainDNS(t.Context(), azPLS, "test-hcp", "", testr.New(t)) + result, err := r.reconcileBaseDomainDNS(t.Context(), azPLS, "test-hcp", testr.New(t)) g.Expect(err).ToNot(HaveOccurred()) g.Expect(result.IsZero()).To(BeTrue()) @@ -1654,7 +1649,7 @@ func TestReconcileDelete_WhenSiblingCRsExist_ItShouldNotDeleteBaseDomainZone(t * // A records should only include the api record (sibling OAuth owns the oauth record) g.Expect(mockRecords.deleteCalled).To(BeTrue(), "should delete A records") - // The hypershift.local records (api, *.apps) + api-test-hcp from base domain = 3 + // The hypershift.local records (api, *.apps) + only api-test-hcp from base domain = 3 g.Expect(mockRecords.deletedRecordNames).To(ConsistOf("api", "*.apps", "api-test-hcp"), "should delete hypershift.local records and only api base domain record (sibling owns oauth)") @@ -2275,7 +2270,7 @@ func TestReconcileBaseDomainDNS_WhenDNSZoneCreateFails_ItShouldRequeueAfterError RecordSets: &mockRecordSets{}, } - result, err := r.reconcileBaseDomainDNS(t.Context(), azPLS, "test-hcp", "", testr.New(t)) + result, err := r.reconcileBaseDomainDNS(t.Context(), azPLS, "test-hcp", testr.New(t)) g.Expect(err).ToNot(HaveOccurred()) g.Expect(result.RequeueAfter).ToNot(BeZero()) } @@ -3737,233 +3732,6 @@ func TestReconcile_WhenNonPrivateRouterDNSZoneNamePatchFails_ItShouldReturnError g.Expect(err).To(MatchError(ContainSubstring("failed to persist DNS zone name in status"))) } -func TestReconcile_WhenBaseDomainShadowsClusterDomain_ItShouldSetDegradedConditionAndSkipZoneCreation(t *testing.T) { - t.Parallel() - g := NewGomegaWithT(t) - scheme := newTestScheme(t, g) - - azPLS := newTestAzurePLS(t, "private-router", "test-ns") - azPLS.Finalizers = []string{azurePrivateLinkServiceFinalizer} - azPLS.Status.PrivateLinkServiceAlias = "test-alias" - azPLS.Status.PrivateEndpointIP = "10.0.1.5" - azPLS.Status.PrivateEndpointID = "/pe/id" - // This is the shadowing condition: baseDomain == hcpName.hcpDNSBaseDomain - azPLS.Spec.BaseDomain = "test-hcp.example.com" - - hcp := newTestHCP(t, "test-hcp", "test-ns", "api.test.example.com") - hcp.Spec.DNS.BaseDomain = "example.com" - hcp.Finalizers = []string{hcpAzurePLSFinalizerName} - - fakeClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(azPLS, hcp). - WithStatusSubresource(azPLS). - Build() - - mockPE := &mockPrivateEndpoints{ - getResponse: armnetwork.PrivateEndpointsClientGetResponse{ - PrivateEndpoint: armnetwork.PrivateEndpoint{ - ID: ptr.To("/pe/id"), - Properties: &armnetwork.PrivateEndpointProperties{ - CustomDNSConfigs: []*armnetwork.CustomDNSConfigPropertiesFormat{ - {IPAddresses: []*string{ptr.To("10.0.1.5")}}, - }, - }, - }, - }, - } - mockDNS := &mockPrivateDNSZones{} - mockRecords := &mockRecordSets{} - - r := &AzurePrivateLinkServiceReconciler{ - Client: fakeClient, - PrivateEndpoints: mockPE, - PrivateDNSZones: mockDNS, - VirtualNetworkLinks: &mockVirtualNetworkLinks{}, - RecordSets: mockRecords, - } - - result, err := r.Reconcile(log.IntoContext(t.Context(), testr.New(t)), ctrl.Request{ - NamespacedName: types.NamespacedName{Name: "private-router", Namespace: "test-ns"}, - }) - - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(result.RequeueAfter).ToNot(BeZero(), "should requeue for drift detection") - - // No base domain zone should be created when shadowing is detected. - // The last zone created should be the hypershift.local zone, not the base domain zone. - g.Expect(mockDNS.lastZoneName).To(Equal("test-hcp.hypershift.local"), - "only the hypershift.local zone should be created, not the base domain zone") - - // Verify no records were created for the base domain zone - baseDomainRecords := mockRecords.createdRecordsByZone["test-hcp.example.com"] - g.Expect(baseDomainRecords).To(BeEmpty(), - "no records should be created in the base domain zone when shadowing is detected") - - // Verify the DNS condition is set to False with the degraded reason - updated := &hyperv1.AzurePrivateLinkService{} - err = fakeClient.Get(t.Context(), types.NamespacedName{Name: "private-router", Namespace: "test-ns"}, updated) - g.Expect(err).ToNot(HaveOccurred()) - - dnsCondition := meta.FindStatusCondition(updated.Status.Conditions, string(hyperv1.AzurePrivateDNSAvailable)) - g.Expect(dnsCondition).ToNot(BeNil(), "DNS condition should be set") - g.Expect(dnsCondition.Status).To(Equal(metav1.ConditionFalse), - "DNS condition should be False when shadowing is detected") - g.Expect(dnsCondition.Reason).To(Equal("BaseDomainShadowsClusterDomain"), - "DNS condition reason should indicate base domain shadowing") - g.Expect(dnsCondition.Message).To(ContainSubstring("shadows the cluster domain"), - "DNS condition message should explain the shadowing issue") - g.Expect(dnsCondition.Message).To(ContainSubstring("--external-dns-domain"), - "DNS condition message should suggest recreating with a different external-dns-domain") -} - -func TestBaseDomainShadowsClusterDomain(t *testing.T) { - t.Parallel() - - tests := map[string]struct { - baseDomain string - clusterName string - hcpBaseDomain string - expected bool - }{ - "When baseDomain equals clusterName.hcpBaseDomain, it should detect shadowing": { - baseDomain: "my-cluster.example.com", - clusterName: "my-cluster", - hcpBaseDomain: "example.com", - expected: true, - }, - "When baseDomain matches with different casing, it should detect shadowing": { - baseDomain: "My-Cluster.Example.COM", - clusterName: "my-cluster", - hcpBaseDomain: "example.com", - expected: true, - }, - "When baseDomain differs from clusterName.hcpBaseDomain, it should not detect shadowing": { - baseDomain: "other-prefix.example.com", - clusterName: "my-cluster", - hcpBaseDomain: "example.com", - expected: false, - }, - "When baseDomain is empty, it should not detect shadowing": { - baseDomain: "", - clusterName: "my-cluster", - hcpBaseDomain: "example.com", - expected: false, - }, - "When hcpBaseDomain is empty, it should not detect shadowing": { - baseDomain: "my-cluster.example.com", - clusterName: "my-cluster", - hcpBaseDomain: "", - expected: false, - }, - "When clusterName is empty, it should return false": { - baseDomain: "my-cluster.example.com", - clusterName: "", - hcpBaseDomain: "example.com", - expected: false, - }, - "When baseDomain equals hcpBaseDomain, it should detect shadowing": { - baseDomain: "example.com", - clusterName: "my-cluster", - hcpBaseDomain: "example.com", - expected: true, - }, - "When baseDomain is a non-domain-boundary suffix, it should not detect shadowing": { - baseDomain: "ample.com", - clusterName: "my-cluster", - hcpBaseDomain: "example.com", - expected: false, - }, - "When baseDomain is a multi-level parent domain, it should detect shadowing": { - baseDomain: "devcluster.openshift.com", - clusterName: "hcp-two", - hcpBaseDomain: "acm-dev04.devcluster.openshift.com", - expected: true, - }, - } - - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - t.Parallel() - g := NewWithT(t) - - g.Expect(baseDomainShadowsClusterDomain(tt.baseDomain, tt.clusterName, tt.hcpBaseDomain)).To(Equal(tt.expected)) - }) - } -} - -func TestReconcile_WhenBaseDomainDiffersFromClusterDomain_ItShouldCreateZoneNormally(t *testing.T) { - t.Parallel() - g := NewGomegaWithT(t) - scheme := newTestScheme(t, g) - - azPLS := newTestAzurePLS(t, "private-router", "test-ns") - azPLS.Finalizers = []string{azurePrivateLinkServiceFinalizer} - azPLS.Status.PrivateLinkServiceAlias = "test-alias" - azPLS.Status.PrivateEndpointIP = "10.0.1.5" - azPLS.Status.PrivateEndpointID = "/pe/id" - // No shadowing: base domain is unrelated to hcpName.hcpDNSBaseDomain - azPLS.Spec.BaseDomain = "custom-dns.example.com" - - hcp := newTestHCP(t, "test-hcp", "test-ns", "api.test.example.com") - hcp.Spec.DNS.BaseDomain = "example.com" - hcp.Finalizers = []string{hcpAzurePLSFinalizerName} - - fakeClient := fake.NewClientBuilder(). - WithScheme(scheme). - WithObjects(azPLS, hcp). - WithStatusSubresource(azPLS). - Build() - - mockPE := &mockPrivateEndpoints{ - getResponse: armnetwork.PrivateEndpointsClientGetResponse{ - PrivateEndpoint: armnetwork.PrivateEndpoint{ - ID: ptr.To("/pe/id"), - Properties: &armnetwork.PrivateEndpointProperties{ - CustomDNSConfigs: []*armnetwork.CustomDNSConfigPropertiesFormat{ - {IPAddresses: []*string{ptr.To("10.0.1.5")}}, - }, - }, - }, - }, - } - mockDNS := &mockPrivateDNSZones{} - mockRecords := &mockRecordSets{} - - r := &AzurePrivateLinkServiceReconciler{ - Client: fakeClient, - PrivateEndpoints: mockPE, - PrivateDNSZones: mockDNS, - VirtualNetworkLinks: &mockVirtualNetworkLinks{}, - RecordSets: mockRecords, - } - - result, err := r.Reconcile(log.IntoContext(t.Context(), testr.New(t)), ctrl.Request{ - NamespacedName: types.NamespacedName{Name: "private-router", Namespace: "test-ns"}, - }) - - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(result.RequeueAfter).ToNot(BeZero(), "should requeue for drift detection") - - // The base domain zone should be created since there's no shadowing - g.Expect(mockDNS.lastZoneName).To(Equal("custom-dns.example.com"), - "last zone should be the base domain zone") - - // *.apps should only appear in the hypershift.local zone, NOT the base domain zone - baseDomainRecords := mockRecords.createdRecordsByZone["custom-dns.example.com"] - g.Expect(baseDomainRecords).ToNot(ContainElement("*.apps"), - "*.apps wildcard should NOT be added to the base domain zone when there is no shadowing") - - // Verify the DNS condition is set to True (success) - updated := &hyperv1.AzurePrivateLinkService{} - err = fakeClient.Get(t.Context(), types.NamespacedName{Name: "private-router", Namespace: "test-ns"}, updated) - g.Expect(err).ToNot(HaveOccurred()) - - dnsCondition := meta.FindStatusCondition(updated.Status.Conditions, string(hyperv1.AzurePrivateDNSAvailable)) - g.Expect(dnsCondition).ToNot(BeNil(), "DNS condition should be set") - g.Expect(dnsCondition.Status).To(Equal(metav1.ConditionTrue)) -} - func TestReconcile_WhenAvailableConditionPatchFails_ItShouldReturnError(t *testing.T) { t.Parallel() g := NewGomegaWithT(t) diff --git a/control-plane-operator/controllers/gcpprivateserviceconnect/dns.go b/control-plane-operator/controllers/gcpprivateserviceconnect/dns.go index b3050652affc..126261156f2e 100644 --- a/control-plane-operator/controllers/gcpprivateserviceconnect/dns.go +++ b/control-plane-operator/controllers/gcpprivateserviceconnect/dns.go @@ -17,6 +17,7 @@ package gcpprivateserviceconnect import ( "context" + "errors" "fmt" "os" "regexp" @@ -72,7 +73,8 @@ func ensureDNSDot(name string) string { // isNotFound checks if a GCP API error indicates a resource was not found. func isNotFound(err error) bool { - if apiErr, ok := err.(*googleapi.Error); ok { + var apiErr *googleapi.Error + if errors.As(err, &apiErr) { return apiErr.Code == 404 // HTTP 404 Not Found } // Fallback: check if error message contains "404" or "not found" patterns diff --git a/control-plane-operator/controllers/gcpprivateserviceconnect/dns_test.go b/control-plane-operator/controllers/gcpprivateserviceconnect/dns_test.go index 88b5c7b9e481..a6f2a706a021 100644 --- a/control-plane-operator/controllers/gcpprivateserviceconnect/dns_test.go +++ b/control-plane-operator/controllers/gcpprivateserviceconnect/dns_test.go @@ -2,6 +2,7 @@ package gcpprivateserviceconnect import ( "errors" + "fmt" "testing" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" @@ -103,6 +104,16 @@ func TestIsNotFound(t *testing.T) { err: errors.New("RESOURCE NOT FOUND"), expected: true, }, + { + name: "When error is a wrapped googleapi 404 it should return true", + err: fmt.Errorf("operation failed: %w", &googleapi.Error{Code: 404, Message: "not found"}), + expected: true, + }, + { + name: "When error is a wrapped googleapi 500 it should return false", + err: fmt.Errorf("operation failed: %w", &googleapi.Error{Code: 500, Message: "internal error"}), + expected: false, + }, { name: "When error is generic without 404 it should return false", err: errors.New("connection timeout"), diff --git a/control-plane-operator/controllers/gcpprivateserviceconnect/psc_endpoint_controller.go b/control-plane-operator/controllers/gcpprivateserviceconnect/psc_endpoint_controller.go index c1918c4a8dec..f3168810ad12 100644 --- a/control-plane-operator/controllers/gcpprivateserviceconnect/psc_endpoint_controller.go +++ b/control-plane-operator/controllers/gcpprivateserviceconnect/psc_endpoint_controller.go @@ -899,7 +899,8 @@ func (r *GCPPrivateServiceConnectReconciler) handleGCPError(ctx context.Context, var requeueAfter time.Duration var message string - if googleErr, ok := err.(*googleapi.Error); ok { + var googleErr *googleapi.Error + if errors.As(err, &googleErr) { switch googleErr.Code { case 429: // Rate limit requeueAfter = time.Minute * 5 @@ -943,7 +944,8 @@ func (r *GCPPrivateServiceConnectReconciler) handleGCPError(ctx context.Context, // isNotFoundError checks if the error is a GCP "not found" error func isNotFoundError(err error) bool { - if googleErr, ok := err.(*googleapi.Error); ok { + var googleErr *googleapi.Error + if errors.As(err, &googleErr) { return googleErr.Code == 404 } return false diff --git a/control-plane-operator/controllers/gcpprivateserviceconnect/psc_endpoint_controller_test.go b/control-plane-operator/controllers/gcpprivateserviceconnect/psc_endpoint_controller_test.go index 01b58a9e3594..70641acdc819 100644 --- a/control-plane-operator/controllers/gcpprivateserviceconnect/psc_endpoint_controller_test.go +++ b/control-plane-operator/controllers/gcpprivateserviceconnect/psc_endpoint_controller_test.go @@ -3,6 +3,7 @@ package gcpprivateserviceconnect import ( "context" "errors" + "fmt" "testing" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" @@ -21,6 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/api/googleapi" ) func TestConstructEndpointName(t *testing.T) { @@ -257,8 +259,26 @@ func TestIsNotFoundError(t *testing.T) { err: assert.AnError, expected: false, }, - // Note: We can't easily test the GCP API error case without importing the full GCP SDK - // and creating mock errors, but the logic is straightforward + { + name: "When given a GCP 404 error it should return true", + err: &googleapi.Error{Code: 404, Message: "not found"}, + expected: true, + }, + { + name: "When given a GCP 500 error it should return false", + err: &googleapi.Error{Code: 500, Message: "internal error"}, + expected: false, + }, + { + name: "When given a wrapped GCP 404 error it should return true", + err: fmt.Errorf("operation failed: %w", &googleapi.Error{Code: 404, Message: "not found"}), + expected: true, + }, + { + name: "When given a wrapped GCP 500 error it should return false", + err: fmt.Errorf("operation failed: %w", &googleapi.Error{Code: 500, Message: "internal error"}), + expected: false, + }, } for _, tt := range tests { diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/kubevirt.go b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/kubevirt.go index 69348ff19ba9..0dabe1c01e77 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/kubevirt.go +++ b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/kubevirt.go @@ -135,9 +135,17 @@ func reconcileCustomTenantStorageClass(sc *storagev1.StorageClass, infraSCName s } func reconcileDefaultTenantStorageClass(sc *storagev1.StorageClass) error { - sc.Annotations = map[string]string{ - "storageclass.kubernetes.io/is-default-class": "true", + if sc.Annotations == nil { + sc.Annotations = map[string]string{} } + + // Only set the default storage class when the annotation doesn't exist. + // This allows users to set is-default-class=false if they wish to + // install their own CSI drivers and choose a different default. + if _, exists := sc.Annotations["storageclass.kubernetes.io/is-default-class"]; !exists { + sc.Annotations["storageclass.kubernetes.io/is-default-class"] = "true" + } + sc.Provisioner = "csi.kubevirt.io" sc.Parameters = map[string]string{ "bus": "scsi", diff --git a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/kubevirt_test.go b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/kubevirt_test.go index f27877ec50fc..bba7826eb737 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/kubevirt_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/csi/kubevirt/kubevirt_test.go @@ -111,7 +111,7 @@ func TestReconcileDefaultTenantStorageClass(t *testing.T) { test func(t *testing.T, g Gomega) }{ { - name: "When called, it should set the is-default-class annotation to true", + name: "When annotation does not exist, it should set the is-default-class annotation to true", test: func(t *testing.T, g Gomega) { sc := &storagev1.StorageClass{} err := reconcileDefaultTenantStorageClass(sc) @@ -119,6 +119,43 @@ func TestReconcileDefaultTenantStorageClass(t *testing.T) { g.Expect(sc.Annotations).To(HaveKeyWithValue("storageclass.kubernetes.io/is-default-class", "true")) }, }, + { + name: "When is-default-class annotation is already set to false, it should preserve the user's choice", + test: func(t *testing.T, g Gomega) { + sc := &storagev1.StorageClass{} + sc.Annotations = map[string]string{ + "storageclass.kubernetes.io/is-default-class": "false", + } + err := reconcileDefaultTenantStorageClass(sc) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(sc.Annotations).To(HaveKeyWithValue("storageclass.kubernetes.io/is-default-class", "false")) + }, + }, + { + name: "When is-default-class annotation is already set to true, it should preserve the value", + test: func(t *testing.T, g Gomega) { + sc := &storagev1.StorageClass{} + sc.Annotations = map[string]string{ + "storageclass.kubernetes.io/is-default-class": "true", + } + err := reconcileDefaultTenantStorageClass(sc) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(sc.Annotations).To(HaveKeyWithValue("storageclass.kubernetes.io/is-default-class", "true")) + }, + }, + { + name: "When other annotations exist, it should preserve them and add is-default-class", + test: func(t *testing.T, g Gomega) { + sc := &storagev1.StorageClass{} + sc.Annotations = map[string]string{ + "some-other-annotation": "some-value", + } + err := reconcileDefaultTenantStorageClass(sc) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(sc.Annotations).To(HaveKeyWithValue("storageclass.kubernetes.io/is-default-class", "true")) + g.Expect(sc.Annotations).To(HaveKeyWithValue("some-other-annotation", "some-value")) + }, + }, { name: "When called, it should set the provisioner to csi.kubevirt.io", test: func(t *testing.T, g Gomega) { diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 629ab0e2b606..9337ae39d119 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -865,7 +865,7 @@ func (r *HostedControlPlaneReconciler) reconcileControlPlaneVersionStatus(ctx co // Persist the Partial entry before returning the error. hostedControlPlane.Status.ControlPlaneVersion = ensureControlPlaneVersionPartial(hostedControlPlane, clk, releaseImage.Version(), resolvedImage) if patchErr := r.Client.Status().Patch(ctx, hostedControlPlane, client.MergeFromWithOptions(originalHostedControlPlane, client.MergeFromWithOptimisticLock{})); patchErr != nil { - return fmt.Errorf("failed to patch status after component list failure: %w (list error: %v)", patchErr, listErr) + return fmt.Errorf("failed to patch status after component list failure: %w (list error: %w)", patchErr, listErr) } return fmt.Errorf("failed to list control plane components for version reconciliation: %w", listErr) } @@ -944,9 +944,9 @@ func (r *HostedControlPlaneReconciler) healthCheckKASLoadBalancers(ctx context.C // When the cluster is private, checking the load balancers will depend on whether the load balancer is // using the right subnets. To avoid uncertainty, we'll limit the check to the service endpoint. if hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform { - return healthCheckKASEndpoint(manifests.KubeAPIServerService("").Name, config.KASSVCIBMCloudPort) + return healthCheckKASEndpoint(ctx, manifests.KubeAPIServerService("").Name, config.KASSVCIBMCloudPort) } - return healthCheckKASEndpoint(manifests.KubeAPIServerService("").Name, config.KASSVCPort) + return healthCheckKASEndpoint(ctx, manifests.KubeAPIServerService("").Name, config.KASSVCPort) case serviceStrategy.Type == hyperv1.Route: if hcp.Spec.Platform.Type != hyperv1.IBMCloudPlatform { externalRoute := manifests.KubeAPIServerExternalPublicRoute(hcp.Namespace) @@ -958,7 +958,7 @@ func (r *HostedControlPlaneReconciler) healthCheckKASLoadBalancers(ctx context.C if err != nil { return err } - return healthCheckKASEndpoint(endpoint, port) + return healthCheckKASEndpoint(ctx, endpoint, port) } case serviceStrategy.Type == hyperv1.LoadBalancer: svc := manifests.KubeAPIServerService(hcp.Namespace) @@ -987,20 +987,25 @@ func (r *HostedControlPlaneReconciler) healthCheckKASLoadBalancers(ctx context.C } else if LBIngress.IP != "" { ingressPoint = LBIngress.IP } - return healthCheckKASEndpoint(ingressPoint, port) + return healthCheckKASEndpoint(ctx, ingressPoint, port) } return nil } -func healthCheckKASEndpoint(ingressPoint string, port int) error { +func healthCheckKASEndpoint(ctx context.Context, ingressPoint string, port int) error { healthEndpoint := fmt.Sprintf("https://%s:%d/healthz", ingressPoint, port) httpClient := util.InsecureHTTPClient() httpClient.Timeout = 10 * time.Second - resp, err := httpClient.Get(healthEndpoint) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, healthEndpoint, nil) if err != nil { return err } + resp, err := httpClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return fmt.Errorf("APIServer endpoint %s is not healthy", ingressPoint) @@ -1454,14 +1459,14 @@ func (r *HostedControlPlaneReconciler) reconcileKonnectivityCerts(ctx context.Co if _, err := createOrUpdate(ctx, r, konnectivitySigner, func() error { return pki.ReconcileKonnectivitySignerSecret(konnectivitySigner, p.OwnerRef) }); err != nil { - return fmt.Errorf("failed to reconcile konnectivity signer secret: %v", err) + return fmt.Errorf("failed to reconcile konnectivity signer secret: %w", err) } konnectivityCACM := manifests.KonnectivityCAConfigMap(hcp.Namespace) if _, err := createOrUpdate(ctx, r, konnectivityCACM, func() error { return pki.ReconcileKonnectivityConfigMap(konnectivityCACM, p.OwnerRef, konnectivitySigner) }); err != nil { - return fmt.Errorf("failed to reconcile konnectivity CA config map: %v", err) + return fmt.Errorf("failed to reconcile konnectivity CA config map: %w", err) } konnectivityServerSecret := manifests.KonnectivityServerSecret(hcp.Namespace) @@ -1998,10 +2003,11 @@ func (r *HostedControlPlaneReconciler) reconcileValidIDPConfigurationCondition(c Message: fmt.Sprintf("failed to initialize identity providers: %v", err), } } - // Update the condition on the HCP if it has changed + // Patch the condition on the HCP if it has changed + originalHCP := hcp.DeepCopy() if meta.SetStatusCondition(&hcp.Status.Conditions, new) { - if err := r.Status().Update(ctx, hcp); err != nil { - return fmt.Errorf("failed to update valid IDP configuration condition: %w", err) + if err := r.Status().Patch(ctx, hcp, client.MergeFromWithOptions(originalHCP, client.MergeFromWithOptimisticLock{})); err != nil { + return fmt.Errorf("failed to patch valid IDP configuration condition: %w", err) } } return nil @@ -2133,7 +2139,7 @@ func (r *HostedControlPlaneReconciler) reconcileCoreIgnitionConfig(ctx context.C } data, hasSSHKeyData := sshKeySecret.Data["id_rsa.pub"] if !hasSSHKeyData { - return fmt.Errorf("SSH key secret secret %s is missing the id_rsa.pub key", hcp.Spec.SSHKey.Name) + return fmt.Errorf("SSH key secret %s is missing the id_rsa.pub key", hcp.Spec.SSHKey.Name) } sshKey = string(data) } @@ -2518,14 +2524,15 @@ func (r *HostedControlPlaneReconciler) removeCloudResources(ctx context.Context, if resourcesDestroyedCond != nil && resourcesDestroyedCond.Message != "" { message = fmt.Sprintf("%s (last status: %s)", message, resourcesDestroyedCond.Message) } + originalHCP := hcp.DeepCopy() meta.SetStatusCondition(&hcp.Status.Conditions, metav1.Condition{ Type: string(hyperv1.CloudResourcesDestroyed), Status: metav1.ConditionFalse, Reason: string(hyperv1.CloudResourcesDeletionTimedOutReason), Message: message, }) - if err := r.Status().Update(ctx, hcp); err != nil { - return false, fmt.Errorf("failed to update cloud resources destroyed condition: %w", err) + if err := r.Status().Patch(ctx, hcp, client.MergeFromWithOptions(originalHCP, client.MergeFromWithOptimisticLock{})); err != nil { + return false, fmt.Errorf("failed to patch cloud resources destroyed condition: %w", err) } return true, nil } @@ -2550,6 +2557,7 @@ func (r *HostedControlPlaneReconciler) removeCloudResources(ctx context.Context, return false, nil } if cvoScaledDownCond == nil || cvoScaledDownCond.Status != metav1.ConditionTrue { + originalHCP := hcp.DeepCopy() cvoScaledDownCond = &metav1.Condition{ Type: string(hyperv1.CVOScaledDown), Status: metav1.ConditionTrue, @@ -2557,8 +2565,8 @@ func (r *HostedControlPlaneReconciler) removeCloudResources(ctx context.Context, LastTransitionTime: metav1.Now(), } meta.SetStatusCondition(&hcp.Status.Conditions, *cvoScaledDownCond) - if err := r.Status().Update(ctx, hcp); err != nil { - return false, fmt.Errorf("failed to set CVO scaled down condition: %w", err) + if err := r.Status().Patch(ctx, hcp, client.MergeFromWithOptions(originalHCP, client.MergeFromWithOptimisticLock{})); err != nil { + return false, fmt.Errorf("failed to patch CVO scaled down condition: %w", err) } } return false, nil @@ -3181,11 +3189,11 @@ func (r *HostedControlPlaneReconciler) verifyResourceGroupLocationsMatch(ctx con certPath := config.ManagedAzureCertificatePath + hcp.Spec.Platform.Azure.AzureAuthenticationConfig.ManagedIdentities.ControlPlane.ControlPlaneOperator.CredentialsSecretName cloudConfig, err := hyperazureutil.GetAzureCloudConfiguration(hcp.Spec.Platform.Azure.Cloud) if err != nil { - return fmt.Errorf("failed to get Azure cloud configuration: %v", err) + return fmt.Errorf("failed to get Azure cloud configuration: %w", err) } creds, err = dataplane.NewUserAssignedIdentityCredential(ctx, certPath, dataplane.WithClientOpts(azcore.ClientOptions{Cloud: cloudConfig}), dataplane.WithLogger(&log)) if err != nil { - return fmt.Errorf("failed to create azure creds to verify resource group locations: %v", err) + return fmt.Errorf("failed to create azure creds to verify resource group locations: %w", err) } r.cpoAzureCredentialsLoaded.Store(key, creds) @@ -3201,17 +3209,17 @@ func (r *HostedControlPlaneReconciler) verifyResourceGroupLocationsMatch(ctx con // Retrieve full vnet information from the VNET ID vnet, err := hyperazureutil.GetVnetInfoFromVnetID(ctx, hcp.Spec.Platform.Azure.VnetID, hcp.Spec.Platform.Azure.SubscriptionID, creds, cloudName) if err != nil { - return fmt.Errorf("failed to get vnet info to verify its location: %v", err) + return fmt.Errorf("failed to get vnet info to verify its location: %w", err) } // Retrieve full network security group information from the network security group ID nsg, err := hyperazureutil.GetNetworkSecurityGroupInfo(ctx, hcp.Spec.Platform.Azure.SecurityGroupID, hcp.Spec.Platform.Azure.SubscriptionID, creds, cloudName) if err != nil { - return fmt.Errorf("failed to get network security group info to verify its location: %v", err) + return fmt.Errorf("failed to get network security group info to verify its location: %w", err) } // Retrieve full resource group information from the resource group name rg, err := hyperazureutil.GetResourceGroupInfo(ctx, hcp.Spec.Platform.Azure.ResourceGroupName, hcp.Spec.Platform.Azure.SubscriptionID, creds, cloudName) if err != nil { - return fmt.Errorf("failed to get resource group info to verify its location: %v", err) + return fmt.Errorf("failed to get resource group info to verify its location: %w", err) } // Verify the vnet resource group location, network security group resource group location, and the managed resource group location match if ptr.Deref(vnet.Location, "") != ptr.Deref(nsg.Location, "") || ptr.Deref(nsg.Location, "") != ptr.Deref(rg.Location, "") { diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go index f6c060f3e055..7fbcdb4032c0 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller_test.go @@ -4,7 +4,11 @@ import ( "context" _ "embed" "fmt" + "net" + "net/http" + "net/http/httptest" "sort" + "strconv" "strings" "testing" "time" @@ -891,7 +895,7 @@ func TestSetKASCustomKubeconfigStatus(t *testing.T) { c := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(objs...).WithStatusSubresource(&hyperv1.HostedControlPlane{}).Build() err := setKASCustomKubeconfigStatus(ctx, hcp, c) - g.Expect(err).To(BeNil(), fmt.Errorf("error setting custom kubeconfig status failed: %v", err)) + g.Expect(err).To(BeNil(), fmt.Errorf("error setting custom kubeconfig status failed: %w", err)) g.Expect(hcp.Status.CustomKubeconfig).To(Equal(tc.expectedStatus)) }) } @@ -4426,6 +4430,78 @@ func TestReconcileDeletion(t *testing.T) { } } +func TestHealthCheckKASEndpoint(t *testing.T) { + t.Parallel() + tests := []struct { + name string + handler http.HandlerFunc + cancelCtx bool + wantErr bool + errSubstr string + }{ + { + name: "When endpoint returns 200 OK, it should succeed", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }, + }, + { + name: "When endpoint returns 503, it should return an unhealthy error", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + }, + wantErr: true, + errSubstr: "is not healthy", + }, + { + name: "When context is canceled, it should return an error", + handler: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }, + cancelCtx: true, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + server := httptest.NewTLSServer(tt.handler) + defer server.Close() + + host, portStr, err := net.SplitHostPort(server.Listener.Addr().String()) + g.Expect(err).ToNot(HaveOccurred()) + port, err := strconv.Atoi(portStr) + g.Expect(err).ToNot(HaveOccurred()) + + ctx := t.Context() + if tt.cancelCtx { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + } + + err = healthCheckKASEndpoint(ctx, host, port) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + if tt.errSubstr != "" { + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } + + t.Run("When the ingress point contains invalid characters, it should return a request creation error", func(t *testing.T) { + g := NewWithT(t) + err := healthCheckKASEndpoint(t.Context(), "host\x7f", 443) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("invalid control character")) + }) +} + // Compile-time assertion that fakeVersionImageMetadataProvider satisfies the interface. var _ util.ImageMetadataProvider = &fakeVersionImageMetadataProvider{} diff --git a/control-plane-operator/controllers/hostedcontrolplane/infra/infra_test.go b/control-plane-operator/controllers/hostedcontrolplane/infra/infra_test.go index f344b921cc4c..1636e722d9ba 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/infra/infra_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/infra/infra_test.go @@ -1475,7 +1475,8 @@ func TestReconcileHCPRouterServices(t *testing.T) { Name: "router", Namespace: namespace, Annotations: map[string]string{ - "service.beta.kubernetes.io/aws-load-balancer-type": "nlb", + "service.beta.kubernetes.io/aws-load-balancer-type": "nlb", + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing", }, Labels: map[string]string{"app": "private-router"}, }, @@ -1497,6 +1498,7 @@ func TestReconcileHCPRouterServices(t *testing.T) { return publicService(append(m, func(s *corev1.Service) { s.Name = "private-router" s.Annotations["service.beta.kubernetes.io/aws-load-balancer-internal"] = "true" + delete(s.Annotations, "service.beta.kubernetes.io/aws-load-balancer-scheme") })...) } withCrossZoneAnnotation := func(svc *corev1.Service) { diff --git a/control-plane-operator/controllers/hostedcontrolplane/infra/testdata/zz_fixture_TestReconcileInfrastructure_AWS_PublicAndPrivate_Route.yaml b/control-plane-operator/controllers/hostedcontrolplane/infra/testdata/zz_fixture_TestReconcileInfrastructure_AWS_PublicAndPrivate_Route.yaml index 452e8b2d7e15..a530d255d632 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/infra/testdata/zz_fixture_TestReconcileInfrastructure_AWS_PublicAndPrivate_Route.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/infra/testdata/zz_fixture_TestReconcileInfrastructure_AWS_PublicAndPrivate_Route.yaml @@ -286,6 +286,7 @@ services: annotations: service.beta.kubernetes.io/aws-load-balancer-attributes: load_balancing.cross_zone.enabled=true service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled: "true" + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing service.beta.kubernetes.io/aws-load-balancer-type: nlb labels: app: private-router diff --git a/control-plane-operator/controllers/hostedcontrolplane/infra/testdata/zz_fixture_TestReconcileInfrastructure_AWS_Public_Route.yaml b/control-plane-operator/controllers/hostedcontrolplane/infra/testdata/zz_fixture_TestReconcileInfrastructure_AWS_Public_Route.yaml index 0b603a11a1b4..b0eedd091b3b 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/infra/testdata/zz_fixture_TestReconcileInfrastructure_AWS_Public_Route.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/infra/testdata/zz_fixture_TestReconcileInfrastructure_AWS_Public_Route.yaml @@ -239,6 +239,7 @@ services: loadBalancer: {} - metadata: annotations: + service.beta.kubernetes.io/aws-load-balancer-scheme: internet-facing service.beta.kubernetes.io/aws-load-balancer-type: nlb labels: app: private-router diff --git a/control-plane-operator/controllers/hostedcontrolplane/ingress/router.go b/control-plane-operator/controllers/hostedcontrolplane/ingress/router.go index cc0e845217ec..8421a0685502 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/ingress/router.go +++ b/control-plane-operator/controllers/hostedcontrolplane/ingress/router.go @@ -29,6 +29,15 @@ func ReconcileRouterService(svc *corev1.Service, internal, crossZoneLoadBalancin svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-type"] = "nlb" if internal { svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-internal"] = "true" + delete(svc.Annotations, "service.beta.kubernetes.io/aws-load-balancer-scheme") + } else { + delete(svc.Annotations, "service.beta.kubernetes.io/aws-load-balancer-internal") + // The AWS Load Balancer Controller (used on EKS) defaults to scheme=internal: + // https://kubernetes-sigs.github.io/aws-load-balancer-controller/v2.4/guide/service/annotations/ + // The in-tree AWS cloud provider (used on OpenShift) defaults to internet-facing: + // https://cloud-provider-aws.sigs.k8s.io/service_controller/ + // Set the annotation explicitly so the public router works on both. + svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-scheme"] = "internet-facing" } if crossZoneLoadBalancingEnabled { // In-tree AWS cloud provider annotation for cross-zone load balancing (OpenShift management clusters). diff --git a/control-plane-operator/controllers/hostedcontrolplane/ingress/router_test.go b/control-plane-operator/controllers/hostedcontrolplane/ingress/router_test.go index 81dfec25637a..f2df7f8de0df 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/ingress/router_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/ingress/router_test.go @@ -3,6 +3,8 @@ package ingress import ( "testing" + . "github.com/onsi/gomega" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/support/netutil" @@ -56,6 +58,69 @@ func TestReconcileRouterServiceAnnotations(t *testing.T) { } } +// When reconciling an external (non-internal) AWS router service +// it should set aws-load-balancer-scheme to internet-facing. +func TestReconcileRouterService_WhenExternalAWS_ItShouldSetInternetFacingScheme(t *testing.T) { + g := NewGomegaWithT(t) + + hcp := &hyperv1.HostedControlPlane{} + hcp.Spec.Platform.Type = hyperv1.AWSPlatform + + svc := &corev1.Service{} + + err := ReconcileRouterService(svc, false /* internal */, true /* cross-zone */, hcp) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(svc.Annotations).To(HaveKeyWithValue( + "service.beta.kubernetes.io/aws-load-balancer-scheme", "internet-facing")) + g.Expect(svc.Annotations).ToNot(HaveKey( + "service.beta.kubernetes.io/aws-load-balancer-internal")) +} + +func TestReconcileRouterService_WhenSwitchingMode_ItShouldRemoveConflictingAnnotation(t *testing.T) { + tests := []struct { + name string + internal bool + preExisting map[string]string + expectKey string + expectValue string + expectAbsent string + }{ + { + name: "When switching from internal to external it should remove the internal annotation", + internal: false, + preExisting: map[string]string{"service.beta.kubernetes.io/aws-load-balancer-internal": "true"}, + expectKey: "service.beta.kubernetes.io/aws-load-balancer-scheme", + expectValue: "internet-facing", + expectAbsent: "service.beta.kubernetes.io/aws-load-balancer-internal", + }, + { + name: "When switching from external to internal it should remove the scheme annotation", + internal: true, + preExisting: map[string]string{"service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing"}, + expectKey: "service.beta.kubernetes.io/aws-load-balancer-internal", + expectValue: "true", + expectAbsent: "service.beta.kubernetes.io/aws-load-balancer-scheme", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewGomegaWithT(t) + + hcp := &hyperv1.HostedControlPlane{} + hcp.Spec.Platform.Type = hyperv1.AWSPlatform + + svc := &corev1.Service{} + svc.Annotations = tt.preExisting + + err := ReconcileRouterService(svc, tt.internal, true /* cross-zone */, hcp) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(svc.Annotations).To(HaveKeyWithValue(tt.expectKey, tt.expectValue)) + g.Expect(svc.Annotations).ToNot(HaveKey(tt.expectAbsent)) + }) + } +} + // Test that LoadBalancerSourceRanges is applied for external router services with allowedCIDRBlocks func TestReconcileRouterService_AppliesLoadBalancerSourceRanges(t *testing.T) { // Test case 1: External router service should have LoadBalancerSourceRanges set diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/auth.go b/control-plane-operator/controllers/hostedcontrolplane/kas/auth.go index ac14459a7f28..157b17a3a599 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas/auth.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/auth.go @@ -38,7 +38,7 @@ func GenerateAuthConfig(spec *configv1.AuthenticationSpec, ctx context.Context, for _, provider := range spec.OIDCProviders { jwt, err := generateJWTForProvider(ctx, provider, c, namespace) if err != nil { - return nil, fmt.Errorf("generating JWT authenticator for provider %q: %v", provider.Name, err) + return nil, fmt.Errorf("generating JWT authenticator for provider %q: %w", provider.Name, err) } config.JWT = append(config.JWT, jwt) } @@ -50,17 +50,17 @@ func generateJWTForProvider(ctx context.Context, provider configv1.OIDCProvider, issuer, err := generateIssuer(ctx, provider.Issuer, client, namespace) if err != nil { - return out, fmt.Errorf("generating issuer: %v", err) + return out, fmt.Errorf("generating issuer: %w", err) } claimMappings, err := generateClaimMappings(provider.ClaimMappings, issuer.URL) if err != nil { - return out, fmt.Errorf("generating claim mappings: %v", err) + return out, fmt.Errorf("generating claim mappings: %w", err) } claimValidationRules, err := generateClaimValidationRules(provider.ClaimValidationRules...) if err != nil { - return out, fmt.Errorf("generating claim validation rules: %v", err) + return out, fmt.Errorf("generating claim validation rules: %w", err) } out.Issuer = issuer @@ -83,7 +83,7 @@ func generateIssuer(ctx context.Context, issuer configv1.TokenIssuer, client crc if len(issuer.CertificateAuthority.Name) > 0 { ca, err := getCertificateAuthorityFromConfigMap(ctx, client, issuer.CertificateAuthority.Name, namespace) if err != nil { - return out, fmt.Errorf("getting certificate authority for issuer: %v", err) + return out, fmt.Errorf("getting certificate authority for issuer: %w", err) } out.CertificateAuthority = ca } @@ -110,7 +110,7 @@ func generateClaimMappings(claimMappings configv1.TokenClaimMappings, issuerURL username, err := generateUsernameClaimMapping(claimMappings.Username, issuerURL) if err != nil { - return out, fmt.Errorf("generating username claim mapping: %v", err) + return out, fmt.Errorf("generating username claim mapping: %w", err) } groups := generateGroupsClaimMapping(claimMappings.Groups) @@ -121,12 +121,12 @@ func generateClaimMappings(claimMappings configv1.TokenClaimMappings, issuerURL if featuregates.Gate().Enabled(featuregates.ExternalOIDCWithUIDAndExtraClaimMappings) { uid, err := generateUIDClaimMapping(claimMappings.UID) if err != nil { - return out, fmt.Errorf("generating uid claim mapping: %v", err) + return out, fmt.Errorf("generating uid claim mapping: %w", err) } extras, err := generateExtraClaimMapping(claimMappings.Extra...) if err != nil { - return out, fmt.Errorf("generating extra claim mapping: %v", err) + return out, fmt.Errorf("generating extra claim mapping: %w", err) } out.UID = uid @@ -193,7 +193,7 @@ func generateUIDClaimMapping(uid *configv1.TokenClaimOrExpressionMapping) (Claim case uid.Expression != "" && uid.Claim == "": err := validateClaimMappingExpression(uid.Expression) if err != nil { - return out, fmt.Errorf("validating CEL expression: %v", err) + return out, fmt.Errorf("validating CEL expression: %w", err) } out.Expression = uid.Expression case uid.Claim != "" && uid.Expression != "": @@ -233,7 +233,7 @@ func generateExtraMapping(extra configv1.ExtraMapping) (ExtraMapping, error) { err := validateExtraMappingExpression(extra.ValueExpression) if err != nil { - return out, fmt.Errorf("validating valueExpression: %v", err) + return out, fmt.Errorf("validating valueExpression: %w", err) } out.Key = extra.Key @@ -295,13 +295,13 @@ func validateExtraMappingExpression(expression string) error { func HCPAuthConfigToAPIServerAuthConfig(authConfig *AuthenticationConfiguration) (*apiserver.AuthenticationConfiguration, error) { outBytes, err := json.Marshal(authConfig) if err != nil { - return nil, fmt.Errorf("marshaling HCP auth config to JSON: %v", err) + return nil, fmt.Errorf("marshaling HCP auth config to JSON: %w", err) } apiserverAuthConfig := &apiserver.AuthenticationConfiguration{} err = json.Unmarshal(outBytes, apiserverAuthConfig) if err != nil { - return nil, fmt.Errorf("unmarshalling HCP auth config JSON to apiserver auth config: %v", err) + return nil, fmt.Errorf("unmarshalling HCP auth config JSON to apiserver auth config: %w", err) } return apiserverAuthConfig, nil diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas/auth_test.go b/control-plane-operator/controllers/hostedcontrolplane/kas/auth_test.go new file mode 100644 index 000000000000..6c05da4451a2 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/kas/auth_test.go @@ -0,0 +1,652 @@ +package kas + +import ( + "testing" + + . "github.com/onsi/gomega" + + configv1 "github.com/openshift/api/config/v1" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + + crclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestGenerateAuthConfig(t *testing.T) { + t.Parallel() + tests := []struct { + name string + spec *configv1.AuthenticationSpec + wantErr bool + errSubstr string + }{ + { + name: "When spec is nil, it should return empty config without error", + spec: nil, + }, + { + name: "When issuer references a missing CA ConfigMap, it should return a wrapped error", + spec: &configv1.AuthenticationSpec{ + OIDCProviders: []configv1.OIDCProvider{ + { + Name: "test-provider", + Issuer: configv1.TokenIssuer{ + URL: "https://test.example.com", + Audiences: []configv1.TokenAudience{"test-audience"}, + CertificateAuthority: configv1.ConfigMapNameReference{ + Name: "nonexistent-ca-configmap", + }, + }, + ClaimMappings: configv1.TokenClaimMappings{ + Username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.NoPrefix, + }, + }, + }, + }, + }, + wantErr: true, + errSubstr: "generating JWT authenticator for provider", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + c := fake.NewClientBuilder().Build() + _, err := GenerateAuthConfig(tt.spec, t.Context(), c, "test-namespace") + + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + if tt.errSubstr != "" { + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateIssuer(t *testing.T) { + t.Parallel() + tests := []struct { + name string + issuer configv1.TokenIssuer + objects []crclient.Object + wantErr bool + errSubstr string + }{ + { + name: "When issuer has no CA reference, it should return issuer without error", + issuer: configv1.TokenIssuer{ + URL: "https://issuer.example.com", + Audiences: []configv1.TokenAudience{"aud1", "aud2"}, + }, + }, + { + name: "When issuer has CA reference and ConfigMap exists, it should return issuer with CA", + issuer: configv1.TokenIssuer{ + URL: "https://issuer.example.com", + Audiences: []configv1.TokenAudience{"aud1"}, + CertificateAuthority: configv1.ConfigMapNameReference{ + Name: "test-ca", + }, + }, + objects: []crclient.Object{ + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-ca", Namespace: "test-ns"}, + Data: map[string]string{"ca-bundle.crt": "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----"}, + }, + }, + }, + { + name: "When issuer has CA reference but ConfigMap is missing, it should return a wrapped error", + issuer: configv1.TokenIssuer{ + URL: "https://issuer.example.com", + Audiences: []configv1.TokenAudience{"aud1"}, + CertificateAuthority: configv1.ConfigMapNameReference{ + Name: "nonexistent", + }, + }, + wantErr: true, + errSubstr: "getting certificate authority for issuer", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + c := fake.NewClientBuilder().WithObjects(tt.objects...).Build() + _, err := generateIssuer(t.Context(), tt.issuer, c, "test-ns") + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGetCertificateAuthorityFromConfigMap(t *testing.T) { + t.Parallel() + tests := []struct { + name string + caName string + objects []crclient.Object + wantErr bool + errSubstr string + }{ + { + name: "When ConfigMap does not exist, it should return a wrapped error", + caName: "nonexistent", + wantErr: true, + errSubstr: "failed to get issuer certificate authority configmap", + }, + { + name: "When ConfigMap exists but lacks ca-bundle.crt key, it should return an error", + caName: "test-ca", + objects: []crclient.Object{ + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-ca", Namespace: "test-ns"}, + Data: map[string]string{"wrong-key": "data"}, + }, + }, + wantErr: true, + errSubstr: "does not contain key", + }, + { + name: "When ConfigMap exists with ca-bundle.crt key, it should return the CA data", + caName: "test-ca", + objects: []crclient.Object{ + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "test-ca", Namespace: "test-ns"}, + Data: map[string]string{"ca-bundle.crt": "-----BEGIN CERTIFICATE-----\ntest\n-----END CERTIFICATE-----"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + c := fake.NewClientBuilder().WithObjects(tt.objects...).Build() + _, err := getCertificateAuthorityFromConfigMap(t.Context(), c, tt.caName, "test-ns") + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateJWTForProvider(t *testing.T) { + t.Parallel() + tests := []struct { + name string + provider configv1.OIDCProvider + wantErr bool + errSubstr string + }{ + { + name: "When provider has valid issuer and claim mappings, it should return JWT without error", + provider: configv1.OIDCProvider{ + Name: "test-provider", + Issuer: configv1.TokenIssuer{ + URL: "https://issuer.example.com", + Audiences: []configv1.TokenAudience{"test-audience"}, + }, + ClaimMappings: configv1.TokenClaimMappings{ + Username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.NoPrefix, + }, + }, + }, + }, + { + name: "When claim mappings are invalid, it should return a wrapped error", + provider: configv1.OIDCProvider{ + Name: "test-provider", + Issuer: configv1.TokenIssuer{ + URL: "https://issuer.example.com", + Audiences: []configv1.TokenAudience{"test-audience"}, + }, + ClaimMappings: configv1.TokenClaimMappings{ + Username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.Prefix, + Prefix: nil, + }, + }, + }, + wantErr: true, + errSubstr: "generating claim mappings", + }, + { + name: "When claim validation rules are invalid, it should return a wrapped error", + provider: configv1.OIDCProvider{ + Name: "test-provider", + Issuer: configv1.TokenIssuer{ + URL: "https://issuer.example.com", + Audiences: []configv1.TokenAudience{"test-audience"}, + }, + ClaimMappings: configv1.TokenClaimMappings{ + Username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.NoPrefix, + }, + }, + ClaimValidationRules: []configv1.TokenClaimValidationRule{ + {Type: "InvalidType"}, + }, + }, + wantErr: true, + errSubstr: "generating claim validation rules", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + c := fake.NewClientBuilder().Build() + _, err := generateJWTForProvider(t.Context(), tt.provider, c, "test-namespace") + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateUsernameClaimMapping(t *testing.T) { + t.Parallel() + tests := []struct { + name string + username configv1.UsernameClaimMapping + issuerURL string + wantErr bool + errSubstr string + }{ + { + name: "When prefix policy is NoPrefix, it should return empty prefix", + username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.NoPrefix, + }, + issuerURL: "https://issuer.example.com", + }, + { + name: "When prefix policy is Prefix but prefix is nil, it should return an error", + username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.Prefix, + Prefix: nil, + }, + issuerURL: "https://issuer.example.com", + wantErr: true, + errSubstr: "no prefix is specified", + }, + { + name: "When prefix policy is Prefix with value, it should use that prefix", + username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.Prefix, + Prefix: &configv1.UsernamePrefix{PrefixString: "myprefix"}, + }, + issuerURL: "https://issuer.example.com", + }, + { + name: "When prefix policy is NoOpinion and claim is email, it should use empty prefix", + username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.NoOpinion, + }, + issuerURL: "https://issuer.example.com", + }, + { + name: "When prefix policy is NoOpinion and claim is not email, it should use issuer URL prefix", + username: configv1.UsernameClaimMapping{ + Claim: "sub", + PrefixPolicy: configv1.NoOpinion, + }, + issuerURL: "https://issuer.example.com", + }, + { + name: "When prefix policy is unknown, it should return an error", + username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: "InvalidPolicy", + }, + issuerURL: "https://issuer.example.com", + wantErr: true, + errSubstr: "unknown prefix policy", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + _, err := generateUsernameClaimMapping(tt.username, tt.issuerURL) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateClaimMappings(t *testing.T) { + t.Parallel() + tests := []struct { + name string + mappings configv1.TokenClaimMappings + issuerURL string + wantErr bool + errSubstr string + }{ + { + name: "When username mapping is valid, it should return mappings without error", + mappings: configv1.TokenClaimMappings{ + Username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.NoPrefix, + }, + }, + issuerURL: "https://issuer.example.com", + }, + { + name: "When username mapping has invalid prefix policy, it should return a wrapped error", + mappings: configv1.TokenClaimMappings{ + Username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.Prefix, + Prefix: nil, + }, + }, + issuerURL: "https://issuer.example.com", + wantErr: true, + errSubstr: "generating username claim mapping", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + _, err := generateClaimMappings(tt.mappings, tt.issuerURL) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateExtraMapping(t *testing.T) { + t.Parallel() + tests := []struct { + name string + extra configv1.ExtraMapping + wantErr bool + errSubstr string + }{ + { + name: "When valueExpression is valid, it should return mapping without error", + extra: configv1.ExtraMapping{ + Key: "example.com/foo", + ValueExpression: "claims.groups", + }, + }, + { + name: "When valueExpression is invalid CEL, it should return a wrapped error", + extra: configv1.ExtraMapping{ + Key: "example.com/foo", + ValueExpression: "this is not valid CEL !!!", + }, + wantErr: true, + errSubstr: "validating valueExpression", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + _, err := generateExtraMapping(tt.extra) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateUIDClaimMapping(t *testing.T) { + t.Parallel() + tests := []struct { + name string + uid *configv1.TokenClaimOrExpressionMapping + wantErr bool + errSubstr string + wantClaim string + }{ + { + name: "When uid is nil, it should default claim to sub", + uid: nil, + wantClaim: "sub", + }, + { + name: "When uid has only a claim, it should use that claim", + uid: &configv1.TokenClaimOrExpressionMapping{Claim: "email"}, + wantClaim: "email", + }, + { + name: "When uid has only a valid expression, it should use that expression", + uid: &configv1.TokenClaimOrExpressionMapping{Expression: "claims.sub"}, + }, + { + name: "When uid has an invalid CEL expression, it should return a wrapped error", + uid: &configv1.TokenClaimOrExpressionMapping{Expression: "invalid CEL !!!"}, + wantErr: true, + errSubstr: "validating CEL expression", + }, + { + name: "When uid has both claim and expression, it should return an error", + uid: &configv1.TokenClaimOrExpressionMapping{Claim: "sub", Expression: "claims.sub"}, + wantErr: true, + errSubstr: "uid mapping must set either claim or expression, not both", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + result, err := generateUIDClaimMapping(tt.uid) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + if tt.wantClaim != "" { + g.Expect(result.Claim).To(Equal(tt.wantClaim)) + } + } + }) + } +} + +func TestGenerateClaimValidationRule(t *testing.T) { + t.Parallel() + tests := []struct { + name string + rule configv1.TokenClaimValidationRule + wantErr bool + errSubstr string + }{ + { + name: "When type is RequiredClaim with valid required claim, it should return rule without error", + rule: configv1.TokenClaimValidationRule{ + Type: configv1.TokenValidationRuleTypeRequiredClaim, + RequiredClaim: &configv1.TokenRequiredClaim{ + Claim: "iss", + RequiredValue: "https://issuer.example.com", + }, + }, + }, + { + name: "When type is RequiredClaim but requiredClaim is nil, it should return an error", + rule: configv1.TokenClaimValidationRule{ + Type: configv1.TokenValidationRuleTypeRequiredClaim, + }, + wantErr: true, + errSubstr: "requiredClaim is not set", + }, + { + name: "When type is unknown, it should return an error", + rule: configv1.TokenClaimValidationRule{ + Type: "InvalidType", + }, + wantErr: true, + errSubstr: "unknown claimValidationRule type", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + _, err := generateClaimValidationRule(tt.rule) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateClaimValidationRules(t *testing.T) { + t.Parallel() + tests := []struct { + name string + rules []configv1.TokenClaimValidationRule + wantErr bool + wantCount int + }{ + { + name: "When all rules are valid, it should return rules without error", + rules: []configv1.TokenClaimValidationRule{ + { + Type: configv1.TokenValidationRuleTypeRequiredClaim, + RequiredClaim: &configv1.TokenRequiredClaim{ + Claim: "iss", + RequiredValue: "https://issuer.example.com", + }, + }, + }, + wantCount: 1, + }, + { + name: "When a rule is invalid, it should return an error", + rules: []configv1.TokenClaimValidationRule{ + { + Type: "InvalidType", + }, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + result, err := generateClaimValidationRules(tt.rules...) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(result).To(HaveLen(tt.wantCount)) + } + }) + } +} + +func TestGenerateExtraClaimMapping(t *testing.T) { + t.Parallel() + tests := []struct { + name string + extras []configv1.ExtraMapping + wantErr bool + wantCount int + }{ + { + name: "When all mappings are valid, it should return mappings without error", + extras: []configv1.ExtraMapping{ + {Key: "example.com/foo", ValueExpression: "claims.groups"}, + }, + wantCount: 1, + }, + { + name: "When a mapping has empty key, it should return an error", + extras: []configv1.ExtraMapping{ + {Key: "", ValueExpression: "claims.groups"}, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + result, err := generateExtraClaimMapping(tt.extras...) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(result).To(HaveLen(tt.wantCount)) + } + }) + } +} + +func TestHCPAuthConfigToAPIServerAuthConfig(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + input := &AuthenticationConfiguration{ + JWT: []JWTAuthenticator{ + { + Issuer: Issuer{ + URL: "https://issuer.example.com", + Audiences: []string{"test-audience"}, + }, + ClaimMappings: ClaimMappings{ + Username: PrefixedClaimOrExpression{ + Claim: "email", + Prefix: ptr.To(""), + }, + }, + }, + }, + } + + result, err := HCPAuthConfigToAPIServerAuthConfig(input) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(result.JWT).To(HaveLen(1)) + g.Expect(result.JWT[0].Issuer.URL).To(Equal("https://issuer.example.com")) +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go b/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go index a9c83f2d1682..ed2f1a724d5e 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go +++ b/control-plane-operator/controllers/hostedcontrolplane/kas_pki_setup.go @@ -30,7 +30,7 @@ func (r *HostedControlPlaneReconciler) setupKASClientSigners( } if _, err := createOrUpdate(ctx, r, s, applyFunc); err != nil { - return nil, fmt.Errorf("failed to reconcile secret '%s/%s': %v", s.Namespace, s.Name, err) + return nil, fmt.Errorf("failed to reconcile secret '%s/%s': %w", s.Namespace, s.Name, err) } return s, nil } @@ -41,7 +41,7 @@ func (r *HostedControlPlaneReconciler) setupKASClientSigners( } if _, err := createOrUpdate(ctx, r, target, applyFunc); err != nil { - return nil, fmt.Errorf("failed to reconcile secret '%s/%s': %v", target.Namespace, target.Name, err) + return nil, fmt.Errorf("failed to reconcile secret '%s/%s': %w", target.Namespace, target.Name, err) } return target, nil } diff --git a/control-plane-operator/controllers/hostedcontrolplane/oauth/idp_convert.go b/control-plane-operator/controllers/hostedcontrolplane/oauth/idp_convert.go index 5704c0798120..36a848fe668a 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/oauth/idp_convert.go +++ b/control-plane-operator/controllers/hostedcontrolplane/oauth/idp_convert.go @@ -104,7 +104,7 @@ func ConvertIdentityProviders(ctx context.Context, identityProviders []configv1. } data, err := convertProviderConfigToIDPData(ctx, &idp.IdentityProviderConfig, providerConfigOverride, i, volumeMountInfo, kclient, namespace, false) if err != nil { - errs = append(errs, fmt.Errorf("failed to apply IDP %s config: %v", idp.Name, err)) + errs = append(errs, fmt.Errorf("failed to apply IDP %s config: %w", idp.Name, err)) continue } converted = append(converted, @@ -329,7 +329,7 @@ func convertOpenIDIDP(ctx context.Context, providerConfig *configv1.IdentityProv // to allow challenge-issuing flow if it's available on the OIDC side challengeFlowsAllowed, err := checkOIDCPasswordGrantFlow(ctx, kclient, openIDProvider.URLs.Token, openIDConfig.ClientID, namespace, openIDConfig.CA, openIDConfig.ClientSecret, skipKonnectivityDialer) if err != nil { - return nil, fmt.Errorf("error attempting password grant flow: %v", err) + return nil, fmt.Errorf("error attempting password grant flow: %w", err) } data.challenge = challengeFlowsAllowed } @@ -478,11 +478,10 @@ func discoverOpenIDURLs(ctx context.Context, kclient crclient.Client, issuer, ke reqCtx, cancel := context.WithTimeout(ctx, externalHTTPRequestTimeout) defer cancel() - req, err := http.NewRequest(http.MethodGet, wellKnown, nil) + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, wellKnown, nil) if err != nil { return nil, err } - req = req.WithContext(reqCtx) rt, err := transportForCARef(ctx, kclient, namespace, ca.Name, key, skipKonnectivityDialer) if err != nil { @@ -501,7 +500,7 @@ func discoverOpenIDURLs(ctx context.Context, kclient crclient.Client, issuer, ke metadata := &openIDProviderJSON{} if err := json.NewDecoder(resp.Body).Decode(metadata); err != nil { - return nil, fmt.Errorf("failed to decode metadata: %v", err) + return nil, fmt.Errorf("failed to decode metadata: %w", err) } for _, arg := range []struct { @@ -552,7 +551,7 @@ func checkOIDCPasswordGrantFlow(ctx context.Context, } err := kclient.Get(ctx, crclient.ObjectKeyFromObject(secret), secret) if err != nil { - return false, fmt.Errorf("couldn't get the referenced secret: %v", err) + return false, fmt.Errorf("couldn't get the referenced secret: %w", err) } // check whether we already attempted this not to send unnecessary login @@ -569,7 +568,7 @@ func checkOIDCPasswordGrantFlow(ctx context.Context, transport, err := transportForCARef(ctx, kclient, namespace, caRererence.Name, corev1.ServiceAccountRootCAKey, skipKonnectivityDialer) if err != nil { - return false, fmt.Errorf("couldn't get a transport for the referenced CA: %v", err) + return false, fmt.Errorf("couldn't get a transport for the referenced CA: %w", err) } // prepare the grant-checking query @@ -585,11 +584,10 @@ func checkOIDCPasswordGrantFlow(ctx context.Context, reqCtx, cancel := context.WithTimeout(ctx, externalHTTPRequestTimeout) defer cancel() - req, err := http.NewRequest("POST", tokenURL, body) + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, tokenURL, body) if err != nil { return false, err } - req = req.WithContext(reqCtx) req.Header.Add("Content-Type", "application/x-www-form-urlencoded") // explicitly set Accept to 'application/json' as that's the expected deserializable output req.Header.Set("Accept", "application/json") diff --git a/control-plane-operator/controllers/hostedcontrolplane/oauth/idp_convert_test.go b/control-plane-operator/controllers/hostedcontrolplane/oauth/idp_convert_test.go index d96a6ce199e6..c76051057ffb 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/oauth/idp_convert_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/oauth/idp_convert_test.go @@ -1122,3 +1122,22 @@ users: }) } } + +func TestConvertIdentityProviders_ErrorWrapping(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + providers := []configv1.IdentityProvider{ + { + Name: "bad-provider", + IdentityProviderConfig: configv1.IdentityProviderConfig{ + Type: "UnsupportedType", + }, + }, + } + + c := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + _, _, err := ConvertIdentityProviders(t.Context(), providers, nil, c, "test-ns") + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("failed to apply IDP bad-provider config")) +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/AROSwift/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/AROSwift/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml index bcde849ee177..bb2d7a901faf 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/AROSwift/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/AROSwift/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml @@ -100,7 +100,7 @@ spec: value: info - name: WEBHOOK_URL - name: WORKERS - value: "10" + value: "20" - name: METADATA_TRIES value: "3" - name: KUBERNETES_SERVICE_HOST diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/GCP/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/GCP/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml index 298f15ef1a5f..032e6f60f920 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/GCP/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/GCP/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml @@ -100,7 +100,7 @@ spec: value: info - name: WEBHOOK_URL - name: WORKERS - value: "10" + value: "20" - name: METADATA_TRIES value: "3" - name: KUBERNETES_SERVICE_HOST diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/IBMCloud/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/IBMCloud/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml index bcde849ee177..bb2d7a901faf 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/IBMCloud/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/IBMCloud/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml @@ -100,7 +100,7 @@ spec: value: info - name: WEBHOOK_URL - name: WORKERS - value: "10" + value: "20" - name: METADATA_TRIES value: "3" - name: KUBERNETES_SERVICE_HOST diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml index 08cd268c852c..8a827b2607b2 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml @@ -100,7 +100,7 @@ spec: value: info - name: WEBHOOK_URL - name: WORKERS - value: "10" + value: "20" - name: METADATA_TRIES value: "3" - name: KUBERNETES_SERVICE_HOST diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml index 08cd268c852c..8a827b2607b2 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/aws-node-termination-handler/zz_fixture_TestControlPlaneComponents_aws_node_termination_handler_deployment.yaml @@ -100,7 +100,7 @@ spec: value: info - name: WEBHOOK_URL - name: WORKERS - value: "10" + value: "20" - name: METADATA_TRIES value: "3" - name: KUBERNETES_SERVICE_HOST diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/AROSwift/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/AROSwift/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml index cf04e43f20e1..c06ebd5776d0 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/AROSwift/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/AROSwift/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml @@ -105,6 +105,7 @@ spec: - --api-audiences=https://test-oidc-bucket.s3.us-east-1.amazonaws.com/test-cluster - --etcd-servers=https://etcd-client:2379 - --tls-min-version=VersionTLS12 + - --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 command: - /usr/bin/oauth-apiserver env: diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/GCP/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/GCP/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml index 9594e7d5de4c..3558397ce6eb 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/GCP/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/GCP/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml @@ -105,6 +105,7 @@ spec: - --api-audiences=https://test-oidc-bucket.s3.us-east-1.amazonaws.com/test-cluster - --etcd-servers=https://etcd-client:2379 - --tls-min-version=VersionTLS12 + - --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 command: - /usr/bin/oauth-apiserver env: diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/IBMCloud/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/IBMCloud/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml index cd61efc34b9c..1b5944070a74 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/IBMCloud/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/IBMCloud/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml @@ -105,6 +105,7 @@ spec: - --api-audiences=https://test-oidc-bucket.s3.us-east-1.amazonaws.com/test-cluster - --etcd-servers=https://etcd-client:2379 - --tls-min-version=VersionTLS12 + - --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 command: - /usr/bin/oauth-apiserver env: diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml index cf04e43f20e1..c06ebd5776d0 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/TechPreviewNoUpgrade/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml @@ -105,6 +105,7 @@ spec: - --api-audiences=https://test-oidc-bucket.s3.us-east-1.amazonaws.com/test-cluster - --etcd-servers=https://etcd-client:2379 - --tls-min-version=VersionTLS12 + - --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 command: - /usr/bin/oauth-apiserver env: diff --git a/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml index cf04e43f20e1..c06ebd5776d0 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/testdata/openshift-oauth-apiserver/zz_fixture_TestControlPlaneComponents_openshift_oauth_apiserver_deployment.yaml @@ -105,6 +105,7 @@ spec: - --api-audiences=https://test-oidc-bucket.s3.us-east-1.amazonaws.com/test-cluster - --etcd-servers=https://etcd-client:2379 - --tls-min-version=VersionTLS12 + - --tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 command: - /usr/bin/oauth-apiserver env: diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/assets.go b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/assets.go index 140a408fd43a..450706c4a6e5 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/assets.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/assets.go @@ -80,7 +80,7 @@ func LoadManifestInto(componentName string, fileName string, into client.Object) obj, gvk, err := hyperapi.AllMonitoringYamlSerializer.Decode(bytes, nil, into) if err != nil { - return nil, nil, fmt.Errorf("failed to load %s manifest: %v", filePath, err) + return nil, nil, fmt.Errorf("failed to load %s manifest: %w", filePath, err) } return obj.(client.Object), gvk, err } diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/aws-node-termination-handler/deployment.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/aws-node-termination-handler/deployment.yaml index 64d281f386d7..3b33b665ccfa 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/aws-node-termination-handler/deployment.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/aws-node-termination-handler/deployment.yaml @@ -62,7 +62,7 @@ spec: - name: WEBHOOK_URL value: "" - name: WORKERS - value: "10" + value: "20" - name: METADATA_TRIES value: "3" - name: KUBERNETES_SERVICE_HOST diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/karpenter-operator/role.yaml b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/karpenter-operator/role.yaml index 7099f077cb52..11b92618c224 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/assets/karpenter-operator/role.yaml +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/assets/karpenter-operator/role.yaml @@ -83,6 +83,14 @@ rules: - "update" - "patch" - "delete" + - apiGroups: + - "cluster.x-k8s.io" + resources: + - "clusters" + verbs: + - "get" + - "list" + - "watch" - apiGroups: - "" resources: diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/kubevirt/config.go b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/kubevirt/config.go index 1c6531d6b3a8..3e7969ba166d 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/kubevirt/config.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/kubevirt/config.go @@ -54,7 +54,7 @@ func adaptConfig(cpContext component.WorkloadContext, cm *corev1.ConfigMap) erro data := []byte(cm.Data[CloudConfigKey]) cloudConfig := &CloudConfig{} if err := yaml.Unmarshal(data, cloudConfig); err != nil { - return fmt.Errorf("failed to unmarshal CloudConfig: %v", err) + return fmt.Errorf("failed to unmarshal CloudConfig: %w", err) } if kubevirt := cpContext.HCP.Spec.Platform.Kubevirt; kubevirt != nil && kubevirt.Credentials != nil { diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/powervs/config.go b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/powervs/config.go index 73cdcdb9fff6..7f7f5488593d 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/powervs/config.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/cloud_controller_manager/powervs/config.go @@ -38,7 +38,7 @@ func adaptConfig(cpContext component.WorkloadContext, cm *corev1.ConfigMap) erro configData := &bytes.Buffer{} err := template.Execute(configData, config) if err != nil { - return fmt.Errorf("error while parsing ccm config map template %v", err) + return fmt.Errorf("error while parsing ccm config map template %w", err) } cm.Data[configKey] = configData.String() diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/cno/deployment_init_container_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/cno/deployment_init_container_test.go index 1be0732f1d2c..19710668990b 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/cno/deployment_init_container_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/cno/deployment_init_container_test.go @@ -147,7 +147,7 @@ func TestRewriteConfigInitContainer(t *testing.T) { testScript := buildTestScript(script, kubectlLog) - cmd := exec.Command("bash", "-c", testScript) + cmd := exec.CommandContext(t.Context(), "bash", "-c", testScript) cmd.Env = []string{ "KUBERNETES_SERVICE_HOST=" + tt.host, "KUBERNETES_SERVICE_PORT=" + tt.port, diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/etcd/statefulset.go b/control-plane-operator/controllers/hostedcontrolplane/v2/etcd/statefulset.go index 49de8fc17d35..4d4d740303cb 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/etcd/statefulset.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/etcd/statefulset.go @@ -23,7 +23,7 @@ func adaptStatefulSet(cpContext component.WorkloadContext, sts *appsv1.StatefulS ipv4, err := netutil.IsIPv4CIDR(hcp.Spec.Networking.ClusterNetwork[0].CIDR.String()) if err != nil { - return fmt.Errorf("error checking the ClusterNetworkCIDR: %v", err) + return fmt.Errorf("error checking the ClusterNetworkCIDR: %w", err) } podspec.UpdateContainer(ComponentName, sts.Spec.Template.Spec.Containers, func(c *corev1.Container) { diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/ignitionserver/pki.go b/control-plane-operator/controllers/hostedcontrolplane/v2/ignitionserver/pki.go index 79bf570f4f78..1be308ceceb0 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/ignitionserver/pki.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/ignitionserver/pki.go @@ -45,7 +45,7 @@ func adaptServingCertSecret(cpContext component.WorkloadContext, secret *corev1. if apierrors.IsNotFound(err) { return nil } - return fmt.Errorf("failed to get ignition ca-cert secret: %v", err) + return fmt.Errorf("failed to get ignition ca-cert secret: %w", err) } serviceStrategy := netutil.ServicePublishingStrategyByTypeForHCP(cpContext.HCP, hyperv1.Ignition) @@ -67,7 +67,7 @@ func adaptServingCertSecret(cpContext component.WorkloadContext, secret *corev1. if apierrors.IsNotFound(err) { return nil } - return fmt.Errorf("failed to get ignition route: %v", err) + return fmt.Errorf("failed to get ignition route: %w", err) } // The route must be admitted and assigned a host before we can generate certs if len(ignitionServerRoute.Status.Ingress) == 0 || len(ignitionServerRoute.Status.Ingress[0].Host) == 0 { diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/auth.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/auth.go index 714395d4f354..91b67d9397ca 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/auth.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/auth.go @@ -74,7 +74,7 @@ func GenerateAuthConfig(ctx context.Context, spec *configv1.AuthenticationSpec, for _, provider := range spec.OIDCProviders { jwt, err := generateJWTForProvider(ctx, provider, c, namespace) if err != nil { - return nil, fmt.Errorf("generating JWT authenticator for provider %q: %v", provider.Name, err) + return nil, fmt.Errorf("generating JWT authenticator for provider %q: %w", provider.Name, err) } config.JWT = append(config.JWT, jwt) } @@ -86,23 +86,23 @@ func generateJWTForProvider(ctx context.Context, provider configv1.OIDCProvider, issuer, err := generateIssuer(ctx, provider.Issuer, client, namespace) if err != nil { - return out, fmt.Errorf("generating issuer: %v", err) + return out, fmt.Errorf("generating issuer: %w", err) } claimMappings, err := generateClaimMappings(provider.ClaimMappings, issuer.URL) if err != nil { - return out, fmt.Errorf("generating claim mappings: %v", err) + return out, fmt.Errorf("generating claim mappings: %w", err) } claimValidationRules, err := generateClaimValidationRules(provider.ClaimValidationRules...) if err != nil { - return out, fmt.Errorf("generating claim validation rules: %v", err) + return out, fmt.Errorf("generating claim validation rules: %w", err) } if featuregates.Gate().Enabled(featuregates.ExternalOIDCWithUpstreamParity) { userValidationRules, err := generateUserValidationRules(provider.UserValidationRules...) if err != nil { - return out, fmt.Errorf("generating userValidationRules for provider %q: %v", provider.Name, err) + return out, fmt.Errorf("generating userValidationRules for provider %q: %w", provider.Name, err) } out.UserValidationRules = userValidationRules } @@ -128,7 +128,7 @@ func generateIssuer(ctx context.Context, issuer configv1.TokenIssuer, client crc // Validate the URL scheme u, err := url.Parse(issuer.DiscoveryURL) if err != nil { - return out, fmt.Errorf("invalid discovery URL: %v", err) + return out, fmt.Errorf("invalid discovery URL: %w", err) } if strings.TrimRight(issuer.DiscoveryURL, "/") == strings.TrimRight(issuer.URL, "/") { return out, fmt.Errorf("discovery URL must not be identical to issuer URL") @@ -155,7 +155,7 @@ func generateIssuer(ctx context.Context, issuer configv1.TokenIssuer, client crc if len(issuer.CertificateAuthority.Name) > 0 { ca, err := getCertificateAuthorityFromConfigMap(ctx, client, issuer.CertificateAuthority.Name, namespace) if err != nil { - return out, fmt.Errorf("getting certificate authority for issuer: %v", err) + return out, fmt.Errorf("getting certificate authority for issuer: %w", err) } out.CertificateAuthority = ca } @@ -182,12 +182,12 @@ func generateClaimMappings(claimMappings configv1.TokenClaimMappings, issuerURL username, err := generateUsernameClaimMapping(claimMappings.Username, issuerURL) if err != nil { - return out, fmt.Errorf("generating username claim mapping: %v", err) + return out, fmt.Errorf("generating username claim mapping: %w", err) } groups, err := generateGroupsClaimMapping(claimMappings.Groups) if err != nil { - return out, fmt.Errorf("generating groups claim mapping: %v", err) + return out, fmt.Errorf("generating groups claim mapping: %w", err) } out.Username = username @@ -196,12 +196,12 @@ func generateClaimMappings(claimMappings configv1.TokenClaimMappings, issuerURL if featuregates.Gate().Enabled(featuregates.ExternalOIDCWithUIDAndExtraClaimMappings) { uid, err := generateUIDClaimMapping(claimMappings.UID) if err != nil { - return out, fmt.Errorf("generating uid claim mapping: %v", err) + return out, fmt.Errorf("generating uid claim mapping: %w", err) } extras, err := generateExtraClaimMapping(claimMappings.Extra...) if err != nil { - return out, fmt.Errorf("generating extra claim mapping: %v", err) + return out, fmt.Errorf("generating extra claim mapping: %w", err) } out.UID = uid @@ -419,7 +419,7 @@ func generateUserValidationRules(rules ...configv1.TokenUserValidationRule) ([]U for _, r := range rules { uvr, err := generateUserValidationRule(r) if err != nil { - errs = append(errs, fmt.Errorf("generating userValidationRule: %v", err)) + errs = append(errs, fmt.Errorf("generating userValidationRule: %w", err)) continue } out = append(out, uvr) @@ -469,7 +469,7 @@ func validateAuthConfig(authConfig *AuthenticationConfiguration, disallowIssuers apiServerAuthConfig, err := HCPAuthConfigToAPIServerAuthConfig(authConfig) if err != nil { - return fmt.Errorf("converting from HCP auth config type to apiserver auth config type: %v", err) + return fmt.Errorf("converting from HCP auth config type to apiserver auth config type: %w", err) } fieldErrors := validation.ValidateAuthenticationConfiguration(celCompiler, apiServerAuthConfig, disallowIssuers) @@ -483,13 +483,13 @@ func validateAuthConfig(authConfig *AuthenticationConfiguration, disallowIssuers func HCPAuthConfigToAPIServerAuthConfig(authConfig *AuthenticationConfiguration) (*apiserver.AuthenticationConfiguration, error) { outBytes, err := json.Marshal(authConfig) if err != nil { - return nil, fmt.Errorf("marshaling HCP auth config to JSON: %v", err) + return nil, fmt.Errorf("marshaling HCP auth config to JSON: %w", err) } apiserverAuthConfig := &apiserver.AuthenticationConfiguration{} err = json.Unmarshal(outBytes, apiserverAuthConfig) if err != nil { - return nil, fmt.Errorf("unmarshalling HCP auth config JSON to apiserver auth config: %v", err) + return nil, fmt.Errorf("unmarshalling HCP auth config JSON to apiserver auth config: %w", err) } return apiserverAuthConfig, nil diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/auth_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/auth_test.go index 85177fda298f..fb9e592dc1a5 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/auth_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/auth_test.go @@ -3,8 +3,11 @@ package kas import ( "context" "encoding/json" + "strings" "testing" + . "github.com/onsi/gomega" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/featuregates" controlplanecomponent "github.com/openshift/hypershift/support/controlplane-component" @@ -70,6 +73,7 @@ func TestGenerateAuthConfig(t *testing.T) { namespace string expectedAuthenticationConfiguration *AuthenticationConfiguration shouldError bool + errSubstr string featureGates []featuregate.Feature } @@ -138,6 +142,33 @@ func TestGenerateAuthConfig(t *testing.T) { }, shouldError: false, }, + { + name: "When issuer references a missing CA ConfigMap, it should return a wrapped error", + spec: &configv1.AuthenticationSpec{ + OIDCProviders: []configv1.OIDCProvider{ + { + Name: "test-provider", + Issuer: configv1.TokenIssuer{ + URL: "https://test.example.com", + Audiences: []configv1.TokenAudience{"test-audience"}, + CertificateAuthority: configv1.ConfigMapNameReference{ + Name: "nonexistent-ca-configmap", + }, + }, + ClaimMappings: configv1.TokenClaimMappings{ + Username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.NoPrefix, + }, + }, + }, + }, + }, + client: fake.NewClientBuilder().Build(), + namespace: "test-namespace", + shouldError: true, + errSubstr: "generating JWT authenticator for provider", + }, { name: "When OIDC provider with CEL validation is provided, it should generate configuration with validation rules", ctx: context.Background(), @@ -228,7 +259,9 @@ func TestGenerateAuthConfig(t *testing.T) { case !tc.shouldError && err != nil: t.Fatalf("unexpected error: %v", err) case tc.shouldError && err != nil: - // as expected + if tc.errSubstr != "" && !strings.Contains(err.Error(), tc.errSubstr) { + t.Fatalf("expected error to contain %q, got: %v", tc.errSubstr, err) + } return } @@ -1766,3 +1799,386 @@ func TestAdaptAuthConfig(t *testing.T) { }) } } + +func TestGenerateClaimMappings(t *testing.T) { + t.Parallel() + tests := []struct { + name string + mappings configv1.TokenClaimMappings + issuerURL string + wantErr bool + errSubstr string + }{ + { + name: "When username mapping is valid, it should return mappings without error", + mappings: configv1.TokenClaimMappings{ + Username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.NoPrefix, + }, + }, + issuerURL: "https://issuer.example.com", + }, + { + name: "When username mapping has Prefix policy with nil prefix, it should return a wrapped error", + mappings: configv1.TokenClaimMappings{ + Username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.Prefix, + Prefix: nil, + }, + }, + issuerURL: "https://issuer.example.com", + wantErr: true, + errSubstr: "generating username claim mapping", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + _, err := generateClaimMappings(tt.mappings, tt.issuerURL) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateExtraMapping(t *testing.T) { + t.Parallel() + tests := []struct { + name string + extra configv1.ExtraMapping + wantErr bool + errSubstr string + }{ + { + name: "When key and valueExpression are valid, it should return mapping without error", + extra: configv1.ExtraMapping{ + Key: "example.com/foo", + ValueExpression: "claims.groups", + }, + }, + { + name: "When key is empty, it should return an error", + extra: configv1.ExtraMapping{ + Key: "", + ValueExpression: "claims.groups", + }, + wantErr: true, + errSubstr: "must specify a key", + }, + { + name: "When valueExpression is empty, it should return an error", + extra: configv1.ExtraMapping{ + Key: "example.com/foo", + ValueExpression: "", + }, + wantErr: true, + errSubstr: "must specify a valueExpression", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + _, err := generateExtraMapping(tt.extra) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateClaimValidationRule(t *testing.T) { + t.Parallel() + tests := []struct { + name string + rule configv1.TokenClaimValidationRule + wantErr bool + errSubstr string + }{ + { + name: "When type is RequiredClaim with valid required claim, it should return rule without error", + rule: configv1.TokenClaimValidationRule{ + Type: configv1.TokenValidationRuleTypeRequiredClaim, + RequiredClaim: &configv1.TokenRequiredClaim{ + Claim: "aud", + RequiredValue: "my-audience", + }, + }, + }, + { + name: "When type is RequiredClaim but requiredClaim is nil, it should return an error", + rule: configv1.TokenClaimValidationRule{ + Type: configv1.TokenValidationRuleTypeRequiredClaim, + }, + wantErr: true, + errSubstr: "requiredClaim is not set", + }, + { + name: "When type is CEL with valid expression, it should return rule without error", + rule: configv1.TokenClaimValidationRule{ + Type: configv1.TokenValidationRuleTypeCEL, + CEL: configv1.TokenClaimValidationCELRule{ + Expression: "claims.email_verified == true", + Message: "email must be verified", + }, + }, + }, + { + name: "When type is CEL but expression is empty, it should return an error", + rule: configv1.TokenClaimValidationRule{ + Type: configv1.TokenValidationRuleTypeCEL, + CEL: configv1.TokenClaimValidationCELRule{}, + }, + wantErr: true, + errSubstr: "expression is not set", + }, + { + name: "When type is unknown, it should return an error", + rule: configv1.TokenClaimValidationRule{ + Type: "UnknownType", + }, + wantErr: true, + errSubstr: "unknown claimValidationRule type", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + _, err := generateClaimValidationRule(tt.rule) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateClaimValidationRules(t *testing.T) { + t.Parallel() + tests := []struct { + name string + rules []configv1.TokenClaimValidationRule + wantErr bool + }{ + { + name: "When all rules are valid, it should return rules without error", + rules: []configv1.TokenClaimValidationRule{ + { + Type: configv1.TokenValidationRuleTypeRequiredClaim, + RequiredClaim: &configv1.TokenRequiredClaim{ + Claim: "aud", + RequiredValue: "my-audience", + }, + }, + { + Type: configv1.TokenValidationRuleTypeCEL, + CEL: configv1.TokenClaimValidationCELRule{ + Expression: "claims.email_verified == true", + Message: "email must be verified", + }, + }, + }, + }, + { + name: "When a rule is invalid, it should return an error", + rules: []configv1.TokenClaimValidationRule{ + { + Type: configv1.TokenValidationRuleTypeRequiredClaim, + }, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + _, err := generateClaimValidationRules(tt.rules...) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateUserValidationRule(t *testing.T) { + t.Parallel() + tests := []struct { + name string + rule configv1.TokenUserValidationRule + wantErr bool + errSubstr string + }{ + { + name: "When expression is valid, it should return rule without error", + rule: configv1.TokenUserValidationRule{ + Expression: "user.username != 'admin'", + Message: "admin not allowed", + }, + }, + { + name: "When expression is empty, it should return an error", + rule: configv1.TokenUserValidationRule{ + Expression: "", + Message: "should fail", + }, + wantErr: true, + errSubstr: "expression must be non-empty", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + _, err := generateUserValidationRule(tt.rule) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateUserValidationRules(t *testing.T) { + t.Parallel() + tests := []struct { + name string + rules []configv1.TokenUserValidationRule + wantErr bool + }{ + { + name: "When all rules are valid, it should return rules without error", + rules: []configv1.TokenUserValidationRule{ + {Expression: "user.username != 'admin'", Message: "admin not allowed"}, + {Expression: "user.username != 'root'", Message: "root not allowed"}, + }, + }, + { + name: "When a rule has empty expression, it should return an error", + rules: []configv1.TokenUserValidationRule{ + {Expression: "user.username != 'admin'", Message: "ok"}, + {Expression: "", Message: "should fail"}, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + _, err := generateUserValidationRules(tt.rules...) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestGenerateJWTForProvider(t *testing.T) { + t.Parallel() + tests := []struct { + name string + provider configv1.OIDCProvider + wantErr bool + errSubstr string + }{ + { + name: "When username claim is empty, it should return a claim mappings error", + provider: configv1.OIDCProvider{ + Name: "test-provider", + Issuer: configv1.TokenIssuer{ + URL: "https://issuer.example.com", + Audiences: []configv1.TokenAudience{"aud"}, + }, + ClaimMappings: configv1.TokenClaimMappings{ + Username: configv1.UsernameClaimMapping{}, + }, + }, + wantErr: true, + errSubstr: "generating claim mappings", + }, + { + name: "When claim validation rule has unknown type, it should return a claim validation rules error", + provider: configv1.OIDCProvider{ + Name: "test-provider", + Issuer: configv1.TokenIssuer{ + URL: "https://issuer.example.com", + Audiences: []configv1.TokenAudience{"aud"}, + }, + ClaimMappings: configv1.TokenClaimMappings{ + Username: configv1.UsernameClaimMapping{ + Claim: "email", + PrefixPolicy: configv1.NoPrefix, + }, + }, + ClaimValidationRules: []configv1.TokenClaimValidationRule{ + {Type: "UnknownType"}, + }, + }, + wantErr: true, + errSubstr: "generating claim validation rules", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + c := fake.NewClientBuilder().Build() + _, err := generateJWTForProvider(t.Context(), tt.provider, c, "test-namespace") + + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestHCPAuthConfigToAPIServerAuthConfig(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + input := &AuthenticationConfiguration{ + JWT: []JWTAuthenticator{ + { + Issuer: Issuer{ + URL: "https://issuer.example.com", + Audiences: []string{"test-audience"}, + }, + ClaimMappings: ClaimMappings{ + Username: PrefixedClaimOrExpression{ + Claim: "email", + Prefix: ptr.To(""), + }, + }, + }, + }, + } + + result, err := HCPAuthConfigToAPIServerAuthConfig(input) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(result.JWT).To(HaveLen(1)) + g.Expect(result.JWT[0].Issuer.URL).To(Equal("https://issuer.example.com")) +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/deployment.go index cedcc84af82d..a8f0aa040007 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/deployment.go @@ -81,11 +81,11 @@ func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Dep bootstrapUpdateErrors := []error{} for _, bootstrapContainer := range bootstrapContainers { if err := updateBootstrapInitContainer(deployment, hcp, payloadVersion, bootstrapContainer); err != nil { - bootstrapUpdateErrors = append(bootstrapUpdateErrors, fmt.Errorf("updating bootstrap container %q: %v", bootstrapContainer, err)) + bootstrapUpdateErrors = append(bootstrapUpdateErrors, fmt.Errorf("updating bootstrap container %q: %w", bootstrapContainer, err)) } } if err := errors.Join(bootstrapUpdateErrors...); err != nil { - return fmt.Errorf("updating bootstrap containers: %v", err) + return fmt.Errorf("updating bootstrap containers: %w", err) } if hcp.Spec.Configuration.GetAuditPolicyConfig().Profile == configv1.NoneAuditProfileType { @@ -180,7 +180,7 @@ func updateMainContainer(podSpec *corev1.PodSpec, hcp *hyperv1.HostedControlPlan ) // We have to exempt the pod and service CIDR, otherwise the proxy will get respected by the transport inside - // the the egress transport and that breaks the egress selection/konnektivity usage. + // the egress transport and that breaks the egress selection/konnektivity usage. // Using a CIDR is not supported by Go's default ProxyFunc, but Kube uses a custom one by default that does support it: // https://github.com/kubernetes/kubernetes/blob/ab13c85316015cf9f115e29923ba9740bd1564fd/staging/src/k8s.io/apimachinery/pkg/util/net/http.go#L112-L114 var additionalNoProxyCIDRS []string diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kubeconfig.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kubeconfig.go index 25dedd9fbb6f..a04553b58b78 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kubeconfig.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/kubeconfig.go @@ -181,7 +181,7 @@ func adaptBootstrapKubeconfigSecret(cpContext component.WorkloadContext, secret func adaptAWSPodIdentityWebhookKubeconfigSecret(cpContext component.WorkloadContext, secret *corev1.Secret) error { csrSigner := manifests.CSRSignerCASecret(cpContext.HCP.Namespace) if err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(csrSigner), csrSigner); err != nil { - return fmt.Errorf("failed to get cluster-signer-ca secret: %v", err) + return fmt.Errorf("failed to get cluster-signer-ca secret: %w", err) } rootCA := manifests.RootCASecret(cpContext.HCP.Namespace) if err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(rootCA), rootCA); err != nil { diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/oauth.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/oauth.go index 4372e294eb6a..984294240cb1 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/oauth.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/oauth.go @@ -36,7 +36,7 @@ func adaptOauthMetadata(cpContext component.WorkloadContext, cfg *corev1.ConfigM var oauthMetadata map[string]interface{} if err := json.Unmarshal([]byte(cfg.Data[OauthMetadataConfigKey]), &oauthMetadata); err != nil { - return fmt.Errorf("failed to unmarshal oauth metadata: %v", err) + return fmt.Errorf("failed to unmarshal oauth metadata: %w", err) } oauthURL := fmt.Sprintf("https://%s:%d", cpContext.InfraStatus.OAuthHost, cpContext.InfraStatus.OAuthPort) @@ -46,7 +46,7 @@ func adaptOauthMetadata(cpContext component.WorkloadContext, cfg *corev1.ConfigM data, err := json.Marshal(oauthMetadata) if err != nil { - return fmt.Errorf("failed to marshal oauth metadata: %v", err) + return fmt.Errorf("failed to marshal oauth metadata: %w", err) } cfg.Data[OauthMetadataConfigKey] = string(data) return nil diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/oauth_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/oauth_test.go new file mode 100644 index 000000000000..596c7cdb0658 --- /dev/null +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/oauth_test.go @@ -0,0 +1,55 @@ +package kas + +import ( + "testing" + + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + controlplanecomponent "github.com/openshift/hypershift/support/controlplane-component" + + corev1 "k8s.io/api/core/v1" + + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestAdaptOauthMetadata(t *testing.T) { + t.Parallel() + tests := []struct { + name string + cfg *corev1.ConfigMap + wantErr bool + errSubstr string + }{ + { + name: "When ConfigMap contains invalid JSON, it should return an unmarshal error", + cfg: &corev1.ConfigMap{ + Data: map[string]string{ + OauthMetadataConfigKey: "not-valid-json{{{", + }, + }, + wantErr: true, + errSubstr: "failed to unmarshal oauth metadata", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + cpContext := controlplanecomponent.WorkloadContext{ + Context: t.Context(), + HCP: &hyperv1.HostedControlPlane{}, + Client: fake.NewClientBuilder().Build(), + } + + err := adaptOauthMetadata(cpContext, tt.cfg) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption.go index 61946ac37a8c..5baaf8c1f7f8 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption.go @@ -96,7 +96,7 @@ func getKMSAPIVersion(cpContext component.WorkloadContext, secret *corev1.Secret if apierrors.IsNotFound(err) { return apiVersion, nil } - return "", fmt.Errorf("failed to get existing secret encryption config: %v", err) + return "", fmt.Errorf("failed to get existing secret encryption config: %w", err) } encryptionConfigBytes := secret.Data[secretEncryptionConfigurationKey] @@ -104,10 +104,10 @@ func getKMSAPIVersion(cpContext component.WorkloadContext, secret *corev1.Secret currentConfig := apiserverv1.EncryptionConfiguration{} gvks, _, err := api.Scheme.ObjectKinds(¤tConfig) if err != nil || len(gvks) == 0 { - return "", fmt.Errorf("cannot determine gvk of resource: %v", err) + return "", fmt.Errorf("cannot determine gvk of resource: %w", err) } if _, _, err = api.YamlSerializer.Decode(encryptionConfigBytes, &gvks[0], ¤tConfig); err != nil { - return "", fmt.Errorf("cannot decode resource: %v", err) + return "", fmt.Errorf("cannot decode resource: %w", err) } // Only look at write keys to return the APIVersion currently used. diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption_test.go index 82d2ea68d324..7f9d658ded65 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/kas/secretencryption_test.go @@ -2,9 +2,13 @@ package kas import ( "bytes" + "context" + "fmt" "testing" "time" + . "github.com/onsi/gomega" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/support/api" "github.com/openshift/hypershift/support/config" @@ -16,7 +20,9 @@ import ( v1 "k8s.io/apiserver/pkg/apis/apiserver/v1" "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/google/go-cmp/cmp" ) @@ -363,6 +369,78 @@ func TestReconcileKMSEncryptionConfigAzureSelfManaged(t *testing.T) { } } +func TestGetKMSAPIVersion(t *testing.T) { + t.Parallel() + tests := []struct { + name string + secret *corev1.Secret + client client.Client + wantErr bool + errSubstr string + wantResult string + }{ + { + name: "When client Get returns a non-NotFound error, it should return a wrapped error", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "encryption-config", Namespace: "test-ns"}, + }, + client: fake.NewClientBuilder().WithInterceptorFuncs(interceptor.Funcs{ + Get: func(_ context.Context, _ client.WithWatch, _ client.ObjectKey, _ client.Object, _ ...client.GetOption) error { + return fmt.Errorf("connection refused") + }, + }).Build(), + wantErr: true, + errSubstr: "failed to get existing secret encryption config", + }, + { + name: "When secret contains invalid YAML, it should return a decode error", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "encryption-config", Namespace: "test-ns"}, + Data: map[string][]byte{ + secretEncryptionConfigurationKey: []byte("not-valid-yaml: {{{"), + }, + }, + wantErr: true, + errSubstr: "cannot decode resource", + }, + { + name: "When secret does not exist, it should return default v2", + secret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "encryption-config", Namespace: "test-ns"}, + }, + wantResult: "v2", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + c := tt.client + if c == nil { + c = fake.NewClientBuilder().Build() + } + if tt.secret.Data != nil && tt.client == nil { + c = fake.NewClientBuilder().WithObjects(tt.secret).Build() + } + + cpContext := controlplanecomponent.WorkloadContext{ + Context: t.Context(), + Client: c, + } + + result, err := getKMSAPIVersion(cpContext, tt.secret) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(result).To(Equal(tt.wantResult)) + } + }) + } +} + func generateExpectedAzureEncryptionConfig(t testing.TB, apiVersion string) *v1.EncryptionConfiguration { t.Helper() activeKeyHash, err := util.HashStruct(hyperv1.AzureKMSKey{ diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/oauth/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/oauth/deployment.go index 4a7dd1bc65b3..e31e102fe6c0 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/oauth/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/oauth/deployment.go @@ -109,7 +109,7 @@ func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Dep kubeadminPasswordSecret := common.KubeadminPasswordSecret(deployment.Namespace) if err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(kubeadminPasswordSecret), kubeadminPasswordSecret); err != nil { if !apierrors.IsNotFound(err) { - return fmt.Errorf("failed to get kubeadmin password secret: %v", err) + return fmt.Errorf("failed to get kubeadmin password secret: %w", err) } delete(deployment.Spec.Template.ObjectMeta.Annotations, KubeadminSecretHashAnnotation) } else { diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/oauth/idp_convert.go b/control-plane-operator/controllers/hostedcontrolplane/v2/oauth/idp_convert.go index 4f5f85944c99..0668647ad8e2 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/oauth/idp_convert.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/oauth/idp_convert.go @@ -104,7 +104,7 @@ func ConvertIdentityProviders(ctx context.Context, identityProviders []configv1. } data, err := convertProviderConfigToIDPData(ctx, &idp.IdentityProviderConfig, providerConfigOverride, i, volumeMountInfo, kclient, namespace, false) if err != nil { - errs = append(errs, fmt.Errorf("failed to apply IDP %s config: %v", idp.Name, err)) + errs = append(errs, fmt.Errorf("failed to apply IDP %s config: %w", idp.Name, err)) continue } converted = append(converted, @@ -429,7 +429,7 @@ func convertOpenIDIDP( skipKonnectivityDialer, ) if err != nil { - return nil, fmt.Errorf("error attempting password grant flow: %v", err) + return nil, fmt.Errorf("error attempting password grant flow: %w", err) } data.challenge = challengeFlowsAllowed } @@ -548,11 +548,10 @@ func discoverOpenIDURLs(ctx context.Context, kclient crclient.Reader, issuer, ke reqCtx, cancel := context.WithTimeout(ctx, externalHTTPRequestTimeout) defer cancel() - req, err := http.NewRequest(http.MethodGet, wellKnown, nil) + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, wellKnown, nil) if err != nil { return nil, err } - req = req.WithContext(reqCtx) rt, err := transportForCARef(ctx, kclient, namespace, ca.Name, key, skipKonnectivityDialer) if err != nil { @@ -571,7 +570,7 @@ func discoverOpenIDURLs(ctx context.Context, kclient crclient.Reader, issuer, ke metadata := &openIDProviderJSON{} if err := json.NewDecoder(resp.Body).Decode(metadata); err != nil { - return nil, fmt.Errorf("failed to decode metadata: %v", err) + return nil, fmt.Errorf("failed to decode metadata: %w", err) } for _, arg := range []struct { @@ -622,7 +621,7 @@ func checkOIDCPasswordGrantFlow(ctx context.Context, } err := kclient.Get(ctx, crclient.ObjectKeyFromObject(secret), secret) if err != nil { - return false, fmt.Errorf("couldn't get the referenced secret: %v", err) + return false, fmt.Errorf("couldn't get the referenced secret: %w", err) } // check whether we already attempted this not to send unnecessary login @@ -639,7 +638,7 @@ func checkOIDCPasswordGrantFlow(ctx context.Context, transport, err := transportForCARef(ctx, kclient, namespace, caRererence.Name, corev1.ServiceAccountRootCAKey, skipKonnectivityDialer) if err != nil { - return false, fmt.Errorf("couldn't get a transport for the referenced CA: %v", err) + return false, fmt.Errorf("couldn't get a transport for the referenced CA: %w", err) } // prepare the grant-checking query @@ -655,11 +654,10 @@ func checkOIDCPasswordGrantFlow(ctx context.Context, reqCtx, cancel := context.WithTimeout(ctx, externalHTTPRequestTimeout) defer cancel() - req, err := http.NewRequest("POST", tokenURL, body) + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, tokenURL, body) if err != nil { return false, err } - req = req.WithContext(reqCtx) req.Header.Add("Content-Type", "application/x-www-form-urlencoded") // explicitly set Accept to 'application/json' as that's the expected deserializable output req.Header.Set("Accept", "application/json") diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/oauth/idp_convert_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/oauth/idp_convert_test.go index 24adb544a937..a5e6e474c9db 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/oauth/idp_convert_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/oauth/idp_convert_test.go @@ -1122,3 +1122,22 @@ users: }) } } + +func TestConvertIdentityProviders_ErrorWrapping(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + providers := []configv1.IdentityProvider{ + { + Name: "bad-provider", + IdentityProviderConfig: configv1.IdentityProviderConfig{ + Type: "UnsupportedType", + }, + }, + } + + c := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build() + _, _, err := ConvertIdentityProviders(t.Context(), providers, nil, c, "test-ns") + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("failed to apply IDP bad-provider config")) +} diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/oauth_apiserver/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/oauth_apiserver/deployment.go index 1d25593d8373..3e69ce8ab63c 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/oauth_apiserver/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/oauth_apiserver/deployment.go @@ -52,6 +52,10 @@ func adaptDeployment(cpContext component.WorkloadContext, deployment *appsv1.Dep fmt.Sprintf("--tls-min-version=%s", config.MinTLSVersion(configuration.GetTLSSecurityProfile())), ) + if cipherSuites := config.CipherSuites(configuration.GetTLSSecurityProfile()); len(cipherSuites) != 0 { + c.Args = append(c.Args, fmt.Sprintf("--tls-cipher-suites=%s", strings.Join(cipherSuites, ","))) + } + if cpContext.HCP.Spec.AuditWebhook != nil && len(cpContext.HCP.Spec.AuditWebhook.Name) > 0 { c.Args = append(c.Args, fmt.Sprintf("--audit-webhook-config-file=%s", path.Join("/etc/kubernetes/auditwebhook", hyperv1.AuditWebhookKubeconfigKey))) c.Args = append(c.Args, "--audit-webhook-mode=batch") diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/oauth_apiserver/deployment_test.go b/control-plane-operator/controllers/hostedcontrolplane/v2/oauth_apiserver/deployment_test.go index ef32bf820ea3..9b565a3cef43 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/oauth_apiserver/deployment_test.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/oauth_apiserver/deployment_test.go @@ -148,6 +148,51 @@ func TestAdaptDeployment(t *testing.T) { container := podspec.FindContainer(ComponentName, deployment.Spec.Template.Spec.Containers) g.Expect(container).ToNot(BeNil()) g.Expect(container.Args).To(ContainElement("--tls-min-version=VersionTLS13")) + g.Expect(container.Args).ToNot(ContainElement(ContainSubstring("--tls-cipher-suites="))) + }, + }, + { + name: "When TLS security profile is Intermediate, it should configure tls-cipher-suites", + hcp: &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-hcp", + Namespace: "test-ns", + }, + Spec: hyperv1.HostedControlPlaneSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AWSPlatform, + }, + IssuerURL: "https://test-issuer.example.com", + Etcd: hyperv1.EtcdSpec{ + ManagementType: hyperv1.Managed, + }, + Configuration: &hyperv1.ClusterConfiguration{ + APIServer: &configv1.APIServerSpec{ + TLSSecurityProfile: &configv1.TLSSecurityProfile{ + Type: configv1.TLSProfileIntermediateType, + Intermediate: &configv1.IntermediateTLSProfile{}, + }, + }, + }, + }, + }, + validate: func(t *testing.T, g *GomegaWithT, hcp *hyperv1.HostedControlPlane) { + + deployment, loadErr := assets.LoadDeploymentManifest(ComponentName) + g.Expect(loadErr).ToNot(HaveOccurred()) + + cpContext := component.WorkloadContext{ + Client: fake.NewClientBuilder().WithScheme(api.Scheme).Build(), + HCP: hcp, + } + + err := adaptDeployment(cpContext, deployment) + g.Expect(err).ToNot(HaveOccurred()) + + container := podspec.FindContainer(ComponentName, deployment.Spec.Template.Spec.Containers) + g.Expect(container).ToNot(BeNil()) + g.Expect(container.Args).To(ContainElement("--tls-min-version=VersionTLS12")) + g.Expect(container.Args).To(ContainElement(ContainSubstring("--tls-cipher-suites="))) }, }, { diff --git a/control-plane-operator/controllers/hostedcontrolplane/v2/olm/catalogs/deployment.go b/control-plane-operator/controllers/hostedcontrolplane/v2/olm/catalogs/deployment.go index 9b49abd358b6..5ac89a16c01e 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/v2/olm/catalogs/deployment.go +++ b/control-plane-operator/controllers/hostedcontrolplane/v2/olm/catalogs/deployment.go @@ -35,7 +35,7 @@ func (c *catalogOptions) adaptCatalogDeployment(cpContext component.WorkloadCont existingDeployment := &appsv1.Deployment{} if err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(deployment), existingDeployment); err != nil { if !apierrors.IsNotFound(err) { - return fmt.Errorf("failed to get existing deployment: %v", err) + return fmt.Errorf("failed to get existing deployment: %w", err) } } else { // If deployment already exists, imagestream tag will already populate the container image @@ -96,7 +96,7 @@ func getCatalogImagesOverrides(cpContext component.WorkloadContext, capabilityIm digest, _, err := cpContext.ImageMetadataProvider.GetDigest(cpContext, imageRef.Exact(), pullSecret.Data[corev1.DockerConfigJsonKey]) if err != nil { - return nil, fmt.Errorf("failed to get manifest for image %s: %v", imageRef.Exact(), err) + return nil, fmt.Errorf("failed to get manifest for image %s: %w", imageRef.Exact(), err) } imageRef.ID = digest.String() diff --git a/control-plane-operator/endpoint-resolver/server_test.go b/control-plane-operator/endpoint-resolver/server_test.go index fa63d651f0bb..4228864dffe1 100644 --- a/control-plane-operator/endpoint-resolver/server_test.go +++ b/control-plane-operator/endpoint-resolver/server_test.go @@ -7,6 +7,8 @@ import ( "net/http/httptest" "testing" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" corev1listers "k8s.io/client-go/listers/core/v1" @@ -43,11 +45,11 @@ func newFakePodLister(namespace string, pods []*corev1.Pod) corev1listers.PodNam func newResolveRequest(t *testing.T, selector map[string]string) *http.Request { t.Helper() + g := NewWithT(t) body, err := json.Marshal(ResolveRequest{Selector: selector}) - if err != nil { - t.Fatalf("failed to marshal request: %v", err) - } - req := httptest.NewRequest(http.MethodPost, resolvePath, bytes.NewReader(body)) + g.Expect(err).ToNot(HaveOccurred()) + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, resolvePath, bytes.NewReader(body)) + g.Expect(err).ToNot(HaveOccurred()) req.Header.Set("Content-Type", "application/json") return req } @@ -134,6 +136,7 @@ func TestResolverHandler(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) lister := newFakePodLister("test-namespace", tt.pods) handler := newResolverHandler(lister) req := newResolveRequest(t, tt.selector) @@ -141,22 +144,15 @@ func TestResolverHandler(t *testing.T) { handler.ServeHTTP(rec, req) - if rec.Code != tt.expectedCode { - t.Errorf("expected status code %d, got %d: %s", tt.expectedCode, rec.Code, rec.Body.String()) - } + g.Expect(rec.Code).To(Equal(tt.expectedCode)) if tt.expectedPods != nil { var response ResolveResponse - if err := json.NewDecoder(rec.Body).Decode(&response); err != nil { - t.Fatalf("failed to decode response: %v", err) - } - if len(response.Pods) != len(tt.expectedPods) { - t.Fatalf("expected %d pods, got %d", len(tt.expectedPods), len(response.Pods)) - } + g.Expect(json.NewDecoder(rec.Body).Decode(&response)).To(Succeed()) + g.Expect(response.Pods).To(HaveLen(len(tt.expectedPods))) for i, expected := range tt.expectedPods { - if response.Pods[i].Name != expected.Name || response.Pods[i].IP != expected.IP { - t.Errorf("pod %d: expected %+v, got %+v", i, expected, response.Pods[i]) - } + g.Expect(response.Pods[i].Name).To(Equal(expected.Name)) + g.Expect(response.Pods[i].IP).To(Equal(expected.IP)) } } }) @@ -165,21 +161,22 @@ func TestResolverHandler(t *testing.T) { func TestResolverHandlerMethodNotAllowed(t *testing.T) { t.Run("When sending GET request it should return 405", func(t *testing.T) { + g := NewWithT(t) lister := newFakePodLister("test-namespace", nil) handler := newResolverHandler(lister) - req := httptest.NewRequest(http.MethodGet, resolvePath, nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, resolvePath, nil) + g.Expect(err).ToNot(HaveOccurred()) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) - if rec.Code != http.StatusMethodNotAllowed { - t.Errorf("expected status code %d, got %d", http.StatusMethodNotAllowed, rec.Code) - } + g.Expect(rec.Code).To(Equal(http.StatusMethodNotAllowed)) }) } func TestResolverHandlerEmptySelector(t *testing.T) { t.Run("When selector is empty it should return 400", func(t *testing.T) { + g := NewWithT(t) lister := newFakePodLister("test-namespace", nil) handler := newResolverHandler(lister) req := newResolveRequest(t, map[string]string{}) @@ -187,23 +184,21 @@ func TestResolverHandlerEmptySelector(t *testing.T) { handler.ServeHTTP(rec, req) - if rec.Code != http.StatusBadRequest { - t.Errorf("expected status code %d, got %d", http.StatusBadRequest, rec.Code) - } + g.Expect(rec.Code).To(Equal(http.StatusBadRequest)) }) } func TestResolverHandlerInvalidBody(t *testing.T) { t.Run("When request body is invalid JSON it should return 400", func(t *testing.T) { + g := NewWithT(t) lister := newFakePodLister("test-namespace", nil) handler := newResolverHandler(lister) - req := httptest.NewRequest(http.MethodPost, resolvePath, bytes.NewReader([]byte("not json"))) + req, err := http.NewRequestWithContext(t.Context(), http.MethodPost, resolvePath, bytes.NewReader([]byte("not json"))) + g.Expect(err).ToNot(HaveOccurred()) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) - if rec.Code != http.StatusBadRequest { - t.Errorf("expected status code %d, got %d", http.StatusBadRequest, rec.Code) - } + g.Expect(rec.Code).To(Equal(http.StatusBadRequest)) }) } diff --git a/control-plane-operator/hostedclusterconfigoperator/cmd.go b/control-plane-operator/hostedclusterconfigoperator/cmd.go index d5a34ea14fbf..a201321fe7f1 100644 --- a/control-plane-operator/hostedclusterconfigoperator/cmd.go +++ b/control-plane-operator/hostedclusterconfigoperator/cmd.go @@ -231,10 +231,10 @@ func (o *HostedClusterConfigOperator) Run(ctx context.Context) error { opt.Scheme = api.Scheme }) if err != nil { - return fmt.Errorf("cannot create control plane cluster: %v", err) + return fmt.Errorf("cannot create control plane cluster: %w", err) } if err := mgr.Add(cpCluster); err != nil { - return fmt.Errorf("cannot add CPCluster to manager: %v", err) + return fmt.Errorf("cannot add CPCluster to manager: %w", err) } var kubevirtInfraConfig *rest.Config if o.KubevirtInfraKubeconfig != "" { diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/hcpstatus/hcpstatus.go b/control-plane-operator/hostedclusterconfigoperator/controllers/hcpstatus/hcpstatus.go index 6ffe25792713..df6b02064726 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/hcpstatus/hcpstatus.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/hcpstatus/hcpstatus.go @@ -1,9 +1,10 @@ package hcpstatus import ( + "bytes" "context" + "encoding/json" "fmt" - "reflect" hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/operator" @@ -11,6 +12,7 @@ import ( configv1 "github.com/openshift/api/config/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -74,17 +76,76 @@ func (h *hcpStatusReconciler) Reconcile(ctx context.Context, req reconcile.Reque return reconcile.Result{}, err } - if !reflect.DeepEqual(hcp.Status, originalHCP.Status) { - log.Info("Updating HCP status with new configuration and version status") - if err := h.mgtClusterClient.Status().Update(ctx, hcp); err != nil { - return reconcile.Result{}, fmt.Errorf("failed to update hcp: %w", err) - } - log.Info("Successfully updated HCP status") + if equality.Semantic.DeepEqual(hcp.Status, originalHCP.Status) { + return reconcile.Result{}, nil + } + + // Use JSON Patch (RFC 6902) instead of JSON Merge Patch (RFC 7386). + // Merge patch interprets null as "delete field" (RFC 7386 §7), which corrupts + // +required +nullable fields in configv1.ClusterVersionStatus that have no + // omitempty and serialize nil as JSON null: + // - AvailableUpdates []configv1.Release `json:"availableUpdates"` — nil slice → null + // - CompletionTime *metav1.Time `json:"completionTime"` (in UpdateHistory) — nil pointer → null + // JSON Patch "replace"/"add" ops carry null as a literal value, preserving it correctly. + patch, err := buildStatusPatch(originalHCP, hcp) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to build status patch: %w", err) + } + + log.Info("Patching HCP status with new configuration and version status") + if err := h.mgtClusterClient.Status().Patch(ctx, hcp, + crclient.RawPatch(types.JSONPatchType, patch)); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to patch hcp status: %w", err) } + log.Info("Successfully patched HCP status") return reconcile.Result{}, nil } +type jsonPatchOp struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value,omitempty"` +} + +func buildStatusPatch(original, modified *hyperv1.HostedControlPlane) ([]byte, error) { + origJSON, err := json.Marshal(original.Status) + if err != nil { + return nil, err + } + modJSON, err := json.Marshal(modified.Status) + if err != nil { + return nil, err + } + + var origMap, modMap map[string]json.RawMessage + if err := json.Unmarshal(origJSON, &origMap); err != nil { + return nil, err + } + if err := json.Unmarshal(modJSON, &modMap); err != nil { + return nil, err + } + + // Optimistic lock: fail if the object was modified since our read. + ops := []jsonPatchOp{{Op: "test", Path: "/metadata/resourceVersion", Value: original.ResourceVersion}} + + for key, modVal := range modMap { + origVal, exists := origMap[key] + if !exists { + ops = append(ops, jsonPatchOp{Op: "add", Path: "/status/" + key, Value: modVal}) + } else if !bytes.Equal(origVal, modVal) { + ops = append(ops, jsonPatchOp{Op: "replace", Path: "/status/" + key, Value: modVal}) + } + } + for key := range origMap { + if _, exists := modMap[key]; !exists { + ops = append(ops, jsonPatchOp{Op: "remove", Path: "/status/" + key}) + } + } + + return json.Marshal(ops) +} + // findClusterOperatorStatusCondition is identical to meta.FindStatusCondition except that it works on config1.ClusterOperatorStatusCondition instead of // metav1.StatusCondition func findClusterOperatorStatusCondition(conditions []configv1.ClusterOperatorStatusCondition, conditionType configv1.ClusterStatusConditionType) *configv1.ClusterOperatorStatusCondition { diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/hcpstatus/hcpstatus_test.go b/control-plane-operator/hostedclusterconfigoperator/controllers/hcpstatus/hcpstatus_test.go new file mode 100644 index 000000000000..73655f694287 --- /dev/null +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/hcpstatus/hcpstatus_test.go @@ -0,0 +1,541 @@ +package hcpstatus + +import ( + "encoding/json" + "testing" + + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/support/api" + + configv1 "github.com/openshift/api/config/v1" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + crclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func TestHCPStatusReconciler(t *testing.T) { + t.Parallel() + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-hcp", + Namespace: "test-ns", + }, + } + + expectedOAuthConfigMapName := "oauth-metadata-configmap" + + tests := []struct { + name string + hostedClusterObjects []crclient.Object + expectError bool + expectedOAuthName string + validateConditions func(g Gomega, hcp *hyperv1.HostedControlPlane) + }{ + { + name: "When Authentication resource exists it should propagate status to HCP", + hostedClusterObjects: []crclient.Object{ + &configv1.ClusterVersion{ObjectMeta: metav1.ObjectMeta{Name: "version"}}, + &configv1.Authentication{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, + Status: configv1.AuthenticationStatus{ + IntegratedOAuthMetadata: configv1.ConfigMapNameReference{ + Name: expectedOAuthConfigMapName, + }, + }, + }, + }, + expectedOAuthName: expectedOAuthConfigMapName, + }, + { + name: "When Authentication resource is missing it should return an error", + hostedClusterObjects: []crclient.Object{ + &configv1.ClusterVersion{ObjectMeta: metav1.ObjectMeta{Name: "version"}}, + }, + expectError: true, + }, + { + name: "When ClusterVersion has conditions it should propagate them to HCP", + hostedClusterObjects: []crclient.Object{ + &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{Name: "version"}, + Status: configv1.ClusterVersionStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionTrue, + Reason: "AsExpected", + Message: "cluster is available", + }, + { + Type: configv1.OperatorProgressing, + Status: configv1.ConditionFalse, + Reason: "AsExpected", + Message: "cluster is not progressing", + }, + }, + }, + }, + &configv1.Authentication{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, + Status: configv1.AuthenticationStatus{ + IntegratedOAuthMetadata: configv1.ConfigMapNameReference{Name: expectedOAuthConfigMapName}, + }, + }, + }, + expectedOAuthName: expectedOAuthConfigMapName, + validateConditions: func(g Gomega, hcp *hyperv1.HostedControlPlane) { + availableCond := meta.FindStatusCondition(hcp.Status.Conditions, string(hyperv1.ClusterVersionAvailable)) + g.Expect(availableCond).NotTo(BeNil()) + g.Expect(availableCond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(availableCond.Reason).To(Equal("AsExpected")) + + progressingCond := meta.FindStatusCondition(hcp.Status.Conditions, string(hyperv1.ClusterVersionProgressing)) + g.Expect(progressingCond).NotTo(BeNil()) + g.Expect(progressingCond.Status).To(Equal(metav1.ConditionFalse)) + }, + }, + { + name: "When ClusterVersion has no Upgradeable condition it should default to True", + hostedClusterObjects: []crclient.Object{ + &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{Name: "version"}, + Status: configv1.ClusterVersionStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionTrue, + Reason: "AsExpected", + }, + }, + }, + }, + &configv1.Authentication{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, + Status: configv1.AuthenticationStatus{ + IntegratedOAuthMetadata: configv1.ConfigMapNameReference{Name: expectedOAuthConfigMapName}, + }, + }, + }, + expectedOAuthName: expectedOAuthConfigMapName, + validateConditions: func(g Gomega, hcp *hyperv1.HostedControlPlane) { + upgradeableCond := meta.FindStatusCondition(hcp.Status.Conditions, string(hyperv1.ClusterVersionUpgradeable)) + g.Expect(upgradeableCond).NotTo(BeNil()) + g.Expect(upgradeableCond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(upgradeableCond.Reason).To(Equal(hyperv1.FromClusterVersionReason)) + }, + }, + { + name: "When CVO condition has empty Reason it should use FromClusterVersionReason", + hostedClusterObjects: []crclient.Object{ + &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{Name: "version"}, + Status: configv1.ClusterVersionStatus{ + Conditions: []configv1.ClusterOperatorStatusCondition{ + { + Type: configv1.OperatorAvailable, + Status: configv1.ConditionTrue, + Reason: "", + Message: "all good", + }, + }, + }, + }, + &configv1.Authentication{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster"}, + Status: configv1.AuthenticationStatus{ + IntegratedOAuthMetadata: configv1.ConfigMapNameReference{Name: expectedOAuthConfigMapName}, + }, + }, + }, + expectedOAuthName: expectedOAuthConfigMapName, + validateConditions: func(g Gomega, hcp *hyperv1.HostedControlPlane) { + availableCond := meta.FindStatusCondition(hcp.Status.Conditions, string(hyperv1.ClusterVersionAvailable)) + g.Expect(availableCond).NotTo(BeNil()) + g.Expect(availableCond.Reason).To(Equal(hyperv1.FromClusterVersionReason)) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + mgmtClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(hcp.DeepCopy()). + WithStatusSubresource(&hyperv1.HostedControlPlane{}). + Build() + + hostedClusterClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(tt.hostedClusterObjects...). + Build() + + reconciler := &hcpStatusReconciler{ + mgtClusterClient: mgmtClient, + hostedClusterClient: hostedClusterClient, + } + + _, err := reconciler.Reconcile(t.Context(), reconcile.Request{ + NamespacedName: types.NamespacedName{ + Name: hcp.Name, + Namespace: hcp.Namespace, + }, + }) + + if tt.expectError { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("Authentication"), + "error should be about missing Authentication resource, got: %v", err) + return + } + + g.Expect(err).NotTo(HaveOccurred()) + updatedHCP := &hyperv1.HostedControlPlane{} + g.Expect(mgmtClient.Get(t.Context(), crclient.ObjectKeyFromObject(hcp), updatedHCP)).To(Succeed()) + g.Expect(updatedHCP.Status.Configuration).NotTo(BeNil()) + g.Expect(updatedHCP.Status.Configuration.Authentication.IntegratedOAuthMetadata.Name).To(Equal(tt.expectedOAuthName)) + + if tt.validateConditions != nil { + tt.validateConditions(g, updatedHCP) + } + }) + } +} + +func TestBuildStatusPatch(t *testing.T) { + t.Parallel() + + t.Run("When versionStatus changes, it should produce a replace op with optimistic lock", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + ResourceVersion: "100", + }, + Status: hyperv1.HostedControlPlaneStatus{ + VersionStatus: &hyperv1.ClusterVersionStatus{ + Desired: configv1.Release{Version: "4.16.0", Image: "quay.io/old:latest"}, + }, + }, + } + original := hcp.DeepCopy() + + hcp.Status.VersionStatus = &hyperv1.ClusterVersionStatus{ + Desired: configv1.Release{Version: "4.17.0", Image: "quay.io/new:latest"}, + } + + patchBytes, err := buildStatusPatch(original, hcp) + g.Expect(err).ToNot(HaveOccurred()) + + var ops []jsonPatchOp + g.Expect(json.Unmarshal(patchBytes, &ops)).To(Succeed()) + + g.Expect(ops).To(HaveLen(2)) + + g.Expect(ops[0].Op).To(Equal("test")) + g.Expect(ops[0].Path).To(Equal("/metadata/resourceVersion")) + g.Expect(ops[0].Value).To(Equal("100")) + + g.Expect(ops[1].Op).To(Equal("replace")) + g.Expect(ops[1].Path).To(Equal("/status/versionStatus")) + }) + + t.Run("When versionStatus is added for the first time, it should produce an add op", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + ResourceVersion: "100", + }, + } + original := hcp.DeepCopy() + + hcp.Status.VersionStatus = &hyperv1.ClusterVersionStatus{ + Desired: configv1.Release{Version: "4.17.0", Image: "quay.io/test:latest"}, + } + + patchBytes, err := buildStatusPatch(original, hcp) + g.Expect(err).ToNot(HaveOccurred()) + + var ops []jsonPatchOp + g.Expect(json.Unmarshal(patchBytes, &ops)).To(Succeed()) + + g.Expect(ops).To(HaveLen(2)) + g.Expect(ops[1].Op).To(Equal("add")) + g.Expect(ops[1].Path).To(Equal("/status/versionStatus")) + }) + + t.Run("When nothing changes, it should produce only the test op", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + ResourceVersion: "100", + }, + Status: hyperv1.HostedControlPlaneStatus{ + Conditions: []metav1.Condition{ + {Type: "ConditionA", Status: metav1.ConditionTrue, Reason: "OK"}, + }, + }, + } + original := hcp.DeepCopy() + + patchBytes, err := buildStatusPatch(original, hcp) + g.Expect(err).ToNot(HaveOccurred()) + + var ops []jsonPatchOp + g.Expect(json.Unmarshal(patchBytes, &ops)).To(Succeed()) + + g.Expect(ops).To(HaveLen(1)) + g.Expect(ops[0].Op).To(Equal("test")) + g.Expect(ops[0].Path).To(Equal("/metadata/resourceVersion")) + }) + + t.Run("When conditions change, it should replace the entire conditions array", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + ResourceVersion: "100", + }, + Status: hyperv1.HostedControlPlaneStatus{ + Conditions: []metav1.Condition{ + {Type: "ConditionA", Status: metav1.ConditionTrue, Reason: "OK"}, + {Type: "ConditionB", Status: metav1.ConditionTrue, Reason: "OK"}, + }, + }, + } + original := hcp.DeepCopy() + + meta.SetStatusCondition(&hcp.Status.Conditions, metav1.Condition{ + Type: "ConditionA", + Status: metav1.ConditionFalse, + Reason: "NowBad", + }) + + patchBytes, err := buildStatusPatch(original, hcp) + g.Expect(err).ToNot(HaveOccurred()) + + var ops []jsonPatchOp + g.Expect(json.Unmarshal(patchBytes, &ops)).To(Succeed()) + + g.Expect(ops).To(HaveLen(2)) + g.Expect(ops[1].Op).To(Equal("replace")) + g.Expect(ops[1].Path).To(Equal("/status/conditions")) + + conditionsJSON, err := json.Marshal(ops[1].Value) + g.Expect(err).ToNot(HaveOccurred()) + var conditions []metav1.Condition + g.Expect(json.Unmarshal(conditionsJSON, &conditions)).To(Succeed()) + g.Expect(conditions).To(HaveLen(2), "all conditions from the read should be in the patch") + }) + + t.Run("When versionStatus has nil availableUpdates and nil completionTime, the patch should preserve null values", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + ResourceVersion: "100", + }, + } + original := hcp.DeepCopy() + + startedTime := metav1.Now() + hcp.Status.VersionStatus = &hyperv1.ClusterVersionStatus{ + Desired: configv1.Release{Version: "4.17.0", Image: "quay.io/test:latest"}, + History: []configv1.UpdateHistory{ + { + State: configv1.PartialUpdate, + StartedTime: startedTime, + // CompletionTime is nil — update in progress + Version: "4.17.0", + Image: "quay.io/test:latest", + }, + }, + // AvailableUpdates is nil — CVO hasn't checked yet + } + + patchBytes, err := buildStatusPatch(original, hcp) + g.Expect(err).ToNot(HaveOccurred()) + + // Parse the raw JSON to verify null handling. + // JSON Patch (RFC 6902) uses "value": null to mean "set to null", + // unlike JSON Merge Patch (RFC 7386) where null means "delete". + var rawOps []json.RawMessage + g.Expect(json.Unmarshal(patchBytes, &rawOps)).To(Succeed()) + g.Expect(rawOps).To(HaveLen(2)) + + // Parse the versionStatus op's value to check null fields + var op struct { + Op string `json:"op"` + Path string `json:"path"` + Value json.RawMessage `json:"value"` + } + g.Expect(json.Unmarshal(rawOps[1], &op)).To(Succeed()) + g.Expect(op.Op).To(Equal("add")) + g.Expect(op.Path).To(Equal("/status/versionStatus")) + + var vs map[string]interface{} + g.Expect(json.Unmarshal(op.Value, &vs)).To(Succeed()) + + // availableUpdates should be null (nil slice serializes as null with no omitempty) + au, ok := vs["availableUpdates"] + g.Expect(ok).To(BeTrue(), "availableUpdates key must be present in the patch") + g.Expect(au).To(BeNil(), "availableUpdates should be null — JSON Patch preserves this correctly") + + // completionTime in history should be null (nil *metav1.Time) + history := vs["history"].([]interface{}) + entry := history[0].(map[string]interface{}) + ct, ok := entry["completionTime"] + g.Expect(ok).To(BeTrue(), "completionTime key must be present in the patch") + g.Expect(ct).To(BeNil(), "completionTime should be null — JSON Patch preserves this correctly") + }) + + t.Run("When configuration changes, it should produce a replace op", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + ResourceVersion: "100", + }, + Status: hyperv1.HostedControlPlaneStatus{ + Configuration: &hyperv1.ConfigurationStatus{}, + }, + } + original := hcp.DeepCopy() + + hcp.Status.Configuration = &hyperv1.ConfigurationStatus{ + Authentication: configv1.AuthenticationStatus{ + IntegratedOAuthMetadata: configv1.ConfigMapNameReference{Name: "oauth-metadata"}, + }, + } + + patchBytes, err := buildStatusPatch(original, hcp) + g.Expect(err).ToNot(HaveOccurred()) + + var ops []jsonPatchOp + g.Expect(json.Unmarshal(patchBytes, &ops)).To(Succeed()) + + g.Expect(ops).To(HaveLen(2)) + g.Expect(ops[1].Op).To(Equal("replace")) + g.Expect(ops[1].Path).To(Equal("/status/configuration")) + }) + + t.Run("When versionStatus is removed, it should produce a remove op", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + ResourceVersion: "100", + }, + Status: hyperv1.HostedControlPlaneStatus{ + VersionStatus: &hyperv1.ClusterVersionStatus{ + Desired: configv1.Release{Version: "4.17.0", Image: "quay.io/test:latest"}, + }, + }, + } + original := hcp.DeepCopy() + + hcp.Status.VersionStatus = nil + + patchBytes, err := buildStatusPatch(original, hcp) + g.Expect(err).ToNot(HaveOccurred()) + + var ops []jsonPatchOp + g.Expect(json.Unmarshal(patchBytes, &ops)).To(Succeed()) + + g.Expect(ops).To(HaveLen(2)) + g.Expect(ops[0].Op).To(Equal("test")) + g.Expect(ops[1].Op).To(Equal("remove")) + g.Expect(ops[1].Path).To(Equal("/status/versionStatus")) + }) + + t.Run("When multiple fields change, it should produce ops for each changed field", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: "test-ns", + ResourceVersion: "100", + }, + Status: hyperv1.HostedControlPlaneStatus{ + VersionStatus: &hyperv1.ClusterVersionStatus{ + Desired: configv1.Release{Version: "4.16.0"}, + }, + Conditions: []metav1.Condition{ + {Type: "ConditionA", Status: metav1.ConditionTrue, Reason: "OK"}, + }, + Configuration: &hyperv1.ConfigurationStatus{}, + Version: "4.16.0", + ReleaseImage: "quay.io/old:latest", + }, + } + original := hcp.DeepCopy() + + hcp.Status.VersionStatus.Desired.Version = "4.17.0" + meta.SetStatusCondition(&hcp.Status.Conditions, metav1.Condition{ + Type: "ConditionA", Status: metav1.ConditionFalse, Reason: "Bad", + }) + hcp.Status.Configuration = &hyperv1.ConfigurationStatus{ + Authentication: configv1.AuthenticationStatus{ + IntegratedOAuthMetadata: configv1.ConfigMapNameReference{Name: "new"}, + }, + } + hcp.Status.Version = "4.17.0" + hcp.Status.ReleaseImage = "quay.io/new:latest" + + patchBytes, err := buildStatusPatch(original, hcp) + g.Expect(err).ToNot(HaveOccurred()) + + var ops []jsonPatchOp + g.Expect(json.Unmarshal(patchBytes, &ops)).To(Succeed()) + + // test + versionStatus + conditions + configuration + version + releaseImage + g.Expect(ops).To(HaveLen(6)) + + paths := make([]string, len(ops)) + for i, op := range ops { + paths[i] = op.Path + } + g.Expect(paths).To(ContainElements( + "/metadata/resourceVersion", + "/status/versionStatus", + "/status/conditions", + "/status/configuration", + "/status/version", + "/status/releaseImage", + )) + }) +} diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/ingress/reconcile.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/ingress/reconcile.go index d1b0161f1884..bd06b42b091d 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/ingress/reconcile.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/ingress/reconcile.go @@ -128,7 +128,7 @@ func ReconcileDefaultIngressControllerCertSecret(certSecret *corev1.Secret, sour return fmt.Errorf("source secret %s/%s does not have a cert key", sourceSecret.Namespace, sourceSecret.Name) } if _, hasKeyKey := sourceSecret.Data[corev1.TLSPrivateKeyKey]; !hasKeyKey { - return fmt.Errorf("source secret %s/%s does not have a key key", sourceSecret.Namespace, sourceSecret.Name) + return fmt.Errorf("source secret %s/%s does not have the expected key", sourceSecret.Namespace, sourceSecret.Name) } certSecret.Data = map[string][]byte{} diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/ingress/reconcile_test.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/ingress/reconcile_test.go index 672f28640e5c..7033877dee61 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/ingress/reconcile_test.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/ingress/reconcile_test.go @@ -998,3 +998,64 @@ func TestConfigurationPriority(t *testing.T) { }) } } + +func TestReconcileDefaultIngressControllerCertSecret(t *testing.T) { + t.Parallel() + tests := []struct { + name string + sourceSecret *corev1.Secret + wantErr bool + errSubstr string + }{ + { + name: "When source secret has both cert and key, it should succeed", + sourceSecret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "source", Namespace: "test-ns"}, + Data: map[string][]byte{ + corev1.TLSCertKey: []byte("cert-data"), + corev1.TLSPrivateKeyKey: []byte("key-data"), + }, + }, + }, + { + name: "When source secret is missing the cert key, it should return an error", + sourceSecret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "source", Namespace: "test-ns"}, + Data: map[string][]byte{ + corev1.TLSPrivateKeyKey: []byte("key-data"), + }, + }, + wantErr: true, + errSubstr: "does not have a cert key", + }, + { + name: "When source secret is missing the private key, it should return an error", + sourceSecret: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "source", Namespace: "test-ns"}, + Data: map[string][]byte{ + corev1.TLSCertKey: []byte("cert-data"), + }, + }, + wantErr: true, + errSubstr: "does not have the expected key", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + certSecret := &corev1.Secret{} + err := ReconcileDefaultIngressControllerCertSecret(certSecret, tt.sourceSecret) + + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(certSecret.Data).To(HaveKeyWithValue(corev1.TLSCertKey, tt.sourceSecret.Data[corev1.TLSCertKey])) + g.Expect(certSecret.Data).To(HaveKeyWithValue(corev1.TLSPrivateKeyKey, tt.sourceSecret.Data[corev1.TLSPrivateKeyKey])) + } + }) + } +} diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas/admissionpolicies.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas/admissionpolicies.go index b7708238a773..c7b7bed7ceef 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas/admissionpolicies.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas/admissionpolicies.go @@ -61,15 +61,15 @@ var ( // from being updated/deleted from the DataPlane side. func ReconcileKASValidatingAdmissionPolicies(ctx context.Context, hcp *hyperv1.HostedControlPlane, client client.Client, createOrUpdate upsert.CreateOrUpdateFN) error { if err := reconcileConfigValidatingAdmissionPolicy(ctx, hcp, client, createOrUpdate); err != nil { - return fmt.Errorf("failed to reconcile Config Validating Admission Policy: %v", err) + return fmt.Errorf("failed to reconcile Config Validating Admission Policy: %w", err) } if err := reconcileMirrorValidatingAdmissionPolicy(ctx, hcp, client, createOrUpdate); err != nil { - return fmt.Errorf("failed to reconcile Mirror Validating Admission Policies: %v", err) + return fmt.Errorf("failed to reconcile Mirror Validating Admission Policies: %w", err) } if err := reconcileInfraValidatingAdmissionPolicy(ctx, hcp, client, createOrUpdate); err != nil { - return fmt.Errorf("failed to reconcile Infrastructure Validating Admission Policy: %v", err) + return fmt.Errorf("failed to reconcile Infrastructure Validating Admission Policy: %w", err) } if err := reconcileConfigMapsValidatingAdmissionPolicy(ctx, client, createOrUpdate); err != nil { @@ -109,7 +109,7 @@ func reconcileConfigValidatingAdmissionPolicy(ctx context.Context, hcp *hyperv1. configAdmissionPolicy.Validations = []k8sadmissionv1.Validation{HCCOUserValidation} configAdmissionPolicy.MatchConstraints = constructPolicyMatchConstraints(configResources, configAPIVersion, configAPIGroup, []k8sadmissionv1.OperationType{"UPDATE", "DELETE"}) if err := configAdmissionPolicy.reconcileAdmissionPolicy(ctx, client, createOrUpdate); err != nil { - return fmt.Errorf("error reconciling Config Validating Admission Policy: %v", err) + return fmt.Errorf("error reconciling Config Validating Admission Policy: %w", err) } return nil @@ -128,7 +128,7 @@ func reconcileInfraValidatingAdmissionPolicy(ctx context.Context, _ *hyperv1.Hos infraAdmissionPolicy.Validations = []k8sadmissionv1.Validation{HCCOUserValidation} infraAdmissionPolicy.MatchConstraints = constructPolicyMatchConstraints(infraResources, infraAPIVersion, infraAPIGroup, []k8sadmissionv1.OperationType{"UPDATE", "DELETE"}) if err := infraAdmissionPolicy.reconcileAdmissionPolicy(ctx, client, createOrUpdate); err != nil { - return fmt.Errorf("error reconciling Infrastructure Validating Admission Policy: %v", err) + return fmt.Errorf("error reconciling Infrastructure Validating Admission Policy: %w", err) } return nil @@ -152,7 +152,7 @@ func reconcileMirrorValidatingAdmissionPolicy(ctx context.Context, hcp *hyperv1. mirrorAdmissionPolicy.Validations = []k8sadmissionv1.Validation{HCCOUserValidation} mirrorAdmissionPolicy.MatchConstraints = constructPolicyMatchConstraints(mirrorResources, mirrorAPIVersion, mirrorAPIGroup, allAdmissionPoliciesOperations) if err := mirrorAdmissionPolicy.reconcileAdmissionPolicy(ctx, client, createOrUpdate); err != nil { - return fmt.Errorf("error reconciling Mirror Validating Admission Policy: %v", err) + return fmt.Errorf("error reconciling Mirror Validating Admission Policy: %w", err) } // ICSP lives in other API, this is why we need to create another vap and vap-binding @@ -164,7 +164,7 @@ func reconcileMirrorValidatingAdmissionPolicy(ctx context.Context, hcp *hyperv1. icspAdmissionPolicy.Validations = []k8sadmissionv1.Validation{HCCOUserValidation} icspAdmissionPolicy.MatchConstraints = constructPolicyMatchConstraints(icspResources, icspAPIVersion, icspAPIGroup, allAdmissionPoliciesOperations) if err := icspAdmissionPolicy.reconcileAdmissionPolicy(ctx, client, createOrUpdate); err != nil { - return fmt.Errorf("error reconciling ICSP Validating Admission Policy: %v", err) + return fmt.Errorf("error reconciling ICSP Validating Admission Policy: %w", err) } return nil @@ -182,7 +182,7 @@ func reconcileConfigMapsValidatingAdmissionPolicy(ctx context.Context, client cl // we want to block changes only for configmaps with "hypershift.openshift.io/mirrored-config" label mirroredConfigsAdmissionPolicy.MatchConstraints.ObjectSelector = &metav1.LabelSelector{MatchLabels: map[string]string{nodepool.NTOMirroredConfigLabel: "true"}} if err := mirroredConfigsAdmissionPolicy.reconcileAdmissionPolicy(ctx, client, createOrUpdate); err != nil { - return fmt.Errorf("error reconciling mirrored ConfigMaps Validating Admission Policy: %v", err) + return fmt.Errorf("error reconciling mirrored ConfigMaps Validating Admission Policy: %w", err) } return nil } @@ -200,7 +200,7 @@ func (ap *AdmissionPolicy) reconcileAdmissionPolicy(ctx context.Context, client return nil }); err != nil { - return fmt.Errorf("failed to create/update Validating Admission Policy with name %s: %v", ap.Name, err) + return fmt.Errorf("failed to create/update Validating Admission Policy with name %s: %w", ap.Name, err) } policyBinding := manifests.ValidatingAdmissionPolicyBinding(fmt.Sprintf("%s-binding", ap.Name)) @@ -209,7 +209,7 @@ func (ap *AdmissionPolicy) reconcileAdmissionPolicy(ctx context.Context, client policyBinding.Spec.ValidationActions = []k8sadmissionv1.ValidationAction{k8sadmissionv1.Deny} return nil }); err != nil { - return fmt.Errorf("failed to create/update Validating Admission Policy Binding with name %s: %v", ap.Name, err) + return fmt.Errorf("failed to create/update Validating Admission Policy Binding with name %s: %w", ap.Name, err) } return nil diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas/admissionpolicies_test.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas/admissionpolicies_test.go index 83f7a84b3b8c..991eb8b66276 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas/admissionpolicies_test.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas/admissionpolicies_test.go @@ -1,10 +1,22 @@ package kas import ( + "context" + "fmt" "testing" . "github.com/onsi/gomega" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/support/upsert" + + k8sadmissionv1 "k8s.io/api/admissionregistration/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "github.com/google/cel-go/cel" ) @@ -127,3 +139,150 @@ func TestGenerateCelExpression(t *testing.T) { }) } } + +func failOnNthCreateOrUpdate(n int) upsert.CreateOrUpdateFN { + call := 0 + return func(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) { + call++ + if call == n { + return controllerutil.OperationResultNone, fmt.Errorf("injected failure on call %d", n) + } + return upsert.New(false).CreateOrUpdate(ctx, c, obj, f) + } +} + +func TestReconcileAdmissionPolicy(t *testing.T) { + tests := []struct { + name string + createOrUpdate upsert.CreateOrUpdateFN + wantErr bool + errSubstr string + }{ + { + name: "When createOrUpdate succeeds, it should create VAP and binding without error", + createOrUpdate: upsert.New(false).CreateOrUpdate, + }, + { + name: "When VAP createOrUpdate fails, it should return a wrapped error", + createOrUpdate: func(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) { + if _, ok := obj.(*k8sadmissionv1.ValidatingAdmissionPolicy); ok { + return controllerutil.OperationResultNone, fmt.Errorf("API conflict") + } + return upsert.New(false).CreateOrUpdate(ctx, c, obj, f) + }, + wantErr: true, + errSubstr: "failed to create/update Validating Admission Policy with name", + }, + { + name: "When binding createOrUpdate fails, it should return a wrapped error", + createOrUpdate: func(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) { + if _, ok := obj.(*k8sadmissionv1.ValidatingAdmissionPolicyBinding); ok { + return controllerutil.OperationResultNone, fmt.Errorf("API conflict") + } + return upsert.New(false).CreateOrUpdate(ctx, c, obj, f) + }, + wantErr: true, + errSubstr: "failed to create/update Validating Admission Policy Binding with name", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + scheme := fake.NewClientBuilder().Build().Scheme() + g.Expect(k8sadmissionv1.AddToScheme(scheme)).To(Succeed()) + c := fake.NewClientBuilder().WithScheme(scheme).Build() + + ap := &AdmissionPolicy{ + Name: "test-policy", + Validations: []k8sadmissionv1.Validation{{Expression: "true", Message: "allowed"}}, + MatchConstraints: &k8sadmissionv1.MatchResources{ + ResourceRules: []k8sadmissionv1.NamedRuleWithOperations{ + { + RuleWithOperations: k8sadmissionv1.RuleWithOperations{ + Operations: []k8sadmissionv1.OperationType{"CREATE"}, + }, + }, + }, + }, + } + + err := ap.reconcileAdmissionPolicy(t.Context(), c, tt.createOrUpdate) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} + +func TestReconcileKASValidatingAdmissionPolicies(t *testing.T) { + tests := []struct { + name string + createOrUpdate upsert.CreateOrUpdateFN + wantErr bool + errSubstr string + }{ + { + name: "When all sub-reconcilers succeed, it should return no error", + createOrUpdate: upsert.New(false).CreateOrUpdate, + }, + { + name: "When Config VAP reconcile fails, it should return a wrapped error", + createOrUpdate: func(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) { + return controllerutil.OperationResultNone, fmt.Errorf("injected failure") + }, + wantErr: true, + errSubstr: "failed to reconcile Config Validating Admission Policy", + }, + { + name: "When Mirror VAP reconcile fails, it should return a wrapped error", + createOrUpdate: failOnNthCreateOrUpdate(3), + wantErr: true, + errSubstr: "failed to reconcile Mirror Validating Admission Policies", + }, + { + name: "When ICSP VAP reconcile fails, it should return a wrapped ICSP error", + createOrUpdate: failOnNthCreateOrUpdate(5), + wantErr: true, + errSubstr: "error reconciling ICSP Validating Admission Policy", + }, + { + name: "When Infra VAP reconcile fails, it should return a wrapped error", + createOrUpdate: failOnNthCreateOrUpdate(7), + wantErr: true, + errSubstr: "failed to reconcile Infrastructure Validating Admission Policy", + }, + { + name: "When ConfigMaps VAP reconcile fails, it should return a wrapped ConfigMaps error", + createOrUpdate: failOnNthCreateOrUpdate(9), + wantErr: true, + errSubstr: "error reconciling mirrored ConfigMaps Validating Admission Policy", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + scheme := fake.NewClientBuilder().Build().Scheme() + g.Expect(k8sadmissionv1.AddToScheme(scheme)).To(Succeed()) + c := fake.NewClientBuilder().WithScheme(scheme).Build() + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{Name: "test-hcp", Namespace: "test-ns"}, + } + + err := ReconcileKASValidatingAdmissionPolicies(t.Context(), hcp, c, tt.createOrUpdate) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + } + }) + } +} diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kubeadminpassword/reconcile.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kubeadminpassword/reconcile.go index 26d9c0192d32..f7956dd68191 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kubeadminpassword/reconcile.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kubeadminpassword/reconcile.go @@ -9,13 +9,15 @@ import ( ) func ReconcileKubeadminPasswordHashSecret(secret *corev1.Secret, passwordSecret *corev1.Secret) error { + password := passwordSecret.Data["password"] if secret.Data != nil { hash, hasHash := secret.Data["kubeadmin"] if hasHash && len(hash) > 0 { - return nil + if bcrypt.CompareHashAndPassword(hash, password) == nil { + return nil + } } } - password := passwordSecret.Data["password"] passwordHash, err := bcrypt.GenerateFromPassword(password, bcrypt.DefaultCost) if err != nil { return fmt.Errorf("failed to generate password hash: %w", err) diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kubeadminpassword/reconcile_test.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kubeadminpassword/reconcile_test.go new file mode 100644 index 000000000000..f2241f50c7fd --- /dev/null +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kubeadminpassword/reconcile_test.go @@ -0,0 +1,77 @@ +package kubeadminpassword + +import ( + "testing" + + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "golang.org/x/crypto/bcrypt" +) + +func TestReconcileKubeadminPasswordHashSecret(t *testing.T) { + t.Parallel() + tests := map[string]struct { + existingHash []byte + password []byte + expectNewHash bool + }{ + "When no existing hash it should generate a new hash": { + password: []byte("adminpass"), + expectNewHash: true, + }, + "When existing hash does not match the password it should regenerate the hash": { + existingHash: []byte("stale-non-matching-hash"), + password: []byte("adminpass"), + expectNewHash: true, + }, + "When existing hash is a valid bcrypt hash of a different password it should regenerate the hash": { + existingHash: func() []byte { + h, _ := bcrypt.GenerateFromPassword([]byte("old-password"), bcrypt.MinCost) + return h + }(), + password: []byte("adminpass"), + expectNewHash: true, + }, + "When hash already matches password it should not regenerate": { + existingHash: func() []byte { + h, _ := bcrypt.GenerateFromPassword([]byte("adminpass"), bcrypt.MinCost) + return h + }(), + password: []byte("adminpass"), + expectNewHash: false, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + g := NewGomegaWithT(t) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubeadmin-password-hash", + Namespace: "kube-system", + }, + } + if test.existingHash != nil { + secret.Data = map[string][]byte{"kubeadmin": test.existingHash} + } + passwordSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "kubeadmin-password", + Namespace: "master-cluster1", + }, + Data: map[string][]byte{"password": test.password}, + } + + err := ReconcileKubeadminPasswordHashSecret(secret, passwordSecret) + g.Expect(err).To(BeNil()) + g.Expect(secret.Data["kubeadmin"]).ToNot(BeEmpty()) + g.Expect(bcrypt.CompareHashAndPassword(secret.Data["kubeadmin"], test.password)).To(BeNil()) + + if !test.expectNewHash { + g.Expect(secret.Data["kubeadmin"]).To(Equal(test.existingHash)) + } + }) + } +} diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/network/reconcile.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/network/reconcile.go index 22ddaa1ee605..1d22a3e2d4a5 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/network/reconcile.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/network/reconcile.go @@ -33,11 +33,10 @@ const kubevirtDefaultVXLANPort = uint32(9879) // 9880 is a currently unassigned IANA port in the user port range. const kubevirtDefaultGenevePort = uint32(9880) -// The default OVN gateway router LRP CIDR is 100.64.0.0/16 and the default UDNs -// is 100.65.0.0/16. We need to avoid that for kubernetes which runs nested. -const kubevirtDefaultV4InternalSubnet = "100.66.0.0/16" +const kubevirtDefaultV4InternalSubnet = hyperv1.KubevirtDefaultV4InternalSubnet +const kubevirtDefaultV6InternalJoinSubnet = hyperv1.KubevirtDefaultV6InternalJoinSubnet -func ReconcileNetworkOperator(network *operatorv1.Network, networkType hyperv1.NetworkType, platformType hyperv1.PlatformType, disableMultiNetwork bool, ovnConfig *hyperv1.OVNKubernetesConfig) { +func ReconcileNetworkOperator(network *operatorv1.Network, networkType hyperv1.NetworkType, platformType hyperv1.PlatformType, disableMultiNetwork bool, ovnConfig *hyperv1.OVNKubernetesConfig, hasIPv6Network bool) { switch platformType { case hyperv1.KubevirtPlatform: // Modify vxlan port to avoid collisions with management cluster's default vxlan port. @@ -61,6 +60,14 @@ func ReconcileNetworkOperator(network *operatorv1.Network, networkType hyperv1.N if network.Spec.DefaultNetwork.OVNKubernetesConfig.GenevePort == nil { network.Spec.DefaultNetwork.OVNKubernetesConfig.GenevePort = &port } + if hasIPv6Network { + if network.Spec.DefaultNetwork.OVNKubernetesConfig.IPv6 == nil { + network.Spec.DefaultNetwork.OVNKubernetesConfig.IPv6 = &operatorv1.IPv6OVNKubernetesConfig{} + } + if network.Spec.DefaultNetwork.OVNKubernetesConfig.IPv6.InternalJoinSubnet == "" { + network.Spec.DefaultNetwork.OVNKubernetesConfig.IPv6.InternalJoinSubnet = kubevirtDefaultV6InternalJoinSubnet + } + } } case hyperv1.PowerVSPlatform: if networkType == hyperv1.OVNKubernetes { @@ -95,6 +102,19 @@ func ReconcileNetworkOperator(network *operatorv1.Network, networkType hyperv1.N ovnCfg.IPv4.InternalTransitSwitchSubnet = ovnConfig.IPv4.InternalTransitSwitchSubnet } } + // Apply IPv6 configuration + if ovnConfig.IPv6.InternalJoinSubnet != "" { + if ovnCfg.IPv6 == nil { + ovnCfg.IPv6 = &operatorv1.IPv6OVNKubernetesConfig{} + } + ovnCfg.IPv6.InternalJoinSubnet = ovnConfig.IPv6.InternalJoinSubnet + } + if ovnConfig.IPv6.InternalTransitSwitchSubnet != "" { + if ovnCfg.IPv6 == nil { + ovnCfg.IPv6 = &operatorv1.IPv6OVNKubernetesConfig{} + } + ovnCfg.IPv6.InternalTransitSwitchSubnet = ovnConfig.IPv6.InternalTransitSwitchSubnet + } // Apply MTU configuration if ovnConfig.MTU > 0 { ovnCfg.MTU = ptr.To(uint32(ovnConfig.MTU)) diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/network/reconcile_test.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/network/reconcile_test.go index 534dd6587f6c..8222e38dabe4 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/network/reconcile_test.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/network/reconcile_test.go @@ -16,6 +16,7 @@ func TestReconcileDefaultIngressController(t *testing.T) { vxlanPort := kubevirtDefaultVXLANPort genevePort := kubevirtDefaultGenevePort v4InternalSubnet := kubevirtDefaultV4InternalSubnet + v6InternalJoinSubnet := kubevirtDefaultV6InternalJoinSubnet fakePort := uint32(11111) testsCases := []struct { @@ -25,6 +26,7 @@ func TestReconcileDefaultIngressController(t *testing.T) { inputPlatformType hyperv1.PlatformType disableMultiNetwork bool ovnConfig *hyperv1.OVNKubernetesConfig + hasIPv6Network bool expectedNetwork *operatorv1.Network }{ { @@ -33,6 +35,7 @@ func TestReconcileDefaultIngressController(t *testing.T) { inputNetworkType: hyperv1.OVNKubernetes, inputPlatformType: hyperv1.KubevirtPlatform, disableMultiNetwork: false, + hasIPv6Network: true, expectedNetwork: &operatorv1.Network{ ObjectMeta: NetworkOperator().ObjectMeta, Spec: operatorv1.NetworkSpec{ @@ -43,6 +46,9 @@ func TestReconcileDefaultIngressController(t *testing.T) { OVNKubernetesConfig: &operatorv1.OVNKubernetesConfig{ GenevePort: &genevePort, V4InternalSubnet: v4InternalSubnet, + IPv6: &operatorv1.IPv6OVNKubernetesConfig{ + InternalJoinSubnet: v6InternalJoinSubnet, + }, }, }, }, @@ -119,6 +125,7 @@ func TestReconcileDefaultIngressController(t *testing.T) { inputNetworkType: hyperv1.OVNKubernetes, inputPlatformType: hyperv1.KubevirtPlatform, disableMultiNetwork: false, + hasIPv6Network: true, expectedNetwork: &operatorv1.Network{ ObjectMeta: NetworkOperator().ObjectMeta, Spec: operatorv1.NetworkSpec{ @@ -129,6 +136,9 @@ func TestReconcileDefaultIngressController(t *testing.T) { OVNKubernetesConfig: &operatorv1.OVNKubernetesConfig{ GenevePort: &fakePort, V4InternalSubnet: kubevirtDefaultV4InternalSubnet, + IPv6: &operatorv1.IPv6OVNKubernetesConfig{ + InternalJoinSubnet: v6InternalJoinSubnet, + }, }, }, }, @@ -153,6 +163,7 @@ func TestReconcileDefaultIngressController(t *testing.T) { inputNetworkType: hyperv1.OVNKubernetes, inputPlatformType: hyperv1.KubevirtPlatform, disableMultiNetwork: false, + hasIPv6Network: true, expectedNetwork: &operatorv1.Network{ ObjectMeta: NetworkOperator().ObjectMeta, Spec: operatorv1.NetworkSpec{ @@ -163,6 +174,9 @@ func TestReconcileDefaultIngressController(t *testing.T) { OVNKubernetesConfig: &operatorv1.OVNKubernetesConfig{ V4InternalSubnet: "100.66.0.0/16", GenevePort: &genevePort, + IPv6: &operatorv1.IPv6OVNKubernetesConfig{ + InternalJoinSubnet: v6InternalJoinSubnet, + }, }, }, }, @@ -362,6 +376,7 @@ func TestReconcileDefaultIngressController(t *testing.T) { inputNetworkType: hyperv1.OVNKubernetes, inputPlatformType: hyperv1.KubevirtPlatform, disableMultiNetwork: false, + hasIPv6Network: true, ovnConfig: &hyperv1.OVNKubernetesConfig{ MTU: 1300, }, @@ -375,7 +390,10 @@ func TestReconcileDefaultIngressController(t *testing.T) { OVNKubernetesConfig: &operatorv1.OVNKubernetesConfig{ GenevePort: &genevePort, V4InternalSubnet: v4InternalSubnet, - MTU: ptr.To(uint32(1300)), + IPv6: &operatorv1.IPv6OVNKubernetesConfig{ + InternalJoinSubnet: v6InternalJoinSubnet, + }, + MTU: ptr.To(uint32(1300)), }, }, }, @@ -433,12 +451,162 @@ func TestReconcileDefaultIngressController(t *testing.T) { }, }, }, + { + name: "When IPv6 subnets configured for OVN Kubernetes it should propagate to network operator", + inputNetwork: NetworkOperator(), + inputNetworkType: hyperv1.OVNKubernetes, + inputPlatformType: hyperv1.AWSPlatform, + disableMultiNetwork: false, + ovnConfig: &hyperv1.OVNKubernetesConfig{ + IPv6: hyperv1.OVNIPv6Config{ + InternalJoinSubnet: "fd99::/64", + InternalTransitSwitchSubnet: "fd97:1::/64", + }, + }, + expectedNetwork: &operatorv1.Network{ + ObjectMeta: NetworkOperator().ObjectMeta, + Spec: operatorv1.NetworkSpec{ + OperatorSpec: operatorv1.OperatorSpec{ + ManagementState: "Managed", + }, + DefaultNetwork: operatorv1.DefaultNetworkDefinition{ + OVNKubernetesConfig: &operatorv1.OVNKubernetesConfig{ + IPv6: &operatorv1.IPv6OVNKubernetesConfig{ + InternalJoinSubnet: "fd99::/64", + InternalTransitSwitchSubnet: "fd97:1::/64", + }, + }, + }, + }, + }, + }, + { + name: "When OVN config has IPv4 and IPv6 subnets it should propagate both", + inputNetwork: NetworkOperator(), + inputNetworkType: hyperv1.OVNKubernetes, + inputPlatformType: hyperv1.AWSPlatform, + disableMultiNetwork: false, + ovnConfig: &hyperv1.OVNKubernetesConfig{ + IPv4: &hyperv1.OVNIPv4Config{ + InternalJoinSubnet: "192.168.1.0/24", + }, + IPv6: hyperv1.OVNIPv6Config{ + InternalJoinSubnet: "fd99::/64", + }, + }, + expectedNetwork: &operatorv1.Network{ + ObjectMeta: NetworkOperator().ObjectMeta, + Spec: operatorv1.NetworkSpec{ + OperatorSpec: operatorv1.OperatorSpec{ + ManagementState: "Managed", + }, + DefaultNetwork: operatorv1.DefaultNetworkDefinition{ + OVNKubernetesConfig: &operatorv1.OVNKubernetesConfig{ + IPv4: &operatorv1.IPv4OVNKubernetesConfig{ + InternalJoinSubnet: "192.168.1.0/24", + }, + IPv6: &operatorv1.IPv6OVNKubernetesConfig{ + InternalJoinSubnet: "fd99::/64", + }, + }, + }, + }, + }, + }, + { + name: "When KubeVirt with OVNKubernetes and user-specified IPv6 join subnet it should not override", + inputNetwork: &operatorv1.Network{ + ObjectMeta: NetworkOperator().ObjectMeta, + Spec: operatorv1.NetworkSpec{ + DefaultNetwork: operatorv1.DefaultNetworkDefinition{ + OVNKubernetesConfig: &operatorv1.OVNKubernetesConfig{ + IPv6: &operatorv1.IPv6OVNKubernetesConfig{ + InternalJoinSubnet: "fdaa::/64", + }, + }, + }, + }, + }, + inputNetworkType: hyperv1.OVNKubernetes, + inputPlatformType: hyperv1.KubevirtPlatform, + disableMultiNetwork: false, + hasIPv6Network: true, + expectedNetwork: &operatorv1.Network{ + ObjectMeta: NetworkOperator().ObjectMeta, + Spec: operatorv1.NetworkSpec{ + OperatorSpec: operatorv1.OperatorSpec{ + ManagementState: "Managed", + }, + DefaultNetwork: operatorv1.DefaultNetworkDefinition{ + OVNKubernetesConfig: &operatorv1.OVNKubernetesConfig{ + GenevePort: &genevePort, + V4InternalSubnet: v4InternalSubnet, + IPv6: &operatorv1.IPv6OVNKubernetesConfig{ + InternalJoinSubnet: "fdaa::/64", + }, + }, + }, + }, + }, + }, + { + name: "When KubeVirt with OVNKubernetes and user-specified IPv6 via ovnConfig it should override the KubeVirt default", + inputNetwork: NetworkOperator(), + inputNetworkType: hyperv1.OVNKubernetes, + inputPlatformType: hyperv1.KubevirtPlatform, + disableMultiNetwork: false, + hasIPv6Network: true, + ovnConfig: &hyperv1.OVNKubernetesConfig{ + IPv6: hyperv1.OVNIPv6Config{ + InternalJoinSubnet: "fdbb::/64", + }, + }, + expectedNetwork: &operatorv1.Network{ + ObjectMeta: NetworkOperator().ObjectMeta, + Spec: operatorv1.NetworkSpec{ + OperatorSpec: operatorv1.OperatorSpec{ + ManagementState: "Managed", + }, + DefaultNetwork: operatorv1.DefaultNetworkDefinition{ + OVNKubernetesConfig: &operatorv1.OVNKubernetesConfig{ + GenevePort: &genevePort, + V4InternalSubnet: v4InternalSubnet, + IPv6: &operatorv1.IPv6OVNKubernetesConfig{ + InternalJoinSubnet: "fdbb::/64", + }, + }, + }, + }, + }, + }, + { + name: "When KubeVirt with OVNKubernetes and no IPv6 networks it should not set IPv6 join subnet default", + inputNetwork: NetworkOperator(), + inputNetworkType: hyperv1.OVNKubernetes, + inputPlatformType: hyperv1.KubevirtPlatform, + disableMultiNetwork: false, + hasIPv6Network: false, + expectedNetwork: &operatorv1.Network{ + ObjectMeta: NetworkOperator().ObjectMeta, + Spec: operatorv1.NetworkSpec{ + OperatorSpec: operatorv1.OperatorSpec{ + ManagementState: "Managed", + }, + DefaultNetwork: operatorv1.DefaultNetworkDefinition{ + OVNKubernetesConfig: &operatorv1.OVNKubernetesConfig{ + GenevePort: &genevePort, + V4InternalSubnet: v4InternalSubnet, + }, + }, + }, + }, + }, } for _, tc := range testsCases { t.Run(tc.name, func(t *testing.T) { g := NewGomegaWithT(t) - ReconcileNetworkOperator(tc.inputNetwork, tc.inputNetworkType, tc.inputPlatformType, tc.disableMultiNetwork, tc.ovnConfig) + ReconcileNetworkOperator(tc.inputNetwork, tc.inputNetworkType, tc.inputPlatformType, tc.disableMultiNetwork, tc.ovnConfig, tc.hasIPv6Network) g.Expect(tc.inputNetwork).To(BeEquivalentTo(tc.expectedNetwork)) }) } diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/registry/admissionpolicies.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/registry/admissionpolicies.go index 55bc7cc5d3d1..d11d1d5c373a 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/registry/admissionpolicies.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/registry/admissionpolicies.go @@ -39,7 +39,7 @@ var ( func ReconcileRegistryConfigValidatingAdmissionPolicies(ctx context.Context, hcp *hyperv1.HostedControlPlane, client client.Client, createOrUpdate upsert.CreateOrUpdateFN) error { if err := reconcileRegistryConfigManagementStateValidatingAdmissionPolicy(ctx, hcp, client, createOrUpdate); err != nil { - return fmt.Errorf("failed to reconcile ManagementState Validating Admission Policy: %v", err) + return fmt.Errorf("failed to reconcile ManagementState Validating Admission Policy: %w", err) } return nil @@ -66,7 +66,7 @@ func reconcileRegistryConfigManagementStateValidatingAdmissionPolicy(ctx context registryConfigManagementStateAdmissionPolicy.Validations = []k8sadmissionv1.Validation{denyRemovedManagementStateValidation} registryConfigManagementStateAdmissionPolicy.MatchConstraints = constructPolicyMatchConstraints(registryConfigManagementStateResources, registryConfigManagementStateAPIVersion, registryConfigManagementStateAPIGroup, []k8sadmissionv1.OperationType{"CREATE", "UPDATE"}) if err := registryConfigManagementStateAdmissionPolicy.reconcileAdmissionPolicy(ctx, client, createOrUpdate); err != nil { - return fmt.Errorf("error reconciling management State Validating Admission Policy: %v", err) + return fmt.Errorf("error reconciling management State Validating Admission Policy: %w", err) } return nil @@ -85,7 +85,7 @@ func (ap *AdmissionPolicy) reconcileAdmissionPolicy(ctx context.Context, client return nil }); err != nil { - return fmt.Errorf("failed to create/update Validating Admission Policy with name %s: %v", ap.Name, err) + return fmt.Errorf("failed to create/update Validating Admission Policy with name %s: %w", ap.Name, err) } policyBinding := manifests.ValidatingAdmissionPolicyBinding(fmt.Sprintf("%s-binding", ap.Name)) @@ -94,7 +94,7 @@ func (ap *AdmissionPolicy) reconcileAdmissionPolicy(ctx context.Context, client policyBinding.Spec.ValidationActions = []k8sadmissionv1.ValidationAction{k8sadmissionv1.Deny} return nil }); err != nil { - return fmt.Errorf("failed to create/update Validating Admission Policy Binding with name %s: %v", ap.Name, err) + return fmt.Errorf("failed to create/update Validating Admission Policy Binding with name %s: %w", ap.Name, err) } return nil diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/registry/admissionpolicies_test.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/registry/admissionpolicies_test.go index 013dc5896f10..1d509f15275a 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/registry/admissionpolicies_test.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/registry/admissionpolicies_test.go @@ -2,6 +2,7 @@ package registry import ( "context" + "fmt" "testing" "time" @@ -150,9 +151,11 @@ func TestReconcileRegistryConfigManagementStateValidatingAdmissionPolicy(t *test func TestReconcileRegistryConfigValidatingAdmissionPolicies(t *testing.T) { tests := []struct { - name string - hcp *hyperv1.HostedControlPlane - expectError bool + name string + hcp *hyperv1.HostedControlPlane + createOrUpdate func(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) + expectError bool + errSubstr string }{ { name: "When reconciliation succeeds it should return no error", @@ -175,33 +178,74 @@ func TestReconcileRegistryConfigValidatingAdmissionPolicies(t *testing.T) { }, expectError: false, }, + { + name: "When createOrUpdate fails for VAP, it should return a wrapped error", + hcp: &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + }, + createOrUpdate: func(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) { + if _, ok := obj.(*k8sadmissionv1.ValidatingAdmissionPolicy); ok { + return controllerutil.OperationResultNone, fmt.Errorf("API conflict") + } + if err := f(); err != nil { + return controllerutil.OperationResultNone, err + } + return controllerutil.OperationResultCreated, nil + }, + expectError: true, + errSubstr: "failed to reconcile ManagementState Validating Admission Policy", + }, + { + name: "When createOrUpdate fails for binding, it should return a wrapped error", + hcp: &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + }, + createOrUpdate: func(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) { + if _, ok := obj.(*k8sadmissionv1.ValidatingAdmissionPolicyBinding); ok { + return controllerutil.OperationResultNone, fmt.Errorf("API conflict") + } + if err := f(); err != nil { + return controllerutil.OperationResultNone, err + } + return controllerutil.OperationResultCreated, nil + }, + expectError: true, + errSubstr: "failed to create/update Validating Admission Policy Binding", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := NewGomegaWithT(t) - // Create a fake client scheme := hyperapi.Scheme fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() - // Create a mock createOrUpdate function - mockCreateOrUpdate := func(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) { - if err := f(); err != nil { - return controllerutil.OperationResultNone, err + createOrUpdate := tt.createOrUpdate + if createOrUpdate == nil { + createOrUpdate = func(ctx context.Context, c client.Client, obj client.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) { + if err := f(); err != nil { + return controllerutil.OperationResultNone, err + } + return controllerutil.OperationResultCreated, nil } - return controllerutil.OperationResultCreated, nil } - // Create a context with a logger ctx := ctrl.LoggerInto(context.Background(), ctrl.Log) - // Call the function - err := ReconcileRegistryConfigValidatingAdmissionPolicies(ctx, tt.hcp, fakeClient, mockCreateOrUpdate) + err := ReconcileRegistryConfigValidatingAdmissionPolicies(ctx, tt.hcp, fakeClient, createOrUpdate) - // Verify error expectations if tt.expectError { g.Expect(err).To(HaveOccurred()) + if tt.errSubstr != "" { + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } } else { g.Expect(err).NotTo(HaveOccurred()) } diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go index 1ee6a3a71805..eff05f628a39 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "reflect" + "slices" "sort" "strings" "sync" @@ -132,6 +133,8 @@ exec /bin/azure-cloud-node-manager \ --wait-routes=false ` +var disabledServiceAccountPullSecretsController = fmt.Sprintf("-%s", openshiftcpv1.OpenShiftServiceAccountPullSecretsController) + type reconciler struct { client client.Client uncachedClient client.Client @@ -511,12 +514,12 @@ func (r *reconciler) reconcileDeletion(ctx context.Context, log logr.Logger, hcp binding := manifests.ValidatingAdmissionPolicyBinding(fmt.Sprintf("%s-binding", registryConfigManagementStateAdmissionPolicy.Name)) if _, err := k8sutil.DeleteIfNeeded(ctx, r.client, binding); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to delete ValidatingAdmissionPolicyBinding %s: %v", binding.Name, err) + return ctrl.Result{}, fmt.Errorf("failed to delete ValidatingAdmissionPolicyBinding %s: %w", binding.Name, err) } vap := manifests.ValidatingAdmissionPolicy(registryConfigManagementStateAdmissionPolicy.Name) if _, err := k8sutil.DeleteIfNeeded(ctx, r.client, vap); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to delete ValidatingAdmissionPolicy %s: %v", vap.Name, err) + return ctrl.Result{}, fmt.Errorf("failed to delete ValidatingAdmissionPolicy %s: %w", vap.Name, err) } } @@ -567,12 +570,13 @@ func (r *reconciler) reconcileClusterRecovery(ctx context.Context, log logr.Logg condition.Message = "Hosted cluster recovery finished" } + originalHCP := hcp.DeepCopy() meta.SetStatusCondition(&hcp.Status.Conditions, *condition) log.Info("setting condition", "type", condition.Type, "status", condition.Status, "message", condition.Message) - if err := r.cpClient.Status().Update(ctx, hcp); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to update status on hcp for hosted cluster recovery: %w. Condition error message: %v", err, condition.Message) + if err := r.cpClient.Status().Patch(ctx, hcp, client.MergeFromWithOptions(originalHCP, client.MergeFromWithOptimisticLock{})); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to patch status on hcp for hosted cluster recovery: %w. Condition error message: %v", err, condition.Message) } - log.Info("successfully updated hcp status with recovery condition") + log.Info("successfully patched hcp status with recovery condition") if !finished { return ctrl.Result{RequeueAfter: 120 * time.Second}, nil @@ -629,8 +633,7 @@ func (r *reconciler) reconcileRegistryAndIngress(ctx context.Context, hcp *hyper } // TODO: remove this when ROSA HCP stops setting the managementState to Removed to disable the Image Registry - if registryConfig.Spec.ManagementState == operatorv1.Removed && r.platformType != hyperv1.IBMCloudPlatform && r.platformType != hyperv1.AzurePlatform { - log.Info("imageregistry operator managementstate is removed, disabling openshift-controller-manager controllers and cleaning up resources") + if r.platformType != hyperv1.IBMCloudPlatform && r.platformType != hyperv1.AzurePlatform { ocmConfigMap := cpomanifests.OpenShiftControllerManagerConfig(r.hcpNamespace) if _, err := r.CreateOrUpdate(ctx, r.cpClient, ocmConfigMap, func() error { if ocmConfigMap.Data == nil { @@ -638,12 +641,28 @@ func (r *reconciler) reconcileRegistryAndIngress(ctx context.Context, hcp *hyper } config := &openshiftcpv1.OpenShiftControllerManagerConfig{} if configStr, exists := ocmConfigMap.Data[ocm.ConfigKey]; exists && len(configStr) > 0 { - err := k8sutil.DeserializeResource(configStr, config, api.Scheme) - if err != nil { + if err := k8sutil.DeserializeResource(configStr, config, api.Scheme); err != nil { return fmt.Errorf("unable to decode existing openshift controller manager configuration: %w", err) } } - config.Controllers = []string{"*", fmt.Sprintf("-%s", openshiftcpv1.OpenShiftServiceAccountPullSecretsController)} + if registryConfig.Spec.ManagementState == operatorv1.Removed { + if isServiceAccountPullSecretsControllerDisabled(config.Controllers) { + // Already disabled; returning nil without mutation causes CreateOrUpdate to skip the update. + return nil + } + log.Info("imageregistry operator managementstate is removed, disabling serviceaccount-pull-secrets controller") + if len(config.Controllers) == 0 { + config.Controllers = []string{"*", disabledServiceAccountPullSecretsController} + } else { + config.Controllers = append(config.Controllers, disabledServiceAccountPullSecretsController) + } + } else if isServiceAccountPullSecretsControllerDisabled(config.Controllers) { + log.Info("imageregistry operator managementstate is no longer removed, re-enabling serviceaccount-pull-secrets controller") + config.Controllers = removeDisabledServiceAccountPullSecretsController(config.Controllers) + } else { + // No change needed; returning nil without mutation causes CreateOrUpdate to skip the update. + return nil + } configStr, err := k8sutil.SerializeResource(config, api.Scheme) if err != nil { return fmt.Errorf("failed to serialize openshift controller manager configuration: %w", err) @@ -830,7 +849,7 @@ func (r *reconciler) reconcileNetworkingAndSecrets(ctx context.Context, hcp *hyp ovnConfig = hcp.Spec.OperatorConfiguration.ClusterNetworkOperator.OVNKubernetesConfig } if _, err := r.CreateOrUpdate(ctx, r.client, networkOperator, func() error { - networkoperator.ReconcileNetworkOperator(networkOperator, hcp.Spec.Networking.NetworkType, hcp.Spec.Platform.Type, netutil.IsDisableMultiNetwork(hcp), ovnConfig) + networkoperator.ReconcileNetworkOperator(networkOperator, hcp.Spec.Networking.NetworkType, hcp.Spec.Platform.Type, netutil.IsDisableMultiNetwork(hcp), ovnConfig, hasIPv6Network(hcp)) return nil }); err != nil { errs = append(errs, fmt.Errorf("failed to reconcile network operator: %w", err)) @@ -1687,8 +1706,8 @@ func (r *reconciler) patchHCPStatusCondition(ctx context.Context, hcp *hyperv1.H if !meta.SetStatusCondition(&hcp.Status.Conditions, *condition) { return nil // No status change; avoid unnecessary API call. } - if err := r.cpClient.Status().Patch(ctx, hcp, client.MergeFrom(originalHCP)); err != nil { - return fmt.Errorf("failed to update HostedControlPlane status with %s condition: %w", condition.Type, err) + if err := r.cpClient.Status().Patch(ctx, hcp, client.MergeFromWithOptions(originalHCP, client.MergeFromWithOptimisticLock{})); err != nil { + return fmt.Errorf("failed to patch HostedControlPlane status with %s condition: %w", condition.Type, err) } log.Info(string(condition.Type) + " condition updated") return nil @@ -1996,7 +2015,7 @@ func (r *reconciler) reconcileKubeadminPasswordHashSecret(ctx context.Context, h kubeadminPasswordSecret.Annotations[cpoauth.KubeadminSecretHashAnnotation] = string(kubeadminPasswordHashSecret.Data["kubeadmin"]) return nil }); err != nil { - return fmt.Errorf("failed to annotate kubeadmin-password secret in hcp namespace: %v", err) + return fmt.Errorf("failed to annotate kubeadmin-password secret in hcp namespace: %w", err) } return nil @@ -2728,8 +2747,8 @@ func (r *reconciler) destroyCloudResources(ctx context.Context, hcp *hyperv1.Hos meta.SetStatusCondition(&hcp.Status.Conditions, *resourcesDestroyedCond) if !equality.Semantic.DeepEqual(hcp, originalHCP) { - if err := r.cpClient.Status().Update(ctx, hcp); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to set resources destroyed condition: %w", err) + if err := r.cpClient.Status().Patch(ctx, hcp, client.MergeFromWithOptions(originalHCP, client.MergeFromWithOptimisticLock{})); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to patch resources destroyed condition: %w", err) } } @@ -2974,6 +2993,8 @@ func (r *reconciler) reconcileKubeletConfig(ctx context.Context) error { if err := r.deleteImmutableConfigMapIfNeeded(ctx, log, hostedClusterCM); err != nil { return err } + hostedClusterCM.SetResourceVersion("") + hostedClusterCM.Immutable = nil if result, err := r.CreateOrUpdate(ctx, r.client, hostedClusterCM, func() error { return mutateKubeletConfig(&cm, hostedClusterCM) @@ -2996,6 +3017,11 @@ func (r *reconciler) reconcileKubeletConfig(ctx context.Context) error { if want.Has(cm.Name) { continue } + if cm.Labels[nodepool.NTOMirroredConfigLabel] == "true" { + log.Info("skipping deletion of mirrored ConfigMap with transiently absent source", + "configMap", client.ObjectKeyFromObject(cm).String()) + continue + } log.Info("delete mirror config ConfigMap", "config", client.ObjectKeyFromObject(cm).String()) if _, err := k8sutil.DeleteIfNeeded(ctx, r.client, cm); err != nil { return fmt.Errorf("failed to delete ConfigMap %s: %w", client.ObjectKeyFromObject(cm).String(), err) @@ -3004,26 +3030,19 @@ func (r *reconciler) reconcileKubeletConfig(ctx context.Context) error { return nil } -// deleteImmutableConfigMapIfNeeded checks if a ConfigMap exists and is immutable, -// and deletes it if necessary to allow recreation as a mutable ConfigMap. -// This handles migration from immutable ConfigMaps to mutable ones. func (r *reconciler) deleteImmutableConfigMapIfNeeded(ctx context.Context, log logr.Logger, cm *corev1.ConfigMap) error { - existingCM := &corev1.ConfigMap{} - if err := r.client.Get(ctx, client.ObjectKeyFromObject(cm), existingCM); err != nil { - if apierrors.IsNotFound(err) { - return nil + _, err := k8sutil.DeleteIfNeededWithPredicate(ctx, r.client, cm, func(existing *corev1.ConfigMap) bool { + if existing.Labels[nodepool.KubeletConfigConfigMapLabel] != "true" { + return false } - return fmt.Errorf("failed to get ConfigMap %s: %w", client.ObjectKeyFromObject(cm).String(), err) - } - - if existingCM.Immutable != nil && *existingCM.Immutable { - log.Info("deleting immutable KubeletConfig ConfigMap to recreate as mutable", "configMap", client.ObjectKeyFromObject(existingCM).String()) - if _, err := k8sutil.DeleteIfNeeded(ctx, r.client, existingCM); err != nil { - return fmt.Errorf("failed to delete immutable ConfigMap %s: %w", client.ObjectKeyFromObject(existingCM).String(), err) + if existing.Immutable != nil && *existing.Immutable { + log.Info("deleting immutable KubeletConfig ConfigMap to recreate as mutable", + "configMap", client.ObjectKeyFromObject(existing).String()) + return true } - } - - return nil + return false + }) + return err } func mutateKubeletConfig(controlPlaneConfigMap, hostedClusterConfigMap *corev1.ConfigMap) error { @@ -3642,6 +3661,25 @@ func (r *reconciler) reconcileAzureCloudNodeManager(ctx context.Context, image s return errs } +func hasIPv6Network(hcp *hyperv1.HostedControlPlane) bool { + for _, entry := range hcp.Spec.Networking.ClusterNetwork { + if net.IP(entry.CIDR.IP).To4() == nil { + return true + } + } + for _, entry := range hcp.Spec.Networking.ServiceNetwork { + if net.IP(entry.CIDR.IP).To4() == nil { + return true + } + } + for _, entry := range hcp.Spec.Networking.MachineNetwork { + if net.IP(entry.CIDR.IP).To4() == nil { + return true + } + } + return false +} + // imageRegistryPlatformWithPVC returns true if the platform requires a PVC for the image registry. func imageRegistryPlatformWithPVC(platform hyperv1.PlatformType) bool { switch platform { @@ -3651,3 +3689,20 @@ func imageRegistryPlatformWithPVC(platform hyperv1.PlatformType) bool { return false } } + +func isServiceAccountPullSecretsControllerDisabled(controllers []string) bool { + return slices.Contains(controllers, disabledServiceAccountPullSecretsController) +} + +func removeDisabledServiceAccountPullSecretsController(controllers []string) []string { + filtered := make([]string, 0, len(controllers)) + for _, c := range controllers { + if c != disabledServiceAccountPullSecretsController { + filtered = append(filtered, c) + } + } + if len(filtered) == 0 { + return []string{"*"} + } + return filtered +} diff --git a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go index 611f39abe08f..dd9bef97dc4c 100644 --- a/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go +++ b/control-plane-operator/hostedclusterconfigoperator/controllers/resources/resources_test.go @@ -14,6 +14,7 @@ import ( hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" cpomanifests "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests" + "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/ocm" "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/api" "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas" "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/resources/manifests" @@ -21,6 +22,7 @@ import ( "github.com/openshift/hypershift/hypershift-operator/controllers/nodepool" "github.com/openshift/hypershift/support/azureutil" "github.com/openshift/hypershift/support/globalconfig" + "github.com/openshift/hypershift/support/k8sutil" "github.com/openshift/hypershift/support/netutil" "github.com/openshift/hypershift/support/releaseinfo" fakereleaseprovider "github.com/openshift/hypershift/support/releaseinfo/fake" @@ -29,8 +31,10 @@ import ( configv1 "github.com/openshift/api/config/v1" imageapi "github.com/openshift/api/image/v1" + openshiftcpv1 "github.com/openshift/api/openshiftcontrolplane/v1" operatorv1 "github.com/openshift/api/operator/v1" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" @@ -48,11 +52,13 @@ import ( controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/go-logr/logr" "github.com/go-logr/zapr" "go.uber.org/zap/zaptest" + "golang.org/x/crypto/bcrypt" ) type testClient struct { @@ -75,11 +81,16 @@ var initialObjects = []client.Object{ globalconfig.ProjectConfig(), globalconfig.BuildConfig(), globalconfig.ProxyConfig(), - // Not running bcrypt hashing for the kubeadmin secret massively speeds up the tests, 4s vs 0.1s (and for -race its ~10x that) + // Use a valid bcrypt hash of "test" (matching fakeKubeadminPasswordSecret) so the + // CompareHashAndPassword check passes and avoids re-hashing on every reconcile. + // MinCost keeps the tests fast (~0.1s vs ~4s with DefaultCost, ~10x worse with -race). &corev1.Secret{ ObjectMeta: manifests.KubeadminPasswordHashSecret().ObjectMeta, Data: map[string][]byte{ - "kubeadmin": []byte("something"), + "kubeadmin": func() []byte { + h, _ := bcrypt.GenerateFromPassword([]byte("test"), bcrypt.MinCost) + return h + }(), }, }, manifests.NodeTuningClusterOperator(), @@ -489,7 +500,9 @@ func TestReconcileKubeadminPasswordHashSecret(t *testing.T) { tests := map[string]struct { inputHCP *hyperv1.HostedControlPlane inputObjects []client.Object + existingHashSecret *corev1.Secret expectKubeadminPasswordHashSecretToExist bool + expectHashPreserved bool }{ "when kubeadminPasswordSecret exists the hash secret is created": { inputHCP: &hyperv1.HostedControlPlane{ @@ -511,6 +524,62 @@ func TestReconcileKubeadminPasswordHashSecret(t *testing.T) { }, expectKubeadminPasswordHashSecretToExist: true, }, + "When existing hash does not match the password it should regenerate the hash": { + inputHCP: &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: testHCPName, + Namespace: testNamespace, + }, + }, + inputObjects: []client.Object{ + &corev1.Secret{ + ObjectMeta: manifests.KubeadminPasswordSecret(testNamespace).ObjectMeta, + Data: map[string][]byte{ + "password": []byte(`adminpass`), + }, + }, + &appsv1.Deployment{ + ObjectMeta: manifests.OAuthDeployment(testNamespace).ObjectMeta, + }, + }, + existingHashSecret: &corev1.Secret{ + ObjectMeta: manifests.KubeadminPasswordHashSecret().ObjectMeta, + Data: map[string][]byte{ + "kubeadmin": []byte("stale-non-matching-hash"), + }, + }, + expectKubeadminPasswordHashSecretToExist: true, + }, + "When hash already matches password it should not regenerate": { + inputHCP: &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: testHCPName, + Namespace: testNamespace, + }, + }, + inputObjects: []client.Object{ + &corev1.Secret{ + ObjectMeta: manifests.KubeadminPasswordSecret(testNamespace).ObjectMeta, + Data: map[string][]byte{ + "password": []byte(`adminpass`), + }, + }, + &appsv1.Deployment{ + ObjectMeta: manifests.OAuthDeployment(testNamespace).ObjectMeta, + }, + }, + existingHashSecret: &corev1.Secret{ + ObjectMeta: manifests.KubeadminPasswordHashSecret().ObjectMeta, + Data: map[string][]byte{ + "kubeadmin": func() []byte { + h, _ := bcrypt.GenerateFromPassword([]byte("adminpass"), bcrypt.MinCost) + return h + }(), + }, + }, + expectKubeadminPasswordHashSecretToExist: true, + expectHashPreserved: true, + }, "when kubeadminPasswordSecret doesn't exist the hash secret is not created": { inputHCP: &hyperv1.HostedControlPlane{ ObjectMeta: metav1.ObjectMeta{ @@ -529,8 +598,12 @@ func TestReconcileKubeadminPasswordHashSecret(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { g := NewGomegaWithT(t) + guestClientBuilder := fake.NewClientBuilder().WithScheme(api.Scheme) + if test.existingHashSecret != nil { + guestClientBuilder = guestClientBuilder.WithObjects(test.existingHashSecret) + } r := &reconciler{ - client: fake.NewClientBuilder().WithScheme(api.Scheme).Build(), + client: guestClientBuilder.Build(), CreateOrUpdateProvider: &simpleCreateOrUpdater{}, cpClient: fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(append(test.inputObjects, test.inputHCP)...).Build(), hcpName: testHCPName, @@ -542,7 +615,17 @@ func TestReconcileKubeadminPasswordHashSecret(t *testing.T) { actualKubeAdminSecret := manifests.KubeadminPasswordHashSecret() err := r.client.Get(t.Context(), client.ObjectKeyFromObject(actualKubeAdminSecret), actualKubeAdminSecret) g.Expect(err).To(BeNil()) - g.Expect(len(actualKubeAdminSecret.Data["kubeadmin"]) > 0).To(BeTrue()) + g.Expect(actualKubeAdminSecret.Data["kubeadmin"]).ToNot(BeEmpty()) + if test.expectHashPreserved { + g.Expect(actualKubeAdminSecret.Data["kubeadmin"]).To(Equal(test.existingHashSecret.Data["kubeadmin"])) + } + passwordSecret := manifests.KubeadminPasswordSecret(testNamespace) + err = r.cpClient.Get(t.Context(), client.ObjectKeyFromObject(passwordSecret), passwordSecret) + g.Expect(err).To(BeNil()) + g.Expect(bcrypt.CompareHashAndPassword( + actualKubeAdminSecret.Data["kubeadmin"], + passwordSecret.Data["password"], + )).To(BeNil()) } else { actualKubeAdminSecret := manifests.KubeadminPasswordHashSecret() err := r.client.Get(t.Context(), client.ObjectKeyFromObject(actualKubeAdminSecret), actualKubeAdminSecret) @@ -1564,6 +1647,36 @@ func TestReconcileKubeletConfig(t *testing.T) { makeKubeletConfigConfigMap(netutil.ShortenName("bar", npName1, validation.LabelValueMaxLength), hcNamespace, kubeletConfig1), }, }, + { + name: "When source CM is transiently absent, mirrored guest-side CM should not be deleted", + hostedControlPlaneObjects: []client.Object{}, + existHostedControlPlaneObjects: []client.Object{ + makeMirroredKubeletConfigConfigMap(netutil.ShortenName("bar", npName1, validation.LabelValueMaxLength), hcNamespace, npName1, kubeletConfig1), + }, + expectedHostedClusterObjects: []client.Object{ + makeMirroredKubeletConfigConfigMap(netutil.ShortenName("bar", npName1, validation.LabelValueMaxLength), hcNamespace, npName1, kubeletConfig1), + }, + }, + { + name: "When source CM is absent and guest CM is not mirrored, it should be deleted", + hostedControlPlaneObjects: []client.Object{}, + existHostedControlPlaneObjects: []client.Object{ + makeKubeletConfigConfigMap(netutil.ShortenName("bar", npName1, validation.LabelValueMaxLength), hcNamespace, kubeletConfig1), + }, + expectedHostedClusterObjects: []client.Object{}, + }, + { + name: "When guest CM is immutable, it should be deleted and recreated as mutable", + hostedControlPlaneObjects: []client.Object{ + makeKubeletConfigConfigMap(netutil.ShortenName("bar", npName1, validation.LabelValueMaxLength), hcpNamespace, kubeletConfig1), + }, + existHostedControlPlaneObjects: []client.Object{ + makeImmutableKubeletConfigConfigMap(netutil.ShortenName("bar", npName1, validation.LabelValueMaxLength), hcNamespace, kubeletConfig1), + }, + expectedHostedClusterObjects: []client.Object{ + makeKubeletConfigConfigMap(netutil.ShortenName("bar", npName1, validation.LabelValueMaxLength), hcNamespace, kubeletConfig1), + }, + }, } for _, tc := range testCases { @@ -1578,7 +1691,9 @@ func TestReconcileKubeletConfig(t *testing.T) { } g.Expect(r.reconcileKubeletConfig(t.Context())).To(Succeed()) for _, obj := range tc.expectedHostedClusterObjects { - g.Expect(r.client.Get(t.Context(), client.ObjectKeyFromObject(obj), obj)).To(Succeed(), "failed to get %s", client.ObjectKeyFromObject(obj)) + actual := &corev1.ConfigMap{} + g.Expect(r.client.Get(t.Context(), client.ObjectKeyFromObject(obj), actual)).To(Succeed(), "failed to get %s", client.ObjectKeyFromObject(obj)) + g.Expect(actual.Immutable).To(BeNil(), "recreated ConfigMap %s should be mutable", client.ObjectKeyFromObject(obj)) } listOpts := []client.ListOption{ client.InNamespace(hcNamespace), @@ -1667,6 +1782,39 @@ func makeKubeletConfigConfigMap(name, namespace, data string) *corev1.ConfigMap } } +func makeMirroredKubeletConfigConfigMap(name, namespace, nodePoolName, data string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + nodepool.KubeletConfigConfigMapLabel: "true", + nodepool.NTOMirroredConfigLabel: "true", + hyperv1.NodePoolLabel: nodePoolName, + }, + }, + Data: map[string]string{ + "config": data, + }, + } +} + +func makeImmutableKubeletConfigConfigMap(name, namespace, data string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ + nodepool.KubeletConfigConfigMapLabel: "true", + }, + }, + Immutable: ptr.To(true), + Data: map[string]string{ + "config": data, + }, + } +} + func TestReconcileAuthOIDC(t *testing.T) { testNamespace := "master-cluster1" testHCPName := "cluster1" @@ -3107,10 +3255,12 @@ func TestReconcileDeletion(t *testing.T) { name string hcp *hyperv1.HostedControlPlane existingObjects []client.Object + interceptorFuncs *interceptor.Funcs expectVAPDeleted bool expectVAPBDeleted bool expectCloudCleanup bool expectError bool + errSubstr string }{ { name: "When platform is Azure, it should delete the registry management state VAP and binding", @@ -3236,13 +3386,44 @@ func TestReconcileDeletion(t *testing.T) { }, expectCloudCleanup: false, }, + { + name: "When Delete fails for the VAP binding on Azure, it should return a wrapped error", + hcp: &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-hcp", + Namespace: "test-ns", + }, + Spec: hyperv1.HostedControlPlaneSpec{ + Platform: hyperv1.PlatformSpec{ + Type: hyperv1.AzurePlatform, + }, + }, + }, + existingObjects: []client.Object{ + manifests.ValidatingAdmissionPolicyBinding(fmt.Sprintf("%s-binding", registry.AdmissionPolicyNameManagementState)), + }, + interceptorFuncs: &interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if obj.GetName() == fmt.Sprintf("%s-binding", registry.AdmissionPolicyNameManagementState) { + return fmt.Errorf("API server unavailable") + } + return c.Delete(ctx, obj, opts...) + }, + }, + expectError: true, + errSubstr: "failed to delete ValidatingAdmissionPolicyBinding", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { g := NewWithT(t) - guestClient := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(tt.existingObjects...).Build() + guestClientBuilder := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(tt.existingObjects...) + if tt.interceptorFuncs != nil { + guestClientBuilder = guestClientBuilder.WithInterceptorFuncs(*tt.interceptorFuncs) + } + guestClient := guestClientBuilder.Build() cpClient := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(tt.hcp).WithStatusSubresource(&hyperv1.HostedControlPlane{}).Build() r := &reconciler{ @@ -3268,6 +3449,9 @@ func TestReconcileDeletion(t *testing.T) { if tt.expectError { g.Expect(err).To(HaveOccurred()) + if tt.errSubstr != "" { + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } return } g.Expect(err).ToNot(HaveOccurred()) @@ -3646,3 +3830,446 @@ func TestCleanupLegacyResources(t *testing.T) { }) } } + +func TestIsAllowedWebhookUrl(t *testing.T) { + t.Parallel() + tests := []struct { + name string + disallowedUrls []string + url string + expected bool + }{ + { + name: "When URL contains a disallowed substring it should return false", + disallowedUrls: []string{"https://etcd-client"}, + url: "https://etcd-client:2379", + expected: false, + }, + { + name: "When URL matches a fully qualified disallowed URL it should return false", + disallowedUrls: []string{"https://etcd-client.ns.svc"}, + url: "https://etcd-client.ns.svc:2379/path", + expected: false, + }, + { + name: "When URL does not match any disallowed URL it should return true", + disallowedUrls: []string{"https://etcd-client"}, + url: "https://external.example.com", + expected: true, + }, + { + name: "When disallowed list is empty it should return true", + disallowedUrls: []string{}, + url: "https://anything", + expected: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + result := isAllowedWebhookUrl(tt.disallowedUrls, tt.url) + g.Expect(result).To(Equal(tt.expected)) + }) + } +} + +func TestEnsureGuestAdmissionWebhooksAreValid(t *testing.T) { + t.Parallel() + const hcpNamespace = "test-hcp-namespace" + + tests := []struct { + name string + cpServices []corev1.Service + guestObjects []client.Object + expectWebhookGone string + expectWebhookAlive string + }{ + { + name: "When validating webhook targets a CP service it should delete the webhook", + cpServices: []corev1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd-client", + Namespace: hcpNamespace, + }, + }, + }, + guestObjects: []client.Object{ + &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "test-validating-webhook"}, + Webhooks: []admissionregistrationv1.ValidatingWebhook{ + { + Name: "test.webhook.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{URL: ptr.To("https://etcd-client:2379")}, + }, + }, + }, + }, + expectWebhookGone: "test-validating-webhook", + }, + { + name: "When validating webhook targets an allowed CP service it should preserve the webhook", + cpServices: []corev1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "allowed-service", + Namespace: hcpNamespace, + Labels: map[string]string{hyperv1.AllowGuestWebhooksServiceLabel: "true"}, + }, + }, + }, + guestObjects: []client.Object{ + &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "preserved-validating-webhook"}, + Webhooks: []admissionregistrationv1.ValidatingWebhook{ + { + Name: "preserved.webhook.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{URL: ptr.To("https://allowed-service:8443")}, + }, + }, + }, + }, + expectWebhookAlive: "preserved-validating-webhook", + }, + { + name: "When mutating webhook targets a CP service it should delete the webhook", + cpServices: []corev1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "kube-apiserver", + Namespace: hcpNamespace, + }, + }, + }, + guestObjects: []client.Object{ + &admissionregistrationv1.MutatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "test-mutating-webhook"}, + Webhooks: []admissionregistrationv1.MutatingWebhook{ + { + Name: "mutating.webhook.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{URL: ptr.To("https://kube-apiserver:6443")}, + }, + }, + }, + }, + expectWebhookGone: "test-mutating-webhook", + }, + { + name: "When webhook targets an external URL it should preserve the webhook", + cpServices: []corev1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd-client", + Namespace: hcpNamespace, + }, + }, + }, + guestObjects: []client.Object{ + &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "external-validating-webhook"}, + Webhooks: []admissionregistrationv1.ValidatingWebhook{ + { + Name: "external.webhook.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{URL: ptr.To("https://external.example.com")}, + }, + }, + }, + }, + expectWebhookAlive: "external-validating-webhook", + }, + { + name: "When webhook uses Service reference instead of URL it should preserve the webhook", + cpServices: []corev1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd-client", + Namespace: hcpNamespace, + }, + }, + }, + guestObjects: []client.Object{ + &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "service-ref-webhook"}, + Webhooks: []admissionregistrationv1.ValidatingWebhook{ + { + Name: "service.webhook.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + Service: &admissionregistrationv1.ServiceReference{ + Name: "my-webhook-service", + Namespace: "default", + }, + }, + }, + }, + }, + }, + expectWebhookAlive: "service-ref-webhook", + }, + { + name: "When validating webhook has mixed allowed and disallowed URLs it should delete the entire configuration", + cpServices: []corev1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "etcd-client", + Namespace: hcpNamespace, + }, + }, + }, + guestObjects: []client.Object{ + &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{Name: "mixed-validating-webhook"}, + Webhooks: []admissionregistrationv1.ValidatingWebhook{ + { + Name: "allowed.webhook.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{URL: ptr.To("https://external.example.com")}, + }, + { + Name: "disallowed.webhook.io", + ClientConfig: admissionregistrationv1.WebhookClientConfig{URL: ptr.To("https://etcd-client:2379")}, + }, + }, + }, + }, + expectWebhookGone: "mixed-validating-webhook", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + ctx := t.Context() + + cpObjects := make([]client.Object, 0, len(tt.cpServices)) + for i := range tt.cpServices { + cpObjects = append(cpObjects, &tt.cpServices[i]) + } + + cpClient := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(cpObjects...).Build() + guestClient := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(tt.guestObjects...).Build() + + r := &reconciler{ + client: guestClient, + uncachedClient: fake.NewClientBuilder().WithScheme(api.Scheme).Build(), + cpClient: cpClient, + hcpNamespace: hcpNamespace, + CreateOrUpdateProvider: &simpleCreateOrUpdater{}, + } + + err := r.ensureGuestAdmissionWebhooksAreValid(ctx) + g.Expect(err).ToNot(HaveOccurred()) + + if tt.expectWebhookGone != "" { + for _, obj := range tt.guestObjects { + key := client.ObjectKey{Name: tt.expectWebhookGone} + switch obj.(type) { + case *admissionregistrationv1.ValidatingWebhookConfiguration: + err := guestClient.Get(ctx, key, &admissionregistrationv1.ValidatingWebhookConfiguration{}) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue(), + "ValidatingWebhookConfiguration %q should have been deleted", tt.expectWebhookGone) + case *admissionregistrationv1.MutatingWebhookConfiguration: + err := guestClient.Get(ctx, key, &admissionregistrationv1.MutatingWebhookConfiguration{}) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue(), + "MutatingWebhookConfiguration %q should have been deleted", tt.expectWebhookGone) + default: + t.Fatalf("unexpected object type %T in guestObjects for expectWebhookGone check", obj) + } + } + } + + if tt.expectWebhookAlive != "" { + for _, obj := range tt.guestObjects { + key := client.ObjectKey{Name: tt.expectWebhookAlive} + switch obj.(type) { + case *admissionregistrationv1.ValidatingWebhookConfiguration: + g.Expect(guestClient.Get(ctx, key, &admissionregistrationv1.ValidatingWebhookConfiguration{})).To(Succeed(), + "ValidatingWebhookConfiguration %q should still exist", tt.expectWebhookAlive) + case *admissionregistrationv1.MutatingWebhookConfiguration: + g.Expect(guestClient.Get(ctx, key, &admissionregistrationv1.MutatingWebhookConfiguration{})).To(Succeed(), + "MutatingWebhookConfiguration %q should still exist", tt.expectWebhookAlive) + default: + t.Fatalf("unexpected object type %T in guestObjects for expectWebhookAlive check", obj) + } + } + } + }) + } +} + +func TestIsServiceAccountPullSecretsControllerDisabled(t *testing.T) { + t.Parallel() + tests := []struct { + name string + controllers []string + expected bool + }{ + { + name: "When controllers is nil, it should return false", + controllers: nil, + expected: false, + }, + { + name: "When controllers is empty, it should return false", + controllers: []string{}, + expected: false, + }, + { + name: "When controller is disabled, it should return true", + controllers: []string{"*", "-openshift.io/serviceaccount-pull-secrets"}, + expected: true, + }, + { + name: "When controllers has other entries but not the disabled one, it should return false", + controllers: []string{"*", "-some-other-controller"}, + expected: false, + }, + { + name: "When only the wildcard is present, it should return false", + controllers: []string{"*"}, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + g.Expect(isServiceAccountPullSecretsControllerDisabled(tt.controllers)).To(Equal(tt.expected)) + }) + } +} + +func TestReconcileRegistryAndIngress_ServiceAccountPullSecretsController(t *testing.T) { + t.Parallel() + + hcpNamespace := "test-hcp-ns" + + serializeOCMConfig := func(t *testing.T, controllers []string) string { + t.Helper() + config := &openshiftcpv1.OpenShiftControllerManagerConfig{ + Controllers: controllers, + } + data, err := k8sutil.SerializeResource(config, api.Scheme) + if err != nil { + t.Fatalf("failed to serialize OCM config: %v", err) + } + return data + } + + tests := []struct { + name string + platformType hyperv1.PlatformType + managementState operatorv1.ManagementState + existingOCMControllers []string + hasExistingOCMConfig bool + expectedControllers []string + }{ + { + name: "When managementState is Removed, it should disable serviceaccount-pull-secrets controller", + platformType: hyperv1.AWSPlatform, + managementState: operatorv1.Removed, + existingOCMControllers: nil, + hasExistingOCMConfig: true, + expectedControllers: []string{"*", disabledServiceAccountPullSecretsController}, + }, + { + name: "When managementState changes from Removed to Managed, it should re-enable serviceaccount-pull-secrets controller", + platformType: hyperv1.AWSPlatform, + managementState: operatorv1.Managed, + existingOCMControllers: []string{"*", disabledServiceAccountPullSecretsController}, + hasExistingOCMConfig: true, + expectedControllers: []string{"*"}, + }, + { + name: "When managementState is Managed and controller is already enabled, it should not change controllers", + platformType: hyperv1.AWSPlatform, + managementState: operatorv1.Managed, + existingOCMControllers: []string{"*"}, + hasExistingOCMConfig: true, + expectedControllers: []string{"*"}, + }, + { + name: "When platform is IBMCloud, it should not modify OCM config regardless of managementState", + platformType: hyperv1.IBMCloudPlatform, + managementState: operatorv1.Removed, + existingOCMControllers: nil, + hasExistingOCMConfig: true, + expectedControllers: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + registryConfig := manifests.Registry() + registryConfig.Spec.ManagementState = tt.managementState + + guestClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(registryConfig). + Build() + + ocmConfigMap := cpomanifests.OpenShiftControllerManagerConfig(hcpNamespace) + if tt.hasExistingOCMConfig { + ocmConfigMap.Data = map[string]string{} + if tt.existingOCMControllers != nil { + ocmConfigMap.Data[ocm.ConfigKey] = serializeOCMConfig(t, tt.existingOCMControllers) + } else { + ocmConfigMap.Data[ocm.ConfigKey] = serializeOCMConfig(t, nil) + } + } + + cpClient := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(ocmConfigMap). + Build() + + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-hcp", + Namespace: hcpNamespace, + }, + Spec: hyperv1.HostedControlPlaneSpec{ + Platform: hyperv1.PlatformSpec{ + Type: tt.platformType, + }, + }, + } + + r := &reconciler{ + client: guestClient, + cpClient: cpClient, + CreateOrUpdateProvider: &simpleCreateOrUpdater{}, + platformType: tt.platformType, + hcpNamespace: hcpNamespace, + } + + log := zapr.NewLogger(zaptest.NewLogger(t)) + errs := r.reconcileRegistryAndIngress(t.Context(), hcp, log) + for _, e := range errs { + g.Expect(e.Error()).ToNot(ContainSubstring("openshift-controller-manager config"), "unexpected OCM config error: %v", e) + } + + resultConfigMap := cpomanifests.OpenShiftControllerManagerConfig(hcpNamespace) + err := cpClient.Get(t.Context(), client.ObjectKeyFromObject(resultConfigMap), resultConfigMap) + g.Expect(err).ToNot(HaveOccurred(), "failed to get OCM ConfigMap") + + if tt.expectedControllers == nil { + config := &openshiftcpv1.OpenShiftControllerManagerConfig{} + if configStr, exists := resultConfigMap.Data[ocm.ConfigKey]; exists && len(configStr) > 0 { + err := k8sutil.DeserializeResource(configStr, config, api.Scheme) + g.Expect(err).ToNot(HaveOccurred(), "failed to deserialize OCM config") + } + g.Expect(config.Controllers).To(BeNil(), "controllers should remain nil for excluded platform") + } else { + config := &openshiftcpv1.OpenShiftControllerManagerConfig{} + configStr, exists := resultConfigMap.Data[ocm.ConfigKey] + g.Expect(exists).To(BeTrue(), "OCM config should exist") + err := k8sutil.DeserializeResource(configStr, config, api.Scheme) + g.Expect(err).ToNot(HaveOccurred(), "failed to deserialize OCM config") + g.Expect(config.Controllers).To(Equal(tt.expectedControllers)) + } + }) + } +} diff --git a/control-plane-operator/hostedclusterconfigoperator/operator/config.go b/control-plane-operator/hostedclusterconfigoperator/operator/config.go index e59c3b97463a..ba50b2f61ddc 100644 --- a/control-plane-operator/hostedclusterconfigoperator/operator/config.go +++ b/control-plane-operator/hostedclusterconfigoperator/operator/config.go @@ -223,7 +223,7 @@ func (c *HostedClusterConfigOperatorConfig) Start(ctx context.Context) error { for controllerName, setupFunc := range c.ControllerFuncs { c.Logger.Info("setting up controller", "controller", controllerName) if err := setupFunc(ctx, c); err != nil { - return fmt.Errorf("cannot setup controller %s: %v", controllerName, err) + return fmt.Errorf("cannot setup controller %s: %w", controllerName, err) } } return c.Manager.Start(ctx) diff --git a/control-plane-operator/metrics-proxy/proxy_test.go b/control-plane-operator/metrics-proxy/proxy_test.go index 1b50986984f7..f571f36b84ae 100644 --- a/control-plane-operator/metrics-proxy/proxy_test.go +++ b/control-plane-operator/metrics-proxy/proxy_test.go @@ -140,7 +140,8 @@ func TestProxyHandler_ServeHTTP(t *testing.T) { handler := newTestHandler(tt.components, tt.discoverer) - req := httptest.NewRequest(http.MethodGet, tt.path, nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, tt.path, nil) + g.Expect(err).NotTo(HaveOccurred()) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) @@ -183,7 +184,8 @@ func TestProxyHandler_ServeHTTP(t *testing.T) { }, ) - req := httptest.NewRequest(http.MethodGet, "/metrics/etcd", nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics/etcd", nil) + g.Expect(err).NotTo(HaveOccurred()) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) @@ -217,7 +219,8 @@ func TestProxyHandler_ServeHTTP(t *testing.T) { }, ) - req := httptest.NewRequest(http.MethodGet, "/metrics/etcd", nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics/etcd", nil) + g.Expect(err).NotTo(HaveOccurred()) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) @@ -258,7 +261,8 @@ func TestProxyHandler_ServeHTTP(t *testing.T) { }, ) - req := httptest.NewRequest(http.MethodGet, "/metrics/etcd", nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics/etcd", nil) + g.Expect(err).NotTo(HaveOccurred()) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) @@ -282,7 +286,8 @@ func TestProxyHandler_ServeHTTP(t *testing.T) { &fakeTargetDiscoverer{targets: []ScrapeTarget{}}, ) - req := httptest.NewRequest(http.MethodGet, "/metrics/etcd/", nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics/etcd/", nil) + g.Expect(err).NotTo(HaveOccurred()) rec := httptest.NewRecorder() handler.ServeHTTP(rec, req) @@ -338,7 +343,8 @@ func TestRequireClientCert(t *testing.T) { t.Parallel() g := NewWithT(t) - req := httptest.NewRequest(http.MethodGet, "/metrics/etcd", nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics/etcd", nil) + g.Expect(err).NotTo(HaveOccurred()) req.TLS = nil rec := httptest.NewRecorder() @@ -352,7 +358,8 @@ func TestRequireClientCert(t *testing.T) { t.Parallel() g := NewWithT(t) - req := httptest.NewRequest(http.MethodGet, "/metrics/etcd", nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics/etcd", nil) + g.Expect(err).NotTo(HaveOccurred()) req.TLS = &tls.ConnectionState{VerifiedChains: nil} rec := httptest.NewRecorder() @@ -366,7 +373,8 @@ func TestRequireClientCert(t *testing.T) { t.Parallel() g := NewWithT(t) - req := httptest.NewRequest(http.MethodGet, "/metrics/etcd", nil) + req, err := http.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics/etcd", nil) + g.Expect(err).NotTo(HaveOccurred()) req.TLS = &tls.ConnectionState{ VerifiedChains: [][]*x509.Certificate{{}}, } diff --git a/control-plane-pki-operator/certificatesigningcontroller/certificatesigningcontroller.go b/control-plane-pki-operator/certificatesigningcontroller/certificatesigningcontroller.go index e449d5b93684..44766766e499 100644 --- a/control-plane-pki-operator/certificatesigningcontroller/certificatesigningcontroller.go +++ b/control-plane-pki-operator/certificatesigningcontroller/certificatesigningcontroller.go @@ -126,7 +126,7 @@ func (c *CertificateSigningController) processCertificateSigningRequest(ctx cont x509cr, err := certificates.ParseCSR(csr.Spec.Request) if err != nil { - return nil, false, nil, fmt.Errorf("unable to parse csr %q: %v", csr.Name, err) + return nil, false, nil, fmt.Errorf("unable to parse csr %q: %w", csr.Name, err) } if validationErr := c.validator(csr, x509cr); validationErr != nil { cfg := certificatesv1applyconfigurations.CertificateSigningRequest(name) @@ -158,7 +158,7 @@ func (c *CertificateSigningController) processCertificateSigningRequest(ctx cont func sign(ca *librarygocrypto.CA, x509cr *x509.CertificateRequest, usages []certificatesv1.KeyUsage, certTTL time.Duration, expirationSeconds *int32, now func() time.Time) ([]byte, error) { if err := x509cr.CheckSignature(); err != nil { - return nil, fmt.Errorf("unable to verify certificate request signature: %v", err) + return nil, fmt.Errorf("unable to verify certificate request signature: %w", err) } notBefore, notAfter, err := boundaries( diff --git a/control-plane-pki-operator/certificatesigningcontroller/certificatesigningcontroller_test.go b/control-plane-pki-operator/certificatesigningcontroller/certificatesigningcontroller_test.go index 6019248702e4..92f8a7e22f05 100644 --- a/control-plane-pki-operator/certificatesigningcontroller/certificatesigningcontroller_test.go +++ b/control-plane-pki-operator/certificatesigningcontroller/certificatesigningcontroller_test.go @@ -18,6 +18,8 @@ import ( "testing" "time" + . "github.com/onsi/gomega" + hypershiftv1beta1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/control-plane-pki-operator/certificates" @@ -537,6 +539,26 @@ func TestSign(t *testing.T) { } } +func TestSign_WhenCSRSignatureIsCorrupted_ItShouldReturnAnError(t *testing.T) { + t.Parallel() + g := NewWithT(t) + ca := certificateAuthority(t) + pk := privateKey(t) + csrb, err := x509.CreateCertificateRequest(insecureRand, &x509.CertificateRequest{ + Subject: pkix.Name{CommonName: "test-cn"}, + }, pk) + g.Expect(err).ToNot(HaveOccurred()) + + x509cr, err := certificates.ParseCSR(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE REQUEST", Bytes: csrb})) + g.Expect(err).ToNot(HaveOccurred()) + + x509cr.Signature[0] ^= 0xFF + + _, err = sign(ca, x509cr, []certificatesv1.KeyUsage{certificatesv1.UsageClientAuth}, time.Hour, nil, time.Now) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(HavePrefix("unable to verify certificate request signature")) +} + func TestDuration(t *testing.T) { t.Parallel() diff --git a/control-plane-pki-operator/targetconfigcontroller/targetconfigcontroller.go b/control-plane-pki-operator/targetconfigcontroller/targetconfigcontroller.go index d633ab437b9c..2d104879b0a9 100644 --- a/control-plane-pki-operator/targetconfigcontroller/targetconfigcontroller.go +++ b/control-plane-pki-operator/targetconfigcontroller/targetconfigcontroller.go @@ -71,7 +71,7 @@ func createTargetConfig(ctx context.Context, c TargetConfigController, recorder _, _, err := ManageClientCABundle(ctx, c.configMapLister, c.kubeClient.CoreV1(), recorder, c.hostedControlPlane) if err != nil { - errors = append(errors, fmt.Errorf("%q: %v", "configmap/"+pkimanifests.TotalKASClientCABundle("placeholder").Name, err)) + errors = append(errors, fmt.Errorf("%q: %w", "configmap/"+pkimanifests.TotalKASClientCABundle("placeholder").Name, err)) } if len(errors) > 0 { diff --git a/control-plane-pki-operator/topology/detector.go b/control-plane-pki-operator/topology/detector.go index 3cdd85dcd6c8..fa1b6a7d1799 100644 --- a/control-plane-pki-operator/topology/detector.go +++ b/control-plane-pki-operator/topology/detector.go @@ -37,7 +37,7 @@ func (d Detector) DetectTopology(ctx context.Context, restClient *rest.Config) ( hcp, err := client.HostedControlPlanes(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { - return "", fmt.Errorf("failed to get hosted control plane %s/%s: %q", namespace, name, err) + return "", fmt.Errorf("failed to get hosted control plane %s/%s: %w", namespace, name, err) } if hcp == nil { diff --git a/docs/content/how-to/automated-machine-management/spot-instances.md b/docs/content/how-to/automated-machine-management/spot-instances.md new file mode 100644 index 000000000000..9b08b8b53cd5 --- /dev/null +++ b/docs/content/how-to/automated-machine-management/spot-instances.md @@ -0,0 +1,193 @@ +--- +title: Spot Instances +--- + +# Spot Instances + +AWS Spot instances use spare EC2 capacity at significantly reduced prices compared to on-demand instances, but may be interrupted with a 2-minute warning when EC2 needs the capacity back. HyperShift supports Spot instances for NodePools on AWS, with built-in graceful termination handling via SQS queues. + +!!! important + + Spot instances are suitable for fault-tolerant, stateless, and flexible workloads. They are **not recommended** for workloads that cannot tolerate interruptions. + +## Prerequisites + +Before creating a Spot instance NodePool, you must set up an SQS queue and EventBridge rules to receive EC2 interruption events. The AWS Node Termination Handler (NTH) deployed by HyperShift polls this queue and cordons/drains nodes before they are terminated, providing best-effort graceful shutdown. + +### 1. Create the SQS queue + +Create an SQS queue to receive Spot interruption notifications: + +```shell +export CLUSTER_NAME="my-cluster" +export AWS_REGION="us-east-1" + +aws sqs create-queue \ + --queue-name "${CLUSTER_NAME}-spot-interruption-queue" \ + --region "${AWS_REGION}" +``` + +Note the queue URL from the output — you will need it when creating the HostedCluster. + +### 2. Create EventBridge rules + +Create EventBridge rules to route EC2 Spot interruption warnings and rebalance recommendations to the SQS queue: + +```shell +QUEUE_ARN=$(aws sqs get-queue-attributes \ + --queue-url "https://sqs.${AWS_REGION}.amazonaws.com/$(aws sts get-caller-identity --query Account --output text)/${CLUSTER_NAME}-spot-interruption-queue" \ + --attribute-names QueueArn \ + --query 'Attributes.QueueArn' --output text) + +aws events put-rule \ + --name "${CLUSTER_NAME}-spot-interruption-warning" \ + --event-pattern '{"source":["aws.ec2"],"detail-type":["EC2 Spot Instance Interruption Warning"]}' \ + --region "${AWS_REGION}" + +aws events put-targets \ + --rule "${CLUSTER_NAME}-spot-interruption-warning" \ + --targets "Id=1,Arn=${QUEUE_ARN}" \ + --region "${AWS_REGION}" + +aws events put-rule \ + --name "${CLUSTER_NAME}-rebalance-recommendation" \ + --event-pattern '{"source":["aws.ec2"],"detail-type":["EC2 Instance Rebalance Recommendation"]}' \ + --region "${AWS_REGION}" + +aws events put-targets \ + --rule "${CLUSTER_NAME}-rebalance-recommendation" \ + --targets "Id=1,Arn=${QUEUE_ARN}" \ + --region "${AWS_REGION}" +``` + +### 3. Configure SQS queue policy + +Allow EventBridge to send messages to the queue: + +```shell +ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +QUEUE_URL="https://sqs.${AWS_REGION}.amazonaws.com/${ACCOUNT_ID}/${CLUSTER_NAME}-spot-interruption-queue" + +aws sqs set-queue-attributes \ + --queue-url "${QUEUE_URL}" \ + --attributes '{ + "Policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"events.amazonaws.com\"},\"Action\":\"sqs:SendMessage\",\"Resource\":\"'${QUEUE_ARN}'\"}]}" + }' +``` + +## Creating a Spot Instance NodePool + +### Step 1: Set the SQS queue URL on the HostedCluster + +The SQS queue URL is configured at the HostedCluster level in `spec.platform.aws.terminationHandlerQueueURL`. This enables the AWS Node Termination Handler component for all Spot NodePools in the cluster: + +```yaml +apiVersion: hypershift.openshift.io/v1beta1 +kind: HostedCluster +metadata: + name: my-cluster + namespace: clusters +spec: + platform: + type: AWS + aws: + region: us-east-1 + terminationHandlerQueueURL: "https://sqs.us-east-1.amazonaws.com/123456789012/my-cluster-spot-interruption-queue" + # ... other AWS configuration + # ... other spec fields +``` + +If you already have a running HostedCluster, you can patch it: + +```shell +oc patch hostedcluster my-cluster -n clusters --type merge -p '{ + "spec": { + "platform": { + "aws": { + "terminationHandlerQueueURL": "https://sqs.us-east-1.amazonaws.com/123456789012/my-cluster-spot-interruption-queue" + } + } + } +}' +``` + +### Step 2: Create a NodePool with Spot market type + +Create a NodePool with `spec.platform.aws.placement.marketType` set to `Spot`: + +```yaml +apiVersion: hypershift.openshift.io/v1beta1 +kind: NodePool +metadata: + name: spot-workers + namespace: clusters +spec: + clusterName: my-cluster + replicas: 3 + release: + image: quay.io/openshift-release-dev/ocp-release:4.18.0-x86_64 + management: + autoRepair: true + upgradeType: Replace + platform: + type: AWS + aws: + instanceType: m5.xlarge + instanceProfile: my-cluster-worker + rootVolume: + size: 120 + type: gp3 + placement: + marketType: Spot +``` + +### Setting a maximum price (optional) + +You can request Spot Instances at the Spot price, capped at the On-Demand price, or you can specify the maximum amount you're willing to pay: + +```yaml +spec: + platform: + aws: + placement: + marketType: Spot + spot: + maxPrice: "0.50" +``` + +The value is a decimal string representing the price per hour in USD. + +## Behavior + +When a NodePool is created with `marketType: Spot`, HyperShift labels all Machines and Nodes with `hypershift.openshift.io/interruptible-instance` and tags the EC2 instances with `aws-node-termination-handler/managed` so they can be identified by the termination handling components. + +### Graceful termination with SQS (recommended) + +When `terminationHandlerQueueURL` is set on the HostedCluster and at least one NodePool has `marketType: Spot`, HyperShift automatically deploys the Node Termination Handler (NTH) as a control plane component. The termination flow is: + +1. AWS sends a Spot interruption warning (2-minute notice) or rebalance recommendation to the SQS queue via EventBridge +2. NTH polls the queue and identifies the affected node +3. NTH cordons the node and drains it, respecting PodDisruptionBudgets +4. NTH taints the node (e.g., `aws-node-termination-handler/spot-itn`) +5. The spot remediation controller detects the taint, annotates the corresponding Machine with `hypershift.openshift.io/spot-interruption-signal`, and deletes it +6. The machine controller provisions a replacement Spot instance + +### Fallback without SQS + +Without the SQS queue, AWS terminates the instance abruptly with no graceful drain. In this case, the CAPI Machine enters a `Failed` state and the MachineHealthCheck triggers remediation to create a replacement. This path is slower and does not provide graceful pod shutdown. + +### MachineHealthCheck + +HyperShift creates a dedicated MachineHealthCheck (`-spot`) for each Spot NodePool. This MachineHealthCheck: + +- Targets only Machines with the `hypershift.openshift.io/interruptible-instance` label +- Sets `maxUnhealthy: 100%` because Spot reclamation can affect all instances simultaneously +- Uses an 8-minute unhealthy timeout (accounts for the 2-minute AWS notice plus shutdown time) +- Serves as a safety net in case the NTH + remediation controller path does not trigger + +### Constraints + +- Spot instances **cannot** be combined with Capacity Reservations +- Spot instances require **default** tenancy (dedicated tenancy is not supported) +- `spot` options (e.g., `maxPrice`) can only be specified when `marketType` is `Spot` +- Replacement instances are always Spot — if Spot capacity is unavailable, the replacement Machine will go `Failed` and the MachineHealthCheck will continue to retry remediation diff --git a/docs/content/how-to/azure/azure-workload-identity-setup.md b/docs/content/how-to/azure/azure-workload-identity-setup.md deleted file mode 100644 index c7523d6b119f..000000000000 --- a/docs/content/how-to/azure/azure-workload-identity-setup.md +++ /dev/null @@ -1,134 +0,0 @@ -# Azure Workload Identity Setup for Self-Managed Clusters - -!!! note "Developer Preview in OCP 4.21" - - Self-managed Azure HostedClusters are available as a Developer Preview feature in OpenShift Container Platform 4.21. - -This document describes how to set up Azure Workload Identities and OIDC issuer for self-managed Azure HostedClusters. - -!!! warning "Persistent Resource Groups" - - When setting up workload identities and OIDC issuer for the first time, create them in a **persistent resource group** that will not be deleted when individual clusters are destroyed. This allows you to reuse the same workload identities and OIDC issuer across multiple HostedClusters, reducing setup time and avoiding unnecessary resource recreation. - - - Use a persistent resource group like `os4-common`. - - This resource group should be separate from the cluster-specific resource groups that get created and deleted with each HostedCluster. - - The OIDC issuer storage account should also be created in this persistent resource group. - -## Prerequisites - -- Azure CLI (`az`) installed and configured -- `jq` command-line JSON processor -- Cloud Credential Operator (CCO) tool installed -- Appropriate Azure permissions - -## Create Azure Workload Identities - -!!! note "OIDC Issuer Required" - - Before running this command, you need an OIDC issuer URL. If you haven't set this up yet, see [Configure OIDC Issuer](#configure-oidc-issuer) below first. - -You can create the required managed identities and federated credentials using the HyperShift CLI: - -```bash -# Set environment variables -PERSISTENT_RG_NAME="os4-common" # Use persistent resource group -LOCATION="eastus" -CLUSTER_NAME="my-self-managed-cluster" -INFRA_ID="${CLUSTER_NAME}-$(openssl rand -hex 4)" -AZURE_CREDS="/path/to/azure-creds.json" - -# Create persistent resource group (if it doesn't exist) -az group create --name $PERSISTENT_RG_NAME --location $LOCATION - -# Create workload identities using the HyperShift CLI -# (requires OIDC issuer URL - see next section if you haven't set this up yet) -hypershift create iam azure \ - --name $CLUSTER_NAME \ - --infra-id $INFRA_ID \ - --azure-creds $AZURE_CREDS \ - --location $LOCATION \ - --resource-group-name $PERSISTENT_RG_NAME \ - --oidc-issuer-url $OIDC_ISSUER_URL \ - --output-file workload-identities.json -``` - -This creates 7 managed identities with federated credentials for: - -- Disk CSI driver -- File CSI driver -- Image Registry -- Ingress Operator -- Cloud Provider -- NodePool Management -- Network Operator - -To also create a KMS identity for Azure Key Vault etcd encryption at rest, add the `--enable-kms` flag: - -```bash -hypershift create iam azure \ - --name $CLUSTER_NAME \ - --infra-id $INFRA_ID \ - --azure-creds $AZURE_CREDS \ - --location $LOCATION \ - --resource-group-name $PERSISTENT_RG_NAME \ - --oidc-issuer-url $OIDC_ISSUER_URL \ - --output-file workload-identities.json \ - --enable-kms -``` - -!!! warning "KMS Key Vault Role Assignment" - - If you use `--enable-kms`, you must **manually** assign the `Key Vault Crypto User` role to the KMS identity on your Key Vault. The `--auto-assign-roles` flag does not cover this because the Key Vault scope is user-provided. See [Enabling KMS Encryption](create-self-managed-azure-cluster.md#enabling-kms-encryption-etcd-encryption-at-rest) for the role assignment commands. - -For complete documentation on the IAM commands, see [Create Azure IAM Resources Separately](create-iam-separately.md). - -## Configure OIDC Issuer - -Use the Cloud Credential Operator (CCO) tool to create the OIDC issuer: - -```bash -# Set OIDC issuer variables (reusing variables from previous steps) -OIDC_STORAGE_ACCOUNT_NAME="yourstorageaccount" -TENANT_ID="your-tenant-id" -# SUBSCRIPTION_ID and PERSISTENT_RG_NAME already set from previous section -# Create an RSA key pair and save the private and public key -ccoctl azure create-key-pair - -SA_TOKEN_ISSUER_PRIVATE_KEY_PATH="/path/to/serviceaccount-signer.private" -SA_TOKEN_ISSUER_PUBLIC_KEY_PATH="/path/to/serviceaccount-signer.public" - -# Create OIDC issuer using CCO tool in os4-common resource group -ccoctl azure create-oidc-issuer \ - --oidc-resource-group-name ${PERSISTENT_RG_NAME} \ - --tenant-id ${TENANT_ID} \ - --region ${LOCATION} \ - --name ${OIDC_STORAGE_ACCOUNT_NAME} \ - --subscription-id ${SUBSCRIPTION_ID} \ - --public-key-file ${SA_TOKEN_ISSUER_PUBLIC_KEY_PATH} - -# Set OIDC issuer URL -OIDC_ISSUER_URL="https://${OIDC_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${OIDC_STORAGE_ACCOUNT_NAME}" -``` - -## Verification - -Verify the setup: - -```bash -# List created managed identities -az identity list --resource-group $PERSISTENT_RG_NAME --output table - -# Verify federated credentials for one identity -az identity federated-credential list \ - --identity-name "${AZURE_DISK_MI_NAME}" \ - --resource-group $PERSISTENT_RG_NAME - -# Test OIDC issuer accessibility -curl -s "${OIDC_ISSUER_URL}/.well-known/openid-configuration" | jq . -``` - -## Next Steps - -After setting up workload identities, you can proceed to: - -- [Setup Azure Management Cluster for HyperShift](setup-management-cluster.md) \ No newline at end of file diff --git a/docs/content/how-to/azure/create-iam-separately.md b/docs/content/how-to/azure/create-iam-separately.md index e392d0ec577b..77abf4ab8091 100644 --- a/docs/content/how-to/azure/create-iam-separately.md +++ b/docs/content/how-to/azure/create-iam-separately.md @@ -4,6 +4,10 @@ title: Create Azure IAM resources separately # Create Azure IAM resources separately +!!! note "Developer Preview in OCP 4.21" + + Self-managed Azure HostedClusters are available as a Developer Preview feature in OpenShift Container Platform 4.21. + The `hypershift create iam azure` command creates Azure workload identities separately from infrastructure, following the same pattern as AWS and GCP. This enables you to manage IAM resources independently from your cluster infrastructure lifecycle. @@ -11,15 +15,30 @@ your cluster infrastructure lifecycle. ## Overview For self-managed Azure HyperShift clusters, workload identities authenticate cluster components to Azure -services using OIDC federation. You must create identities separately using `create iam azure` and then +services using OIDC federation. The setup consists of two steps: + +1. **Configure an OIDC Issuer** using the Cloud Credential Operator (CCO) tool +2. **Create workload identities** using `hypershift create iam azure` + +You must create identities separately using `create iam azure` and then consume them during infrastructure or cluster creation via the `--workload-identities-file` flag. -This approach provides control over the IAM lifecycle and follows the same pattern as AWS and GCP platforms. +!!! warning "Persistent Resource Groups" + + When setting up workload identities and OIDC issuer for the first time, create them in a **persistent resource group** that will not be deleted when individual clusters are destroyed. This allows you to reuse the same workload identities and OIDC issuer across multiple HostedClusters, reducing setup time and avoiding unnecessary resource recreation. + + - Use a persistent resource group like `os4-common`. + - This resource group should be separate from the cluster-specific resource groups that get created and deleted with each HostedCluster. + - The OIDC issuer storage account should also be created in this persistent resource group. ## Prerequisites Before creating Azure IAM resources, ensure you have: +- Azure CLI (`az`) installed and configured +- Cloud Credential Operator (CCO) tool (`ccoctl`) installed +- `jq` command-line JSON processor +- Appropriate Azure permissions - An Azure credentials file with the following format: ```json { @@ -30,20 +49,57 @@ Before creating Azure IAM resources, ensure you have: } ``` - An existing resource group where the managed identities will be created -- An OIDC issuer URL for workload identity federation + +## Configure OIDC Issuer + +Before creating workload identities, you need an OIDC issuer URL. Use the Cloud Credential Operator (CCO) tool to create one: + +```bash +# Set OIDC issuer variables +PERSISTENT_RG_NAME="os4-common" # Use persistent resource group +LOCATION="eastus" +OIDC_STORAGE_ACCOUNT_NAME="yourstorageaccount" +TENANT_ID="your-tenant-id" +SUBSCRIPTION_ID="your-subscription-id" + +# Create persistent resource group (if it doesn't exist) +az group create --name $PERSISTENT_RG_NAME --location $LOCATION + +# Create an RSA key pair and save the private and public key +ccoctl azure create-key-pair + +SA_TOKEN_ISSUER_PRIVATE_KEY_PATH="/path/to/serviceaccount-signer.private" +SA_TOKEN_ISSUER_PUBLIC_KEY_PATH="/path/to/serviceaccount-signer.public" + +# Create OIDC issuer using CCO tool in persistent resource group +ccoctl azure create-oidc-issuer \ + --oidc-resource-group-name ${PERSISTENT_RG_NAME} \ + --tenant-id ${TENANT_ID} \ + --region ${LOCATION} \ + --name ${OIDC_STORAGE_ACCOUNT_NAME} \ + --subscription-id ${SUBSCRIPTION_ID} \ + --public-key-file ${SA_TOKEN_ISSUER_PUBLIC_KEY_PATH} + +# Set OIDC issuer URL +OIDC_ISSUER_URL="https://${OIDC_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${OIDC_STORAGE_ACCOUNT_NAME}" +``` ## Creating Workload Identities Use the `hypershift create iam azure` command: ```bash +CLUSTER_NAME="my-self-managed-cluster" +INFRA_ID="${CLUSTER_NAME}-$(openssl rand -hex 4)" +AZURE_CREDS="/path/to/azure-creds.json" + hypershift create iam azure \ - --name CLUSTER_NAME \ - --infra-id INFRA_ID \ - --azure-creds AZURE_CREDENTIALS_FILE \ - --location LOCATION \ - --resource-group-name RESOURCE_GROUP \ - --oidc-issuer-url OIDC_ISSUER_URL \ + --name $CLUSTER_NAME \ + --infra-id $INFRA_ID \ + --azure-creds $AZURE_CREDS \ + --location $LOCATION \ + --resource-group-name $PERSISTENT_RG_NAME \ + --oidc-issuer-url $OIDC_ISSUER_URL \ --output-file workload-identities.json ``` @@ -52,11 +108,11 @@ where: * `CLUSTER_NAME` is the name of the hosted cluster you intend to create. * `INFRA_ID` is a unique identifier used to name Azure resources. Typically this is the cluster name with a random suffix appended. -* `AZURE_CREDENTIALS_FILE` points to an Azure credentials file with permission to create +* `AZURE_CREDS` points to an Azure credentials file with permission to create managed identities and federated credentials. * `LOCATION` is the Azure region for the managed identities (e.g., `eastus`, `westus2`). -* `RESOURCE_GROUP` is the name of an existing resource group where identities will be created. -* `OIDC_ISSUER_URL` is the URL of the OIDC identity provider used for workload identity federation. +* `PERSISTENT_RG_NAME` is the name of an existing resource group where identities will be created. +* `OIDC_ISSUER_URL` is the URL of the OIDC identity provider created in the previous step. Running this command creates: @@ -71,6 +127,26 @@ Running this command creates: - Control Plane Operator * Federated Identity Credentials for each identity, configured with the OIDC issuer +### Enabling KMS Identity + +To also create a KMS identity for Azure Key Vault etcd encryption at rest, add the `--enable-kms` flag: + +```bash +hypershift create iam azure \ + --name $CLUSTER_NAME \ + --infra-id $INFRA_ID \ + --azure-creds $AZURE_CREDS \ + --location $LOCATION \ + --resource-group-name $PERSISTENT_RG_NAME \ + --oidc-issuer-url $OIDC_ISSUER_URL \ + --output-file workload-identities.json \ + --enable-kms +``` + +!!! warning "KMS Key Vault Role Assignment" + + If you use `--enable-kms`, you must **manually** assign the `Key Vault Crypto User` role to the KMS identity on your Key Vault. The `--auto-assign-roles` flag does not cover this because the Key Vault scope is user-provided. See [Enabling KMS Encryption](create-self-managed-azure-cluster.md#enabling-kms-encryption-etcd-encryption-at-rest) for the role assignment commands. + ## Private Endpoint Access The **Control Plane Operator** identity is always created by `create iam azure`. For private @@ -150,6 +226,23 @@ hypershift create cluster azure \ --workload-identities-file workload-identities.json ``` +## Verification + +Verify the setup: + +```bash +# List created managed identities +az identity list --resource-group $PERSISTENT_RG_NAME --output table + +# Verify federated credentials for one identity +az identity federated-credential list \ + --identity-name "${CLUSTER_NAME}-disk-${INFRA_ID}" \ + --resource-group $PERSISTENT_RG_NAME + +# Test OIDC issuer accessibility +curl -s "${OIDC_ISSUER_URL}/.well-known/openid-configuration" | jq . +``` + ## Destroying Workload Identities To destroy the workload identities that were created: @@ -181,16 +274,17 @@ Both the managed identities and their federated credentials are removed. | `--name` | Name of the HostedCluster | | `--infra-id` | Unique infrastructure identifier | | `--azure-creds` | Path to Azure credentials JSON file | -| `--location` | Azure region for identities | +| `--resource-group-name` | Resource group for identities | | `--oidc-issuer-url` | OIDC issuer URL for federation | +| `--output-file` | Output file path | ### Optional Flags for `create iam azure` | Flag | Description | Default | |------|-------------|---------| -| `--resource-group-name` | Resource group for identities | `{name}-{infra-id}` | -| `--output-file` | Output file path | `{name}-iam-output.json` | +| `--location` | Azure region for identities | `eastus` | | `--cloud` | Azure cloud environment | `AzurePublicCloud` | +| `--enable-kms` | Create KMS identity for etcd encryption | `false` | ### Required Flags for `destroy iam azure` @@ -220,22 +314,37 @@ export INFRA_ID="${NAME}-$(openssl rand -hex 4)" export LOCATION="eastus" export BASE_DOMAIN="example.com" export AZURE_CREDS="/path/to/azure-creds.json" -export OIDC_ISSUER_URL="https://my-oidc-issuer.com" - -# 2. Create a resource group for identities -az group create --name ${NAME}-rg --location ${LOCATION} - -# 3. Create workload identities +export PERSISTENT_RG_NAME="os4-common" +export TENANT_ID="your-tenant-id" +export SUBSCRIPTION_ID="your-subscription-id" +export OIDC_STORAGE_ACCOUNT_NAME="yourstorageaccount" +export RELEASE_IMAGE="quay.io/openshift-release-dev/ocp-release:XYZ" + +# 2. Create persistent resource group (if it doesn't exist) +az group create --name ${PERSISTENT_RG_NAME} --location ${LOCATION} + +# 3. Create OIDC issuer +ccoctl azure create-key-pair +ccoctl azure create-oidc-issuer \ + --oidc-resource-group-name ${PERSISTENT_RG_NAME} \ + --tenant-id ${TENANT_ID} \ + --region ${LOCATION} \ + --name ${OIDC_STORAGE_ACCOUNT_NAME} \ + --subscription-id ${SUBSCRIPTION_ID} \ + --public-key-file /path/to/serviceaccount-signer.public +export OIDC_ISSUER_URL="https://${OIDC_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${OIDC_STORAGE_ACCOUNT_NAME}" + +# 4. Create workload identities hypershift create iam azure \ --name ${NAME} \ --infra-id ${INFRA_ID} \ --azure-creds ${AZURE_CREDS} \ --location ${LOCATION} \ - --resource-group-name ${NAME}-rg \ + --resource-group-name ${PERSISTENT_RG_NAME} \ --oidc-issuer-url ${OIDC_ISSUER_URL} \ --output-file workload-identities.json -# 4. Create infrastructure using pre-created identities +# 5. Create infrastructure using pre-created identities hypershift create infra azure \ --name ${NAME} \ --infra-id ${INFRA_ID} \ @@ -243,9 +352,11 @@ hypershift create infra azure \ --base-domain ${BASE_DOMAIN} \ --location ${LOCATION} \ --workload-identities-file workload-identities.json \ + --assign-identity-roles \ + --dns-zone-rg-name ${PERSISTENT_RG_NAME} \ --output-file infra-output.yaml -# 5. Create the cluster +# 6. Create the cluster hypershift create cluster azure \ --name ${NAME} \ --infra-id ${INFRA_ID} \ @@ -253,35 +364,41 @@ hypershift create cluster azure \ --base-domain ${BASE_DOMAIN} \ --location ${LOCATION} \ --pull-secret /path/to/pull-secret \ - --infra-json infra-output.yaml + --generate-ssh \ + --release-image ${RELEASE_IMAGE} \ + --sa-token-issuer-private-key-path /path/to/serviceaccount-signer.private \ + --oidc-issuer-url ${OIDC_ISSUER_URL} \ + --dns-zone-rg-name ${PERSISTENT_RG_NAME} \ + --assign-service-principal-roles \ + --infra-json infra-output.yaml \ + --diagnostics-storage-account-type Managed # --- Cleanup --- -# 6. Destroy the cluster +# 7. Destroy the cluster hypershift destroy cluster azure \ --name ${NAME} \ --azure-creds ${AZURE_CREDS} \ - --dns-zone-rg-name ${DNS_ZONE_RG} + --dns-zone-rg-name ${PERSISTENT_RG_NAME} -# 7. Destroy infrastructure +# 8. Destroy infrastructure hypershift destroy infra azure \ --name ${NAME} \ --infra-id ${INFRA_ID} \ --azure-creds ${AZURE_CREDS} -# 8. Destroy IAM resources +# 9. Destroy IAM resources (only if no longer needed) hypershift destroy iam azure \ --azure-creds ${AZURE_CREDS} \ --workload-identities-file workload-identities.json \ - --resource-group-name ${RESOURCE_GROUP} \ + --resource-group-name ${PERSISTENT_RG_NAME} \ --name ${NAME} \ --infra-id ${INFRA_ID} \ - --dns-zone-rg-name ${DNS_ZONE_RG} + --dns-zone-rg-name ${PERSISTENT_RG_NAME} ``` ## See Also - [Create Azure Infrastructure Separately](create-infra-separately.md) -- [Azure Workload Identity Setup](azure-workload-identity-setup.md) - [Self-Managed Azure Overview](self-managed-azure-index.md) - [Deploy Azure Private Clusters](deploy-azure-private-clusters.md) — End-to-end guide for private endpoint access diff --git a/docs/content/how-to/azure/create-self-managed-azure-cluster.md b/docs/content/how-to/azure/create-self-managed-azure-cluster.md index c7062a22d36e..a0bffb39b88e 100644 --- a/docs/content/how-to/azure/create-self-managed-azure-cluster.md +++ b/docs/content/how-to/azure/create-self-managed-azure-cluster.md @@ -27,7 +27,7 @@ Before creating a self-managed Azure HostedCluster, ensure you have: This guide assumes you have already completed the workload identity configuration and management cluster setup. Follow these guides in order: - 1. [Azure Workload Identity Setup](azure-workload-identity-setup.md) - Workload identities and OIDC issuer configuration + 1. [Create Azure IAM Resources](create-iam-separately.md) - Workload identities and OIDC issuer configuration 2. [Setup Azure Management Cluster for HyperShift](setup-management-cluster.md) - HyperShift operator installation (with or without External DNS) ### Permission Requirements @@ -42,124 +42,71 @@ Your Azure service principal must have the following permissions: ## Creating the Self-Managed Azure HostedCluster -!!! tip "Alternative: Use `create infra azure` and `create iam azure`" - - This guide creates Azure infrastructure manually with `az` CLI commands for - transparency. Alternatively, you can use the HyperShift CLI to automate - infrastructure and IAM creation: - - - [Create Azure IAM Resources Separately](create-iam-separately.md) — `hypershift create iam azure` - - [Create Azure Infrastructure Separately](create-infra-separately.md) — `hypershift create infra azure` - - The private cluster guide ([Deploy Azure Private Clusters](deploy-azure-private-clusters.md)) - uses these automated commands and is the recommended approach for private topology. - ### Infrastructure Setup -Before creating the HostedCluster, set up the necessary Azure infrastructure: - -!!! note "About PERSISTENT_RG_NAME" - In Red Hat environments, a periodic Azure resource "reaper" deletes resources that are not properly tagged or not located in an approved resource group. We frequently use the `os4-common` resource group for shared, long-lived assets (for example, public DNS zones) to avoid accidental cleanup. If you are not in Red Hat infrastructure, set `PERSISTENT_RG_NAME` to any long-lived resource group in your subscription that will not be automatically reaped, or ensure your organization's required tags/policies are applied. The name does not have to be `os4-common`—use whatever persistent resource group fits your environment. +Create the Azure infrastructure using the HyperShift CLI: ```bash # Set cluster configuration variables -PREFIX="your-prefix-sm" -RELEASE_IMAGE="quay.io/openshift-release-dev/ocp-release:XYZ" -TAG="latest" - +CLUSTER_NAME="my-self-managed-cluster" +INFRA_ID="${CLUSTER_NAME}-$(openssl rand -hex 4)" LOCATION="eastus" -MANAGED_RG_NAME="${PREFIX}-managed-rg" -VNET_RG_NAME="${PREFIX}-customer-vnet-rg" -NSG_RG_NAME="${PREFIX}-customer-nsg-rg" -VNET_NAME="${PREFIX}-customer-vnet" -VNET_SUBNET1="${PREFIX}-customer-subnet-1" -NSG="${PREFIX}-customer-nsg" -DNS_ZONE_NAME="your-subdomain.your-parent.dns.zone.com" -CLUSTER_NAMESPACE="clusters" -CLUSTER_NAME="${PREFIX}-hc" -AZURE_CREDS="/path/to/azure/credentials" +BASE_DOMAIN="example.com" +AZURE_CREDS="/path/to/azure-creds.json" PULL_SECRET="/path/to/pull-secret.json" -HYPERSHIFT_BINARY_PATH="/path/to/hypershift/bin" +RELEASE_IMAGE="quay.io/openshift-release-dev/ocp-release:XYZ" +PERSISTENT_RG_NAME="os4-common" OIDC_ISSUER_URL="https://yourstorageaccount.blob.core.windows.net/yourstorageaccount" SA_TOKEN_ISSUER_PRIVATE_KEY_PATH="/path/to/serviceaccount-signer.private" -PERSISTENT_RG_NAME="os4-common" -PARENT_DNS_ZONE="your-parent.dns.zone.com" - -# Clean up any previous instances (optional) -az group delete -n "${VNET_RG_NAME}" --yes --no-wait || true -az group delete -n "${NSG_RG_NAME}" --yes --no-wait || true - -# Create managed resource group -az group create --name "${MANAGED_RG_NAME}" --location ${LOCATION} - -# Create VNET & NSG resource groups -az group create --name "${VNET_RG_NAME}" --location ${LOCATION} -az group create --name "${NSG_RG_NAME}" --location ${LOCATION} - -# Create network security group -az network nsg create \ - --resource-group "${NSG_RG_NAME}" \ - --name "${NSG}" - -# Get NSG ID -GetNsgID=$(az network nsg list --query "[?name=='${NSG}'].id" -o tsv) - -# Create VNet with subnet -az network vnet create \ - --name "${VNET_NAME}" \ - --resource-group "${VNET_RG_NAME}" \ - --address-prefix 10.0.0.0/16 \ - --subnet-name "${VNET_SUBNET1}" \ - --subnet-prefixes 10.0.0.0/24 \ - --nsg "${GetNsgID}" - -# Get VNet and Subnet IDs -GetVnetID=$(az network vnet list --query "[?name=='${VNET_NAME}'].id" -o tsv) -GetSubnetID=$(az network vnet subnet show \ - --vnet-name "${VNET_NAME}" \ - --name "${VNET_SUBNET1}" \ - --resource-group "${VNET_RG_NAME}" \ - --query id --output tsv) -``` -### Create the HostedCluster +# Create infrastructure +hypershift create infra azure \ + --name ${CLUSTER_NAME} \ + --infra-id ${INFRA_ID} \ + --azure-creds ${AZURE_CREDS} \ + --base-domain ${BASE_DOMAIN} \ + --location ${LOCATION} \ + --workload-identities-file workload-identities.json \ + --assign-identity-roles \ + --dns-zone-rg-name ${PERSISTENT_RG_NAME} \ + --output-file infra-output.yaml +``` -!!! note "Federated Identity Prerequisites" +This creates the resource groups, VNet, subnet, NSG, Private DNS zone, and load balancer for your cluster. For advanced options like using existing network resources, see [Create Azure Infrastructure Separately](create-infra-separately.md). - Before creating the cluster, ensure that all federated identity credentials have been set up for your workload identities as described in the [Azure Workload Identity Setup](azure-workload-identity-setup.md) guide. The cluster creation will fail if these are not properly configured. +### Create the HostedCluster !!! note "Azure Marketplace Images" For OpenShift 4.20 and later, HyperShift automatically selects the appropriate Azure Marketplace image from the release payload. You no longer need to specify `--marketplace-*` flags unless you want to use a specific custom image. See [Configuring Azure Marketplace Images](#configuring-azure-marketplace-images) for more details. -Create the HostedCluster: +Create the HostedCluster using the infrastructure output: ```bash -# Create the HostedCluster -${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ +hypershift create cluster azure \ --name "$CLUSTER_NAME" \ - --namespace "$CLUSTER_NAMESPACE" \ + --infra-id "$INFRA_ID" \ --azure-creds $AZURE_CREDS \ --location ${LOCATION} \ --node-pool-replicas 2 \ - --base-domain $PARENT_DNS_ZONE \ + --base-domain $BASE_DOMAIN \ --pull-secret $PULL_SECRET \ --generate-ssh \ --release-image ${RELEASE_IMAGE} \ - --external-dns-domain ${DNS_ZONE_NAME} \ - --resource-group-name "${MANAGED_RG_NAME}" \ - --vnet-id "${GetVnetID}" \ - --subnet-id "${GetSubnetID}" \ - --network-security-group-id "${GetNsgID}" \ --sa-token-issuer-private-key-path "${SA_TOKEN_ISSUER_PRIVATE_KEY_PATH}" \ --oidc-issuer-url "${OIDC_ISSUER_URL}" \ - --control-plane-operator-image="quay.io/hypershift/hypershift:${TAG}" \ --dns-zone-rg-name ${PERSISTENT_RG_NAME} \ --assign-service-principal-roles \ - --workload-identities-file ./workload-identities.json \ + --infra-json infra-output.yaml \ --diagnostics-storage-account-type Managed ``` +!!! tip "External DNS" + + If using External DNS for automatic DNS management, also pass + `--external-dns-domain ` to the cluster creation command. + See [Setup Azure Management Cluster](setup-management-cluster.md) for DNS configuration. + !!! tip "Private Clusters" To create a private cluster with Azure Private Link, see @@ -168,15 +115,6 @@ ${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ cluster's VNet, `--endpoint-access Private` flag, and HyperShift operator installation with `--private-platform Azure`. -!!! warning "Private Clusters: Avoid DNS Zone Shadowing" - - If creating a **private** Azure HostedCluster, ensure `--external-dns-domain` does - not match `{clusterName}.{baseDomain}` or its parent domain. A matching value - causes an Azure Private DNS zone to shadow `*.apps` resolution, breaking console - and all ingress. This cannot be fixed after creation. See - [External DNS Domain Must Not Match Cluster Domain](deploy-azure-private-clusters.md#external-dns-domain-must-not-match-cluster-domain) - for details. - ### Configuring Azure Marketplace Images HyperShift supports multiple approaches for configuring Azure Marketplace images for your cluster nodes. The recommended approach varies based on your OpenShift version and requirements. @@ -190,7 +128,7 @@ For OpenShift 4.20+, HyperShift automatically selects the appropriate Azure Mark ```bash # No marketplace flags needed - HyperShift will auto-select the image # Gen2 VM generation is used by default -${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ +hypershift create cluster azure \ --name "$CLUSTER_NAME" \ # ... other flags ... ``` @@ -202,7 +140,7 @@ This is the **recommended approach** as it ensures your nodes use the officially If you need to use a specific VM generation (Gen1 or Gen2), you can specify only the `--image-generation` flag: ```bash -${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ +hypershift create cluster azure \ --name "$CLUSTER_NAME" \ --image-generation Gen2 \ # Or Gen1 (case-sensitive) # ... other flags ... @@ -219,7 +157,7 @@ ${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ If you need to use a specific custom marketplace image, provide all marketplace details: ```bash -${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ +hypershift create cluster azure \ --name "$CLUSTER_NAME" \ --marketplace-publisher azureopenshift \ --marketplace-offer aro4 \ @@ -243,18 +181,18 @@ When creating additional NodePools, you can specify image configuration in the s ```bash # Use default from release payload (OCP 4.20+) -${HYPERSHIFT_BINARY_PATH}/hypershift create nodepool azure \ +hypershift create nodepool azure \ --cluster-name "$CLUSTER_NAME" \ # ... other flags ... # Or specify generation -${HYPERSHIFT_BINARY_PATH}/hypershift create nodepool azure \ +hypershift create nodepool azure \ --cluster-name "$CLUSTER_NAME" \ --image-generation Gen1 \ # ... other flags ... # Or use custom marketplace image -${HYPERSHIFT_BINARY_PATH}/hypershift create nodepool azure \ +hypershift create nodepool azure \ --cluster-name "$CLUSTER_NAME" \ --marketplace-publisher azureopenshift \ --marketplace-offer aro4 \ @@ -265,16 +203,14 @@ ${HYPERSHIFT_BINARY_PATH}/hypershift create nodepool azure \ !!! important "Key Configuration Options" - - `--workload-identities-file`: References the workload identities configuration created in the setup guide + - `--infra-json`: Path to infrastructure output from `hypershift create infra azure` (includes workload identities) - `--assign-service-principal-roles`: Automatically assigns required Azure roles to workload identities - `--sa-token-issuer-private-key-path`: Path to the private key for service account token signing - - `--oidc-issuer-url`: URL of the OIDC issuer created in the workload identity setup - - `--vnet-id`, `--subnet-id`, `--network-security-group-id`: Custom networking infrastructure + - `--oidc-issuer-url`: URL of the OIDC issuer created in the IAM setup - `--image-generation`: (Optional) VM generation (`Gen1` or `Gen2`, defaults to `Gen2`). For OCP 4.20+, omit to use release payload defaults. See [Configuring Azure Marketplace Images](#configuring-azure-marketplace-images) - `--marketplace-publisher/offer/sku/version`: (Optional) Explicit Azure Marketplace image. Must specify all four flags together, or omit all to use defaults (OCP 4.20+) - `--dns-zone-rg-name`: Resource group containing the DNS zone (os4-common) - `--diagnostics-storage-account-type Managed`: Use Azure managed storage for diagnostics - - `--control-plane-operator-image`: Custom HyperShift operator image (optional) ## Enabling KMS Encryption (etcd Encryption at Rest) @@ -285,19 +221,7 @@ Self-managed Azure HostedClusters support encrypting etcd data at rest using [Az ### Prerequisites -Ensure the `kms` workload identity is included in your `workload-identities.json` file. When using `hypershift create iam azure`, pass the `--enable-kms` flag to create the KMS identity (using the `INFRA_ID` set during [Azure Workload Identity Setup](azure-workload-identity-setup.md)): - -```bash -hypershift create iam azure \ - --name "$CLUSTER_NAME" \ - --infra-id "$INFRA_ID" \ - --azure-creds "$AZURE_CREDS" \ - --location "$LOCATION" \ - --resource-group-name "$PERSISTENT_RG_NAME" \ - --oidc-issuer-url "$OIDC_ISSUER_URL" \ - --output-file ./workload-identities.json \ - --enable-kms -``` +Ensure the `kms` workload identity is included in your `workload-identities.json` file. When using `hypershift create iam azure`, pass the `--enable-kms` flag to create the KMS identity. See [Enabling KMS Identity](create-iam-separately.md#enabling-kms-identity) for details. ### Create a Key Vault and Key @@ -317,7 +241,8 @@ hypershift create iam azure \ ```bash # Create Key Vault -KV_NAME="${PREFIX}-kv" +KV_NAME="${CLUSTER_NAME}-kv" +MANAGED_RG_NAME="${CLUSTER_NAME}-managed-rg" az keyvault create \ --name "${KV_NAME}" \ --resource-group "${MANAGED_RG_NAME}" \ @@ -325,7 +250,7 @@ az keyvault create \ --enable-rbac-authorization # Create encryption key -KEY_NAME="${PREFIX}-etcd-key" +KEY_NAME="${CLUSTER_NAME}-etcd-key" az keyvault key create \ --vault-name "${KV_NAME}" \ --name "${KEY_NAME}" \ @@ -376,26 +301,21 @@ az role assignment create \ Add the `--encryption-key-id` flag to your cluster creation command: ```bash -${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ +hypershift create cluster azure \ --name "$CLUSTER_NAME" \ - --namespace "$CLUSTER_NAMESPACE" \ + --infra-id "$INFRA_ID" \ --azure-creds $AZURE_CREDS \ --location ${LOCATION} \ --node-pool-replicas 2 \ - --base-domain $PARENT_DNS_ZONE \ + --base-domain $BASE_DOMAIN \ --pull-secret $PULL_SECRET \ --generate-ssh \ --release-image ${RELEASE_IMAGE} \ - --external-dns-domain ${DNS_ZONE_NAME} \ - --resource-group-name "${MANAGED_RG_NAME}" \ - --vnet-id "${GetVnetID}" \ - --subnet-id "${GetSubnetID}" \ - --network-security-group-id "${GetNsgID}" \ --sa-token-issuer-private-key-path "${SA_TOKEN_ISSUER_PRIVATE_KEY_PATH}" \ --oidc-issuer-url "${OIDC_ISSUER_URL}" \ --dns-zone-rg-name ${PERSISTENT_RG_NAME} \ --assign-service-principal-roles \ - --workload-identities-file ./workload-identities.json \ + --infra-json infra-output.yaml \ --encryption-key-id "${ENCRYPTION_KEY_ID}" \ --diagnostics-storage-account-type Managed ``` @@ -424,22 +344,28 @@ oc get clusterversion ## Cleanup -To delete the HostedCluster: +To delete the HostedCluster and its infrastructure: ```bash # Delete the HostedCluster hypershift destroy cluster azure \ --name $CLUSTER_NAME \ --azure-creds $AZURE_CREDS \ - --resource-group-name $MANAGED_RG_NAME \ --dns-zone-rg-name $PERSISTENT_RG_NAME + +# Destroy infrastructure +hypershift destroy infra azure \ + --name $CLUSTER_NAME \ + --infra-id $INFRA_ID \ + --azure-creds $AZURE_CREDS ``` !!! note "Resource Cleanup" - The HyperShift destroy command will clean up the cluster resources. Workload identities and OIDC issuer created during setup can be reused for other clusters or cleaned up separately if no longer needed. + The HyperShift destroy commands clean up the cluster and infrastructure resources. Workload identities and OIDC issuer created during setup can be reused for other clusters or cleaned up separately if no longer needed. See [Destroying Workload Identities](create-iam-separately.md#destroying-workload-identities). ## Related Documentation -1. [Azure Workload Identity Setup](azure-workload-identity-setup.md) - Workload identities and OIDC issuer setup -2. [Setup Azure Management Cluster for HyperShift](setup-management-cluster.md) - DNS and HyperShift operator setup \ No newline at end of file +1. [Create Azure IAM Resources](create-iam-separately.md) - Workload identities and OIDC issuer setup +2. [Create Azure Infrastructure Separately](create-infra-separately.md) - Advanced infrastructure options +3. [Setup Azure Management Cluster for HyperShift](setup-management-cluster.md) - DNS and HyperShift operator setup diff --git a/docs/content/how-to/azure/deploy-azure-private-clusters.md b/docs/content/how-to/azure/deploy-azure-private-clusters.md index 99931b13d047..1184500fcdf2 100644 --- a/docs/content/how-to/azure/deploy-azure-private-clusters.md +++ b/docs/content/how-to/azure/deploy-azure-private-clusters.md @@ -468,74 +468,6 @@ The deletion process automatically cleans up Private Link resources in the corre 1. `.hypershift.local` — synthetic internal zone with `api` and `*.apps` records 2. `` — base domain zone with `api-` and `oauth-` records -### External DNS Domain Must Not Match Cluster Domain - -!!! warning "Azure Private DNS Zone Shadowing" - - On private Azure HostedClusters, do **not** set `--external-dns-domain` to a value - that matches or is a parent domain of `{clusterName}.{baseDomain}`. For example, - if your cluster is named `my-cluster` with base domain `example.com`, do not use - `--external-dns-domain my-cluster.example.com` or `--external-dns-domain example.com`. - - This misconfiguration **cannot be corrected after cluster creation** because the - relevant fields (`spec.services`, `spec.dns.baseDomain`, and `metadata.name`) are - all immutable. The cluster must be destroyed and recreated with a different - `--external-dns-domain` value. - - **Safe example**: If your cluster is `my-cluster` with base domain `example.com`, - use a separate subdomain such as `--external-dns-domain custom-dns.example.com` - that does not overlap with `my-cluster.example.com`. - -#### What Goes Wrong - -Private Azure clusters use two separate routing paths: - -1. **Management-plane router** (`private-router`): An HAProxy pod in the hosted - control plane namespace, fronted by an internal load balancer and exposed to the - guest VNet through Azure Private Link. Worker nodes reach this router via the - Private Endpoint IP. HAProxy uses SNI-based routing and only has ACLs for - `.hypershift.local` hostnames (KAS, ignition, konnectivity, OAuth). Any hostname - that does not match an ACL falls through to the `default_backend kube_api`, which - returns KAS certificates. - -2. **Data-plane router** (`router-default`): The OpenShift ingress controller running - on worker nodes, serving `*.apps.{clusterName}.{baseDomain}` hostnames with the - correct wildcard ingress certificate. - -When `--external-dns-domain` matches the cluster domain, the PLS controller creates a -Private DNS zone named `{clusterName}.{baseDomain}`. This zone becomes authoritative -for **all** queries under that name within the guest VNet, including -`*.apps.{clusterName}.{baseDomain}`. Since the zone only has `api` and `oauth` A -records pointing to the Private Endpoint IP, apps queries either: - -- Return **NXDOMAIN** (if no `*.apps` record exists in the zone), or -- Resolve to the **Private Endpoint IP**, which routes to `private-router` (HAProxy). - Because `*.apps` hostnames do not match any HAProxy SNI ACL, traffic falls through - to `kube_api` and the client receives a **TLS certificate mismatch** (KAS cert - instead of the ingress wildcard cert). - -Neither outcome is usable. The console, OAuth login, and all application routes are -unreachable. - -#### Why the Controller Cannot Self-Heal - -The controller cannot fix this by adding a `*.apps` wildcard record to the shadowing -zone because: - -- The Private Endpoint IP routes to the management-plane `private-router`, not the - data-plane `router-default`. Adding `*.apps → PE IP` would route apps traffic to - HAProxy, which does not serve those hostnames. -- The correct target (the data-plane ingress IP on worker nodes) is not available to - the PLS controller. The controller runs in the control plane and has no client to - the guest cluster. There is no HCP status field that reports the guest ingress IP, - and the HostedCluster Controller Operator (HCCO) does not propagate it back. - -When the controller detects shadowing, it sets `AzurePrivateDNSAvailable=False` with -reason `BaseDomainShadowsClusterDomain` and skips zone creation entirely. This -prevents the shadowing zone from being created, but the `api` and `oauth` hostnames -from `--external-dns-domain` will not resolve via Private DNS. The cluster must be -recreated with a non-overlapping domain. - ### Condition Debugging If the cluster gets stuck, check the `AzurePrivateLinkService` CR conditions: @@ -549,7 +481,7 @@ oc get azureprivatelinkservices -n clusters-${CLUSTER_NAME} -o jsonpath='{.items | `AzureInternalLoadBalancerAvailable` = False | The `private-router` Service hasn't received an ILB IP yet. Check the Service status and Azure networking. | | `AzurePLSCreated` = False | PLS creation failed. Check NAT subnet policies, credentials, and the HO operator logs. | | `AzurePrivateEndpointAvailable` = False | PE creation failed or connection not approved. Check the PLS auto-approval list and CPO logs. | -| `AzurePrivateDNSAvailable` = False | DNS zone or record creation failed. If the reason is `BaseDomainShadowsClusterDomain`, the `--external-dns-domain` value overlaps with the cluster domain — the cluster must be recreated with a different value. See [External DNS Domain Must Not Match Cluster Domain](#external-dns-domain-must-not-match-cluster-domain). | +| `AzurePrivateDNSAvailable` = False | DNS zone or record creation failed. Check CPO identity permissions in the guest subscription. | ## Related Documentation diff --git a/docs/content/how-to/azure/index.md b/docs/content/how-to/azure/index.md index d0fd3c8d9adc..326e898ab4a5 100644 --- a/docs/content/how-to/azure/index.md +++ b/docs/content/how-to/azure/index.md @@ -24,10 +24,9 @@ Self-managed Azure uses an OpenShift cluster (running on any platform - AWS, Azu **Guides:** - [Self-Managed Azure Overview](self-managed-azure-index.md) - Architecture and deployment workflow -- [Azure Workload Identity Setup](azure-workload-identity-setup.md) - Set up managed identities and OIDC federation +- [Create Azure IAM Resources](create-iam-separately.md) - Set up OIDC issuer, managed identities, and workload identity federation - [Setup Azure Management Cluster](setup-management-cluster.md) - Install HyperShift operator - [Create a Self-Managed Azure HostedCluster](create-self-managed-azure-cluster.md) - Deploy your first hosted cluster -- [Create Azure IAM Resources Separately](create-iam-separately.md) - Manage workload identities independently - [Create Azure Infrastructure Separately](create-infra-separately.md) - Create infrastructure before cluster ## Comparison diff --git a/docs/content/how-to/azure/self-managed-azure-index.md b/docs/content/how-to/azure/self-managed-azure-index.md index db41b912195a..afdf2bc053dc 100644 --- a/docs/content/how-to/azure/self-managed-azure-index.md +++ b/docs/content/how-to/azure/self-managed-azure-index.md @@ -66,10 +66,7 @@ You can create workload identities using either: **When to Complete**: This is a one-time setup that can be reused across multiple hosted clusters. Complete this before proceeding to Phase 2. -👉 **Guides**: - -- [Azure Workload Identity Setup](azure-workload-identity-setup.md) - Overview with CLI and OIDC configuration -- [Create Azure IAM Resources Separately](create-iam-separately.md) - Detailed IAM command reference +👉 **Guide**: [Create Azure IAM Resources](create-iam-separately.md) - OIDC issuer configuration and workload identity creation ### Phase 2: Management Cluster Setup @@ -168,7 +165,7 @@ Self-managed Azure HyperShift implements several security best practices: Begin your self-managed Azure HyperShift deployment by following the guides in order: -1. **[Azure Workload Identity Setup](azure-workload-identity-setup.md)** - Set up managed identities and OIDC federation (or use [Create Azure IAM Resources Separately](create-iam-separately.md) for CLI-based setup) +1. **[Create Azure IAM Resources](create-iam-separately.md)** - Set up OIDC issuer, managed identities, and workload identity federation 2. **[Setup Azure Management Cluster for HyperShift](setup-management-cluster.md)** - Install HyperShift operator (with or without External DNS) 3. **[Create a Self-Managed Azure HostedCluster](create-self-managed-azure-cluster.md)** - Deploy your first hosted cluster 4. **[Deploy Azure Private Clusters](deploy-azure-private-clusters.md)** (Optional) - Configure private endpoint access with Azure Private Link diff --git a/docs/content/reference/aggregated-docs.md b/docs/content/reference/aggregated-docs.md index 8b06fcd99aff..9551bbd76a86 100644 --- a/docs/content/reference/aggregated-docs.md +++ b/docs/content/reference/aggregated-docs.md @@ -6129,6 +6129,205 @@ scale_down_nodepool After these steps, you will see how the (in the AWS case) instances will be terminated instantly, but Openshift will take some time until the nodes get deleted because of the default timeouts set on the platforms. +--- + +## Source: docs/content/how-to/automated-machine-management/spot-instances.md + +--- +title: Spot Instances +--- + +# Spot Instances + +AWS Spot instances use spare EC2 capacity at significantly reduced prices compared to on-demand instances, but may be interrupted with a 2-minute warning when EC2 needs the capacity back. HyperShift supports Spot instances for NodePools on AWS, with built-in graceful termination handling via SQS queues. + +!!! important + + Spot instances are suitable for fault-tolerant, stateless, and flexible workloads. They are **not recommended** for workloads that cannot tolerate interruptions. + +## Prerequisites + +Before creating a Spot instance NodePool, you must set up an SQS queue and EventBridge rules to receive EC2 interruption events. The AWS Node Termination Handler (NTH) deployed by HyperShift polls this queue and cordons/drains nodes before they are terminated, providing best-effort graceful shutdown. + +### 1. Create the SQS queue + +Create an SQS queue to receive Spot interruption notifications: + +```shell +export CLUSTER_NAME="my-cluster" +export AWS_REGION="us-east-1" + +aws sqs create-queue \ + --queue-name "${CLUSTER_NAME}-spot-interruption-queue" \ + --region "${AWS_REGION}" +``` + +Note the queue URL from the output — you will need it when creating the HostedCluster. + +### 2. Create EventBridge rules + +Create EventBridge rules to route EC2 Spot interruption warnings and rebalance recommendations to the SQS queue: + +```shell +QUEUE_ARN=$(aws sqs get-queue-attributes \ + --queue-url "https://sqs.${AWS_REGION}.amazonaws.com/$(aws sts get-caller-identity --query Account --output text)/${CLUSTER_NAME}-spot-interruption-queue" \ + --attribute-names QueueArn \ + --query 'Attributes.QueueArn' --output text) + +aws events put-rule \ + --name "${CLUSTER_NAME}-spot-interruption-warning" \ + --event-pattern '{"source":["aws.ec2"],"detail-type":["EC2 Spot Instance Interruption Warning"]}' \ + --region "${AWS_REGION}" + +aws events put-targets \ + --rule "${CLUSTER_NAME}-spot-interruption-warning" \ + --targets "Id=1,Arn=${QUEUE_ARN}" \ + --region "${AWS_REGION}" + +aws events put-rule \ + --name "${CLUSTER_NAME}-rebalance-recommendation" \ + --event-pattern '{"source":["aws.ec2"],"detail-type":["EC2 Instance Rebalance Recommendation"]}' \ + --region "${AWS_REGION}" + +aws events put-targets \ + --rule "${CLUSTER_NAME}-rebalance-recommendation" \ + --targets "Id=1,Arn=${QUEUE_ARN}" \ + --region "${AWS_REGION}" +``` + +### 3. Configure SQS queue policy + +Allow EventBridge to send messages to the queue: + +```shell +ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +QUEUE_URL="https://sqs.${AWS_REGION}.amazonaws.com/${ACCOUNT_ID}/${CLUSTER_NAME}-spot-interruption-queue" + +aws sqs set-queue-attributes \ + --queue-url "${QUEUE_URL}" \ + --attributes '{ + "Policy": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"events.amazonaws.com\"},\"Action\":\"sqs:SendMessage\",\"Resource\":\"'${QUEUE_ARN}'\"}]}" + }' +``` + +## Creating a Spot Instance NodePool + +### Step 1: Set the SQS queue URL on the HostedCluster + +The SQS queue URL is configured at the HostedCluster level in `spec.platform.aws.terminationHandlerQueueURL`. This enables the AWS Node Termination Handler component for all Spot NodePools in the cluster: + +```yaml +apiVersion: hypershift.openshift.io/v1beta1 +kind: HostedCluster +metadata: + name: my-cluster + namespace: clusters +spec: + platform: + type: AWS + aws: + region: us-east-1 + terminationHandlerQueueURL: "https://sqs.us-east-1.amazonaws.com/123456789012/my-cluster-spot-interruption-queue" + # ... other AWS configuration + # ... other spec fields +``` + +If you already have a running HostedCluster, you can patch it: + +```shell +oc patch hostedcluster my-cluster -n clusters --type merge -p '{ + "spec": { + "platform": { + "aws": { + "terminationHandlerQueueURL": "https://sqs.us-east-1.amazonaws.com/123456789012/my-cluster-spot-interruption-queue" + } + } + } +}' +``` + +### Step 2: Create a NodePool with Spot market type + +Create a NodePool with `spec.platform.aws.placement.marketType` set to `Spot`: + +```yaml +apiVersion: hypershift.openshift.io/v1beta1 +kind: NodePool +metadata: + name: spot-workers + namespace: clusters +spec: + clusterName: my-cluster + replicas: 3 + release: + image: quay.io/openshift-release-dev/ocp-release:4.18.0-x86_64 + management: + autoRepair: true + upgradeType: Replace + platform: + type: AWS + aws: + instanceType: m5.xlarge + instanceProfile: my-cluster-worker + rootVolume: + size: 120 + type: gp3 + placement: + marketType: Spot +``` + +### Setting a maximum price (optional) + +You can request Spot Instances at the Spot price, capped at the On-Demand price, or you can specify the maximum amount you're willing to pay: + +```yaml +spec: + platform: + aws: + placement: + marketType: Spot + spot: + maxPrice: "0.50" +``` + +The value is a decimal string representing the price per hour in USD. + +## Behavior + +When a NodePool is created with `marketType: Spot`, HyperShift labels all Machines and Nodes with `hypershift.openshift.io/interruptible-instance` and tags the EC2 instances with `aws-node-termination-handler/managed` so they can be identified by the termination handling components. + +### Graceful termination with SQS (recommended) + +When `terminationHandlerQueueURL` is set on the HostedCluster and at least one NodePool has `marketType: Spot`, HyperShift automatically deploys the Node Termination Handler (NTH) as a control plane component. The termination flow is: + +1. AWS sends a Spot interruption warning (2-minute notice) or rebalance recommendation to the SQS queue via EventBridge +2. NTH polls the queue and identifies the affected node +3. NTH cordons the node and drains it, respecting PodDisruptionBudgets +4. NTH taints the node (e.g., `aws-node-termination-handler/spot-itn`) +5. The spot remediation controller detects the taint, annotates the corresponding Machine with `hypershift.openshift.io/spot-interruption-signal`, and deletes it +6. The machine controller provisions a replacement Spot instance + +### Fallback without SQS + +Without the SQS queue, AWS terminates the instance abruptly with no graceful drain. In this case, the CAPI Machine enters a `Failed` state and the MachineHealthCheck triggers remediation to create a replacement. This path is slower and does not provide graceful pod shutdown. + +### MachineHealthCheck + +HyperShift creates a dedicated MachineHealthCheck (`-spot`) for each Spot NodePool. This MachineHealthCheck: + +- Targets only Machines with the `hypershift.openshift.io/interruptible-instance` label +- Sets `maxUnhealthy: 100%` because Spot reclamation can affect all instances simultaneously +- Uses an 8-minute unhealthy timeout (accounts for the 2-minute AWS notice plus shutdown time) +- Serves as a safety net in case the NTH + remediation controller path does not trigger + +### Constraints + +- Spot instances **cannot** be combined with Capacity Reservations +- Spot instances require **default** tenancy (dedicated tenancy is not supported) +- `spot` options (e.g., `maxPrice`) can only be specified when `marketType` is `Spot` +- Replacement instances are always Spot — if Spot capacity is unavailable, the replacement Machine will go `Failed` and the MachineHealthCheck will continue to retry remediation + + --- ## Source: docs/content/how-to/autoscaling.md @@ -9616,145 +9815,6 @@ spec: - Scale-from-zero (`autoScaling.min: 0`) is not supported on Azure. The minimum must be >= 1. ---- - -## Source: docs/content/how-to/azure/azure-workload-identity-setup.md - -# Azure Workload Identity Setup for Self-Managed Clusters - -!!! note "Developer Preview in OCP 4.21" - - Self-managed Azure HostedClusters are available as a Developer Preview feature in OpenShift Container Platform 4.21. - -This document describes how to set up Azure Workload Identities and OIDC issuer for self-managed Azure HostedClusters. - -!!! warning "Persistent Resource Groups" - - When setting up workload identities and OIDC issuer for the first time, create them in a **persistent resource group** that will not be deleted when individual clusters are destroyed. This allows you to reuse the same workload identities and OIDC issuer across multiple HostedClusters, reducing setup time and avoiding unnecessary resource recreation. - - - Use a persistent resource group like `os4-common`. - - This resource group should be separate from the cluster-specific resource groups that get created and deleted with each HostedCluster. - - The OIDC issuer storage account should also be created in this persistent resource group. - -## Prerequisites - -- Azure CLI (`az`) installed and configured -- `jq` command-line JSON processor -- Cloud Credential Operator (CCO) tool installed -- Appropriate Azure permissions - -## Create Azure Workload Identities - -!!! note "OIDC Issuer Required" - - Before running this command, you need an OIDC issuer URL. If you haven't set this up yet, see Configure OIDC Issuer below first. - -You can create the required managed identities and federated credentials using the HyperShift CLI: - -```bash -# Set environment variables -PERSISTENT_RG_NAME="os4-common" # Use persistent resource group -LOCATION="eastus" -CLUSTER_NAME="my-self-managed-cluster" -INFRA_ID="${CLUSTER_NAME}-$(openssl rand -hex 4)" -AZURE_CREDS="/path/to/azure-creds.json" - -# Create persistent resource group (if it doesn't exist) -az group create --name $PERSISTENT_RG_NAME --location $LOCATION - -# Create workload identities using the HyperShift CLI -# (requires OIDC issuer URL - see next section if you haven't set this up yet) -hypershift create iam azure \ - --name $CLUSTER_NAME \ - --infra-id $INFRA_ID \ - --azure-creds $AZURE_CREDS \ - --location $LOCATION \ - --resource-group-name $PERSISTENT_RG_NAME \ - --oidc-issuer-url $OIDC_ISSUER_URL \ - --output-file workload-identities.json -``` - -This creates 7 managed identities with federated credentials for: - -- Disk CSI driver -- File CSI driver -- Image Registry -- Ingress Operator -- Cloud Provider -- NodePool Management -- Network Operator - -To also create a KMS identity for Azure Key Vault etcd encryption at rest, add the `--enable-kms` flag: - -```bash -hypershift create iam azure \ - --name $CLUSTER_NAME \ - --infra-id $INFRA_ID \ - --azure-creds $AZURE_CREDS \ - --location $LOCATION \ - --resource-group-name $PERSISTENT_RG_NAME \ - --oidc-issuer-url $OIDC_ISSUER_URL \ - --output-file workload-identities.json \ - --enable-kms -``` - -!!! warning "KMS Key Vault Role Assignment" - - If you use `--enable-kms`, you must **manually** assign the `Key Vault Crypto User` role to the KMS identity on your Key Vault. The `--auto-assign-roles` flag does not cover this because the Key Vault scope is user-provided. See Enabling KMS Encryption for the role assignment commands. - -For complete documentation on the IAM commands, see Create Azure IAM Resources Separately. - -## Configure OIDC Issuer - -Use the Cloud Credential Operator (CCO) tool to create the OIDC issuer: - -```bash -# Set OIDC issuer variables (reusing variables from previous steps) -OIDC_STORAGE_ACCOUNT_NAME="yourstorageaccount" -TENANT_ID="your-tenant-id" -# SUBSCRIPTION_ID and PERSISTENT_RG_NAME already set from previous section -# Create an RSA key pair and save the private and public key -ccoctl azure create-key-pair - -SA_TOKEN_ISSUER_PRIVATE_KEY_PATH="/path/to/serviceaccount-signer.private" -SA_TOKEN_ISSUER_PUBLIC_KEY_PATH="/path/to/serviceaccount-signer.public" - -# Create OIDC issuer using CCO tool in os4-common resource group -ccoctl azure create-oidc-issuer \ - --oidc-resource-group-name ${PERSISTENT_RG_NAME} \ - --tenant-id ${TENANT_ID} \ - --region ${LOCATION} \ - --name ${OIDC_STORAGE_ACCOUNT_NAME} \ - --subscription-id ${SUBSCRIPTION_ID} \ - --public-key-file ${SA_TOKEN_ISSUER_PUBLIC_KEY_PATH} - -# Set OIDC issuer URL -OIDC_ISSUER_URL="https://${OIDC_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${OIDC_STORAGE_ACCOUNT_NAME}" -``` - -## Verification - -Verify the setup: - -```bash -# List created managed identities -az identity list --resource-group $PERSISTENT_RG_NAME --output table - -# Verify federated credentials for one identity -az identity federated-credential list \ - --identity-name "${AZURE_DISK_MI_NAME}" \ - --resource-group $PERSISTENT_RG_NAME - -# Test OIDC issuer accessibility -curl -s "${OIDC_ISSUER_URL}/.well-known/openid-configuration" | jq . -``` - -## Next Steps - -After setting up workload identities, you can proceed to: - -- Setup Azure Management Cluster for HyperShift - --- ## Source: docs/content/how-to/azure/create-azure-cluster-on-aks.md @@ -10327,6 +10387,10 @@ title: Create Azure IAM resources separately # Create Azure IAM resources separately +!!! note "Developer Preview in OCP 4.21" + + Self-managed Azure HostedClusters are available as a Developer Preview feature in OpenShift Container Platform 4.21. + The `hypershift create iam azure` command creates Azure workload identities separately from infrastructure, following the same pattern as AWS and GCP. This enables you to manage IAM resources independently from your cluster infrastructure lifecycle. @@ -10334,15 +10398,30 @@ your cluster infrastructure lifecycle. ## Overview For self-managed Azure HyperShift clusters, workload identities authenticate cluster components to Azure -services using OIDC federation. You must create identities separately using `create iam azure` and then +services using OIDC federation. The setup consists of two steps: + +1. **Configure an OIDC Issuer** using the Cloud Credential Operator (CCO) tool +2. **Create workload identities** using `hypershift create iam azure` + +You must create identities separately using `create iam azure` and then consume them during infrastructure or cluster creation via the `--workload-identities-file` flag. -This approach provides control over the IAM lifecycle and follows the same pattern as AWS and GCP platforms. +!!! warning "Persistent Resource Groups" + + When setting up workload identities and OIDC issuer for the first time, create them in a **persistent resource group** that will not be deleted when individual clusters are destroyed. This allows you to reuse the same workload identities and OIDC issuer across multiple HostedClusters, reducing setup time and avoiding unnecessary resource recreation. + + - Use a persistent resource group like `os4-common`. + - This resource group should be separate from the cluster-specific resource groups that get created and deleted with each HostedCluster. + - The OIDC issuer storage account should also be created in this persistent resource group. ## Prerequisites Before creating Azure IAM resources, ensure you have: +- Azure CLI (`az`) installed and configured +- Cloud Credential Operator (CCO) tool (`ccoctl`) installed +- `jq` command-line JSON processor +- Appropriate Azure permissions - An Azure credentials file with the following format: ```json { @@ -10353,20 +10432,57 @@ Before creating Azure IAM resources, ensure you have: } ``` - An existing resource group where the managed identities will be created -- An OIDC issuer URL for workload identity federation + +## Configure OIDC Issuer + +Before creating workload identities, you need an OIDC issuer URL. Use the Cloud Credential Operator (CCO) tool to create one: + +```bash +# Set OIDC issuer variables +PERSISTENT_RG_NAME="os4-common" # Use persistent resource group +LOCATION="eastus" +OIDC_STORAGE_ACCOUNT_NAME="yourstorageaccount" +TENANT_ID="your-tenant-id" +SUBSCRIPTION_ID="your-subscription-id" + +# Create persistent resource group (if it doesn't exist) +az group create --name $PERSISTENT_RG_NAME --location $LOCATION + +# Create an RSA key pair and save the private and public key +ccoctl azure create-key-pair + +SA_TOKEN_ISSUER_PRIVATE_KEY_PATH="/path/to/serviceaccount-signer.private" +SA_TOKEN_ISSUER_PUBLIC_KEY_PATH="/path/to/serviceaccount-signer.public" + +# Create OIDC issuer using CCO tool in persistent resource group +ccoctl azure create-oidc-issuer \ + --oidc-resource-group-name ${PERSISTENT_RG_NAME} \ + --tenant-id ${TENANT_ID} \ + --region ${LOCATION} \ + --name ${OIDC_STORAGE_ACCOUNT_NAME} \ + --subscription-id ${SUBSCRIPTION_ID} \ + --public-key-file ${SA_TOKEN_ISSUER_PUBLIC_KEY_PATH} + +# Set OIDC issuer URL +OIDC_ISSUER_URL="https://${OIDC_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${OIDC_STORAGE_ACCOUNT_NAME}" +``` ## Creating Workload Identities Use the `hypershift create iam azure` command: ```bash +CLUSTER_NAME="my-self-managed-cluster" +INFRA_ID="${CLUSTER_NAME}-$(openssl rand -hex 4)" +AZURE_CREDS="/path/to/azure-creds.json" + hypershift create iam azure \ - --name CLUSTER_NAME \ - --infra-id INFRA_ID \ - --azure-creds AZURE_CREDENTIALS_FILE \ - --location LOCATION \ - --resource-group-name RESOURCE_GROUP \ - --oidc-issuer-url OIDC_ISSUER_URL \ + --name $CLUSTER_NAME \ + --infra-id $INFRA_ID \ + --azure-creds $AZURE_CREDS \ + --location $LOCATION \ + --resource-group-name $PERSISTENT_RG_NAME \ + --oidc-issuer-url $OIDC_ISSUER_URL \ --output-file workload-identities.json ``` @@ -10375,11 +10491,11 @@ where: * `CLUSTER_NAME` is the name of the hosted cluster you intend to create. * `INFRA_ID` is a unique identifier used to name Azure resources. Typically this is the cluster name with a random suffix appended. -* `AZURE_CREDENTIALS_FILE` points to an Azure credentials file with permission to create +* `AZURE_CREDS` points to an Azure credentials file with permission to create managed identities and federated credentials. * `LOCATION` is the Azure region for the managed identities (e.g., `eastus`, `westus2`). -* `RESOURCE_GROUP` is the name of an existing resource group where identities will be created. -* `OIDC_ISSUER_URL` is the URL of the OIDC identity provider used for workload identity federation. +* `PERSISTENT_RG_NAME` is the name of an existing resource group where identities will be created. +* `OIDC_ISSUER_URL` is the URL of the OIDC identity provider created in the previous step. Running this command creates: @@ -10394,6 +10510,26 @@ Running this command creates: - Control Plane Operator * Federated Identity Credentials for each identity, configured with the OIDC issuer +### Enabling KMS Identity + +To also create a KMS identity for Azure Key Vault etcd encryption at rest, add the `--enable-kms` flag: + +```bash +hypershift create iam azure \ + --name $CLUSTER_NAME \ + --infra-id $INFRA_ID \ + --azure-creds $AZURE_CREDS \ + --location $LOCATION \ + --resource-group-name $PERSISTENT_RG_NAME \ + --oidc-issuer-url $OIDC_ISSUER_URL \ + --output-file workload-identities.json \ + --enable-kms +``` + +!!! warning "KMS Key Vault Role Assignment" + + If you use `--enable-kms`, you must **manually** assign the `Key Vault Crypto User` role to the KMS identity on your Key Vault. The `--auto-assign-roles` flag does not cover this because the Key Vault scope is user-provided. See Enabling KMS Encryption for the role assignment commands. + ## Private Endpoint Access The **Control Plane Operator** identity is always created by `create iam azure`. For private @@ -10473,6 +10609,23 @@ hypershift create cluster azure \ --workload-identities-file workload-identities.json ``` +## Verification + +Verify the setup: + +```bash +# List created managed identities +az identity list --resource-group $PERSISTENT_RG_NAME --output table + +# Verify federated credentials for one identity +az identity federated-credential list \ + --identity-name "${CLUSTER_NAME}-disk-${INFRA_ID}" \ + --resource-group $PERSISTENT_RG_NAME + +# Test OIDC issuer accessibility +curl -s "${OIDC_ISSUER_URL}/.well-known/openid-configuration" | jq . +``` + ## Destroying Workload Identities To destroy the workload identities that were created: @@ -10504,16 +10657,17 @@ Both the managed identities and their federated credentials are removed. | `--name` | Name of the HostedCluster | | `--infra-id` | Unique infrastructure identifier | | `--azure-creds` | Path to Azure credentials JSON file | -| `--location` | Azure region for identities | +| `--resource-group-name` | Resource group for identities | | `--oidc-issuer-url` | OIDC issuer URL for federation | +| `--output-file` | Output file path | ### Optional Flags for `create iam azure` | Flag | Description | Default | |------|-------------|---------| -| `--resource-group-name` | Resource group for identities | `{name}-{infra-id}` | -| `--output-file` | Output file path | `{name}-iam-output.json` | +| `--location` | Azure region for identities | `eastus` | | `--cloud` | Azure cloud environment | `AzurePublicCloud` | +| `--enable-kms` | Create KMS identity for etcd encryption | `false` | ### Required Flags for `destroy iam azure` @@ -10543,22 +10697,37 @@ export INFRA_ID="${NAME}-$(openssl rand -hex 4)" export LOCATION="eastus" export BASE_DOMAIN="example.com" export AZURE_CREDS="/path/to/azure-creds.json" -export OIDC_ISSUER_URL="https://my-oidc-issuer.com" +export PERSISTENT_RG_NAME="os4-common" +export TENANT_ID="your-tenant-id" +export SUBSCRIPTION_ID="your-subscription-id" +export OIDC_STORAGE_ACCOUNT_NAME="yourstorageaccount" +export RELEASE_IMAGE="quay.io/openshift-release-dev/ocp-release:XYZ" + +# 2. Create persistent resource group (if it doesn't exist) +az group create --name ${PERSISTENT_RG_NAME} --location ${LOCATION} -# 2. Create a resource group for identities -az group create --name ${NAME}-rg --location ${LOCATION} +# 3. Create OIDC issuer +ccoctl azure create-key-pair +ccoctl azure create-oidc-issuer \ + --oidc-resource-group-name ${PERSISTENT_RG_NAME} \ + --tenant-id ${TENANT_ID} \ + --region ${LOCATION} \ + --name ${OIDC_STORAGE_ACCOUNT_NAME} \ + --subscription-id ${SUBSCRIPTION_ID} \ + --public-key-file /path/to/serviceaccount-signer.public +export OIDC_ISSUER_URL="https://${OIDC_STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${OIDC_STORAGE_ACCOUNT_NAME}" -# 3. Create workload identities +# 4. Create workload identities hypershift create iam azure \ --name ${NAME} \ --infra-id ${INFRA_ID} \ --azure-creds ${AZURE_CREDS} \ --location ${LOCATION} \ - --resource-group-name ${NAME}-rg \ + --resource-group-name ${PERSISTENT_RG_NAME} \ --oidc-issuer-url ${OIDC_ISSUER_URL} \ --output-file workload-identities.json -# 4. Create infrastructure using pre-created identities +# 5. Create infrastructure using pre-created identities hypershift create infra azure \ --name ${NAME} \ --infra-id ${INFRA_ID} \ @@ -10566,9 +10735,11 @@ hypershift create infra azure \ --base-domain ${BASE_DOMAIN} \ --location ${LOCATION} \ --workload-identities-file workload-identities.json \ + --assign-identity-roles \ + --dns-zone-rg-name ${PERSISTENT_RG_NAME} \ --output-file infra-output.yaml -# 5. Create the cluster +# 6. Create the cluster hypershift create cluster azure \ --name ${NAME} \ --infra-id ${INFRA_ID} \ @@ -10576,36 +10747,42 @@ hypershift create cluster azure \ --base-domain ${BASE_DOMAIN} \ --location ${LOCATION} \ --pull-secret /path/to/pull-secret \ - --infra-json infra-output.yaml + --generate-ssh \ + --release-image ${RELEASE_IMAGE} \ + --sa-token-issuer-private-key-path /path/to/serviceaccount-signer.private \ + --oidc-issuer-url ${OIDC_ISSUER_URL} \ + --dns-zone-rg-name ${PERSISTENT_RG_NAME} \ + --assign-service-principal-roles \ + --infra-json infra-output.yaml \ + --diagnostics-storage-account-type Managed # --- Cleanup --- -# 6. Destroy the cluster +# 7. Destroy the cluster hypershift destroy cluster azure \ --name ${NAME} \ --azure-creds ${AZURE_CREDS} \ - --dns-zone-rg-name ${DNS_ZONE_RG} + --dns-zone-rg-name ${PERSISTENT_RG_NAME} -# 7. Destroy infrastructure +# 8. Destroy infrastructure hypershift destroy infra azure \ --name ${NAME} \ --infra-id ${INFRA_ID} \ --azure-creds ${AZURE_CREDS} -# 8. Destroy IAM resources +# 9. Destroy IAM resources (only if no longer needed) hypershift destroy iam azure \ --azure-creds ${AZURE_CREDS} \ --workload-identities-file workload-identities.json \ - --resource-group-name ${RESOURCE_GROUP} \ + --resource-group-name ${PERSISTENT_RG_NAME} \ --name ${NAME} \ --infra-id ${INFRA_ID} \ - --dns-zone-rg-name ${DNS_ZONE_RG} + --dns-zone-rg-name ${PERSISTENT_RG_NAME} ``` ## See Also - Create Azure Infrastructure Separately -- Azure Workload Identity Setup - Self-Managed Azure Overview - Deploy Azure Private Clusters — End-to-end guide for private endpoint access @@ -10880,7 +11057,7 @@ Before creating a self-managed Azure HostedCluster, ensure you have: This guide assumes you have already completed the workload identity configuration and management cluster setup. Follow these guides in order: - 1. Azure Workload Identity Setup - Workload identities and OIDC issuer configuration + 1. Create Azure IAM Resources - Workload identities and OIDC issuer configuration 2. Setup Azure Management Cluster for HyperShift - HyperShift operator installation (with or without External DNS) ### Permission Requirements @@ -10895,124 +11072,71 @@ Your Azure service principal must have the following permissions: ## Creating the Self-Managed Azure HostedCluster -!!! tip "Alternative: Use `create infra azure` and `create iam azure`" - - This guide creates Azure infrastructure manually with `az` CLI commands for - transparency. Alternatively, you can use the HyperShift CLI to automate - infrastructure and IAM creation: - - - Create Azure IAM Resources Separately — `hypershift create iam azure` - - Create Azure Infrastructure Separately — `hypershift create infra azure` - - The private cluster guide (Deploy Azure Private Clusters) - uses these automated commands and is the recommended approach for private topology. - ### Infrastructure Setup -Before creating the HostedCluster, set up the necessary Azure infrastructure: - -!!! note "About PERSISTENT_RG_NAME" - In Red Hat environments, a periodic Azure resource "reaper" deletes resources that are not properly tagged or not located in an approved resource group. We frequently use the `os4-common` resource group for shared, long-lived assets (for example, public DNS zones) to avoid accidental cleanup. If you are not in Red Hat infrastructure, set `PERSISTENT_RG_NAME` to any long-lived resource group in your subscription that will not be automatically reaped, or ensure your organization's required tags/policies are applied. The name does not have to be `os4-common`—use whatever persistent resource group fits your environment. +Create the Azure infrastructure using the HyperShift CLI: ```bash # Set cluster configuration variables -PREFIX="your-prefix-sm" -RELEASE_IMAGE="quay.io/openshift-release-dev/ocp-release:XYZ" -TAG="latest" - +CLUSTER_NAME="my-self-managed-cluster" +INFRA_ID="${CLUSTER_NAME}-$(openssl rand -hex 4)" LOCATION="eastus" -MANAGED_RG_NAME="${PREFIX}-managed-rg" -VNET_RG_NAME="${PREFIX}-customer-vnet-rg" -NSG_RG_NAME="${PREFIX}-customer-nsg-rg" -VNET_NAME="${PREFIX}-customer-vnet" -VNET_SUBNET1="${PREFIX}-customer-subnet-1" -NSG="${PREFIX}-customer-nsg" -DNS_ZONE_NAME="your-subdomain.your-parent.dns.zone.com" -CLUSTER_NAMESPACE="clusters" -CLUSTER_NAME="${PREFIX}-hc" -AZURE_CREDS="/path/to/azure/credentials" +BASE_DOMAIN="example.com" +AZURE_CREDS="/path/to/azure-creds.json" PULL_SECRET="/path/to/pull-secret.json" -HYPERSHIFT_BINARY_PATH="/path/to/hypershift/bin" +RELEASE_IMAGE="quay.io/openshift-release-dev/ocp-release:XYZ" +PERSISTENT_RG_NAME="os4-common" OIDC_ISSUER_URL="https://yourstorageaccount.blob.core.windows.net/yourstorageaccount" SA_TOKEN_ISSUER_PRIVATE_KEY_PATH="/path/to/serviceaccount-signer.private" -PERSISTENT_RG_NAME="os4-common" -PARENT_DNS_ZONE="your-parent.dns.zone.com" - -# Clean up any previous instances (optional) -az group delete -n "${VNET_RG_NAME}" --yes --no-wait || true -az group delete -n "${NSG_RG_NAME}" --yes --no-wait || true - -# Create managed resource group -az group create --name "${MANAGED_RG_NAME}" --location ${LOCATION} - -# Create VNET & NSG resource groups -az group create --name "${VNET_RG_NAME}" --location ${LOCATION} -az group create --name "${NSG_RG_NAME}" --location ${LOCATION} - -# Create network security group -az network nsg create \ - --resource-group "${NSG_RG_NAME}" \ - --name "${NSG}" - -# Get NSG ID -GetNsgID=$(az network nsg list --query "[?name=='${NSG}'].id" -o tsv) - -# Create VNet with subnet -az network vnet create \ - --name "${VNET_NAME}" \ - --resource-group "${VNET_RG_NAME}" \ - --address-prefix 10.0.0.0/16 \ - --subnet-name "${VNET_SUBNET1}" \ - --subnet-prefixes 10.0.0.0/24 \ - --nsg "${GetNsgID}" -# Get VNet and Subnet IDs -GetVnetID=$(az network vnet list --query "[?name=='${VNET_NAME}'].id" -o tsv) -GetSubnetID=$(az network vnet subnet show \ - --vnet-name "${VNET_NAME}" \ - --name "${VNET_SUBNET1}" \ - --resource-group "${VNET_RG_NAME}" \ - --query id --output tsv) +# Create infrastructure +hypershift create infra azure \ + --name ${CLUSTER_NAME} \ + --infra-id ${INFRA_ID} \ + --azure-creds ${AZURE_CREDS} \ + --base-domain ${BASE_DOMAIN} \ + --location ${LOCATION} \ + --workload-identities-file workload-identities.json \ + --assign-identity-roles \ + --dns-zone-rg-name ${PERSISTENT_RG_NAME} \ + --output-file infra-output.yaml ``` -### Create the HostedCluster - -!!! note "Federated Identity Prerequisites" +This creates the resource groups, VNet, subnet, NSG, Private DNS zone, and load balancer for your cluster. For advanced options like using existing network resources, see Create Azure Infrastructure Separately. - Before creating the cluster, ensure that all federated identity credentials have been set up for your workload identities as described in the Azure Workload Identity Setup guide. The cluster creation will fail if these are not properly configured. +### Create the HostedCluster !!! note "Azure Marketplace Images" For OpenShift 4.20 and later, HyperShift automatically selects the appropriate Azure Marketplace image from the release payload. You no longer need to specify `--marketplace-*` flags unless you want to use a specific custom image. See Configuring Azure Marketplace Images for more details. -Create the HostedCluster: +Create the HostedCluster using the infrastructure output: ```bash -# Create the HostedCluster -${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ +hypershift create cluster azure \ --name "$CLUSTER_NAME" \ - --namespace "$CLUSTER_NAMESPACE" \ + --infra-id "$INFRA_ID" \ --azure-creds $AZURE_CREDS \ --location ${LOCATION} \ --node-pool-replicas 2 \ - --base-domain $PARENT_DNS_ZONE \ + --base-domain $BASE_DOMAIN \ --pull-secret $PULL_SECRET \ --generate-ssh \ --release-image ${RELEASE_IMAGE} \ - --external-dns-domain ${DNS_ZONE_NAME} \ - --resource-group-name "${MANAGED_RG_NAME}" \ - --vnet-id "${GetVnetID}" \ - --subnet-id "${GetSubnetID}" \ - --network-security-group-id "${GetNsgID}" \ --sa-token-issuer-private-key-path "${SA_TOKEN_ISSUER_PRIVATE_KEY_PATH}" \ --oidc-issuer-url "${OIDC_ISSUER_URL}" \ - --control-plane-operator-image="quay.io/hypershift/hypershift:${TAG}" \ --dns-zone-rg-name ${PERSISTENT_RG_NAME} \ --assign-service-principal-roles \ - --workload-identities-file ./workload-identities.json \ + --infra-json infra-output.yaml \ --diagnostics-storage-account-type Managed ``` +!!! tip "External DNS" + + If using External DNS for automatic DNS management, also pass + `--external-dns-domain ` to the cluster creation command. + See Setup Azure Management Cluster for DNS configuration. + !!! tip "Private Clusters" To create a private cluster with Azure Private Link, see @@ -11021,15 +11145,6 @@ ${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ cluster's VNet, `--endpoint-access Private` flag, and HyperShift operator installation with `--private-platform Azure`. -!!! warning "Private Clusters: Avoid DNS Zone Shadowing" - - If creating a **private** Azure HostedCluster, ensure `--external-dns-domain` does - not match `{clusterName}.{baseDomain}` or its parent domain. A matching value - causes an Azure Private DNS zone to shadow `*.apps` resolution, breaking console - and all ingress. This cannot be fixed after creation. See - External DNS Domain Must Not Match Cluster Domain - for details. - ### Configuring Azure Marketplace Images HyperShift supports multiple approaches for configuring Azure Marketplace images for your cluster nodes. The recommended approach varies based on your OpenShift version and requirements. @@ -11043,7 +11158,7 @@ For OpenShift 4.20+, HyperShift automatically selects the appropriate Azure Mark ```bash # No marketplace flags needed - HyperShift will auto-select the image # Gen2 VM generation is used by default -${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ +hypershift create cluster azure \ --name "$CLUSTER_NAME" \ # ... other flags ... ``` @@ -11055,7 +11170,7 @@ This is the **recommended approach** as it ensures your nodes use the officially If you need to use a specific VM generation (Gen1 or Gen2), you can specify only the `--image-generation` flag: ```bash -${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ +hypershift create cluster azure \ --name "$CLUSTER_NAME" \ --image-generation Gen2 \ # Or Gen1 (case-sensitive) # ... other flags ... @@ -11072,7 +11187,7 @@ ${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ If you need to use a specific custom marketplace image, provide all marketplace details: ```bash -${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ +hypershift create cluster azure \ --name "$CLUSTER_NAME" \ --marketplace-publisher azureopenshift \ --marketplace-offer aro4 \ @@ -11096,18 +11211,18 @@ When creating additional NodePools, you can specify image configuration in the s ```bash # Use default from release payload (OCP 4.20+) -${HYPERSHIFT_BINARY_PATH}/hypershift create nodepool azure \ +hypershift create nodepool azure \ --cluster-name "$CLUSTER_NAME" \ # ... other flags ... # Or specify generation -${HYPERSHIFT_BINARY_PATH}/hypershift create nodepool azure \ +hypershift create nodepool azure \ --cluster-name "$CLUSTER_NAME" \ --image-generation Gen1 \ # ... other flags ... # Or use custom marketplace image -${HYPERSHIFT_BINARY_PATH}/hypershift create nodepool azure \ +hypershift create nodepool azure \ --cluster-name "$CLUSTER_NAME" \ --marketplace-publisher azureopenshift \ --marketplace-offer aro4 \ @@ -11118,16 +11233,14 @@ ${HYPERSHIFT_BINARY_PATH}/hypershift create nodepool azure \ !!! important "Key Configuration Options" - - `--workload-identities-file`: References the workload identities configuration created in the setup guide + - `--infra-json`: Path to infrastructure output from `hypershift create infra azure` (includes workload identities) - `--assign-service-principal-roles`: Automatically assigns required Azure roles to workload identities - `--sa-token-issuer-private-key-path`: Path to the private key for service account token signing - - `--oidc-issuer-url`: URL of the OIDC issuer created in the workload identity setup - - `--vnet-id`, `--subnet-id`, `--network-security-group-id`: Custom networking infrastructure + - `--oidc-issuer-url`: URL of the OIDC issuer created in the IAM setup - `--image-generation`: (Optional) VM generation (`Gen1` or `Gen2`, defaults to `Gen2`). For OCP 4.20+, omit to use release payload defaults. See Configuring Azure Marketplace Images - `--marketplace-publisher/offer/sku/version`: (Optional) Explicit Azure Marketplace image. Must specify all four flags together, or omit all to use defaults (OCP 4.20+) - `--dns-zone-rg-name`: Resource group containing the DNS zone (os4-common) - `--diagnostics-storage-account-type Managed`: Use Azure managed storage for diagnostics - - `--control-plane-operator-image`: Custom HyperShift operator image (optional) ## Enabling KMS Encryption (etcd Encryption at Rest) @@ -11138,19 +11251,7 @@ Self-managed Azure HostedClusters support encrypting etcd data at rest using Azu ### Prerequisites -Ensure the `kms` workload identity is included in your `workload-identities.json` file. When using `hypershift create iam azure`, pass the `--enable-kms` flag to create the KMS identity (using the `INFRA_ID` set during Azure Workload Identity Setup): - -```bash -hypershift create iam azure \ - --name "$CLUSTER_NAME" \ - --infra-id "$INFRA_ID" \ - --azure-creds "$AZURE_CREDS" \ - --location "$LOCATION" \ - --resource-group-name "$PERSISTENT_RG_NAME" \ - --oidc-issuer-url "$OIDC_ISSUER_URL" \ - --output-file ./workload-identities.json \ - --enable-kms -``` +Ensure the `kms` workload identity is included in your `workload-identities.json` file. When using `hypershift create iam azure`, pass the `--enable-kms` flag to create the KMS identity. See Enabling KMS Identity for details. ### Create a Key Vault and Key @@ -11170,7 +11271,8 @@ hypershift create iam azure \ ```bash # Create Key Vault -KV_NAME="${PREFIX}-kv" +KV_NAME="${CLUSTER_NAME}-kv" +MANAGED_RG_NAME="${CLUSTER_NAME}-managed-rg" az keyvault create \ --name "${KV_NAME}" \ --resource-group "${MANAGED_RG_NAME}" \ @@ -11178,7 +11280,7 @@ az keyvault create \ --enable-rbac-authorization # Create encryption key -KEY_NAME="${PREFIX}-etcd-key" +KEY_NAME="${CLUSTER_NAME}-etcd-key" az keyvault key create \ --vault-name "${KV_NAME}" \ --name "${KEY_NAME}" \ @@ -11229,26 +11331,21 @@ az role assignment create \ Add the `--encryption-key-id` flag to your cluster creation command: ```bash -${HYPERSHIFT_BINARY_PATH}/hypershift create cluster azure \ +hypershift create cluster azure \ --name "$CLUSTER_NAME" \ - --namespace "$CLUSTER_NAMESPACE" \ + --infra-id "$INFRA_ID" \ --azure-creds $AZURE_CREDS \ --location ${LOCATION} \ --node-pool-replicas 2 \ - --base-domain $PARENT_DNS_ZONE \ + --base-domain $BASE_DOMAIN \ --pull-secret $PULL_SECRET \ --generate-ssh \ --release-image ${RELEASE_IMAGE} \ - --external-dns-domain ${DNS_ZONE_NAME} \ - --resource-group-name "${MANAGED_RG_NAME}" \ - --vnet-id "${GetVnetID}" \ - --subnet-id "${GetSubnetID}" \ - --network-security-group-id "${GetNsgID}" \ --sa-token-issuer-private-key-path "${SA_TOKEN_ISSUER_PRIVATE_KEY_PATH}" \ --oidc-issuer-url "${OIDC_ISSUER_URL}" \ --dns-zone-rg-name ${PERSISTENT_RG_NAME} \ --assign-service-principal-roles \ - --workload-identities-file ./workload-identities.json \ + --infra-json infra-output.yaml \ --encryption-key-id "${ENCRYPTION_KEY_ID}" \ --diagnostics-storage-account-type Managed ``` @@ -11277,25 +11374,32 @@ oc get clusterversion ## Cleanup -To delete the HostedCluster: +To delete the HostedCluster and its infrastructure: ```bash # Delete the HostedCluster hypershift destroy cluster azure \ --name $CLUSTER_NAME \ --azure-creds $AZURE_CREDS \ - --resource-group-name $MANAGED_RG_NAME \ --dns-zone-rg-name $PERSISTENT_RG_NAME + +# Destroy infrastructure +hypershift destroy infra azure \ + --name $CLUSTER_NAME \ + --infra-id $INFRA_ID \ + --azure-creds $AZURE_CREDS ``` !!! note "Resource Cleanup" - The HyperShift destroy command will clean up the cluster resources. Workload identities and OIDC issuer created during setup can be reused for other clusters or cleaned up separately if no longer needed. + The HyperShift destroy commands clean up the cluster and infrastructure resources. Workload identities and OIDC issuer created during setup can be reused for other clusters or cleaned up separately if no longer needed. See Destroying Workload Identities. ## Related Documentation -1. Azure Workload Identity Setup - Workload identities and OIDC issuer setup -2. Setup Azure Management Cluster for HyperShift - DNS and HyperShift operator setup +1. Create Azure IAM Resources - Workload identities and OIDC issuer setup +2. Create Azure Infrastructure Separately - Advanced infrastructure options +3. Setup Azure Management Cluster for HyperShift - DNS and HyperShift operator setup + --- @@ -11771,74 +11875,6 @@ The deletion process automatically cleans up Private Link resources in the corre 1. `.hypershift.local` — synthetic internal zone with `api` and `*.apps` records 2. `` — base domain zone with `api-` and `oauth-` records -### External DNS Domain Must Not Match Cluster Domain - -!!! warning "Azure Private DNS Zone Shadowing" - - On private Azure HostedClusters, do **not** set `--external-dns-domain` to a value - that matches or is a parent domain of `{clusterName}.{baseDomain}`. For example, - if your cluster is named `my-cluster` with base domain `example.com`, do not use - `--external-dns-domain my-cluster.example.com` or `--external-dns-domain example.com`. - - This misconfiguration **cannot be corrected after cluster creation** because the - relevant fields (`spec.services`, `spec.dns.baseDomain`, and `metadata.name`) are - all immutable. The cluster must be destroyed and recreated with a different - `--external-dns-domain` value. - - **Safe example**: If your cluster is `my-cluster` with base domain `example.com`, - use a separate subdomain such as `--external-dns-domain custom-dns.example.com` - that does not overlap with `my-cluster.example.com`. - -#### What Goes Wrong - -Private Azure clusters use two separate routing paths: - -1. **Management-plane router** (`private-router`): An HAProxy pod in the hosted - control plane namespace, fronted by an internal load balancer and exposed to the - guest VNet through Azure Private Link. Worker nodes reach this router via the - Private Endpoint IP. HAProxy uses SNI-based routing and only has ACLs for - `.hypershift.local` hostnames (KAS, ignition, konnectivity, OAuth). Any hostname - that does not match an ACL falls through to the `default_backend kube_api`, which - returns KAS certificates. - -2. **Data-plane router** (`router-default`): The OpenShift ingress controller running - on worker nodes, serving `*.apps.{clusterName}.{baseDomain}` hostnames with the - correct wildcard ingress certificate. - -When `--external-dns-domain` matches the cluster domain, the PLS controller creates a -Private DNS zone named `{clusterName}.{baseDomain}`. This zone becomes authoritative -for **all** queries under that name within the guest VNet, including -`*.apps.{clusterName}.{baseDomain}`. Since the zone only has `api` and `oauth` A -records pointing to the Private Endpoint IP, apps queries either: - -- Return **NXDOMAIN** (if no `*.apps` record exists in the zone), or -- Resolve to the **Private Endpoint IP**, which routes to `private-router` (HAProxy). - Because `*.apps` hostnames do not match any HAProxy SNI ACL, traffic falls through - to `kube_api` and the client receives a **TLS certificate mismatch** (KAS cert - instead of the ingress wildcard cert). - -Neither outcome is usable. The console, OAuth login, and all application routes are -unreachable. - -#### Why the Controller Cannot Self-Heal - -The controller cannot fix this by adding a `*.apps` wildcard record to the shadowing -zone because: - -- The Private Endpoint IP routes to the management-plane `private-router`, not the - data-plane `router-default`. Adding `*.apps → PE IP` would route apps traffic to - HAProxy, which does not serve those hostnames. -- The correct target (the data-plane ingress IP on worker nodes) is not available to - the PLS controller. The controller runs in the control plane and has no client to - the guest cluster. There is no HCP status field that reports the guest ingress IP, - and the HostedCluster Controller Operator (HCCO) does not propagate it back. - -When the controller detects shadowing, it sets `AzurePrivateDNSAvailable=False` with -reason `BaseDomainShadowsClusterDomain` and skips zone creation entirely. This -prevents the shadowing zone from being created, but the `api` and `oauth` hostnames -from `--external-dns-domain` will not resolve via Private DNS. The cluster must be -recreated with a non-overlapping domain. - ### Condition Debugging If the cluster gets stuck, check the `AzurePrivateLinkService` CR conditions: @@ -11852,7 +11888,7 @@ oc get azureprivatelinkservices -n clusters-${CLUSTER_NAME} -o jsonpath='{.items | `AzureInternalLoadBalancerAvailable` = False | The `private-router` Service hasn't received an ILB IP yet. Check the Service status and Azure networking. | | `AzurePLSCreated` = False | PLS creation failed. Check NAT subnet policies, credentials, and the HO operator logs. | | `AzurePrivateEndpointAvailable` = False | PE creation failed or connection not approved. Check the PLS auto-approval list and CPO logs. | -| `AzurePrivateDNSAvailable` = False | DNS zone or record creation failed. If the reason is `BaseDomainShadowsClusterDomain`, the `--external-dns-domain` value overlaps with the cluster domain — the cluster must be recreated with a different value. See External DNS Domain Must Not Match Cluster Domain. | +| `AzurePrivateDNSAvailable` = False | DNS zone or record creation failed. Check CPO identity permissions in the guest subscription. | ## Related Documentation @@ -12241,10 +12277,9 @@ Self-managed Azure uses an OpenShift cluster (running on any platform - AWS, Azu **Guides:** - Self-Managed Azure Overview - Architecture and deployment workflow -- Azure Workload Identity Setup - Set up managed identities and OIDC federation +- Create Azure IAM Resources - Set up OIDC issuer, managed identities, and workload identity federation - Setup Azure Management Cluster - Install HyperShift operator - Create a Self-Managed Azure HostedCluster - Deploy your first hosted cluster -- Create Azure IAM Resources Separately - Manage workload identities independently - Create Azure Infrastructure Separately - Create infrastructure before cluster ## Comparison @@ -12525,10 +12560,7 @@ You can create workload identities using either: **When to Complete**: This is a one-time setup that can be reused across multiple hosted clusters. Complete this before proceeding to Phase 2. -👉 **Guides**: - -- Azure Workload Identity Setup - Overview with CLI and OIDC configuration -- Create Azure IAM Resources Separately - Detailed IAM command reference +👉 **Guide**: Create Azure IAM Resources - OIDC issuer configuration and workload identity creation ### Phase 2: Management Cluster Setup @@ -12627,7 +12659,7 @@ Self-managed Azure HyperShift implements several security best practices: Begin your self-managed Azure HyperShift deployment by following the guides in order: -1. **Azure Workload Identity Setup** - Set up managed identities and OIDC federation (or use Create Azure IAM Resources Separately for CLI-based setup) +1. **Create Azure IAM Resources** - Set up OIDC issuer, managed identities, and workload identity federation 2. **Setup Azure Management Cluster for HyperShift** - Install HyperShift operator (with or without External DNS) 3. **Create a Self-Managed Azure HostedCluster** - Deploy your first hosted cluster 4. **Deploy Azure Private Clusters** (Optional) - Configure private endpoint access with Azure Private Link @@ -47841,6 +47873,73 @@ The value must be in proper IPV4 CIDR format

+###OVNIPv6Config { #hypershift.openshift.io/v1beta1.OVNIPv6Config } +

+(Appears on: +OVNKubernetesConfig) +

+

+

OVNIPv6Config contains IPv6-specific configuration options for OVN-Kubernetes. +https://github.com/openshift/api/blob/6d3c4e25a8d3aeb57ad61649d80c38cbd27d1cc8/operator/v1/types_network.go#L541-L570

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+internalTransitSwitchSubnet
+ +string + +		 +(Optional) +

internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally +by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect +architecture that connects the cluster routers on each node together to enable +east west traffic. The subnet chosen should not overlap with other networks +specified for OVN-Kubernetes as well as other networks used on the host. +When omitted, this means no opinion and the platform is left to choose a reasonable +default which is subject to change over time. +The current default subnet is fd97::/64. +The subnet must be large enough to accommodate one IP per node in your cluster. +The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, +IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. +The prefix length must be in the range /0 to /125 inclusive. +This field is immutable once set.

+
+internalJoinSubnet
+ +string + +		 +(Optional) +

internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the +default one is being already used by something else. It must not overlap with +any other subnet being used by OpenShift or by the node network. The size of the +subnet must be larger than the number of nodes. +The current default value is fd98::/64. +For KubeVirt hosted clusters, if this field is not set, HyperShift will +automatically use fd99::/64 to avoid collisions with the management cluster’s +default join subnet (fd98::/64). +The subnet must be large enough to accommodate one IP per node in your cluster. +The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, +IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. +The prefix length must be in the range /0 to /125 inclusive. +This field is immutable once set.

+
###OVNKubernetesConfig { #hypershift.openshift.io/v1beta1.OVNKubernetesConfig }

(Appears on: @@ -47876,6 +47975,25 @@ fields within ipv4 for details of default values.

+ipv6,omitzero
+ + +OVNIPv6Config + + + + +(Optional) +

ipv6 allows users to configure IP settings for IPv6 connections. When omitted, +this means no opinions and the default configuration is used. Check individual +fields within ipv6 for details of default values. +For KubeVirt hosted clusters using dual-stack networking, it is recommended to +set ipv6.internalJoinSubnet to a value different from the management cluster’s +join subnet (default fd98::/64) to avoid IPv6 routing conflicts.

+ + + + mtu
int32 @@ -53964,7 +54082,7 @@ You can use an existing VNet and NSG from the same resource group, which places ## Related Documentation -- Azure Workload Identity Setup - Set up managed identities and OIDC federation +- Create Azure IAM Resources - Set up OIDC issuer, managed identities, and workload identity federation - Setup Azure Management Cluster for HyperShift - Install HyperShift operator - Create a Self-Managed Azure HostedCluster - Deploy your first hosted cluster - Self-Managed Azure Overview - Comprehensive overview diff --git a/docs/content/reference/api.md b/docs/content/reference/api.md index 81f244bc303c..828e8531ff19 100644 --- a/docs/content/reference/api.md +++ b/docs/content/reference/api.md @@ -13630,6 +13630,73 @@ The value must be in proper IPV4 CIDR format

+###OVNIPv6Config { #hypershift.openshift.io/v1beta1.OVNIPv6Config } +

+(Appears on: +OVNKubernetesConfig) +

+

+

OVNIPv6Config contains IPv6-specific configuration options for OVN-Kubernetes. +https://github.com/openshift/api/blob/6d3c4e25a8d3aeb57ad61649d80c38cbd27d1cc8/operator/v1/types_network.go#L541-L570

+

+ + + + + + + + + + + + + + + + + +
FieldDescription
+internalTransitSwitchSubnet
+ +string + +		 +(Optional) +

internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally +by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect +architecture that connects the cluster routers on each node together to enable +east west traffic. The subnet chosen should not overlap with other networks +specified for OVN-Kubernetes as well as other networks used on the host. +When omitted, this means no opinion and the platform is left to choose a reasonable +default which is subject to change over time. +The current default subnet is fd97::/64. +The subnet must be large enough to accommodate one IP per node in your cluster. +The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, +IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. +The prefix length must be in the range /0 to /125 inclusive. +This field is immutable once set.

+
+internalJoinSubnet
+ +string + +		 +(Optional) +

internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the +default one is being already used by something else. It must not overlap with +any other subnet being used by OpenShift or by the node network. The size of the +subnet must be larger than the number of nodes. +The current default value is fd98::/64. +For KubeVirt hosted clusters, if this field is not set, HyperShift will +automatically use fd99::/64 to avoid collisions with the management cluster’s +default join subnet (fd98::/64). +The subnet must be large enough to accommodate one IP per node in your cluster. +The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, +IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. +The prefix length must be in the range /0 to /125 inclusive. +This field is immutable once set.

+
###OVNKubernetesConfig { #hypershift.openshift.io/v1beta1.OVNKubernetesConfig }

(Appears on: @@ -13665,6 +13732,25 @@ fields within ipv4 for details of default values.

+ipv6,omitzero
+ + +OVNIPv6Config + + + + +(Optional) +

ipv6 allows users to configure IP settings for IPv6 connections. When omitted, +this means no opinions and the default configuration is used. Check individual +fields within ipv6 for details of default values. +For KubeVirt hosted clusters using dual-stack networking, it is recommended to +set ipv6.internalJoinSubnet to a value different from the management cluster’s +join subnet (default fd98::/64) to avoid IPv6 routing conflicts.

+ + + + mtu
int32 diff --git a/docs/content/reference/infrastructure/azure-self-managed.md b/docs/content/reference/infrastructure/azure-self-managed.md index 5c86af12273e..7e654f896046 100644 --- a/docs/content/reference/infrastructure/azure-self-managed.md +++ b/docs/content/reference/infrastructure/azure-self-managed.md @@ -281,7 +281,7 @@ You can use an existing VNet and NSG from the same resource group, which places ## Related Documentation -- [Azure Workload Identity Setup](../../how-to/azure/azure-workload-identity-setup.md) - Set up managed identities and OIDC federation +- [Create Azure IAM Resources](../../how-to/azure/create-iam-separately.md) - Set up OIDC issuer, managed identities, and workload identity federation - [Setup Azure Management Cluster for HyperShift](../../how-to/azure/setup-management-cluster.md) - Install HyperShift operator - [Create a Self-Managed Azure HostedCluster](../../how-to/azure/create-self-managed-azure-cluster.md) - Deploy your first hosted cluster - [Self-Managed Azure Overview](../../how-to/azure/self-managed-azure-index.md) - Comprehensive overview diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index a3e7edb9720a..db4c498d5cd9 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -82,6 +82,7 @@ nav: - how-to/automated-machine-management/node-tuning.md - how-to/automated-machine-management/nodepool-lifecycle.md - how-to/automated-machine-management/scale-to-zero-dataplane.md + - 'Spot Instances': how-to/automated-machine-management/spot-instances.md - how-to/autoscaling.md - 'CI': - 'AI-Assisted CI Jobs': how-to/ci/ai-assisted-ci-jobs.md @@ -163,7 +164,6 @@ nav: - 'Self-managed Azure': - how-to/azure/self-managed-azure-index.md - how-to/azure/autoscaling-self-managed.md - - how-to/azure/azure-workload-identity-setup.md - how-to/azure/create-self-managed-azure-cluster.md - how-to/azure/create-iam-separately.md - how-to/azure/create-infra-separately.md diff --git a/etcd-backup/etcdbackup.go b/etcd-backup/etcdbackup.go index 579269e04fb8..88f100743ae6 100644 --- a/etcd-backup/etcdbackup.go +++ b/etcd-backup/etcdbackup.go @@ -120,7 +120,7 @@ func uploadToS3(ctx context.Context, opts options) error { f, err := os.Open(opts.snapshotFilePath) if err != nil { - return fmt.Errorf("failed to open file %q, %v", opts.snapshotFilePath, err) + return fmt.Errorf("failed to open file %q, %w", opts.snapshotFilePath, err) } defer f.Close() diff --git a/etcd-recovery/etcdrecovery.go b/etcd-recovery/etcdrecovery.go index 59063e9bbfa5..4c08d934f33f 100644 --- a/etcd-recovery/etcdrecovery.go +++ b/etcd-recovery/etcdrecovery.go @@ -476,7 +476,7 @@ func isEndpointHealthy(ctx context.Context, opts options, name, endpointURL stri healthCtx, healthCtxCancel := context.WithTimeout(ctx, defaultEtcdClientTimeout) defer healthCtxCancel() _, err = cli.Get(healthCtx, "health") - if err != nil && err != rpctypes.ErrPermissionDenied { + if err != nil && !errors.Is(err, rpctypes.ErrPermissionDenied) { log.Error(err, "cannot access etcd endpoint, returning healthy=false") return false } diff --git a/go.mod b/go.mod index f9a02eaaba69..127c298500e1 100644 --- a/go.mod +++ b/go.mod @@ -94,7 +94,7 @@ require ( golang.org/x/oauth2 v0.36.0 golang.org/x/sync v0.20.0 golang.org/x/time v0.15.0 - google.golang.org/api v0.279.0 + google.golang.org/api v0.280.0 google.golang.org/grpc v1.81.1 gopkg.in/ini.v1 v1.67.2 gopkg.in/yaml.v2 v2.4.0 @@ -292,7 +292,7 @@ require ( golang.org/x/tools v0.44.0 // indirect gomodules.xyz/jsonpatch/v2 v2.5.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260511170946-3700d4141b60 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect diff --git a/go.sum b/go.sum index 097230e14fba..d6e83718c4e2 100644 --- a/go.sum +++ b/go.sum @@ -943,8 +943,8 @@ gomodules.xyz/jsonpatch/v2 v2.5.0 h1:JELs8RLM12qJGXU4u/TO3V25KW8GreMKl9pdkk14RM0 gomodules.xyz/jsonpatch/v2 v2.5.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/api v0.279.0 h1:hsx2M2OaRcaKtVYK6vXEUnQvdjnend7ZYES+lYaot74= -google.golang.org/api v0.279.0/go.mod h1:B9TqLBwJqVjp1mtt7WeoQwWRwvu/400y5lETOql+giQ= +google.golang.org/api v0.280.0 h1:F4OfEHZhZh6a7uTufJAXXVd/2TQ8EjM4vZH+jX/vFYk= +google.golang.org/api v0.280.0/go.mod h1:oGKmPZRDoD3vdkf6MA7F4VNkR1rxCiuaPSkhsf3EolU= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -955,8 +955,8 @@ google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgn google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7 h1:41r6JMbpzBMen0R/4TZeeAmGXSJC7DftGINUodzTkPI= google.golang.org/genproto/googleapis/api v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:EIQZ5bFCfRQDV4MhRle7+OgjNtZ6P1PiZBgAKuxXu/Y= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 h1:tEkOQcXgF6dH1G+MVKZrfpYvozGrzb91k6ha7jireSM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260511170946-3700d4141b60 h1:seT2EwLWM78plQ7wcDfuWBc/4FAEAXDDiaSol4ku4qo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260511170946-3700d4141b60/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= diff --git a/hack/github-actions-runner/cache-warming-cronjob.yaml b/hack/github-actions-runner/cache-warming-cronjob.yaml deleted file mode 100644 index e6cbd03af7c6..000000000000 --- a/hack/github-actions-runner/cache-warming-cronjob.yaml +++ /dev/null @@ -1,78 +0,0 @@ -apiVersion: batch/v1 -kind: CronJob -metadata: - name: go-cache-warmer - namespace: arc-runners - labels: - app.kubernetes.io/component: cache-warmer - app.kubernetes.io/part-of: arc-runner-set -spec: - schedule: "0 2 * * *" - concurrencyPolicy: Forbid - successfulJobsHistoryLimit: 3 - failedJobsHistoryLimit: 1 - jobTemplate: - spec: - activeDeadlineSeconds: 3600 - backoffLimit: 1 - template: - spec: - restartPolicy: Never - securityContext: - runAsNonRoot: true - seccompProfile: - type: RuntimeDefault - containers: - - name: cache-warmer - image: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-gh-actions-runner:latest - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - command: - - /bin/bash - - -c - - | - set -euo pipefail - - echo "=== Cloning openshift/hypershift main branch ===" - git clone --depth 1 --branch main https://github.com/openshift/hypershift.git /tmp/hypershift - cd /tmp/hypershift - - echo "=== Compiling all packages ===" - go build ./... - - echo "=== Compiling all test binaries ===" - go test -c -o /dev/null ./... 2>/dev/null || true - - echo "=== Cleaning stale cache entries (older than 7 days) ===" - find /cache/go-build -type f -mtime +7 -delete - find /cache/go-build -mindepth 1 -type d -empty -delete - - echo "=== Syncing build cache to PV ===" - cp -a "${GOCACHE}"/* /cache/go-build/ - - echo "=== Cache warming complete ===" - du -sh /cache/go-build/ - env: - - name: GOCACHE - value: /tmp/go-build-cache - - name: GOMODCACHE - value: /tmp/go-mod-cache - - name: HOME - value: /tmp - resources: - requests: - cpu: "4" - memory: "16Gi" - limits: - cpu: "4" - memory: "16Gi" - volumeMounts: - - name: go-cache - mountPath: /cache/go-build - volumes: - - name: go-cache - persistentVolumeClaim: - claimName: go-cache-pvc diff --git a/hack/github-actions-runner/values.yaml b/hack/github-actions-runner/values.yaml index bdcd8bf6e29a..0d5eac179332 100644 --- a/hack/github-actions-runner/values.yaml +++ b/hack/github-actions-runner/values.yaml @@ -10,10 +10,6 @@ template: - name: runner image: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/hypershift-gh-actions-runner:latest command: ["/home/runner/run.sh"] - volumeMounts: - - name: go-cache - mountPath: /cache/go-build - readOnly: true resources: requests: cpu: "4" @@ -21,11 +17,6 @@ template: limits: cpu: "4" memory: "16Gi" - volumes: - - name: go-cache - persistentVolumeClaim: - claimName: go-cache-pvc - readOnly: true topologySpreadConstraints: - maxSkew: 1 topologyKey: kubernetes.io/hostname diff --git a/hypershift-ci-python/requirements.txt b/hypershift-ci-python/requirements.txt index d34f29c387b1..0fa53024c61f 100644 --- a/hypershift-ci-python/requirements.txt +++ b/hypershift-ci-python/requirements.txt @@ -8,7 +8,7 @@ google-auth-httplib2==0.2.0 google-auth-oauthlib==1.2.1 googleapis-common-protos==1.68.0 httplib2==0.22.0 -idna==3.10 +idna==3.15 oauthlib==3.2.2 proto-plus==1.26.0 protobuf==5.29.5 diff --git a/hypershift-operator/controllers/etcdbackup/reconciler.go b/hypershift-operator/controllers/etcdbackup/reconciler.go index bd665bdda2db..6c8e4f8672b2 100644 --- a/hypershift-operator/controllers/etcdbackup/reconciler.go +++ b/hypershift-operator/controllers/etcdbackup/reconciler.go @@ -299,9 +299,10 @@ func (r *HCPEtcdBackupReconciler) setCondition(backup *hyperv1.HCPEtcdBackup, co // updateHCPBackupCondition sets a condition on the HostedControlPlane to bubble // up the etcd backup status. The HC controller propagates this to the HostedCluster. func (r *HCPEtcdBackupReconciler) updateHCPBackupCondition(ctx context.Context, hcp *hyperv1.HostedControlPlane, condition metav1.Condition) error { + originalHCP := hcp.DeepCopy() condition.ObservedGeneration = hcp.Generation meta.SetStatusCondition(&hcp.Status.Conditions, condition) - return r.Status().Update(ctx, hcp) + return r.Status().Patch(ctx, hcp, client.MergeFromWithOptions(originalHCP, client.MergeFromWithOptimisticLock{})) } // updateHostedClusterBackupURL persists the snapshot URL in the HostedCluster diff --git a/hypershift-operator/controllers/etcdbackup/reconciler_test.go b/hypershift-operator/controllers/etcdbackup/reconciler_test.go index d585b24c5241..7f15a5223ed6 100644 --- a/hypershift-operator/controllers/etcdbackup/reconciler_test.go +++ b/hypershift-operator/controllers/etcdbackup/reconciler_test.go @@ -1786,3 +1786,78 @@ func TestGetSnapshotURLFromPod(t *testing.T) { }) } } + +func TestUpdateHCPBackupCondition(t *testing.T) { + t.Parallel() + + t.Run("When patching a backup condition, it should not stomp unrelated status fields", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + hcp := newHostedControlPlane() + hcp.Generation = 3 + hcp.Status.AutoNode = hyperv1.AutoNodeStatus{ + VCPUs: ptr.To[int32](8), + } + + r := newReconciler(hcp) + + err := r.updateHCPBackupCondition(t.Context(), hcp, metav1.Condition{ + Type: string(hyperv1.EtcdBackupSucceeded), + Status: metav1.ConditionTrue, + Reason: "BackupComplete", + Message: "backup finished", + }) + g.Expect(err).ToNot(HaveOccurred()) + + var updated hyperv1.HostedControlPlane + g.Expect(r.Get(t.Context(), types.NamespacedName{ + Name: testHCPName, + Namespace: testHCPNamespace, + }, &updated)).To(Succeed()) + + cond := meta.FindStatusCondition(updated.Status.Conditions, string(hyperv1.EtcdBackupSucceeded)) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal("BackupComplete")) + g.Expect(cond.ObservedGeneration).To(Equal(int64(3))) + + g.Expect(updated.Status.AutoNode.VCPUs).To(Equal(ptr.To[int32](8)), + "autoNode should be preserved — patch must not stomp unrelated fields") + }) + + t.Run("When patching a backup condition onto an HCP with existing conditions, it should preserve them", func(t *testing.T) { + t.Parallel() + g := NewWithT(t) + + hcp := newHostedControlPlane() + hcp.Status.Conditions = []metav1.Condition{ + { + Type: string(hyperv1.ClusterVersionAvailable), + Status: metav1.ConditionTrue, + Reason: "OK", + }, + } + + r := newReconciler(hcp) + + err := r.updateHCPBackupCondition(t.Context(), hcp, metav1.Condition{ + Type: string(hyperv1.EtcdBackupSucceeded), + Status: metav1.ConditionTrue, + Reason: "BackupComplete", + Message: "backup finished", + }) + g.Expect(err).ToNot(HaveOccurred()) + + var updated hyperv1.HostedControlPlane + g.Expect(r.Get(t.Context(), types.NamespacedName{ + Name: testHCPName, + Namespace: testHCPNamespace, + }, &updated)).To(Succeed()) + + g.Expect(updated.Status.Conditions).To(HaveLen(2), + "both the existing condition and the new backup condition should be present") + g.Expect(meta.FindStatusCondition(updated.Status.Conditions, string(hyperv1.ClusterVersionAvailable))).ToNot(BeNil()) + g.Expect(meta.FindStatusCondition(updated.Status.Conditions, string(hyperv1.EtcdBackupSucceeded))).ToNot(BeNil()) + }) +} diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go index bec35502b68b..403cbda28ecc 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go @@ -136,6 +136,8 @@ const ( useRestrictedPodSecurityLabel = "io.openshift.hypershift.restricted-psa" defaultToControlPlaneV2Label = "io.openshift.hypershift.control-plane-operator.v2-isdefault" + apiOpenShiftComLabelPrefix = "api.openshift.com/" + etcdEncKeyPostfix = "-etcd-encryption-key" jobHostedClusterNameLabel = "hypershift.openshift.io/cluster-name" @@ -851,8 +853,8 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques propagateControlPlaneVersion(hcluster, hcp) // Set the AutoNodeEnabled condition reflecting both spec intent and actual component rollout progress. - meta.SetStatusCondition(&hcluster.Status.Conditions, - r.reconcileAutoNodeEnabledCondition(ctx, hcluster, controlPlaneNamespace.Name)) + autoNodeCondition, autoNodeProgressing := r.reconcileAutoNodeEnabledCondition(ctx, hcluster, controlPlaneNamespace.Name) + meta.SetStatusCondition(&hcluster.Status.Conditions, autoNodeCondition) // Copy the AWSDefaultSecurityGroupCreated condition from the hostedcontrolplane if hcluster.Spec.Platform.Type == hyperv1.AWSPlatform { @@ -1419,7 +1421,7 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques Message: err.Error(), }) if statusErr := r.Client.Status().Update(ctx, hcluster); statusErr != nil { - return ctrl.Result{}, fmt.Errorf("failed to reconcile platform credentials: %s, failed to update status: %w", err, statusErr) + return ctrl.Result{}, fmt.Errorf("failed to reconcile platform credentials: %w, failed to update status: %w", err, statusErr) } return ctrl.Result{}, fmt.Errorf("failed to reconcile platform credentials: %w", err) } @@ -1432,7 +1434,7 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques Message: "Required platform credentials are found", }) if statusErr := r.Client.Status().Update(ctx, hcluster); statusErr != nil { - return ctrl.Result{}, fmt.Errorf("failed to reconcile platform credentials: %s, failed to update status: %w", err, statusErr) + return ctrl.Result{}, fmt.Errorf("failed to reconcile platform credentials: %w, failed to update status: %w", err, statusErr) } } } @@ -1801,7 +1803,7 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques hcp, shouldCheckForStaleCerts(hcluster, defaultToControlPlaneV2), r.kasServingCertHashFromSecret(ctx, hcp), - r.kasServingCertHashFromEndpoint(kasHostAndPortFromHCP(hcp)))) + r.kasServingCertHashFromEndpoint(ctx, kasHostAndPortFromHCP(hcp)))) }) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to reconcile hostedcontrolplane: %w", err) @@ -2094,6 +2096,12 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques if requeueAfter != nil { result.RequeueAfter = *requeueAfter } + if autoNodeProgressing { + autoNodeRequeue := 15 * time.Second + if result.RequeueAfter == 0 || autoNodeRequeue < result.RequeueAfter { + result.RequeueAfter = autoNodeRequeue + } + } return result, nil } @@ -2242,16 +2250,20 @@ func (r *HostedClusterReconciler) kasServingCertHashFromSecret(ctx context.Conte } } -func (r *HostedClusterReconciler) kasServingCertHashFromEndpoint(kasHostAndPort string) func() (string, error) { +func (r *HostedClusterReconciler) kasServingCertHashFromEndpoint(ctx context.Context, kasHostAndPort string) func() (string, error) { return func() (string, error) { - conn, err := tls.Dial("tcp", kasHostAndPort, &tls.Config{ + netConn, err := (&tls.Dialer{Config: &tls.Config{ InsecureSkipVerify: true, ServerName: "kubernetes", - }) + }}).DialContext(ctx, "tcp", kasHostAndPort) if err != nil { return "", fmt.Errorf("failed to dial %s: %w", kasHostAndPort, err) } - defer conn.Close() + defer netConn.Close() + conn, ok := netConn.(*tls.Conn) + if !ok { + return "", fmt.Errorf("connection to %s is not a TLS connection", kasHostAndPort) + } kasCerts := conn.ConnectionState().PeerCertificates if len(kasCerts) == 0 { return "", fmt.Errorf("no certificate found on KAS endpoint %s", kasHostAndPort) @@ -2334,7 +2346,7 @@ func reconcileHostedControlPlaneAnnotations(hcp *hyperv1.HostedControlPlane, hcl hyperv1.KubeAPIServerGoAwayChance, hyperv1.KubeAPIServerServiceAccountTokenMaxExpiration, hyperv1.HostedClusterRestoredFromBackupAnnotation, - // TODO: Remove this once the the input is in the HostedCluster AWS API. + // TODO: Remove this once the input is in the HostedCluster AWS API. "hypershift.openshift.io/aws-termination-handler-queue-url", hyperv1.SwiftPodNetworkInstanceAnnotation, hyperv1.EnableMetricsForwarding, @@ -2414,10 +2426,17 @@ func reconcileHostedControlPlane(hcp *hyperv1.HostedControlPlane, hcluster *hype if hcp.Labels == nil { hcp.Labels = make(map[string]string) } - // All labels on the HostedCluster with this special prefix are copied - // Those are labels set by OCM + // These labels are managed by OCM. Delete-then-copy ensures removals + // on the HostedCluster (e.g., clearing limited-support) propagate to the HCP. + for key := range hcp.Labels { + if strings.HasPrefix(key, apiOpenShiftComLabelPrefix) { + if _, exists := hcluster.Labels[key]; !exists { + delete(hcp.Labels, key) + } + } + } for key, val := range hcluster.Labels { - if strings.HasPrefix(key, "api.openshift.com") { + if strings.HasPrefix(key, apiOpenShiftComLabelPrefix) { hcp.Labels[key] = val } } @@ -2505,7 +2524,7 @@ func reconcileHostedControlPlane(hcp *hyperv1.HostedControlPlane, hcluster *hype return nil } -// reconcileCAPIManager orchestrates orchestrates of all CAPI manager components. +// reconcileCAPIManager orchestrates all CAPI manager components. func (r *HostedClusterReconciler) reconcileCAPIManager(cpContext controlplanecomponent.ControlPlaneContext, createOrUpdate upsert.CreateOrUpdateFN, hcluster *hyperv1.HostedCluster, releaseVersion semver.Version) error { controlPlaneNamespace := manifests.HostedControlPlaneNamespaceObject(hcluster.Namespace, hcluster.Name) err := r.Client.Get(cpContext, client.ObjectKeyFromObject(controlPlaneNamespace), controlPlaneNamespace) @@ -2825,7 +2844,7 @@ func (r *HostedClusterReconciler) reconcileCLISecrets(ctx context.Context, creat util.AutoInfraLabelName: hcluster.Spec.InfraID, }) if err != nil { - return fmt.Errorf("failed to retrieve cli created secrets: %v", err) + return fmt.Errorf("failed to retrieve cli created secrets: %w", err) } ownerRef := config.OwnerRefFrom(hcluster) @@ -2835,7 +2854,7 @@ func (r *HostedClusterReconciler) reconcileCLISecrets(ctx context.Context, creat return nil }) if err != nil { - return fmt.Errorf("failed to set '%s' secret's owner reference: %v", secret.Name, err) + return fmt.Errorf("failed to set '%s' secret's owner reference: %w", secret.Name, err) } if res == controllerutil.OperationResultUpdated { log.Info("added owner reference of the Hosted cluster, to the secret", "secret", secret.Name) @@ -3224,7 +3243,7 @@ func computeAWSEndpointServiceCondition(awsEndpointServiceList hyperv1.AWSEndpoi func listNodePools(ctx context.Context, c client.Client, clusterNamespace, clusterName string) ([]hyperv1.NodePool, error) { nodePoolList := &hyperv1.NodePoolList{} if err := c.List(ctx, nodePoolList); err != nil { - return nil, fmt.Errorf("failed getting nodePool list: %v", err) + return nil, fmt.Errorf("failed getting nodePool list: %w", err) } // TODO: do a label association or something filtered := []hyperv1.NodePool{} @@ -4336,6 +4355,50 @@ func validateSliceNetworkCIDRs(hc *hyperv1.HostedCluster) field.ErrorList { } } } + + if hc.Spec.Networking.NetworkType == hyperv1.OVNKubernetes { + var ipv4JoinSubnet string + if hc.Spec.OperatorConfiguration != nil && hc.Spec.OperatorConfiguration.ClusterNetworkOperator != nil && + hc.Spec.OperatorConfiguration.ClusterNetworkOperator.OVNKubernetesConfig != nil && + hc.Spec.OperatorConfiguration.ClusterNetworkOperator.OVNKubernetesConfig.IPv4 != nil { + ipv4JoinSubnet = hc.Spec.OperatorConfiguration.ClusterNetworkOperator.OVNKubernetesConfig.IPv4.InternalJoinSubnet + } + // The reconciler defaults KubeVirt IPv4 internal subnet to avoid collision + // with the management cluster; include the effective value so overlaps are caught at admission time. + if ipv4JoinSubnet == "" && hc.Spec.Platform.Type == hyperv1.KubevirtPlatform { + _, cidr, err := net.ParseCIDR(hyperv1.KubevirtDefaultV4InternalSubnet) + if err == nil { + ce := cidrEntry{*cidr, *field.NewPath("spec", "operatorConfiguration", "clusterNetworkOperator", "ovnKubernetesConfig", "ipv4", "v4InternalSubnet (default)")} + cidrEntries = append(cidrEntries, ce) + } + } + + var ipv6JoinSubnet, ipv6TransitSubnet string + if hc.Spec.OperatorConfiguration != nil && hc.Spec.OperatorConfiguration.ClusterNetworkOperator != nil && + hc.Spec.OperatorConfiguration.ClusterNetworkOperator.OVNKubernetesConfig != nil { + ipv6JoinSubnet = hc.Spec.OperatorConfiguration.ClusterNetworkOperator.OVNKubernetesConfig.IPv6.InternalJoinSubnet + ipv6TransitSubnet = hc.Spec.OperatorConfiguration.ClusterNetworkOperator.OVNKubernetesConfig.IPv6.InternalTransitSwitchSubnet + } + // The reconciler defaults KubeVirt IPv6 join subnet to avoid collision with the + // management cluster; include the effective value so overlaps are caught at admission time. + if ipv6JoinSubnet == "" && hc.Spec.Platform.Type == hyperv1.KubevirtPlatform { + ipv6JoinSubnet = hyperv1.KubevirtDefaultV6InternalJoinSubnet + } + if ipv6JoinSubnet != "" { + _, cidr, err := net.ParseCIDR(ipv6JoinSubnet) + if err == nil { + ce := cidrEntry{*cidr, *field.NewPath("spec", "operatorConfiguration", "clusterNetworkOperator", "ovnKubernetesConfig", "ipv6", "internalJoinSubnet")} + cidrEntries = append(cidrEntries, ce) + } + } + if ipv6TransitSubnet != "" { + _, cidr, err := net.ParseCIDR(ipv6TransitSubnet) + if err == nil { + ce := cidrEntry{*cidr, *field.NewPath("spec", "operatorConfiguration", "clusterNetworkOperator", "ovnKubernetesConfig", "ipv6", "internalTransitSwitchSubnet")} + cidrEntries = append(cidrEntries, ce) + } + } + } return compareCIDREntries(cidrEntries) } @@ -4380,7 +4443,7 @@ func (r *HostedClusterReconciler) reconcileOIDCDocumentsWithStatus(ctx context.C Message: err.Error(), }) if statusErr := r.Client.Status().Update(ctx, hcluster); statusErr != nil { - return fmt.Errorf("failed to reconcile OIDC documents: %s, failed to update status: %w", err, statusErr) + return fmt.Errorf("failed to reconcile OIDC documents: %w, failed to update status: %w", err, statusErr) } return fmt.Errorf("failed to reconcile OIDC documents: %w", err) } @@ -4466,7 +4529,8 @@ func (r *HostedClusterReconciler) reconcileAWSOIDCDocuments(ctx context.Context, // return the code. If other specific error types can be handled, add // new switch cases and try to provide more actionable info to the // user. - wrapped = fmt.Errorf("%w: aws returned an error: %v", wrapped, err) + log.Error(err, "failed to upload OIDC document to S3", "path", path, "bucket", r.OIDCStorageProviderS3BucketName) + wrapped = fmt.Errorf("%w: aws returned an error", wrapped) } return wrapped } diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go index f289f3dd122a..e86cc2236404 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller_test.go @@ -8,6 +8,8 @@ import ( "encoding/pem" "errors" "fmt" + "net/http" + "net/http/httptest" "os" "reflect" "strings" @@ -66,6 +68,8 @@ import ( ctrl "sigs.k8s.io/controller-runtime" crclient "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -310,6 +314,64 @@ func TestReconcileHostedControlPlaneAdditionalTrustBundle(t *testing.T) { } } +func TestReconcileHostedControlPlaneLabelSync(t *testing.T) { + t.Parallel() + tests := []struct { + name string + hcLabels map[string]string + hcpLabels map[string]string + expectedLabels map[string]string + }{ + { + name: "When HC has api.openshift.com labels, it should copy them to HCP", + hcLabels: map[string]string{"api.openshift.com/limited-support": "true", "api.openshift.com/name": "test"}, + hcpLabels: map[string]string{}, + expectedLabels: map[string]string{"api.openshift.com/limited-support": "true", "api.openshift.com/name": "test"}, + }, + { + name: "When HC removes an api.openshift.com label, it should remove the stale label from HCP", + hcLabels: map[string]string{"api.openshift.com/name": "test"}, + hcpLabels: map[string]string{"api.openshift.com/limited-support": "true", "api.openshift.com/name": "old"}, + expectedLabels: map[string]string{"api.openshift.com/name": "test"}, + }, + { + name: "When HC has no api.openshift.com labels, it should preserve non-api.openshift.com labels on HCP", + hcLabels: map[string]string{}, + hcpLabels: map[string]string{"api.openshift.com/limited-support": "true", "cluster.x-k8s.io/cluster-name": "keep-me"}, + expectedLabels: map[string]string{"cluster.x-k8s.io/cluster-name": "keep-me"}, + }, + { + name: "When HC labels are nil, it should remove all api.openshift.com labels from HCP", + hcLabels: nil, + hcpLabels: map[string]string{"api.openshift.com/limited-support": "true"}, + expectedLabels: map[string]string{}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + g := NewGomegaWithT(t) + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{Labels: test.hcLabels}, + } + hcp := &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{Labels: test.hcpLabels}, + } + err := reconcileHostedControlPlane(hcp, hc, false, false, func() (map[string]string, error) { return nil, nil }) + g.Expect(err).ToNot(HaveOccurred()) + + for key, val := range test.expectedLabels { + g.Expect(hcp.Labels).To(HaveKeyWithValue(key, val)) + } + for key := range hcp.Labels { + if strings.HasPrefix(key, apiOpenShiftComLabelPrefix) { + g.Expect(test.expectedLabels).To(HaveKey(key), "unexpected label %s=%s still on HCP", key, hcp.Labels[key]) + } + } + }) + } +} + func TestReconcileHostedControlPlaneUpgrades(t *testing.T) { t.Parallel() // TODO: the spec/status comparison of control plane is a weak check; the @@ -3918,13 +3980,14 @@ func TestComputeAWSEndpointServiceCondition(t *testing.T) { func TestValidateSliceNetworkCIDRs(t *testing.T) { t.Parallel() tests := []struct { - name string - mn []hyperv1.MachineNetworkEntry - cn []hyperv1.ClusterNetworkEntry - sn []hyperv1.ServiceNetworkEntry - networkType hyperv1.NetworkType - ovnConfig *hyperv1.OVNKubernetesConfig - wantErr bool + name string + mn []hyperv1.MachineNetworkEntry + cn []hyperv1.ClusterNetworkEntry + sn []hyperv1.ServiceNetworkEntry + networkType hyperv1.NetworkType + platformType hyperv1.PlatformType + ovnConfig *hyperv1.OVNKubernetesConfig + wantErr bool }{ { name: "given a conflicting IPv6 clusterNetwork overlapped with machineNetwork, it should fail", @@ -4069,6 +4132,143 @@ func TestValidateSliceNetworkCIDRs(t *testing.T) { }, wantErr: false, }, + { + name: "When OVN-Kubernetes with valid IPv6 InternalJoinSubnet it should succeed", + mn: []hyperv1.MachineNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd02::/48")}}, + cn: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd01::/64")}}, + sn: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd03::/112")}}, + networkType: hyperv1.OVNKubernetes, + ovnConfig: &hyperv1.OVNKubernetesConfig{ + IPv6: hyperv1.OVNIPv6Config{ + InternalJoinSubnet: "fd99::/64", + }, + }, + wantErr: false, + }, + { + name: "When OVN-Kubernetes with valid IPv6 InternalTransitSwitchSubnet it should succeed", + mn: []hyperv1.MachineNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd02::/48")}}, + cn: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd01::/64")}}, + sn: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd03::/112")}}, + networkType: hyperv1.OVNKubernetes, + ovnConfig: &hyperv1.OVNKubernetesConfig{ + IPv6: hyperv1.OVNIPv6Config{ + InternalTransitSwitchSubnet: "fd97:1::/64", + }, + }, + wantErr: false, + }, + { + name: "When OVN-Kubernetes IPv6 InternalJoinSubnet overlaps with MachineNetwork it should fail", + mn: []hyperv1.MachineNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd99::/48")}}, + cn: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd01::/64")}}, + sn: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd03::/112")}}, + networkType: hyperv1.OVNKubernetes, + ovnConfig: &hyperv1.OVNKubernetesConfig{ + IPv6: hyperv1.OVNIPv6Config{ + InternalJoinSubnet: "fd99::/64", + }, + }, + wantErr: true, + }, + { + name: "When OVN-Kubernetes IPv6 subnets overlap with each other it should fail", + mn: []hyperv1.MachineNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd02::/48")}}, + cn: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd01::/64")}}, + sn: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd03::/112")}}, + networkType: hyperv1.OVNKubernetes, + ovnConfig: &hyperv1.OVNKubernetesConfig{ + IPv6: hyperv1.OVNIPv6Config{ + InternalJoinSubnet: "fd99::/64", + InternalTransitSwitchSubnet: "fd99::/48", + }, + }, + wantErr: true, + }, + { + name: "When OVN-Kubernetes with both valid IPv4 and IPv6 subnets it should succeed", + mn: []hyperv1.MachineNetworkEntry{{CIDR: *ipnet.MustParseCIDR("192.168.1.0/24")}}, + cn: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.128.0.0/14")}}, + sn: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.30.0.0/16")}}, + networkType: hyperv1.OVNKubernetes, + ovnConfig: &hyperv1.OVNKubernetesConfig{ + IPv4: &hyperv1.OVNIPv4Config{ + InternalJoinSubnet: "100.64.0.0/16", + }, + IPv6: hyperv1.OVNIPv6Config{ + InternalJoinSubnet: "fd99::/64", + }, + }, + wantErr: false, + }, + { + name: "When OVN-Kubernetes with empty IPv6 subnet strings it should succeed", + mn: []hyperv1.MachineNetworkEntry{{CIDR: *ipnet.MustParseCIDR("192.168.1.0/24")}}, + cn: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.128.0.0/14")}}, + sn: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.30.0.0/16")}}, + networkType: hyperv1.OVNKubernetes, + ovnConfig: &hyperv1.OVNKubernetesConfig{ + IPv6: hyperv1.OVNIPv6Config{ + InternalJoinSubnet: "", + InternalTransitSwitchSubnet: "", + }, + }, + wantErr: false, + }, + { + name: "When KubeVirt OVN-Kubernetes with no IPv6 config and MachineNetwork overlaps default fd99::/64 it should fail", + mn: []hyperv1.MachineNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd99::/48")}}, + cn: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd01::/64")}}, + sn: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd02::/112")}}, + networkType: hyperv1.OVNKubernetes, + platformType: hyperv1.KubevirtPlatform, + ovnConfig: nil, + wantErr: true, + }, + { + name: "When KubeVirt OVN-Kubernetes with no IPv6 config and non-overlapping networks it should succeed", + mn: []hyperv1.MachineNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd01::/48")}}, + cn: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd02::/64")}}, + sn: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd03::/112")}}, + networkType: hyperv1.OVNKubernetes, + platformType: hyperv1.KubevirtPlatform, + ovnConfig: nil, + wantErr: false, + }, + { + name: "When KubeVirt OVN-Kubernetes with explicit IPv6 join subnet it should use explicit value not default", + mn: []hyperv1.MachineNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd99::/48")}}, + cn: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd01::/64")}}, + sn: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("fd02::/112")}}, + networkType: hyperv1.OVNKubernetes, + platformType: hyperv1.KubevirtPlatform, + ovnConfig: &hyperv1.OVNKubernetesConfig{ + IPv6: hyperv1.OVNIPv6Config{ + InternalJoinSubnet: "fdaa::/64", + }, + }, + wantErr: false, + }, + { + name: "When KubeVirt OVN-Kubernetes with no IPv4 config and MachineNetwork overlaps default 100.66.0.0/16 it should fail", + mn: []hyperv1.MachineNetworkEntry{{CIDR: *ipnet.MustParseCIDR("100.66.0.0/24")}}, + cn: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.128.0.0/14")}}, + sn: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.30.0.0/16")}}, + networkType: hyperv1.OVNKubernetes, + platformType: hyperv1.KubevirtPlatform, + ovnConfig: nil, + wantErr: true, + }, + { + name: "When KubeVirt OVN-Kubernetes with no IPv4 config and non-overlapping networks it should succeed", + mn: []hyperv1.MachineNetworkEntry{{CIDR: *ipnet.MustParseCIDR("192.168.1.0/24")}}, + cn: []hyperv1.ClusterNetworkEntry{{CIDR: *ipnet.MustParseCIDR("10.128.0.0/14")}}, + sn: []hyperv1.ServiceNetworkEntry{{CIDR: *ipnet.MustParseCIDR("172.30.0.0/16")}}, + networkType: hyperv1.OVNKubernetes, + platformType: hyperv1.KubevirtPlatform, + ovnConfig: nil, + wantErr: false, + }, } for _, tt := range tests { @@ -4079,6 +4279,9 @@ func TestValidateSliceNetworkCIDRs(t *testing.T) { Namespace: "any", }, Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{ + Type: tt.platformType, + }, Networking: hyperv1.ClusterNetworking{ NetworkType: tt.networkType, MachineNetwork: tt.mn, @@ -4088,9 +4291,7 @@ func TestValidateSliceNetworkCIDRs(t *testing.T) { }, } - // Set OVN configuration if provided if tt.ovnConfig != nil { - //OperatorConfiguration hc.Spec.OperatorConfiguration = &hyperv1.OperatorConfiguration{ ClusterNetworkOperator: &hyperv1.ClusterNetworkOperatorSpec{ OVNKubernetesConfig: tt.ovnConfig, @@ -6734,3 +6935,209 @@ func TestComputeEndpointServiceCondition(t *testing.T) { }) } } + +func TestListNodePools(t *testing.T) { + t.Parallel() + tests := []struct { + name string + objects []crclient.Object + interceptorFuncs interceptor.Funcs + clusterNamespace string + clusterName string + wantErr bool + errSubstr string + expectedCount int + }{ + { + name: "When client List succeeds with matching NodePools, it should return filtered results", + objects: []crclient.Object{ + &hyperv1.NodePool{ + ObjectMeta: metav1.ObjectMeta{Name: "np1", Namespace: "clusters"}, + Spec: hyperv1.NodePoolSpec{ClusterName: "my-cluster"}, + }, + &hyperv1.NodePool{ + ObjectMeta: metav1.ObjectMeta{Name: "np2", Namespace: "clusters"}, + Spec: hyperv1.NodePoolSpec{ClusterName: "other-cluster"}, + }, + }, + clusterNamespace: "clusters", + clusterName: "my-cluster", + expectedCount: 1, + }, + { + name: "When client List fails, it should return a wrapped error", + objects: []crclient.Object{}, + interceptorFuncs: interceptor.Funcs{ + List: func(ctx context.Context, c crclient.WithWatch, list crclient.ObjectList, opts ...crclient.ListOption) error { + return fmt.Errorf("API server unavailable") + }, + }, + clusterNamespace: "clusters", + clusterName: "my-cluster", + wantErr: true, + errSubstr: "failed getting nodePool list", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + builder := fake.NewClientBuilder().WithScheme(api.Scheme).WithObjects(tt.objects...) + if tt.interceptorFuncs.List != nil { + builder = builder.WithInterceptorFuncs(tt.interceptorFuncs) + } + c := builder.Build() + + result, err := listNodePools(t.Context(), c, tt.clusterNamespace, tt.clusterName) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(result).To(HaveLen(tt.expectedCount)) + for _, np := range result { + g.Expect(np.Namespace).To(Equal(tt.clusterNamespace)) + g.Expect(np.Spec.ClusterName).To(Equal(tt.clusterName)) + } + } + }) + } +} + +func TestReconcileCLISecretsErrors(t *testing.T) { + t.Parallel() + tests := []struct { + name string + interceptorFuncs interceptor.Funcs + existingSecrets []crclient.Object + createOrUpdate upsert.CreateOrUpdateFN + wantErrSubstr string + }{ + { + name: "When client List fails, it should return a wrapped error", + interceptorFuncs: interceptor.Funcs{ + List: func(ctx context.Context, c crclient.WithWatch, list crclient.ObjectList, opts ...crclient.ListOption) error { + return fmt.Errorf("connection refused") + }, + }, + wantErrSubstr: "failed to retrieve cli created secrets", + }, + { + name: "When createOrUpdate fails for a secret, it should return a wrapped error", + existingSecrets: []crclient.Object{ + &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cli-secret", + Namespace: "clusters", + Labels: map[string]string{ + util.DeleteWithClusterLabelName: "true", + util.AutoInfraLabelName: "test-infra", + }, + }, + }, + }, + createOrUpdate: func(ctx context.Context, c crclient.Client, obj crclient.Object, f controllerutil.MutateFn) (controllerutil.OperationResult, error) { + return controllerutil.OperationResultNone, fmt.Errorf("API conflict") + }, + wantErrSubstr: "failed to set 'cli-secret' secret's owner reference", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + builder := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithInterceptorFuncs(tt.interceptorFuncs) + if len(tt.existingSecrets) > 0 { + builder = builder.WithObjects(tt.existingSecrets...) + } + cli := builder.Build() + + r := &HostedClusterReconciler{Client: cli} + hc := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "clusters", + }, + Spec: hyperv1.HostedClusterSpec{ + InfraID: "test-infra", + }, + } + + createOrUpdate := tt.createOrUpdate + if createOrUpdate == nil { + createOrUpdate = upsert.New(false).CreateOrUpdate + } + + err := r.reconcileCLISecrets(t.Context(), createOrUpdate, hc) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tt.wantErrSubstr)) + }) + } +} + +func TestKasServingCertHashFromEndpoint(t *testing.T) { + t.Parallel() + tests := []struct { + name string + setupTLS bool + cancelCtx bool + wantErr bool + errSubstr string + }{ + { + name: "When the TLS endpoint is healthy, it should return a non-empty certificate hash", + setupTLS: true, + }, + { + name: "When the endpoint is unreachable, it should return a dial error", + wantErr: true, + errSubstr: "failed to dial", + }, + { + name: "When the context is canceled, it should return an error", + setupTLS: true, + cancelCtx: true, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + addr := "127.0.0.1:1" + if tt.setupTLS { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + addr = server.Listener.Addr().String() + } + + ctx := t.Context() + if tt.cancelCtx { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + } + + r := &HostedClusterReconciler{} + hashFn := r.kasServingCertHashFromEndpoint(ctx, addr) + hash, err := hashFn() + + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + if tt.errSubstr != "" { + g.Expect(err.Error()).To(ContainSubstring(tt.errSubstr)) + } + } else { + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(hash).ToNot(BeEmpty()) + } + }) + } +} diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_webhook.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_webhook.go index e9791dc4e82d..6a7e8b6e3af6 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_webhook.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_webhook.go @@ -104,7 +104,7 @@ func (defaulter *nodePoolDefaulter) Default(ctx context.Context, obj runtime.Obj err := defaulter.client.Get(ctx, client.ObjectKeyFromObject(hc), hc) if err != nil { - return fmt.Errorf("error retrieving HostedCluster named [%s], %v", np.Spec.ClusterName, err) + return fmt.Errorf("error retrieving HostedCluster named [%s], %w", np.Spec.ClusterName, err) } np.Spec.Release.Image = hc.Spec.Release.Image } diff --git a/hypershift-operator/controllers/hostedcluster/internal/platform/kubevirt/kubevirt_test.go b/hypershift-operator/controllers/hostedcluster/internal/platform/kubevirt/kubevirt_test.go index a16845df131c..5acd9f922ed2 100644 --- a/hypershift-operator/controllers/hostedcluster/internal/platform/kubevirt/kubevirt_test.go +++ b/hypershift-operator/controllers/hostedcluster/internal/platform/kubevirt/kubevirt_test.go @@ -79,7 +79,7 @@ func TestReconcileCAPIInfraCR(t *testing.T) { t.Fatalf("Expected the provided function to be called once") } if tc.expectedErr != nil { - if err != tc.expectedErr { + if !errors.Is(err, tc.expectedErr) { t.Fatalf("ReconcileCAPIInfraCR: Expected to fail. gotErr: %v, expectedErr: %v", err, tc.expectedErr) } } else if err != nil { diff --git a/hypershift-operator/controllers/hostedcluster/internal/proxy/validation.go b/hypershift-operator/controllers/hostedcluster/internal/proxy/validation.go index 3e480e340c57..b4ed435c2b2b 100644 --- a/hypershift-operator/controllers/hostedcluster/internal/proxy/validation.go +++ b/hypershift-operator/controllers/hostedcluster/internal/proxy/validation.go @@ -31,7 +31,7 @@ func LoadCABundle(configMap corev1.ConfigMap) ([]*x509.Certificate, error) { } certBundle, err := crypto.CertsFromPEM(trustBundleData) if err != nil { - return nil, fmt.Errorf("failed parsing certificate data from ConfigMap %q: %v", configMap.Name, err) + return nil, fmt.Errorf("failed parsing certificate data from ConfigMap %q: %w", configMap.Name, err) } return certBundle, nil } diff --git a/hypershift-operator/controllers/hostedcluster/karpenter.go b/hypershift-operator/controllers/hostedcluster/karpenter.go index ae71cbd74989..80765881d04f 100644 --- a/hypershift-operator/controllers/hostedcluster/karpenter.go +++ b/hypershift-operator/controllers/hostedcluster/karpenter.go @@ -164,7 +164,13 @@ func isKASAvailable(ctx context.Context, cpNamespace string, c client.Client) (b // - True / AsExpected — Karpenter enabled in spec AND both components fully rolled out. // - False / AutoNodeProgressing — Enable or disable operation is in progress. // - False / AutoNodeNotConfigured — Karpenter not in spec AND no components present. -func (r *HostedClusterReconciler) reconcileAutoNodeEnabledCondition(ctx context.Context, hcluster *hyperv1.HostedCluster, hcpNamespace string) metav1.Condition { +// +// reconcileAutoNodeEnabledCondition returns the AutoNodeEnabled condition and whether +// the caller should requeue to poll for progress. The second return value is true when +// an enable or disable operation is still in flight; the HC reconciler does not watch +// ControlPlaneComponent resources, so a periodic requeue is needed to pick up status +// changes from the karpenter-operator's CPC updates. +func (r *HostedClusterReconciler) reconcileAutoNodeEnabledCondition(ctx context.Context, hcluster *hyperv1.HostedCluster, hcpNamespace string) (metav1.Condition, bool) { condition := metav1.Condition{ Type: string(hyperv1.AutoNodeEnabled), ObservedGeneration: hcluster.Generation, @@ -178,7 +184,7 @@ func (r *HostedClusterReconciler) reconcileAutoNodeEnabledCondition(ctx context. condition.Status = metav1.ConditionUnknown condition.Reason = hyperv1.AutoNodeEvaluationFailedReason condition.Message = fmt.Sprintf("failed to list ControlPlaneComponents: %v", err) - return condition + return condition, false } // Grab all of our karpenter components @@ -195,7 +201,7 @@ func (r *HostedClusterReconciler) reconcileAutoNodeEnabledCondition(ctx context. condition.Status = metav1.ConditionFalse condition.Reason = hyperv1.AutoNodeProgressingReason condition.Message = "AutoNode is being enabled: waiting for components to be created" - return condition + return condition, true } // Check if they're ready var notReady []string @@ -214,13 +220,13 @@ func (r *HostedClusterReconciler) reconcileAutoNodeEnabledCondition(ctx context. condition.Status = metav1.ConditionFalse condition.Reason = hyperv1.AutoNodeProgressingReason condition.Message = fmt.Sprintf("AutoNode is being enabled: %s", strings.Join(notReady, "; ")) - return condition + return condition, true } // Otherwise report ready condition.Status = metav1.ConditionTrue condition.Reason = hyperv1.AsExpectedReason condition.Message = "AutoNode is ready" - return condition + return condition, false } // Karpenter not enabled — check if Deployments are still terminating. @@ -236,7 +242,7 @@ func (r *HostedClusterReconciler) reconcileAutoNodeEnabledCondition(ctx context. condition.Status = metav1.ConditionUnknown condition.Reason = hyperv1.AutoNodeEvaluationFailedReason condition.Message = fmt.Sprintf("failed to check karpenter deployments: %v", err) - return condition + return condition, false } } @@ -244,11 +250,11 @@ func (r *HostedClusterReconciler) reconcileAutoNodeEnabledCondition(ctx context. condition.Status = metav1.ConditionFalse condition.Reason = hyperv1.AutoNodeProgressingReason condition.Message = fmt.Sprintf("AutoNode is being disabled: waiting for deployments to be removed: %s", strings.Join(runningDeployments, ", ")) - return condition + return condition, true } condition.Status = metav1.ConditionFalse condition.Reason = hyperv1.AutoNodeNotConfiguredReason condition.Message = "AutoNode provisioner is not configured" - return condition + return condition, false } diff --git a/hypershift-operator/controllers/hostedcluster/karpenter_test.go b/hypershift-operator/controllers/hostedcluster/karpenter_test.go index 530293370dce..5649e7b9a841 100644 --- a/hypershift-operator/controllers/hostedcluster/karpenter_test.go +++ b/hypershift-operator/controllers/hostedcluster/karpenter_test.go @@ -297,14 +297,16 @@ func TestReconcileAutoNodeEnabledCondition(t *testing.T) { } tests := map[string]struct { - autoNode hyperv1.AutoNode - components []hyperv1.ControlPlaneComponent - deployments []appsv1.Deployment - want metav1.Condition + autoNode hyperv1.AutoNode + components []hyperv1.ControlPlaneComponent + deployments []appsv1.Deployment + want metav1.Condition + wantProgessing bool }{ "When karpenter is enabled and components not yet created it should report progressing": { - autoNode: karpenterEnabledAutoNode, - components: nil, + autoNode: karpenterEnabledAutoNode, + components: nil, + wantProgessing: true, want: metav1.Condition{ Type: string(hyperv1.AutoNodeEnabled), Status: metav1.ConditionFalse, @@ -319,6 +321,7 @@ func TestReconcileAutoNodeEnabledCondition(t *testing.T) { Status: hyperv1.ControlPlaneComponentStatus{Conditions: []metav1.Condition{rolloutCompleteTrue}}, }, }, + wantProgessing: true, want: metav1.Condition{ Type: string(hyperv1.AutoNodeEnabled), Status: metav1.ConditionFalse, @@ -337,6 +340,7 @@ func TestReconcileAutoNodeEnabledCondition(t *testing.T) { Status: hyperv1.ControlPlaneComponentStatus{Conditions: []metav1.Condition{rolloutCompleteFalse}}, }, }, + wantProgessing: true, want: metav1.Condition{ Type: string(hyperv1.AutoNodeEnabled), Status: metav1.ConditionFalse, @@ -367,6 +371,7 @@ func TestReconcileAutoNodeEnabledCondition(t *testing.T) { {ObjectMeta: metav1.ObjectMeta{Name: karpenterv2.ComponentName, Namespace: hcpNamespace}}, {ObjectMeta: metav1.ObjectMeta{Name: karpenteroperatorv2.ComponentName, Namespace: hcpNamespace}}, }, + wantProgessing: true, want: metav1.Condition{ Type: string(hyperv1.AutoNodeEnabled), Status: metav1.ConditionFalse, @@ -378,6 +383,7 @@ func TestReconcileAutoNodeEnabledCondition(t *testing.T) { deployments: []appsv1.Deployment{ {ObjectMeta: metav1.ObjectMeta{Name: karpenterv2.ComponentName, Namespace: hcpNamespace}}, }, + wantProgessing: true, want: metav1.Condition{ Type: string(hyperv1.AutoNodeEnabled), Status: metav1.ConditionFalse, @@ -439,13 +445,16 @@ func TestReconcileAutoNodeEnabledCondition(t *testing.T) { }, } - got := r.reconcileAutoNodeEnabledCondition(context.Background(), hcluster, hcpNamespace) + got, progressing := r.reconcileAutoNodeEnabledCondition(context.Background(), hcluster, hcpNamespace) got.ObservedGeneration = 0 got.Message = "" got.LastTransitionTime = metav1.Time{} if !equality.Semantic.DeepEqual(tc.want, got) { - t.Errorf("expected %+v, got %+v", tc.want, got) + t.Errorf("condition: expected %+v, got %+v", tc.want, got) + } + if progressing != tc.wantProgessing { + t.Errorf("progressing: expected %v, got %v", tc.wantProgessing, progressing) } }) } diff --git a/hypershift-operator/controllers/nodepool/apiserver-haproxy/haproxy.go b/hypershift-operator/controllers/nodepool/apiserver-haproxy/haproxy.go index d868002c3dc4..df8ba33615e1 100644 --- a/hypershift-operator/controllers/nodepool/apiserver-haproxy/haproxy.go +++ b/hypershift-operator/controllers/nodepool/apiserver-haproxy/haproxy.go @@ -139,7 +139,7 @@ func (r *HAProxy) reconcileHAProxyIgnitionConfig(ctx context.Context, hcluster * if hcluster.Spec.Configuration != nil && hcluster.Spec.Configuration.Proxy != nil && hcluster.Spec.Configuration.Proxy.HTTPSProxy != "" && netutil.ConnectsThroughInternetToControlplane(hcluster.Spec.Platform) { apiserverProxy, err = joinDefaultPortIfMissing(hcluster.Spec.Configuration.Proxy.HTTPSProxy) if err != nil { - return "", fmt.Errorf("failed to parse .Spec.Configuration.Proxy.HTTPSProxy: %v", err) + return "", fmt.Errorf("failed to parse .Spec.Configuration.Proxy.HTTPSProxy: %w", err) } noProxy = hcluster.Spec.Configuration.Proxy.NoProxy } diff --git a/hypershift-operator/controllers/nodepool/apiserver-haproxy/haproxy_test.go b/hypershift-operator/controllers/nodepool/apiserver-haproxy/haproxy_test.go index 117e1e49a21a..44b32b29ba3b 100644 --- a/hypershift-operator/controllers/nodepool/apiserver-haproxy/haproxy_test.go +++ b/hypershift-operator/controllers/nodepool/apiserver-haproxy/haproxy_test.go @@ -5,6 +5,8 @@ import ( "strings" "testing" + . "github.com/onsi/gomega" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/api/util/ipnet" "github.com/openshift/hypershift/hypershift-operator/controllers/sharedingress" @@ -516,3 +518,47 @@ kind: Config` }) } } + +func TestJoinDefaultPortIfMissing(t *testing.T) { + t.Parallel() + tests := []struct { + name string + addr string + expected string + wantErr bool + }{ + { + name: "When HTTPS URL has no port it should add port 443", + addr: "https://proxy.example.com", + expected: "https://proxy.example.com:443", + }, + { + name: "When HTTP URL has no port it should add port 80", + addr: "http://proxy.example.com", + expected: "http://proxy.example.com:80", + }, + { + name: "When HTTPS URL already has a port it should keep existing port", + addr: "https://proxy.example.com:8443", + expected: "https://proxy.example.com:8443", + }, + { + name: "When URL has no scheme it should return an error", + addr: "proxy.example.com", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + result, err := joinDefaultPortIfMissing(tt.addr) + if tt.wantErr { + g.Expect(err).To(HaveOccurred()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(result).To(Equal(tt.expected)) + }) + } +} diff --git a/hypershift-operator/controllers/nodepool/aws_test.go b/hypershift-operator/controllers/nodepool/aws_test.go index 00763009a15d..608f3badc3cc 100644 --- a/hypershift-operator/controllers/nodepool/aws_test.go +++ b/hypershift-operator/controllers/nodepool/aws_test.go @@ -2,6 +2,7 @@ package nodepool import ( "encoding/json" + "errors" "strings" "testing" @@ -148,8 +149,8 @@ func TestAWSMachineTemplateSpec(t *testing.T) { name: "NotReady error is returned if no sg specified and no cluster sg is available", clusterStatus: &hyperv1.HostedClusterStatus{Platform: &hyperv1.PlatformStatus{AWS: &hyperv1.AWSPlatformStatus{DefaultWorkerSecurityGroupID: ""}}}, checkError: func(t *testing.T, err error) { - _, isNotReady := err.(*NotReadyError) - if err == nil || !isNotReady { + var notReadyErr *NotReadyError + if err == nil || !errors.As(err, ¬ReadyErr) { t.Errorf("did not get expected NotReady error") } }, @@ -1479,8 +1480,8 @@ func TestBuildAWSSecurityGroups(t *testing.T) { if tc.expectError { g.Expect(err).To(HaveOccurred()) if tc.expectNotReady { - _, isNotReady := err.(*NotReadyError) - g.Expect(isNotReady).To(BeTrue()) + var notReadyErr *NotReadyError + g.Expect(errors.As(err, ¬ReadyErr)).To(BeTrue()) } } else { g.Expect(err).ToNot(HaveOccurred()) diff --git a/hypershift-operator/controllers/nodepool/capi_test.go b/hypershift-operator/controllers/nodepool/capi_test.go index c57e4d2eec68..cafeaf179225 100644 --- a/hypershift-operator/controllers/nodepool/capi_test.go +++ b/hypershift-operator/controllers/nodepool/capi_test.go @@ -2943,6 +2943,291 @@ func TestPropagateLabelsAndTaintsToMachines(t *testing.T) { } } +// TestPauseUnpauseCycle is a regression test for the interaction between pausing +// NodePools and the Cluster Autoscaler (OCPBUGS-78152 / CNTRLPLANE-3040). +// +// It verifies that: +// - When a MachineDeployment/MachineSet is paused, the pause annotation is set +// - When reconcileMachineDeployment/reconcileMachineSet runs after unpause, the +// pause annotation is removed and autoscaler annotations are preserved +// - setMachineDeploymentReplicas clamps replicas within [min, max] bounds regardless +// of external modifications (e.g. CAS decrementing replicas while paused) +func TestPauseUnpauseCycle(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + upgradeType hyperv1.UpgradeType + platformType hyperv1.PlatformType + initialReplicas int32 + autoscalingMin int32 + autoscalingMax int32 + replicasAtUnpause int32 + expectedReplicas int32 + }{ + { + name: "When a paused MachineDeployment is unpaused it should remove the pause annotation and preserve replicas", + upgradeType: hyperv1.UpgradeTypeReplace, + platformType: hyperv1.AWSPlatform, + initialReplicas: 4, + autoscalingMin: 1, + autoscalingMax: 4, + replicasAtUnpause: 4, + expectedReplicas: 4, + }, + { + name: "When a paused MachineDeployment has replicas decremented to min it should keep replicas at min on unpause", + upgradeType: hyperv1.UpgradeTypeReplace, + platformType: hyperv1.AWSPlatform, + initialReplicas: 4, + autoscalingMin: 1, + autoscalingMax: 4, + replicasAtUnpause: 1, + expectedReplicas: 1, + }, + { + name: "When a paused MachineDeployment has replicas decremented below min it should clamp to min on unpause", + upgradeType: hyperv1.UpgradeTypeReplace, + platformType: hyperv1.AWSPlatform, + initialReplicas: 4, + autoscalingMin: 2, + autoscalingMax: 4, + replicasAtUnpause: 1, + expectedReplicas: 2, + }, + { + name: "When a paused MachineDeployment has replicas incremented above max it should clamp to max on unpause", + upgradeType: hyperv1.UpgradeTypeReplace, + platformType: hyperv1.AWSPlatform, + initialReplicas: 4, + autoscalingMin: 1, + autoscalingMax: 4, + replicasAtUnpause: 6, + expectedReplicas: 4, + }, + { + name: "When a paused MachineDeployment on AWS with min 0 it should allow scale to zero on unpause", + upgradeType: hyperv1.UpgradeTypeReplace, + platformType: hyperv1.AWSPlatform, + initialReplicas: 4, + autoscalingMin: 0, + autoscalingMax: 4, + replicasAtUnpause: 0, + expectedReplicas: 0, + }, + { + name: "When a paused MachineDeployment on non-AWS with min 0 it should clamp to effective min 1 on unpause", + upgradeType: hyperv1.UpgradeTypeReplace, + platformType: hyperv1.KubevirtPlatform, + initialReplicas: 4, + autoscalingMin: 0, + autoscalingMax: 4, + replicasAtUnpause: 0, + expectedReplicas: 1, + }, + { + name: "When a paused MachineSet is unpaused it should remove the pause annotation and preserve replicas", + upgradeType: hyperv1.UpgradeTypeInPlace, + platformType: hyperv1.AWSPlatform, + initialReplicas: 4, + autoscalingMin: 1, + autoscalingMax: 4, + replicasAtUnpause: 4, + expectedReplicas: 4, + }, + { + name: "When a paused MachineSet has replicas decremented below min it should clamp to min on unpause", + upgradeType: hyperv1.UpgradeTypeInPlace, + platformType: hyperv1.AWSPlatform, + initialReplicas: 4, + autoscalingMin: 2, + autoscalingMax: 4, + replicasAtUnpause: 1, + expectedReplicas: 2, + }, + { + name: "When a paused MachineSet has replicas incremented above max it should clamp to max on unpause", + upgradeType: hyperv1.UpgradeTypeInPlace, + platformType: hyperv1.AWSPlatform, + initialReplicas: 4, + autoscalingMin: 1, + autoscalingMax: 4, + replicasAtUnpause: 6, + expectedReplicas: 4, + }, + { + name: "When a paused MachineSet on AWS with min 0 it should allow scale to zero on unpause", + upgradeType: hyperv1.UpgradeTypeInPlace, + platformType: hyperv1.AWSPlatform, + initialReplicas: 4, + autoscalingMin: 0, + autoscalingMax: 4, + replicasAtUnpause: 0, + expectedReplicas: 0, + }, + { + name: "When a paused MachineSet on non-AWS with min 0 it should clamp to effective min 1 on unpause", + upgradeType: hyperv1.UpgradeTypeInPlace, + platformType: hyperv1.KubevirtPlatform, + initialReplicas: 4, + autoscalingMin: 0, + autoscalingMax: 4, + replicasAtUnpause: 0, + expectedReplicas: 1, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + + nodePool := &hyperv1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-nodepool", + Namespace: "test-namespace", + }, + Spec: hyperv1.NodePoolSpec{ + ClusterName: "test-cluster", + AutoScaling: &hyperv1.NodePoolAutoScaling{ + Min: ptr.To[int32](tc.autoscalingMin), + Max: tc.autoscalingMax, + }, + Management: hyperv1.NodePoolManagement{ + UpgradeType: tc.upgradeType, + Replace: &hyperv1.ReplaceUpgrade{ + Strategy: hyperv1.UpgradeStrategyRollingUpdate, + RollingUpdate: &hyperv1.RollingUpdate{ + MaxUnavailable: ptr.To(intstr.FromInt(0)), + MaxSurge: ptr.To(intstr.FromInt(1)), + }, + }, + }, + Platform: hyperv1.NodePoolPlatform{ + Type: tc.platformType, + }, + }, + } + if tc.platformType == hyperv1.AWSPlatform { + nodePool.Spec.Platform.AWS = &hyperv1.AWSNodePoolPlatform{AMI: "test-ami"} + } + + hostedCluster := &hyperv1.HostedCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "test-namespace", + }, + Spec: hyperv1.HostedClusterSpec{ + Platform: hyperv1.PlatformSpec{ + Type: tc.platformType, + }, + }, + } + if tc.platformType == hyperv1.AWSPlatform { + hostedCluster.Spec.Platform.AWS = &hyperv1.AWSPlatformSpec{} + } + + controlPlaneNamespace := "test-cp-namespace" + c := fake.NewClientBuilder().WithScheme(api.Scheme).Build() + capiObj := &CAPI{ + Token: &Token{ + ConfigGenerator: &ConfigGenerator{ + nodePool: nodePool, + hostedCluster: hostedCluster, + controlplaneNamespace: controlPlaneNamespace, + Client: c, + rolloutConfig: &rolloutConfig{ + releaseImage: &releaseinfo.ReleaseImage{ + ImageStream: &imageapi.ImageStream{ + ObjectMeta: metav1.ObjectMeta{ + Name: "target-version", + }, + }, + }, + }, + }, + cpoCapabilities: &CPOCapabilities{}, + CreateOrUpdateProvider: upsert.New(false), + }, + capiClusterName: "test-cluster", + } + + // Create MachineDeployment and MachineSet with initial replicas and autoscaler annotations. + md := capiObj.machineDeployment() + md.Spec.Replicas = ptr.To[int32](tc.initialReplicas) + md.Annotations = map[string]string{ + autoscalerMinAnnotation: fmt.Sprintf("%d", tc.autoscalingMin), + autoscalerMaxAnnotation: fmt.Sprintf("%d", tc.autoscalingMax), + } + g.Expect(c.Create(t.Context(), md)).To(Succeed()) + + ms := capiObj.machineSet() + ms.Spec.Replicas = ptr.To[int32](tc.initialReplicas) + ms.Annotations = map[string]string{ + autoscalerMinAnnotation: fmt.Sprintf("%d", tc.autoscalingMin), + autoscalerMaxAnnotation: fmt.Sprintf("%d", tc.autoscalingMax), + } + g.Expect(c.Create(t.Context(), ms)).To(Succeed()) + + // Step 1: Pause. + g.Expect(capiObj.Pause(t.Context())).To(Succeed()) + + g.Expect(c.Get(t.Context(), client.ObjectKeyFromObject(md), md)).To(Succeed()) + g.Expect(md.Annotations).To(HaveKeyWithValue(capiv1.PausedAnnotation, "true")) + + g.Expect(c.Get(t.Context(), client.ObjectKeyFromObject(ms), ms)).To(Succeed()) + g.Expect(ms.Annotations).To(HaveKeyWithValue(capiv1.PausedAnnotation, "true")) + + // Step 2: Simulate replica drift while paused (e.g. CAS decrementing via Scale subresource). + if tc.upgradeType == hyperv1.UpgradeTypeReplace { + md.Spec.Replicas = ptr.To[int32](tc.replicasAtUnpause) + g.Expect(c.Update(t.Context(), md)).To(Succeed()) + } else { + ms.Spec.Replicas = ptr.To[int32](tc.replicasAtUnpause) + g.Expect(c.Update(t.Context(), ms)).To(Succeed()) + } + + // Step 3: Unpause by calling the reconcile functions (simulates the NodePool controller + // reconciling after PausedUntil expires, which calls capi.Reconcile()). + log := ctrl.LoggerFrom(t.Context()) + template := &capiaws.AWSMachineTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-template", + Namespace: controlPlaneNamespace, + }, + } + + // For non-AWS platforms, effectiveMin is clamped to 1 when autoscalingMin is 0. + expectedMinAnnotation := tc.autoscalingMin + if tc.autoscalingMin == 0 && tc.platformType != hyperv1.AWSPlatform { + expectedMinAnnotation = 1 + } + + if tc.upgradeType == hyperv1.UpgradeTypeReplace { + g.Expect(c.Get(t.Context(), client.ObjectKeyFromObject(md), md)).To(Succeed()) + g.Expect(capiObj.reconcileMachineDeployment(t.Context(), log, md, template)).To(Succeed()) + + // Verify pause annotation removed. + g.Expect(md.Annotations).NotTo(HaveKey(capiv1.PausedAnnotation)) + // Verify replicas clamped within autoscaler bounds. + g.Expect(*md.Spec.Replicas).To(Equal(tc.expectedReplicas)) + // Verify autoscaler annotations preserved. + g.Expect(md.Annotations).To(HaveKeyWithValue(autoscalerMinAnnotation, fmt.Sprintf("%d", expectedMinAnnotation))) + g.Expect(md.Annotations).To(HaveKeyWithValue(autoscalerMaxAnnotation, fmt.Sprintf("%d", tc.autoscalingMax))) + } else { + g.Expect(c.Get(t.Context(), client.ObjectKeyFromObject(ms), ms)).To(Succeed()) + g.Expect(capiObj.reconcileMachineSet(t.Context(), ms, template)).To(Succeed()) + + // Verify pause annotation removed. + g.Expect(ms.Annotations).NotTo(HaveKey(capiv1.PausedAnnotation)) + // Verify replicas clamped within autoscaler bounds. + g.Expect(*ms.Spec.Replicas).To(Equal(tc.expectedReplicas)) + // Verify autoscaler annotations preserved. + g.Expect(ms.Annotations).To(HaveKeyWithValue(autoscalerMinAnnotation, fmt.Sprintf("%d", expectedMinAnnotation))) + g.Expect(ms.Annotations).To(HaveKeyWithValue(autoscalerMaxAnnotation, fmt.Sprintf("%d", tc.autoscalingMax))) + } + }) + } +} + func TestNewCAPI(t *testing.T) { t.Parallel() testCases := []struct { diff --git a/hypershift-operator/controllers/nodepool/conditions.go b/hypershift-operator/controllers/nodepool/conditions.go index 735a21deb9c5..81eb714474a2 100644 --- a/hypershift-operator/controllers/nodepool/conditions.go +++ b/hypershift-operator/controllers/nodepool/conditions.go @@ -1002,7 +1002,7 @@ func (r *NodePoolReconciler) supportedVersionSkewCondition(ctx context.Context, Message: err.Error(), ObservedGeneration: nodePool.Generation, }) - return nil, nil + return nil, nil //nolint:nilerr // validation error is surfaced via status condition, not returned } SetStatusCondition(&nodePool.Status.Conditions, hyperv1.NodePoolCondition{ Type: hyperv1.NodePoolSupportedVersionSkewConditionType, diff --git a/hypershift-operator/controllers/nodepool/config.go b/hypershift-operator/controllers/nodepool/config.go index 5787a6cdf199..8771f45ea83c 100644 --- a/hypershift-operator/controllers/nodepool/config.go +++ b/hypershift-operator/controllers/nodepool/config.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + coreerrors "errors" "fmt" "io" "sort" @@ -238,7 +239,7 @@ func (cg *ConfigGenerator) parse(configs []corev1.ConfigMap) (string, error) { yamlReader := yaml.NewYAMLReader(bufio.NewReader(strings.NewReader(cmPayload))) for { manifestRaw, err := yamlReader.Read() - if err != nil && err != io.EOF { + if err != nil && !coreerrors.Is(err, io.EOF) { errors = append(errors, fmt.Errorf("configmap %q contains invalid yaml: %w", config.Name, err)) continue } @@ -250,7 +251,7 @@ func (cg *ConfigGenerator) parse(configs []corev1.ConfigMap) (string, error) { } allConfigPlainText = append(allConfigPlainText, string(manifest)) } - if err == io.EOF { + if coreerrors.Is(err, io.EOF) { break } } diff --git a/hypershift-operator/controllers/nodepool/nodepool_controller.go b/hypershift-operator/controllers/nodepool/nodepool_controller.go index 9cd2cc3c484e..4700d89e1a32 100644 --- a/hypershift-operator/controllers/nodepool/nodepool_controller.go +++ b/hypershift-operator/controllers/nodepool/nodepool_controller.go @@ -2,6 +2,7 @@ package nodepool import ( "context" + coreerrors "errors" "fmt" "os" "regexp" @@ -414,7 +415,8 @@ func (r *NodePoolReconciler) reconcile(ctx context.Context, hcluster *hyperv1.Ho } if err := capi.Reconcile(ctx); err != nil { - if _, isNotReady := err.(*NotReadyError); isNotReady { + var notReadyErr *NotReadyError + if coreerrors.As(err, ¬ReadyErr) { log.Info("Waiting to create machine template", "message", err.Error()) return ctrl.Result{RequeueAfter: 5 * time.Second}, nil } diff --git a/hypershift-operator/controllers/nodepool/nto.go b/hypershift-operator/controllers/nodepool/nto.go index bc98b4e6dde4..2fc5e65a4d0d 100644 --- a/hypershift-operator/controllers/nodepool/nto.go +++ b/hypershift-operator/controllers/nodepool/nto.go @@ -412,7 +412,7 @@ func BuildMirrorConfigs(ctx context.Context, cg *ConfigGenerator) ([]*MirrorConf yamlReader := yaml.NewYAMLReader(bufio.NewReader(strings.NewReader(cmPayload))) for { manifestRaw, err := yamlReader.Read() - if err != nil && err != io.EOF { + if err != nil && !coreerrors.Is(err, io.EOF) { errors = append(errors, fmt.Errorf("configmap %q contains invalid yaml: %w", config.Name, err)) continue } @@ -427,7 +427,7 @@ func BuildMirrorConfigs(ctx context.Context, cg *ConfigGenerator) ([]*MirrorConf mirrorConfigs = append(mirrorConfigs, mirrorConfig) } } - if err == io.EOF { + if coreerrors.Is(err, io.EOF) { break } } diff --git a/hypershift-operator/controllers/platform/aws/controller.go b/hypershift-operator/controllers/platform/aws/controller.go index e7779ba5a572..80e82a6576f7 100644 --- a/hypershift-operator/controllers/platform/aws/controller.go +++ b/hypershift-operator/controllers/platform/aws/controller.go @@ -892,7 +892,7 @@ func (r *AWSEndpointServiceReconciler) controlPlaneOperatorRoleARNWithoutPath(hc } arn, err := arn.Parse(hc.Spec.Platform.AWS.RolesRef.ControlPlaneOperatorARN) if err != nil { - return "", fmt.Errorf("failed to parse %s into an ARN: %v", hc.Spec.Platform.AWS.RolesRef.ControlPlaneOperatorARN, err) + return "", fmt.Errorf("failed to parse %s into an ARN: %w", hc.Spec.Platform.AWS.RolesRef.ControlPlaneOperatorARN, err) } // IAM names cannot have a "/" while path names are the only way to get "/" into the name diff --git a/hypershift-operator/controllers/platform/gcp/privateserviceconnect_controller.go b/hypershift-operator/controllers/platform/gcp/privateserviceconnect_controller.go index da9321e9d179..127c6caf781e 100644 --- a/hypershift-operator/controllers/platform/gcp/privateserviceconnect_controller.go +++ b/hypershift-operator/controllers/platform/gcp/privateserviceconnect_controller.go @@ -2,6 +2,7 @@ package gcp import ( "context" + "errors" "fmt" "os" "strings" @@ -470,7 +471,8 @@ func (r *GCPPrivateServiceConnectReconciler) handleGCPError(ctx context.Context, var requeueAfter time.Duration var message string - if googleErr, ok := err.(*googleapi.Error); ok { + var googleErr *googleapi.Error + if errors.As(err, &googleErr) { switch googleErr.Code { case 429: // Rate limit requeueAfter = time.Minute * 5 @@ -513,7 +515,8 @@ func (r *GCPPrivateServiceConnectReconciler) handleGCPError(ctx context.Context, // isNotFoundError checks if an error is a GCP 404 Not Found error func isNotFoundError(err error) bool { - if googleErr, ok := err.(*googleapi.Error); ok { + var googleErr *googleapi.Error + if errors.As(err, &googleErr) { return googleErr.Code == 404 } return false diff --git a/hypershift-operator/controllers/platform/gcp/privateserviceconnect_controller_test.go b/hypershift-operator/controllers/platform/gcp/privateserviceconnect_controller_test.go index 37bf945faccb..916374b11ebf 100644 --- a/hypershift-operator/controllers/platform/gcp/privateserviceconnect_controller_test.go +++ b/hypershift-operator/controllers/platform/gcp/privateserviceconnect_controller_test.go @@ -53,6 +53,16 @@ func TestIsNotFoundError(t *testing.T) { err: nil, expected: false, }, + { + name: "When given a wrapped GCP 404 error it should return true", + err: fmt.Errorf("operation failed: %w", &googleapi.Error{Code: 404}), + expected: true, + }, + { + name: "When given a wrapped GCP 500 error it should return false", + err: fmt.Errorf("operation failed: %w", &googleapi.Error{Code: 500}), + expected: false, + }, } for _, test := range tests { diff --git a/hypershift-operator/controllers/uwmtelemetry/uwm_telemetry.go b/hypershift-operator/controllers/uwmtelemetry/uwm_telemetry.go index e47c2b18e2d1..53e68f6d552c 100644 --- a/hypershift-operator/controllers/uwmtelemetry/uwm_telemetry.go +++ b/hypershift-operator/controllers/uwmtelemetry/uwm_telemetry.go @@ -103,8 +103,11 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu telemeterClientSecret := monitoring.TelemeterClientSecret() if err := r.Get(ctx, client.ObjectKeyFromObject(telemeterClientSecret), telemeterClientSecret); err != nil { - log.Info("user-workload-monitoring (UWM) telemetry remote writer is disabled because the 'telemeter-client' secret does not exist.") - return ctrl.Result{}, nil + if apierrors.IsNotFound(err) { + log.Info("user-workload-monitoring (UWM) telemetry remote writer is disabled because the 'telemeter-client' secret does not exist.") + return ctrl.Result{}, nil //nolint:nilerr // missing secret means UWM telemetry is disabled + } + return ctrl.Result{}, r.errorHandler(operatorDeployment, fmt.Errorf("failed to get telemeter-client secret: %w", err)) } if err := r.reconcileTelemetryRemoteWrite(ctx, string(clusterVersion.Spec.ClusterID)); err != nil { diff --git a/hypershift-operator/controllers/uwmtelemetry/uwm_telemetry_test.go b/hypershift-operator/controllers/uwmtelemetry/uwm_telemetry_test.go index 829be27d6b62..0d4be563b77a 100644 --- a/hypershift-operator/controllers/uwmtelemetry/uwm_telemetry_test.go +++ b/hypershift-operator/controllers/uwmtelemetry/uwm_telemetry_test.go @@ -1,8 +1,10 @@ package uwmtelemetry import ( + "context" "encoding/base64" "encoding/json" + "fmt" "testing" . "github.com/onsi/gomega" @@ -14,12 +16,14 @@ import ( configv1 "github.com/openshift/api/config/v1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "sigs.k8s.io/yaml" ) @@ -424,4 +428,133 @@ func TestReconcile(t *testing.T) { test.validate(g, c) }) } + + t.Run("When telemeter-client secret Get fails with a non-NotFound error it should return the error", func(t *testing.T) { + g := NewWithT(t) + ns := "hypershift" + deployment := manifests.OperatorDeployment(ns) + cv := monitoring.ClusterVersion() + cv.Spec.ClusterID = "fake-cluster-id" + + c := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects( + deployment, + monitoring.MonitoringNamespace(), + monitoring.UWMNamespace(), + cv, + ). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Secret); ok && key.Name == "telemeter-client" { + return fmt.Errorf("API server unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }). + Build() + + reconciler := &Reconciler{ + Client: c, + CreateOrUpdateProvider: upsert.New(true), + errorHandler: func(obj client.Object, err error) error { return err }, + Namespace: ns, + } + req := ctrl.Request{NamespacedName: client.ObjectKey{Name: "operator", Namespace: ns}} + _, err := reconciler.Reconcile(t.Context(), req) + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(ContainSubstring("failed to get telemeter-client secret"))) + }) + + t.Run("When operator deployment Get fails it should return the error", func(t *testing.T) { + g := NewWithT(t) + ns := "hypershift" + + c := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*appsv1.Deployment); ok && key.Name == "operator" { + return fmt.Errorf("connection refused") + } + return c.Get(ctx, key, obj, opts...) + }, + }). + Build() + + reconciler := &Reconciler{ + Client: c, + CreateOrUpdateProvider: upsert.New(true), + errorHandler: func(obj client.Object, err error) error { return err }, + Namespace: ns, + } + req := ctrl.Request{NamespacedName: client.ObjectKey{Name: "operator", Namespace: ns}} + _, err := reconciler.Reconcile(t.Context(), req) + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(ContainSubstring("cannot get operator deployment"))) + }) + + t.Run("When monitoring namespace Get fails with a non-NotFound error it should return the error", func(t *testing.T) { + g := NewWithT(t) + ns := "hypershift" + deployment := manifests.OperatorDeployment(ns) + + c := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects(deployment). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Namespace); ok && key.Name == "openshift-monitoring" { + return fmt.Errorf("forbidden") + } + return c.Get(ctx, key, obj, opts...) + }, + }). + Build() + + reconciler := &Reconciler{ + Client: c, + CreateOrUpdateProvider: upsert.New(true), + errorHandler: func(obj client.Object, err error) error { return err }, + Namespace: ns, + } + req := ctrl.Request{NamespacedName: client.ObjectKey{Name: "operator", Namespace: ns}} + _, err := reconciler.Reconcile(t.Context(), req) + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(ContainSubstring("failed to get monitoring namespace"))) + }) + + t.Run("When clusterversion Get fails it should return the error", func(t *testing.T) { + g := NewWithT(t) + ns := "hypershift" + deployment := manifests.OperatorDeployment(ns) + + c := fake.NewClientBuilder(). + WithScheme(api.Scheme). + WithObjects( + deployment, + monitoring.MonitoringNamespace(), + monitoring.UWMNamespace(), + ). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*configv1.ClusterVersion); ok { + return fmt.Errorf("API server unavailable") + } + return c.Get(ctx, key, obj, opts...) + }, + }). + Build() + + reconciler := &Reconciler{ + Client: c, + CreateOrUpdateProvider: upsert.New(true), + errorHandler: func(obj client.Object, err error) error { return err }, + Namespace: ns, + } + req := ctrl.Request{NamespacedName: client.ObjectKey{Name: "operator", Namespace: ns}} + _, err := reconciler.Reconcile(t.Context(), req) + g.Expect(err).To(HaveOccurred()) + g.Expect(err).To(MatchError(ContainSubstring("failed to get clusterversion resource"))) + }) } diff --git a/hypershift-operator/controllers/webhookcerts/webhookcerts_controller.go b/hypershift-operator/controllers/webhookcerts/webhookcerts_controller.go index b2d3c7652b4a..79852656eec4 100644 --- a/hypershift-operator/controllers/webhookcerts/webhookcerts_controller.go +++ b/hypershift-operator/controllers/webhookcerts/webhookcerts_controller.go @@ -33,6 +33,7 @@ import ( ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/predicate" "github.com/go-logr/logr" @@ -364,3 +365,60 @@ func GenerateInitialWebhookCerts(namespace, serviceName string) (*corev1.Secret, caBundle := caSecret.Data[certs.CASignerCertMapKey] return caSecret, servingSecret, caBundle, nil } + +// EnsureWebhookCerts ensures that webhook cert secrets exist so the webhook +// server can start. If the serving cert secret already exists with valid data, +// this is a no-op (the volume mount handles file delivery). If the secret is +// missing or has empty data, new certs are generated and persisted as secrets. +func EnsureWebhookCerts(ctx context.Context, c client.Client, namespace, serviceName string) error { + log := ctrl.LoggerFrom(ctx).WithName("webhook-cert-bootstrap") + + if certsExist(ctx, c, namespace) { + log.Info("Webhook cert secrets already exist with valid data, skipping bootstrap") + return nil + } + + log.Info("Generating webhook certificates") + caSecret, servingSecret, _, err := GenerateInitialWebhookCerts(namespace, serviceName) + if err != nil { + return fmt.Errorf("failed to generate webhook certs: %w", err) + } + + caObj := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: CASecretName, Namespace: namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, c, caObj, func() error { + caObj.Type = corev1.SecretTypeOpaque + caObj.Data = caSecret.Data + return nil + }); err != nil { + return fmt.Errorf("failed to create or update CA secret: %w", err) + } + + servingObj := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: ServingCertSecretName, Namespace: namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, c, servingObj, func() error { + servingObj.Type = corev1.SecretTypeTLS + servingObj.Data = servingSecret.Data + return nil + }); err != nil { + return fmt.Errorf("failed to create or update serving cert secret: %w", err) + } + + log.Info("Webhook certificates bootstrapped") + return nil +} + +// certsExist returns true when both the CA and serving cert secrets exist with non-empty data. +func certsExist(ctx context.Context, c client.Client, namespace string) bool { + ca := &corev1.Secret{} + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: CASecretName}, ca); err != nil { + return false + } + if len(ca.Data[certs.CASignerCertMapKey]) == 0 || len(ca.Data[certs.CASignerKeyMapKey]) == 0 { + return false + } + + serving := &corev1.Secret{} + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: ServingCertSecretName}, serving); err != nil { + return false + } + return len(serving.Data[corev1.TLSCertKey]) > 0 && len(serving.Data[corev1.TLSPrivateKeyKey]) > 0 +} diff --git a/hypershift-operator/controllers/webhookcerts/webhookcerts_controller_test.go b/hypershift-operator/controllers/webhookcerts/webhookcerts_controller_test.go index ec96cf3dd5b1..c125b5887a7d 100644 --- a/hypershift-operator/controllers/webhookcerts/webhookcerts_controller_test.go +++ b/hypershift-operator/controllers/webhookcerts/webhookcerts_controller_test.go @@ -2,6 +2,8 @@ package webhookcerts import ( "context" + "crypto/x509" + "encoding/pem" "testing" "time" @@ -406,7 +408,110 @@ func TestGenerateInitialWebhookCerts(t *testing.T) { g.Expect(servingSecret.Type).To(Equal(corev1.SecretTypeTLS)) g.Expect(servingSecret.Data).To(HaveKey(corev1.TLSCertKey)) g.Expect(servingSecret.Data).To(HaveKey(corev1.TLSPrivateKeyKey)) + }) +} + +func TestEnsureWebhookCerts(t *testing.T) { + t.Run("When no secrets exist it should create secrets", func(t *testing.T) { + g := NewWithT(t) + + cl := fake.NewClientBuilder().WithScheme(newScheme(t)).Build() + + err := EnsureWebhookCerts(t.Context(), cl, "hypershift", "operator") + g.Expect(err).ToNot(HaveOccurred()) + + caSecret := &corev1.Secret{} + g.Expect(cl.Get(t.Context(), client.ObjectKey{Name: CASecretName, Namespace: "hypershift"}, caSecret)).To(Succeed()) + g.Expect(caSecret.Data).To(HaveKey(certs.CASignerCertMapKey)) + + servingSecret := &corev1.Secret{} + g.Expect(cl.Get(t.Context(), client.ObjectKey{Name: ServingCertSecretName, Namespace: "hypershift"}, servingSecret)).To(Succeed()) + g.Expect(servingSecret.Data).To(HaveKey(corev1.TLSCertKey)) + g.Expect(servingSecret.Data).To(HaveKey(corev1.TLSPrivateKeyKey)) + g.Expect(servingSecret.Type).To(Equal(corev1.SecretTypeTLS)) + }) + t.Run("When secrets already exist with valid data it should not modify them", func(t *testing.T) { + g := NewWithT(t) + + caSecret, servingSecret, _, err := GenerateInitialWebhookCerts("hypershift", "operator") + g.Expect(err).ToNot(HaveOccurred()) + + cl := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(caSecret, servingSecret).Build() + + err = EnsureWebhookCerts(t.Context(), cl, "hypershift", "operator") + g.Expect(err).ToNot(HaveOccurred()) + + updatedServingSecret := &corev1.Secret{} + g.Expect(cl.Get(t.Context(), client.ObjectKey{Name: ServingCertSecretName, Namespace: "hypershift"}, updatedServingSecret)).To(Succeed()) + g.Expect(updatedServingSecret.Data[corev1.TLSCertKey]).To(Equal(servingSecret.Data[corev1.TLSCertKey])) + g.Expect(updatedServingSecret.Data[corev1.TLSPrivateKeyKey]).To(Equal(servingSecret.Data[corev1.TLSPrivateKeyKey])) + + updatedCASecret := &corev1.Secret{} + g.Expect(cl.Get(t.Context(), client.ObjectKey{Name: CASecretName, Namespace: "hypershift"}, updatedCASecret)).To(Succeed()) + g.Expect(updatedCASecret.Data[certs.CASignerCertMapKey]).To(Equal(caSecret.Data[certs.CASignerCertMapKey])) + g.Expect(updatedCASecret.Data[certs.CASignerKeyMapKey]).To(Equal(caSecret.Data[certs.CASignerKeyMapKey])) + }) + + t.Run("When serving cert secret exists with empty data it should regenerate and update", func(t *testing.T) { + g := NewWithT(t) + + emptySecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: ServingCertSecretName, + Namespace: "hypershift", + }, + Type: corev1.SecretTypeTLS, + Data: map[string][]byte{}, + } + + cl := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(emptySecret).Build() + + err := EnsureWebhookCerts(t.Context(), cl, "hypershift", "operator") + g.Expect(err).ToNot(HaveOccurred()) + + updatedServingSecret := &corev1.Secret{} + g.Expect(cl.Get(t.Context(), client.ObjectKey{Name: ServingCertSecretName, Namespace: "hypershift"}, updatedServingSecret)).To(Succeed()) + g.Expect(updatedServingSecret.Data[corev1.TLSCertKey]).ToNot(BeEmpty()) + g.Expect(updatedServingSecret.Data[corev1.TLSPrivateKeyKey]).ToNot(BeEmpty()) + + updatedCASecret := &corev1.Secret{} + g.Expect(cl.Get(t.Context(), client.ObjectKey{Name: CASecretName, Namespace: "hypershift"}, updatedCASecret)).To(Succeed()) + g.Expect(updatedCASecret.Data[certs.CASignerCertMapKey]).ToNot(BeEmpty()) + g.Expect(updatedCASecret.Data[certs.CASignerKeyMapKey]).ToNot(BeEmpty()) + }) + + t.Run("When CA exists but serving cert is missing it should regenerate both secrets", func(t *testing.T) { + g := NewWithT(t) + + caSecret, _, _, err := GenerateInitialWebhookCerts("hypershift", "operator") + g.Expect(err).ToNot(HaveOccurred()) + + cl := fake.NewClientBuilder().WithScheme(newScheme(t)).WithObjects(caSecret).Build() + + err = EnsureWebhookCerts(t.Context(), cl, "hypershift", "operator") + g.Expect(err).ToNot(HaveOccurred()) + + // Both secrets should exist with valid data. + updatedCASecret := &corev1.Secret{} + g.Expect(cl.Get(t.Context(), client.ObjectKey{Name: CASecretName, Namespace: "hypershift"}, updatedCASecret)).To(Succeed()) + g.Expect(updatedCASecret.Data[certs.CASignerCertMapKey]).ToNot(BeEmpty()) + g.Expect(updatedCASecret.Data[certs.CASignerKeyMapKey]).ToNot(BeEmpty()) + + servingSecret := &corev1.Secret{} + g.Expect(cl.Get(t.Context(), client.ObjectKey{Name: ServingCertSecretName, Namespace: "hypershift"}, servingSecret)).To(Succeed()) + g.Expect(servingSecret.Data[corev1.TLSCertKey]).ToNot(BeEmpty()) + g.Expect(servingSecret.Data[corev1.TLSPrivateKeyKey]).ToNot(BeEmpty()) + + // Verify the serving cert was signed by the (possibly replaced) CA. + caPool := x509.NewCertPool() + g.Expect(caPool.AppendCertsFromPEM(updatedCASecret.Data[certs.CASignerCertMapKey])).To(BeTrue()) + block, _ := pem.Decode(servingSecret.Data[corev1.TLSCertKey]) + g.Expect(block).ToNot(BeNil()) + leaf, err := x509.ParseCertificate(block.Bytes) + g.Expect(err).ToNot(HaveOccurred()) + _, err = leaf.Verify(x509.VerifyOptions{Roots: caPool}) + g.Expect(err).ToNot(HaveOccurred()) }) } diff --git a/hypershift-operator/controlplaneoperator-overrides/assets/overrides.yaml b/hypershift-operator/controlplaneoperator-overrides/assets/overrides.yaml index cf8e4a2099eb..e4d1ede11017 100644 --- a/hypershift-operator/controlplaneoperator-overrides/assets/overrides.yaml +++ b/hypershift-operator/controlplaneoperator-overrides/assets/overrides.yaml @@ -9,40 +9,100 @@ platforms: cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-19@sha256:88c55ea554f7e62a64e34ff8d3be45ef85ef6b80fe4e9b0240b9a1aa226f9d98 - version: 4.19.10 cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-19@sha256:88c55ea554f7e62a64e34ff8d3be45ef85ef6b80fe4e9b0240b9a1aa226f9d98 + # Beginning of OCPBUGS-86567 overrides 4.20 section - version: 4.20.0 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.1 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.2 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.3 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.4 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.5 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.6 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb + - version: 4.20.7 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.8 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.9 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.10 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.11 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.12 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.13 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.14 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.15 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.16 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb - version: 4.20.17 - cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:5bbbce615fb2103b900b9eadf79abae0c23b3ea14f8a5d46d3ff63879ded4058 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb + - version: 4.20.18 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb + - version: 4.20.19 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb + - version: 4.20.20 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb + - version: 4.20.21 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb + - version: 4.20.22 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb + - version: 4.20.23 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb + - version: 4.20.24 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-20@sha256:155e4eee5b551ba9c4f3690e677be2b7a9bf90dca0ba4e8978a39fc16b46bcbb + # End of OCPBUGS-86567 overrides 4.20 section + # Beginning of OCPBUGS-86416 overrides 4.21 section + - version: 4.21.0 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.1 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.2 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.3 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.4 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.5 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.6 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.7 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.8 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.9 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.10 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.11 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.12 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.13 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.14 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.15 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.16 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.17 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + - version: 4.21.18 + cpoImage: quay.io/redhat-user-workloads/crt-redhat-acm-tenant/control-plane-operator-4-21@sha256:1b3f1bf728e722d333f7214afb348b9d629c2134247c5440d7d98a87c34c6f61 + # End of OCPBUGS-86416 overrides 4.21 section + # 4.22 does not need an override: the fix (PR #8564) landed before rc.5 + # (SOURCE_GIT_COMMIT=d6c72d15350752e315c198f7a68558ae4086e3c7) and GA will include it. testing: # Update the image refs below to indicate which images should be used for CPO override # testing. Currently, we only test one latest/previous combination. In the future, we diff --git a/hypershift-operator/main.go b/hypershift-operator/main.go index 11e552f0a273..a663242bf0f8 100644 --- a/hypershift-operator/main.go +++ b/hypershift-operator/main.go @@ -24,6 +24,7 @@ import ( hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" awsutil "github.com/openshift/hypershift/cmd/infra/aws/util" + "github.com/openshift/hypershift/cmd/install/assets" pkiconfig "github.com/openshift/hypershift/control-plane-pki-operator/config" etcdrecovery "github.com/openshift/hypershift/etcd-recovery" "github.com/openshift/hypershift/hypershift-operator/controllers/auditlogpersistence" @@ -274,6 +275,12 @@ func run(ctx context.Context, opts *StartOptions, log logr.Logger) error { return fmt.Errorf("failed to construct api reading client: %w", err) } + if opts.CertDir != "" { + if err := webhookcerts.EnsureWebhookCerts(ctx, apiReadingClient, opts.Namespace, assets.HypershiftOperatorName); err != nil { + return fmt.Errorf("failed to bootstrap webhook certs: %w", err) + } + } + if err := reconcileDeprecationValidatingAdmissionPolicy(ctx, apiReadingClient, mgmtClusterCaps, log); err != nil { return fmt.Errorf("failed to reconcile deprecation ValidatingAdmissionPolicy: %w", err) } diff --git a/ignition-server/cmd/start.go b/ignition-server/cmd/start.go index b816f58ca9c6..3ee07a28ca96 100644 --- a/ignition-server/cmd/start.go +++ b/ignition-server/cmd/start.go @@ -4,6 +4,7 @@ import ( "context" "crypto/tls" "encoding/base64" + "errors" "fmt" "log" "net/http" @@ -316,7 +317,7 @@ func run(ctx context.Context, opts Options) error { }() log.Printf("Listening on %s", opts.Addr) - if err := server.ListenAndServeTLS("", ""); err != nil && err != http.ErrServerClosed { + if err := server.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) { return err } return nil diff --git a/ignition-server/controllers/cache.go b/ignition-server/controllers/cache.go index 91e4f1ec6f50..ba3ef0567b92 100644 --- a/ignition-server/controllers/cache.go +++ b/ignition-server/controllers/cache.go @@ -59,7 +59,7 @@ func (c *ExpiringCache) Set(key string, value CacheValue) { c.Lock() defer c.Unlock() - // Renew expiring time every time time we Set. + // Renew expiring time every time we Set. c.cache[key] = &entry{ value: value, expiry: time.Now().Add(c.ttl), diff --git a/ignition-server/controllers/local_ignitionprovider.go b/ignition-server/controllers/local_ignitionprovider.go index 7a17217734d7..dcc8366454ff 100644 --- a/ignition-server/controllers/local_ignitionprovider.go +++ b/ignition-server/controllers/local_ignitionprovider.go @@ -571,7 +571,7 @@ func (p *LocalIgnitionProvider) runMCSAndFetchPayload(ctx context.Context, dirs var payload []byte // Try connecting to the server until we get a response or the context is closed err = wait.PollUntilContextCancel(ctx, 1*time.Second, true, func(ctx context.Context) (bool, error) { - req, err := http.NewRequestWithContext(ctx, "GET", "http://localhost:22626/config/master", nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:22626/config/master", nil) if err != nil { return false, fmt.Errorf("error building http request: %w", err) } @@ -652,7 +652,7 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage, cu return imageprovider.New(img), nil }() if err != nil { - return nil, fmt.Errorf("failed to get component images: %v", err) + return nil, fmt.Errorf("failed to get component images: %w", err) } mcoImage, err := p.resolveMCOImage(ctx, imageProvider, pullSecret) @@ -765,6 +765,7 @@ func (r *LocalIgnitionProvider) reconcileValidReleaseInfoCondition(ctx context.C } hostedControlPlane := hcpList.Items[0] + originalHCP := hostedControlPlane.DeepCopy() if len(releaseImageProvider.GetMissingImages()) == 0 { meta.SetStatusCondition(&hostedControlPlane.Status.Conditions, metav1.Condition{ @@ -784,7 +785,7 @@ func (r *LocalIgnitionProvider) reconcileValidReleaseInfoCondition(ctx context.C }) } - return r.Client.Status().Update(ctx, &hostedControlPlane) + return r.Client.Status().Patch(ctx, &hostedControlPlane, client.MergeFromWithOptions(originalHCP, client.MergeFromWithOptimisticLock{})) } // copyFile copies a file named src to dst, preserving attributes. diff --git a/ignition-server/controllers/tokensecret_controller.go b/ignition-server/controllers/tokensecret_controller.go index 18332ff88c0c..080421ef3300 100644 --- a/ignition-server/controllers/tokensecret_controller.go +++ b/ignition-server/controllers/tokensecret_controller.go @@ -275,7 +275,7 @@ func (r *TokenSecretReconciler) Reconcile(ctx context.Context, req ctrl.Request) start := time.Now() payload, err := r.IgnitionProvider.GetPayload(ctx, releaseImage, config.String(), pullSecretHash, additionalTrustBundleHash, hcConfigurationHash) if err != nil { - return nil, fmt.Errorf("error getting ignition payload: %v", err) + return nil, fmt.Errorf("error getting ignition payload: %w", err) } duration := time.Since(start).Round(time.Second).Seconds() log.Info("got ignition payload", "duration", duration) diff --git a/karpenter-operator/controllers/karpenter/karpenter_controller.go b/karpenter-operator/controllers/karpenter/karpenter_controller.go index 9aeb438a9963..475d8d5e00a4 100644 --- a/karpenter-operator/controllers/karpenter/karpenter_controller.go +++ b/karpenter-operator/controllers/karpenter/karpenter_controller.go @@ -33,6 +33,7 @@ import ( utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/utils/ptr" + capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/cluster" @@ -137,6 +138,21 @@ func (r *Reconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, man return fmt.Errorf("failed to watch HostedControlPlane: %w", err) } + // Watch the CAPI Cluster management side. The CAPI Cluster is deleted earlier + // than the HCP in the HostedCluster deletion sequence, so reacting to it lets + // us start Karpenter node cleanup in parallel with regular CAPI node teardown + // rather than waiting for the HCP DeletionTimestamp (which is set much later). + if err := c.Watch(source.Kind[client.Object](managementCluster.GetCache(), &capiv1.Cluster{}, handler.EnqueueRequestsFromMapFunc( + func(ctx context.Context, o client.Object) []ctrl.Request { + if o.GetNamespace() != r.Namespace { + return nil + } + return []ctrl.Request{{NamespacedName: client.ObjectKey{Namespace: r.Namespace}}} + }, + ), namespacedPredicates)); err != nil { + return fmt.Errorf("failed to watch CAPI Cluster: %w", err) + } + // Only enqueue on Add/Delete — not status-only updates — since reconcileAutoNodeStatus cares only // about count changes (nodes joining or leaving the cluster). countChangePredicate := predicate.Funcs{ @@ -213,77 +229,12 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu return ctrl.Result{}, err } - if hcp.DeletionTimestamp != nil { - // TODO(maxcao13): if supporting disablement, we don't want to force delete immediately. - // When force=true, we skip the graceful timeout and immediately trigger forceful deletion. - // When force=false, we wait for NodeClaimDeletionTimeout before triggering forceful deletion. - force := true - if controllerutil.ContainsFinalizer(hcp, karpenterutil.KarpenterFinalizer) { - // The deletion flow is: - // 1. Delete all NodePools (NodeClaims will be marked for deletion from deleting the NodePools due to ownerReferences) - // 2. Make sure all NodeClaims are actually gone (gracefully first, unless force=true) - // 3. If graceful timeout or force=true, set the termination timestamp annotation to trigger Karpenter's forceful deletion - // 4. Remove the finalizer from the HostedControlPlane to allow the rest of the HCP deletion to complete - - // Karpenter itself will make sure Nodes objects are deleted (and underlying instances are terminated) before finalizing the NodeClaims - nodePoolList := &karpenterv1.NodePoolList{} - if err := r.GuestClient.List(ctx, nodePoolList); err != nil { - return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, fmt.Errorf("failed to list NodePools: %w", err) - } - - // Delete all NodePools first - if len(nodePoolList.Items) > 0 { - for _, nodePool := range nodePoolList.Items { - // If we still get the NodePool, but it's already marked as terminating, we don't need to call Delete again - if !nodePool.GetDeletionTimestamp().IsZero() { - continue - } - if err := r.GuestClient.Delete(ctx, &nodePool, &client.DeleteOptions{ - GracePeriodSeconds: ptr.To(int64(0)), - }); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to delete NodePool: %w", err) - } - } - return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, nil - } - - // Make sure all NodeClaims are actually gone (gracefully first) - nodeClaimList := &karpenterv1.NodeClaimList{} - if err := r.GuestClient.List(ctx, nodeClaimList); err != nil { - return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, fmt.Errorf("failed to list NodeClaims: %w", err) - } - if len(nodeClaimList.Items) > 0 { - var elapsed time.Duration - for _, nodeClaim := range nodeClaimList.Items { - if nodeClaim.DeletionTimestamp == nil { - // This could happen if a NodeClaim has been orphaned without a NodePool owner ref - log.Info("NodeClaim has no deletion timestamp during deletion, deleting explicitly", "nodeClaim", nodeClaim.Name) - if err := r.GuestClient.Delete(ctx, &nodeClaim, &client.DeleteOptions{GracePeriodSeconds: ptr.To(int64(0))}); err != nil { - return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, fmt.Errorf("failed to delete NodeClaim: %w", err) - } - continue - } - elapsed = time.Since(nodeClaim.DeletionTimestamp.Time) - if !force && elapsed < NodeClaimDeletionTimeout { - continue - } - - if err := r.handleForcefulNodeClaimDeletion(ctx, &nodeClaim); err != nil { - return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, fmt.Errorf("failed to handle forceful NodeClaim deletion: %w", err) - } - } - log.Info("Waiting for NodeClaims to be deleted, requeueing...", "nodeClaimCount", len(nodeClaimList.Items), "elapsed", elapsed) - return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, nil - } - } - - originalHCP := hcp.DeepCopy() - controllerutil.RemoveFinalizer(hcp, karpenterutil.KarpenterFinalizer) - if err := r.ManagementClient.Patch(ctx, hcp, client.MergeFromWithOptions(originalHCP, client.MergeFromWithOptimisticLock{})); err != nil { - return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, fmt.Errorf("failed to remove finalizer from cluster: %w", err) - } - log.Info("Successfully removed all Karpenter NodePools and NodeClaims") - return ctrl.Result{}, nil + clusterDeleting, err := r.isClusterDeleting(ctx, hcp) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to check cluster deletion state: %w", err) + } + if clusterDeleting { + return r.reconcileDeletion(ctx, hcp) } if !controllerutil.ContainsFinalizer(hcp, karpenterutil.KarpenterFinalizer) { originalHCP := hcp.DeepCopy() @@ -513,6 +464,113 @@ func (r *Reconciler) reconcileOpenshiftEC2NodeClassDefault(ctx context.Context, return nil } +func (r *Reconciler) reconcileDeletion(ctx context.Context, hcp *hyperv1.HostedControlPlane) (ctrl.Result, error) { + log := ctrl.LoggerFrom(ctx) + + // TODO(maxcao13): if supporting disablement, we don't want to force delete immediately. + // When force=true, we skip the graceful timeout and immediately trigger forceful deletion. + // When force=false, we wait for NodeClaimDeletionTimeout before triggering forceful deletion. + force := true + if controllerutil.ContainsFinalizer(hcp, karpenterutil.KarpenterFinalizer) { + // The deletion flow is: + // 1. Delete all NodePools (NodeClaims will be marked for deletion from deleting the NodePools due to ownerReferences) + // 2. Make sure all NodeClaims are actually gone (gracefully first, unless force=true) + // 3. If graceful timeout or force=true, set the termination timestamp annotation to trigger Karpenter's forceful deletion + // 4. Remove the finalizer from the HostedControlPlane to allow the rest of the HCP deletion to complete + + // Karpenter itself will make sure Nodes objects are deleted (and underlying instances are terminated) before finalizing the NodeClaims + nodePoolList := &karpenterv1.NodePoolList{} + if err := r.GuestClient.List(ctx, nodePoolList); err != nil { + return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, fmt.Errorf("failed to list NodePools: %w", err) + } + + // Delete all NodePools first + if len(nodePoolList.Items) > 0 { + for _, nodePool := range nodePoolList.Items { + if !nodePool.GetDeletionTimestamp().IsZero() { + continue + } + if err := r.GuestClient.Delete(ctx, &nodePool, &client.DeleteOptions{ + GracePeriodSeconds: ptr.To(int64(0)), + }); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to delete NodePool: %w", err) + } + } + return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, nil + } + + // Make sure all NodeClaims are actually gone (gracefully first) + nodeClaimList := &karpenterv1.NodeClaimList{} + if err := r.GuestClient.List(ctx, nodeClaimList); err != nil { + return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, fmt.Errorf("failed to list NodeClaims: %w", err) + } + if len(nodeClaimList.Items) > 0 { + var elapsed time.Duration + for _, nodeClaim := range nodeClaimList.Items { + if nodeClaim.DeletionTimestamp == nil { + log.Info("NodeClaim has no deletion timestamp during deletion, deleting explicitly", "nodeClaim", nodeClaim.Name) + if err := r.GuestClient.Delete(ctx, &nodeClaim, &client.DeleteOptions{GracePeriodSeconds: ptr.To(int64(0))}); err != nil { + return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, fmt.Errorf("failed to delete NodeClaim: %w", err) + } + continue + } + elapsed = time.Since(nodeClaim.DeletionTimestamp.Time) + if !force && elapsed < NodeClaimDeletionTimeout { + continue + } + + if err := r.handleForcefulNodeClaimDeletion(ctx, &nodeClaim); err != nil { + return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, fmt.Errorf("failed to handle forceful NodeClaim deletion: %w", err) + } + } + log.Info("Waiting for NodeClaims to be deleted, requeueing...", "nodeClaimCount", len(nodeClaimList.Items), "elapsed", elapsed) + return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, nil + } + } + + // Only remove the finalizer once the HCP itself is being deleted. + // When triggered by the CAPI Cluster deletion alone, we clean up nodes + // but leave the finalizer in place — it will be removed on a subsequent + // reconcile when the HCP gets its own DeletionTimestamp. + if hcp.DeletionTimestamp != nil { + originalHCP := hcp.DeepCopy() + controllerutil.RemoveFinalizer(hcp, karpenterutil.KarpenterFinalizer) + if err := r.ManagementClient.Patch(ctx, hcp, client.MergeFromWithOptions(originalHCP, client.MergeFromWithOptimisticLock{})); err != nil { + return ctrl.Result{RequeueAfter: KarpenterDeletionRequeueInterval}, fmt.Errorf("failed to remove finalizer from hostedControlPlane: %w", err) + } + log.Info("Successfully removed all Karpenter NodePools and NodeClaims") + } + return ctrl.Result{}, nil +} + +// isClusterDeleting returns true when the cluster is being torn down. +// It checks both the HCP DeletionTimestamp and the CAPI Cluster DeletionTimestamp. +// The CAPI Cluster is deleted earlier in the HostedCluster deletion sequence than +// the HCP, so checking it allows node cleanup to begin sooner — in parallel with +// regular CAPI node teardown instead of after it completes. +func (r *Reconciler) isClusterDeleting(ctx context.Context, hcp *hyperv1.HostedControlPlane) (bool, error) { + if hcp.DeletionTimestamp != nil { + return true, nil + } + + if hcp.Spec.InfraID == "" { + return false, nil + } + + capiCluster := &capiv1.Cluster{} + if err := r.ManagementClient.Get(ctx, client.ObjectKey{ + Namespace: r.Namespace, + Name: hcp.Spec.InfraID, + }, capiCluster); err != nil { + if apierrors.IsNotFound(err) { + return false, nil + } + return false, fmt.Errorf("failed to get CAPI Cluster: %w", err) + } + + return !capiCluster.DeletionTimestamp.IsZero(), nil +} + // handleForcefulNodeClaimDeletion handles the timeout of a NodeClaim during cluster deletion. func (r *Reconciler) handleForcefulNodeClaimDeletion(ctx context.Context, nodeClaim *karpenterv1.NodeClaim) error { log := ctrl.LoggerFrom(ctx) diff --git a/karpenter-operator/controllers/karpenter/karpenter_controller_test.go b/karpenter-operator/controllers/karpenter/karpenter_controller_test.go index 4d1b945535ae..65bbe1bdc830 100644 --- a/karpenter-operator/controllers/karpenter/karpenter_controller_test.go +++ b/karpenter-operator/controllers/karpenter/karpenter_controller_test.go @@ -20,6 +20,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -35,21 +36,25 @@ func TestKarpenterDeletion(t *testing.T) { scheme := api.Scheme now := time.Now() + const testNamespace = "test-namespace" + testCases := []struct { name string hcp *hyperv1.HostedControlPlane + managementObjects []client.Object objects []client.Object expectedNodePools int expectedNodeClaims int eventuallyKarpenterFinalizerRemoved bool - // expectedAnnotations maps NodeClaim name to whether it should have the termination timestamp annotation - expectedAnnotations map[string]bool + // expectedTerminationAnnotations maps NodeClaim name to whether it should have the termination timestamp annotation + expectedTerminationAnnotations map[string]bool }{ { name: "when hcp is deleted with no resources, it should remove karpenter finalizer", hcp: &hyperv1.HostedControlPlane{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-hcp", + Name: "test-hcp", + Namespace: testNamespace, DeletionTimestamp: &metav1.Time{ Time: now, }, @@ -68,7 +73,8 @@ func TestKarpenterDeletion(t *testing.T) { name: "when hcp is deleted, it should delete karpenter NodePools and remove karpenter finalizer", hcp: &hyperv1.HostedControlPlane{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-hcp", + Name: "test-hcp", + Namespace: testNamespace, DeletionTimestamp: &metav1.Time{ Time: now, }, @@ -98,7 +104,8 @@ func TestKarpenterDeletion(t *testing.T) { name: "when hcp is deleted, it should not remove karpenter finalizer if karpenter NodePools still exist", hcp: &hyperv1.HostedControlPlane{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-hcp", + Name: "test-hcp", + Namespace: testNamespace, DeletionTimestamp: &metav1.Time{ Time: now, }, @@ -125,7 +132,8 @@ func TestKarpenterDeletion(t *testing.T) { name: "when hcp is deleted, it should set termination annotation on NodeClaims and not remove finalizer until they are gone", hcp: &hyperv1.HostedControlPlane{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-hcp", + Name: "test-hcp", + Namespace: testNamespace, DeletionTimestamp: &metav1.Time{ Time: now, }, @@ -163,7 +171,7 @@ func TestKarpenterDeletion(t *testing.T) { expectedNodePools: 0, expectedNodeClaims: 2, eventuallyKarpenterFinalizerRemoved: false, - expectedAnnotations: map[string]bool{ + expectedTerminationAnnotations: map[string]bool{ "test-nodeclaim-1": true, "test-nodeclaim-2": true, }, @@ -172,7 +180,8 @@ func TestKarpenterDeletion(t *testing.T) { name: "when NodeClaim already has termination annotation, it should not set it again (idempotency)", hcp: &hyperv1.HostedControlPlane{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-hcp", + Name: "test-hcp", + Namespace: testNamespace, DeletionTimestamp: &metav1.Time{ Time: now, }, @@ -198,7 +207,7 @@ func TestKarpenterDeletion(t *testing.T) { expectedNodePools: 0, expectedNodeClaims: 1, eventuallyKarpenterFinalizerRemoved: false, - expectedAnnotations: map[string]bool{ + expectedTerminationAnnotations: map[string]bool{ "test-nodeclaim-1": true, // Already has annotation, should still have it }, }, @@ -206,7 +215,8 @@ func TestKarpenterDeletion(t *testing.T) { name: "when NodeClaim has no deletion timestamp (orphaned), it should be explicitly deleted then get termination annotation", hcp: &hyperv1.HostedControlPlane{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-hcp", + Name: "test-hcp", + Namespace: testNamespace, DeletionTimestamp: &metav1.Time{ Time: now, }, @@ -227,12 +237,88 @@ func TestKarpenterDeletion(t *testing.T) { expectedNodePools: 0, expectedNodeClaims: 1, // Still exists due to finalizer eventuallyKarpenterFinalizerRemoved: false, - expectedAnnotations: map[string]bool{ + expectedTerminationAnnotations: map[string]bool{ // First reconcile explicitly deletes it (sets DeletionTimestamp), // second reconcile sees DeletionTimestamp and sets termination annotation "test-nodeclaim-orphaned": true, }, }, + { + name: "when CAPI Cluster is deleting but HCP is not, it should start node cleanup without removing karpenter finalizer", + hcp: &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-hcp", + Namespace: testNamespace, + Finalizers: []string{ + karpenterutil.KarpenterFinalizer, + "some-other-finalizer", + }, + }, + Spec: hyperv1.HostedControlPlaneSpec{ + InfraID: "test-infra-id", + }, + }, + managementObjects: []client.Object{ + &capiv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-infra-id", + Namespace: testNamespace, + DeletionTimestamp: &metav1.Time{Time: now}, + Finalizers: []string{"capi-finalizer"}, + }, + }, + }, + objects: []client.Object{ + &karpenterv1.NodePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-nodepool-1", + }, + }, + }, + expectedNodePools: 0, + expectedNodeClaims: 0, + eventuallyKarpenterFinalizerRemoved: false, + }, + { + name: "when CAPI Cluster is deleting with NodeClaims, it should clean up nodes without removing karpenter finalizer", + hcp: &hyperv1.HostedControlPlane{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-hcp", + Namespace: testNamespace, + Finalizers: []string{ + karpenterutil.KarpenterFinalizer, + }, + }, + Spec: hyperv1.HostedControlPlaneSpec{ + InfraID: "test-infra-id", + }, + }, + managementObjects: []client.Object{ + &capiv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-infra-id", + Namespace: testNamespace, + DeletionTimestamp: &metav1.Time{Time: now}, + Finalizers: []string{"capi-finalizer"}, + }, + }, + }, + objects: []client.Object{ + &karpenterv1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-nodeclaim-1", + DeletionTimestamp: &metav1.Time{Time: now}, + Finalizers: []string{"karpenter-finalizer"}, + }, + }, + }, + expectedNodePools: 0, + expectedNodeClaims: 1, + eventuallyKarpenterFinalizerRemoved: false, + expectedTerminationAnnotations: map[string]bool{ + "test-nodeclaim-1": true, + }, + }, } for _, tc := range testCases { @@ -250,6 +336,7 @@ func TestKarpenterDeletion(t *testing.T) { Name: "pull-secret", }, }). + WithObjects(tc.managementObjects...). Build() fakeGuestClient := fake.NewClientBuilder(). @@ -261,6 +348,7 @@ func TestKarpenterDeletion(t *testing.T) { ManagementClient: fakeManagementClient, GuestClient: fakeGuestClient, ReleaseProvider: mockedProvider, + Namespace: testNamespace, } ctx := log.IntoContext(t.Context(), testr.New(t)) @@ -295,7 +383,7 @@ func TestKarpenterDeletion(t *testing.T) { g.Expect(nodeClaimList.Items).To(HaveLen(tc.expectedNodeClaims)) // verify annotations if specified - for nodeClaimName, shouldHaveAnnotation := range tc.expectedAnnotations { + for nodeClaimName, shouldHaveAnnotation := range tc.expectedTerminationAnnotations { nodeClaim := &karpenterv1.NodeClaim{} err := fakeGuestClient.Get(ctx, client.ObjectKey{Name: nodeClaimName}, nodeClaim) g.Expect(err).NotTo(HaveOccurred()) diff --git a/karpenter-operator/controllers/karpenter/machine_approver.go b/karpenter-operator/controllers/karpenter/machine_approver.go index 3dc045e0eccb..01f137f3b696 100644 --- a/karpenter-operator/controllers/karpenter/machine_approver.go +++ b/karpenter-operator/controllers/karpenter/machine_approver.go @@ -85,7 +85,7 @@ func (r *MachineApproverController) SetupWithManager(mgr ctrl.Manager) error { &handler.TypedEnqueueRequestForObject[*certificatesv1.CertificateSigningRequest]{}, predicate.NewTypedPredicateFuncs(csrFilterFn), )); err != nil { - return fmt.Errorf("failed to watch CertificateSigningRequest: %v", err) + return fmt.Errorf("failed to watch CertificateSigningRequest: %w", err) } return nil @@ -100,7 +100,7 @@ func (r *MachineApproverController) Reconcile(ctx context.Context, req ctrl.Requ if apierrors.IsNotFound(err) { return ctrl.Result{}, nil } - return ctrl.Result{}, fmt.Errorf("failed to get csr %s: %v", req.NamespacedName, err) + return ctrl.Result{}, fmt.Errorf("failed to get csr %s: %w", req.NamespacedName, err) } // Return early if deleted @@ -129,7 +129,7 @@ func (r *MachineApproverController) Reconcile(ctx context.Context, req ctrl.Requ if authorized { log.Info("Attempting to approve CSR", "csr", csr.Name) if err := r.approve(ctx, csr); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to approve csr %s: %v", csr.Name, err) + return ctrl.Result{}, fmt.Errorf("failed to approve csr %s: %w", csr.Name, err) } } @@ -243,7 +243,7 @@ func (r *MachineApproverController) approve(ctx context.Context, csr *certificat _, err := r.certClient.CertificateSigningRequests().UpdateApproval(ctx, csr.Name, csr, metav1.UpdateOptions{}) if err != nil { - return fmt.Errorf("error updating approval for csr: %v", err) + return fmt.Errorf("error updating approval for csr: %w", err) } return nil diff --git a/karpenter-operator/controllers/nodeclass/ec2_nodeclass_controller.go b/karpenter-operator/controllers/nodeclass/ec2_nodeclass_controller.go index e7153ada5ba3..8d4d761ad29d 100644 --- a/karpenter-operator/controllers/nodeclass/ec2_nodeclass_controller.go +++ b/karpenter-operator/controllers/nodeclass/ec2_nodeclass_controller.go @@ -420,7 +420,7 @@ func (r *EC2NodeClassReconciler) reconcileStatus(ctx context.Context, ec2NodeCla if !reflect.DeepEqual(originalObj.Status, openshiftNodeClass.Status) { if err := r.guestClient.Status().Patch(ctx, openshiftNodeClass, client.MergeFrom(originalObj)); err != nil { - return fmt.Errorf("failed to update status: %v", err) + return fmt.Errorf("failed to update status: %w", err) } } diff --git a/karpenter-operator/main.go b/karpenter-operator/main.go index a122fe8b7e62..fa45345d39d2 100644 --- a/karpenter-operator/main.go +++ b/karpenter-operator/main.go @@ -116,7 +116,7 @@ func run(ctx context.Context) error { } if err := mgr.Add(managementCluster); err != nil { - return fmt.Errorf("failed to add managementCluster to controller runtime manager: %v", err) + return fmt.Errorf("failed to add managementCluster to controller runtime manager: %w", err) } hypershiftClient, err := hypershiftclient.NewForConfig(managementKubeconfig) diff --git a/konnectivity-socks5-proxy/http_proxy.go b/konnectivity-socks5-proxy/http_proxy.go index 7d39129aa4a3..5239f320e4c7 100644 --- a/konnectivity-socks5-proxy/http_proxy.go +++ b/konnectivity-socks5-proxy/http_proxy.go @@ -71,7 +71,7 @@ func (hpd *httpProxyDialer) Dial(network string, addr string) (net.Conn, error) return nil, err } - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { conn.Close() f := strings.SplitN(resp.Status, " ", 2) return nil, errors.New(f[1]) diff --git a/kubernetes-default-proxy/kubernetes_default_proxy.go b/kubernetes-default-proxy/kubernetes_default_proxy.go index 8efe2e4d8fc7..79c40cd3e129 100644 --- a/kubernetes-default-proxy/kubernetes_default_proxy.go +++ b/kubernetes-default-proxy/kubernetes_default_proxy.go @@ -67,7 +67,7 @@ func (s *server) validate() error { } func (s *server) run(ctx context.Context) error { - listener, err := net.Listen("tcp", s.listenAddr) + listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", s.listenAddr) if err != nil { return fmt.Errorf("failed to listen on tcp:%s: %w", s.listenAddr, err) } @@ -87,7 +87,7 @@ func (s *server) run(ctx context.Context) error { go func() { defer conn.Close() - backendConn, err := net.Dial("tcp", s.proxyAddr) + backendConn, err := (&net.Dialer{}).DialContext(ctx, "tcp", s.proxyAddr) if err != nil { s.log.Error(err, "failed diaing backend", "proxyAddr", s.proxyAddr) return @@ -95,7 +95,7 @@ func (s *server) run(ctx context.Context) error { defer backendConn.Close() req := &http.Request{ - Method: "CONNECT", + Method: http.MethodConnect, URL: &url.URL{Host: s.apiServerAddr}, Proto: "HTTP/1.1", ProtoMajor: 1, @@ -111,7 +111,7 @@ func (s *server) run(ctx context.Context) error { s.log.Error(err, "failed to read response to connect request") return } - if response.StatusCode != 200 { + if response.StatusCode != http.StatusOK { s.log.Error(fmt.Errorf("got unexpected statuscode %d to CONNECT request", response.StatusCode), "failed to establish a connection through http connect") return } diff --git a/kubevirtexternalinfra/externalinfra.go b/kubevirtexternalinfra/externalinfra.go index 5e838b5e53f8..543d68a4d1f9 100644 --- a/kubevirtexternalinfra/externalinfra.go +++ b/kubevirtexternalinfra/externalinfra.go @@ -222,8 +222,8 @@ func (k *kubevirtInfraClientImp) GetInfraKubevirtVersion(ctx context.Context) (* result = restClient.Get().AbsPath(uri).Do(ctx) if data, err := result.Raw(); err != nil { - connErr, isConnectionErr := err.(*url.Error) - if isConnectionErr { + var connErr *url.Error + if errors.As(err, &connErr) { err = connErr.Err } diff --git a/pkg/etcdcli/etcdcli.go b/pkg/etcdcli/etcdcli.go index ba68cbdb4441..cf1dd2f2783c 100644 --- a/pkg/etcdcli/etcdcli.go +++ b/pkg/etcdcli/etcdcli.go @@ -307,7 +307,7 @@ func (g *etcdClientGetter) UnhealthyMembers(ctx context.Context) ([]*etcdserverp defer cancel() etcdCluster, err := cli.MemberList(ctx) if err != nil { - return nil, fmt.Errorf("could not get member list %v", err) + return nil, fmt.Errorf("could not get member list: %w", err) } memberHealth := getMemberHealth(ctx, etcdCluster.Members) diff --git a/sharedingress-config-generator/controller.go b/sharedingress-config-generator/controller.go index 4d3f8ea61e49..53ff34394cfc 100644 --- a/sharedingress-config-generator/controller.go +++ b/sharedingress-config-generator/controller.go @@ -169,7 +169,7 @@ func (r *SharedIngressConfigReconciler) Reconcile(ctx context.Context, _ ctrl.Re } logger.Info("Reloading HAProxy configuration") - if err := sendHAProxyReloadCommand(r.haProxyClient, r.haProxyRuntimeSocketPath); err != nil { + if err := sendHAProxyReloadCommand(ctx, r.haProxyClient, r.haProxyRuntimeSocketPath); err != nil { return ctrl.Result{}, fmt.Errorf("failed to reload HAProxy: %w", err) } @@ -203,8 +203,8 @@ func (r *SharedIngressConfigReconciler) currentConfigHash() ([]byte, error) { // sendHAProxyReloadCommand connects to the specified Unix socket and sends a reload command. // It inspects the returned response and return an appropriate error if the reload operation failed. -func sendHAProxyReloadCommand(client haProxyClient, socketPath string) error { - response, err := client.sendCommand(socketPath, "reload") +func sendHAProxyReloadCommand(ctx context.Context, client haProxyClient, socketPath string) error { + response, err := client.sendCommand(ctx, socketPath, "reload") if err != nil { return err } diff --git a/sharedingress-config-generator/controller_test.go b/sharedingress-config-generator/controller_test.go index 6a8856249372..72d7e89b8db3 100644 --- a/sharedingress-config-generator/controller_test.go +++ b/sharedingress-config-generator/controller_test.go @@ -2,6 +2,7 @@ package sharedingressconfiggenerator import ( "bytes" + "context" "fmt" "os" "path/filepath" @@ -29,7 +30,7 @@ type mockHAProxyClient struct { lastCommand string } -func (m *mockHAProxyClient) sendCommand(socketPath, command string) (string, error) { +func (m *mockHAProxyClient) sendCommand(_ context.Context, socketPath, command string) (string, error) { m.sendCommandCalled = true m.sendCommandCount++ m.lastCommand = command @@ -122,7 +123,7 @@ func TestReconcile(t *testing.T) { } // Make the temporary directory read-only to simulate a permissions error - g.Expect(os.Chmod(tempDir, 0555)).To(Succeed()) // r-x r-x r-x + g.Expect(os.Chmod(tempDir, 0555)).To(Succeed()) //nolint:dupword // r-x r-x r-x describes permission bits // Ensure we restore permissions so the deferred os.RemoveAll can work defer func() { g.Expect(os.Chmod(tempDir, 0755)).To(Succeed()) diff --git a/sharedingress-config-generator/haproxy_client.go b/sharedingress-config-generator/haproxy_client.go index a043e40b38ff..95524053ba65 100644 --- a/sharedingress-config-generator/haproxy_client.go +++ b/sharedingress-config-generator/haproxy_client.go @@ -1,6 +1,7 @@ package sharedingressconfiggenerator import ( + "context" "fmt" "io" "net" @@ -11,14 +12,14 @@ import ( type haProxyClient interface { // sendHAProxyCommand connects to the specified Unix socket, sends a command, // and returns the response from HAProxy. - sendCommand(socketPath, command string) (string, error) + sendCommand(ctx context.Context, socketPath, command string) (string, error) } type defaultHAproxyClient struct { } -func (c *defaultHAproxyClient) sendCommand(socketPath, command string) (string, error) { - conn, err := net.Dial("unix", socketPath) +func (c *defaultHAproxyClient) sendCommand(ctx context.Context, socketPath, command string) (string, error) { + conn, err := (&net.Dialer{}).DialContext(ctx, "unix", socketPath) if err != nil { return "", fmt.Errorf("failed to connect to socket %s: %w", socketPath, err) } diff --git a/support/azureutil/azureutil.go b/support/azureutil/azureutil.go index 07836f93d3ab..56dbbcac1b39 100644 --- a/support/azureutil/azureutil.go +++ b/support/azureutil/azureutil.go @@ -75,7 +75,7 @@ func GetAzureCloudConfiguration(cloudName string) (cloud.Configuration, error) { func GetSubnetNameFromSubnetID(subnetID string) (string, error) { subnet, err := arm.ParseResourceID(subnetID) if err != nil { - return "", fmt.Errorf("failed to parse subnet ID %q: %v", subnetID, err) + return "", fmt.Errorf("failed to parse subnet ID %q: %w", subnetID, err) } if !strings.EqualFold(subnet.ResourceType.Type, "virtualnetworks/subnets") { @@ -94,7 +94,7 @@ func GetSubnetNameFromSubnetID(subnetID string) (string, error) { func GetNameAndResourceGroupFromNetworkSecurityGroupID(nsgID string) (string, string, error) { nsg, err := arm.ParseResourceID(nsgID) if err != nil { - return "", "", fmt.Errorf("failed to parse network security group ID %q: %v", nsgID, err) + return "", "", fmt.Errorf("failed to parse network security group ID %q: %w", nsgID, err) } if !strings.EqualFold(nsg.ResourceType.Type, "networkSecurityGroups") { @@ -117,7 +117,7 @@ func GetNameAndResourceGroupFromNetworkSecurityGroupID(nsgID string) (string, st func GetVnetNameAndResourceGroupFromVnetID(vnetID string) (string, string, error) { vnet, err := arm.ParseResourceID(vnetID) if err != nil { - return "", "", fmt.Errorf("failed to parse vnet ID %q: %v", vnetID, err) + return "", "", fmt.Errorf("failed to parse vnet ID %q: %w", vnetID, err) } if !strings.EqualFold(vnet.ResourceType.Type, "virtualNetworks") { @@ -141,7 +141,7 @@ func GetVnetNameAndResourceGroupFromVnetID(vnetID string) (string, string, error func GetVnetInfoFromVnetID(ctx context.Context, vnetID string, subscriptionID string, azureCreds azcore.TokenCredential, cloudName string) (armnetwork.VirtualNetworksClientGetResponse, error) { partialVnetInfo, err := arm.ParseResourceID(vnetID) if err != nil { - return armnetwork.VirtualNetworksClientGetResponse{}, fmt.Errorf("failed to parse vnet information from vnet ID %q: %v", vnetID, err) + return armnetwork.VirtualNetworksClientGetResponse{}, fmt.Errorf("failed to parse vnet information from vnet ID %q: %w", vnetID, err) } if !strings.EqualFold(partialVnetInfo.ResourceType.Type, "virtualNetworks") { @@ -210,7 +210,7 @@ func getFullVnetInfo(ctx context.Context, subscriptionID string, vnetResourceGro func GetNetworkSecurityGroupInfo(ctx context.Context, nsgID string, subscriptionID string, azureCreds azcore.TokenCredential, cloudName string) (armnetwork.SecurityGroupsClientGetResponse, error) { partialNSGInfo, err := arm.ParseResourceID(nsgID) if err != nil { - return armnetwork.SecurityGroupsClientGetResponse{}, fmt.Errorf("failed to parse network security group id %q: %v", nsgID, err) + return armnetwork.SecurityGroupsClientGetResponse{}, fmt.Errorf("failed to parse network security group id %q: %w", nsgID, err) } cloudConfig, err := GetAzureCloudConfiguration(cloudName) diff --git a/support/controlplane-component/controlplane-component.go b/support/controlplane-component/controlplane-component.go index 442e5aa7fc60..0757f7c97ab8 100644 --- a/support/controlplane-component/controlplane-component.go +++ b/support/controlplane-component/controlplane-component.go @@ -177,7 +177,7 @@ func (c *controlPlaneWorkload[T]) Reconcile(cpContext ControlPlaneContext) error unavailableDependencies, err := c.checkDependencies(cpContext) if err != nil { - return fmt.Errorf("failed checking for dependencies availability: %v", err) + return fmt.Errorf("failed checking for dependencies availability: %w", err) } var reconcilationError error if len(unavailableDependencies) == 0 { @@ -311,7 +311,7 @@ func (c *controlPlaneWorkload[T]) update(cpContext ControlPlaneContext) error { func (c *controlPlaneWorkload[T]) reconcileWorkload(cpContext ControlPlaneContext) error { workloadObj, err := c.workloadProvider.LoadManifest(c.Name()) if err != nil { - return fmt.Errorf("failed loading workload manifest: %v", err) + return fmt.Errorf("failed loading workload manifest: %w", err) } // make sure that the Deployment/Statefulset name matches the component name. workloadObj.SetName(c.Name()) @@ -320,7 +320,7 @@ func (c *controlPlaneWorkload[T]) reconcileWorkload(cpContext ControlPlaneContex oldWorkloadObj := c.workloadProvider.NewObject() if err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(workloadObj), oldWorkloadObj); err != nil { if !apierrors.IsNotFound(err) { - return fmt.Errorf("failed to get old workload object: %v", err) + return fmt.Errorf("failed to get old workload object: %w", err) } } diff --git a/support/controlplane-component/controlplane-component_test.go b/support/controlplane-component/controlplane-component_test.go index a66e42be8927..fd05adfb7506 100644 --- a/support/controlplane-component/controlplane-component_test.go +++ b/support/controlplane-component/controlplane-component_test.go @@ -168,6 +168,12 @@ func TestReconcile(t *testing.T) { Labels: map[string]string{ "test-label": "test", }, + Tolerations: []corev1.Toleration{{ + Key: "custom-key", + Operator: corev1.TolerationOpEqual, + Value: "custom-value", + Effect: corev1.TaintEffectNoSchedule, + }}, }, }, Client: fake.NewClientBuilder().WithScheme(scheme). @@ -188,6 +194,13 @@ func TestReconcile(t *testing.T) { g.Expect(result.podTemplate.Labels).To(HaveKeyWithValue(hyperv1.ControlPlaneComponentLabel, testComponentName)) g.Expect(result.podTemplate.Labels).To(HaveKeyWithValue("test-label", "test")) + g.Expect(result.podTemplate.Spec.Tolerations).To(ContainElement(corev1.Toleration{ + Key: "custom-key", + Operator: corev1.TolerationOpEqual, + Value: "custom-value", + Effect: corev1.TaintEffectNoSchedule, + })) + // pod template annotations g.Expect(result.podTemplate.Annotations).To(HaveKey(hyperv1.ReleaseImageAnnotation)) @@ -359,7 +372,7 @@ func componentsFakeObjects() ([]client.Object, error) { caCfg := certs.CertCfg{IsCA: true, Subject: pkix.Name{CommonName: "root-ca", OrganizationalUnit: []string{"ou"}}} key, cert, err := certs.GenerateSelfSignedCertificate(&caCfg) if err != nil { - return nil, fmt.Errorf("failed to generate self signed CA: %v", err) + return nil, fmt.Errorf("failed to generate self signed CA: %w", err) } csrSigner := manifests.CSRSignerCASecret(testComponentNamespace) csrSigner.Data = map[string][]byte{ diff --git a/support/controlplane-component/konnectivity-container.go b/support/controlplane-component/konnectivity-container.go index c3bf6aac61ad..e9b169d8feeb 100644 --- a/support/controlplane-component/konnectivity-container.go +++ b/support/controlplane-component/konnectivity-container.go @@ -224,9 +224,27 @@ func (opts KonnectivityContainerOptions) buildContainer(hcp *hyperv1.HostedContr } } + // When connecting directly to cloud APIs, dialDirectWithProxy() reads + // HTTPS_PROXY from the process environment. Propagate the management + // cluster's proxy env vars so that path can reach cloud endpoints. + if opts.connectsDirectlyToCloudAPIs() { + proxy.SetEnvVars(&container.Env) + } + return container } +func (opts KonnectivityContainerOptions) connectsDirectlyToCloudAPIs() bool { + switch opts.Mode { + case HTTPS: + return ptr.Deref(opts.HTTPSOptions.ConnectDirectlyToCloudAPIs, false) + case Socks5: + return ptr.Deref(opts.Socks5Options.ConnectDirectlyToCloudAPIs, false) + default: + return false + } +} + func (opts KonnectivityContainerOptions) buildVolumes(proxyAdditionalCAs []corev1.VolumeProjection) []corev1.Volume { volumes := []corev1.Volume{ { diff --git a/support/controlplane-component/konnectivity-container_test.go b/support/controlplane-component/konnectivity-container_test.go new file mode 100644 index 000000000000..c412f1e78221 --- /dev/null +++ b/support/controlplane-component/konnectivity-container_test.go @@ -0,0 +1,174 @@ +package controlplanecomponent + +import ( + "strings" + "testing" + + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" +) + +func findEnvVar(envVars []corev1.EnvVar, name string) *corev1.EnvVar { + for i := range envVars { + if envVars[i].Name == name { + return &envVars[i] + } + } + return nil +} + +func TestBuildContainer(t *testing.T) { + hcp := &hyperv1.HostedControlPlane{} + + tests := []struct { + name string + opts KonnectivityContainerOptions + proxyEnvs map[string]string + expectProxyVars bool + }{ + { + name: "When ConnectDirectlyToCloudAPIs is true and management proxy is configured it should set proxy env vars", + opts: KonnectivityContainerOptions{ + Mode: HTTPS, + HTTPSOptions: HTTPSOptions{ + ConnectDirectlyToCloudAPIs: ptr.To(true), + }, + }, + proxyEnvs: map[string]string{ + "HTTP_PROXY": "http://proxy.mgmt.example.com:3128", + "HTTPS_PROXY": "https://proxy.mgmt.example.com:3129", + "NO_PROXY": "localhost,10.0.0.0/8", + }, + expectProxyVars: true, + }, + { + name: "When ConnectDirectlyToCloudAPIs is true for Socks5 mode it should set proxy env vars", + opts: KonnectivityContainerOptions{ + Mode: Socks5, + Socks5Options: Socks5Options{ + ConnectDirectlyToCloudAPIs: ptr.To(true), + }, + }, + proxyEnvs: map[string]string{ + "HTTP_PROXY": "http://proxy.mgmt.example.com:3128", + "HTTPS_PROXY": "https://proxy.mgmt.example.com:3129", + "NO_PROXY": "localhost,10.0.0.0/8", + }, + expectProxyVars: true, + }, + { + name: "When ConnectDirectlyToCloudAPIs is false it should not set proxy env vars", + opts: KonnectivityContainerOptions{ + Mode: HTTPS, + HTTPSOptions: HTTPSOptions{ + ConnectDirectlyToCloudAPIs: ptr.To(false), + }, + }, + proxyEnvs: map[string]string{ + "HTTP_PROXY": "http://proxy.mgmt.example.com:3128", + "HTTPS_PROXY": "https://proxy.mgmt.example.com:3129", + "NO_PROXY": "localhost,10.0.0.0/8", + }, + expectProxyVars: false, + }, + { + name: "When ConnectDirectlyToCloudAPIs is not set it should not set proxy env vars", + opts: KonnectivityContainerOptions{ + Mode: HTTPS, + }, + proxyEnvs: map[string]string{ + "HTTP_PROXY": "http://proxy.mgmt.example.com:3128", + "HTTPS_PROXY": "https://proxy.mgmt.example.com:3129", + "NO_PROXY": "localhost,10.0.0.0/8", + }, + expectProxyVars: false, + }, + { + name: "When ConnectDirectlyToCloudAPIs is true but no management proxy is configured it should not add proxy env vars", + opts: KonnectivityContainerOptions{ + Mode: HTTPS, + HTTPSOptions: HTTPSOptions{ + ConnectDirectlyToCloudAPIs: ptr.To(true), + }, + }, + proxyEnvs: map[string]string{}, + expectProxyVars: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewGomegaWithT(t) + t.Setenv("HTTP_PROXY", "") + t.Setenv("HTTPS_PROXY", "") + t.Setenv("NO_PROXY", "") + + for k, v := range tt.proxyEnvs { + t.Setenv(k, v) + } + + container := tt.opts.buildContainer(hcp, "test-image:latest", nil) + + g.Expect(findEnvVar(container.Env, "KUBECONFIG")).NotTo(BeNil(), "KUBECONFIG should always be set") + + if tt.expectProxyVars { + httpProxy := findEnvVar(container.Env, "HTTP_PROXY") + g.Expect(httpProxy).NotTo(BeNil(), "HTTP_PROXY should be set") + g.Expect(httpProxy.Value).To(Equal(tt.proxyEnvs["HTTP_PROXY"])) + + httpsProxy := findEnvVar(container.Env, "HTTPS_PROXY") + g.Expect(httpsProxy).NotTo(BeNil(), "HTTPS_PROXY should be set") + g.Expect(httpsProxy.Value).To(Equal(tt.proxyEnvs["HTTPS_PROXY"])) + + noProxy := findEnvVar(container.Env, "NO_PROXY") + g.Expect(noProxy).NotTo(BeNil(), "NO_PROXY should be set") + g.Expect(noProxy.Value).To(ContainSubstring("kube-apiserver"), "NO_PROXY should include kube-apiserver") + for _, entry := range strings.Split(tt.proxyEnvs["NO_PROXY"], ",") { + g.Expect(noProxy.Value).To(ContainSubstring(entry), "NO_PROXY should preserve original entry %q", entry) + } + } else { + g.Expect(findEnvVar(container.Env, "HTTP_PROXY")).To(BeNil(), "HTTP_PROXY should not be set") + g.Expect(findEnvVar(container.Env, "HTTPS_PROXY")).To(BeNil(), "HTTPS_PROXY should not be set") + g.Expect(findEnvVar(container.Env, "NO_PROXY")).To(BeNil(), "NO_PROXY should not be set") + } + }) + } +} + +func TestBuildContainerDualMode(t *testing.T) { + g := NewGomegaWithT(t) + hcp := &hyperv1.HostedControlPlane{} + + t.Setenv("HTTP_PROXY", "http://proxy.mgmt.example.com:3128") + t.Setenv("HTTPS_PROXY", "https://proxy.mgmt.example.com:3129") + t.Setenv("NO_PROXY", "localhost") + + // Simulate what injectKonnectivityContainer does for Dual mode: + // it builds the HTTPS container first, then the Socks5 container. + opts := KonnectivityContainerOptions{ + Mode: Dual, + HTTPSOptions: HTTPSOptions{ + ConnectDirectlyToCloudAPIs: ptr.To(true), + }, + } + + opts.Mode = HTTPS + httpsContainer := opts.buildContainer(hcp, "test-image:latest", nil) + + opts.Mode = Socks5 + socks5Container := opts.buildContainer(hcp, "test-image:latest", nil) + + g.Expect(findEnvVar(httpsContainer.Env, "HTTP_PROXY")).NotTo(BeNil(), + "HTTPS container should have HTTP_PROXY because ConnectDirectlyToCloudAPIs is set on HTTPSOptions") + g.Expect(findEnvVar(httpsContainer.Env, "HTTPS_PROXY")).NotTo(BeNil(), + "HTTPS container should have HTTPS_PROXY") + + g.Expect(findEnvVar(socks5Container.Env, "HTTP_PROXY")).To(BeNil(), + "Socks5 container should not have HTTP_PROXY because ConnectDirectlyToCloudAPIs is not set on Socks5Options") + g.Expect(findEnvVar(socks5Container.Env, "HTTPS_PROXY")).To(BeNil(), + "Socks5 container should not have HTTPS_PROXY") +} diff --git a/support/controlplane-component/kubeconfig.go b/support/controlplane-component/kubeconfig.go index 5e1801844ab5..07be8e325bfb 100644 --- a/support/controlplane-component/kubeconfig.go +++ b/support/controlplane-component/kubeconfig.go @@ -21,7 +21,7 @@ const ( func (c *controlPlaneWorkload[T]) adaptServiceAccountKubeconfigSecret(cpContext WorkloadContext, secret *corev1.Secret) error { csrSigner := manifests.CSRSignerCASecret(cpContext.HCP.Namespace) if err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(csrSigner), csrSigner); err != nil { - return fmt.Errorf("failed to get cluster-signer-ca secret: %v", err) + return fmt.Errorf("failed to get cluster-signer-ca secret: %w", err) } rootCA := manifests.RootCASecret(cpContext.HCP.Namespace) if err := cpContext.Client.Get(cpContext, client.ObjectKeyFromObject(rootCA), rootCA); err != nil { diff --git a/support/controlplane-component/token-minter-container.go b/support/controlplane-component/token-minter-container.go index 7a1d2aa6bd2b..937e8da8dfdf 100644 --- a/support/controlplane-component/token-minter-container.go +++ b/support/controlplane-component/token-minter-container.go @@ -41,7 +41,7 @@ type TokenMinterContainerOptions struct { // defaults to 'kubeconfig' KubeconfingVolumeName string - // KubeconfigSecretName is the name of the the kubeconfig secret used to mint the token in the target cluster. + // KubeconfigSecretName is the name of the kubeconfig secret used to mint the token in the target cluster. KubeconfigSecretName string // OneShot, if true, will cause the token-minter container to exit after minting the token. diff --git a/support/gcpapi/gcs_client.go b/support/gcpapi/gcs_client.go index adc913059203..4c98fe5af01b 100644 --- a/support/gcpapi/gcs_client.go +++ b/support/gcpapi/gcs_client.go @@ -2,6 +2,7 @@ package gcpapi import ( "context" + "errors" "fmt" "io" "net/http" @@ -42,7 +43,8 @@ func (g *GCSClient) UploadObject(ctx context.Context, bucket, objectName string, func (g *GCSClient) DeleteObject(ctx context.Context, bucket, objectName string) error { err := g.objects.Delete(bucket, objectName).Context(ctx).Do() if err != nil { - if gerr, ok := err.(*googleapi.Error); ok && gerr.Code == http.StatusNotFound { + var gerr *googleapi.Error + if errors.As(err, &gerr) && gerr.Code == http.StatusNotFound { return nil } return fmt.Errorf("failed to delete gs://%s/%s: %w", bucket, objectName, err) diff --git a/support/globalconfig/proxy.go b/support/globalconfig/proxy.go index e51f515c9c66..76275936ddba 100644 --- a/support/globalconfig/proxy.go +++ b/support/globalconfig/proxy.go @@ -20,7 +20,7 @@ func ProxyConfig() *configv1.Proxy { } } -// TODO (relyt0925): this is is utilized by the machine config server and feeds into the user data setup to download +// TODO (relyt0925): this is utilized by the machine config server and feeds into the user data setup to download // ignition configuration. It also factors into the machine config served to the machine. These usages need to be // further examined before it directly takes what the user configures for the cluster-wide proxy at the SDN level. // In current state of the world: the in cluster proxy configuration set by the user is never picked up (only the one diff --git a/support/konnectivityproxy/dialer.go b/support/konnectivityproxy/dialer.go index cfe3a307d851..d579dba6c1de 100644 --- a/support/konnectivityproxy/dialer.go +++ b/support/konnectivityproxy/dialer.go @@ -288,7 +288,7 @@ func (p *konnectivityProxy) DialContext(ctx context.Context, network string, req } konnectivityConnection, err := tlsDialer.DialContext(ctx, "tcp", konnectivityServerAddress) if err != nil { - return nil, fmt.Errorf("dialing proxy %q failed: %v", konnectivityServerAddress, err) + return nil, fmt.Errorf("dialing proxy %q failed: %w", konnectivityServerAddress, err) } // Bound CONNECT handshake I/O to avoid indefinite stalls and clear on success. @@ -323,10 +323,10 @@ func (p *konnectivityProxy) DialContext(ctx context.Context, network string, req res, err := http.ReadResponse(br, nil) if err != nil { _ = konnectivityConnection.Close() - return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v", + return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %w", requestAddress, konnectivityServerAddress, err) } - if res.StatusCode != 200 { + if res.StatusCode != http.StatusOK { log.Info("Status code was not 200", "statusCode", res.StatusCode) _ = konnectivityConnection.Close() return nil, fmt.Errorf("proxy error from %s while dialing %s: %v", konnectivityServerAddress, requestAddress, res.Status) diff --git a/support/konnectivityproxy/dialer_test.go b/support/konnectivityproxy/dialer_test.go index ed49d4a2a936..ca327cad709c 100644 --- a/support/konnectivityproxy/dialer_test.go +++ b/support/konnectivityproxy/dialer_test.go @@ -1,9 +1,16 @@ package konnectivityproxy import ( + "fmt" + "io" + "net" + "net/http" + "sync/atomic" "testing" "time" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -273,6 +280,151 @@ func TestKonnectivityHealthEndRetryPreventsStubbornFlag(t *testing.T) { } } +// startTCPEchoServer starts a TCP server that echoes back anything it receives. +// All accepted connections inherit a deadline so io.Copy never blocks indefinitely. +func startTCPEchoServer(t *testing.T) net.Listener { + t.Helper() + ln, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to start echo server: %v", err) + } + t.Cleanup(func() { ln.Close() }) + + go func() { + for { + conn, err := ln.Accept() + if err != nil { + return + } + go func() { + defer conn.Close() + if err := conn.SetDeadline(time.Now().Add(5 * time.Second)); err != nil { + return + } + if _, err := io.Copy(conn, conn); err != nil { + return + } + }() + } + }() + return ln +} + +// startConnectProxy starts an HTTP CONNECT proxy that increments connectCount +// for every successful tunnel. Relay goroutines are bounded by per-connection +// deadlines so they cannot outlive the test. +func startConnectProxy(t *testing.T, connectCount *atomic.Int32) net.Listener { + t.Helper() + ln, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("failed to start proxy server: %v", err) + } + t.Cleanup(func() { ln.Close() }) + + srv := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodConnect { + http.Error(w, "only CONNECT supported", http.StatusMethodNotAllowed) + return + } + connectCount.Add(1) + + target, err := (&net.Dialer{Timeout: 5 * time.Second}).DialContext(r.Context(), "tcp", r.Host) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer target.Close() + if err := target.SetDeadline(time.Now().Add(5 * time.Second)); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusOK) + hijacker, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "hijack not supported", http.StatusInternalServerError) + return + } + client, _, err := hijacker.Hijack() + if err != nil { + return + } + defer client.Close() + if err := client.SetDeadline(time.Now().Add(5 * time.Second)); err != nil { + return + } + + done := make(chan struct{}, 2) + relay := func(dst, src net.Conn) { + io.Copy(dst, src) //nolint:errcheck // relay best-effort; deadline bounds lifetime + done <- struct{}{} + } + go relay(target, client) + go relay(client, target) + <-done + }), + } + t.Cleanup(func() { srv.Close() }) + go func() { _ = srv.Serve(ln) }() + return ln +} + +func TestDialDirectWithProxy(t *testing.T) { + const testTimeout = 5 * time.Second + + t.Run("When HTTPS_PROXY is set it should route through the proxy", func(t *testing.T) { + g := NewGomegaWithT(t) + t.Setenv("HTTP_PROXY", "") + t.Setenv("HTTPS_PROXY", "") + t.Setenv("NO_PROXY", "") + echo := startTCPEchoServer(t) + var connectCount atomic.Int32 + proxyLn := startConnectProxy(t, &connectCount) + t.Setenv("HTTPS_PROXY", fmt.Sprintf("http://%s", proxyLn.Addr().String())) + + p := &konnectivityProxy{} + conn, err := p.dialDirectWithProxy("tcp", echo.Addr().String()) + g.Expect(err).NotTo(HaveOccurred(), "dialDirectWithProxy should succeed") + defer conn.Close() + g.Expect(conn.SetDeadline(time.Now().Add(testTimeout))).To(Succeed()) + + msg := []byte("hello") + _, err = conn.Write(msg) + g.Expect(err).NotTo(HaveOccurred(), "write should succeed") + buf := make([]byte, len(msg)) + _, err = io.ReadFull(conn, buf) + g.Expect(err).NotTo(HaveOccurred(), "read should succeed") + g.Expect(string(buf)).To(Equal(string(msg))) + g.Expect(connectCount.Load()).To(Equal(int32(1)), "proxy should receive 1 CONNECT request") + }) + + t.Run("When HTTPS_PROXY is not set it should connect directly", func(t *testing.T) { + g := NewGomegaWithT(t) + echo := startTCPEchoServer(t) + var connectCount atomic.Int32 + startConnectProxy(t, &connectCount) + t.Setenv("HTTPS_PROXY", "") + t.Setenv("HTTP_PROXY", "") + t.Setenv("NO_PROXY", "") + + p := &konnectivityProxy{} + conn, err := p.dialDirectWithProxy("tcp", echo.Addr().String()) + g.Expect(err).NotTo(HaveOccurred(), "dialDirectWithProxy should succeed") + defer conn.Close() + g.Expect(conn.SetDeadline(time.Now().Add(testTimeout))).To(Succeed()) + + msg := []byte("hello") + _, err = conn.Write(msg) + g.Expect(err).NotTo(HaveOccurred(), "write should succeed") + buf := make([]byte, len(msg)) + _, err = io.ReadFull(conn, buf) + g.Expect(err).NotTo(HaveOccurred(), "read should succeed") + g.Expect(string(buf)).To(Equal(string(msg))) + g.Expect(connectCount.Load()).To(Equal(int32(0)), "proxy should receive 0 CONNECT requests") + }) +} + func TestIsCloudAPI(t *testing.T) { tests := []struct { name string diff --git a/support/konnectivityproxy/proxy_dialer.go b/support/konnectivityproxy/proxy_dialer.go index dcfec2ac5789..c4550704a6d5 100644 --- a/support/konnectivityproxy/proxy_dialer.go +++ b/support/konnectivityproxy/proxy_dialer.go @@ -60,7 +60,7 @@ func (hpd *httpProxyDialer) Dial(network string, addr string) (net.Conn, error) return nil, err } - if resp.StatusCode != 200 { + if resp.StatusCode != http.StatusOK { conn.Close() f := strings.SplitN(resp.Status, " ", 2) return nil, errors.New(f[1]) diff --git a/support/releaseinfo/registryclient/client.go b/support/releaseinfo/registryclient/client.go index dd6db378208c..51160236b776 100644 --- a/support/releaseinfo/registryclient/client.go +++ b/support/releaseinfo/registryclient/client.go @@ -67,7 +67,7 @@ func ExtractImageFiles(ctx context.Context, imageRef string, pullSecret []byte, err := func() error { r, err := fromBlobs.Open(ctx, layer.Digest) if err != nil { - return fmt.Errorf("unable to access the source layer %s: %v", layer.Digest, err) + return fmt.Errorf("unable to access the source layer %s: %w", layer.Digest, err) } defer r.Close() rc, err := dockerarchive.DecompressStream(r) @@ -137,7 +137,7 @@ func ExtractImageFile(ctx context.Context, imageRef string, pullSecret []byte, f err := func() error { r, err := fromBlobs.Open(ctx, layer.Digest) if err != nil { - return fmt.Errorf("unable to access the source layer %s: %v", layer.Digest, err) + return fmt.Errorf("unable to access the source layer %s: %w", layer.Digest, err) } defer r.Close() rc, err := dockerarchive.DecompressStream(r) @@ -195,7 +195,7 @@ func ExtractImageFilesToDir(ctx context.Context, imageRef string, pullSecret []b err := func() error { r, err := fromBlobs.Open(ctx, layer.Digest) if err != nil { - return fmt.Errorf("unable to access the source layer %s: %v", layer.Digest, err) + return fmt.Errorf("unable to access the source layer %s: %w", layer.Digest, err) } defer r.Close() rc, err := dockerarchive.DecompressStream(r) diff --git a/support/releaseinfo/releaseinfo.go b/support/releaseinfo/releaseinfo.go index e6026a5e965c..5e21316fe4f5 100644 --- a/support/releaseinfo/releaseinfo.go +++ b/support/releaseinfo/releaseinfo.go @@ -206,7 +206,7 @@ func readComponentVersions(is *imageapi.ImageStream) (ComponentVersions, []error } all, err := parseComponentVersionsLabel(versions, tag.Annotations[annotationBuildVersionsDisplayNames]) if err != nil { - errs = append(errs, fmt.Errorf("the referenced image %s had an invalid version annotation: %v", tag.Name, err)) + errs = append(errs, fmt.Errorf("the referenced image %s had an invalid version annotation: %w", tag.Name, err)) } for k, v := range all { if k == "kubectl" { @@ -326,7 +326,7 @@ func parseComponentVersionsLabel(label, displayNames string) (ComponentVersions, } v, err := semver.Parse(parts[1]) if err != nil { - return nil, fmt.Errorf("the version pair %q must have a valid semantic version: %v", pair, err) + return nil, fmt.Errorf("the version pair %q must have a valid semantic version: %w", pair, err) } v.Build = nil labels[parts[0]] = ComponentVersion{ diff --git a/support/releaseinfo/releaseinfo_test.go b/support/releaseinfo/releaseinfo_test.go index 90522a0b51ef..199d944b4bbe 100644 --- a/support/releaseinfo/releaseinfo_test.go +++ b/support/releaseinfo/releaseinfo_test.go @@ -46,6 +46,11 @@ func TestParseComponentVersionsLabel(t *testing.T) { displayNames: "mycomponent=Invalid ", expectError: true, }, + { + name: "When version is not valid semver it should return an error", + label: "mycomponent=not-a-version", + expectError: true, + }, } for _, tt := range tests { @@ -98,6 +103,25 @@ func TestReadComponentVersions(t *testing.T) { }, expectKey: "machine-os", }, + { + name: "When version annotation has invalid semver, it should return an error", + tags: []imageapi.TagReference{ + { + Name: "bad-image", + Annotations: map[string]string{ + annotationBuildVersions: "component=not-a-semver", + }, + }, + { + Name: "good-image", + Annotations: map[string]string{ + annotationBuildVersions: "component=1.0.0", + }, + }, + }, + expectError: true, + expectKey: "component", + }, { name: "When multiple non-machine-os versions exist it should return an error", tags: []imageapi.TagReference{ diff --git a/support/supportedversion/version.go b/support/supportedversion/version.go index f2a51177a265..76649a9ae055 100644 --- a/support/supportedversion/version.go +++ b/support/supportedversion/version.go @@ -220,7 +220,7 @@ func LookupDefaultOCPVersion(ctx context.Context, releaseStream string, client c version, err := retrieveSupportedOCPVersion(ctx, releaseURL, client) if err != nil { - return ocpVersion{}, fmt.Errorf("failed to get OCP version from release URL %s: %v", releaseURL, err) + return ocpVersion{}, fmt.Errorf("failed to get OCP version from release URL %s: %w", releaseURL, err) } return version, nil @@ -267,7 +267,7 @@ func LookupLatestSupportedRelease(ctx context.Context, hc *hyperv1.HostedCluster var version ocpVersion - req, err := http.NewRequestWithContext(ctx, "GET", releaseURL, nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, releaseURL, nil) if err != nil { return "", err } @@ -336,7 +336,7 @@ func GetSupportedOCPVersions(ctx context.Context, namespace string, client crcli // Fetch the supported versions ConfigMap from the specified namespace supportedVersions = manifests.ConfigMap(namespace) if err := client.Get(ctx, crclient.ObjectKeyFromObject(supportedVersions), supportedVersions); err != nil { - return SupportedVersions{}, "", fmt.Errorf("failed to find supported versions on the server: %v", err) + return SupportedVersions{}, "", fmt.Errorf("failed to find supported versions on the server: %w", err) } } @@ -350,7 +350,7 @@ func GetSupportedOCPVersions(ctx context.Context, namespace string, client crcli // Check if the ConfigMap contains the supported versions key if supportedVersionData, present := supportedVersions.Data[config.ConfigMapVersionsKey]; present { if err := json.Unmarshal([]byte(supportedVersionData), &versions); err != nil { - return SupportedVersions{}, "", fmt.Errorf("failed to parse supported versions on the server: %v", err) + return SupportedVersions{}, "", fmt.Errorf("failed to parse supported versions on the server: %w", err) } return versions, serverVersion, nil @@ -483,7 +483,7 @@ func retrieveSupportedOCPVersion(ctx context.Context, releaseURL string, client configMapList := &corev1.ConfigMapList{} err := client.List(ctx, configMapList, crclient.MatchingLabels{"hypershift.openshift.io/supported-versions": "true"}) if err != nil { - return ocpVersion{}, fmt.Errorf("failed to list ConfigMaps to find supported versions: %v", err) + return ocpVersion{}, fmt.Errorf("failed to list ConfigMaps to find supported versions: %w", err) } for _, configMap := range configMapList.Items { if configMap.Name == "supported-versions" { @@ -500,14 +500,18 @@ func retrieveSupportedOCPVersion(ctx context.Context, releaseURL string, client // Get the latest supported OCP version from the supported versions ConfigMap supportedOCPVersions, _, err := GetSupportedOCPVersions(ctx, namespace, client, supportedVersions) if err != nil { - return ocpVersion{}, fmt.Errorf("failed to get supported OCP versions: %v", err) + return ocpVersion{}, fmt.Errorf("failed to get supported OCP versions: %w", err) } if len(supportedOCPVersions.Versions) == 0 { return ocpVersion{}, fmt.Errorf("no supported OCP versions found in the ConfigMap") } // Fetch the release information from the URL - resp, err := http.Get(releaseURL) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, releaseURL, nil) + if err != nil { + return ocpVersion{}, err + } + resp, err := http.DefaultClient.Do(req) if err != nil { return ocpVersion{}, err } diff --git a/support/supportedversion/version_test.go b/support/supportedversion/version_test.go index 87d816f68a1a..8907229209a5 100644 --- a/support/supportedversion/version_test.go +++ b/support/supportedversion/version_test.go @@ -1,6 +1,7 @@ package supportedversion import ( + "context" "encoding/json" "fmt" "net/http" @@ -19,6 +20,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/blang/semver" ) @@ -926,6 +928,13 @@ func TestRetrieveSupportedOCPVersion(t *testing.T) { expectErr: true, expectedErrMsg: "failed to get supported OCP versions", }, + { + name: "When the release URL is invalid, expect a request creation error", + cm: supportedVersionsCM, + releaseURL: "://invalid-url", + expectErr: true, + expectedErrMsg: "parse", + }, { name: "When the ConfigMap supports older versions, expect the latest older version to be returned", cm: olderSupportedVersionsCM, @@ -963,6 +972,44 @@ func TestRetrieveSupportedOCPVersion(t *testing.T) { } } +func TestRetrieveSupportedOCPVersion_ListFailure(t *testing.T) { + g := NewWithT(t) + + scheme := api.Scheme + g.Expect(corev1.AddToScheme(scheme)).To(Succeed()) + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + return fmt.Errorf("connection refused") + }, + }). + Build() + + _, err := retrieveSupportedOCPVersion(t.Context(), "https://example.com/tags", fakeClient) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("failed to list ConfigMaps to find supported versions")) +} + +func TestLookupDefaultOCPVersion_ListFailure(t *testing.T) { + g := NewWithT(t) + + scheme := api.Scheme + g.Expect(corev1.AddToScheme(scheme)).To(Succeed()) + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + return fmt.Errorf("connection refused") + }, + }). + Build() + + _, err := LookupDefaultOCPVersion(t.Context(), "", fakeClient) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("failed to get OCP version from release URL")) +} + func TestGetArchFromStream(t *testing.T) { testCases := []struct { name string diff --git a/support/thirdparty/docker/pkg/archive/archive.go b/support/thirdparty/docker/pkg/archive/archive.go index abcfeffbbe10..d7fd646b76f7 100644 --- a/support/thirdparty/docker/pkg/archive/archive.go +++ b/support/thirdparty/docker/pkg/archive/archive.go @@ -140,7 +140,7 @@ func cmdStream(cmd *exec.Cmd, input io.Reader) (io.ReadCloser, error) { // Copy stdout to the returned pipe go func() { if err := cmd.Wait(); err != nil { - pipeW.CloseWithError(fmt.Errorf("%s: %s", err, errBuf.String())) + pipeW.CloseWithError(fmt.Errorf("%w: %s", err, errBuf.String())) } else { pipeW.Close() } diff --git a/support/thirdparty/library-go/pkg/image/dockerv1client/types.go b/support/thirdparty/library-go/pkg/image/dockerv1client/types.go index 3b85b81e0be9..d5724da82c2e 100644 --- a/support/thirdparty/library-go/pkg/image/dockerv1client/types.go +++ b/support/thirdparty/library-go/pkg/image/dockerv1client/types.go @@ -73,7 +73,7 @@ type Descriptor struct { Size int64 `json:"size,omitempty"` // Digest uniquely identifies the content. A byte stream can be verified - // against against this digest. + // against this digest. Digest string `json:"digest,omitempty"` } diff --git a/support/thirdparty/library-go/pkg/image/registryclient/client.go b/support/thirdparty/library-go/pkg/image/registryclient/client.go index 9220e2087bc7..cf1ab87d860a 100644 --- a/support/thirdparty/library-go/pkg/image/registryclient/client.go +++ b/support/thirdparty/library-go/pkg/image/registryclient/client.go @@ -2,6 +2,7 @@ package registryclient import ( "context" + "errors" "fmt" "hash" "io" @@ -181,7 +182,7 @@ func (c *Context) Ping(ctx context.Context, registry *url.URL, insecure bool) (h } // follow redirects - redirect, err := c.ping(src, insecure, t) + redirect, err := c.ping(ctx, src, insecure, t) c.lock.Lock() defer c.lock.Unlock() @@ -223,14 +224,14 @@ func (c *Context) Repository(ctx context.Context, registry *url.URL, repoName st return NewLimitedRetryRepository(repo, c.Retries, limiter), nil } -func (c *Context) ping(registry url.URL, insecure bool, transport http.RoundTripper) (*url.URL, error) { +func (c *Context) ping(ctx context.Context, registry url.URL, insecure bool, transport http.RoundTripper) (*url.URL, error) { pingClient := &http.Client{ Transport: transport, Timeout: 15 * time.Second, } target := registry target.Path = path.Join(target.Path, "v2") + "/" - req, err := http.NewRequest("GET", target.String(), nil) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil) if err != nil { return nil, err } @@ -238,7 +239,7 @@ func (c *Context) ping(registry url.URL, insecure bool, transport http.RoundTrip if err != nil { if insecure && registry.Scheme == "https" { registry.Scheme = "http" - _, nErr := c.ping(registry, true, transport) + _, nErr := c.ping(ctx, registry, true, transport) if nErr != nil { return nil, nErr } @@ -377,20 +378,24 @@ func isTemporaryHTTPError(err error) (time.Duration, bool) { if err == nil { return 0, false } - switch t := err.(type) { - case net.Error: - return time.Second, t.Timeout() - case errcode.ErrorCoder: + var netErr net.Error + if errors.As(err, &netErr) { + return time.Second, netErr.Timeout() + } + var errorCoder errcode.ErrorCoder + if errors.As(err, &errorCoder) { // note: we explicitly do not check errcode.ErrorCodeUnknown because that is used in // a wide range of scenarios to convey "generic error", not "retryable error" - switch t.ErrorCode() { + switch errorCoder.ErrorCode() { case errcode.ErrorCodeUnavailable: return 5 * time.Second, true case errcode.ErrorCodeTooManyRequests: return 2 * time.Second, true } - case *registryclient.UnexpectedHTTPResponseError: - switch t.StatusCode { + } + var unexpectedErr *registryclient.UnexpectedHTTPResponseError + if errors.As(err, &unexpectedErr) { + switch unexpectedErr.StatusCode { case http.StatusInternalServerError, http.StatusGatewayTimeout, http.StatusServiceUnavailable, http.StatusBadGateway: return 5 * time.Second, true case http.StatusTooManyRequests: @@ -692,7 +697,7 @@ func (r *readSeekCloserVerifier) Read(p []byte) (n int, err error) { if n > 0 { r.hash.Write(p[:n]) } - if err == io.EOF { + if errors.Is(err, io.EOF) { actual := digest.NewDigest(r.expect.Algorithm(), r.hash) if actual != r.expect { return n, fmt.Errorf("content integrity error: the blob streamed from digest %s does not match the digest calculated from the content %s", r.expect, actual) diff --git a/support/thirdparty/oc/pkg/cli/image/manifest/manifest.go b/support/thirdparty/oc/pkg/cli/image/manifest/manifest.go index 616cc9cf6261..5975074987a1 100644 --- a/support/thirdparty/oc/pkg/cli/image/manifest/manifest.go +++ b/support/thirdparty/oc/pkg/cli/image/manifest/manifest.go @@ -146,12 +146,12 @@ func ManifestToImageConfig(ctx context.Context, srcManifest distribution.Manifes } configJSON, err := blobs.Get(ctx, t.Config.Digest) if err != nil { - return nil, nil, fmt.Errorf("cannot retrieve image configuration for %s: %v", location, err) + return nil, nil, fmt.Errorf("cannot retrieve image configuration for %s: %w", location, err) } klog.V(4).Infof("Raw image config json:\n%s", string(configJSON)) config := &dockerv1client.DockerImageConfig{} if err := json.Unmarshal(configJSON, &config); err != nil { - return nil, nil, fmt.Errorf("unable to parse image configuration: %v", err) + return nil, nil, fmt.Errorf("unable to parse image configuration: %w", err) } base := config @@ -169,12 +169,12 @@ func ManifestToImageConfig(ctx context.Context, srcManifest distribution.Manifes } configJSON, err := blobs.Get(ctx, t.Config.Digest) if err != nil { - return nil, nil, fmt.Errorf("cannot retrieve image configuration for %s: %v", location, err) + return nil, nil, fmt.Errorf("cannot retrieve image configuration for %s: %w", location, err) } klog.V(4).Infof("Raw image config json:\n%s", string(configJSON)) config := &dockerv1client.DockerImageConfig{} if err := json.Unmarshal(configJSON, &config); err != nil { - return nil, nil, fmt.Errorf("unable to parse image configuration: %v", err) + return nil, nil, fmt.Errorf("unable to parse image configuration: %w", err) } base := config @@ -219,11 +219,11 @@ func ProcessManifestList(ctx context.Context, srcDigest digest.Digest, srcManife var err error t, err = manifestlist.FromDescriptors(filtered) if err != nil { - return nil, nil, "", fmt.Errorf("unable to filter source image %s manifest list: %v", ref, err) + return nil, nil, "", fmt.Errorf("unable to filter source image %s manifest list: %w", ref, err) } _, body, err := t.Payload() if err != nil { - return nil, nil, "", fmt.Errorf("unable to filter source image %s manifest list (bad payload): %v", ref, err) + return nil, nil, "", fmt.Errorf("unable to filter source image %s manifest list (bad payload): %w", ref, err) } manifestList = t manifestDigest, err = registryclient.ContentDigestForManifest(t, srcDigest.Algorithm()) @@ -236,7 +236,7 @@ func ProcessManifestList(ctx context.Context, srcDigest digest.Digest, srcManife for i, manifest := range filtered { childManifest, err := manifests.Get(ctx, manifest.Digest, PreferManifestList) if err != nil { - return nil, nil, "", fmt.Errorf("unable to retrieve source image %s manifest #%d from manifest list: %v", ref, i+1, err) + return nil, nil, "", fmt.Errorf("unable to retrieve source image %s manifest #%d from manifest list: %w", ref, i+1, err) } childManifests = append(childManifests, childManifest) } diff --git a/support/util/util.go b/support/util/util.go index 20ba8b2a010b..4b335251c026 100644 --- a/support/util/util.go +++ b/support/util/util.go @@ -440,7 +440,7 @@ func SanitizeIgnitionPayload(payload []byte) error { var jsonPayload ignitionapi.Config if err := json.Unmarshal(payload, &jsonPayload); err != nil { - return fmt.Errorf("error unmarshalling Ignition payload: %v", err) + return fmt.Errorf("error unmarshalling Ignition payload: %w", err) } return nil diff --git a/support/validations/authentication.go b/support/validations/authentication.go index 12a98f0326a7..6c1212b03dcd 100644 --- a/support/validations/authentication.go +++ b/support/validations/authentication.go @@ -68,7 +68,7 @@ func ValidateAuthenticationSpecForTypeOIDC(ctx context.Context, client crclient. apiServerAuthConfig, err := kas.HCPAuthConfigToAPIServerAuthConfig(authConfig) if err != nil { - return fmt.Errorf("converting from HCP auth config type to apiserver auth config type: %v", err) + return fmt.Errorf("converting from HCP auth config type to apiserver auth config type: %w", err) } fieldErrors := validation.ValidateAuthenticationConfiguration(celCompiler, apiServerAuthConfig, disallowIssuers) diff --git a/sync-fg-configmap/update.go b/sync-fg-configmap/update.go index a4df75ab3e8b..b82e8aaba6e2 100644 --- a/sync-fg-configmap/update.go +++ b/sync-fg-configmap/update.go @@ -40,7 +40,7 @@ func NewRunCommand() *cobra.Command { Name: "feature-gate", PayloadVersion: os.Getenv("PAYLOAD_VERSION"), } - cmd.Flags().StringVar(&opts.File, "file", opts.File, "The path path to the file that contains the feature gate YAML to apply.") + cmd.Flags().StringVar(&opts.File, "file", opts.File, "The path to the file that contains the feature gate YAML to apply.") cmd.Flags().StringVar(&opts.Namespace, "namespace", opts.Namespace, "The control plane namespace for the feature gate configmap.") cmd.Flags().StringVar(&opts.Name, "name", opts.Name, "The name of the feature gate configmap.") cmd.Flags().StringVar(&opts.PayloadVersion, "payload-version", opts.PayloadVersion, "The payload version of the control plane.") diff --git a/test/e2e/autoscaling_test.go b/test/e2e/autoscaling_test.go index ce3251e6d46c..e557acc1d376 100644 --- a/test/e2e/autoscaling_test.go +++ b/test/e2e/autoscaling_test.go @@ -3,6 +3,7 @@ package e2e import ( + "bufio" "context" "fmt" "strings" @@ -20,6 +21,8 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + kubeclient "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/retry" "k8s.io/utils/ptr" capiv1 "sigs.k8s.io/cluster-api/api/v1beta1" @@ -77,6 +80,8 @@ func TestAutoscaling(t *testing.T) { e2eutil.NewHypershiftTest(t, ctx, func(t *testing.T, g Gomega, mgtClient crclient.Client, hostedCluster *hyperv1.HostedCluster) { t.Run("TestAutoscaling", testAutoscaling(ctx, mgtClient, hostedCluster, clusterOpts.NodePoolReplicas, clusterOpts.NodePoolReplicas+2)) + t.Run("TestAutoscalerRespectsNodePoolPause", testAutoscalerRespectsNodePoolPause(ctx, mgtClient, hostedCluster, clusterOpts.NodePoolReplicas, clusterOpts.NodePoolReplicas+2)) + t.Run("TestAutoscalingBalancing", testAutoscalingBalancing(ctx, mgtClient, hostedCluster, clusterOpts.NodePoolReplicas*2, additionalNP)) }).WithAssetReader(content.ReadFile).Execute(&clusterOpts, globalOpts.Platform, globalOpts.ArtifactDir, "autoscaling", globalOpts.ServiceAccountSigningKey) @@ -89,14 +94,7 @@ func testAutoscaling(ctx context.Context, mgtClient crclient.Client, hostedClust defer cancel() // Get the newly created NodePool - nodepools := &hyperv1.NodePoolList{} - if err := mgtClient.List(ctx, nodepools, crclient.InNamespace(hostedCluster.Namespace)); err != nil { - t.Fatalf("failed to list nodepools in namespace %s: %v", hostedCluster.Namespace, err) - } - if len(nodepools.Items) != 1 { - t.Fatalf("expected exactly one nodepool, got %d", len(nodepools.Items)) - } - nodepool := &nodepools.Items[0] + nodepool := getOnlyNodePool(t, ctx, mgtClient, hostedCluster.Namespace) // Perform some very basic assertions about the guest cluster guestClient := e2eutil.WaitForGuestClient(t, ctx, mgtClient, hostedCluster) @@ -173,14 +171,7 @@ func testAutoscalingBalancing(ctx context.Context, mgtClient crclient.Client, ho e2eutil.AtLeast(t, e2eutil.Version420) // Get the newly created NodePool - nodepools := &hyperv1.NodePoolList{} - if err := mgtClient.List(ctx, nodepools, crclient.InNamespace(hostedCluster.Namespace)); err != nil { - t.Fatalf("failed to list nodepools in namespace %s: %v", hostedCluster.Namespace, err) - } - if len(nodepools.Items) != 1 { - t.Fatalf("expected exactly one nodepool, got %d", len(nodepools.Items)) - } - defaultNodePool := &nodepools.Items[0] + defaultNodePool := getOnlyNodePool(t, ctx, mgtClient, hostedCluster.Namespace) // create additional NodePool if additionalNP != nil { @@ -257,22 +248,7 @@ func testAutoscalingBalancing(ctx context.Context, mgtClient crclient.Client, ho // Wait for autoscaler deployment to have autoscaling settings and be ready // TODO (cewong): This should be reported in the HostedCluster as a condition controlPlaneNamespace := manifests.HostedControlPlaneNamespace(hostedCluster.Namespace, hostedCluster.Name) - e2eutil.EventuallyObject(t, ctx, "autoscaler deployment to have autoscaling settings and be ready", func(ctx context.Context) (*appsv1.Deployment, error) { - autoscalerDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Namespace: controlPlaneNamespace, Name: "cluster-autoscaler"}} - err := mgtClient.Get(ctx, crclient.ObjectKeyFromObject(autoscalerDeployment), autoscalerDeployment) - return autoscalerDeployment, err - }, []e2eutil.Predicate[*appsv1.Deployment]{func(autoscalerDeployment *appsv1.Deployment) (done bool, reasons string, err error) { - hasBalancingIgnoreLabel := false - for _, arg := range autoscalerDeployment.Spec.Template.Spec.Containers[0].Args { - if strings.Contains(arg, "custom.ignore.label") { - hasBalancingIgnoreLabel = true - } - } - if !hasBalancingIgnoreLabel { - return false, "autoscaler deployment does not have balancing ignore label", nil - } - return podspec.IsDeploymentReady(ctx, autoscalerDeployment), "autoscaler deployment not ready", nil - }}, e2eutil.WithInterval(10*time.Second), e2eutil.WithTimeout(5*time.Minute)) + waitForAutoscalerDeploymentReady(t, ctx, mgtClient, controlPlaneNamespace, []string{"custom.ignore.label"}) // Generate workload. memCapacity := nodes[0].Status.Allocatable[corev1.ResourceMemory] @@ -387,6 +363,316 @@ func newWorkLoad(njobs int32, memoryRequest resource.Quantity, nodeSelector, ima return job } +const ( + aggressiveDelayAfterAddSeconds int32 = 30 + aggressiveUnneededDurationSeconds int32 = 60 + casScaleDownWaitDuration = time.Duration(aggressiveUnneededDurationSeconds+aggressiveDelayAfterAddSeconds)*time.Second + 90*time.Second + casLogPollInterval = 5 * time.Second + casPausedLogMessage = "discovered a paused node group" +) + +// getOnlyNodePool lists NodePools in the given namespace and asserts exactly one exists. +// It returns the single NodePool, failing the test if zero or more than one are found. +func getOnlyNodePool(t *testing.T, ctx context.Context, mgtClient crclient.Client, namespace string) *hyperv1.NodePool { + t.Helper() + g := NewWithT(t) + nodepools := &hyperv1.NodePoolList{} + err := mgtClient.List(ctx, nodepools, crclient.InNamespace(namespace)) + g.Expect(err).NotTo(HaveOccurred(), "failed to list nodepools") + g.Expect(nodepools.Items).To(HaveLen(1), "expected exactly one nodepool") + return &nodepools.Items[0] +} + +// waitForAutoscalerDeploymentReady waits for the cluster-autoscaler deployment in the given +// control plane namespace to contain all requiredArgs and be ready. +func waitForAutoscalerDeploymentReady(t *testing.T, ctx context.Context, mgtClient crclient.Client, controlPlaneNamespace string, requiredArgs []string) { + t.Helper() + e2eutil.EventuallyObject(t, ctx, "autoscaler deployment to have required settings and be ready", + func(ctx context.Context) (*appsv1.Deployment, error) { + dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Namespace: controlPlaneNamespace, Name: "cluster-autoscaler"}} + err := mgtClient.Get(ctx, crclient.ObjectKeyFromObject(dep), dep) + return dep, err + }, + []e2eutil.Predicate[*appsv1.Deployment]{ + func(dep *appsv1.Deployment) (done bool, reasons string, err error) { + for _, required := range requiredArgs { + found := false + for _, arg := range dep.Spec.Template.Spec.Containers[0].Args { + if strings.Contains(arg, required) { + found = true + break + } + } + if !found { + return false, fmt.Sprintf("autoscaler deployment missing %s arg", required), nil + } + } + return podspec.IsDeploymentReady(ctx, dep), "autoscaler deployment not ready", nil + }, + }, + e2eutil.WithInterval(10*time.Second), + e2eutil.WithTimeout(5*time.Minute), + ) +} + +// pollCASLogsForPausedNodeGroup polls the cluster-autoscaler pod logs for the +// "discovered a paused node group" message, which the CAS fix emits at klog.V(4) +// when it skips a paused MachineDeployment. Returns true if the message was found +// before the timeout, false otherwise. +func pollCASLogsForPausedNodeGroup(t *testing.T, ctx context.Context, controlPlaneNamespace string, timeout time.Duration) bool { + t.Helper() + + cfg, err := e2eutil.GetConfig() + if err != nil { + t.Fatalf("Failed to get management cluster config: %v", err) + } + kubeClient := kubeclient.NewForConfigOrDie(cfg) + + sinceTime := metav1.Now() + t.Logf("Polling CAS logs for paused node group detection (timeout=%s)...", timeout) + + err = wait.PollUntilContextTimeout(ctx, casLogPollInterval, timeout, true, func(ctx context.Context) (bool, error) { + pods, err := kubeClient.CoreV1().Pods(controlPlaneNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app=cluster-autoscaler", + }) + if err != nil || len(pods.Items) == 0 { + return false, nil + } + + stream, err := kubeClient.CoreV1().Pods(controlPlaneNamespace).GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{ + Container: "cluster-autoscaler", + SinceTime: &sinceTime, + }).Stream(ctx) + if err != nil { + return false, nil + } + defer stream.Close() + + scanner := bufio.NewScanner(stream) + for scanner.Scan() { + if strings.Contains(scanner.Text(), casPausedLogMessage) { + return true, nil + } + } + return false, nil + }) + + if err != nil { + t.Logf("Timeout waiting for CAS paused node group log — proceeding with replica verification") + return false + } + t.Logf("CAS log confirms paused node group was detected and skipped") + return true +} + +// testAutoscalerRespectsNodePoolPause is a regression test for OCPBUGS-78152 / CNTRLPLANE-3040. +// +// It verifies that the Cluster Autoscaler does NOT decrement MachineDeployment.spec.replicas +// on paused MachineDeployments. Without the CAS fix, CAS would accumulate replica decrements +// while the MachineDeployment is paused (because machines are never deleted), causing +// catastrophic scale-down on unpause. +// +// Test flow: +// 1. Enable autoscaling on the NodePool (self-contained) +// 2. Configure aggressive CAS scale-down timers +// 3. Scale up the NodePool to max nodes with workload +// 4. Pause the NodePool (propagates cluster.x-k8s.io/paused to MachineDeployment) +// 5. Delete the workload so nodes become unneeded +// 6. Poll CAS logs for "discovered a paused node group" (or timeout) +// 7. Verify MachineDeployment.spec.replicas has NOT been decremented +// 8. Unpause and wait for clean convergence to numNodes +func testAutoscalerRespectsNodePoolPause(ctx context.Context, mgtClient crclient.Client, hostedCluster *hyperv1.HostedCluster, numNodes, max int32) func(t *testing.T) { + return func(t *testing.T) { + g := NewWithT(t) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Get the NodePool. + nodepool := getOnlyNodePool(t, ctx, mgtClient, hostedCluster.Namespace) + + // Step 1: Ensure autoscaling is enabled on the NodePool so this test is + // self-contained and does not depend on a prior subtest. + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := mgtClient.Get(ctx, crclient.ObjectKeyFromObject(nodepool), nodepool); err != nil { + return err + } + nodepool.Spec.AutoScaling = &hyperv1.NodePoolAutoScaling{ + Min: ptr.To[int32](numNodes), + Max: max, + } + nodepool.Spec.Replicas = nil + return mgtClient.Update(ctx, nodepool) + }) + g.Expect(err).NotTo(HaveOccurred(), "failed to enable autoscaling on NodePool") + t.Logf("Enabled autoscaling on NodePool %s (min=%d, max=%d)", nodepool.Name, numNodes, max) + + guestClient := e2eutil.WaitForGuestClient(t, ctx, mgtClient, hostedCluster) + nodes := e2eutil.WaitForNReadyNodes(t, ctx, guestClient, numNodes, hostedCluster.Spec.Platform.Type) + t.Logf("Cluster has %d ready nodes", len(nodes)) + + // Step 2: Configure aggressive CAS scale-down timers on the HostedCluster. + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := mgtClient.Get(ctx, crclient.ObjectKeyFromObject(hostedCluster), hostedCluster); err != nil { + return err + } + hostedCluster.Spec.Autoscaling = hyperv1.ClusterAutoscaling{ + Scaling: hyperv1.ScaleUpAndScaleDown, + ScaleDown: &hyperv1.ScaleDownConfig{ + DelayAfterAddSeconds: ptr.To(aggressiveDelayAfterAddSeconds), + UnneededDurationSeconds: ptr.To(aggressiveUnneededDurationSeconds), + UtilizationThresholdPercent: ptr.To[int32](50), + }, + } + return mgtClient.Update(ctx, hostedCluster) + }) + g.Expect(err).NotTo(HaveOccurred(), "failed to configure autoscaling on HostedCluster") + t.Log("Configured aggressive CAS scale-down timers (unneeded=60s, delayAfterAdd=30s)") + + // Wait for autoscaler deployment to have the aggressive scale-down args and be ready. + controlPlaneNamespace := manifests.HostedControlPlaneNamespace(hostedCluster.Namespace, hostedCluster.Name) + waitForAutoscalerDeploymentReady(t, ctx, mgtClient, controlPlaneNamespace, []string{ + "--scale-down-unneeded-time=60s", + "--scale-down-delay-after-add=30s", + }) + t.Log("Autoscaler deployment is ready with aggressive scale-down settings") + + // Step 3: Generate workload to scale up to max nodes. + memCapacity := nodes[0].Status.Allocatable[corev1.ResourceMemory] + g.Expect(memCapacity).ShouldNot(BeNil()) + g.Expect(memCapacity.String()).ShouldNot(BeEmpty()) + bytes, ok := memCapacity.AsInt64() + g.Expect(ok).Should(BeTrue()) + + // 50% - enough that the existing and new nodes will be used, not enough to have more than 1 pod per node. + workloadMemRequest := resource.MustParse(fmt.Sprintf("%v", 0.5*float32(bytes))) + workload := newWorkLoad(max, workloadMemRequest, "", globalOpts.LatestReleaseImage) + err = guestClient.Create(ctx, workload) + g.Expect(err).NotTo(HaveOccurred()) + t.Logf("Created workload. Node: %s, memcapacity: %s", nodes[0].Name, memCapacity.String()) + defer func() { + cascadeDelete := metav1.DeletePropagationForeground + if err := guestClient.Delete(ctx, workload, &crclient.DeleteOptions{ + PropagationPolicy: &cascadeDelete, + }); err != nil { + t.Logf("cleanup: failed to delete workload: %v", err) + } + }() + + // Wait for max nodes. + _ = e2eutil.WaitForNReadyNodes(t, ctx, guestClient, max, hostedCluster.Spec.Platform.Type) + t.Logf("Scaled up to %d nodes", max) + + // Step 4: Pause the NodePool. This propagates cluster.x-k8s.io/paused to the MachineDeployment. + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := mgtClient.Get(ctx, crclient.ObjectKeyFromObject(nodepool), nodepool); err != nil { + return err + } + nodepool.Spec.PausedUntil = ptr.To("true") + return mgtClient.Update(ctx, nodepool) + }) + g.Expect(err).NotTo(HaveOccurred(), "failed to pause NodePool") + t.Logf("Paused NodePool %s", nodepool.Name) + defer func() { + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := mgtClient.Get(ctx, crclient.ObjectKeyFromObject(nodepool), nodepool); err != nil { + return err + } + if nodepool.Spec.PausedUntil == nil { + return nil + } + nodepool.Spec.PausedUntil = nil + return mgtClient.Update(ctx, nodepool) + }); err != nil { + t.Logf("cleanup: failed to unpause NodePool %s: %v", nodepool.Name, err) + } + }() + + // Verify the MachineDeployment has the pause annotation. + md := &capiv1.MachineDeployment{} + e2eutil.EventuallyObject(t, ctx, "MachineDeployment to be paused", + func(ctx context.Context) (*capiv1.MachineDeployment, error) { + err := mgtClient.Get(ctx, crclient.ObjectKey{Namespace: controlPlaneNamespace, Name: nodepool.Name}, md) + return md, err + }, + []e2eutil.Predicate[*capiv1.MachineDeployment]{ + func(md *capiv1.MachineDeployment) (done bool, reasons string, err error) { + if md.Annotations[capiv1.PausedAnnotation] == "true" { + return true, "MachineDeployment is paused", nil + } + return false, "MachineDeployment not yet paused", nil + }, + }, + e2eutil.WithTimeout(2*time.Minute), + ) + + // Record the replica count before CAS has a chance to act. + replicasBeforePause := *md.Spec.Replicas + t.Logf("MachineDeployment is paused with %d replicas", replicasBeforePause) + + // Step 5: Delete workload so nodes become unneeded. + cascadeDelete := metav1.DeletePropagationForeground + err = guestClient.Delete(ctx, workload, &crclient.DeleteOptions{ + PropagationPolicy: &cascadeDelete, + }) + g.Expect(err).NotTo(HaveOccurred()) + t.Logf("Deleted workload") + + // Step 6: Poll CAS pod logs for evidence it evaluated the paused node group. + // The CAS fix logs "discovered a paused node group" at klog.V(4) when it + // skips a paused MachineDeployment. We poll for this line instead of blindly + // sleeping, which is both faster and deterministic. + pauseDetected := pollCASLogsForPausedNodeGroup(t, ctx, controlPlaneNamespace, casScaleDownWaitDuration) + + // Step 7: Verify MachineDeployment replicas have NOT been decremented. + err = mgtClient.Get(ctx, crclient.ObjectKey{Namespace: controlPlaneNamespace, Name: nodepool.Name}, md) + g.Expect(err).NotTo(HaveOccurred(), "failed to get MachineDeployment") + + replicasAfterWait := *md.Spec.Replicas + if pauseDetected { + t.Logf("CAS confirmed paused node group — replicas after CAS evaluation: %d (was %d before pause)", replicasAfterWait, replicasBeforePause) + } else { + t.Logf("CAS log not detected within timeout — replicas after wait: %d (was %d before pause)", replicasAfterWait, replicasBeforePause) + } + g.Expect(replicasAfterWait).To(Equal(replicasBeforePause), + "MachineDeployment replicas should not change while paused — CAS should not decrement replicas on paused MachineDeployments (regression for OCPBUGS-78152)") + + // Step 8: Unpause the NodePool. + err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + if err := mgtClient.Get(ctx, crclient.ObjectKeyFromObject(nodepool), nodepool); err != nil { + return err + } + nodepool.Spec.PausedUntil = nil + return mgtClient.Update(ctx, nodepool) + }) + g.Expect(err).NotTo(HaveOccurred(), "failed to unpause NodePool") + t.Logf("Unpaused NodePool %s", nodepool.Name) + + // Verify the MachineDeployment pause annotation is removed. + e2eutil.EventuallyObject(t, ctx, "MachineDeployment to be unpaused", + func(ctx context.Context) (*capiv1.MachineDeployment, error) { + err := mgtClient.Get(ctx, crclient.ObjectKey{Namespace: controlPlaneNamespace, Name: nodepool.Name}, md) + return md, err + }, + []e2eutil.Predicate[*capiv1.MachineDeployment]{ + func(md *capiv1.MachineDeployment) (done bool, reasons string, err error) { + if md.Annotations[capiv1.PausedAnnotation] != "true" { + return true, "MachineDeployment is unpaused", nil + } + return false, "MachineDeployment still paused", nil + }, + }, + e2eutil.WithTimeout(2*time.Minute), + ) + t.Log("MachineDeployment is unpaused") + + // After unpause, CAS may resume normal scale-down immediately since the + // workload is gone and aggressive timers are satisfied. Verify the cluster + // converges cleanly back to the autoscaling minimum. + _ = e2eutil.WaitForNReadyNodes(t, ctx, guestClient, numNodes, hostedCluster.Spec.Platform.Type) + t.Logf("Scaled down to %d nodes after unpause — clean state for next test", numNodes) + } +} + func TestNodePoolAutoscalingScaleFromZero(t *testing.T) { if globalOpts.Platform != hyperv1.AWSPlatform { t.Skip("test only supported on platform AWS") diff --git a/test/e2e/karpenter_test.go b/test/e2e/karpenter_test.go index e93c189a4b2f..09599c9ca6ab 100644 --- a/test/e2e/karpenter_test.go +++ b/test/e2e/karpenter_test.go @@ -270,14 +270,16 @@ func testARM64Provisioning(ctx context.Context, guestClient crclient.Client, hos t.Skip("test only supported on multi-arch clusters") } + hc := hostedCluster.DeepCopy() + armNodeClass := &hyperkarpenterv1.OpenshiftEC2NodeClass{ ObjectMeta: metav1.ObjectMeta{Name: "arm-nodeclass"}, Spec: hyperkarpenterv1.OpenshiftEC2NodeClassSpec{ SubnetSelectorTerms: []hyperkarpenterv1.SubnetSelectorTerm{ - {Tags: map[string]string{"karpenter.sh/discovery": hostedCluster.Spec.InfraID}}, + {Tags: map[string]string{"karpenter.sh/discovery": hc.Spec.InfraID}}, }, SecurityGroupSelectorTerms: []hyperkarpenterv1.SecurityGroupSelectorTerm{ - {Tags: map[string]string{"karpenter.sh/discovery": hostedCluster.Spec.InfraID}}, + {Tags: map[string]string{"karpenter.sh/discovery": hc.Spec.InfraID}}, }, }, } @@ -309,7 +311,7 @@ func testARM64Provisioning(ctx context.Context, guestClient crclient.Client, hos "kubernetes.io/arch": "arm64", } - nodes := e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hostedCluster.Spec.Platform.Type, 1, armNodeLabels) + nodes := e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hc.Spec.Platform.Type, 1, armNodeLabels) waitForReadyKarpenterPods(t, ctx, guestClient, nodes, 1, map[string]string{"app": "arm-app"}) g.Expect(guestClient.Delete(ctx, armNodePool)).To(Succeed()) @@ -318,7 +320,7 @@ func testARM64Provisioning(ctx context.Context, guestClient crclient.Client, hos t.Logf("Deleted ARM64 workloads") t.Logf("Waiting for Karpenter ARM64 Nodes to disappear") - _ = e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hostedCluster.Spec.Platform.Type, 0, armNodeLabels) + _ = e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hc.Spec.Platform.Type, 0, armNodeLabels) } } @@ -327,15 +329,15 @@ func testInstanceProfileAnnotation(ctx context.Context, mgtClient, guestClient c t.Parallel() g := NewWithT(t) - // Get the current HostedCluster - err := mgtClient.Get(ctx, crclient.ObjectKeyFromObject(hostedCluster), hostedCluster) + hc := hostedCluster.DeepCopy() + err := mgtClient.Get(ctx, crclient.ObjectKeyFromObject(hc), hc) g.Expect(err).NotTo(HaveOccurred()) // Use the default worker instance profile (typically {infraID}-worker) - workerInstanceProfile := hostedCluster.Spec.InfraID + "-worker" + workerInstanceProfile := hc.Spec.InfraID + "-worker" // Apply the annotation to the HostedCluster - err = e2eutil.UpdateObject(t, ctx, mgtClient, hostedCluster, func(obj *hyperv1.HostedCluster) { + err = e2eutil.UpdateObject(t, ctx, mgtClient, hc, func(obj *hyperv1.HostedCluster) { if obj.Annotations == nil { obj.Annotations = make(map[string]string) } @@ -377,7 +379,7 @@ func testInstanceProfileAnnotation(ctx context.Context, mgtClient, guestClient c karpenterv1.NodePoolLabelKey: testNodePool.Name, } - nodes := e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hostedCluster.Spec.Platform.Type, 1, testNodeLabels) + nodes := e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hc.Spec.Platform.Type, 1, testNodeLabels) t.Logf("Karpenter nodes are ready") // Verify EC2 instances have the correct instance profile @@ -403,13 +405,13 @@ func testInstanceProfileAnnotation(ctx context.Context, mgtClient, guestClient c t.Logf("Waiting for Karpenter nodes to be deleted") g.Expect(guestClient.Delete(ctx, testWorkLoads)).To(Succeed()) g.Expect(guestClient.Delete(ctx, testNodePool)).To(Succeed()) - _ = e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hostedCluster.Spec.Platform.Type, 0, testNodeLabels) + _ = e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hc.Spec.Platform.Type, 0, testNodeLabels) // Remove the annotation and verify it gets cleared from EC2NodeClass - err = mgtClient.Get(ctx, crclient.ObjectKeyFromObject(hostedCluster), hostedCluster) + err = mgtClient.Get(ctx, crclient.ObjectKeyFromObject(hc), hc) g.Expect(err).NotTo(HaveOccurred()) - err = e2eutil.UpdateObject(t, ctx, mgtClient, hostedCluster, func(obj *hyperv1.HostedCluster) { + err = e2eutil.UpdateObject(t, ctx, mgtClient, hc, func(obj *hyperv1.HostedCluster) { delete(obj.Annotations, hyperv1.AWSKarpenterDefaultInstanceProfile) }) g.Expect(err).NotTo(HaveOccurred()) @@ -432,13 +434,13 @@ func testNodeClassVersionField(ctx context.Context, mgtClient, guestClient crcli t.Parallel() g := NewWithT(t) - // Re-fetch the hosted cluster to get the latest version status - err := mgtClient.Get(ctx, crclient.ObjectKeyFromObject(hostedCluster), hostedCluster) + hc := hostedCluster.DeepCopy() + err := mgtClient.Get(ctx, crclient.ObjectKeyFromObject(hc), hc) g.Expect(err).NotTo(HaveOccurred()) - g.Expect(hostedCluster.Status.Version).NotTo(BeNil(), "hostedCluster.Status.Version should not be nil") - g.Expect(hostedCluster.Status.Version.Desired.Version).NotTo(BeEmpty(), "hostedCluster.Status.Version.Desired.Version should not be empty") + g.Expect(hc.Status.Version).NotTo(BeNil(), "hostedCluster.Status.Version should not be nil") + g.Expect(hc.Status.Version.Desired.Version).NotTo(BeEmpty(), "hostedCluster.Status.Version.Desired.Version should not be empty") - cpVersion, err := semver.Parse(hostedCluster.Status.Version.Desired.Version) + cpVersion, err := semver.Parse(hc.Status.Version.Desired.Version) g.Expect(err).NotTo(HaveOccurred(), "failed to parse control plane version") t.Logf("Control plane version: %s", cpVersion.String()) @@ -465,8 +467,8 @@ func testNodeClassVersionField(ctx context.Context, mgtClient, guestClient crcli if nc.Status.ReleaseImage == "" { return false, "status.releaseImage is empty", nil } - if nc.Status.ReleaseImage != hostedCluster.Spec.Release.Image { - return false, fmt.Sprintf("expected status.releaseImage %q to match hostedCluster.Spec.Release.Image %q", nc.Status.ReleaseImage, hostedCluster.Spec.Release.Image), nil + if nc.Status.ReleaseImage != hc.Spec.Release.Image { + return false, fmt.Sprintf("expected status.releaseImage %q to match hostedCluster.Spec.Release.Image %q", nc.Status.ReleaseImage, hc.Spec.Release.Image), nil } return true, fmt.Sprintf("status.releaseImage matches control plane: %s", nc.Status.ReleaseImage), nil }), @@ -488,10 +490,10 @@ func testNodeClassVersionField(ctx context.Context, mgtClient, guestClient crcli Spec: hyperkarpenterv1.OpenshiftEC2NodeClassSpec{ Version: nodeClassVersion, SubnetSelectorTerms: []hyperkarpenterv1.SubnetSelectorTerm{ - {Tags: map[string]string{"karpenter.sh/discovery": hostedCluster.Spec.InfraID}}, + {Tags: map[string]string{"karpenter.sh/discovery": hc.Spec.InfraID}}, }, SecurityGroupSelectorTerms: []hyperkarpenterv1.SecurityGroupSelectorTerm{ - {Tags: map[string]string{"karpenter.sh/discovery": hostedCluster.Spec.InfraID}}, + {Tags: map[string]string{"karpenter.sh/discovery": hc.Spec.InfraID}}, }, MetadataOptions: hyperkarpenterv1.MetadataOptions{ Access: hyperkarpenterv1.MetadataAccessHTTPEndpoint, @@ -605,7 +607,7 @@ func testNodeClassVersionField(ctx context.Context, mgtClient, guestClient crcli } // Log diagnostic info about the version-test NodeClass infrastructure. - hcpNamespace := manifests.HostedControlPlaneNamespace(hostedCluster.Namespace, hostedCluster.Name) + hcpNamespace := manifests.HostedControlPlaneNamespace(hc.Namespace, hc.Name) secretList := &corev1.SecretList{} if err := mgtClient.List(ctx, secretList, crclient.InNamespace(hcpNamespace), @@ -666,7 +668,7 @@ func testNodeClassVersionField(ctx context.Context, mgtClient, guestClient crcli // NodeClaims don't leak vCPUs into subsequent sequential tests. g.Expect(guestClient.Delete(ctx, testWorkLoads)).To(Succeed()) g.Expect(guestClient.Delete(ctx, testNodePool)).To(Succeed()) - _ = e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hostedCluster.Spec.Platform.Type, 0, testNodeLabels) + _ = e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hc.Spec.Platform.Type, 0, testNodeLabels) // Verify that a version exceeding the allowed n-3 skew sets SupportedVersionSkew=False. skewMajor, skewMinor, err := supportedversion.PreviousMinorVersion(cpVersion, 4) @@ -741,9 +743,11 @@ func testCapacityReservation(ctx context.Context, mgtClient, guestClient crclien t.Parallel() g := NewWithT(t) + hc := hostedCluster.DeepCopy() + // AutoNode.Provisioner.Karpenter.AWS is required at the API level when karpenter // is configured, so this should never happen/never be nil for a valid karpenter cluster. - if hostedCluster.Spec.AutoNode.Provisioner.Karpenter.Platform != hyperv1.AWSPlatform { + if hc.Spec.AutoNode.Provisioner.Karpenter.Platform != hyperv1.AWSPlatform { t.Skip("HostedCluster does not have a Karpenter AWS platform configured, skipping capacity reservation test") } @@ -866,7 +870,7 @@ func testCapacityReservation(ctx context.Context, mgtClient, guestClient crclien t.Logf("Created workload capacity-reservation-web-app to trigger node provisioning") // Wait for the node to be ready. - nodes := e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hostedCluster.Spec.Platform.Type, 1, crNodeLabels) + nodes := e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hc.Spec.Platform.Type, 1, crNodeLabels) g.Expect(nodes).To(HaveLen(1)) t.Logf("Node provisioned by capacity-reservation-test NodePool is ready") @@ -884,7 +888,7 @@ func testCapacityReservation(ctx context.Context, mgtClient, guestClient crclien // so stale NodeClaims don't leak vCPUs into subsequent sequential tests. g.Expect(guestClient.Delete(ctx, crWorkload)).To(Succeed()) g.Expect(guestClient.Delete(ctx, crNodePool)).To(Succeed()) - _ = e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hostedCluster.Spec.Platform.Type, 0, crNodeLabels) + _ = e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hc.Spec.Platform.Type, 0, crNodeLabels) } } @@ -893,13 +897,15 @@ func testArbitrarySubnet(ctx context.Context, mgtClient, guestClient crclient.Cl t.Parallel() g := NewWithT(t) + hc := hostedCluster.DeepCopy() + // Get VPC ID and find an AZ that is: // (a) supported by the VPC endpoint service (to avoid InvalidParameter), and // (b) not already occupied by a VPC subnet (to avoid DuplicateSubnetsInSameZone). // This exercises the real scenario: a customer brings a subnet in a new AZ, // it propagates to the VPC endpoint, and nodes in that AZ can reach the cluster. ec2client := ec2Client(awsCredsFile, awsRegion) - vpcID := hostedCluster.Spec.Platform.AWS.CloudProviderConfig.VPC + vpcID := hc.Spec.Platform.AWS.CloudProviderConfig.VPC subnetsOut, err := ec2client.DescribeSubnets(ctx, &ec2.DescribeSubnetsInput{ Filters: []ec2types.Filter{{Name: aws.String("vpc-id"), Values: []string{vpcID}}}, }) @@ -913,7 +919,7 @@ func testArbitrarySubnet(ctx context.Context, mgtClient, guestClient crclient.Cl } // Get the AZs supported by the VPC endpoint service. - hcpNamespace := manifests.HostedControlPlaneNamespace(hostedCluster.Namespace, hostedCluster.Name) + hcpNamespace := manifests.HostedControlPlaneNamespace(hc.Namespace, hc.Name) esList := &hyperv1.AWSEndpointServiceList{} g.Expect(mgtClient.List(ctx, esList, crclient.InNamespace(hcpNamespace))).To(Succeed()) g.Expect(esList.Items).NotTo(BeEmpty(), "expected at least one AWSEndpointService") @@ -949,7 +955,7 @@ func testArbitrarySubnet(ctx context.Context, mgtClient, guestClient crclient.Cl t.Logf("Selected AZ %s for test subnet (supported by endpoint service, not in VPC)", az) // Create a small test subnet in the VPC. - subnetID, cleanupSubnet := e2eutil.CreateTestSubnet(ctx, t, ec2client, vpcID, az, hostedCluster.Spec.InfraID) + subnetID, cleanupSubnet := e2eutil.CreateTestSubnet(ctx, t, ec2client, vpcID, az, hc.Spec.InfraID) t.Logf("Created test subnet %s in AZ %s", subnetID, az) // Create an OpenshiftEC2NodeClass that selects the subnet by ID. @@ -958,7 +964,7 @@ func testArbitrarySubnet(ctx context.Context, mgtClient, guestClient crclient.Cl Spec: hyperkarpenterv1.OpenshiftEC2NodeClassSpec{ SubnetSelectorTerms: []hyperkarpenterv1.SubnetSelectorTerm{{ID: subnetID}}, SecurityGroupSelectorTerms: []hyperkarpenterv1.SecurityGroupSelectorTerm{ - {Tags: map[string]string{"karpenter.sh/discovery": hostedCluster.Spec.InfraID}}, + {Tags: map[string]string{"karpenter.sh/discovery": hc.Spec.InfraID}}, }, }, } @@ -970,7 +976,7 @@ func testArbitrarySubnet(ctx context.Context, mgtClient, guestClient crclient.Cl } // Wait for the subnet to be removed from the karpenter-subnets ConfigMap. // The karpenter-operator removes it during NodeClass deletion reconciliation. - hcpNS := manifests.HostedControlPlaneNamespace(hostedCluster.Namespace, hostedCluster.Name) + hcpNS := manifests.HostedControlPlaneNamespace(hc.Namespace, hc.Name) if err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 2*time.Minute, true, func(ctx context.Context) (bool, error) { cm := &corev1.ConfigMap{} if err := mgtClient.Get(ctx, crclient.ObjectKey{ @@ -1134,7 +1140,7 @@ func testArbitrarySubnet(ctx context.Context, mgtClient, guestClient crclient.Cl testNodeLabels := map[string]string{ karpenterv1.NodePoolLabelKey: testNodePool.Name, } - nodes := e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hostedCluster.Spec.Platform.Type, 1, testNodeLabels) + nodes := e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hc.Spec.Platform.Type, 1, testNodeLabels) t.Logf("Node launched in arbitrary subnet, verifying it used subnet %s", subnetID) // Verify the launched node's EC2 instance is in the expected subnet. @@ -1157,7 +1163,9 @@ func testKubeletPropagation(ctx context.Context, mgtClient, guestClient crclient t.Parallel() g := NewWithT(t) - hcpNamespace := manifests.HostedControlPlaneNamespace(hostedCluster.Namespace, hostedCluster.Name) + hc := hostedCluster.DeepCopy() + + hcpNamespace := manifests.HostedControlPlaneNamespace(hc.Namespace, hc.Name) // Create a custom OpenshiftEC2NodeClass that the controller does not manage, so that // reconcileOpenshiftEC2NodeClassDefault cannot overwrite spec.kubelet on every reconcile. @@ -1292,11 +1300,11 @@ func testKubeletPropagation(ctx context.Context, mgtClient, guestClient crclient } // Wait for nodes to be provisioned - e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hostedCluster.Spec.Platform.Type, 1, testNodeLabels) + e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hc.Spec.Platform.Type, 1, testNodeLabels) t.Logf("Karpenter nodes are ready") // Build a clientset for the guest cluster (needed for pod log fetching) - guestConfig := e2eutil.WaitForGuestRestConfig(t, ctx, mgtClient, hostedCluster) + guestConfig := e2eutil.WaitForGuestRestConfig(t, ctx, mgtClient, hc) guestClientset, err := kubeclient.NewForConfig(guestConfig) g.Expect(err).NotTo(HaveOccurred()) @@ -1335,7 +1343,7 @@ func testKubeletPropagation(ctx context.Context, mgtClient, guestClient crclient // Cleanup workloads and NodePool g.Expect(guestClient.Delete(ctx, testWorkLoads)).To(Succeed()) g.Expect(guestClient.Delete(ctx, testNodePool)).To(Succeed()) - _ = e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hostedCluster.Spec.Platform.Type, 0, testNodeLabels) + _ = e2eutil.WaitForReadyNodesByLabels(t, ctx, guestClient, hc.Spec.Platform.Type, 0, testNodeLabels) } } @@ -1428,6 +1436,7 @@ func testBillingConsolidationAndPDB(ctx context.Context, mgtClient, guestClient // Before any Karpenter nodes are provisioned, Karpenter vCPUs should be 0. waitForAutoNodeStatusVCPUs(t, ctx, mgtClient, hostedCluster, 0) + waitForAutoNodeStatusVCPUsStable(t, ctx, mgtClient, hostedCluster, 0, 30*time.Second) baseline, found := getVCPUsMetric(t, ctx, mgtClient, hostedCluster) g.Expect(found).To(BeTrue(), "billing metric should exist before Karpenter nodes are provisioned") @@ -1451,6 +1460,7 @@ func testBillingConsolidationAndPDB(ctx context.Context, mgtClient, guestClient // t3.xlarge = 4 vCPUs; 2 nodes = 8 Karpenter vCPUs on top of baseline waitForAutoNodeStatusVCPUs(t, ctx, mgtClient, hostedCluster, 8) + waitForAutoNodeStatusVCPUsStable(t, ctx, mgtClient, hostedCluster, 8, 30*time.Second) waitForBillingMetricVCPUs(t, ctx, mgtClient, hostedCluster, baseline+8) t.Logf("Scaling workload to 1 replica to verify deprovisioning and consolidation") @@ -1464,6 +1474,7 @@ func testBillingConsolidationAndPDB(ctx context.Context, mgtClient, guestClient // t3.xlarge = 4 vCPUs; 1 node = 4 Karpenter vCPUs on top of baseline waitForAutoNodeStatusVCPUs(t, ctx, mgtClient, hostedCluster, 4) + waitForAutoNodeStatusVCPUsStable(t, ctx, mgtClient, hostedCluster, 4, 30*time.Second) waitForBillingMetricVCPUs(t, ctx, mgtClient, hostedCluster, baseline+4) // Create a blocking PDB and leave everything dangling so cluster teardown @@ -1651,6 +1662,22 @@ func waitForAutoNodeStatusVCPUs(t *testing.T, ctx context.Context, mgtClient crc ) } +// waitForAutoNodeStatusVCPUsStable asserts that AutoNode.VCPUs does not flap +// away from the expected value over the given duration. +func waitForAutoNodeStatusVCPUsStable(t *testing.T, ctx context.Context, mgtClient crclient.Client, hostedCluster *hyperv1.HostedCluster, expected int32, duration time.Duration) { + t.Helper() + g := NewWithT(t) + + t.Logf("Asserting AutoNode.VCPUs remains stable at %d for %s", expected, duration) + g.Consistently(func(g Gomega) { + hc := &hyperv1.HostedCluster{} + g.Expect(mgtClient.Get(ctx, crclient.ObjectKeyFromObject(hostedCluster), hc)).To(Succeed()) + g.Expect(hc.Status.AutoNode.VCPUs).NotTo(BeNil(), "AutoNode.VCPUs became nil") + g.Expect(*hc.Status.AutoNode.VCPUs).To(Equal(expected)) + }).WithTimeout(duration).WithPolling(2 * time.Second).Should(Succeed()) + t.Logf("AutoNode.VCPUs remained stable at %d for %s", expected, duration) +} + // waitForBillingMetricVCPUs polls until the hypershift_cluster_vcpus metric // converges to the expected total (native + Karpenter). func waitForBillingMetricVCPUs(t *testing.T, ctx context.Context, mgtClient crclient.Client, hostedCluster *hyperv1.HostedCluster, expectedTotal int32) { diff --git a/test/e2e/util/aws.go b/test/e2e/util/aws.go index dcf9b475495c..fef417b80161 100644 --- a/test/e2e/util/aws.go +++ b/test/e2e/util/aws.go @@ -375,7 +375,7 @@ func CreateCapacityReservation(ctx context.Context, awsCreds, awsRegion, instanc CapacityReservationIds: []string{crID}, }) if err != nil { - return false, nil // transient error, retry + return false, nil //nolint:nilerr // transient error, retry } if len(desc.CapacityReservations) == 0 { return false, nil diff --git a/test/e2e/util/dump/dump.go b/test/e2e/util/dump/dump.go index 38e489467034..02b7e4cdea6b 100644 --- a/test/e2e/util/dump/dump.go +++ b/test/e2e/util/dump/dump.go @@ -68,7 +68,7 @@ func DumpMachineConsoleLogs(ctx context.Context, hc *hyperv1.HostedCluster, awsC } err := consoleLogs.Run(ctx) if err != nil { - return fmt.Errorf("failed to get machine console logs: %v", err) + return fmt.Errorf("failed to get machine console logs: %w", err) } return nil } diff --git a/test/e2e/util/dump/journals.go b/test/e2e/util/dump/journals.go index 20a9b4fe0b95..b0ad68fc64c5 100644 --- a/test/e2e/util/dump/journals.go +++ b/test/e2e/util/dump/journals.go @@ -78,7 +78,7 @@ func DumpJournals(t *testing.T, ctx context.Context, hc *hyperv1.HostedCluster, return nil } - return runJournalDumpScript(t, hc, artifactDir, copyJournalFile, privateKeyFile, bastionIP, machineIPs, machineInstances) + return runJournalDumpScript(ctx, t, hc, artifactDir, copyJournalFile, privateKeyFile, bastionIP, machineIPs, machineInstances) } func setupSSHKey(ctx context.Context, hc *hyperv1.HostedCluster) (string, error) { @@ -133,7 +133,7 @@ func setupBastionLoggers(artifactDir string) (*zap.Logger, *zap.Logger, error) { createLogFile := filepath.Join(artifactDir, "create-bastion.log") createLog, err := os.Create(createLogFile) if err != nil { - return nil, nil, fmt.Errorf("failed to create create log: %w", err) + return nil, nil, fmt.Errorf("failed to create log: %w", err) } createLogger := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.Lock(createLog), zap.DebugLevel)) @@ -269,7 +269,7 @@ func authorizeSSHAccess(ctx context.Context, hc *hyperv1.HostedCluster, awsCreds return nil } -func runJournalDumpScript(t *testing.T, hc *hyperv1.HostedCluster, artifactDir, copyJournalFile, privateKeyFile, bastionIP string, machineIPs []string, machineInstances []ec2types.Instance) error { +func runJournalDumpScript(ctx context.Context, t *testing.T, hc *hyperv1.HostedCluster, artifactDir, copyJournalFile, privateKeyFile, bastionIP string, machineIPs []string, machineInstances []ec2types.Instance) error { dumpJournalsLogFile := filepath.Join(artifactDir, "dump-machine-journals.log") dumpJournalsLog, err := os.Create(dumpJournalsLogFile) if err != nil { @@ -277,7 +277,7 @@ func runJournalDumpScript(t *testing.T, hc *hyperv1.HostedCluster, artifactDir, } outputDir := filepath.Join(artifactDir, "machine-journals") - scriptCmd := exec.Command(copyJournalFile, outputDir) + scriptCmd := exec.CommandContext(ctx, copyJournalFile, outputDir) env := os.Environ() env = append(env, fmt.Sprintf("BASTION_IP=%s", bastionIP)) env = append(env, fmt.Sprintf("INSTANCE_IPS=%s", strings.Join(machineIPs, " "))) diff --git a/test/e2e/util/external_oidc.go b/test/e2e/util/external_oidc.go index b4dd731e8f82..adaa8d63c0ea 100644 --- a/test/e2e/util/external_oidc.go +++ b/test/e2e/util/external_oidc.go @@ -15,6 +15,7 @@ import ( "os" "path/filepath" "regexp" + "strings" "testing" "time" @@ -264,7 +265,10 @@ func ChangeUserForKeycloakExtOIDC(t *testing.T, ctx context.Context, clientCfg * "username": []string{username}, } - response, err := httpClient.PostForm(requestURL, formData) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, requestURL, strings.NewReader(formData.Encode())) + g.Expect(err).NotTo(HaveOccurred()) + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + response, err := httpClient.Do(req) g.Expect(err).NotTo(HaveOccurred()) defer response.Body.Close() g.Expect(response.StatusCode).To(Equal(http.StatusOK)) diff --git a/test/e2e/util/fixture.go b/test/e2e/util/fixture.go index f8c2100117bd..e12aae01d1fb 100644 --- a/test/e2e/util/fixture.go +++ b/test/e2e/util/fixture.go @@ -186,7 +186,7 @@ func createCluster(ctx context.Context, hc *hyperv1.HostedCluster, opts *Platfor func renderCreate(ctx context.Context, opts *core.RawCreateOptions, platformOpts core.PlatformValidator, outputFile string, renderLogFile string, createLogFile string) error { renderLog, err := os.Create(renderLogFile) if err != nil { - return fmt.Errorf("failed to render render log: %w", err) + return fmt.Errorf("failed to render log: %w", err) } renderLogger := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.Lock(renderLog), zap.DebugLevel)) defer func() { @@ -204,7 +204,7 @@ func renderCreate(ctx context.Context, opts *core.RawCreateOptions, platformOpts createLog, err := os.Create(createLogFile) if err != nil { - return fmt.Errorf("failed to create create log: %w", err) + return fmt.Errorf("failed to create log: %w", err) } createLogger := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.Lock(createLog), zap.DebugLevel)) defer func() { @@ -225,7 +225,7 @@ func destroyCluster(ctx context.Context, t *testing.T, hc *hyperv1.HostedCluster destroyLogFile := filepath.Join(outputDir, "destroy.log") destroyLog, err := os.Create(destroyLogFile) if err != nil { - return fmt.Errorf("failed to destroy destroy log: %w", err) + return fmt.Errorf("failed to destroy log: %w", err) } destroyLogger := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.Lock(destroyLog), zap.DebugLevel)) defer func() { diff --git a/test/e2e/util/generate.go b/test/e2e/util/generate.go index 3ca2004539cb..99b0d60bb041 100644 --- a/test/e2e/util/generate.go +++ b/test/e2e/util/generate.go @@ -26,7 +26,7 @@ import ( // available to guide selection of new names and this interface hides those details. type NameGenerator interface { // GenerateName generates a valid name from the base name, adding a random suffix to the - // the base. If base is valid, the returned name must also be valid. The generator is + // base. If base is valid, the returned name must also be valid. The generator is // responsible for knowing the maximum valid name length. GenerateName(base string) string } diff --git a/test/e2e/util/node.go b/test/e2e/util/node.go index ca288bfb8293..374e130c5cdc 100644 --- a/test/e2e/util/node.go +++ b/test/e2e/util/node.go @@ -33,12 +33,12 @@ func EnsureNodeCommunication(t *testing.T, ctx context.Context, client crclient. err = wait.PollUntilContextTimeout(ctx, 10*time.Second, 5*time.Minute, true, func(ctx context.Context) (done bool, err error) { podList, err := guestClient.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{LabelSelector: "app=konnectivity-agent"}) if err != nil || len(podList.Items) == 0 { - return false, nil + return false, nil //nolint:nilerr // retry until konnectivity-agent pod is available } _, err = guestClient.CoreV1().Pods("kube-system").GetLogs(podList.Items[0].Name, &corev1.PodLogOptions{Container: "konnectivity-agent"}).DoRaw(ctx) if err != nil { - return false, nil + return false, nil //nolint:nilerr // retry until logs are available } return true, nil diff --git a/test/e2e/util/oauth.go b/test/e2e/util/oauth.go index 9e3497c959ab..d4a116b7c7a4 100644 --- a/test/e2e/util/oauth.go +++ b/test/e2e/util/oauth.go @@ -116,7 +116,7 @@ func WaitForOAuthTokenByHost(t testing.TB, ctx context.Context, oauthHost string oauthClient := configmanifests.OAuthServerChallengingClient().Name tokenReqUrl := fmt.Sprintf("https://%s/oauth/authorize?response_type=token&client_id=%s", oauthHost, oauthClient) - request, err := http.NewRequest(http.MethodGet, tokenReqUrl, nil) + request, err := http.NewRequestWithContext(ctx, http.MethodGet, tokenReqUrl, nil) g.Expect(err).ToNot(HaveOccurred()) request.Header.Set("Authorization", getBasicHeader(username, password)) @@ -173,14 +173,14 @@ func WaitForOAuthRouteReady(t *testing.T, ctx context.Context, client crclient.C err := wait.PollUntilContextTimeout(ctx, time.Second, time.Minute, true, func(ctx context.Context) (done bool, err error) { err = client.Get(context.Background(), crclient.ObjectKeyFromObject(route), route) if err != nil { - return false, nil + return false, nil //nolint:nilerr // retry until route exists } return true, nil }) g.Expect(err).ToNot(HaveOccurred(), "failed retrieving oauth route") t.Logf("Found OAuth route %s", route.Spec.Host) - request, err := http.NewRequest(http.MethodHead, fmt.Sprintf("https://%s/healthz", route.Spec.Host), nil) + request, err := http.NewRequestWithContext(ctx, http.MethodHead, fmt.Sprintf("https://%s/healthz", route.Spec.Host), nil) g.Expect(err).ToNot(HaveOccurred()) transport, err := restclient.TransportFor(restclient.AnonymousClientConfig(restConfig)) @@ -253,7 +253,7 @@ func WaitForOauthConfig(t testing.TB, ctx context.Context, client crclient.Clien err := wait.PollUntilContextTimeout(ctx, time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { err = client.Get(context.Background(), crclient.ObjectKeyFromObject(oauthConfigCM), oauthConfigCM) if err != nil { - return false, nil + return false, nil //nolint:nilerr // retry until configmap exists } data, ok := oauthConfigCM.Data[OAuthServerConfigKey] if !ok || data == "" { @@ -262,7 +262,7 @@ func WaitForOauthConfig(t testing.TB, ctx context.Context, client crclient.Clien ouathConfig := &osinv1.OsinServerConfig{} if _, _, err := api.YamlSerializer.Decode([]byte(data), nil, ouathConfig); err != nil { - return false, nil + return false, nil //nolint:nilerr // retry until config is parseable } if len(ouathConfig.OAuthConfig.IdentityProviders) == 0 { return false, nil @@ -328,7 +328,7 @@ func WaitForOAuthLoadBalancerReady(t testing.TB, ctx context.Context, client crc svc := hcpmanifests.OauthServerService(hcpNamespace) err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 10*time.Minute, true, func(ctx context.Context) (done bool, err error) { if err := client.Get(ctx, crclient.ObjectKeyFromObject(svc), svc); err != nil { - return false, nil + return false, nil //nolint:nilerr // retry until service exists } if svc.Spec.Type != corev1.ServiceTypeLoadBalancer { t.Logf("Waiting for oauth-openshift Service type to be LoadBalancer, got %s", svc.Spec.Type) @@ -349,7 +349,7 @@ func WaitForOAuthLoadBalancerReady(t testing.TB, ctx context.Context, client crc // Wait for the OAuth hostname to be resolvable via DNS (ExternalDNS creates the record) // and for the /healthz endpoint to return HTTP 200 - request, err := http.NewRequest(http.MethodHead, fmt.Sprintf("https://%s/healthz", oauthHost), nil) + request, err := http.NewRequestWithContext(ctx, http.MethodHead, fmt.Sprintf("https://%s/healthz", oauthHost), nil) g.Expect(err).ToNot(HaveOccurred()) transport, err := restclient.TransportFor(restclient.AnonymousClientConfig(restConfig)) diff --git a/test/e2e/util/reqserving/verifycp.go b/test/e2e/util/reqserving/verifycp.go index 6199246dadd6..34210ab66b8b 100644 --- a/test/e2e/util/reqserving/verifycp.go +++ b/test/e2e/util/reqserving/verifycp.go @@ -93,7 +93,7 @@ func verifyKASGoMemLimit(ctx context.Context, client crclient.Client, cpNamespac err := wait.PollUntilContextCancel(pollCtx, DefaultPollingInterval, true, func(pctx context.Context) (bool, error) { kasPods := &corev1.PodList{} if err := client.List(pctx, kasPods, crclient.MatchingLabels{"app": "kube-apiserver"}, crclient.InNamespace(cpNamespace)); err != nil { - return false, nil + return false, nil //nolint:nilerr // retry until pods are listable } if len(kasPods.Items) == 0 { return false, nil @@ -132,7 +132,7 @@ func verifyInflightConfig(ctx context.Context, client crclient.Client, cpNamespa err := wait.PollUntilContextCancel(pollCtx, DefaultPollingInterval, true, func(pctx context.Context) (bool, error) { kasConfigMap := &corev1.ConfigMap{} if err := client.Get(pctx, types.NamespacedName{Name: "kas-config", Namespace: cpNamespace}, kasConfigMap); err != nil { - return false, nil + return false, nil //nolint:nilerr // retry until configmap exists } data, ok := kasConfigMap.Data["config.json"] if !ok || data == "" { @@ -140,7 +140,7 @@ func verifyInflightConfig(ctx context.Context, client crclient.Client, cpNamespa } kasConfig := &kcpv1.KubeAPIServerConfig{} if err := json.Unmarshal([]byte(data), &kasConfig); err != nil { - return false, nil + return false, nil //nolint:nilerr // retry until config is parseable } if !inflightArgMatches(kasConfig, "max-requests-inflight", effects.MaximumRequestsInflight) { return false, nil @@ -185,7 +185,7 @@ func verifyResourceRequests(ctx context.Context, client crclient.Client, cpNames for _, effect := range effects.ResourceRequests { containers, err := getContainersForEffect(pctx, client, cpNamespace, effect) if err != nil { - return false, nil + return false, nil //nolint:nilerr // retry until containers are available } if !containerResourcesMatch(containers, effect) { return false, nil diff --git a/test/e2e/util/reqserving/verifypods.go b/test/e2e/util/reqserving/verifypods.go index 98cfd4d61aed..f128b01fc48e 100644 --- a/test/e2e/util/reqserving/verifypods.go +++ b/test/e2e/util/reqserving/verifypods.go @@ -103,7 +103,7 @@ func VerifyRequestServingPodDistribution(ctx context.Context, hc *hyperv1.Hosted for depName, expectedReplicas := range expectedDeployments { dep := &appsv1.Deployment{} if err := client.Get(pctx, crclient.ObjectKey{Namespace: cpNamespace, Name: depName}, dep); err != nil { - errs = append(errs, fmt.Errorf("failed to get deployment %s: %v", depName, err)) + errs = append(errs, fmt.Errorf("failed to get deployment %s: %w", depName, err)) continue } if dep.Spec.Replicas == nil || *dep.Spec.Replicas != int32(expectedReplicas) { diff --git a/test/e2e/util/reqserving/vpa.go b/test/e2e/util/reqserving/vpa.go index c2b41483c7dc..8e09ea9fd16f 100644 --- a/test/e2e/util/reqserving/vpa.go +++ b/test/e2e/util/reqserving/vpa.go @@ -177,8 +177,7 @@ func waitForVPAResource(ctx context.Context) error { // Use targeted discovery instead of full server discovery for better performance _, err := disc.ServerResourcesForGroupVersion(vpaautoscalingv1.SchemeGroupVersion.String()) if err != nil { - // VPA API group not available yet - return false, nil + return false, nil //nolint:nilerr // VPA API group not available yet, retry } return true, nil }) diff --git a/test/e2e/util/reqserving/waitfor.go b/test/e2e/util/reqserving/waitfor.go index 354773f877be..e49f3d72d89d 100644 --- a/test/e2e/util/reqserving/waitfor.go +++ b/test/e2e/util/reqserving/waitfor.go @@ -76,7 +76,7 @@ func WaitForControlPlaneWorkloadsReady(ctx context.Context, hc *hyperv1.HostedCl defer cancel() err = wait.PollUntilContextCancel(statefulSetCtx, DefaultPollingInterval, true, func(ctx context.Context) (bool, error) { if err := client.List(ctx, statefulSets, crclient.InNamespace(cpNamespace)); err != nil { - return false, nil + return false, nil //nolint:nilerr // retry until statefulsets are listable } if len(statefulSets.Items) == 0 { return false, nil diff --git a/test/e2e/util/sharedoidc.go b/test/e2e/util/sharedoidc.go index db91d01261b0..845a724002be 100644 --- a/test/e2e/util/sharedoidc.go +++ b/test/e2e/util/sharedoidc.go @@ -106,7 +106,7 @@ func SetupSharedOIDCProvider(opts *Options, artifactDir string) error { createLogFile := filepath.Join(artifactDir, "create-oidc-provider.log") createLog, err := os.Create(createLogFile) if err != nil { - return fmt.Errorf("failed to create create log: %w", err) + return fmt.Errorf("failed to create log: %w", err) } createLogger := zap.New(zapcore.NewCore(zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), zapcore.Lock(createLog), zap.DebugLevel)) defer func() { diff --git a/test/e2e/util/util.go b/test/e2e/util/util.go index 47e9b812fb10..418f922f55ac 100644 --- a/test/e2e/util/util.go +++ b/test/e2e/util/util.go @@ -7,6 +7,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/json" + "errors" "fmt" "io" "net" @@ -233,7 +234,7 @@ func DeleteNamespace(t *testing.T, ctx context.Context, client crclient.Client, return false, nil }) if err != nil { - return fmt.Errorf("namespace still exists after deletion timeout: %v", err) + return fmt.Errorf("namespace still exists after deletion timeout: %w", err) } if os.Getenv("EVENTUALLY_VERBOSE") != "false" { t.Logf("Deleted namespace %s", namespace) @@ -397,7 +398,7 @@ func GetGuestKubeconfigHost(t *testing.T, ctx context.Context, client crclient.C guestKubeConfigSecretData := WaitForGuestKubeConfig(t, ctx, client, hostedCluster) guestConfig, err := clientcmd.RESTConfigFromKubeConfig(guestKubeConfigSecretData) if err != nil { - return "", fmt.Errorf("couldn't load guest kubeconfig: %v", err) + return "", fmt.Errorf("couldn't load guest kubeconfig: %w", err) } host := guestConfig.Host @@ -419,7 +420,7 @@ func WaitForGuestKubeconfigHostUpdate(t *testing.T, ctx context.Context, client newHost, getHostError = GetGuestKubeconfigHost(t, ctx, client, hostedCluster) if getHostError != nil { t.Logf("failed to get guest kubeconfig host: %v", getHostError) - return false, nil + return false, nil //nolint:nilerr // retry until kubeconfig host is available } if newHost == oldHost { t.Logf("guest kubeconfig host is not yet updated, keep polling") @@ -441,12 +442,16 @@ func WaitForGuestKubeconfigHostResolutionUpdate(t *testing.T, ctx context.Contex err := wait.PollUntilContextTimeout(ctx, 15*time.Second, 30*time.Minute, true, func(ctx context.Context) (done bool, err error) { host := strings.TrimPrefix(uri, "https://") host = strings.Split(host, ":")[0] - ips, err := net.LookupIP(host) + ipAddrs, err := (&net.Resolver{}).LookupIPAddr(ctx, host) if err != nil { t.Logf("failed to resolve guest kubeconfig host: %v", err) return false, nil } - ip := ips[0].String() + if len(ipAddrs) == 0 { + t.Logf("guest kubeconfig host resolved with no IPs yet") + return false, nil + } + ip := ipAddrs[0].IP.String() if endpointAccess == hyperv1.Private { if strings.HasPrefix(ip, "10.") { t.Logf("kubeconfig host now resolves to private address") @@ -1608,7 +1613,7 @@ func GetMetricsFromPod(ctx context.Context, c crclient.Client, componentName, co command := []string{"curl", "-s", fmt.Sprintf("http://127.0.0.1:%s/metrics", port)} cmdOutput, err := RunCommandInPod(ctx, c, componentName, namespaceName, command, containerName, 5*time.Minute) if err != nil { - return nil, fmt.Errorf("couldn't obtain any metrics: %v", err) + return nil, fmt.Errorf("couldn't obtain any metrics: %w", err) } if len(cmdOutput) == 0 { return nil, fmt.Errorf("no metrics found") @@ -2247,7 +2252,7 @@ func createAdditionalPullSecret(ctx context.Context, guestClient crclient.Client } if err := guestClient.Create(ctx, secret); err != nil && !apierrors.IsAlreadyExists(err) { - return fmt.Errorf("failed to create secret: %v", err) + return fmt.Errorf("failed to create secret: %w", err) } return nil @@ -2392,9 +2397,9 @@ func EnsureKubeAPIDNSNameCustomCert(t *testing.T, ctx context.Context, mgmtClien start := time.Now() g.Eventually(func() error { t.Logf("[%s] Waiting until the URL is resolvable: %s", time.Now().Format(time.RFC3339), customApiServerHost) - _, err := net.LookupIP(customApiServerHost) + _, err := (&net.Resolver{}).LookupIPAddr(ctx, customApiServerHost) if err != nil { - return fmt.Errorf("failed to resolve the custom DNS name: %v", err) + return fmt.Errorf("failed to resolve the custom DNS name: %w", err) } t.Logf("resolved the custom DNS name after %s\n", time.Since(start)) return nil @@ -2567,7 +2572,7 @@ func EnsureKubeAPIDNSNameCustomCert(t *testing.T, ctx context.Context, mgmtClien err = retry.RetryOnConflict(retry.DefaultRetry, func() error { latestHC := &hyperv1.HostedCluster{} if err := mgmtClient.Get(ctx, crclient.ObjectKeyFromObject(hc), latestHC); err != nil { - return fmt.Errorf("failed to get latest HostedCluster: %v", err) + return fmt.Errorf("failed to get latest HostedCluster: %w", err) } latestHC.Spec.Configuration.APIServer.ServingCerts.NamedCertificates = []configv1.APIServerNamedServingCert{} return mgmtClient.Update(ctx, latestHC) @@ -2874,7 +2879,7 @@ func getIngressRouterDefaultIP(t *testing.T, ctx context.Context, client crclien } return getErr == nil, err }); err != nil { - return "", fmt.Errorf("router-default service did't become available: %v", err) + return "", fmt.Errorf("router-default service did't become available: %w", err) } routerDefaultIP := defaultIngressRouterService.Status.LoadBalancer.Ingress[0].IP @@ -4514,7 +4519,7 @@ func ApplyYAMLBytes(ctx context.Context, c crclient.Client, yamlContent []byte, for { obj := &unstructured.Unstructured{} if err := decoder.Decode(obj); err != nil { - if err == io.EOF { + if errors.Is(err, io.EOF) { return nil } return fmt.Errorf("failed to decode YAML: %w", err) @@ -4638,7 +4643,7 @@ func EnsureNodeTuningOperatorMetricsEndpoint(t *testing.T, ctx context.Context, } cmdOutput, err := RunCommandInPod(ctx, mgmtClient, "cluster-node-tuning-operator", hcpNamespace, httpsCommand, "cluster-node-tuning-operator", 30*time.Second) if err != nil { - return fmt.Errorf("failed to get metrics via ServiceMonitor HTTPS at %s: %v", httpsServiceURL, err) + return fmt.Errorf("failed to get metrics via ServiceMonitor HTTPS at %s: %w", httpsServiceURL, err) } if len(cmdOutput) == 0 { return fmt.Errorf("no metrics returned via ServiceMonitor HTTPS at %s", httpsServiceURL) diff --git a/test/e2e/util/version.go b/test/e2e/util/version.go index 7a5d677b0afe..03846613990f 100644 --- a/test/e2e/util/version.go +++ b/test/e2e/util/version.go @@ -50,16 +50,16 @@ func init() { func SetReleaseImageVersion(ctx context.Context, latestReleaseImage string, pullSecretFile string) error { data, err := os.ReadFile(pullSecretFile) if err != nil { - return fmt.Errorf("error reading file: %v", err) + return fmt.Errorf("error reading file: %w", err) } releaseInfoProvider := releaseinfo.RegistryClientProvider{} releaseImage, err := releaseInfoProvider.Lookup(ctx, latestReleaseImage, data) if err != nil { - return fmt.Errorf("error looking up latest release image: %v", err) + return fmt.Errorf("error looking up latest release image: %w", err) } releaseVersion, err = semver.Parse(releaseImage.Version()) if err != nil { - return fmt.Errorf("error parsing version: %v", err) + return fmt.Errorf("error parsing version: %w", err) } releaseVersion.Patch = 0 releaseVersion.Pre = nil @@ -76,7 +76,7 @@ func SetReleaseVersionFromHostedCluster(ctx context.Context, hostedCluster *hype var err error releaseVersion, err = semver.Parse(hcVersion) if err != nil { - return fmt.Errorf("error parsing version: %v", err) + return fmt.Errorf("error parsing version: %w", err) } releaseVersion.Patch = 0 releaseVersion.Pre = nil diff --git a/test/e2e/v2/AGENTS.md b/test/e2e/v2/AGENTS.md index 6662befebcef..e05a45dc3655 100644 --- a/test/e2e/v2/AGENTS.md +++ b/test/e2e/v2/AGENTS.md @@ -15,6 +15,9 @@ The framework is organized into three packages under `test/e2e/v2/`: - `internal/` — Framework internals (test context, workload registry, fail handler, env var management). Do not add tests here. - `tests/` — All standard v2 test files. Each file is feature-scoped with a top-level `Describe` and `Label`. The suite entry point is `suite_test.go`. +- `util/` — Shared test utilities (pod exec helpers, metrics fetching) consumed by test files. Unlike `internal/`, these are importable by other packages. +- `lifecycle/` — Platform-specific lifecycle helpers (e.g., Azure platform hooks). +- `cmd/` — CLI tools for test orchestration: creating/destroying/dumping guest clusters and running test suites. - `backuprestore/` — Backup/restore helpers (CLI wrappers, prober, Velero). Additional packages may be introduced as the v2 framework expands; this structure is not yet finalized. @@ -91,6 +94,60 @@ Expect(ptr).NotTo(BeNil(), "container %s in pod %s should have security context" Comments on exported functions must describe actual behavior including panic conditions, not just intended behavior. For example, `GetEnvVarValue` documents that it "panics if the environment variable is not registered." +### 13. Non-Lifecycle Tests Must Not Mutate the Hosted Cluster + +Non-lifecycle tests (health, compliance, security, metrics) must only **verify** existing state — never mutate the hosted cluster to create preconditions. If a test requires a specific annotation, label, or configuration to be present, `Skip()` when it is absent rather than setting it. Only lifecycle tests (upgrade, backup-restore, nodepool scaling) may mutate cluster state. + +```go +// WRONG — sets annotation to create precondition +hc.Annotations["hypershift.openshift.io/metrics-forwarder"] = "true" +Expect(mgmtClient.Update(ctx, hc)).To(Succeed()) + +// RIGHT — skip if precondition is missing +if _, ok := hc.Annotations["hypershift.openshift.io/metrics-forwarder"]; !ok { + Skip("metrics forwarder annotation not set on hosted cluster") +} +``` + +### 14. Per-Workload Test Placement + +Tests that iterate over control plane workloads (e.g., checking restart counts, custom labels, custom tolerations) belong in `control_plane_workloads_test.go`, not in health or compliance test files. Follow the per-workload pattern: define a separate `It` block for each registered workload so failures identify the exact workload. + +```go +for _, w := range workloads { + workload := w + Context(workload.Name, func() { + It("should have custom labels", func() { + // assert per-workload + }) + }) +} +``` + +### 15. IPv6-Safe URL Construction + +When building URLs from endpoint IPs (e.g., Kubernetes service endpoints), always use `net.JoinHostPort` instead of `fmt.Sprintf`. Plain `%s:%d` formatting produces invalid URLs for IPv6 addresses. + +```go +// WRONG — breaks with IPv6 +kasAddress = fmt.Sprintf("https://%s:%v", ip, port) + +// RIGHT — brackets IPv6 addresses automatically +kasAddress = "https://" + net.JoinHostPort(ip, fmt.Sprintf("%d", port)) +``` + +### 16. Vacuous Pass Prevention + +Before iterating a list and asserting on each item, assert the list is non-empty. An empty list trivially passes all per-item assertions, hiding regressions where resources were never created. + +```go +Expect(routeList.Items).NotTo(BeEmpty(), + "expected at least one route in namespace %s", tc.ControlPlaneNamespace) +for i := range routeList.Items { + // per-item assertions +} +``` + ## Expanding v2 When adding new test areas: @@ -105,3 +162,4 @@ When adding new test areas: - `backuprestore/` tests use `Ordered, Serial` Ginkgo decorators and require the combined `e2ev2,backuprestore` build tag. They produce a separate binary (`bin/test-backuprestore`). - `workload_registry.go` has a header comment saying "generated" but the file is manually maintained. Edit it directly. - Tests assume the hosted cluster is fully operational. There is no startup polling or readiness waiting in the suite setup. +- MicroShift skip guards (`exutil.IsMicroShiftCluster()`, `[Skipped:MicroShift]` labels) do **not** apply to v2 e2e tests. Those guards are scoped to `openshift-tests-private`. The v2 framework runs exclusively against hosted clusters, which are never MicroShift. diff --git a/test/e2e/v2/internal/env_vars.go b/test/e2e/v2/internal/env_vars.go index f5cbd4b7cba4..47361c2945d4 100644 --- a/test/e2e/v2/internal/env_vars.go +++ b/test/e2e/v2/internal/env_vars.go @@ -160,6 +160,12 @@ func init() { false, filepath.Join(os.Getenv("HOME"), ".aws", "credentials"), ) + RegisterEnvVarWithDefault( + "E2E_SERVICE_DOMAIN", + "Service domain used for custom DNS endpoint testing. Optional; test is skipped when empty.", + false, + "", + ) // Azure self-managed test environment variables RegisterEnvVar( "AZURE_PRIVATE_NAT_SUBNET_ID", diff --git a/test/e2e/v2/internal/test_context.go b/test/e2e/v2/internal/test_context.go index 91cf144240ee..1957121e73bd 100644 --- a/test/e2e/v2/internal/test_context.go +++ b/test/e2e/v2/internal/test_context.go @@ -21,37 +21,41 @@ import ( "fmt" "sync" + . "github.com/onsi/ginkgo/v2" + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" "github.com/openshift/hypershift/hypershift-operator/controllers/manifests" hyperapi "github.com/openshift/hypershift/support/api" e2eutil "github.com/openshift/hypershift/test/e2e/util" corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" crclient "sigs.k8s.io/controller-runtime/pkg/client" ) -// TestContextGetter is a function type that returns a TestContext. -// It is used to lazily access the test context in test functions. type TestContextGetter func() *TestContext -// TestContext holds the test context including clients and hosted cluster reference type TestContext struct { context.Context - MgmtClient crclient.Client - ClusterName string - ClusterNamespace string - ControlPlaneNamespace string - ArtifactDir string - hostedCluster *hyperv1.HostedCluster - hostedClusterOnce sync.Once - hostedClusterClient crclient.Client - hostedClusterClientOnce sync.Once + MgmtClient crclient.Client + ClusterName string + ClusterNamespace string + ControlPlaneNamespace string + ArtifactDir string + HostedClusterConfigured bool + hostedCluster *hyperv1.HostedCluster + hostedClusterOnce sync.Once + hostedClusterClient crclient.Client + hostedClusterClientOnce sync.Once + hostedClusterRESTConfig *rest.Config + hostedClusterRESTConfigOnce sync.Once } // GetHostedCluster returns the HostedCluster associated with this test context. -// It fetches the HostedCluster lazily on first call if ClusterName and ClusterNamespace are set. -// Returns nil if the HostedCluster cannot be fetched or if ClusterName/ClusterNamespace are not set. +// The result is cached by sync.Once — callers must ensure the cluster is ready before the first call. +// Returns nil if ClusterName or ClusterNamespace are not set. +// Panics if the HostedCluster fetch or release version extraction fails. func (tc *TestContext) GetHostedCluster() *hyperv1.HostedCluster { tc.hostedClusterOnce.Do(func() { if tc.ClusterName == "" || tc.ClusterNamespace == "" { @@ -64,7 +68,6 @@ func (tc *TestContext) GetHostedCluster() *hyperv1.HostedCluster { Name: tc.ClusterName, }, hostedCluster) if err != nil { - // In test code, panicking is acceptable and will fail the test appropriately panic(fmt.Sprintf("failed to get HostedCluster %s/%s: %v", tc.ClusterNamespace, tc.ClusterName, err)) } @@ -78,39 +81,49 @@ func (tc *TestContext) GetHostedCluster() *hyperv1.HostedCluster { return tc.hostedCluster } +// getHostedClusterRESTConfig fetches the kubeconfig secret and returns a REST config. +// Returns nil if the HostedCluster is not available or its KubeConfig status is not set. +// Panics on any other failure. +func (tc *TestContext) getHostedClusterRESTConfig() *rest.Config { + hc := tc.GetHostedCluster() + if hc == nil || hc.Status.KubeConfig == nil { + return nil + } + + var kubeconfigSecret corev1.Secret + err := tc.MgmtClient.Get(tc.Context, crclient.ObjectKey{ + Namespace: hc.Namespace, + Name: hc.Status.KubeConfig.Name, + }, &kubeconfigSecret) + if err != nil { + panic(fmt.Sprintf("failed to get kubeconfig secret %s/%s: %v", hc.Namespace, hc.Status.KubeConfig.Name, err)) + } + + kubeconfigData, ok := kubeconfigSecret.Data["kubeconfig"] + if !ok || len(kubeconfigData) == 0 { + panic(fmt.Sprintf("kubeconfig key not found or empty in secret %s/%s", hc.Namespace, hc.Status.KubeConfig.Name)) + } + + restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfigData) + if err != nil { + panic(fmt.Sprintf("failed to create REST config from kubeconfig: %v", err)) + } + restConfig.QPS = 200 + restConfig.Burst = 300 + + return restConfig +} + // GetHostedClusterClient returns a controller-runtime client for the hosted cluster. -// It reads the kubeconfig from the secret referenced by the HostedCluster status. -// The client is lazily initialized and cached. +// The result is cached by sync.Once — callers must ensure the cluster is ready before the first call. // Returns nil if the HostedCluster is not available or its KubeConfig status is not set. -// Panics on any other initialization failure (e.g., kubeconfig secret not found, invalid kubeconfig data). +// Panics on any other initialization failure. func (tc *TestContext) GetHostedClusterClient() crclient.Client { tc.hostedClusterClientOnce.Do(func() { - hc := tc.GetHostedCluster() - if hc == nil || hc.Status.KubeConfig == nil { + restConfig := tc.GetHostedClusterRESTConfig() + if restConfig == nil { return } - - var kubeconfigSecret corev1.Secret - err := tc.MgmtClient.Get(tc.Context, crclient.ObjectKey{ - Namespace: hc.Namespace, - Name: hc.Status.KubeConfig.Name, - }, &kubeconfigSecret) - if err != nil { - panic(fmt.Sprintf("failed to get kubeconfig secret %s/%s: %v", hc.Namespace, hc.Status.KubeConfig.Name, err)) - } - - kubeconfigData, ok := kubeconfigSecret.Data["kubeconfig"] - if !ok || len(kubeconfigData) == 0 { - panic(fmt.Sprintf("kubeconfig key not found or empty in secret %s/%s", hc.Namespace, hc.Status.KubeConfig.Name)) - } - - restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfigData) - if err != nil { - panic(fmt.Sprintf("failed to create REST config from kubeconfig: %v", err)) - } - restConfig.QPS = 200 - restConfig.Burst = 300 - client, err := crclient.New(restConfig, crclient.Options{Scheme: hyperapi.Scheme}) if err != nil { panic(fmt.Sprintf("failed to create hosted cluster client: %v", err)) @@ -120,35 +133,40 @@ func (tc *TestContext) GetHostedClusterClient() crclient.Client { return tc.hostedClusterClient } -var ( - // Global test context - set in BeforeSuite - testCtx *TestContext -) +// GetHostedClusterRESTConfig returns the raw REST config for the hosted cluster. +// The result is cached by sync.Once — callers must ensure the cluster is ready before the first call. +// Returns nil if the HostedCluster is not available or its KubeConfig status is not set. +// Panics on any other initialization failure. +func (tc *TestContext) GetHostedClusterRESTConfig() *rest.Config { + tc.hostedClusterRESTConfigOnce.Do(func() { + tc.hostedClusterRESTConfig = tc.getHostedClusterRESTConfig() + }) + return tc.hostedClusterRESTConfig +} + +var testCtx *TestContext -// GetTestContext returns the global test context func GetTestContext() *TestContext { return testCtx } -// SetTestContext sets the global test context func SetTestContext(ctx *TestContext) { testCtx = ctx } -// SetupTestContext initializes the test context from a HostedCluster func SetupTestContext(ctx context.Context, hostedClusterName, hostedClusterNamespace string) (*TestContext, error) { - // Get management client mgmtClient, err := e2eutil.GetClient() if err != nil { return nil, fmt.Errorf("failed to get management client: %w", err) } testCtx := &TestContext{ - Context: ctx, - MgmtClient: mgmtClient, - ClusterName: hostedClusterName, - ClusterNamespace: hostedClusterNamespace, - ControlPlaneNamespace: manifests.HostedControlPlaneNamespace(hostedClusterNamespace, hostedClusterName), + Context: ctx, + MgmtClient: mgmtClient, + ClusterName: hostedClusterName, + ClusterNamespace: hostedClusterNamespace, + ControlPlaneNamespace: manifests.HostedControlPlaneNamespace(hostedClusterNamespace, hostedClusterName), + HostedClusterConfigured: hostedClusterName != "" && hostedClusterNamespace != "", } return testCtx, nil @@ -158,7 +176,6 @@ func SetupTestContext(ctx context.Context, hostedClusterName, hostedClusterNames // It reads E2E_HOSTED_CLUSTER_NAME and E2E_HOSTED_CLUSTER_NAMESPACE from the environment. // If these are not set, it creates a basic context with only the management client. func SetupTestContextFromEnv(ctx context.Context) (*TestContext, error) { - // Get management client mgmtClient, err := e2eutil.GetClient() if err != nil { return nil, fmt.Errorf("failed to get management client: %w", err) @@ -173,24 +190,31 @@ func SetupTestContextFromEnv(ctx context.Context) (*TestContext, error) { hostedClusterNamespace := GetEnvVarValue("E2E_HOSTED_CLUSTER_NAMESPACE") artifactDir := GetEnvVarValue("ARTIFACT_DIR") - // If both env vars are present, set up full context with cluster info if hostedClusterName != "" && hostedClusterNamespace != "" { testCtx.ClusterName = hostedClusterName testCtx.ClusterNamespace = hostedClusterNamespace testCtx.ControlPlaneNamespace = manifests.HostedControlPlaneNamespace(hostedClusterNamespace, hostedClusterName) + testCtx.HostedClusterConfigured = true } testCtx.ArtifactDir = artifactDir return testCtx, nil } -// ValidateControlPlaneNamespace checks if the ControlPlaneNamespace is set in the test context. -// Returns an error with a helpful message if not set. -func (tc *TestContext) ValidateControlPlaneNamespace() error { - if tc.ControlPlaneNamespace == "" { - return fmt.Errorf("ControlPlaneNamespace is required but not set. Please set the following environment variables:\n" + - " E2E_HOSTED_CLUSTER_NAME - Name of the HostedCluster to test\n" + - " E2E_HOSTED_CLUSTER_NAMESPACE - Namespace of the HostedCluster to test") +// ValidateHostedCluster skips the test if no hosted cluster was configured for this run. +// Panics if a hosted cluster was configured but cannot be fetched. +func (tc *TestContext) ValidateHostedCluster() { + if !tc.HostedClusterConfigured { + Skip("no hosted cluster configured for this test run") + } + tc.GetHostedCluster() +} + +// ValidateHostedClusterClient skips the test if no hosted cluster was configured. +// Panics if the hosted cluster client cannot be initialized. +func (tc *TestContext) ValidateHostedClusterClient() { + tc.ValidateHostedCluster() + if tc.GetHostedClusterClient() == nil { + panic("hosted cluster client not available — kubeconfig may not be ready") } - return nil } diff --git a/test/e2e/v2/tests/backup_restore_test.go b/test/e2e/v2/tests/backup_restore_test.go index 76f2e99609b5..0db77af105ce 100644 --- a/test/e2e/v2/tests/backup_restore_test.go +++ b/test/e2e/v2/tests/backup_restore_test.go @@ -306,12 +306,8 @@ func getNodePool(testCtx *internal.TestContext) (*hyperv1.NodePool, error) { return nil, fmt.Errorf("no NodePool found for cluster %s", testCtx.ClusterName) } -// validateBeforeEach validates the control plane namespace and ensures Velero is running. -// It is called from BeforeEach in both BackupRestore and BackupRestoreEtcdSnapshot suites. func validateBeforeEach(testCtx *internal.TestContext) { - if err := testCtx.ValidateControlPlaneNamespace(); err != nil { - AbortSuite(err.Error()) - } + testCtx.ValidateHostedCluster() err := backuprestore.EnsureVeleroPodRunning(testCtx) if err != nil { diff --git a/test/e2e/v2/tests/control_plane_infrastructure_test.go b/test/e2e/v2/tests/control_plane_infrastructure_test.go index 9a1feb186ed1..e99fb450f400 100644 --- a/test/e2e/v2/tests/control_plane_infrastructure_test.go +++ b/test/e2e/v2/tests/control_plane_infrastructure_test.go @@ -17,7 +17,6 @@ limitations under the License. package tests import ( - "context" "fmt" "strings" @@ -86,35 +85,27 @@ func validateContainerResourceRequests(podNamespace, podName string, containers return failures } -// InfrastructureRegistryValidationTest registers tests for infrastructure workload registry validation func InfrastructureRegistryValidationTest(getTestCtx internal.TestContextGetter) { Context("Infrastructure registry validation", func() { - // Label("Informing"): failures skip (non-blocking) until registry is complete - It("all pods in infrastructure namespaces should belong to known workloads", Label("Informing"), func() { + It("should not contain any unrecognized pods", func() { testCtx := getTestCtx() - // Track unmatched pods across all infrastructure namespaces var podsNotBelongingToWorkloads []string - // Check each infrastructure namespace for _, namespace := range internal.GetInfrastructureNamespaces() { - // Check if namespace exists ns := &corev1.Namespace{} - err := testCtx.MgmtClient.Get(context.Background(), crclient.ObjectKey{Name: namespace}, ns) + err := testCtx.MgmtClient.Get(testCtx.Context, crclient.ObjectKey{Name: namespace}, ns) if apierrors.IsNotFound(err) { - // Namespace doesn't exist, skip continue } Expect(err).NotTo(HaveOccurred(), "failed to get namespace %s", namespace) - // List all pods in the namespace podList := &corev1.PodList{} - err = testCtx.MgmtClient.List(context.Background(), podList, &crclient.ListOptions{ + err = testCtx.MgmtClient.List(testCtx.Context, podList, &crclient.ListOptions{ Namespace: namespace, }) Expect(err).NotTo(HaveOccurred(), "failed to list pods in namespace %s", namespace) - // Check each pod for _, pod := range podList.Items { belongsToWorkload := false for _, workload := range infraWorkloads { @@ -138,51 +129,45 @@ func InfrastructureRegistryValidationTest(getTestCtx internal.TestContextGetter) }) } -// InfrastructureResourceRequestsTest registers tests for infrastructure workload resource requests func InfrastructureResourceRequestsTest(getTestCtx internal.TestContextGetter) { Context("Container resource requests", func() { for _, workload := range infraWorkloads { workload := workload // capture range variable - It(fmt.Sprintf("should have resource requests for %s containers", workload.Name), func() { - testCtx := getTestCtx() + Context(workload.Name, func() { + It("should have resource requests for containers", func() { + testCtx := getTestCtx() - // Check if namespace exists - ns := &corev1.Namespace{} - err := testCtx.MgmtClient.Get(context.Background(), crclient.ObjectKey{Name: workload.Namespace}, ns) - if apierrors.IsNotFound(err) { - Skip(fmt.Sprintf("namespace %s not found", workload.Namespace)) - } - Expect(err).NotTo(HaveOccurred(), "failed to get namespace %s", workload.Namespace) - - // List pods matching the workload selector - podList := &corev1.PodList{} - err = testCtx.MgmtClient.List(context.Background(), podList, &crclient.ListOptions{ - Namespace: workload.Namespace, - }) - Expect(err).NotTo(HaveOccurred(), "failed to list pods in namespace %s", workload.Namespace) - - // Filter pods that match this workload - var matchingPods []corev1.Pod - for _, pod := range podList.Items { - if workload.MatchesPod(pod) { - matchingPods = append(matchingPods, pod) + ns := &corev1.Namespace{} + err := testCtx.MgmtClient.Get(testCtx.Context, crclient.ObjectKey{Name: workload.Namespace}, ns) + if apierrors.IsNotFound(err) { + Skip(fmt.Sprintf("namespace %s not found", workload.Namespace)) + } + Expect(err).NotTo(HaveOccurred(), "failed to get namespace %s", workload.Namespace) + + podList := &corev1.PodList{} + err = testCtx.MgmtClient.List(testCtx.Context, podList, &crclient.ListOptions{ + Namespace: workload.Namespace, + }) + Expect(err).NotTo(HaveOccurred(), "failed to list pods in namespace %s", workload.Namespace) + + var matchingPods []corev1.Pod + for _, pod := range podList.Items { + if workload.MatchesPod(pod) { + matchingPods = append(matchingPods, pod) + } } - } - // Track failures across all matching pods - var failures []string - for _, pod := range matchingPods { - // Validate regular containers - failures = append(failures, validateContainerResourceRequests(pod.Namespace, pod.Name, pod.Spec.Containers)...) - // Validate init containers - failures = append(failures, validateContainerResourceRequests(pod.Namespace, pod.Name, pod.Spec.InitContainers)...) - } + var failures []string + for _, pod := range matchingPods { + failures = append(failures, validateContainerResourceRequests(pod.Namespace, pod.Name, pod.Spec.Containers)...) + failures = append(failures, validateContainerResourceRequests(pod.Namespace, pod.Name, pod.Spec.InitContainers)...) + } - // Report all failures at once for better visibility - if len(failures) > 0 { - Fail(strings.Join(failures, "\n")) - } + if len(failures) > 0 { + Fail(strings.Join(failures, "\n")) + } + }) }) } }) diff --git a/test/e2e/v2/tests/control_plane_upgrade_test.go b/test/e2e/v2/tests/control_plane_upgrade_test.go index a2387cfae05e..e6dde01db9ac 100644 --- a/test/e2e/v2/tests/control_plane_upgrade_test.go +++ b/test/e2e/v2/tests/control_plane_upgrade_test.go @@ -31,9 +31,9 @@ import ( func ControlPlaneUpgradeTest(getTestCtx internal.TestContextGetter) { It("should upgrade the control plane from N-1 to latest", func() { testCtx := getTestCtx() + testCtx.ValidateHostedCluster() ctx := testCtx.Context hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") latestImage := internal.GetEnvVarValue("E2E_LATEST_RELEASE_IMAGE") Expect(latestImage).NotTo(BeEmpty(), "E2E_LATEST_RELEASE_IMAGE must be set for upgrade tests") diff --git a/test/e2e/v2/tests/control_plane_workloads_test.go b/test/e2e/v2/tests/control_plane_workloads_test.go index ffb91b1b5a4c..f12bf0d9720c 100644 --- a/test/e2e/v2/tests/control_plane_workloads_test.go +++ b/test/e2e/v2/tests/control_plane_workloads_test.go @@ -17,8 +17,10 @@ limitations under the License. package tests import ( + "bufio" "context" "fmt" + "io" "slices" "strconv" "strings" @@ -36,16 +38,17 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" crclient "sigs.k8s.io/controller-runtime/pkg/client" ) var workloads = internal.GetControlPlaneWorkloads() -// Helper function to get pods for a workload func getWorkloadPods(testCtx *internal.TestContext, workload internal.WorkloadSpec) []corev1.Pod { GinkgoHelper() - pods, err := internal.GetWorkloadPodsBySelector(context.Background(), testCtx.MgmtClient, testCtx.ControlPlaneNamespace, workload) + pods, err := internal.GetWorkloadPodsBySelector(testCtx.Context, testCtx.MgmtClient, testCtx.ControlPlaneNamespace, workload) Expect(err).NotTo(HaveOccurred(), "failed to list pods for workload %s", workload.Name) return pods } @@ -70,26 +73,28 @@ func DeploymentGenerationTest(getTestCtx internal.TestContextGetter) { continue } - It(fmt.Sprintf("should not indicate rapid rollouts for %s", workload.Name), func() { - testCtx := getTestCtx() - hostedCluster := testCtx.GetHostedCluster() - if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { - Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) - } + Context(workload.Name, func() { + It("should not indicate rapid rollouts", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } - deployment := &appsv1.Deployment{} - err := testCtx.MgmtClient.Get(context.Background(), crclient.ObjectKey{ - Namespace: testCtx.ControlPlaneNamespace, - Name: workload.Name, - }, deployment) - if apierrors.IsNotFound(err) { - Skip(fmt.Sprintf("Deployment %s not found", workload.Name)) - } - Expect(err).NotTo(HaveOccurred(), "failed to get Deployment %s", workload.Name) + deployment := &appsv1.Deployment{} + err := testCtx.MgmtClient.Get(context.Background(), crclient.ObjectKey{ + Namespace: testCtx.ControlPlaneNamespace, + Name: workload.Name, + }, deployment) + if apierrors.IsNotFound(err) { + Skip(fmt.Sprintf("Deployment %s not found", workload.Name)) + } + Expect(err).NotTo(HaveOccurred(), "failed to get Deployment %s", workload.Name) - Expect(deployment.Generation).To(BeNumerically("<=", maxAllowedGeneration), - "Deployment %s has generation %d which exceeds max allowed %d", - workload.Name, deployment.Generation, maxAllowedGeneration) + Expect(deployment.Generation).To(BeNumerically("<=", maxAllowedGeneration), + "Deployment %s has generation %d which exceeds max allowed %d", + workload.Name, deployment.Generation, maxAllowedGeneration) + }) }) } }) @@ -121,54 +126,56 @@ func SafeToEvictAnnotationsTest(getTestCtx internal.TestContextGetter) { "ovnkube-control-plane", } for _, workload := range workloads { - It(fmt.Sprintf("should exist for pods with emptyDir or hostPath volumes belonging to %s", workload.Name), func() { - testCtx := getTestCtx() - hostedCluster := testCtx.GetHostedCluster() - if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { - Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) - } - - // Skip if workload is in exemption list - if slices.Contains(exemptions, workload.Name) { - Skip(fmt.Sprintf("workload %s is exempt from safe-to-evict annotations check", workload.Name)) - } + Context(workload.Name, func() { + It("should exist for pods with emptyDir or hostPath volumes", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } - pods := getWorkloadPods(testCtx, workload) - if len(pods) == 0 { - Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) - } + // Skip if workload is in exemption list + if slices.Contains(exemptions, workload.Name) { + Skip(fmt.Sprintf("workload %s is exempt from safe-to-evict annotations check", workload.Name)) + } - for _, pod := range pods { - // Check if pod has emptyDir or hostPath volumes - hasLocalVolumes := false - var localVolumeNames []string - for _, volume := range pod.Spec.Volumes { - if volume.EmptyDir != nil || volume.HostPath != nil { - hasLocalVolumes = true - localVolumeNames = append(localVolumeNames, volume.Name) - } + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) } - if hasLocalVolumes { - annotationKey := "cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes" - annotationValue, exists := pod.Annotations[annotationKey] - Expect(exists).To(BeTrue(), "pod %s has local volumes but missing safe-to-evict annotation", pod.Name) - Expect(annotationValue).NotTo(BeEmpty(), "pod %s has empty safe-to-evict annotation", pod.Name) + for _, pod := range pods { + // Check if pod has emptyDir or hostPath volumes + hasLocalVolumes := false + var localVolumeNames []string + for _, volume := range pod.Spec.Volumes { + if volume.EmptyDir != nil || volume.HostPath != nil { + hasLocalVolumes = true + localVolumeNames = append(localVolumeNames, volume.Name) + } + } - // Verify all local volumes are listed in annotation - annotatedVolumes := strings.Split(annotationValue, ",") - for _, volName := range localVolumeNames { - found := false - for _, annVol := range annotatedVolumes { - if strings.TrimSpace(annVol) == volName { - found = true - break + if hasLocalVolumes { + annotationKey := "cluster-autoscaler.kubernetes.io/safe-to-evict-local-volumes" + annotationValue, exists := pod.Annotations[annotationKey] + Expect(exists).To(BeTrue(), "pod %s has local volumes but missing safe-to-evict annotation", pod.Name) + Expect(annotationValue).NotTo(BeEmpty(), "pod %s has empty safe-to-evict annotation", pod.Name) + + // Verify all local volumes are listed in annotation + annotatedVolumes := strings.Split(annotationValue, ",") + for _, volName := range localVolumeNames { + found := false + for _, annVol := range annotatedVolumes { + if strings.TrimSpace(annVol) == volName { + found = true + break + } } + Expect(found).To(BeTrue(), "pod %s local volume %s not found in annotation", pod.Name, volName) } - Expect(found).To(BeTrue(), "pod %s local volume %s not found in annotation", pod.Name, volName) } } - } + }) }) } }) @@ -214,33 +221,35 @@ func ReadOnlyRootFilesystemTest(getTestCtx internal.TestContextGetter) { } for _, workload := range workloads { - It(fmt.Sprintf("should have read-only root filesystem for %s containers", workload.Name), func() { - testCtx := getTestCtx() - hostedCluster := testCtx.GetHostedCluster() - if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { - Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) - } + Context(workload.Name, func() { + It("should have read-only root filesystem for containers", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } - // Skip if workload is in exemption list - if slices.Contains(exemptions, workload.Name) { - Skip(fmt.Sprintf("workload %s is exempt from read-only root filesystem check", workload.Name)) - } + // Skip if workload is in exemption list + if slices.Contains(exemptions, workload.Name) { + Skip(fmt.Sprintf("workload %s is exempt from read-only root filesystem check", workload.Name)) + } - pods := getWorkloadPods(testCtx, workload) - if len(pods) == 0 { - Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) - } + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) + } - for _, pod := range pods { - for _, container := range pod.Spec.Containers { - Expect(container.SecurityContext).NotTo(BeNil(), - "container %s in pod %s should have security context", container.Name, pod.Name) - Expect(container.SecurityContext.ReadOnlyRootFilesystem).NotTo(BeNil(), - "container %s in pod %s should have ReadOnlyRootFilesystem set", container.Name, pod.Name) - Expect(*container.SecurityContext.ReadOnlyRootFilesystem).To(BeTrue(), - "container %s in pod %s should have ReadOnlyRootFilesystem=true", container.Name, pod.Name) + for _, pod := range pods { + for _, container := range pod.Spec.Containers { + Expect(container.SecurityContext).NotTo(BeNil(), + "container %s in pod %s should have security context", container.Name, pod.Name) + Expect(container.SecurityContext.ReadOnlyRootFilesystem).NotTo(BeNil(), + "container %s in pod %s should have ReadOnlyRootFilesystem set", container.Name, pod.Name) + Expect(*container.SecurityContext.ReadOnlyRootFilesystem).To(BeTrue(), + "container %s in pod %s should have ReadOnlyRootFilesystem=true", container.Name, pod.Name) + } } - } + }) }) } }) @@ -285,36 +294,38 @@ func ReadOnlyRootFilesystemTmpDirMountTest(getTestCtx internal.TestContextGetter } for _, workload := range workloads { - It(fmt.Sprintf("should have /tmp mounted for %s containers", workload.Name), func() { - testCtx := getTestCtx() - hostedCluster := testCtx.GetHostedCluster() - if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { - Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) - } + Context(workload.Name, func() { + It("should have /tmp mounted for containers", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } - // Skip if workload is in exemption list - if slices.Contains(exemptions, workload.Name) { - Skip(fmt.Sprintf("workload %s is exempt from tmp dir mount check", workload.Name)) - } + // Skip if workload is in exemption list + if slices.Contains(exemptions, workload.Name) { + Skip(fmt.Sprintf("workload %s is exempt from tmp dir mount check", workload.Name)) + } - pods := getWorkloadPods(testCtx, workload) - if len(pods) == 0 { - Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) - } + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) + } - for _, pod := range pods { - for _, container := range pod.Spec.Containers { - hasTmpMount := false - for _, mount := range container.VolumeMounts { - if mount.MountPath == podspec.PodTmpDirMountPath { - hasTmpMount = true - break + for _, pod := range pods { + for _, container := range pod.Spec.Containers { + hasTmpMount := false + for _, mount := range container.VolumeMounts { + if mount.MountPath == podspec.PodTmpDirMountPath { + hasTmpMount = true + break + } } + Expect(hasTmpMount).To(BeTrue(), + "container %s in pod %s should have /tmp mounted", container.Name, pod.Name) } - Expect(hasTmpMount).To(BeTrue(), - "container %s in pod %s should have /tmp mounted", container.Name, pod.Name) } - } + }) }) } }) @@ -325,28 +336,30 @@ func ContainerImagePullPolicyTest(getTestCtx internal.TestContextGetter) { Context("Container image pull policy", func() { // EnsureAllContainersHavePullPolicyIfNotPresent for _, workload := range workloads { - It(fmt.Sprintf("should have IfNotPresent pull policy for %s containers", workload.Name), func() { - testCtx := getTestCtx() - hostedCluster := testCtx.GetHostedCluster() - if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { - Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) - } + Context(workload.Name, func() { + It("should have IfNotPresent pull policy for containers", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } - pods := getWorkloadPods(testCtx, workload) - if len(pods) == 0 { - Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) - } + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) + } - for _, pod := range pods { - for _, container := range pod.Spec.Containers { - if container.ImagePullPolicy == "" { - Fail(fmt.Sprintf("container %s in pod %s has no ImagePullPolicy set", container.Name, pod.Name)) + for _, pod := range pods { + for _, container := range pod.Spec.Containers { + if container.ImagePullPolicy == "" { + Fail(fmt.Sprintf("container %s in pod %s has no ImagePullPolicy set", container.Name, pod.Name)) + } + Expect(container.ImagePullPolicy).To(Equal(corev1.PullIfNotPresent), + "container %s in pod %s should have ImagePullPolicy=IfNotPresent, got %s", + container.Name, pod.Name, container.ImagePullPolicy) } - Expect(container.ImagePullPolicy).To(Equal(corev1.PullIfNotPresent), - "container %s in pod %s should have ImagePullPolicy=IfNotPresent, got %s", - container.Name, pod.Name, container.ImagePullPolicy) } - } + }) }) } }) @@ -373,36 +386,38 @@ func ContainerTerminationMessagePolicyTest(getTestCtx internal.TestContextGetter } for _, workload := range workloads { - It(fmt.Sprintf("should have FallbackToLogsOnError termination message policy for %s containers", workload.Name), func() { - testCtx := getTestCtx() - hostedCluster := testCtx.GetHostedCluster() - if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { - Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) - } - - // Skip if workload is in exemption list - if slices.Contains(exemptions, workload.Name) { - Skip(fmt.Sprintf("workload %s is exempt from termination message policy check", workload.Name)) - } + Context(workload.Name, func() { + It("should have FallbackToLogsOnError termination message policy for containers", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } - // Skip KubeVirt related pods - pods := getWorkloadPods(testCtx, workload) - if len(pods) == 0 { - Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) - } + // Skip if workload is in exemption list + if slices.Contains(exemptions, workload.Name) { + Skip(fmt.Sprintf("workload %s is exempt from termination message policy check", workload.Name)) + } - for _, pod := range pods { - for _, initContainer := range pod.Spec.InitContainers { - Expect(initContainer.TerminationMessagePolicy).To(Equal(corev1.TerminationMessageFallbackToLogsOnError), - "initContainer %s in pod %s should have TerminationMessagePolicy=FallbackToLogsOnError", - initContainer.Name, pod.Name) + // Skip KubeVirt related pods + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) } - for _, container := range pod.Spec.Containers { - Expect(container.TerminationMessagePolicy).To(Equal(corev1.TerminationMessageFallbackToLogsOnError), - "container %s in pod %s should have TerminationMessagePolicy=FallbackToLogsOnError", - container.Name, pod.Name) + + for _, pod := range pods { + for _, initContainer := range pod.Spec.InitContainers { + Expect(initContainer.TerminationMessagePolicy).To(Equal(corev1.TerminationMessageFallbackToLogsOnError), + "initContainer %s in pod %s should have TerminationMessagePolicy=FallbackToLogsOnError", + initContainer.Name, pod.Name) + } + for _, container := range pod.Spec.Containers { + Expect(container.TerminationMessagePolicy).To(Equal(corev1.TerminationMessageFallbackToLogsOnError), + "container %s in pod %s should have TerminationMessagePolicy=FallbackToLogsOnError", + container.Name, pod.Name) + } } - } + }) }) } }) @@ -413,30 +428,32 @@ func ContainerResourceRequestsTest(getTestCtx internal.TestContextGetter) { Context("Container resource requests", func() { // EnsureHCPContainersHaveResourceRequests for _, workload := range workloads { - It(fmt.Sprintf("should have resource requests for %s containers", workload.Name), func() { - testCtx := getTestCtx() - hostedCluster := testCtx.GetHostedCluster() - if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { - Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) - } + Context(workload.Name, func() { + It("should have resource requests for containers", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } - pods := getWorkloadPods(testCtx, workload) - if len(pods) == 0 { - Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) - } + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) + } - for _, pod := range pods { - for _, container := range pod.Spec.Containers { - Expect(container.Resources.Requests).NotTo(BeNil(), - "container %s in pod %s should have resource requests", container.Name, pod.Name) - _, hasCPU := container.Resources.Requests[corev1.ResourceCPU] - Expect(hasCPU).To(BeTrue(), - "container %s in pod %s should have CPU resource request", container.Name, pod.Name) - _, hasMemory := container.Resources.Requests[corev1.ResourceMemory] - Expect(hasMemory).To(BeTrue(), - "container %s in pod %s should have memory resource request", container.Name, pod.Name) + for _, pod := range pods { + for _, container := range pod.Spec.Containers { + Expect(container.Resources.Requests).NotTo(BeNil(), + "container %s in pod %s should have resource requests", container.Name, pod.Name) + _, hasCPU := container.Resources.Requests[corev1.ResourceCPU] + Expect(hasCPU).To(BeTrue(), + "container %s in pod %s should have CPU resource request", container.Name, pod.Name) + _, hasMemory := container.Resources.Requests[corev1.ResourceMemory] + Expect(hasMemory).To(BeTrue(), + "container %s in pod %s should have memory resource request", container.Name, pod.Name) + } } - } + }) }) } }) @@ -448,23 +465,25 @@ func PodPriorityTest(getTestCtx internal.TestContextGetter) { // EnsureNoPodsWithTooHighPriority const maxAllowedPriority = 100002000 for _, workload := range workloads { - It(fmt.Sprintf("should not have too high priority for %s pods", workload.Name), func() { - testCtx := getTestCtx() - hostedCluster := testCtx.GetHostedCluster() - if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { - Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) - } + Context(workload.Name, func() { + It("should not have too high priority for pods", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } - pods := getWorkloadPods(testCtx, workload) - if len(pods) == 0 { - Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) - } + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) + } - for _, pod := range pods { - if pod.Spec.Priority != nil && *pod.Spec.Priority > maxAllowedPriority { - Fail(fmt.Sprintf("pod %s has priority %d which exceeds maximum allowed %d", pod.Name, *pod.Spec.Priority, maxAllowedPriority)) + for _, pod := range pods { + if pod.Spec.Priority != nil && *pod.Spec.Priority > maxAllowedPriority { + Fail(fmt.Sprintf("pod %s has priority %d which exceeds maximum allowed %d", pod.Name, *pod.Spec.Priority, maxAllowedPriority)) + } } - } + }) }) } }) @@ -531,29 +550,31 @@ func ServiceAccountTokenMountingTest(getTestCtx internal.TestContextGetter) { } for _, workload := range workloads { - It(fmt.Sprintf("should not mount service account token unless necessary for %s pods", workload.Name), func() { - testCtx := getTestCtx() - hostedCluster := testCtx.GetHostedCluster() - if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { - Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) - } + Context(workload.Name, func() { + It("should not mount service account token unless necessary for pods", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } - // Skip if workload is in exemption list - if slices.Contains(exemptions, workload.Name) { - Skip(fmt.Sprintf("workload %s is exempt from service account token mounting check", workload.Name)) - } + // Skip if workload is in exemption list + if slices.Contains(exemptions, workload.Name) { + Skip(fmt.Sprintf("workload %s is exempt from service account token mounting check", workload.Name)) + } - pods := getWorkloadPods(testCtx, workload) - if len(pods) == 0 { - Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) - } + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) + } - for _, pod := range pods { - for _, volume := range pod.Spec.Volumes { - Expect(volume.Name).NotTo(HavePrefix("kube-api-access-"), - "pod %s should not have kube-api-access-* volume mounted", pod.Name) + for _, pod := range pods { + for _, volume := range pod.Spec.Volumes { + Expect(volume.Name).NotTo(HavePrefix("kube-api-access-"), + "pod %s should not have kube-api-access-* volume mounted", pod.Name) + } } - } + }) }) } }) @@ -572,118 +593,120 @@ func PodAffinitiesAndTolerationsTest(getTestCtx internal.TestContextGetter) { }) for _, workload := range workloads { - It(fmt.Sprintf("should have correct affinities and tolerations for %s pods", workload.Name), func() { - testCtx := getTestCtx() - hostedCluster := testCtx.GetHostedCluster() - if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { - Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) - } - - // SRO is being removed in 4.18 - if workload.Name == "shared-resource-csi-driver-operator" { - Skip("shared-resource-csi-driver-operator is exempt from affinities and tolerations check") - } + Context(workload.Name, func() { + It("should have correct affinities and tolerations for pods", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } - if workload.Name == "virt-launcher" || workload.Name == "vmi-console-debug" { - Skip("virt-launcher and vmi-console-debug are exempt from affinities and tolerations check") - } + // SRO is being removed in 4.18 + if workload.Name == "shared-resource-csi-driver-operator" { + Skip("shared-resource-csi-driver-operator is exempt from affinities and tolerations check") + } - pods := getWorkloadPods(testCtx, workload) - if len(pods) == 0 { - Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) - } + if workload.Name == "virt-launcher" || workload.Name == "vmi-console-debug" { + Skip("virt-launcher and vmi-console-debug are exempt from affinities and tolerations check") + } - controlPlaneLabelTolerationKey := "hypershift.openshift.io/control-plane" - clusterNodeSchedulingAffinityWeight := 100 - controlPlaneNodeSchedulingAffinityWeight := clusterNodeSchedulingAffinityWeight / 2 - colocationLabelKey := "hypershift.openshift.io/hosted-control-plane" - - var expectedTolerations []corev1.Toleration - switch workload.Name { - case "aws-ebs-csi-driver-operator": - expectedTolerations = []corev1.Toleration{ - { - Key: controlPlaneLabelTolerationKey, - Operator: corev1.TolerationOpExists, - }, - { - Key: hyperv1.HostedClusterLabel, - Operator: corev1.TolerationOpEqual, - Value: testCtx.ControlPlaneNamespace, - }, + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) } - default: - expectedTolerations = []corev1.Toleration{ - { - Key: controlPlaneLabelTolerationKey, - Operator: corev1.TolerationOpEqual, - Value: "true", - Effect: corev1.TaintEffectNoSchedule, - }, - { - Key: hyperv1.HostedClusterLabel, - Operator: corev1.TolerationOpEqual, - Value: testCtx.ControlPlaneNamespace, - Effect: corev1.TaintEffectNoSchedule, - }, + + controlPlaneLabelTolerationKey := "hypershift.openshift.io/control-plane" + clusterNodeSchedulingAffinityWeight := 100 + controlPlaneNodeSchedulingAffinityWeight := clusterNodeSchedulingAffinityWeight / 2 + colocationLabelKey := "hypershift.openshift.io/hosted-control-plane" + + var expectedTolerations []corev1.Toleration + switch workload.Name { + case "aws-ebs-csi-driver-operator": + expectedTolerations = []corev1.Toleration{ + { + Key: controlPlaneLabelTolerationKey, + Operator: corev1.TolerationOpExists, + }, + { + Key: hyperv1.HostedClusterLabel, + Operator: corev1.TolerationOpEqual, + Value: testCtx.ControlPlaneNamespace, + }, + } + default: + expectedTolerations = []corev1.Toleration{ + { + Key: controlPlaneLabelTolerationKey, + Operator: corev1.TolerationOpEqual, + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: hyperv1.HostedClusterLabel, + Operator: corev1.TolerationOpEqual, + Value: testCtx.ControlPlaneNamespace, + Effect: corev1.TaintEffectNoSchedule, + }, + } } - } - for _, pod := range pods { - for _, expectedTol := range expectedTolerations { - found := false - for _, tol := range pod.Spec.Tolerations { - if tol.Key == expectedTol.Key && tol.Operator == expectedTol.Operator && tol.Value == expectedTol.Value && tol.Effect == expectedTol.Effect { - found = true - break + for _, pod := range pods { + for _, expectedTol := range expectedTolerations { + found := false + for _, tol := range pod.Spec.Tolerations { + if tol.Key == expectedTol.Key && tol.Operator == expectedTol.Operator && tol.Value == expectedTol.Value && tol.Effect == expectedTol.Effect { + found = true + break + } } + Expect(found).To(BeTrue(), "pod %s should have toleration %+v", pod.Name, expectedTol) } - Expect(found).To(BeTrue(), "pod %s should have toleration %+v", pod.Name, expectedTol) - } - - // Check affinities - Expect(pod.Spec.Affinity).NotTo(BeNil(), "pod %s should have affinity", pod.Name) - Expect(pod.Spec.Affinity.NodeAffinity).NotTo(BeNil(), "pod %s should have node affinity", pod.Name) - Expect(pod.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution).NotTo(BeEmpty(), - "pod %s should have preferred node affinity", pod.Name) - - // Check for control plane node affinity - hasControlPlaneAffinity := false - for _, term := range pod.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution { - if term.Weight == int32(controlPlaneNodeSchedulingAffinityWeight) { - for _, req := range term.Preference.MatchExpressions { - if req.Key == controlPlaneLabelTolerationKey && req.Operator == corev1.NodeSelectorOpIn { - hasControlPlaneAffinity = true - break + + // Check affinities + Expect(pod.Spec.Affinity).NotTo(BeNil(), "pod %s should have affinity", pod.Name) + Expect(pod.Spec.Affinity.NodeAffinity).NotTo(BeNil(), "pod %s should have node affinity", pod.Name) + Expect(pod.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution).NotTo(BeEmpty(), + "pod %s should have preferred node affinity", pod.Name) + + // Check for control plane node affinity + hasControlPlaneAffinity := false + for _, term := range pod.Spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution { + if term.Weight == int32(controlPlaneNodeSchedulingAffinityWeight) { + for _, req := range term.Preference.MatchExpressions { + if req.Key == controlPlaneLabelTolerationKey && req.Operator == corev1.NodeSelectorOpIn { + hasControlPlaneAffinity = true + break + } } } } - } - Expect(hasControlPlaneAffinity).To(BeTrue(), "pod %s should have control plane node affinity", pod.Name) + Expect(hasControlPlaneAffinity).To(BeTrue(), "pod %s should have control plane node affinity", pod.Name) - // Check for pod affinity - Expect(pod.Spec.Affinity.PodAffinity).NotTo(BeNil(), "pod %s should have pod affinity", pod.Name) - Expect(pod.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution).NotTo(BeEmpty(), - "pod %s should have preferred pod affinity", pod.Name) + // Check for pod affinity + Expect(pod.Spec.Affinity.PodAffinity).NotTo(BeNil(), "pod %s should have pod affinity", pod.Name) + Expect(pod.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution).NotTo(BeEmpty(), + "pod %s should have preferred pod affinity", pod.Name) - hasColocationAffinity := false - for _, term := range pod.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution { - if term.Weight != 100 || term.PodAffinityTerm.LabelSelector == nil { - continue - } - for _, value := range term.PodAffinityTerm.LabelSelector.MatchLabels { - if value == testCtx.ControlPlaneNamespace { + hasColocationAffinity := false + for _, term := range pod.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution { + if term.Weight != 100 || term.PodAffinityTerm.LabelSelector == nil { + continue + } + for _, value := range term.PodAffinityTerm.LabelSelector.MatchLabels { + if value == testCtx.ControlPlaneNamespace { + hasColocationAffinity = true + break + } + } + if term.PodAffinityTerm.LabelSelector.MatchLabels[colocationLabelKey] == testCtx.ControlPlaneNamespace { hasColocationAffinity = true break } } - if term.PodAffinityTerm.LabelSelector.MatchLabels[colocationLabelKey] == testCtx.ControlPlaneNamespace { - hasColocationAffinity = true - break - } + Expect(hasColocationAffinity).To(BeTrue(), "pod %s should have colocation pod affinity", pod.Name) } - Expect(hasColocationAffinity).To(BeTrue(), "pod %s should have colocation pod affinity", pod.Name) - } + }) }) } }) @@ -692,29 +715,22 @@ func PodAffinitiesAndTolerationsTest(getTestCtx internal.TestContextGetter) { // WorkloadRegistryValidationTest registers tests for workload registry validation func WorkloadRegistryValidationTest(getTestCtx internal.TestContextGetter) { Context("Workload registry validation", func() { - // Label("Informing"): failures skip (non-blocking) until registry is complete - It("all pods should belong to predefined workloads", Label("Informing"), func() { + It("should not contain any unrecognized pods", func() { testCtx := getTestCtx() - _ = testCtx.GetHostedCluster() // unused but kept for consistency - // List all pods in control plane namespace podList := &corev1.PodList{} - err := testCtx.MgmtClient.List(context.Background(), podList, &crclient.ListOptions{ + err := testCtx.MgmtClient.List(testCtx.Context, podList, &crclient.ListOptions{ Namespace: testCtx.ControlPlaneNamespace, }) Expect(err).NotTo(HaveOccurred(), "failed to list pods in control plane namespace") - // Build a map of workload selectors for quick lookup workloadSelectors := make(map[string]labels.Selector) for _, workload := range workloads { selector := labels.SelectorFromSet(workload.PodSelector) workloadSelectors[workload.Name] = selector } - // Check each pod var podsNotBelongingToWorkloads []string for _, pod := range podList.Items { - // Skip system pods (if any) - // Check if pod belongs to any workload belongsToWorkload := false for _, selector := range workloadSelectors { if selector.Matches(labels.Set(pod.Labels)) { @@ -778,39 +794,251 @@ func SecurityContextUIDTest(getTestCtx internal.TestContextGetter) { } for _, workload := range workloads { - It(fmt.Sprintf("should have expected RunAsUser UID for %s pods", workload.Name), func() { - testCtx := getTestCtx() - hostedCluster := testCtx.GetHostedCluster() - if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { - Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) - } + Context(workload.Name, func() { + It("should have expected RunAsUser UID for pods", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } - // Skip if workload is in exemption list - if slices.Contains(exemptions, workload.Name) { - Skip(fmt.Sprintf("workload %s is exempt from security context UID check", workload.Name)) - } + // Skip if workload is in exemption list + if slices.Contains(exemptions, workload.Name) { + Skip(fmt.Sprintf("workload %s is exempt from security context UID check", workload.Name)) + } - pods := getWorkloadPods(testCtx, workload) - if len(pods) == 0 { - Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) - } + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) + } + + for _, pod := range pods { + runAsUser := func() *int64 { + if pod.Spec.SecurityContext == nil || pod.Spec.SecurityContext.RunAsUser == nil { + return nil + } + return pod.Spec.SecurityContext.RunAsUser + }() - for _, pod := range pods { - runAsUser := func() *int64 { - if pod.Spec.SecurityContext == nil || pod.Spec.SecurityContext.RunAsUser == nil { - return nil + if runAsUser == nil { + Fail(fmt.Sprintf("pod %s/%s: RunAsUser is not set (expected UID %d)", pod.Namespace, pod.Name, expectedUID)) } - return pod.Spec.SecurityContext.RunAsUser - }() - if runAsUser == nil { - Fail(fmt.Sprintf("pod %s/%s: RunAsUser is not set (expected UID %d)", pod.Namespace, pod.Name, expectedUID)) + Expect(*runAsUser).To(Equal(expectedUID), + "pod %s/%s: RunAsUser %d does not match expected UID %d", + pod.Namespace, pod.Name, *runAsUser, expectedUID) } + }) + }) + } + }) +} - Expect(*runAsUser).To(Equal(expectedUID), - "pod %s/%s: RunAsUser %d does not match expected UID %d", - pod.Namespace, pod.Name, *runAsUser, expectedUID) - } +func isCertificateTriggeredRestart(ctx context.Context, client crclient.Client, pod *corev1.Pod) bool { + hcpList := &hyperv1.HostedControlPlaneList{} + if err := client.List(ctx, hcpList, crclient.InNamespace(pod.Namespace)); err != nil { + fmt.Fprintf(GinkgoWriter, "couldn't list HostedControlPlanes; pod namespace: %s, pod name: %s, error: %v\n", pod.Namespace, pod.Name, err) + return false + } + for _, hcp := range hcpList.Items { + if restartAnnotation, ok := hcp.Annotations[hyperv1.RestartDateAnnotation]; ok { + if strings.HasPrefix(restartAnnotation, "CertHash:") { + return true + } + } + } + return false +} + +func isLeaderElectionFailure(ctx context.Context, client kubernetes.Interface, pod *corev1.Pod, containerName string) bool { + req := client.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{ + Container: containerName, + Previous: true, + TailLines: ptr.To[int64](10), + }) + podLogs, err := req.Stream(ctx) + if err != nil { + fmt.Fprintf(GinkgoWriter, "couldn't stream pod log; pod namespace: %s, pod name: %s, error: %v\n", pod.Namespace, pod.Name, err) + return false + } + defer podLogs.Close() + + scanner := bufio.NewScanner(podLogs) + scanner.Buffer(make([]byte, 256*1024), 512*1024) + for scanner.Scan() { + if strings.Contains(strings.ToLower(scanner.Text()), "election lost") { + return true + } + } + // Drain remaining data to avoid broken pipe + _, _ = io.Copy(io.Discard, podLogs) + if err := scanner.Err(); err != nil { + fmt.Fprintf(GinkgoWriter, "failed to read pod log; pod namespace: %s, pod name: %s, error: %v\n", pod.Namespace, pod.Name, err) + } + return false +} + +func NoCrashingPodsTest(getTestCtx internal.TestContextGetter) { + Context("No crashing pods", func() { + crashTolerations := map[string]int32{ + "ingress-operator": 20, + "cloud-credential-operator": 20, + "olm-operator": 20, + "catalog-operator": 20, + "certified-operators-catalog": 20, + "community-operators-catalog": 20, + "redhat-operators-catalog": 20, + "redhat-marketplace-catalog": 20, + "openstack-manila-csi-controllerplugin": 20, + "kubevirt-csi": 20, + "aws-ebs-csi-driver-controller": 1, + "network-node-identity": 1, + "kubevirt-cloud-controller-manager": 2, + "gcp-cloud-controller-manager": 1, + "dns-operator": 5, + } + + for _, workload := range workloads { + Context(workload.Name, func() { + It("should have no crashing pods", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } + + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) + } + + var defaultCrashToleration int32 + if hostedCluster.Spec.Platform.Type == hyperv1.KubevirtPlatform { + kvPlatform := hostedCluster.Spec.Platform.Kubevirt + if kvPlatform != nil && kvPlatform.Credentials != nil { + defaultCrashToleration = 1 + } + if kvPlatform != nil && hostedCluster.Annotations != nil { + mgmtPlatform, annotationExists := hostedCluster.Annotations[hyperv1.ManagementPlatformAnnotation] + if annotationExists && mgmtPlatform == string(hyperv1.AzurePlatform) { + defaultCrashToleration = 1 + } + } + } + + toleration := defaultCrashToleration + if t, ok := crashTolerations[workload.Name]; ok { + toleration = t + } + + var k8sClient kubernetes.Interface + for _, pod := range pods { + for _, containerStatus := range pod.Status.ContainerStatuses { + if containerStatus.RestartCount <= toleration { + continue + } + if strings.HasPrefix(pod.Name, "kube-controller-manager-") { + if isCertificateTriggeredRestart(testCtx.Context, testCtx.MgmtClient, &pod) { + continue + } + } + if k8sClient == nil { + mgmtRestConfig, err := e2eutil.GetConfig() + Expect(err).NotTo(HaveOccurred(), "failed to get management REST config for log inspection") + k8sClient, err = kubernetes.NewForConfig(mgmtRestConfig) + Expect(err).NotTo(HaveOccurred(), "failed to create kubernetes clientset for log inspection") + } + if isLeaderElectionFailure(testCtx.Context, k8sClient, &pod, containerStatus.Name) { + continue + } + Expect(containerStatus.RestartCount).To(BeNumerically("<=", toleration), + "container %s in pod %s has too many restarts (%d > %d)", + containerStatus.Name, pod.Name, containerStatus.RestartCount, toleration) + } + } + }) + }) + } + }) +} + +// CustomLabelsTest registers per-workload tests for custom label propagation +func CustomLabelsTest(getTestCtx internal.TestContextGetter) { + Context("Custom labels", Label("Informing"), func() { + BeforeEach(func() { + e2eutil.GinkgoAtLeast(e2eutil.Version419) + }) + + exemptions := []string{ + "virt-launcher", + "vmi-console-debug", + } + + for _, workload := range workloads { + Context(workload.Name, func() { + It("should propagate custom labels to pods", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } + if slices.Contains(exemptions, workload.Name) { + Skip(fmt.Sprintf("workload %s is exempt from custom labels check", workload.Name)) + } + + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) + } + + for _, pod := range pods { + Expect(pod.Labels).To(HaveKeyWithValue("hypershift-e2e-test-label", "test"), + "pod %s should have custom label hypershift-e2e-test-label=test", pod.Name) + } + }) + }) + } + }) +} + +// CustomTolerationsTest registers per-workload tests for custom toleration propagation +func CustomTolerationsTest(getTestCtx internal.TestContextGetter) { + Context("Custom tolerations", Label("Informing"), func() { + BeforeEach(func() { + e2eutil.GinkgoAtLeast(e2eutil.Version419) + }) + + exemptions := []string{ + "virt-launcher", + "vmi-console-debug", + } + + for _, workload := range workloads { + Context(workload.Name, func() { + It("should propagate custom tolerations to pods", func() { + testCtx := getTestCtx() + hostedCluster := testCtx.GetHostedCluster() + if internal.ShouldSkipWorkloadForPlatform(workload, hostedCluster) { + Skip(fmt.Sprintf("workload %s is platform-specific and doesn't match cluster platform", workload.Name)) + } + if slices.Contains(exemptions, workload.Name) { + Skip(fmt.Sprintf("workload %s is exempt from custom tolerations check", workload.Name)) + } + + pods := getWorkloadPods(testCtx, workload) + if len(pods) == 0 { + Skip(fmt.Sprintf("no pods found for workload %s", workload.Name)) + } + + for _, pod := range pods { + Expect(pod.Spec.Tolerations).To(ContainElement(corev1.Toleration{ + Key: "hypershift-e2e-test-toleration", + Operator: corev1.TolerationOpEqual, + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }), "pod %s should have custom toleration hypershift-e2e-test-toleration", pod.Name) + } + }) }) } }) @@ -820,6 +1048,7 @@ func SecurityContextUIDTest(getTestCtx internal.TestContextGetter) { func RegisterControlPlaneWorkloadsTests(getTestCtx internal.TestContextGetter) { WorkloadRegistryValidationTest(getTestCtx) DeploymentGenerationTest(getTestCtx) + NoCrashingPodsTest(getTestCtx) SafeToEvictAnnotationsTest(getTestCtx) ReadOnlyRootFilesystemTest(getTestCtx) ReadOnlyRootFilesystemTmpDirMountTest(getTestCtx) @@ -829,6 +1058,8 @@ func RegisterControlPlaneWorkloadsTests(getTestCtx internal.TestContextGetter) { PodPriorityTest(getTestCtx) ServiceAccountTokenMountingTest(getTestCtx) PodAffinitiesAndTolerationsTest(getTestCtx) + CustomLabelsTest(getTestCtx) + CustomTolerationsTest(getTestCtx) SecurityContextUIDTest(getTestCtx) } @@ -841,9 +1072,7 @@ var _ = Describe("Control Plane Workloads", Label("control-plane-workloads"), fu testCtx = internal.GetTestContext() Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") - if err := testCtx.ValidateControlPlaneNamespace(); err != nil { - AbortSuite(err.Error()) - } + testCtx.ValidateHostedCluster() }) RegisterControlPlaneWorkloadsTests(func() *internal.TestContext { return testCtx }) diff --git a/test/e2e/v2/tests/etcd_chaos_test.go b/test/e2e/v2/tests/etcd_chaos_test.go index e10a15b5561d..6298e5b5f2ff 100644 --- a/test/e2e/v2/tests/etcd_chaos_test.go +++ b/test/e2e/v2/tests/etcd_chaos_test.go @@ -122,11 +122,11 @@ func EtcdSingleMemberRecoveryTest(getTestCtx internal.TestContextGetter) { func EtcdKillRandomMembersTest(getTestCtx internal.TestContextGetter) { It("should preserve data when random members are repeatedly killed", func() { testCtx := getTestCtx() + testCtx.ValidateHostedClusterClient() ctx := testCtx.Context cpNamespace := testCtx.ControlPlaneNamespace guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") // Create marker data that should survive the chaos markerCM := createMarkerConfigMap(ctx, guestClient) @@ -169,11 +169,11 @@ func EtcdKillRandomMembersTest(getTestCtx internal.TestContextGetter) { func EtcdKillAllMembersTest(getTestCtx internal.TestContextGetter) { It("should preserve data when all members are killed simultaneously", func() { testCtx := getTestCtx() + testCtx.ValidateHostedClusterClient() ctx := testCtx.Context cpNamespace := testCtx.ControlPlaneNamespace guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") // Create marker data that should survive the chaos markerCM := createMarkerConfigMap(ctx, guestClient) diff --git a/test/e2e/v2/tests/hosted_cluster_ccm_test.go b/test/e2e/v2/tests/hosted_cluster_ccm_test.go index c8c4f6cd49b7..c277788035af 100644 --- a/test/e2e/v2/tests/hosted_cluster_ccm_test.go +++ b/test/e2e/v2/tests/hosted_cluster_ccm_test.go @@ -42,7 +42,7 @@ func GCPCloudControllerManagerTest(getTestCtx internal.TestContextGetter) { } }) - Context("When nodes are initialized by the CCM", func() { + When("nodes are initialized by the CCM", func() { var nodes *corev1.NodeList BeforeEach(func() { diff --git a/test/e2e/v2/tests/hosted_cluster_compliance_test.go b/test/e2e/v2/tests/hosted_cluster_compliance_test.go new file mode 100644 index 000000000000..5ba6405c6c3a --- /dev/null +++ b/test/e2e/v2/tests/hosted_cluster_compliance_test.go @@ -0,0 +1,82 @@ +//go:build e2ev2 + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/support/netutil" + "github.com/openshift/hypershift/test/e2e/v2/internal" + + routev1 "github.com/openshift/api/route/v1" + + crclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +// RegisterHostedClusterComplianceTests registers all hosted cluster compliance tests. +func RegisterHostedClusterComplianceTests(getTestCtx internal.TestContextGetter) { + EnsureAllRoutesUseHCPRouterTest(getTestCtx) +} + +func EnsureAllRoutesUseHCPRouterTest(getTestCtx internal.TestContextGetter) { + When("routes are created in the control plane namespace", func() { + It("should label all routes for the per-HCP router", Label("routes"), func() { + tc := getTestCtx() + hostedCluster := tc.GetHostedCluster() + Expect(hostedCluster).NotTo(BeNil(), "hosted cluster must be configured") + + isRoute := false + for _, svc := range hostedCluster.Spec.Services { + if svc.Service == hyperv1.APIServer && svc.Type == hyperv1.Route { + isRoute = true + break + } + } + if !isRoute { + Skip("route test only applies when APIServer is exposed via Route") + } + + routeList := &routev1.RouteList{} + Expect(tc.MgmtClient.List(tc.Context, routeList, crclient.InNamespace(tc.ControlPlaneNamespace))).To(Succeed()) + Expect(routeList.Items).NotTo(BeEmpty(), + "expected at least one route in control plane namespace %s", tc.ControlPlaneNamespace) + + for i := range routeList.Items { + route := &routeList.Items[i] + original := route.DeepCopy() + netutil.AddHCPRouteLabel(route) + Expect(route.Labels).To(Equal(original.Labels), + "route %s is missing the label to use the per-HCP router", route.Name) + } + }) + }) +} + +var _ = Describe("Hosted Cluster Compliance", Label("hosted-cluster-compliance"), func() { + var testCtx *internal.TestContext + + BeforeEach(func() { + testCtx = internal.GetTestContext() + Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + + testCtx.ValidateHostedCluster() + }) + + RegisterHostedClusterComplianceTests(func() *internal.TestContext { return testCtx }) +}) diff --git a/test/e2e/v2/tests/hosted_cluster_dns_test.go b/test/e2e/v2/tests/hosted_cluster_dns_test.go new file mode 100644 index 000000000000..ef3967bc676c --- /dev/null +++ b/test/e2e/v2/tests/hosted_cluster_dns_test.go @@ -0,0 +1,81 @@ +//go:build e2ev2 + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + "github.com/openshift/hypershift/support/netutil" + e2eutil "github.com/openshift/hypershift/test/e2e/util" + "github.com/openshift/hypershift/test/e2e/v2/internal" +) + +// RegisterHostedClusterDNSTests registers DNS-related hosted cluster tests. +func RegisterHostedClusterDNSTests(getTestCtx internal.TestContextGetter) { + EnsureKubeAPIDNSNameCustomCertTest(getTestCtx) +} + +func EnsureKubeAPIDNSNameCustomCertTest(getTestCtx internal.TestContextGetter) { + When("KubeAPIDNSName and custom certificate are configured", func() { + PIt("should make KAS reachable via the custom DNS endpoint", func() { + tc := getTestCtx() + hostedCluster := tc.GetHostedCluster() + if e2eutil.IsLessThan(e2eutil.Version419) { + Skip("custom DNS name test requires version >= 4.19") + } + if hostedCluster.Spec.Platform.Type == hyperv1.KubevirtPlatform { + Skip("custom DNS name test not supported on KubeVirt platform") + } + + if !netutil.IsPublicHC(hostedCluster) { + Skip("custom DNS name test requires a public hosted cluster") + } + + serviceDomain := internal.GetEnvVarValue("E2E_SERVICE_DOMAIN") + if serviceDomain == "" { + Skip("E2E_SERVICE_DOMAIN not set; skipping custom DNS name test") + } + + // The full implementation would: + // 1. Generate a custom TLS cert via e2eutil.GenerateCustomCertificate() + // 2. Create a cert secret in the HCP namespace + // 3. Update HC with KubeAPIDNSName and custom serving cert reference + // 4. Wait for custom kubeconfig status to appear (30-min timeout) + // 5. Create ExternalName Service with DNS annotation + // 6. Wait for DNS resolution and KAS reachability + // 7. Validate custom kubeconfig status and secret + // 8. Defer full HC state restoration + // + // This test is marked pending until the full DNS lifecycle is wired up. + Expect(serviceDomain).NotTo(BeEmpty()) + }) + }) +} + +var _ = Describe("Hosted Cluster DNS", Label("hosted-cluster-dns"), func() { + var testCtx *internal.TestContext + + BeforeEach(func() { + testCtx = internal.GetTestContext() + Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedCluster() + }) + + RegisterHostedClusterDNSTests(func() *internal.TestContext { return testCtx }) +}) diff --git a/test/e2e/v2/tests/hosted_cluster_health_test.go b/test/e2e/v2/tests/hosted_cluster_health_test.go new file mode 100644 index 000000000000..787e5f21047c --- /dev/null +++ b/test/e2e/v2/tests/hosted_cluster_health_test.go @@ -0,0 +1,198 @@ +//go:build e2ev2 + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + hcc "github.com/openshift/hypershift/hypershift-operator/controllers/hostedcluster" + "github.com/openshift/hypershift/support/conditions" + hyperutil "github.com/openshift/hypershift/support/util" + e2eutil "github.com/openshift/hypershift/test/e2e/util" + "github.com/openshift/hypershift/test/e2e/v2/internal" + + configv1 "github.com/openshift/api/config/v1" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/api/meta" + + crclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +// RegisterHostedClusterHealthTests registers all hosted cluster health test specs. +func RegisterHostedClusterHealthTests(getTestCtx internal.TestContextGetter) { + ValidateHostedClusterConditionsTest(getTestCtx) + EnsureCAPIFinalizersTest(getTestCtx) + EnsureFeatureGateStatusTest(getTestCtx) + EnsurePayloadArchSetCorrectlyTest(getTestCtx) + ValidateConfigurationStatusTest(getTestCtx) +} + +func ValidateHostedClusterConditionsTest(getTestCtx internal.TestContextGetter) { + When("hosted cluster is operational", func() { + It("should have all expected conditions with correct status", func() { + tc := getTestCtx() + hostedCluster := tc.GetHostedCluster() + + expectedConditions := conditions.ExpectedHCConditions(hostedCluster) + delete(expectedConditions, hyperv1.KubeVirtNodesLiveMigratable) + if e2eutil.IsLessThan(e2eutil.Version421) { + delete(expectedConditions, hyperv1.DataPlaneConnectionAvailable) + } + if e2eutil.IsLessThan(e2eutil.Version422) { + delete(expectedConditions, hyperv1.ControlPlaneConnectionAvailable) + delete(expectedConditions, hyperv1.ValidKubeVirtInfraNetworkPolicyRBAC) + } + + Eventually(func(g Gomega) { + hc := &hyperv1.HostedCluster{} + g.Expect(tc.MgmtClient.Get(tc.Context, crclient.ObjectKeyFromObject(hostedCluster), hc)).To(Succeed()) + for condType, expectedStatus := range expectedConditions { + condition := meta.FindStatusCondition(hc.Status.Conditions, string(condType)) + g.Expect(condition).NotTo(BeNil(), "condition %s should be present", condType) + g.Expect(condition.Status).To(Equal(expectedStatus), "condition %s should have status %s", condType, expectedStatus) + } + }, 10*time.Minute, 10*time.Second).Should(Succeed()) + }) + }) +} + +func EnsureCAPIFinalizersTest(getTestCtx internal.TestContextGetter) { + When("CAPI components are deployed", func() { + It("should have component finalizers on all CAPI deployments", func() { + tc := getTestCtx() + for _, name := range hcc.CAPIComponents { + deployment := &appsv1.Deployment{} + Expect(tc.MgmtClient.Get(tc.Context, crclient.ObjectKey{ + Name: name, + Namespace: tc.ControlPlaneNamespace, + }, deployment)).To(Succeed(), "failed to get CAPI deployment %s", name) + Expect(controllerutil.ContainsFinalizer(deployment, hcc.ControlPlaneComponentFinalizer)).To(BeTrue(), + "CAPI deployment %s should have finalizer %s", name, hcc.ControlPlaneComponentFinalizer) + } + }) + }) +} + +func EnsureFeatureGateStatusTest(getTestCtx internal.TestContextGetter) { + When("hosted cluster version is completed", func() { + It("should have feature gate status matching cluster version", func() { + tc := getTestCtx() + if e2eutil.IsLessThan(e2eutil.Version419) { + Skip("Feature gate status test requires version >= 4.19") + } + tc.ValidateHostedClusterClient() + hcClient := tc.GetHostedClusterClient() + + var currentVersion string + Eventually(func(g Gomega) { + cv := &configv1.ClusterVersion{} + g.Expect(hcClient.Get(tc.Context, crclient.ObjectKey{Name: "version"}, cv)).To(Succeed()) + g.Expect(cv.Status.History).NotTo(BeEmpty()) + g.Expect(cv.Status.History[0].State).To(Equal(configv1.CompletedUpdate)) + currentVersion = cv.Status.History[0].Version + }, 30*time.Minute, 30*time.Second).Should(Succeed()) + + Eventually(func(g Gomega) { + fg := &configv1.FeatureGate{} + g.Expect(hcClient.Get(tc.Context, crclient.ObjectKey{Name: "cluster"}, fg)).To(Succeed()) + found := false + for _, details := range fg.Status.FeatureGates { + if details.Version == currentVersion { + found = true + break + } + } + g.Expect(found).To(BeTrue(), "version %s not found in FeatureGate status", currentVersion) + }, 10*time.Minute, 10*time.Second).Should(Succeed()) + }) + }) +} + +func EnsurePayloadArchSetCorrectlyTest(getTestCtx internal.TestContextGetter) { + When("hosted cluster has a release image", func() { + It("should set payload arch status correctly", func() { + tc := getTestCtx() + hostedCluster := tc.GetHostedCluster() + + imageMetadataProvider := &hyperutil.RegistryClientImageMetadataProvider{} + Eventually(func(g Gomega) { + hc := &hyperv1.HostedCluster{} + g.Expect(tc.MgmtClient.Get(tc.Context, crclient.ObjectKeyFromObject(hostedCluster), hc)).To(Succeed()) + g.Expect(hc.Status.PayloadArch).NotTo(BeEmpty(), "PayloadArch should be set") + payloadArch, err := hyperutil.DetermineHostedClusterPayloadArch(tc.Context, tc.MgmtClient, hc, imageMetadataProvider) + g.Expect(err).NotTo(HaveOccurred(), "failed to determine payload arch") + g.Expect(payloadArch).To(Equal(hc.Status.PayloadArch)) + }, 30*time.Minute, time.Minute).Should(Succeed()) + }) + }) +} + +func ValidateConfigurationStatusTest(getTestCtx internal.TestContextGetter) { + When("hosted cluster authentication is configured", func() { + It("should propagate configuration status consistently", func() { + tc := getTestCtx() + hostedCluster := tc.GetHostedCluster() + if e2eutil.IsLessThan(e2eutil.Version421) { + Skip("Configuration status requires version >= 4.21") + } + tc.ValidateHostedClusterClient() + hcClient := tc.GetHostedClusterClient() + + Eventually(func(g Gomega) { + var hostedClusterAuth configv1.Authentication + g.Expect(hcClient.Get(tc.Context, crclient.ObjectKey{Name: "cluster"}, &hostedClusterAuth)).To(Succeed()) + + var hcp hyperv1.HostedControlPlane + g.Expect(tc.MgmtClient.Get(tc.Context, crclient.ObjectKey{ + Name: hostedCluster.Name, + Namespace: tc.ControlPlaneNamespace, + }, &hcp)).To(Succeed()) + g.Expect(hcp.Status.Configuration).NotTo(BeNil(), "HCP configuration status not set") + + var hc hyperv1.HostedCluster + g.Expect(tc.MgmtClient.Get(tc.Context, crclient.ObjectKeyFromObject(hostedCluster), &hc)).To(Succeed()) + g.Expect(hc.Status.Configuration).NotTo(BeNil(), "HC configuration status not set") + + g.Expect(hcp.Status.Configuration.Authentication).To(Equal(hostedClusterAuth.Status), + "HCP authentication status should match hosted cluster Authentication resource") + g.Expect(hc.Status.Configuration.Authentication).To(Equal(hostedClusterAuth.Status), + "HC authentication status should match hosted cluster Authentication resource") + g.Expect(hcp.Status.Configuration.Authentication).To(Equal(hc.Status.Configuration.Authentication), + "HCP and HC authentication status should be consistent") + }, 10*time.Minute, 10*time.Second).Should(Succeed()) + }) + }) +} + +var _ = Describe("Hosted Cluster Health", Label("hosted-cluster-health"), func() { + var testCtx *internal.TestContext + + BeforeEach(func() { + testCtx = internal.GetTestContext() + Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + + testCtx.ValidateHostedCluster() + }) + + RegisterHostedClusterHealthTests(func() *internal.TestContext { return testCtx }) +}) diff --git a/test/e2e/v2/tests/hosted_cluster_image_registry_test.go b/test/e2e/v2/tests/hosted_cluster_image_registry_test.go index 7dcbc84dbf5a..7db2070eea5d 100644 --- a/test/e2e/v2/tests/hosted_cluster_image_registry_test.go +++ b/test/e2e/v2/tests/hosted_cluster_image_registry_test.go @@ -48,8 +48,8 @@ func RegisterHostedClusterImageRegistryTests(getTestCtx internal.TestContextGett // automatically skipped. func ImageRegistryCapabilityEnabledTest(getTestCtx internal.TestContextGetter) { var ( - tc *internal.TestContext - hc *hyperv1.HostedCluster + tc *internal.TestContext + hc *hyperv1.HostedCluster hostedClusterClient crclient.Client ) @@ -245,7 +245,10 @@ func ImageRegistryCapabilityDisabledTest(getTestCtx internal.TestContextGetter) ns.Name = "image-registry-test-namespace" Expect(hostedClusterClient.Create(tc.Context, ns)).To(Succeed()) DeferCleanup(func() { - _ = hostedClusterClient.Delete(tc.Context, ns) + err := hostedClusterClient.Delete(tc.Context, ns) + if err != nil && !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred(), "cleanup: failed to delete test namespace %s", ns.Name) + } }) Eventually(func(g Gomega) { @@ -281,9 +284,7 @@ var _ = Describe("Hosted Cluster Image Registry", Label("hosted-cluster-image-re testCtx = internal.GetTestContext() Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") - if err := testCtx.ValidateControlPlaneNamespace(); err != nil { - AbortSuite(err.Error()) - } + testCtx.ValidateHostedCluster() }) RegisterHostedClusterImageRegistryTests(func() *internal.TestContext { return testCtx }) diff --git a/test/e2e/v2/tests/hosted_cluster_metrics_test.go b/test/e2e/v2/tests/hosted_cluster_metrics_test.go new file mode 100644 index 000000000000..68db4f75bd57 --- /dev/null +++ b/test/e2e/v2/tests/hosted_cluster_metrics_test.go @@ -0,0 +1,322 @@ +//go:build e2ev2 + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + hcmetrics "github.com/openshift/hypershift/hypershift-operator/controllers/hostedcluster/metrics" + npmetrics "github.com/openshift/hypershift/hypershift-operator/controllers/nodepool/metrics" + azureutil "github.com/openshift/hypershift/support/azureutil" + e2eutil "github.com/openshift/hypershift/test/e2e/util" + "github.com/openshift/hypershift/test/e2e/v2/internal" + v2util "github.com/openshift/hypershift/test/e2e/v2/util" + + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + dto "github.com/prometheus/client_model/go" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/client-go/kubernetes" + + crclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func expectMetricHasLabel(g Gomega, families map[string]*dto.MetricFamily, metricName, labelName, labelValue string) { + family, ok := families[metricName] + g.Expect(ok).To(BeTrue(), "metric %s should exist", metricName) + hasMatch := false + for _, m := range family.Metric { + for _, l := range m.GetLabel() { + if l.GetName() == labelName && l.GetValue() == labelValue { + hasMatch = true + } + } + } + g.Expect(hasMatch).To(BeTrue(), "metric %s should have label %s=%s", metricName, labelName, labelValue) +} + +func RegisterHostedClusterMetricsTests(getTestCtx internal.TestContextGetter) { + ValidateMetricsTest(getTestCtx) + EnsureMetricsForwarderWorkingTest(getTestCtx) + EnsureNodeTuningOperatorMetricsEndpointTest(getTestCtx) +} + +func ValidateMetricsTest(getTestCtx internal.TestContextGetter) { + When("HyperShift operator is running", func() { + It("should expose expected metrics at the metrics endpoint", func() { + tc := getTestCtx() + hostedCluster := tc.GetHostedCluster() + if hostedCluster.Spec.Platform.Type == hyperv1.NonePlatform { + Skip("metrics test skipped for None platform") + } + + mgmtRestConfig, err := e2eutil.GetConfig() + Expect(err).NotTo(HaveOccurred(), "should be able to load management cluster REST config") + + clientset, err := kubernetes.NewForConfig(mgmtRestConfig) + Expect(err).NotTo(HaveOccurred(), "should be able to create kubernetes clientset") + + hoNamespace := "hypershift" + hcName := hostedCluster.Name + + Eventually(func(g Gomega) { + currentPods := &corev1.PodList{} + g.Expect(tc.MgmtClient.List(tc.Context, currentPods, + crclient.InNamespace(hoNamespace), + crclient.MatchingLabels{"app": "operator"}, + )).To(Succeed(), "should be able to list pods in the hypershift namespace") + g.Expect(currentPods.Items).NotTo(BeEmpty(), "hypershift-operator pod should exist") + + var runningPodName string + for _, p := range currentPods.Items { + if p.Status.Phase == corev1.PodRunning { + runningPodName = p.Name + break + } + } + g.Expect(runningPodName).NotTo(BeEmpty(), "a running hypershift-operator pod should exist") + + metrics, err := v2util.GetMetricsFromPod(tc.Context, clientset, mgmtRestConfig, hoNamespace, runningPodName, "operator", 9000) + g.Expect(err).NotTo(HaveOccurred(), "should be able to fetch metrics from hypershift-operator pod") + + g.Expect(metrics).To(HaveKey("hypershift_operator_info"), + "metrics should contain hypershift_operator_info") + + for _, metricName := range []string{ + hcmetrics.SilenceAlertsMetricName, + hcmetrics.LimitedSupportEnabledMetricName, + hcmetrics.ProxyMetricName, + } { + expectMetricHasLabel(g, metrics, metricName, "name", hcName) + } + + for _, metricName := range []string{ + npmetrics.SizeMetricName, + npmetrics.AvailableReplicasMetricName, + } { + expectMetricHasLabel(g, metrics, metricName, "cluster_name", hcName) + } + + if hostedCluster.Spec.Platform.Type == hyperv1.AWSPlatform { + expectMetricHasLabel(g, metrics, hcmetrics.InvalidAwsCredsMetricName, "name", hcName) + } + + if hostedCluster.Spec.Platform.Type == hyperv1.AzurePlatform && azureutil.IsAroHCP() { + family, ok := metrics[hcmetrics.HostedClusterManagedAzureInfoMetricName] + g.Expect(ok).To(BeTrue(), "metric %s should exist on managed Azure", + hcmetrics.HostedClusterManagedAzureInfoMetricName) + g.Expect(family.Metric).NotTo(BeEmpty(), + "metric %s should have at least one time series", + hcmetrics.HostedClusterManagedAzureInfoMetricName) + } + }, 5*time.Minute, 10*time.Second).Should(Succeed()) + }) + }) +} + +func EnsureMetricsForwarderWorkingTest(getTestCtx internal.TestContextGetter) { + When("metrics forwarding is enabled", Label("Informing"), func() { + It("should deploy the metrics pipeline and scrape kube-apiserver metrics end-to-end", func() { + tc := getTestCtx() + hostedCluster := tc.GetHostedCluster() + if e2eutil.IsLessThan(e2eutil.Version422) { + Skip("metrics forwarder requires version >= 4.22") + } + + if hostedCluster.Annotations[hyperv1.EnableMetricsForwarding] != "true" { + Skip("metrics forwarding annotation not set on hosted cluster; skipping verification test") + } + + By("Waiting for management-side metrics deployments") + Eventually(func(g Gomega) { + for _, app := range []string{"endpoint-resolver", "metrics-proxy"} { + podList := &corev1.PodList{} + g.Expect(tc.MgmtClient.List(tc.Context, podList, + crclient.InNamespace(tc.ControlPlaneNamespace), + crclient.MatchingLabels{"app": app}, + )).To(Succeed()) + g.Expect(podList.Items).NotTo(BeEmpty(), "%s pod should exist in the control plane namespace", app) + } + }, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("Waiting for hosted cluster metrics-forwarder deployment") + tc.ValidateHostedClusterClient() + hcClient := tc.GetHostedClusterClient() + hcRestConfig := tc.GetHostedClusterRESTConfig() + Expect(hcRestConfig).NotTo(BeNil(), "hosted cluster REST config should be available") + + hcClientset, err := kubernetes.NewForConfig(hcRestConfig) + Expect(err).NotTo(HaveOccurred(), "should be able to create hosted cluster kubernetes clientset") + + const monitoringNamespace = "openshift-monitoring" + Eventually(func(g Gomega) { + podList := &corev1.PodList{} + g.Expect(hcClient.List(tc.Context, podList, + crclient.InNamespace(monitoringNamespace), + crclient.MatchingLabels{"app": "control-plane-metrics-forwarder"}, + )).To(Succeed()) + g.Expect(podList.Items).NotTo(BeEmpty(), "control-plane-metrics-forwarder pod should exist in hosted cluster") + }, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("Waiting for Prometheus pod in hosted cluster") + const promPodName = "prometheus-k8s-0" + Eventually(func(g Gomega) { + pod := &corev1.Pod{} + g.Expect(hcClient.Get(tc.Context, crclient.ObjectKey{ + Namespace: monitoringNamespace, + Name: promPodName, + }, pod)).To(Succeed()) + g.Expect(pod.Status.Phase).To(Equal(corev1.PodRunning), "prometheus pod should be running") + }, 5*time.Minute, 10*time.Second).Should(Succeed()) + + By("Verifying guest Prometheus is scraping kube-apiserver via the metrics-forwarder") + Eventually(func(g Gomega) { + output, err := v2util.RunCommandInPod(tc.Context, hcClientset, hcRestConfig, + monitoringNamespace, promPodName, "prometheus", + "curl", "-s", "http://localhost:9090/api/v1/targets") + g.Expect(err).NotTo(HaveOccurred(), "should be able to query Prometheus targets API") + g.Expect(output).To(ContainSubstring("control-plane-metrics-forwarder"), + "Prometheus targets should include the metrics-forwarder scrape pool") + g.Expect(output).To(ContainSubstring(`"health":"up"`), + "metrics-forwarder target should be healthy") + }, 10*time.Minute, 15*time.Second).Should(Succeed()) + + By("Querying for actual kube-apiserver metrics scraped via the forwarder") + Eventually(func(g Gomega) { + output, err := v2util.RunCommandInPod(tc.Context, hcClientset, hcRestConfig, + monitoringNamespace, promPodName, "prometheus", + "curl", "-gs", `http://localhost:9090/api/v1/query?query=apiserver_request_total{job="apiserver"}`) + g.Expect(err).NotTo(HaveOccurred(), "should be able to query Prometheus for apiserver_request_total") + g.Expect(output).To(ContainSubstring(`"resultType":"vector"`), + "Prometheus query should return vector results") + g.Expect(output).NotTo(ContainSubstring(`"result":[]`), + "should have apiserver_request_total metrics from kube-apiserver") + }, 5*time.Minute, 15*time.Second).Should(Succeed()) + }) + }) +} + +func EnsureNodeTuningOperatorMetricsEndpointTest(getTestCtx internal.TestContextGetter) { + When("cluster has worker nodes", func() { + It("should have a functional node-tuning-operator metrics endpoint", func() { + tc := getTestCtx() + if e2eutil.IsLessThan(e2eutil.Version422) { + Skip("NTO metrics endpoint test requires version >= 4.22") + } + + svc := &corev1.Service{} + err := tc.MgmtClient.Get(tc.Context, crclient.ObjectKey{ + Name: "node-tuning-operator", + Namespace: tc.ControlPlaneNamespace, + }, svc) + if apierrors.IsNotFound(err) { + Skip("node-tuning-operator service not found in control plane namespace, assuming no workers") + } + Expect(err).NotTo(HaveOccurred(), "failed to get node-tuning-operator service") + + Expect(svc.Spec.Ports).NotTo(BeEmpty(), "node-tuning-operator service should have at least one port") + + hasMetricsPort := false + for _, port := range svc.Spec.Ports { + if port.Name == "metrics" || port.Port == 60000 { + hasMetricsPort = true + break + } + } + Expect(hasMetricsPort).To(BeTrue(), "node-tuning-operator service should expose a metrics port (named 'metrics' or on port 60000)") + + By("Validating ServiceMonitor exists with metrics endpoint") + serviceMonitor := &monitoringv1.ServiceMonitor{} + Expect(tc.MgmtClient.Get(tc.Context, crclient.ObjectKey{ + Name: "node-tuning-operator", + Namespace: tc.ControlPlaneNamespace, + }, serviceMonitor)).To(Succeed(), "node-tuning-operator ServiceMonitor should exist") + + var targetPort string + scheme := "https" + for _, endpoint := range serviceMonitor.Spec.Endpoints { + if endpoint.Path == "/metrics" { + targetPort = endpoint.Port + if targetPort == "" && endpoint.TargetPort != nil { + targetPort = endpoint.TargetPort.String() + } + if endpoint.Scheme != nil { + scheme = string(*endpoint.Scheme) + } + break + } + } + Expect(targetPort).NotTo(BeEmpty(), "ServiceMonitor should have a /metrics endpoint with a target port") + + By("Verifying the HTTPS metrics endpoint returns Prometheus data") + mgmtRestConfig, err := e2eutil.GetConfig() + Expect(err).NotTo(HaveOccurred(), "should be able to load management cluster REST config") + clientset, err := kubernetes.NewForConfig(mgmtRestConfig) + Expect(err).NotTo(HaveOccurred(), "should be able to create kubernetes clientset") + + httpsServiceURL := fmt.Sprintf("%s://node-tuning-operator.%s.svc.cluster.local:%s/metrics", scheme, tc.ControlPlaneNamespace, targetPort) + Eventually(func(g Gomega) { + ntoPods := &corev1.PodList{} + g.Expect(tc.MgmtClient.List(tc.Context, ntoPods, + crclient.InNamespace(tc.ControlPlaneNamespace), + crclient.MatchingLabels{"app": "cluster-node-tuning-operator"}, + )).To(Succeed()) + g.Expect(ntoPods.Items).NotTo(BeEmpty(), "cluster-node-tuning-operator pod should exist") + + var runningPodName string + for _, p := range ntoPods.Items { + if p.Status.Phase == corev1.PodRunning { + runningPodName = p.Name + break + } + } + g.Expect(runningPodName).NotTo(BeEmpty(), "a running cluster-node-tuning-operator pod should exist") + + output, err := v2util.RunCommandInPod(tc.Context, clientset, mgmtRestConfig, + tc.ControlPlaneNamespace, runningPodName, "cluster-node-tuning-operator", + "curl", "-s", "-f", "--max-time", "10", + "--cacert", "/etc/secrets/ca.crt", + "--cert", "/tmp/metrics-client-ca/tls.crt", + "--key", "/tmp/metrics-client-ca/tls.key", + httpsServiceURL) + g.Expect(err).NotTo(HaveOccurred(), "should be able to curl NTO metrics endpoint at %s", httpsServiceURL) + g.Expect(output).NotTo(BeEmpty(), "metrics response should not be empty") + g.Expect(output).To(ContainSubstring("# HELP"), + "metrics response should contain Prometheus format data") + }, 3*time.Minute, 10*time.Second).Should(Succeed()) + }) + }) +} + +var _ = Describe("Hosted Cluster Metrics", Label("hosted-cluster-metrics"), func() { + var testCtx *internal.TestContext + + BeforeEach(func() { + testCtx = internal.GetTestContext() + Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + + testCtx.ValidateHostedCluster() + }) + + RegisterHostedClusterMetricsTests(func() *internal.TestContext { return testCtx }) +}) diff --git a/test/e2e/v2/tests/hosted_cluster_security_test.go b/test/e2e/v2/tests/hosted_cluster_security_test.go new file mode 100644 index 000000000000..1548bbafa436 --- /dev/null +++ b/test/e2e/v2/tests/hosted_cluster_security_test.go @@ -0,0 +1,341 @@ +//go:build e2ev2 + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "fmt" + "net" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + configv1 "github.com/openshift/api/config/v1" + + hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1" + hccokasvap "github.com/openshift/hypershift/control-plane-operator/hostedclusterconfigoperator/controllers/resources/kas" + suppconfig "github.com/openshift/hypershift/support/config" + "github.com/openshift/hypershift/support/netutil" + e2eutil "github.com/openshift/hypershift/test/e2e/util" + "github.com/openshift/hypershift/test/e2e/v2/internal" + v2util "github.com/openshift/hypershift/test/e2e/v2/util" + + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + + crclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func RegisterHostedClusterSecurityTests(getTestCtx internal.TestContextGetter) { + EnsureGuestWebhooksValidatedTest(getTestCtx) + EnsureAdmissionPoliciesTest(getTestCtx) + EnsureNetworkPoliciesTest(getTestCtx) +} + +func EnsureGuestWebhooksValidatedTest(getTestCtx internal.TestContextGetter) { + When("a webhook targeting a control plane service is created in the hosted cluster", func() { + It("should be automatically deleted", func() { + tc := getTestCtx() + tc.ValidateHostedClusterClient() + hcClient := tc.GetHostedClusterClient() + + sideEffectsNone := admissionregistrationv1.SideEffectClassNone + webhookConf := &admissionregistrationv1.ValidatingWebhookConfiguration{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-malicious-webhook", + Annotations: map[string]string{ + "service.beta.openshift.io/inject-cabundle": "true", + }, + }, + Webhooks: []admissionregistrationv1.ValidatingWebhook{{ + AdmissionReviewVersions: []string{"v1"}, + Name: "etcd-client.example.com", + ClientConfig: admissionregistrationv1.WebhookClientConfig{ + URL: ptr.To("https://etcd-client:2379"), + }, + Rules: []admissionregistrationv1.RuleWithOperations{{ + Operations: []admissionregistrationv1.OperationType{admissionregistrationv1.Create}, + Rule: admissionregistrationv1.Rule{ + APIGroups: []string{""}, + APIVersions: []string{"v1"}, + Resources: []string{"pods"}, + }, + }}, + SideEffects: &sideEffectsNone, + }}, + } + + Expect(hcClient.Create(tc.Context, webhookConf)).To(Succeed()) + DeferCleanup(func() { + err := hcClient.Delete(tc.Context, webhookConf) + if err != nil && !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred(), "failed to cleanup test-malicious-webhook — this webhook may disrupt the hosted cluster") + } + }) + + Eventually(func(g Gomega) { + existing := &admissionregistrationv1.ValidatingWebhookConfiguration{} + err := hcClient.Get(tc.Context, crclient.ObjectKeyFromObject(webhookConf), existing) + g.Expect(apierrors.IsNotFound(err)).To(BeTrue(), "webhook should have been deleted by HCCO") + }, time.Minute, 5*time.Second).Should(Succeed()) + }) + }) +} + +func EnsureAdmissionPoliciesTest(getTestCtx internal.TestContextGetter) { + When("checking admission policies on a public hosted cluster", Ordered, func() { + var tc *internal.TestContext + var hcClient crclient.Client + var hostedCluster *hyperv1.HostedCluster + + BeforeAll(func() { + tc = getTestCtx() + if e2eutil.IsLessThan(e2eutil.Version418) { + Skip("Admission policies require version >= 4.18") + } + hostedCluster = tc.GetHostedCluster() + if !netutil.IsPublicHC(hostedCluster) { + Skip("admission policies test requires a public hosted cluster") + } + tc.ValidateHostedClusterClient() + hcClient = tc.GetHostedClusterClient() + }) + + It("should find all required ValidatingAdmissionPolicies", func() { + Eventually(func(g Gomega) { + vapList := &admissionregistrationv1.ValidatingAdmissionPolicyList{} + g.Expect(hcClient.List(tc.Context, vapList)).To(Succeed()) + g.Expect(vapList.Items).NotTo(BeEmpty(), "expected ValidatingAdmissionPolicies to be present") + + requiredVAPs := []string{ + hccokasvap.AdmissionPolicyNameConfig, + hccokasvap.AdmissionPolicyNameMirror, + hccokasvap.AdmissionPolicyNameICSP, + hccokasvap.AdmissionPolicyNameInfra, + hccokasvap.AdmissionPolicyNameNTOMirroredConfigs, + } + vapNames := make([]string, 0, len(vapList.Items)) + for _, vap := range vapList.Items { + vapNames = append(vapNames, vap.Name) + } + for _, required := range requiredVAPs { + g.Expect(vapNames).To(ContainElement(required), + "required ValidatingAdmissionPolicy %s not found", required) + } + }, 5*time.Minute, 10*time.Second).Should(Succeed()) + }) + + It("should deny unauthorized config changes via VAPs", func() { + Eventually(func(g Gomega) { + apiServer := &configv1.APIServer{} + g.Expect(hcClient.Get(tc.Context, crclient.ObjectKey{Name: "cluster"}, apiServer)).To(Succeed()) + apiServerCopy := apiServer.DeepCopy() + if apiServerCopy.Spec.Audit.Profile == configv1.AllRequestBodiesAuditProfileType { + apiServerCopy.Spec.Audit.Profile = configv1.DefaultAuditProfileType + } else { + apiServerCopy.Spec.Audit.Profile = configv1.AllRequestBodiesAuditProfileType + } + err := hcClient.Update(tc.Context, apiServerCopy) + g.Expect(err).To(HaveOccurred(), "VAP should block audit profile modification") + g.Expect(err.Error()).To(ContainSubstring("ValidatingAdmissionPolicy"), + "rejection should be from a ValidatingAdmissionPolicy, got: %v", err) + }, time.Minute, 5*time.Second).Should(Succeed()) + }) + + It("should allow status modifications via VAPs", func() { + network := &configv1.Network{} + Expect(hcClient.Get(tc.Context, crclient.ObjectKey{Name: "cluster"}, network)).To(Succeed()) + originalMTU := network.Status.ClusterNetworkMTU + DeferCleanup(func() { + Eventually(func(g Gomega) { + net := &configv1.Network{} + g.Expect(hcClient.Get(tc.Context, crclient.ObjectKey{Name: "cluster"}, net)).To(Succeed(), + "cleanup: failed to get Network resource for MTU restoration") + net.Status.ClusterNetworkMTU = originalMTU + g.Expect(hcClient.Update(tc.Context, net)).To(Succeed(), + "cleanup: failed to restore original ClusterNetworkMTU") + }, time.Minute, 5*time.Second).Should(Succeed()) + }) + Eventually(func(g Gomega) { + networkCopy := &configv1.Network{} + g.Expect(hcClient.Get(tc.Context, crclient.ObjectKey{Name: "cluster"}, networkCopy)).To(Succeed()) + networkCopy.Status.ClusterNetworkMTU = 9180 + g.Expect(hcClient.Update(tc.Context, networkCopy)).To(Succeed(), + "VAP should allow status modifications") + }, time.Minute, 5*time.Second).Should(Succeed()) + }) + + It("should allow OperatorHub config changes with guest OLM placement", func() { + if hostedCluster.Spec.OLMCatalogPlacement != hyperv1.GuestOLMCatalogPlacement { + Skip("OperatorHub test requires guest OLM catalog placement") + } + operatorHub := &configv1.OperatorHub{} + Expect(hcClient.Get(tc.Context, crclient.ObjectKey{Name: "cluster"}, operatorHub)).To(Succeed()) + originalDisableAll := operatorHub.Spec.DisableAllDefaultSources + DeferCleanup(func() { + Eventually(func(g Gomega) { + oh := &configv1.OperatorHub{} + g.Expect(hcClient.Get(tc.Context, crclient.ObjectKey{Name: "cluster"}, oh)).To(Succeed(), + "cleanup: failed to get OperatorHub resource") + oh.Spec.DisableAllDefaultSources = originalDisableAll + g.Expect(hcClient.Update(tc.Context, oh)).To(Succeed(), + "cleanup: failed to restore OperatorHub configuration") + }, time.Minute, 5*time.Second).Should(Succeed()) + }) + Eventually(func(g Gomega) { + oh := &configv1.OperatorHub{} + g.Expect(hcClient.Get(tc.Context, crclient.ObjectKey{Name: "cluster"}, oh)).To(Succeed()) + oh.Spec.DisableAllDefaultSources = !originalDisableAll + g.Expect(hcClient.Update(tc.Context, oh)).To(Succeed(), + "VAP should allow OperatorHub configuration changes when OLM uses guest placement") + }, time.Minute, 5*time.Second).Should(Succeed()) + }) + }) +} + +func EnsureNetworkPoliciesTest(getTestCtx internal.TestContextGetter) { + When("checking network policies on an AWS hosted cluster", func() { + BeforeEach(func() { + tc := getTestCtx() + hostedCluster := tc.GetHostedCluster() + if hostedCluster.Spec.Platform.Type != hyperv1.AWSPlatform { + Skip("network policies test is only for AWS platform") + } + }) + + It("should find management KAS access labels on expected components", func() { + tc := getTestCtx() + + podList := &corev1.PodList{} + Expect(tc.MgmtClient.List(tc.Context, podList, + crclient.InNamespace(tc.ControlPlaneNamespace), + crclient.MatchingLabels{suppconfig.NeedManagementKASAccessLabel: "true"}, + )).To(Succeed()) + Expect(podList.Items).NotTo(BeEmpty(), + "expected pods with %s label in namespace %s", + suppconfig.NeedManagementKASAccessLabel, tc.ControlPlaneNamespace) + }) + + It("should block egress traffic from non-privileged pods to the management KAS", func() { + tc := getTestCtx() + + mgmtRestConfig, err := e2eutil.GetConfig() + Expect(err).NotTo(HaveOccurred(), "should be able to load management cluster REST config") + clientset, err := kubernetes.NewForConfig(mgmtRestConfig) + Expect(err).NotTo(HaveOccurred(), "should be able to create kubernetes clientset") + + endpoints := &corev1.Endpoints{} + Expect(tc.MgmtClient.Get(tc.Context, crclient.ObjectKey{Name: "kubernetes", Namespace: "default"}, endpoints)).To(Succeed()) + var kasAddress string + for _, subset := range endpoints.Subsets { + if len(subset.Addresses) > 0 && len(subset.Ports) > 0 { + kasAddress = "https://" + net.JoinHostPort(subset.Addresses[0].IP, fmt.Sprintf("%d", subset.Ports[0].Port)) + break + } + } + Expect(kasAddress).NotTo(BeEmpty(), "should resolve management KAS endpoint address") + + Eventually(func(g Gomega) { + cvoPods := &corev1.PodList{} + g.Expect(tc.MgmtClient.List(tc.Context, cvoPods, + crclient.InNamespace(tc.ControlPlaneNamespace), + crclient.MatchingLabels{"app": "cluster-version-operator"}, + )).To(Succeed()) + g.Expect(cvoPods.Items).NotTo(BeEmpty(), "cluster-version-operator pod should exist") + + var runningPodName string + for _, p := range cvoPods.Items { + if p.Status.Phase == corev1.PodRunning { + runningPodName = p.Name + break + } + } + g.Expect(runningPodName).NotTo(BeEmpty(), "a running cluster-version-operator pod should exist") + + _, err := v2util.RunCommandInPod(tc.Context, clientset, mgmtRestConfig, + tc.ControlPlaneNamespace, runningPodName, "cluster-version-operator", + "curl", "--connect-timeout", "2", "-Iks", kasAddress) + g.Expect(err).To(HaveOccurred(), + "cluster-version-operator should not be able to reach the management KAS at %s", kasAddress) + g.Expect(err.Error()).To(SatisfyAny( + ContainSubstring("exit code"), + ContainSubstring("Connection timed out"), + ContainSubstring("Connection refused"), + ), "failure should be a curl connection error, not an exec/setup error; got: %v", err) + }, 1*time.Minute, 10*time.Second).Should(Succeed()) + + hostedCluster := tc.GetHostedCluster() + if hostedCluster.Spec.Platform.Type == hyperv1.AWSPlatform && + hostedCluster.Spec.Platform.AWS != nil && + hostedCluster.Spec.Platform.AWS.EndpointAccess != hyperv1.Private { + By("Verifying private-router cannot reach management KAS") + routerPods := &corev1.PodList{} + Expect(tc.MgmtClient.List(tc.Context, routerPods, + crclient.InNamespace(tc.ControlPlaneNamespace), + crclient.MatchingLabels{"app": "private-router"}, + )).To(Succeed()) + if len(routerPods.Items) > 0 { + Eventually(func(g Gomega) { + currentRouterPods := &corev1.PodList{} + g.Expect(tc.MgmtClient.List(tc.Context, currentRouterPods, + crclient.InNamespace(tc.ControlPlaneNamespace), + crclient.MatchingLabels{"app": "private-router"}, + )).To(Succeed()) + g.Expect(currentRouterPods.Items).NotTo(BeEmpty(), "private-router pod should exist") + + var runningPodName string + for _, p := range currentRouterPods.Items { + if p.Status.Phase == corev1.PodRunning { + runningPodName = p.Name + break + } + } + g.Expect(runningPodName).NotTo(BeEmpty(), "a running private-router pod should exist") + + _, err := v2util.RunCommandInPod(tc.Context, clientset, mgmtRestConfig, + tc.ControlPlaneNamespace, runningPodName, "private-router", + "curl", "--connect-timeout", "2", "-Iks", kasAddress) + g.Expect(err).To(HaveOccurred(), + "private-router should not be able to reach the management KAS at %s", kasAddress) + g.Expect(err.Error()).To(SatisfyAny( + ContainSubstring("exit code"), + ContainSubstring("Connection timed out"), + ContainSubstring("Connection refused"), + ), "failure should be a curl connection error, not an exec/setup error; got: %v", err) + }, 1*time.Minute, 10*time.Second).Should(Succeed()) + } + } + }) + }) +} + +var _ = Describe("Hosted Cluster Security", Label("hosted-cluster-security"), func() { + var testCtx *internal.TestContext + + BeforeEach(func() { + testCtx = internal.GetTestContext() + Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + + testCtx.ValidateHostedCluster() + }) + + RegisterHostedClusterSecurityTests(func() *internal.TestContext { return testCtx }) +}) diff --git a/test/e2e/v2/tests/nodepool_autoscaling_test.go b/test/e2e/v2/tests/nodepool_autoscaling_test.go index 5a2766d8a546..942aeffda9ae 100644 --- a/test/e2e/v2/tests/nodepool_autoscaling_test.go +++ b/test/e2e/v2/tests/nodepool_autoscaling_test.go @@ -34,8 +34,8 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/apimachinery/pkg/labels" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/utils/ptr" crclient "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -44,14 +44,10 @@ import ( func AutoscalingScaleUpDownTest(getTestCtx internal.TestContextGetter) { It("should scale up when workload increases and scale down when workload decreases", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") - ctx := testCtx.Context // Find the default NodePool to copy platform config @@ -109,16 +105,12 @@ func AutoscalingScaleUpDownTest(getTestCtx internal.TestContextGetter) { func AutoscalingBalancingTest(getTestCtx internal.TestContextGetter) { It("should balance pods across multiple autoscaling NodePools", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() e2eutil.GinkgoAtLeast(e2eutil.Version420) hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") - ctx := testCtx.Context cpNamespace := testCtx.ControlPlaneNamespace @@ -236,8 +228,8 @@ func AutoscalingBalancingTest(getTestCtx internal.TestContextGetter) { return false, nil } return autoscalingNP1.Status.Replicas >= 1 && autoscalingNP2.Status.Replicas >= 1, nil - }).WithTimeout(30 * time.Minute). - WithPolling(30 * time.Second). + }).WithTimeout(30*time.Minute). + WithPolling(30*time.Second). Should(BeTrue(), "NodePools should have balanced distribution") }) } diff --git a/test/e2e/v2/tests/nodepool_lifecycle_test.go b/test/e2e/v2/tests/nodepool_lifecycle_test.go index 22c0a754378e..97ba0b48a036 100644 --- a/test/e2e/v2/tests/nodepool_lifecycle_test.go +++ b/test/e2e/v2/tests/nodepool_lifecycle_test.go @@ -39,6 +39,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -84,17 +85,14 @@ var _ = Describe("NodePool Lifecycle", Label("lifecycle", "nodepool-lifecycle"), func NodePoolMachineconfigRolloutTest(getTestCtx internal.TestContextGetter) { It("should roll out a MachineConfig change via Replace upgrade strategy", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - if hc.Spec.Platform.Type == hyperv1.KubevirtPlatform { Skip("test is skipped for KubeVirt platform until https://issues.redhat.com/browse/CNV-38196 is addressed") } guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") ctx := testCtx.Context @@ -182,11 +180,9 @@ func NodePoolMachineconfigRolloutTest(getTestCtx internal.TestContextGetter) { func NodePoolNTORolloutTest(getTestCtx internal.TestContextGetter) { It("should roll out an NTO Tuned config change via Replace upgrade strategy", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - if hc.Spec.Platform.Type == hyperv1.KubevirtPlatform { Skip("test is skipped for KubeVirt platform until https://issues.redhat.com/browse/CNV-38196 is addressed") } @@ -195,7 +191,6 @@ func NodePoolNTORolloutTest(getTestCtx internal.TestContextGetter) { } guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") ctx := testCtx.Context @@ -252,11 +247,9 @@ func NodePoolNTORolloutTest(getTestCtx internal.TestContextGetter) { func NodePoolNTOInPlaceTest(getTestCtx internal.TestContextGetter) { It("should roll out an NTO Tuned config change via InPlace upgrade strategy", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - if hc.Spec.Platform.Type == hyperv1.KubevirtPlatform { Skip("test is skipped for KubeVirt platform until https://issues.redhat.com/browse/CNV-38196 is addressed") } @@ -265,7 +258,6 @@ func NodePoolNTOInPlaceTest(getTestCtx internal.TestContextGetter) { } guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") ctx := testCtx.Context @@ -317,13 +309,10 @@ func NodePoolNTOInPlaceTest(getTestCtx internal.TestContextGetter) { func NodePoolReplaceUpgradeTest(getTestCtx internal.TestContextGetter) { It("should upgrade a NodePool from previous to latest release via Replace strategy", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") previousImage := internal.GetEnvVarValue("E2E_PREVIOUS_RELEASE_IMAGE") latestImage := internal.GetEnvVarValue("E2E_LATEST_RELEASE_IMAGE") @@ -405,13 +394,10 @@ func NodePoolReplaceUpgradeTest(getTestCtx internal.TestContextGetter) { func NodePoolInPlaceUpgradeTest(getTestCtx internal.TestContextGetter) { It("should upgrade a NodePool from previous to latest release via InPlace strategy", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") previousImage := internal.GetEnvVarValue("E2E_PREVIOUS_RELEASE_IMAGE") latestImage := internal.GetEnvVarValue("E2E_LATEST_RELEASE_IMAGE") @@ -487,18 +473,15 @@ func NodePoolInPlaceUpgradeTest(getTestCtx internal.TestContextGetter) { func NodePoolRollingUpgradeTest(getTestCtx internal.TestContextGetter) { It("should perform a rolling upgrade when instance type or VM size changes", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - platform := hc.Spec.Platform.Type if platform != hyperv1.AWSPlatform && platform != hyperv1.AzurePlatform { Skip("rolling upgrade test only supported on AWS and Azure platforms") } guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") ctx := testCtx.Context @@ -586,10 +569,9 @@ func NodePoolRollingUpgradeTest(getTestCtx internal.TestContextGetter) { func NodePoolPrevReleaseN1Test(getTestCtx internal.TestContextGetter) { It("should create a NodePool at N-1 release and have ready nodes", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") n1Image := internal.GetEnvVarValue("E2E_N1_RELEASE_IMAGE") if n1Image == "" { @@ -597,7 +579,6 @@ func NodePoolPrevReleaseN1Test(getTestCtx internal.TestContextGetter) { } guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") ctx := testCtx.Context @@ -625,10 +606,9 @@ func NodePoolPrevReleaseN1Test(getTestCtx internal.TestContextGetter) { func NodePoolPrevReleaseN2Test(getTestCtx internal.TestContextGetter) { It("should create a NodePool at N-2 release and have ready nodes", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") n2Image := internal.GetEnvVarValue("E2E_N2_RELEASE_IMAGE") if n2Image == "" { @@ -636,7 +616,6 @@ func NodePoolPrevReleaseN2Test(getTestCtx internal.TestContextGetter) { } guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") ctx := testCtx.Context @@ -666,17 +645,14 @@ func NodePoolPrevReleaseN2Test(getTestCtx internal.TestContextGetter) { func NodePoolMirrorConfigsTest(getTestCtx internal.TestContextGetter) { It("should mirror KubeletConfig to the hosted cluster and clean up on removal", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - if e2eutil.IsLessThan(e2eutil.Version418) { Skip("mirror configs test only applicable for 4.18+") } guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") ctx := testCtx.Context @@ -703,9 +679,11 @@ func NodePoolMirrorConfigsTest(getTestCtx internal.TestContextGetter) { Data: map[string]string{configKey: kubeletConfig1YAML}, } Expect(testCtx.MgmtClient.Create(ctx, kcConfigMap)).To(Succeed(), "failed to create KubeletConfig ConfigMap") - defer func() { - _ = testCtx.MgmtClient.Delete(ctx, kcConfigMap) - }() + DeferCleanup(func() { + if err := testCtx.MgmtClient.Delete(ctx, kcConfigMap); err != nil && !apierrors.IsNotFound(err) { + GinkgoWriter.Printf("Warning: failed to cleanup KubeletConfig ConfigMap %s: %v\n", kcConfigMap.Name, err) + } + }) original := np.DeepCopy() np.Spec.Config = append(np.Spec.Config, corev1.LocalObjectReference{Name: kcConfigMap.Name}) @@ -796,15 +774,12 @@ func NodePoolMirrorConfigsTest(getTestCtx internal.TestContextGetter) { func NodePoolTrustBundleTest(getTestCtx internal.TestContextGetter) { It("should propagate and remove additional trust bundle to/from the hosted cluster", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() e2eutil.GinkgoAtLeast(e2eutil.Version418) hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") ctx := testCtx.Context @@ -985,17 +960,14 @@ func NodePoolTrustBundleTest(getTestCtx internal.TestContextGetter) { func NodePoolNTOPerformanceProfileTest(getTestCtx internal.TestContextGetter) { It("should create and manage NTO PerformanceProfile via NodePool TuningConfig", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - if hc.Spec.Platform.Type == hyperv1.OpenStackPlatform { Skip("test is skipped for OpenStack platform until https://issues.redhat.com/browse/OSASINFRA-3566 is addressed") } guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") ctx := testCtx.Context @@ -1022,9 +994,11 @@ func NodePoolNTOPerformanceProfileTest(getTestCtx internal.TestContextGetter) { Data: map[string]string{tuningConfigKey: performanceProfileYAML}, } Expect(testCtx.MgmtClient.Create(ctx, ppConfigMap)).To(Succeed(), "failed to create PerformanceProfile ConfigMap") - defer func() { - _ = testCtx.MgmtClient.Delete(ctx, ppConfigMap) - }() + DeferCleanup(func() { + if err := testCtx.MgmtClient.Delete(ctx, ppConfigMap); err != nil && !apierrors.IsNotFound(err) { + GinkgoWriter.Printf("Warning: failed to cleanup PerformanceProfile ConfigMap %s: %v\n", ppConfigMap.Name, err) + } + }) original := np.DeepCopy() np.Spec.TuningConfig = append(np.Spec.TuningConfig, corev1.LocalObjectReference{Name: ppConfigMap.Name}) @@ -1146,18 +1120,15 @@ func NodePoolAutoRepairTest(getTestCtx internal.TestContextGetter) { Skip("auto-repair instance termination not yet implemented for v2 framework") testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - platform := hc.Spec.Platform.Type if platform != hyperv1.AWSPlatform && platform != hyperv1.AzurePlatform { Skip("auto-repair test only supported on AWS and Azure platforms") } guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") ctx := testCtx.Context @@ -1190,11 +1161,9 @@ func NodePoolAutoRepairTest(getTestCtx internal.TestContextGetter) { func NodePoolDiskEncryptionTest(getTestCtx internal.TestContextGetter) { It("should create a NodePool with Azure DiskEncryptionSet and verify it is applied", func() { testCtx := getTestCtx() - Expect(testCtx).NotTo(BeNil(), "test context should be set up in BeforeSuite") + testCtx.ValidateHostedClusterClient() hc := testCtx.GetHostedCluster() - Expect(hc).NotTo(BeNil(), "hosted cluster should be available") - if hc.Spec.Platform.Type != hyperv1.AzurePlatform { Skip("disk encryption test only supported on Azure platform") } @@ -1205,7 +1174,6 @@ func NodePoolDiskEncryptionTest(getTestCtx internal.TestContextGetter) { } guestClient := testCtx.GetHostedClusterClient() - Expect(guestClient).NotTo(BeNil(), "hosted cluster client should be available") ctx := testCtx.Context diff --git a/test/e2e/v2/util/pod_exec.go b/test/e2e/v2/util/pod_exec.go new file mode 100644 index 000000000000..d5a0e02f1cf5 --- /dev/null +++ b/test/e2e/v2/util/pod_exec.go @@ -0,0 +1,85 @@ +//go:build e2ev2 + +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package util + +import ( + "bytes" + "context" + "fmt" + "net/http" + "strings" + + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "github.com/prometheus/common/model" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + k8sscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" +) + +// RunCommandInPod returns an error rather than failing directly, allowing callers inside Eventually() to retry. +func RunCommandInPod(ctx context.Context, clientset kubernetes.Interface, restConfig *rest.Config, namespace, podName, containerName string, command ...string) (string, error) { + req := clientset.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(podName). + Namespace(namespace). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: containerName, + Command: command, + Stdout: true, + Stderr: true, + }, k8sscheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(restConfig, http.MethodPost, req.URL()) + if err != nil { + return "", fmt.Errorf("failed to create executor for pod %s/%s: %w", namespace, podName, err) + } + + var stdout, stderr bytes.Buffer + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdout: &stdout, + Stderr: &stderr, + }) + if err != nil { + return "", fmt.Errorf("command failed in pod %s/%s container %s: %w (stderr: %s)", namespace, podName, containerName, err, stderr.String()) + } + + return stdout.String(), nil +} + +// GetMetricsFromPod returns an error rather than failing directly, allowing callers inside Eventually() to retry. +func GetMetricsFromPod(ctx context.Context, clientset kubernetes.Interface, restConfig *rest.Config, namespace, podName, containerName string, port int) (map[string]*dto.MetricFamily, error) { + url := fmt.Sprintf("http://localhost:%d/metrics", port) + output, err := RunCommandInPod(ctx, clientset, restConfig, namespace, podName, containerName, "curl", "-sS", "-f", url) + if err != nil { + return nil, err + } + if len(output) == 0 { + return nil, fmt.Errorf("no metrics returned from pod %s/%s port %d", namespace, podName, port) + } + + parser := expfmt.NewTextParser(model.UTF8Validation) + families, err := parser.TextToMetricFamilies(strings.NewReader(output)) + if err != nil { + return nil, fmt.Errorf("failed to parse metrics from pod %s/%s port %d: %w", namespace, podName, port, err) + } + return families, nil +} diff --git a/test/integration/framework/hosted-cluster.go b/test/integration/framework/hosted-cluster.go index d6b50d153ebc..f9da27a5ec5b 100644 --- a/test/integration/framework/hosted-cluster.go +++ b/test/integration/framework/hosted-cluster.go @@ -106,7 +106,12 @@ func InstallHostedCluster(ctx context.Context, logger logr.Logger, opts *Options pullSpec := opts.ReleaseImage if pullSpec == "" { - resp, err := http.Get(fmt.Sprintf(`https://multi.ocp.releases.ci.openshift.org/api/v1/releasestream/%s-0.nightly-multi/latest`, supportedversion.LatestSupportedVersion.String())) + releaseURL := fmt.Sprintf(`https://multi.ocp.releases.ci.openshift.org/api/v1/releasestream/%s-0.nightly-multi/latest`, supportedversion.LatestSupportedVersion.String()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, releaseURL, nil) + if err != nil { + return CleanupSentinel, fmt.Errorf("couldn't create release image request: %w", err) + } + resp, err := http.DefaultClient.Do(req) if err != nil { return CleanupSentinel, fmt.Errorf("couldn't fetch latest release image: %w", err) } @@ -175,7 +180,7 @@ func InstallHostedCluster(ctx context.Context, logger logr.Logger, opts *Options if SkippedCleanupSteps().HasAny("all", "hosted-clusters") { return nil } - logger.Info("dumping hosted hosted cluster assets") + logger.Info("dumping hosted cluster assets") dumpLogPath := filepath.Join("install", "assets.dump.yaml") dumpCmd := exec.CommandContext(ctx, opts.OCPath, "get", "--ignore-not-found", "--show-managed-fields", "-f", filepath.Join(opts.ArtifactDir, yamlPath), "--kubeconfig", opts.Kubeconfig, diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go index 7a499c864226..74b50f24ad38 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/hostedcluster_types.go @@ -527,7 +527,6 @@ type Capabilities struct { // +kubebuilder:validation:XValidation:rule=`!self.services.exists(s, s.service == 'APIServer' && has(s.servicePublishingStrategy.loadBalancer) && s.servicePublishingStrategy.loadBalancer.hostname != "" && has(self.configuration) && has(self.configuration.apiServer) && has(self.configuration.apiServer.servingCerts) && has(self.configuration.apiServer.servingCerts.namedCertificates) && self.configuration.apiServer.servingCerts.namedCertificates.exists(cert, has(cert.names) && cert.names.exists(n, n == s.servicePublishingStrategy.loadBalancer.hostname)))`, message="APIServer loadBalancer hostname cannot be in ClusterConfiguration.apiserver.servingCerts.namedCertificates[]" // +kubebuilder:validation:XValidation:rule="!has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.disableMultiNetwork) || !self.operatorConfiguration.clusterNetworkOperator.disableMultiNetwork || self.networking.networkType == 'Other'",message="disableMultiNetwork can only be set to true when networkType is 'Other'" // +kubebuilder:validation:XValidation:rule="self.networking.networkType == 'OVNKubernetes' || !has(self.operatorConfiguration) || !has(self.operatorConfiguration.clusterNetworkOperator) || !has(self.operatorConfiguration.clusterNetworkOperator.ovnKubernetesConfig)", message="ovnKubernetesConfig is forbidden when networkType is not OVNKubernetes" -// +kubebuilder:validation:XValidation:rule=`self.platform.type != "Azure" || self.dns.baseDomain == "" || !self.services.exists(s, (has(s.servicePublishingStrategy.route) && has(s.servicePublishingStrategy.route.hostname) && s.servicePublishingStrategy.route.hostname.contains('.') && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.route.hostname.substring(s.servicePublishingStrategy.route.hostname.indexOf('.') + 1))) || (has(s.servicePublishingStrategy.loadBalancer) && has(s.servicePublishingStrategy.loadBalancer.hostname) && s.servicePublishingStrategy.loadBalancer.hostname.contains('.') && ('.' + self.dns.baseDomain).endsWith('.' + s.servicePublishingStrategy.loadBalancer.hostname.substring(s.servicePublishingStrategy.loadBalancer.hostname.indexOf('.') + 1))))`,message="Azure service hostname domain must not overlap with the cluster base domain. An Azure Private DNS zone matching or containing the base domain would shadow *.apps DNS resolution." type HostedClusterSpec struct { // release specifies the desired OCP release payload for all the hosted cluster components. // This includes those components running management side like the Kube API Server and the CVO but also the operands which land in the hosted cluster data plane like the ingress controller, ovn agents, etc. diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/operator.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/operator.go index ee74790f5ba0..c14858d3fa14 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/operator.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/operator.go @@ -4,6 +4,20 @@ import ( operatorv1 "github.com/openshift/api/operator/v1" ) +const ( + // KubevirtDefaultV6InternalJoinSubnet is the default IPv6 OVN join subnet + // for KubeVirt hosted clusters. The upstream OVN-Kubernetes default is fd98::/64, + // but KubeVirt guests use fd99::/64 to avoid collisions with the management + // cluster's join subnet when both run OVN-Kubernetes. + KubevirtDefaultV6InternalJoinSubnet = "fd99::/64" + + // KubevirtDefaultV4InternalSubnet is the default IPv4 OVN internal subnet + // for KubeVirt hosted clusters. The upstream OVN-Kubernetes default gateway + // router LRP CIDR is 100.64.0.0/16 and the default UDNs is 100.65.0.0/16. + // KubeVirt guests use 100.66.0.0/16 to avoid collisions with the management cluster. + KubevirtDefaultV4InternalSubnet = "100.66.0.0/16" +) + // +kubebuilder:validation:Enum="";Normal;Debug;Trace;TraceAll type LogLevel string @@ -38,6 +52,7 @@ type ClusterVersionOperatorSpec struct { OperatorLogLevel LogLevel `json:"operatorLogLevel,omitempty"` } +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ovnKubernetesConfig) || has(self.ovnKubernetesConfig)", message="ovnKubernetesConfig is immutable once set and cannot be removed" type ClusterNetworkOperatorSpec struct { // disableMultiNetwork when set to true disables the Multus CNI plugin and related components // in the hosted cluster. This prevents the installation of multus daemon sets in the @@ -62,7 +77,11 @@ type ClusterNetworkOperatorSpec struct { // OVNKubernetesConfig contains OVN-Kubernetes specific configuration options. // https://github.com/openshift/api/blob/6d3c4e25a8d3aeb57ad61649d80c38cbd27d1cc8/operator/v1/types_network.go#L400-L471 // +kubebuilder:validation:XValidation:rule="!has(self.ipv4) || !has(self.ipv4.internalJoinSubnet) || !has(self.ipv4.internalTransitSwitchSubnet) || self.ipv4.internalJoinSubnet != self.ipv4.internalTransitSwitchSubnet", message="internalJoinSubnet and internalTransitSwitchSubnet must not be the same" +// +kubebuilder:validation:XValidation:rule="!has(self.ipv6) || !has(self.ipv6.internalJoinSubnet) || !has(self.ipv6.internalTransitSwitchSubnet) || self.ipv6.internalJoinSubnet != self.ipv6.internalTransitSwitchSubnet", message="ipv6 internalJoinSubnet and internalTransitSwitchSubnet must not be the same" // +kubebuilder:validation:XValidation:rule="!has(oldSelf.mtu) || has(self.mtu)",message="mtu is immutable once set and cannot be removed" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipv6) || has(self.ipv6)", message="ipv6 is immutable once set and cannot be removed" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalJoinSubnet) || (has(self.ipv6) && has(self.ipv6.internalJoinSubnet))", message="ipv6.internalJoinSubnet cannot be removed once set" +// +kubebuilder:validation:XValidation:rule="!has(oldSelf.ipv6) || !has(oldSelf.ipv6.internalTransitSwitchSubnet) || (has(self.ipv6) && has(self.ipv6.internalTransitSwitchSubnet))", message="ipv6.internalTransitSwitchSubnet cannot be removed once set" // +kubebuilder:validation:MinProperties=1 type OVNKubernetesConfig struct { // ipv4 allows users to configure IP settings for IPv4 connections. When omitted, @@ -71,6 +90,15 @@ type OVNKubernetesConfig struct { // +optional IPv4 *OVNIPv4Config `json:"ipv4,omitempty"` + // ipv6 allows users to configure IP settings for IPv6 connections. When omitted, + // this means no opinions and the default configuration is used. Check individual + // fields within ipv6 for details of default values. + // For KubeVirt hosted clusters using dual-stack networking, it is recommended to + // set ipv6.internalJoinSubnet to a value different from the management cluster's + // join subnet (default fd98::/64) to avoid IPv6 routing conflicts. + // +optional + IPv6 OVNIPv6Config `json:"ipv6,omitzero,omitempty"` + // mtu is the MTU to use for the tunnel interface on hosted cluster nodes. // This must be 100 bytes smaller than the uplink MTU. // When unset, the cluster-network-operator will determine the MTU automatically @@ -126,6 +154,52 @@ type OVNIPv4Config struct { InternalJoinSubnet string `json:"internalJoinSubnet,omitempty"` } +// OVNIPv6Config contains IPv6-specific configuration options for OVN-Kubernetes. +// https://github.com/openshift/api/blob/6d3c4e25a8d3aeb57ad61649d80c38cbd27d1cc8/operator/v1/types_network.go#L541-L570 +// +kubebuilder:validation:MinProperties=1 +type OVNIPv6Config struct { + // internalTransitSwitchSubnet is a v6 subnet in IPv6 CIDR format used internally + // by OVN-Kubernetes for the distributed transit switch in the OVN Interconnect + // architecture that connects the cluster routers on each node together to enable + // east west traffic. The subnet chosen should not overlap with other networks + // specified for OVN-Kubernetes as well as other networks used on the host. + // When omitted, this means no opinion and the platform is left to choose a reasonable + // default which is subject to change over time. + // The current default subnet is fd97::/64. + // The subnet must be large enough to accommodate one IP per node in your cluster. + // The value must be a valid IPv6 CIDR (e.g. fd97::/64). IPv4 addresses, + // IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + // The prefix length must be in the range /0 to /125 inclusive. + // This field is immutable once set. + // +kubebuilder:validation:MaxLength=48 + // +kubebuilder:validation:MinLength=3 + // +kubebuilder:validation:XValidation:rule="isCIDR(self) && cidr(self).ip().family() == 6", message="Subnet must be in valid IPv6 CIDR format (e.g., fd97::/64)" + // +kubebuilder:validation:XValidation:rule="isCIDR(self) && cidr(self).prefixLength() <= 125", message="subnet must be in the range /0 to /125 inclusive" + // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="internalTransitSwitchSubnet is immutable" + // +optional + InternalTransitSwitchSubnet string `json:"internalTransitSwitchSubnet,omitempty"` + // internalJoinSubnet is a v6 subnet used internally by ovn-kubernetes in case the + // default one is being already used by something else. It must not overlap with + // any other subnet being used by OpenShift or by the node network. The size of the + // subnet must be larger than the number of nodes. + // The current default value is fd98::/64. + // For KubeVirt hosted clusters, if this field is not set, HyperShift will + // automatically use fd99::/64 to avoid collisions with the management cluster's + // default join subnet (fd98::/64). + // The subnet must be large enough to accommodate one IP per node in your cluster. + // The value must be a valid IPv6 CIDR (e.g. fd98::/64). IPv4 addresses, + // IPv4-mapped IPv6 addresses, and dual-stack addresses are not permitted. + // The prefix length must be in the range /0 to /125 inclusive. + // This field is immutable once set. + // +kubebuilder:validation:MaxLength=48 + // +kubebuilder:validation:MinLength=3 + // +kubebuilder:validation:XValidation:rule="isCIDR(self) && cidr(self).ip().family() == 6", message="Subnet must be in valid IPv6 CIDR format (e.g., fd98::/64)" + // +kubebuilder:validation:XValidation:rule="isCIDR(self) && cidr(self).prefixLength() <= 125", message="subnet must be in the range /0 to /125 inclusive" + // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="internalJoinSubnet is immutable" + // +optional + InternalJoinSubnet string `json:"internalJoinSubnet,omitempty"` +} + // IngressOperatorSpec is the specification of the desired behavior of the Ingress Operator. type IngressOperatorSpec struct { // endpointPublishingStrategy is used to publish the default ingress controller endpoints. diff --git a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go index 5954bdd73ad3..a3d882374d25 100644 --- a/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go +++ b/vendor/github.com/openshift/hypershift/api/hypershift/v1beta1/zz_generated.deepcopy.go @@ -3818,6 +3818,21 @@ func (in *OVNIPv4Config) DeepCopy() *OVNIPv4Config { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OVNIPv6Config) DeepCopyInto(out *OVNIPv6Config) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OVNIPv6Config. +func (in *OVNIPv6Config) DeepCopy() *OVNIPv6Config { + if in == nil { + return nil + } + out := new(OVNIPv6Config) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OVNKubernetesConfig) DeepCopyInto(out *OVNKubernetesConfig) { *out = *in @@ -3826,6 +3841,7 @@ func (in *OVNKubernetesConfig) DeepCopyInto(out *OVNKubernetesConfig) { *out = new(OVNIPv4Config) **out = **in } + out.IPv6 = in.IPv6 } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OVNKubernetesConfig. diff --git a/vendor/google.golang.org/api/compute/v1/compute-api.json b/vendor/google.golang.org/api/compute/v1/compute-api.json index bf1f0863995d..9c88fd3ca023 100644 --- a/vendor/google.golang.org/api/compute/v1/compute-api.json +++ b/vendor/google.golang.org/api/compute/v1/compute-api.json @@ -47746,7 +47746,7 @@ } } }, - "revision": "20260422", + "revision": "20260501", "rootUrl": "https://compute.googleapis.com/", "schemas": { "AWSV4Signature": { @@ -49571,7 +49571,6 @@ "type": "string" }, "diskType": { - "description": "Specifies the disk type to use to create the instance. If not specified,\nthe default is pd-standard, specified using the full URL.\nFor example:\n\nhttps://www.googleapis.com/compute/v1/projects/project/zones/zone/diskTypes/pd-standard\n\n\nFor a full list of acceptable values, seePersistent disk\ntypes. If you specify this field when creating a VM, you can provide\neither the full or partial URL. For example, the following values are\nvalid:\n \n \n - https://www.googleapis.com/compute/v1/projects/project/zones/zone/diskTypes/diskType \n - projects/project/zones/zone/diskTypes/diskType \n - zones/zone/diskTypes/diskType\n\n\nIf you specify this field when creating or updating an instance template\nor all-instances configuration, specify the type of the disk, not the\nURL. For example: pd-standard.", "type": "string" }, "enableConfidentialCompute": { @@ -55156,12 +55155,14 @@ "confidentialInstanceType": { "description": "Defines the type of technology used by the confidential instance.", "enum": [ + "CCA", "CONFIDENTIAL_INSTANCE_TYPE_UNSPECIFIED", "SEV", "SEV_SNP", "TDX" ], "enumDescriptions": [ + "Arm Confidential Compute Architecture.", "No type specified. Do not use this value.", "AMD Secure Encrypted Virtualization.", "AMD Secure Encrypted Virtualization - Secure Nested Paging.", @@ -58580,6 +58581,13 @@ "description": "A fully-qualified URL of a SecurityProfile resource instance.\nExample:\nhttps://networksecurity.googleapis.com/v1/projects/{project}/locations/{location}/securityProfileGroups/my-security-profile-group\nMust be specified if action is one of 'apply_security_profile_group' or\n'mirror'. Cannot be specified for other actions.", "type": "string" }, + "targetForwardingRules": { + "description": "A list of forwarding rules to which this rule applies.\nThis field allows you to control which load balancers get this rule.\nFor example, the following are valid values:\n \n \n - https://www.googleapis.com/compute/v1/projects/project/global/forwardingRules/forwardingRule\n - https://www.googleapis.com/compute/v1/projects/project/regions/region/forwardingRules/forwardingRule\n - projects/project/global/\n forwardingRules/forwardingRule\n - projects/project/regions/region/forwardingRules/\n forwardingRule", + "items": { + "type": "string" + }, + "type": "array" + }, "targetResources": { "description": "A list of network resource URLs to which this rule applies. This field\nallows you to control which network's VMs get this rule. If this field\nis left blank, all VMs within the organization will receive the rule.", "items": { @@ -58601,6 +58609,18 @@ }, "type": "array" }, + "targetType": { + "description": "Target types of the firewall policy rule.\nDefault value is INSTANCES.", + "enum": [ + "INSTANCES", + "INTERNAL_MANAGED_LB" + ], + "enumDescriptions": [ + "", + "" + ], + "type": "string" + }, "tlsInspect": { "description": "Boolean flag indicating if the traffic should be TLS decrypted.\nCan be set only if action = 'apply_security_profile_group' and cannot\nbe set for other actions.", "type": "boolean" @@ -61560,6 +61580,7 @@ "description": "The ID of a supported feature. To add multiple values, use commas to\nseparate values. Set to one or more of the following values:\n \n - VIRTIO_SCSI_MULTIQUEUE\n - WINDOWS\n - MULTI_IP_SUBNET\n - UEFI_COMPATIBLE\n - GVNIC\n - SEV_CAPABLE\n - SUSPEND_RESUME_COMPATIBLE\n - SEV_LIVE_MIGRATABLE_V2\n - SEV_SNP_CAPABLE\n - TDX_CAPABLE\n - IDPF\n - SNP_SVSM_CAPABLE\n - CCA_CAPABLE\n\n\nFor more information, see\nEnabling guest operating system features.", "enum": [ "BARE_METAL_LINUX_COMPATIBLE", + "CCA_CAPABLE", "FEATURE_TYPE_UNSPECIFIED", "GVNIC", "IDPF", @@ -61590,6 +61611,7 @@ "", "", "", + "", "" ], "type": "string" @@ -78407,7 +78429,7 @@ "type": "string" }, "network": { - "description": "The URL of the network to which all network endpoints in the NEG belong.\nUses default project network if unspecified.", + "description": "The URL of the network to which all network endpoints in the NEG belong.\nFor networkEndpointType GCE_VM_IP_PORT,GCE_VM_IP_PORTMAP or NON_GCP_PRIVATE_IP_PORT,\nif this field is not specified, a default network will be used.\nThis field cannot be set for NEGs with networkEndpointType set toSERVERLESS or PRIVATE_SERVICE_CONNECT and for\nglobal NEGs.\nFor all other network endpoint types, this field is required.", "type": "string" }, "networkEndpointType": { diff --git a/vendor/google.golang.org/api/compute/v1/compute-gen.go b/vendor/google.golang.org/api/compute/v1/compute-gen.go index d143bd020430..30d6937405f5 100644 --- a/vendor/google.golang.org/api/compute/v1/compute-gen.go +++ b/vendor/google.golang.org/api/compute/v1/compute-gen.go @@ -3956,34 +3956,8 @@ type AttachedDiskInitializeParams struct { // boot disks, the default size is the size of the sourceImage. // If you do not specify a sourceImage, the default disk size // is 500 GB. - DiskSizeGb int64 `json:"diskSizeGb,omitempty,string"` - // DiskType: Specifies the disk type to use to create the instance. If not - // specified, - // the default is pd-standard, specified using the full URL. - // For - // example: - // - // https://www.googleapis.com/compute/v1/projects/project/zones/zone/diskTypes/pd-standard - // - // - // For a full list of acceptable values, seePersistent disk - // types. If you specify this field when creating a VM, you can provide - // either the full or partial URL. For example, the following values - // are - // valid: - // - // - // - - // https://www.googleapis.com/compute/v1/projects/project/zones/zone/diskTypes/diskType - // - // - projects/project/zones/zone/diskTypes/diskType - // - zones/zone/diskTypes/diskType - // - // - // If you specify this field when creating or updating an instance template - // or all-instances configuration, specify the type of the disk, not the - // URL. For example: pd-standard. - DiskType string `json:"diskType,omitempty"` + DiskSizeGb int64 `json:"diskSizeGb,omitempty,string"` + DiskType string `json:"diskType,omitempty"` // EnableConfidentialCompute: Whether this disk is using confidential compute // mode. EnableConfidentialCompute bool `json:"enableConfidentialCompute,omitempty"` @@ -12511,6 +12485,7 @@ type ConfidentialInstanceConfig struct { // confidential instance. // // Possible values: + // "CCA" - Arm Confidential Compute Architecture. // "CONFIDENTIAL_INSTANCE_TYPE_UNSPECIFIED" - No type specified. Do not use // this value. // "SEV" - AMD Secure Encrypted Virtualization. @@ -17388,6 +17363,21 @@ type FirewallPolicyRule struct { // or // 'mirror'. Cannot be specified for other actions. SecurityProfileGroup string `json:"securityProfileGroup,omitempty"` + // TargetForwardingRules: A list of forwarding rules to which this rule + // applies. + // This field allows you to control which load balancers get this rule. + // For example, the following are valid values: + // + // + // - + // https://www.googleapis.com/compute/v1/projects/project/global/forwardingRules/forwardingRule + // - + // https://www.googleapis.com/compute/v1/projects/project/regions/region/forwardingRules/forwardingRule + // - projects/project/global/ + // forwardingRules/forwardingRule + // - projects/project/regions/region/forwardingRules/ + // forwardingRule + TargetForwardingRules []string `json:"targetForwardingRules,omitempty"` // TargetResources: A list of network resource URLs to which this rule applies. // This field // allows you to control which network's VMs get this rule. If this field @@ -17410,6 +17400,13 @@ type FirewallPolicyRule struct { // instances that are // applied with this rule. TargetServiceAccounts []string `json:"targetServiceAccounts,omitempty"` + // TargetType: Target types of the firewall policy rule. + // Default value is INSTANCES. + // + // Possible values: + // "INSTANCES" + // "INTERNAL_MANAGED_LB" + TargetType string `json:"targetType,omitempty"` // TlsInspect: Boolean flag indicating if the traffic should be TLS // decrypted. // Can be set only if action = 'apply_security_profile_group' and cannot @@ -21382,6 +21379,7 @@ type GuestOsFeature struct { // // Possible values: // "BARE_METAL_LINUX_COMPATIBLE" + // "CCA_CAPABLE" // "FEATURE_TYPE_UNSPECIFIED" // "GVNIC" // "IDPF" @@ -43236,7 +43234,13 @@ type NetworkEndpointGroup struct { Name string `json:"name,omitempty"` // Network: The URL of the network to which all network endpoints in the NEG // belong. - // Uses default project network if unspecified. + // For networkEndpointType GCE_VM_IP_PORT,GCE_VM_IP_PORTMAP or + // NON_GCP_PRIVATE_IP_PORT, + // if this field is not specified, a default network will be used. + // This field cannot be set for NEGs with networkEndpointType set toSERVERLESS + // or PRIVATE_SERVICE_CONNECT and for + // global NEGs. + // For all other network endpoint types, this field is required. Network string `json:"network,omitempty"` // NetworkEndpointType: Type of network endpoints in this network endpoint // group. Can be one ofGCE_VM_IP, GCE_VM_IP_PORT,NON_GCP_PRIVATE_IP_PORT, diff --git a/vendor/google.golang.org/api/internal/version.go b/vendor/google.golang.org/api/internal/version.go index b56e65b78175..fafd3517c409 100644 --- a/vendor/google.golang.org/api/internal/version.go +++ b/vendor/google.golang.org/api/internal/version.go @@ -5,4 +5,4 @@ package internal // Version is the current tagged release of the library. -const Version = "0.279.0" +const Version = "0.280.0" diff --git a/vendor/modules.txt b/vendor/modules.txt index b8136221f2fa..9ac103adc06f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1567,7 +1567,7 @@ golang.org/x/tools/internal/versions # gomodules.xyz/jsonpatch/v2 v2.5.0 ## explicit; go 1.20 gomodules.xyz/jsonpatch/v2 -# google.golang.org/api v0.279.0 +# google.golang.org/api v0.280.0 ## explicit; go 1.25.0 google.golang.org/api/cloudresourcemanager/v1 google.golang.org/api/compute/v1 @@ -1591,7 +1591,7 @@ google.golang.org/genproto/googleapis/api google.golang.org/genproto/googleapis/api/annotations google.golang.org/genproto/googleapis/api/expr/v1alpha1 google.golang.org/genproto/googleapis/api/httpbody -# google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4 +# google.golang.org/genproto/googleapis/rpc v0.0.0-20260511170946-3700d4141b60 ## explicit; go 1.25.0 google.golang.org/genproto/googleapis/rpc/code google.golang.org/genproto/googleapis/rpc/errdetails