From 976c61e0d11088ce8dd883d343676cefff2ad91d Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Sun, 16 Aug 2026 23:00:36 -0400 Subject: [PATCH 01/44] feat(gcp-hcp): implement ArgoCD sync validation with Ginkgo v2 tests Implements GCP-557: Replace placeholder ArgoCD validation step with real Ginkgo v2 tests Changes: - ci-operator config: Add gcp-hcp-e2e-tests image build from Dockerfile.e2e - verify-argocd-sync commands.sh: Replace placeholder with full implementation - Authenticate with WIF credential from hypershift-gcp-wif-auth step - Generate region and MC cluster kubeconfigs via gcloud CLI - Run Ginkgo v2 test binary with JUnit output - Export REGION_KUBECONFIG and MC_KUBECONFIG to test environment - verify-argocd-sync ref.yaml: Update step configuration - Use gcp-hcp-e2e-tests image (from gcp-hcp-infra repo) - Increase resources: 500m CPU, 1Gi memory (for Go client libraries) - Increase timeout: 60min (accommodates 30min sync + retries + setup) - Update documentation with dependencies and environment variables Architecture: - Bash wrapper: WIF auth + kubeconfig generation via gcloud CLI - Ginkgo test binary: Pure Go validation using controller-runtime/client-go - No gcloud SDK in test image (only in wrapper script) - JUnit XML output to ARTIFACT_DIR for Prow test grid Test flow: 1. Wait for ArgoCD namespace (20min timeout) 2. Wait for ArgoCD server deployment ready (20min timeout) 3. Poll Applications for Synced status (30min timeout, configurable) 4. Auto-retry stuck apps every 2 minutes 5. Optional MC cluster validation (skips if MC_KUBECONFIG not set) Related: openshift-online/gcp-hcp-infra commit 29b7ce7f --- .../openshift-online-gcp-hcp-infra-main.yaml | 3 + .../gcp-hcp-verify-argocd-sync-commands.sh | 83 ++++++++++++++----- .../gcp-hcp-verify-argocd-sync-ref.yaml | 33 ++++++-- 3 files changed, 91 insertions(+), 28 deletions(-) diff --git a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml index d4478cd20b2de..f85cf35e0c479 100644 --- a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml +++ b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml @@ -5,6 +5,9 @@ build_root: namespace: ci tag: latest use_build_cache: true +images: +- dockerfile_path: Dockerfile.e2e + to: gcp-hcp-e2e-tests prowgen: expose: true private: true diff --git a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh index 863e37b504240..c38fa435f29ea 100755 --- a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh @@ -1,29 +1,74 @@ #!/usr/bin/env bash set -euo pipefail -echo "=== GCP HCP E2E Test Placeholder ===" +echo "=== GCP HCP ArgoCD Sync Validation ===" echo "" -echo "Validating provision outputs..." -# Check all expected outputs exist and are non-empty -REQUIRED_OUTPUTS="region-project-id mc-project-id mc-cluster-name mc-cluster-endpoint workspace-name run-id" +# Authenticate with WIF credential (already generated by hypershift-gcp-wif-auth step) +echo "Authenticating with WIF credential..." +if [[ ! -f "${SHARED_DIR}/wif-cred.json" ]]; then + echo "ERROR: WIF credential not found at ${SHARED_DIR}/wif-cred.json" + echo "The hypershift-gcp-wif-auth step must run before this step" + exit 1 +fi -for output in ${REQUIRED_OUTPUTS}; do - if [[ ! -s "${SHARED_DIR}/${output}" ]]; then - echo "ERROR: Missing or empty output: ${output}" - exit 1 - fi - echo " OK ${output}: $(<"${SHARED_DIR}/${output}")" -done +gcloud auth login --cred-file="${SHARED_DIR}/wif-cred.json" --quiet +# Read terraform outputs written by gcp-hcp-tf-provision step +echo "Reading terraform outputs from provision step..." +REGION_PROJECT=$(<"${SHARED_DIR}/region-project-id") +MC_PROJECT=$(<"${SHARED_DIR}/mc-project-id") +MC_CLUSTER_NAME=$(<"${SHARED_DIR}/mc-cluster-name") +MC_CLUSTER_ENDPOINT=$(<"${SHARED_DIR}/mc-cluster-endpoint") +REGION=${GCP_REGION:-us-central1} + +echo " Region Project: ${REGION_PROJECT}" +echo " MC Project: ${MC_PROJECT}" +echo " MC Cluster Name: ${MC_CLUSTER_NAME}" +echo " MC Cluster Endpoint: ${MC_CLUSTER_ENDPOINT}" +echo " Region: ${REGION}" echo "" -echo "Infrastructure Details:" -echo " Region Project: $(<"${SHARED_DIR}/region-project-id")" -echo " MC Project: $(<"${SHARED_DIR}/mc-project-id")" -echo " MC Cluster: $(<"${SHARED_DIR}/mc-cluster-name")" -echo " Workspace: $(<"${SHARED_DIR}/workspace-name")" + +# Generate region cluster kubeconfig +# Note: The provision step currently uses "mc-cluster-name" for the region cluster +# This is a naming inconsistency we're preserving for compatibility +echo "Generating region cluster kubeconfig..." +gcloud container clusters get-credentials "${MC_CLUSTER_NAME}" \ + --region="${REGION}" \ + --project="${REGION_PROJECT}" \ + --quiet + +# Flatten and export to SHARED_DIR +kubectl config view --flatten --minify > "${SHARED_DIR}/region-kubeconfig" +echo " ✓ Region kubeconfig written to ${SHARED_DIR}/region-kubeconfig" + +# Generate MC cluster kubeconfig +# TODO: Once terraform outputs are updated to distinguish region vs MC clusters, +# this will need to use separate cluster names +echo "Generating management cluster kubeconfig..." +gcloud container clusters get-credentials "${MC_CLUSTER_NAME}" \ + --region="${REGION}" \ + --project="${MC_PROJECT}" \ + --quiet + +kubectl config view --flatten --minify > "${SHARED_DIR}/mc-kubeconfig" +echo " ✓ MC kubeconfig written to ${SHARED_DIR}/mc-kubeconfig" echo "" -echo "All provision outputs validated successfully" + +# Set environment variables for Ginkgo test +export REGION_KUBECONFIG="${SHARED_DIR}/region-kubeconfig" +export MC_KUBECONFIG="${SHARED_DIR}/mc-kubeconfig" + +# Optional: Override sync timeouts (defaults to 30m in test code) +# export REGION_SYNC_TIMEOUT="30m" +# export MC_SYNC_TIMEOUT="30m" + +# Run Ginkgo v2 test binary +echo "Running ArgoCD sync validation tests..." +/usr/bin/test-e2e \ + --ginkgo.v \ + --ginkgo.junit-report="${ARTIFACT_DIR}/junit_argocd_sync.xml" \ + --ginkgo.label-filter="argocd" + echo "" -echo "NOTE: This is a placeholder. Real ArgoCD sync validation" -echo " will be implemented in a future story." +echo "✓ ArgoCD sync validation completed successfully" diff --git a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml index f27861752811c..9be3d830ddde1 100644 --- a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml +++ b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml @@ -1,15 +1,30 @@ ref: as: gcp-hcp-verify-argocd-sync - from_image: - namespace: ocp - name: "4.22" - tag: cli + from: gcp-hcp-e2e-tests commands: gcp-hcp-verify-argocd-sync-commands.sh resources: requests: - cpu: 100m - memory: 256Mi - timeout: 5m0s + cpu: 500m + memory: 1Gi + timeout: 60m0s documentation: |- - Placeholder test step that validates provision outputs are available. - Real ArgoCD sync validation will be added in a future story. + Validates that ArgoCD applications have synced successfully on the region + and management clusters after infrastructure provisioning. + + Uses Ginkgo v2 tests to: + - Wait for ArgoCD namespace to be bootstrapped by Fleet Config Sync + - Wait for ArgoCD server deployment to become ready + - Poll all ArgoCD Application CRs for Synced status + - Auto-retry stuck applications every 2 minutes + - Report detailed status and failures + + Depends on: + - hypershift-gcp-wif-auth (must run first to generate WIF credential) + - gcp-hcp-tf-provision (must run to create clusters and write outputs) + + Environment variables (optional): + - REGION_SYNC_TIMEOUT: Timeout for region cluster sync (default: 30m) + - MC_SYNC_TIMEOUT: Timeout for MC cluster sync (default: 30m) + + Outputs: + - ${ARTIFACT_DIR}/junit_argocd_sync.xml: JUnit test results for Prow From 5de968c62bf397b9ec2bb3c8e945d69368f940c3 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Sun, 16 Aug 2026 23:13:57 -0400 Subject: [PATCH 02/44] fix(ci-operator): correct images section format to use items key The images section requires an 'items' key for the list of images. This matches the format used in other ci-operator configs like openshift/hypershift. --- .../gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml index f85cf35e0c479..0bbdcee62f7d7 100644 --- a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml +++ b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml @@ -6,8 +6,9 @@ build_root: tag: latest use_build_cache: true images: -- dockerfile_path: Dockerfile.e2e - to: gcp-hcp-e2e-tests + items: + - dockerfile_path: Dockerfile.e2e + to: gcp-hcp-e2e-tests prowgen: expose: true private: true From e1c0bfdc34ea5a31356972506752da221dad25ac Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Mon, 17 Aug 2026 09:53:33 -0400 Subject: [PATCH 03/44] fix(ci): regenerate Prow jobs for gcp-hcp-e2e-tests image Running make jobs with increased Docker memory (8GB) to regenerate Prow job configurations after adding gcp-hcp-e2e-tests image to ci-operator config. Changes: - Update decoration_config to use sparse checkout for Dockerfile.e2e - Move github-credentials from volumes to oauth_token_secret in decoration - Remove redundant github-credentials volume definitions Resolves generated-config CI check failure. --- ...online-gcp-hcp-infra-main-postsubmits.yaml | 9 ++++---- ...-online-gcp-hcp-infra-main-presubmits.yaml | 21 +++++-------------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml index f144d7f5eb8ac..f3dfd9179d40c 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml @@ -7,7 +7,11 @@ postsubmits: cluster: build03 decorate: true decoration_config: - skip_cloning: true + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner + sparse_checkout_files: + - Dockerfile.e2e labels: ci.openshift.io/generator: prowgen max_concurrency: 1 @@ -54,9 +58,6 @@ postsubmits: readOnly: true serviceAccountName: ci-operator volumes: - - name: github-credentials-openshift-ci-robot-private-git-cloner - secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml index 512aca5dd1c46..c4fe090ff02c9 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml @@ -9,7 +9,11 @@ presubmits: context: ci/prow/check-generated-files decorate: true decoration_config: - skip_cloning: true + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner + sparse_checkout_files: + - Dockerfile.e2e labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -57,9 +61,6 @@ presubmits: readOnly: true serviceAccountName: ci-operator volumes: - - name: github-credentials-openshift-ci-robot-private-git-cloner - secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher @@ -127,9 +128,6 @@ presubmits: readOnly: true serviceAccountName: ci-operator volumes: - - name: github-credentials-openshift-ci-robot-private-git-cloner - secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher @@ -216,9 +214,6 @@ presubmits: - name: ci-pull-credentials secret: secretName: ci-pull-credentials - - name: github-credentials-openshift-ci-robot-private-git-cloner - secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher @@ -303,9 +298,6 @@ presubmits: - name: ci-pull-credentials secret: secretName: ci-pull-credentials - - name: github-credentials-openshift-ci-robot-private-git-cloner - secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher @@ -390,9 +382,6 @@ presubmits: - name: ci-pull-credentials secret: secretName: ci-pull-credentials - - name: github-credentials-openshift-ci-robot-private-git-cloner - secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher From 2defa40246d7ef2d5d5b0b21472953fd2e8eda5f Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Mon, 17 Aug 2026 12:22:08 -0400 Subject: [PATCH 04/44] fix(ci): remove github-credentials volumeMounts after moving to oauth_token_secret The previous commit moved github-credentials to decoration_config oauth_token_secret but left the volumeMount references, causing: 'volumeMount named "github-credentials-openshift-ci-robot-private-git-cloner" is undefined' Removed all volumeMount entries for github-credentials since oauth_token_secret in decoration_config provides credentials without explicit volume mounts. --- ...ift-online-gcp-hcp-infra-main-postsubmits.yaml | 3 --- ...hift-online-gcp-hcp-infra-main-presubmits.yaml | 15 --------------- 2 files changed, 18 deletions(-) diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml index f3dfd9179d40c..63c8f6d46a1b1 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml @@ -44,9 +44,6 @@ postsubmits: - mountPath: /secrets/gcs name: gcs-credentials readOnly: true - - mountPath: /usr/local/github-credentials - name: github-credentials-openshift-ci-robot-private-git-cloner - readOnly: true - mountPath: /secrets/manifest-tool name: manifest-tool-local-pusher readOnly: true diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml index c4fe090ff02c9..50692b6e252f1 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml @@ -47,9 +47,6 @@ presubmits: - mountPath: /secrets/gcs name: gcs-credentials readOnly: true - - mountPath: /usr/local/github-credentials - name: github-credentials-openshift-ci-robot-private-git-cloner - readOnly: true - mountPath: /secrets/manifest-tool name: manifest-tool-local-pusher readOnly: true @@ -114,9 +111,6 @@ presubmits: - mountPath: /secrets/gcs name: gcs-credentials readOnly: true - - mountPath: /usr/local/github-credentials - name: github-credentials-openshift-ci-robot-private-git-cloner - readOnly: true - mountPath: /secrets/manifest-tool name: manifest-tool-local-pusher readOnly: true @@ -191,9 +185,6 @@ presubmits: - mountPath: /secrets/gcs name: gcs-credentials readOnly: true - - mountPath: /usr/local/github-credentials - name: github-credentials-openshift-ci-robot-private-git-cloner - readOnly: true - mountPath: /secrets/manifest-tool name: manifest-tool-local-pusher readOnly: true @@ -275,9 +266,6 @@ presubmits: - mountPath: /secrets/gcs name: gcs-credentials readOnly: true - - mountPath: /usr/local/github-credentials - name: github-credentials-openshift-ci-robot-private-git-cloner - readOnly: true - mountPath: /secrets/manifest-tool name: manifest-tool-local-pusher readOnly: true @@ -359,9 +347,6 @@ presubmits: - mountPath: /secrets/gcs name: gcs-credentials readOnly: true - - mountPath: /usr/local/github-credentials - name: github-credentials-openshift-ci-robot-private-git-cloner - readOnly: true - mountPath: /secrets/manifest-tool name: manifest-tool-local-pusher readOnly: true From 5a93d7296b636affaf669c5673f5d40fd28969c9 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Mon, 17 Aug 2026 12:24:47 -0400 Subject: [PATCH 05/44] fix(security): remove MC_CLUSTER_ENDPOINT from log output Cluster endpoints can expose internal hostnames/IPs. Remove from diagnostic logging per CodeRabbit security finding. --- .../verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh index c38fa435f29ea..bd6e6dd40d0c7 100755 --- a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh @@ -25,7 +25,6 @@ REGION=${GCP_REGION:-us-central1} echo " Region Project: ${REGION_PROJECT}" echo " MC Project: ${MC_PROJECT}" echo " MC Cluster Name: ${MC_CLUSTER_NAME}" -echo " MC Cluster Endpoint: ${MC_CLUSTER_ENDPOINT}" echo " Region: ${REGION}" echo "" From d2cb5bba24c23d28a42add143f66b0859971d445 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Mon, 17 Aug 2026 12:43:58 -0400 Subject: [PATCH 06/44] fix: use distinct region cluster name and make MC kubeconfig conditional - Export region.value.cluster_name as region-cluster-name in provision step - Use REGION_CLUSTER_NAME for region kubeconfig (not MC_CLUSTER_NAME) - Make MC kubeconfig generation conditional (don't fail if MC unavailable) - Tests will skip MC validation if MC_KUBECONFIG not set Addresses CodeRabbit findings: - 3793583558: Use region cluster name for region kubeconfig - 3793583560: Keep region validation available when MC unavailable --- .../gcp-hcp-tf-provision-commands.sh | 3 +- .../gcp-hcp-verify-argocd-sync-commands.sh | 33 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh b/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh index 1da3e047bf4e1..2814d1609b937 100755 --- a/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh @@ -247,6 +247,7 @@ fi # Write individual outputs to SHARED_DIR for downstream steps jq -r '.region.value.project_id // empty' /tmp/tf-outputs.json > "${SHARED_DIR}/region-project-id" +jq -r '.region.value.cluster_name // empty' /tmp/tf-outputs.json > "${SHARED_DIR}/region-cluster-name" jq -r '.management_cluster.value.project_id // empty' /tmp/tf-outputs.json > "${SHARED_DIR}/mc-project-id" jq -r '.management_cluster.value.cluster_name // empty' /tmp/tf-outputs.json > "${SHARED_DIR}/mc-cluster-name" jq -r '.management_cluster.value.cluster_endpoint // empty' /tmp/tf-outputs.json > "${SHARED_DIR}/mc-cluster-endpoint" @@ -256,7 +257,7 @@ echo "${WORKSPACE_NAME}" > "${SHARED_DIR}/workspace-name" echo "${RUN_ID}" > "${SHARED_DIR}/run-id" # Validate critical outputs were written -for output_file in region-project-id mc-project-id mc-cluster-name mc-cluster-endpoint workspace-name run-id; do +for output_file in region-project-id region-cluster-name mc-project-id mc-cluster-name mc-cluster-endpoint workspace-name run-id; do if [[ ! -s "${SHARED_DIR}/${output_file}" ]]; then log "ERROR: Output file ${output_file} is empty or missing" exit 1 diff --git a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh index bd6e6dd40d0c7..a3d2be6d2523d 100755 --- a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh @@ -17,22 +17,21 @@ gcloud auth login --cred-file="${SHARED_DIR}/wif-cred.json" --quiet # Read terraform outputs written by gcp-hcp-tf-provision step echo "Reading terraform outputs from provision step..." REGION_PROJECT=$(<"${SHARED_DIR}/region-project-id") +REGION_CLUSTER_NAME=$(<"${SHARED_DIR}/region-cluster-name") MC_PROJECT=$(<"${SHARED_DIR}/mc-project-id") MC_CLUSTER_NAME=$(<"${SHARED_DIR}/mc-cluster-name") -MC_CLUSTER_ENDPOINT=$(<"${SHARED_DIR}/mc-cluster-endpoint") REGION=${GCP_REGION:-us-central1} -echo " Region Project: ${REGION_PROJECT}" -echo " MC Project: ${MC_PROJECT}" -echo " MC Cluster Name: ${MC_CLUSTER_NAME}" -echo " Region: ${REGION}" +echo " Region Project: ${REGION_PROJECT}" +echo " Region Cluster: ${REGION_CLUSTER_NAME}" +echo " MC Project: ${MC_PROJECT}" +echo " MC Cluster Name: ${MC_CLUSTER_NAME}" +echo " Region: ${REGION}" echo "" # Generate region cluster kubeconfig -# Note: The provision step currently uses "mc-cluster-name" for the region cluster -# This is a naming inconsistency we're preserving for compatibility echo "Generating region cluster kubeconfig..." -gcloud container clusters get-credentials "${MC_CLUSTER_NAME}" \ +gcloud container clusters get-credentials "${REGION_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${REGION_PROJECT}" \ --quiet @@ -41,22 +40,22 @@ gcloud container clusters get-credentials "${MC_CLUSTER_NAME}" \ kubectl config view --flatten --minify > "${SHARED_DIR}/region-kubeconfig" echo " ✓ Region kubeconfig written to ${SHARED_DIR}/region-kubeconfig" -# Generate MC cluster kubeconfig -# TODO: Once terraform outputs are updated to distinguish region vs MC clusters, -# this will need to use separate cluster names +# Generate MC cluster kubeconfig (optional - tests skip MC validation if unavailable) echo "Generating management cluster kubeconfig..." -gcloud container clusters get-credentials "${MC_CLUSTER_NAME}" \ +if gcloud container clusters get-credentials "${MC_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${MC_PROJECT}" \ - --quiet - -kubectl config view --flatten --minify > "${SHARED_DIR}/mc-kubeconfig" -echo " ✓ MC kubeconfig written to ${SHARED_DIR}/mc-kubeconfig" + --quiet 2>/dev/null; then + kubectl config view --flatten --minify > "${SHARED_DIR}/mc-kubeconfig" + echo " ✓ MC kubeconfig written to ${SHARED_DIR}/mc-kubeconfig" + export MC_KUBECONFIG="${SHARED_DIR}/mc-kubeconfig" +else + echo " ⚠ MC cluster unavailable - tests will skip management cluster validation" +fi echo "" # Set environment variables for Ginkgo test export REGION_KUBECONFIG="${SHARED_DIR}/region-kubeconfig" -export MC_KUBECONFIG="${SHARED_DIR}/mc-kubeconfig" # Optional: Override sync timeouts (defaults to 30m in test code) # export REGION_SYNC_TIMEOUT="30m" From 4412c1a5d72628fba7b07f9cf4b16d646f4bb247 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Mon, 17 Aug 2026 12:46:44 -0400 Subject: [PATCH 07/44] fix: address remaining CodeRabbit findings - Update workflow doc to describe real ArgoCD validation (not placeholder) - Add curl timeouts (--connect-timeout 10 --max-time 60) to tfc_api_call - Fix errexit control flow in terraform apply/destroy retry loops (capture command in if statement to prevent set -e exit before retry) Addresses CodeRabbit findings: - 3793583549: Update test phase documentation - 3793583551: Set timeout for TFC API requests - 3793583554: Fix errexit control flow in terraform retry loops --- .../gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml | 2 +- .../tf-deprovision/gcp-hcp-tf-deprovision-commands.sh | 8 ++++++-- .../tf-provision/gcp-hcp-tf-provision-commands.sh | 10 +++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml index a18c3c14236d4..b54122df6206d 100644 --- a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml +++ b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml @@ -15,5 +15,5 @@ workflow: Phase breakdown: - Pre: Authenticate via WIF, provision infrastructure with terraform apply - - Test: Validate outputs and infrastructure (placeholder for now) + - Test: Validate ArgoCD application sync on region and management clusters using Ginkgo v2 tests - Post: Clean up resources with terraform destroy (best-effort) diff --git a/ci-operator/step-registry/gcp-hcp/tf-deprovision/gcp-hcp-tf-deprovision-commands.sh b/ci-operator/step-registry/gcp-hcp/tf-deprovision/gcp-hcp-tf-deprovision-commands.sh index 47f7abcc89cb7..a08ac50e0d44f 100755 --- a/ci-operator/step-registry/gcp-hcp/tf-deprovision/gcp-hcp-tf-deprovision-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/tf-deprovision/gcp-hcp-tf-deprovision-commands.sh @@ -117,8 +117,12 @@ destroy_wait=30 while (( destroy_attempt <= MAX_DESTROY_ATTEMPTS )); do log "DESTROY ATTEMPT: ${destroy_attempt}/${MAX_DESTROY_ATTEMPTS}" - destroy_output=$(terraform destroy -auto-approve -no-color 2>&1) - destroy_exit=$? + # Capture terraform output and exit code without triggering errexit + if destroy_output=$(terraform destroy -auto-approve -no-color 2>&1); then + destroy_exit=0 + else + destroy_exit=$? + fi echo "${destroy_output}" | tee -a "${LOG}" if [[ ${destroy_exit} -eq 0 ]]; then diff --git a/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh b/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh index 2814d1609b937..c30ac4d077445 100755 --- a/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh @@ -30,7 +30,7 @@ tfc_api_call() { local attempt=1 while (( attempt <= max_retries )); do - if output=$(curl -sf "$@" 2>&1); then + if output=$(curl -sf --connect-timeout 10 --max-time 60 "$@" 2>&1); then echo "${output}" return 0 fi @@ -194,8 +194,12 @@ apply_wait=30 while (( apply_attempt <= MAX_APPLY_ATTEMPTS )); do log "APPLY ATTEMPT: ${apply_attempt}/${MAX_APPLY_ATTEMPTS}" - apply_output=$(terraform apply -auto-approve -no-color 2>&1) - apply_exit=$? + # Capture terraform output and exit code without triggering errexit + if apply_output=$(terraform apply -auto-approve -no-color 2>&1); then + apply_exit=0 + else + apply_exit=$? + fi echo "${apply_output}" | tee -a "${LOG}" if [[ ${apply_exit} -eq 0 ]]; then From de07f9bc0c5967bfffc677770ed856ca5a485445 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Mon, 17 Aug 2026 13:38:45 -0400 Subject: [PATCH 08/44] feat(ci): add image promotion for gcp-hcp-e2e-tests Following HyperShift pattern, add promotion config to push the gcp-hcp-e2e-tests image to ci namespace on merge. This replaces the deprecated BuildConfig pattern with modern ci-operator image building and promotion. Without promotion, the image is rebuilt from scratch for every e2e job run. With promotion, the image is pushed to registry on merge and reused by subsequent jobs. Pattern matches openshift/hypershift config: - Build image during CI from Dockerfile.e2e - Promote to ci namespace with latest tag on postsubmit - Step-registry references via 'from: gcp-hcp-e2e-tests' Context: Builds on app.ci are deprecated (per DPTP). The modern pattern is ci-operator images + promotion config instead of BuildConfigs in clusters/app.ci/supplemental-ci-images/. --- .../openshift-online-gcp-hcp-infra-main.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml index 0bbdcee62f7d7..0d764b62fabe3 100644 --- a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml +++ b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml @@ -9,6 +9,14 @@ images: items: - dockerfile_path: Dockerfile.e2e to: gcp-hcp-e2e-tests +promotion: + to: + - additional_images: + gcp-hcp-e2e-tests: gcp-hcp-e2e-tests + excluded_images: + - '*' + namespace: ci + tag: latest prowgen: expose: true private: true From 295ab7a9996512b793004e560fff3124aaa54603 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Mon, 17 Aug 2026 13:39:55 -0400 Subject: [PATCH 09/44] chore(ci): regenerate Prow jobs for image promotion Regenerated after adding promotion config. Creates new postsubmit job 'branch-ci-openshift-online-gcp-hcp-infra-main-images' that pushes gcp-hcp-e2e-tests to ci namespace on merge. --- ...online-gcp-hcp-infra-main-postsubmits.yaml | 71 ++++++++++++ ...-online-gcp-hcp-infra-main-presubmits.yaml | 102 +++++++++++++++++- 2 files changed, 169 insertions(+), 4 deletions(-) diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml index 63c8f6d46a1b1..b3ef9e2714907 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml @@ -44,6 +44,9 @@ postsubmits: - mountPath: /secrets/gcs name: gcs-credentials readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true - mountPath: /secrets/manifest-tool name: manifest-tool-local-pusher readOnly: true @@ -64,3 +67,71 @@ postsubmits: - name: result-aggregator secret: secretName: result-aggregator + - agent: kubernetes + always_run: true + branches: + - ^main$ + decorate: true + decoration_config: + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner + sparse_checkout_files: + - Dockerfile.e2e + labels: + ci-operator.openshift.io/is-promotion: "true" + ci.openshift.io/generator: prowgen + max_concurrency: 1 + name: branch-ci-openshift-online-gcp-hcp-infra-main-images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --image-mirror-push-secret=/etc/push-secret/.dockerconfigjson + - --oauth-token-path=/usr/local/github-credentials/oauth + - --promote + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --target=gcp-hcp-e2e-tests + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/push-secret + name: push-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: push-secret + secret: + secretName: registry-push-credentials-ci-central + - name: result-aggregator + secret: + secretName: result-aggregator diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml index 50692b6e252f1..30a8a4d5608e0 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml @@ -47,6 +47,9 @@ presubmits: - mountPath: /secrets/gcs name: gcs-credentials readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true - mountPath: /secrets/manifest-tool name: manifest-tool-local-pusher readOnly: true @@ -77,7 +80,11 @@ presubmits: context: ci/prow/check-orphan-modules decorate: true decoration_config: - skip_cloning: true + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner + sparse_checkout_files: + - Dockerfile.e2e labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -111,6 +118,9 @@ presubmits: - mountPath: /secrets/gcs name: gcs-credentials readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true - mountPath: /secrets/manifest-tool name: manifest-tool-local-pusher readOnly: true @@ -141,7 +151,11 @@ presubmits: context: ci/prow/e2e-platform decorate: true decoration_config: - skip_cloning: true + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner + sparse_checkout_files: + - Dockerfile.e2e labels: ci-operator.openshift.io/cloud: platform-gcp ci-operator.openshift.io/cloud-cluster-profile: platform-gcp @@ -185,6 +199,9 @@ presubmits: - mountPath: /secrets/gcs name: gcs-credentials readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true - mountPath: /secrets/manifest-tool name: manifest-tool-local-pusher readOnly: true @@ -215,6 +232,69 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )(e2e-platform|remaining-required),?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^main$ + - ^main- + context: ci/prow/images + decorate: true + decoration_config: + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner + sparse_checkout_files: + - Dockerfile.e2e + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-online-gcp-hcp-infra-main-images + rerun_command: /test images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --oauth-token-path=/usr/local/github-credentials/oauth + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --target=gcp-hcp-e2e-tests + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )images,?($|\s.*) - agent: kubernetes always_run: true branches: @@ -224,7 +304,11 @@ presubmits: context: ci/prow/terraform-test decorate: true decoration_config: - skip_cloning: true + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner + sparse_checkout_files: + - Dockerfile.e2e labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -266,6 +350,9 @@ presubmits: - mountPath: /secrets/gcs name: gcs-credentials readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true - mountPath: /secrets/manifest-tool name: manifest-tool-local-pusher readOnly: true @@ -305,7 +392,11 @@ presubmits: context: ci/prow/terraform-validate decorate: true decoration_config: - skip_cloning: true + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner + sparse_checkout_files: + - Dockerfile.e2e labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -347,6 +438,9 @@ presubmits: - mountPath: /secrets/gcs name: gcs-credentials readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true - mountPath: /secrets/manifest-tool name: manifest-tool-local-pusher readOnly: true From a424ee2b4b5452df42dcc18fb2bc1d9376d004d5 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 12:00:32 -0400 Subject: [PATCH 10/44] fix(e2e): split kubeconfig generation into separate step to avoid gcloud dependency in test image The verify-argocd-sync step was trying to run gcloud commands inside the gcp-hcp-e2e-tests image, which only contains the compiled Go test binary. This caused failures with 'gcloud: command not found'. Solution: Create new gcp-hcp-generate-kubeconfigs step that: - Runs in upi-installer image (has gcloud) - Generates static kubeconfigs with embedded access tokens - Saves kubeconfigs to SHARED_DIR for downstream steps The verify-argocd-sync step now: - Runs in gcp-hcp-e2e-tests image (test binary only) - Reads pre-generated kubeconfigs from SHARED_DIR - Executes Ginkgo v2 tests using native Kubernetes client-go This follows the same pattern as HyperShift E2E tests and avoids mixing infrastructure tooling (gcloud) with test execution (Go binary). --- .../gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml | 3 +- .../gcp-hcp/generate-kubeconfigs/OWNERS | 18 +++ .../gcp-hcp-generate-kubeconfigs-commands.sh | 107 ++++++++++++++++++ .../gcp-hcp-generate-kubeconfigs-ref.yaml | 25 ++++ .../gcp-hcp-verify-argocd-sync-commands.sh | 55 ++------- .../gcp-hcp-verify-argocd-sync-ref.yaml | 3 +- 6 files changed, 163 insertions(+), 48 deletions(-) create mode 100644 ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/OWNERS create mode 100755 ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh create mode 100644 ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.yaml diff --git a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml index b54122df6206d..8c770d44dde11 100644 --- a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml +++ b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml @@ -4,6 +4,7 @@ workflow: pre: - ref: hypershift-gcp-wif-auth - ref: gcp-hcp-tf-provision + - ref: gcp-hcp-generate-kubeconfigs test: - ref: gcp-hcp-verify-argocd-sync post: @@ -14,6 +15,6 @@ workflow: runs validation tests, and cleans up resources. Phase breakdown: - - Pre: Authenticate via WIF, provision infrastructure with terraform apply + - Pre: Authenticate via WIF, provision infrastructure with terraform apply, generate kubeconfigs - Test: Validate ArgoCD application sync on region and management clusters using Ginkgo v2 tests - Post: Clean up resources with terraform destroy (best-effort) diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/OWNERS b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/OWNERS new file mode 100644 index 0000000000000..60325af98b50d --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/OWNERS @@ -0,0 +1,18 @@ +approvers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm +reviewers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh new file mode 100755 index 0000000000000..bf2ed1dcfa0b9 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "=== GCP HCP Generate Kubeconfigs ===" +echo "" + +# Authenticate with WIF credential +echo "Authenticating with WIF credential..." +if [[ ! -f "${SHARED_DIR}/wif-cred.json" ]]; then + echo "ERROR: WIF credential not found at ${SHARED_DIR}/wif-cred.json" + exit 1 +fi + +gcloud auth login --cred-file="${SHARED_DIR}/wif-cred.json" --quiet + +# Read terraform outputs +REGION_PROJECT=$(<"${SHARED_DIR}/region-project-id") +REGION_CLUSTER_NAME=$(<"${SHARED_DIR}/region-cluster-name") +MC_PROJECT=$(<"${SHARED_DIR}/mc-project-id") +MC_CLUSTER_NAME=$(<"${SHARED_DIR}/mc-cluster-name") +REGION=${GCP_REGION:-us-central1} + +echo " Region Project: ${REGION_PROJECT}" +echo " Region Cluster: ${REGION_CLUSTER_NAME}" +echo " MC Project: ${MC_PROJECT}" +echo " MC Cluster: ${MC_CLUSTER_NAME}" +echo " Region: ${REGION}" +echo "" + +# Generate region cluster kubeconfig with embedded token +echo "Generating region cluster kubeconfig..." +set +x # Hide sensitive token +REGION_CA=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ + --region="${REGION}" \ + --project="${REGION_PROJECT}" \ + --format='value(masterAuth.clusterCaCertificate)') + +REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ + --region="${REGION}" \ + --project="${REGION_PROJECT}" \ + --format='value(endpoint)') + +ACCESS_TOKEN=$(gcloud auth print-access-token) + +cat > "${SHARED_DIR}/region-kubeconfig" << EOF +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority-data: ${REGION_CA} + server: https://${REGION_ENDPOINT} + name: region-cluster +contexts: +- context: + cluster: region-cluster + user: gcp-user + name: region-context +current-context: region-context +users: +- name: gcp-user + user: + token: ${ACCESS_TOKEN} +EOF + +echo " ✓ Region kubeconfig written" +set -x + +# Generate MC cluster kubeconfig (optional - test skips if unavailable) +echo "Generating management cluster kubeconfig..." +set +x # Hide sensitive token +if MC_CA=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ + --region="${REGION}" \ + --project="${MC_PROJECT}" \ + --format='value(masterAuth.clusterCaCertificate)' 2>/dev/null) && \ + MC_ENDPOINT=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ + --region="${REGION}" \ + --project="${MC_PROJECT}" \ + --format='value(endpoint)' 2>/dev/null); then + + cat > "${SHARED_DIR}/mc-kubeconfig" << EOF +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority-data: ${MC_CA} + server: https://${MC_ENDPOINT} + name: mc-cluster +contexts: +- context: + cluster: mc-cluster + user: gcp-user + name: mc-context +current-context: mc-context +users: +- name: gcp-user + user: + token: ${ACCESS_TOKEN} +EOF + + echo " ✓ MC kubeconfig written" +else + echo " ⚠ MC cluster unavailable - tests will skip MC validation" +fi +set -x + +echo "" +echo "✓ Kubeconfig generation completed" diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.yaml b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.yaml new file mode 100644 index 0000000000000..1b06a88fcfd60 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.yaml @@ -0,0 +1,25 @@ +ref: + as: gcp-hcp-generate-kubeconfigs + from_image: + namespace: ocp + name: "4.22" + tag: upi-installer + commands: gcp-hcp-generate-kubeconfigs-commands.sh + resources: + requests: + cpu: 100m + memory: 256Mi + timeout: 5m0s + documentation: |- + Generates kubeconfigs with embedded access tokens for region and management clusters. + + Uses gcloud to fetch cluster CA and endpoint, then creates static kubeconfigs + with embedded tokens (valid for ~60 minutes, sufficient for CI jobs). + + Depends on: + - hypershift-gcp-wif-auth (must run first) + - gcp-hcp-tf-provision (must write cluster names to SHARED_DIR) + + Outputs: + - ${SHARED_DIR}/region-kubeconfig: Region cluster kubeconfig + - ${SHARED_DIR}/mc-kubeconfig: MC cluster kubeconfig (optional) diff --git a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh index a3d2be6d2523d..b5c164c459ffe 100755 --- a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh @@ -4,58 +4,23 @@ set -euo pipefail echo "=== GCP HCP ArgoCD Sync Validation ===" echo "" -# Authenticate with WIF credential (already generated by hypershift-gcp-wif-auth step) -echo "Authenticating with WIF credential..." -if [[ ! -f "${SHARED_DIR}/wif-cred.json" ]]; then - echo "ERROR: WIF credential not found at ${SHARED_DIR}/wif-cred.json" - echo "The hypershift-gcp-wif-auth step must run before this step" +# Verify kubeconfigs exist (generated by gcp-hcp-generate-kubeconfigs step) +if [[ ! -f "${SHARED_DIR}/region-kubeconfig" ]]; then + echo "ERROR: Region kubeconfig not found at ${SHARED_DIR}/region-kubeconfig" + echo "The gcp-hcp-generate-kubeconfigs step must run before this step" exit 1 fi -gcloud auth login --cred-file="${SHARED_DIR}/wif-cred.json" --quiet - -# Read terraform outputs written by gcp-hcp-tf-provision step -echo "Reading terraform outputs from provision step..." -REGION_PROJECT=$(<"${SHARED_DIR}/region-project-id") -REGION_CLUSTER_NAME=$(<"${SHARED_DIR}/region-cluster-name") -MC_PROJECT=$(<"${SHARED_DIR}/mc-project-id") -MC_CLUSTER_NAME=$(<"${SHARED_DIR}/mc-cluster-name") -REGION=${GCP_REGION:-us-central1} - -echo " Region Project: ${REGION_PROJECT}" -echo " Region Cluster: ${REGION_CLUSTER_NAME}" -echo " MC Project: ${MC_PROJECT}" -echo " MC Cluster Name: ${MC_CLUSTER_NAME}" -echo " Region: ${REGION}" -echo "" - -# Generate region cluster kubeconfig -echo "Generating region cluster kubeconfig..." -gcloud container clusters get-credentials "${REGION_CLUSTER_NAME}" \ - --region="${REGION}" \ - --project="${REGION_PROJECT}" \ - --quiet - -# Flatten and export to SHARED_DIR -kubectl config view --flatten --minify > "${SHARED_DIR}/region-kubeconfig" -echo " ✓ Region kubeconfig written to ${SHARED_DIR}/region-kubeconfig" +# Set environment variables for Ginkgo test +export REGION_KUBECONFIG="${SHARED_DIR}/region-kubeconfig" -# Generate MC cluster kubeconfig (optional - tests skip MC validation if unavailable) -echo "Generating management cluster kubeconfig..." -if gcloud container clusters get-credentials "${MC_CLUSTER_NAME}" \ - --region="${REGION}" \ - --project="${MC_PROJECT}" \ - --quiet 2>/dev/null; then - kubectl config view --flatten --minify > "${SHARED_DIR}/mc-kubeconfig" - echo " ✓ MC kubeconfig written to ${SHARED_DIR}/mc-kubeconfig" +# MC kubeconfig is optional +if [[ -f "${SHARED_DIR}/mc-kubeconfig" ]]; then export MC_KUBECONFIG="${SHARED_DIR}/mc-kubeconfig" + echo "Using MC kubeconfig from ${SHARED_DIR}/mc-kubeconfig" else - echo " ⚠ MC cluster unavailable - tests will skip management cluster validation" + echo "MC kubeconfig not found - tests will skip management cluster validation" fi -echo "" - -# Set environment variables for Ginkgo test -export REGION_KUBECONFIG="${SHARED_DIR}/region-kubeconfig" # Optional: Override sync timeouts (defaults to 30m in test code) # export REGION_SYNC_TIMEOUT="30m" diff --git a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml index 9be3d830ddde1..ee7c4042cac23 100644 --- a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml +++ b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml @@ -19,8 +19,7 @@ ref: - Report detailed status and failures Depends on: - - hypershift-gcp-wif-auth (must run first to generate WIF credential) - - gcp-hcp-tf-provision (must run to create clusters and write outputs) + - gcp-hcp-generate-kubeconfigs (must run first to generate kubeconfigs) Environment variables (optional): - REGION_SYNC_TIMEOUT: Timeout for region cluster sync (default: 30m) From f9acea006e6f54e7569c64a0d5aff005f610a0eb Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 16:11:27 -0400 Subject: [PATCH 11/44] Use DNS/public endpoint for GKE kubeconfig generation Private GKE clusters with enable_private_endpoint=true return the internal endpoint when queried with --format='value(endpoint)'. Test pods cannot reach the private endpoint from the Prow cluster. This change prioritizes DNS-based endpoints (dnsConfig.clusterDns) which are publicly accessible when dns_allow_external_traffic=true, and falls back to the public IP endpoint (privateClusterConfig.publicEndpoint) if DNS endpoints are not configured. This ensures kubectl can access the clusters without requiring gcloud credential helpers or VPN connectivity. --- .../gcp-hcp-generate-kubeconfigs-commands.sh | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh index bf2ed1dcfa0b9..fbc50f0c62923 100755 --- a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh @@ -35,10 +35,19 @@ REGION_CA=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ --project="${REGION_PROJECT}" \ --format='value(masterAuth.clusterCaCertificate)') +# Get DNS endpoint if available (falls back to public IP endpoint) REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${REGION_PROJECT}" \ - --format='value(endpoint)') + --format='value(dnsConfig.clusterDns)') + +# Fallback to public IP endpoint if DNS endpoint is not configured +if [[ -z "${REGION_ENDPOINT}" ]]; then + REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ + --region="${REGION}" \ + --project="${REGION_PROJECT}" \ + --format='value(privateClusterConfig.publicEndpoint)') +fi ACCESS_TOKEN=$(gcloud auth print-access-token) @@ -71,11 +80,24 @@ set +x # Hide sensitive token if MC_CA=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${MC_PROJECT}" \ - --format='value(masterAuth.clusterCaCertificate)' 2>/dev/null) && \ - MC_ENDPOINT=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ - --region="${REGION}" \ - --project="${MC_PROJECT}" \ - --format='value(endpoint)' 2>/dev/null); then + --format='value(masterAuth.clusterCaCertificate)' 2>/dev/null); then + + # Get DNS endpoint if available (falls back to public IP endpoint) + MC_ENDPOINT=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ + --region="${REGION}" \ + --project="${MC_PROJECT}" \ + --format='value(dnsConfig.clusterDns)' 2>/dev/null) + + # Fallback to public IP endpoint if DNS endpoint is not configured + if [[ -z "${MC_ENDPOINT}" ]]; then + MC_ENDPOINT=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ + --region="${REGION}" \ + --project="${MC_PROJECT}" \ + --format='value(privateClusterConfig.publicEndpoint)' 2>/dev/null) + fi + + # Only create kubeconfig if we successfully got an endpoint + if [[ -n "${MC_ENDPOINT}" ]]; then cat > "${SHARED_DIR}/mc-kubeconfig" << EOF apiVersion: v1 @@ -97,7 +119,10 @@ users: token: ${ACCESS_TOKEN} EOF - echo " ✓ MC kubeconfig written" + echo " ✓ MC kubeconfig written" + else + echo " ⚠ MC endpoint unavailable - tests will skip MC validation" + fi else echo " ⚠ MC cluster unavailable - tests will skip MC validation" fi From aa87d3692e20ae58d0b95d9aec85a8abe5fdc275 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 16:17:13 -0400 Subject: [PATCH 12/44] Trigger rehearsal after gcp-hcp-infra PR #1317 merge From 6df18fc4263217a0aa8779f21b0540d5de113a1e Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 16:27:49 -0400 Subject: [PATCH 13/44] Add missing github-credentials volume to e2e-platform job The volumeMount was present but the corresponding volume definition was missing, causing pj-rehearse configuration validation to fail. This volume is needed for ci-operator's --oauth-token-path argument which provides GitHub authentication for private repo access during the test workflow. --- .../openshift-online-gcp-hcp-infra-main-presubmits.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml index 30a8a4d5608e0..f0123fcb9c231 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml @@ -222,6 +222,9 @@ presubmits: - name: ci-pull-credentials secret: secretName: ci-pull-credentials + - name: github-credentials-openshift-ci-robot-private-git-cloner + secret: + secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher From 0db61056511709e23546f821eccb031283cf997d Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 16:32:30 -0400 Subject: [PATCH 14/44] Revert: Remove manually-added github-credentials volume The volume 'github-credentials-openshift-ci-robot-private-git-cloner' is reserved for Prow's decoration system and is automatically added based on the --oauth-token-path argument. Adding it manually causes a conflict: 'volume github-credentials-openshift-ci-robot-private-git-cloner is a reserved for decoration' Prow's decorator handles this volume automatically - no manual definition needed. --- .../openshift-online-gcp-hcp-infra-main-presubmits.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml index f0123fcb9c231..30a8a4d5608e0 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml @@ -222,9 +222,6 @@ presubmits: - name: ci-pull-credentials secret: secretName: ci-pull-credentials - - name: github-credentials-openshift-ci-robot-private-git-cloner - secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher From a284112eb16a16c52a3a7d31caac64404cf66c8a Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 16:54:57 -0400 Subject: [PATCH 15/44] Update Dockerfile path for e2e test relocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updates ci-operator config to use test/Dockerfile.e2e after the e2e test files were moved from repo root to test/ directory. Changes: - dockerfile_path: Dockerfile.e2e → test/Dockerfile.e2e - sparse_checkout_files: Dockerfile.e2e → test/Dockerfile.e2e (6 jobs) This change must be applied atomically with openshift-online/gcp-hcp-infra#1326. Related: GCP-557 --- .../openshift-online-gcp-hcp-infra-main.yaml | 2 +- ...nshift-online-gcp-hcp-infra-main-postsubmits.yaml | 4 ++-- ...enshift-online-gcp-hcp-infra-main-presubmits.yaml | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml index 0d764b62fabe3..1c18ac1d874db 100644 --- a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml +++ b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml @@ -7,7 +7,7 @@ build_root: use_build_cache: true images: items: - - dockerfile_path: Dockerfile.e2e + - dockerfile_path: test/Dockerfile.e2e to: gcp-hcp-e2e-tests promotion: to: diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml index b3ef9e2714907..0c07363bd89ea 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml @@ -11,7 +11,7 @@ postsubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - Dockerfile.e2e + - test/Dockerfile.e2e labels: ci.openshift.io/generator: prowgen max_concurrency: 1 @@ -77,7 +77,7 @@ postsubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - Dockerfile.e2e + - test/Dockerfile.e2e labels: ci-operator.openshift.io/is-promotion: "true" ci.openshift.io/generator: prowgen diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml index 30a8a4d5608e0..49a0935bfeddb 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml @@ -13,7 +13,7 @@ presubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - Dockerfile.e2e + - test/Dockerfile.e2e labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -84,7 +84,7 @@ presubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - Dockerfile.e2e + - test/Dockerfile.e2e labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -155,7 +155,7 @@ presubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - Dockerfile.e2e + - test/Dockerfile.e2e labels: ci-operator.openshift.io/cloud: platform-gcp ci-operator.openshift.io/cloud-cluster-profile: platform-gcp @@ -244,7 +244,7 @@ presubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - Dockerfile.e2e + - test/Dockerfile.e2e labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -308,7 +308,7 @@ presubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - Dockerfile.e2e + - test/Dockerfile.e2e labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -396,7 +396,7 @@ presubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - Dockerfile.e2e + - test/Dockerfile.e2e labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" From 4be2c3b14f493028433da4f753a30848ff197a45 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 17:03:14 -0400 Subject: [PATCH 16/44] Disable Ginkgo color output for cleaner CI logs Adds --ginkgo.no-color flag to test execution. Color codes don't render well in CI log viewers and make output harder to read. Related: GCP-557 --- .../verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh index b5c164c459ffe..5c65b5703c53d 100755 --- a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh @@ -30,6 +30,7 @@ fi echo "Running ArgoCD sync validation tests..." /usr/bin/test-e2e \ --ginkgo.v \ + --ginkgo.no-color \ --ginkgo.junit-report="${ARTIFACT_DIR}/junit_argocd_sync.xml" \ --ginkgo.label-filter="argocd" From 22b79e693e54d61f05792997c4b195e8ded0e8a6 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 17:27:49 -0400 Subject: [PATCH 17/44] Fix build context for relocated test files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sets context_dir: test so Dockerfile.e2e can find go.mod/go.sum in the correct location. The dockerfile_path is relative to context_dir. Without this, the build fails with: error: copier: stat: "/go.mod": no such file or directory Changes: - context_dir: test (new) - dockerfile_path: test/Dockerfile.e2e → Dockerfile.e2e (relative to context_dir) Related: GCP-557 --- .../gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml index 1c18ac1d874db..4d1cb1862aed1 100644 --- a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml +++ b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml @@ -7,7 +7,8 @@ build_root: use_build_cache: true images: items: - - dockerfile_path: test/Dockerfile.e2e + - context_dir: test + dockerfile_path: Dockerfile.e2e to: gcp-hcp-e2e-tests promotion: to: From feb2027c39c2b3fa07952a4223da730eb27b8307 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 18:00:56 -0400 Subject: [PATCH 18/44] Fix sparse checkout by using inputs.src.paths for complete test/ directory The previous approach only checked out test/Dockerfile.e2e via sparse_checkout_files, but the build needs: - test/go.mod - test/go.sum - test/e2e/ Using inputs.src.paths copies the entire test/ directory into the build context, ensuring all required files are available. This replaces sparse checkout with explicit source path mapping: source: /go/src/github.com/openshift-online/gcp-hcp-infra/test dest: . (build context root) Related: GCP-557 --- .../gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml index 4d1cb1862aed1..9c89adc23d65c 100644 --- a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml +++ b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml @@ -9,6 +9,11 @@ images: items: - context_dir: test dockerfile_path: Dockerfile.e2e + inputs: + src: + paths: + - destination_dir: . + source_path: /go/src/github.com/openshift-online/gcp-hcp-infra/test to: gcp-hcp-e2e-tests promotion: to: From 730cd771f61764fe787992fbca11221cedfe8d6c Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 18:52:02 -0400 Subject: [PATCH 19/44] Fix sparse checkout to include entire test/ directory Prowgen auto-generates sparse_checkout_files based on dockerfile_path, but only includes the Dockerfile itself. This causes build failures because go.mod, go.sum, and e2e/ directory are not checked out. Manual fix: Change sparse_checkout_files from 'test/Dockerfile.e2e' to 'test/' to check out the entire test directory. This is the CORRECT and FINAL fix. All required files will now be available: - test/go.mod - test/go.sum - test/e2e/ - test/Dockerfile.e2e Related: GCP-557 --- .../openshift-online-gcp-hcp-infra-main.yaml | 5 ----- ...nshift-online-gcp-hcp-infra-main-postsubmits.yaml | 4 ++-- ...enshift-online-gcp-hcp-infra-main-presubmits.yaml | 12 ++++++------ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml index 9c89adc23d65c..4d1cb1862aed1 100644 --- a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml +++ b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml @@ -9,11 +9,6 @@ images: items: - context_dir: test dockerfile_path: Dockerfile.e2e - inputs: - src: - paths: - - destination_dir: . - source_path: /go/src/github.com/openshift-online/gcp-hcp-infra/test to: gcp-hcp-e2e-tests promotion: to: diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml index 0c07363bd89ea..ded809fb18356 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml @@ -11,7 +11,7 @@ postsubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - test/Dockerfile.e2e + - test/ labels: ci.openshift.io/generator: prowgen max_concurrency: 1 @@ -77,7 +77,7 @@ postsubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - test/Dockerfile.e2e + - test/ labels: ci-operator.openshift.io/is-promotion: "true" ci.openshift.io/generator: prowgen diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml index 49a0935bfeddb..cef5206572f55 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml @@ -13,7 +13,7 @@ presubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - test/Dockerfile.e2e + - test/ labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -84,7 +84,7 @@ presubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - test/Dockerfile.e2e + - test/ labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -155,7 +155,7 @@ presubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - test/Dockerfile.e2e + - test/ labels: ci-operator.openshift.io/cloud: platform-gcp ci-operator.openshift.io/cloud-cluster-profile: platform-gcp @@ -244,7 +244,7 @@ presubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - test/Dockerfile.e2e + - test/ labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -308,7 +308,7 @@ presubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - test/Dockerfile.e2e + - test/ labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -396,7 +396,7 @@ presubmits: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner sparse_checkout_files: - - test/Dockerfile.e2e + - test/ labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" From a0ffe1c74f81629e2d875fec61a2e1756601b844 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 23:41:56 -0400 Subject: [PATCH 20/44] Use private endpoint for cluster connectivity from Prow GKE private clusters are not reachable via public IP from Prow pods. The test was failing with: dial tcp 34.41.118.126:443: i/o timeout This was because we were using publicEndpoint as the fallback when DNS endpoint was unavailable. Fix: Use privateEndpoint instead of publicEndpoint for cluster-to-cluster connectivity. Prow pods run in the same GCP network and can reach private endpoints. Endpoint priority: 1. DNS endpoint (dnsConfig.clusterDns) - preferred 2. Private endpoint (privateClusterConfig.privateEndpoint) - fallback Related: GCP-557 --- .../gcp-hcp-generate-kubeconfigs-commands.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh index fbc50f0c62923..27860f2011eec 100755 --- a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh @@ -41,12 +41,13 @@ REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ --project="${REGION_PROJECT}" \ --format='value(dnsConfig.clusterDns)') -# Fallback to public IP endpoint if DNS endpoint is not configured +# Fallback to private endpoint if DNS endpoint is not configured +# Use private endpoint for cluster-to-cluster connectivity from Prow pod if [[ -z "${REGION_ENDPOINT}" ]]; then REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${REGION_PROJECT}" \ - --format='value(privateClusterConfig.publicEndpoint)') + --format='value(privateClusterConfig.privateEndpoint)') fi ACCESS_TOKEN=$(gcloud auth print-access-token) @@ -88,12 +89,13 @@ if MC_CA=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ --project="${MC_PROJECT}" \ --format='value(dnsConfig.clusterDns)' 2>/dev/null) - # Fallback to public IP endpoint if DNS endpoint is not configured + # Fallback to private endpoint if DNS endpoint is not configured + # Use private endpoint for cluster-to-cluster connectivity from Prow pod if [[ -z "${MC_ENDPOINT}" ]]; then MC_ENDPOINT=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${MC_PROJECT}" \ - --format='value(privateClusterConfig.publicEndpoint)' 2>/dev/null) + --format='value(privateClusterConfig.privateEndpoint)' 2>/dev/null) fi # Only create kubeconfig if we successfully got an endpoint From 1059c7c8f561dfefe75b3ae01f1d7e0ea4aebec2 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 23:42:41 -0400 Subject: [PATCH 21/44] Revert incorrect private endpoint change - need to research actual network topology --- .../gcp-hcp-generate-kubeconfigs-commands.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh index 27860f2011eec..fbc50f0c62923 100755 --- a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh @@ -41,13 +41,12 @@ REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ --project="${REGION_PROJECT}" \ --format='value(dnsConfig.clusterDns)') -# Fallback to private endpoint if DNS endpoint is not configured -# Use private endpoint for cluster-to-cluster connectivity from Prow pod +# Fallback to public IP endpoint if DNS endpoint is not configured if [[ -z "${REGION_ENDPOINT}" ]]; then REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${REGION_PROJECT}" \ - --format='value(privateClusterConfig.privateEndpoint)') + --format='value(privateClusterConfig.publicEndpoint)') fi ACCESS_TOKEN=$(gcloud auth print-access-token) @@ -89,13 +88,12 @@ if MC_CA=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ --project="${MC_PROJECT}" \ --format='value(dnsConfig.clusterDns)' 2>/dev/null) - # Fallback to private endpoint if DNS endpoint is not configured - # Use private endpoint for cluster-to-cluster connectivity from Prow pod + # Fallback to public IP endpoint if DNS endpoint is not configured if [[ -z "${MC_ENDPOINT}" ]]; then MC_ENDPOINT=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${MC_PROJECT}" \ - --format='value(privateClusterConfig.privateEndpoint)' 2>/dev/null) + --format='value(privateClusterConfig.publicEndpoint)' 2>/dev/null) fi # Only create kubeconfig if we successfully got an endpoint From 962f370e833f9b6fc34d9d86c1fb745d8310ab18 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 23:45:12 -0400 Subject: [PATCH 22/44] Use gcloud get-credentials with --dns-endpoint for kubeconfig generation Instead of manually constructing kubeconfigs, use gcloud's built-in get-credentials command with the --dns-endpoint flag. This matches the working manual connection method. Benefits: - Uses the same method that works for manual access - Automatically handles authentication and endpoint selection - Simpler and more maintainable code - gcloud handles DNS endpoint resolution correctly The --dns-endpoint flag tells gcloud to use the DNS-based cluster endpoint (e.g., clustername.p.googleapis.com) instead of IP addresses. Related: GCP-557 --- .../gcp-hcp-generate-kubeconfigs-commands.sh | 95 +++---------------- 1 file changed, 11 insertions(+), 84 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh index fbc50f0c62923..4c51ba0d89c9f 100755 --- a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh @@ -27,106 +27,33 @@ echo " MC Cluster: ${MC_CLUSTER_NAME}" echo " Region: ${REGION}" echo "" -# Generate region cluster kubeconfig with embedded token +# Generate region cluster kubeconfig using gcloud echo "Generating region cluster kubeconfig..." -set +x # Hide sensitive token -REGION_CA=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ +export KUBECONFIG="${SHARED_DIR}/region-kubeconfig" +gcloud container clusters get-credentials "${REGION_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${REGION_PROJECT}" \ - --format='value(masterAuth.clusterCaCertificate)') + --dns-endpoint -# Get DNS endpoint if available (falls back to public IP endpoint) -REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ - --region="${REGION}" \ - --project="${REGION_PROJECT}" \ - --format='value(dnsConfig.clusterDns)') - -# Fallback to public IP endpoint if DNS endpoint is not configured -if [[ -z "${REGION_ENDPOINT}" ]]; then - REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ - --region="${REGION}" \ - --project="${REGION_PROJECT}" \ - --format='value(privateClusterConfig.publicEndpoint)') -fi - -ACCESS_TOKEN=$(gcloud auth print-access-token) - -cat > "${SHARED_DIR}/region-kubeconfig" << EOF -apiVersion: v1 -kind: Config -clusters: -- cluster: - certificate-authority-data: ${REGION_CA} - server: https://${REGION_ENDPOINT} - name: region-cluster -contexts: -- context: - cluster: region-cluster - user: gcp-user - name: region-context -current-context: region-context -users: -- name: gcp-user - user: - token: ${ACCESS_TOKEN} -EOF - -echo " ✓ Region kubeconfig written" -set -x +echo " ✓ Region kubeconfig written to ${SHARED_DIR}/region-kubeconfig" # Generate MC cluster kubeconfig (optional - test skips if unavailable) echo "Generating management cluster kubeconfig..." -set +x # Hide sensitive token -if MC_CA=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ +if gcloud container clusters describe "${MC_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${MC_PROJECT}" \ - --format='value(masterAuth.clusterCaCertificate)' 2>/dev/null); then + --format='value(name)' &>/dev/null; then - # Get DNS endpoint if available (falls back to public IP endpoint) - MC_ENDPOINT=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ + export KUBECONFIG="${SHARED_DIR}/mc-kubeconfig" + gcloud container clusters get-credentials "${MC_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${MC_PROJECT}" \ - --format='value(dnsConfig.clusterDns)' 2>/dev/null) - - # Fallback to public IP endpoint if DNS endpoint is not configured - if [[ -z "${MC_ENDPOINT}" ]]; then - MC_ENDPOINT=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ - --region="${REGION}" \ - --project="${MC_PROJECT}" \ - --format='value(privateClusterConfig.publicEndpoint)' 2>/dev/null) - fi + --dns-endpoint - # Only create kubeconfig if we successfully got an endpoint - if [[ -n "${MC_ENDPOINT}" ]]; then - - cat > "${SHARED_DIR}/mc-kubeconfig" << EOF -apiVersion: v1 -kind: Config -clusters: -- cluster: - certificate-authority-data: ${MC_CA} - server: https://${MC_ENDPOINT} - name: mc-cluster -contexts: -- context: - cluster: mc-cluster - user: gcp-user - name: mc-context -current-context: mc-context -users: -- name: gcp-user - user: - token: ${ACCESS_TOKEN} -EOF - - echo " ✓ MC kubeconfig written" - else - echo " ⚠ MC endpoint unavailable - tests will skip MC validation" - fi + echo " ✓ MC kubeconfig written to ${SHARED_DIR}/mc-kubeconfig" else echo " ⚠ MC cluster unavailable - tests will skip MC validation" fi -set -x echo "" echo "✓ Kubeconfig generation completed" From 9d54061f8c0fdbb8356149005a6188201c9caaa5 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Tue, 18 Aug 2026 23:48:33 -0400 Subject: [PATCH 23/44] Revert "Use gcloud get-credentials with --dns-endpoint for kubeconfig generation" This reverts commit 962f370e833f9b6fc34d9d86c1fb745d8310ab18. --- .../gcp-hcp-generate-kubeconfigs-commands.sh | 95 ++++++++++++++++--- 1 file changed, 84 insertions(+), 11 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh index 4c51ba0d89c9f..fbc50f0c62923 100755 --- a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh @@ -27,33 +27,106 @@ echo " MC Cluster: ${MC_CLUSTER_NAME}" echo " Region: ${REGION}" echo "" -# Generate region cluster kubeconfig using gcloud +# Generate region cluster kubeconfig with embedded token echo "Generating region cluster kubeconfig..." -export KUBECONFIG="${SHARED_DIR}/region-kubeconfig" -gcloud container clusters get-credentials "${REGION_CLUSTER_NAME}" \ +set +x # Hide sensitive token +REGION_CA=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${REGION_PROJECT}" \ - --dns-endpoint + --format='value(masterAuth.clusterCaCertificate)') -echo " ✓ Region kubeconfig written to ${SHARED_DIR}/region-kubeconfig" +# Get DNS endpoint if available (falls back to public IP endpoint) +REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ + --region="${REGION}" \ + --project="${REGION_PROJECT}" \ + --format='value(dnsConfig.clusterDns)') + +# Fallback to public IP endpoint if DNS endpoint is not configured +if [[ -z "${REGION_ENDPOINT}" ]]; then + REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ + --region="${REGION}" \ + --project="${REGION_PROJECT}" \ + --format='value(privateClusterConfig.publicEndpoint)') +fi + +ACCESS_TOKEN=$(gcloud auth print-access-token) + +cat > "${SHARED_DIR}/region-kubeconfig" << EOF +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority-data: ${REGION_CA} + server: https://${REGION_ENDPOINT} + name: region-cluster +contexts: +- context: + cluster: region-cluster + user: gcp-user + name: region-context +current-context: region-context +users: +- name: gcp-user + user: + token: ${ACCESS_TOKEN} +EOF + +echo " ✓ Region kubeconfig written" +set -x # Generate MC cluster kubeconfig (optional - test skips if unavailable) echo "Generating management cluster kubeconfig..." -if gcloud container clusters describe "${MC_CLUSTER_NAME}" \ +set +x # Hide sensitive token +if MC_CA=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${MC_PROJECT}" \ - --format='value(name)' &>/dev/null; then + --format='value(masterAuth.clusterCaCertificate)' 2>/dev/null); then - export KUBECONFIG="${SHARED_DIR}/mc-kubeconfig" - gcloud container clusters get-credentials "${MC_CLUSTER_NAME}" \ + # Get DNS endpoint if available (falls back to public IP endpoint) + MC_ENDPOINT=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${MC_PROJECT}" \ - --dns-endpoint + --format='value(dnsConfig.clusterDns)' 2>/dev/null) + + # Fallback to public IP endpoint if DNS endpoint is not configured + if [[ -z "${MC_ENDPOINT}" ]]; then + MC_ENDPOINT=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ + --region="${REGION}" \ + --project="${MC_PROJECT}" \ + --format='value(privateClusterConfig.publicEndpoint)' 2>/dev/null) + fi - echo " ✓ MC kubeconfig written to ${SHARED_DIR}/mc-kubeconfig" + # Only create kubeconfig if we successfully got an endpoint + if [[ -n "${MC_ENDPOINT}" ]]; then + + cat > "${SHARED_DIR}/mc-kubeconfig" << EOF +apiVersion: v1 +kind: Config +clusters: +- cluster: + certificate-authority-data: ${MC_CA} + server: https://${MC_ENDPOINT} + name: mc-cluster +contexts: +- context: + cluster: mc-cluster + user: gcp-user + name: mc-context +current-context: mc-context +users: +- name: gcp-user + user: + token: ${ACCESS_TOKEN} +EOF + + echo " ✓ MC kubeconfig written" + else + echo " ⚠ MC endpoint unavailable - tests will skip MC validation" + fi else echo " ⚠ MC cluster unavailable - tests will skip MC validation" fi +set -x echo "" echo "✓ Kubeconfig generation completed" From 9da67fc6900ee40c5eb4f3bae0020ab4cdc4e8b6 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Wed, 19 Aug 2026 10:08:02 -0400 Subject: [PATCH 24/44] Use Connect Gateway for cluster access instead of public endpoint Connect Gateway allows secure access to GKE clusters without requiring public master endpoints. This works with private clusters and uses fleet membership (which is already configured). Endpoint format: https://{region}-connectgateway.googleapis.com/v1/projects/{projectNumber}/locations/{region}/gkeMemberships/{clusterName} Benefits: - Works with private master endpoints (no need for enable_private_endpoint=false) - Uses existing fleet enrollment - Same embedded access token pattern (valid for 1 hour) - More secure than public endpoints The kubeconfig uses an embedded access token from WIF which is valid for 1 hour - sufficient for CI jobs that run 30-60 minutes max. Related: GCP-557 --- .../gcp-hcp-generate-kubeconfigs-commands.sh | 90 +++++++++---------- 1 file changed, 42 insertions(+), 48 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh index fbc50f0c62923..14e99241e48f4 100755 --- a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -echo "=== GCP HCP Generate Kubeconfigs ===" +echo "=== GCP HCP Generate Kubeconfigs (Connect Gateway) ===" echo "" # Authenticate with WIF credential @@ -27,37 +27,33 @@ echo " MC Cluster: ${MC_CLUSTER_NAME}" echo " Region: ${REGION}" echo "" -# Generate region cluster kubeconfig with embedded token -echo "Generating region cluster kubeconfig..." +# Generate region cluster kubeconfig with Connect Gateway endpoint +echo "Generating region cluster kubeconfig (Connect Gateway)..." set +x # Hide sensitive token + +# Get project number for Connect Gateway URL +REGION_PROJECT_NUMBER=$(gcloud projects describe "${REGION_PROJECT}" --format='value(projectNumber)') + +# Get cluster CA certificate REGION_CA=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${REGION_PROJECT}" \ --format='value(masterAuth.clusterCaCertificate)') -# Get DNS endpoint if available (falls back to public IP endpoint) -REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ - --region="${REGION}" \ - --project="${REGION_PROJECT}" \ - --format='value(dnsConfig.clusterDns)') - -# Fallback to public IP endpoint if DNS endpoint is not configured -if [[ -z "${REGION_ENDPOINT}" ]]; then - REGION_ENDPOINT=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ - --region="${REGION}" \ - --project="${REGION_PROJECT}" \ - --format='value(privateClusterConfig.publicEndpoint)') -fi +# Build Connect Gateway endpoint +# Format: https://{region}-connectgateway.googleapis.com/v1/projects/{projectNumber}/locations/{region}/gkeMemberships/{clusterName} +REGION_ENDPOINT="${REGION}-connectgateway.googleapis.com/v1/projects/${REGION_PROJECT_NUMBER}/locations/${REGION}/gkeMemberships/${REGION_CLUSTER_NAME}" +# Get access token (valid for 1 hour - sufficient for CI jobs) ACCESS_TOKEN=$(gcloud auth print-access-token) -cat > "${SHARED_DIR}/region-kubeconfig" << EOF +cat > "${SHARED_DIR}/region-kubeconfig" << 'KUBECONFIG_EOF' apiVersion: v1 kind: Config clusters: - cluster: - certificate-authority-data: ${REGION_CA} - server: https://${REGION_ENDPOINT} + certificate-authority-data: REGION_CA_PLACEHOLDER + server: https://REGION_ENDPOINT_PLACEHOLDER name: region-cluster contexts: - context: @@ -68,44 +64,39 @@ current-context: region-context users: - name: gcp-user user: - token: ${ACCESS_TOKEN} -EOF + token: ACCESS_TOKEN_PLACEHOLDER +KUBECONFIG_EOF + +# Replace placeholders +sed -i "s|REGION_CA_PLACEHOLDER|${REGION_CA}|g" "${SHARED_DIR}/region-kubeconfig" +sed -i "s|REGION_ENDPOINT_PLACEHOLDER|${REGION_ENDPOINT}|g" "${SHARED_DIR}/region-kubeconfig" +sed -i "s|ACCESS_TOKEN_PLACEHOLDER|${ACCESS_TOKEN}|g" "${SHARED_DIR}/region-kubeconfig" -echo " ✓ Region kubeconfig written" +echo " ✓ Region kubeconfig written (Connect Gateway)" +echo " Endpoint: https://${REGION_ENDPOINT}" set -x # Generate MC cluster kubeconfig (optional - test skips if unavailable) -echo "Generating management cluster kubeconfig..." +echo "Generating management cluster kubeconfig (Connect Gateway)..." set +x # Hide sensitive token if MC_CA=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ --region="${REGION}" \ --project="${MC_PROJECT}" \ --format='value(masterAuth.clusterCaCertificate)' 2>/dev/null); then - # Get DNS endpoint if available (falls back to public IP endpoint) - MC_ENDPOINT=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ - --region="${REGION}" \ - --project="${MC_PROJECT}" \ - --format='value(dnsConfig.clusterDns)' 2>/dev/null) + # Get MC project number + MC_PROJECT_NUMBER=$(gcloud projects describe "${MC_PROJECT}" --format='value(projectNumber)') - # Fallback to public IP endpoint if DNS endpoint is not configured - if [[ -z "${MC_ENDPOINT}" ]]; then - MC_ENDPOINT=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ - --region="${REGION}" \ - --project="${MC_PROJECT}" \ - --format='value(privateClusterConfig.publicEndpoint)' 2>/dev/null) - fi + # Build Connect Gateway endpoint for MC + MC_ENDPOINT="${REGION}-connectgateway.googleapis.com/v1/projects/${MC_PROJECT_NUMBER}/locations/${REGION}/gkeMemberships/${MC_CLUSTER_NAME}" - # Only create kubeconfig if we successfully got an endpoint - if [[ -n "${MC_ENDPOINT}" ]]; then - - cat > "${SHARED_DIR}/mc-kubeconfig" << EOF + cat > "${SHARED_DIR}/mc-kubeconfig" << 'KUBECONFIG_EOF' apiVersion: v1 kind: Config clusters: - cluster: - certificate-authority-data: ${MC_CA} - server: https://${MC_ENDPOINT} + certificate-authority-data: MC_CA_PLACEHOLDER + server: https://MC_ENDPOINT_PLACEHOLDER name: mc-cluster contexts: - context: @@ -116,17 +107,20 @@ current-context: mc-context users: - name: gcp-user user: - token: ${ACCESS_TOKEN} -EOF + token: ACCESS_TOKEN_PLACEHOLDER +KUBECONFIG_EOF + + # Replace placeholders + sed -i "s|MC_CA_PLACEHOLDER|${MC_CA}|g" "${SHARED_DIR}/mc-kubeconfig" + sed -i "s|MC_ENDPOINT_PLACEHOLDER|${MC_ENDPOINT}|g" "${SHARED_DIR}/mc-kubeconfig" + sed -i "s|ACCESS_TOKEN_PLACEHOLDER|${ACCESS_TOKEN}|g" "${SHARED_DIR}/mc-kubeconfig" - echo " ✓ MC kubeconfig written" - else - echo " ⚠ MC endpoint unavailable - tests will skip MC validation" - fi + echo " ✓ MC kubeconfig written (Connect Gateway)" + echo " Endpoint: https://${MC_ENDPOINT}" else echo " ⚠ MC cluster unavailable - tests will skip MC validation" fi set -x echo "" -echo "✓ Kubeconfig generation completed" +echo "✓ Kubeconfig generation completed (Connect Gateway)" From 0e925ea507b75018b3bbc0b06410dc8a0f32ecd2 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Wed, 19 Aug 2026 12:55:43 -0400 Subject: [PATCH 25/44] Fix Connect Gateway TLS: remove cluster CA from kubeconfig Connect Gateway (connectgateway.googleapis.com) uses Google's public TLS certificates signed by Google Trust Services, not the cluster's self-signed CA. Including certificate-authority-data caused x509 validation failure: tls: failed to verify certificate: x509: certificate signed by unknown authority Fix: omit certificate-authority-data entirely and let the system CA bundle validate Google's public certs. Also removed the now-unnecessary gcloud container clusters describe calls that were only used to fetch the CA certificate. Related: GCP-557 --- .../gcp-hcp-generate-kubeconfigs-commands.sh | 33 ++++++------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh index 14e99241e48f4..6ebec4473e686 100755 --- a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh @@ -34,12 +34,6 @@ set +x # Hide sensitive token # Get project number for Connect Gateway URL REGION_PROJECT_NUMBER=$(gcloud projects describe "${REGION_PROJECT}" --format='value(projectNumber)') -# Get cluster CA certificate -REGION_CA=$(gcloud container clusters describe "${REGION_CLUSTER_NAME}" \ - --region="${REGION}" \ - --project="${REGION_PROJECT}" \ - --format='value(masterAuth.clusterCaCertificate)') - # Build Connect Gateway endpoint # Format: https://{region}-connectgateway.googleapis.com/v1/projects/{projectNumber}/locations/{region}/gkeMemberships/{clusterName} REGION_ENDPOINT="${REGION}-connectgateway.googleapis.com/v1/projects/${REGION_PROJECT_NUMBER}/locations/${REGION}/gkeMemberships/${REGION_CLUSTER_NAME}" @@ -47,12 +41,14 @@ REGION_ENDPOINT="${REGION}-connectgateway.googleapis.com/v1/projects/${REGION_PR # Get access token (valid for 1 hour - sufficient for CI jobs) ACCESS_TOKEN=$(gcloud auth print-access-token) +# Connect Gateway uses Google's public TLS certificates (Google Trust Services), +# NOT the cluster's self-signed CA. No certificate-authority-data needed — +# the system CA bundle validates Google's certs automatically. cat > "${SHARED_DIR}/region-kubeconfig" << 'KUBECONFIG_EOF' apiVersion: v1 kind: Config clusters: - cluster: - certificate-authority-data: REGION_CA_PLACEHOLDER server: https://REGION_ENDPOINT_PLACEHOLDER name: region-cluster contexts: @@ -68,34 +64,26 @@ users: KUBECONFIG_EOF # Replace placeholders -sed -i "s|REGION_CA_PLACEHOLDER|${REGION_CA}|g" "${SHARED_DIR}/region-kubeconfig" sed -i "s|REGION_ENDPOINT_PLACEHOLDER|${REGION_ENDPOINT}|g" "${SHARED_DIR}/region-kubeconfig" sed -i "s|ACCESS_TOKEN_PLACEHOLDER|${ACCESS_TOKEN}|g" "${SHARED_DIR}/region-kubeconfig" -echo " ✓ Region kubeconfig written (Connect Gateway)" +echo " Region kubeconfig written (Connect Gateway)" echo " Endpoint: https://${REGION_ENDPOINT}" set -x # Generate MC cluster kubeconfig (optional - test skips if unavailable) echo "Generating management cluster kubeconfig (Connect Gateway)..." set +x # Hide sensitive token -if MC_CA=$(gcloud container clusters describe "${MC_CLUSTER_NAME}" \ - --region="${REGION}" \ - --project="${MC_PROJECT}" \ - --format='value(masterAuth.clusterCaCertificate)' 2>/dev/null); then - - # Get MC project number - MC_PROJECT_NUMBER=$(gcloud projects describe "${MC_PROJECT}" --format='value(projectNumber)') - +if MC_PROJECT_NUMBER=$(gcloud projects describe "${MC_PROJECT}" --format='value(projectNumber)' 2>/dev/null); then + # Build Connect Gateway endpoint for MC MC_ENDPOINT="${REGION}-connectgateway.googleapis.com/v1/projects/${MC_PROJECT_NUMBER}/locations/${REGION}/gkeMemberships/${MC_CLUSTER_NAME}" - + cat > "${SHARED_DIR}/mc-kubeconfig" << 'KUBECONFIG_EOF' apiVersion: v1 kind: Config clusters: - cluster: - certificate-authority-data: MC_CA_PLACEHOLDER server: https://MC_ENDPOINT_PLACEHOLDER name: mc-cluster contexts: @@ -111,16 +99,15 @@ users: KUBECONFIG_EOF # Replace placeholders - sed -i "s|MC_CA_PLACEHOLDER|${MC_CA}|g" "${SHARED_DIR}/mc-kubeconfig" sed -i "s|MC_ENDPOINT_PLACEHOLDER|${MC_ENDPOINT}|g" "${SHARED_DIR}/mc-kubeconfig" sed -i "s|ACCESS_TOKEN_PLACEHOLDER|${ACCESS_TOKEN}|g" "${SHARED_DIR}/mc-kubeconfig" - echo " ✓ MC kubeconfig written (Connect Gateway)" + echo " MC kubeconfig written (Connect Gateway)" echo " Endpoint: https://${MC_ENDPOINT}" else - echo " ⚠ MC cluster unavailable - tests will skip MC validation" + echo " MC cluster unavailable - tests will skip MC validation" fi set -x echo "" -echo "✓ Kubeconfig generation completed (Connect Gateway)" +echo "Kubeconfig generation completed (Connect Gateway)" From 6c6079d618c2d550ef9f90a088554f0bccc85feb Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Wed, 19 Aug 2026 14:02:55 -0400 Subject: [PATCH 26/44] Workaround Firestore provider bug: import orphaned databases on 409 hashicorp/terraform-provider-google#22533: on freshly created projects, google_firestore_database has a read-after-write consistency bug. The provider creates the database successfully, but the immediate GET returns 404 (~80ms after creation), so the resource is dropped from state. The next apply attempt fails with 409 'Database already exists' because the database exists in GCP but not in state. Fix: detect the 409 pattern in apply output, extract the MC project ID from terraform state/outputs, and terraform import the orphaned database resources before retrying. This restores state consistency so the retry succeeds. Affects ~60% of freshly created projects per upstream bug report. Related: GCP-557 --- .../gcp-hcp-tf-provision-commands.sh | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh b/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh index c30ac4d077445..61ba27b0a80ff 100755 --- a/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh @@ -187,6 +187,66 @@ log "TFC workspace: https://app.terraform.io/app/${TFC_ORG}/workspaces/${WORKSPA # Errors that retrying cannot fix — fail fast instead of wasting time NON_TRANSIENT_ERRORS="quota.*exceeded|forbidden|invalid.*configuration|unauthorized" +# Workaround for hashicorp/terraform-provider-google#22533: +# google_firestore_database has a read-after-write consistency bug on newly +# created projects. The provider creates the database, but the immediate GET +# returns 404, so the resource is dropped from state. The next apply attempt +# then fails with 409 "already exists" because the database is in GCP but not +# in state. Fix: detect the 409, extract the project ID, and import the +# orphaned resources before retrying. +import_orphaned_firestore() { + local output="$1" + + # Check for the Firestore 409 pattern + if ! echo "${output}" | grep -q "google_firestore_database.*Database already exists"; then + return 1 + fi + + log "Detected Firestore provider bug (hashicorp/terraform-provider-google#22533)" + log "Database exists in GCP but not in state — attempting import..." + + # Extract MC project ID from terraform state (the project resource is + # created before Firestore, so it should be in state) + local mc_project + mc_project=$(terraform output -json 2>/dev/null | jq -r '.management_cluster.value.project_id // empty' 2>/dev/null || echo "") + + if [[ -z "${mc_project}" ]]; then + log "WARNING: Could not extract MC project ID from outputs, trying state..." + mc_project=$(terraform show -json 2>/dev/null | \ + jq -r '.. | objects | select(.address? == "module.management_cluster.module.project.google_project.main") | .values.project_id // empty' 2>/dev/null || echo "") + fi + + if [[ -z "${mc_project}" ]]; then + log "ERROR: Could not determine MC project ID for import" + return 1 + fi + + log "MC Project ID: ${mc_project}" + + local imported=0 + for db_name in specs status; do + local address="module.management_cluster.google_firestore_database.${db_name}" + local import_id="projects/${mc_project}/databases/${db_name}" + + # Only import if the error mentions this specific database + if echo "${output}" | grep -q "google_firestore_database.${db_name}"; then + log "Importing ${address} <- ${import_id}" + if terraform import -no-color "${address}" "${import_id}" 2>&1 | tee -a "${LOG}"; then + log "Successfully imported ${db_name} database" + imported=$((imported + 1)) + else + log "WARNING: Failed to import ${db_name} database" + fi + fi + done + + if [[ ${imported} -gt 0 ]]; then + log "Imported ${imported} orphaned Firestore database(s)" + return 0 + fi + return 1 +} + MAX_APPLY_ATTEMPTS=5 apply_attempt=1 apply_wait=30 @@ -217,6 +277,12 @@ while (( apply_attempt <= MAX_APPLY_ATTEMPTS )); do exit 1 fi + # Handle Firestore provider bug: import orphaned databases before retry + if import_orphaned_firestore "${apply_output}"; then + log "Firestore import succeeded — retrying apply immediately" + apply_wait=10 # Short wait after import + fi + if (( apply_attempt < MAX_APPLY_ATTEMPTS )); then log "Transient failure — waiting ${apply_wait}s before retry..." log "This is common due to GCP eventual consistency (IAM propagation, API enablement)" From e6ecbf09874b5140ba38c5f94a33151edfff2288 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Wed, 19 Aug 2026 21:36:12 -0400 Subject: [PATCH 27/44] Fix MC Connect Gateway: use region project for fleet membership MC clusters are registered in the region project's fleet, not their own project's fleet. The Connect Gateway URL must reference the project that owns the fleet membership. Before: projects/{MC_PROJECT_NUMBER}/locations/.../gkeMemberships/... After: projects/{REGION_PROJECT_NUMBER}/locations/.../gkeMemberships/... This fixes the 403 Forbidden error when accessing MC clusters via Connect Gateway. The region project number is already available from the region kubeconfig generation earlier in the script. Related: GCP-557 --- .../gcp-hcp-generate-kubeconfigs-commands.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh index 6ebec4473e686..daeb52d8af75b 100755 --- a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh @@ -74,10 +74,11 @@ set -x # Generate MC cluster kubeconfig (optional - test skips if unavailable) echo "Generating management cluster kubeconfig (Connect Gateway)..." set +x # Hide sensitive token -if MC_PROJECT_NUMBER=$(gcloud projects describe "${MC_PROJECT}" --format='value(projectNumber)' 2>/dev/null); then +if gcloud projects describe "${MC_PROJECT}" --format='value(projectNumber)' &>/dev/null; then - # Build Connect Gateway endpoint for MC - MC_ENDPOINT="${REGION}-connectgateway.googleapis.com/v1/projects/${MC_PROJECT_NUMBER}/locations/${REGION}/gkeMemberships/${MC_CLUSTER_NAME}" + # MC clusters are registered in the REGION project's fleet (not their own). + # Connect Gateway URLs must reference the project that owns the fleet membership. + MC_ENDPOINT="${REGION}-connectgateway.googleapis.com/v1/projects/${REGION_PROJECT_NUMBER}/locations/${REGION}/gkeMemberships/${MC_CLUSTER_NAME}" cat > "${SHARED_DIR}/mc-kubeconfig" << 'KUBECONFIG_EOF' apiVersion: v1 From a2b653c60d214e5a13f0aab1d7e94f0f88981d1a Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Wed, 19 Aug 2026 22:39:49 -0400 Subject: [PATCH 28/44] Increase ArgoCD sync validation timeouts - Step timeout: 60m -> 90m - Ginkgo suite timeout: add --ginkgo.timeout=85m (5m grace before step kill) Initial conservative timeouts to establish a baseline for how long ArgoCD app sync takes in e2e. Can be scaled back once we know what stable looks like. Individual per-app sync timeouts (45m) are configured in gcp-hcp-infra test code via env vars. Related: GCP-557 --- .../verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh | 1 + .../verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh index 5c65b5703c53d..8f1288497fd97 100755 --- a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh @@ -31,6 +31,7 @@ echo "Running ArgoCD sync validation tests..." /usr/bin/test-e2e \ --ginkgo.v \ --ginkgo.no-color \ + --ginkgo.timeout=85m \ --ginkgo.junit-report="${ARTIFACT_DIR}/junit_argocd_sync.xml" \ --ginkgo.label-filter="argocd" diff --git a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml index ee7c4042cac23..d42fc83129ee6 100644 --- a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml +++ b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml @@ -6,7 +6,7 @@ ref: requests: cpu: 500m memory: 1Gi - timeout: 60m0s + timeout: 90m0s documentation: |- Validates that ArgoCD applications have synced successfully on the region and management clusters after infrastructure provisioning. From 7347008e5a18c8c3fa304dbaa1d1745a12e4cc5b Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Thu, 20 Aug 2026 11:08:06 -0400 Subject: [PATCH 29/44] Add pre-destroy-cleanup step to remove resources blocking terraform destroy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ArgoCD-deployed applications create GCP resources that terraform does not manage and cannot delete: - Gateway API resources (HTTPRoute, Gateway) trigger GKE to create Network Endpoint Groups (NEGs) that hold VPC references - external-dns creates DNS records in regional zones - These block terraform destroy with 'network in use' and 'zone not empty' errors The pre-destroy-cleanup step runs before tf-deprovision and: 1. Stops ArgoCD (scale down, delete all Applications/ApplicationSets) 2. Deletes Gateway API resources to trigger GKE NEG cleanup 3. Deletes DNS records from regional managed zones 4. Force-deletes remaining NEGs via gcloud Uses best_effort: true so cleanup failures don't block the job. Uses Connect Gateway with fresh WIF token (original may have expired). Does NOT delete resources terraform handles (IAM, KCC resources, Spanner — these cascade with project deletion). Related: GCP-557 --- .../gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml | 3 +- .../gcp-hcp/pre-destroy-cleanup/OWNERS | 18 ++ .../gcp-hcp-pre-destroy-cleanup-commands.sh | 254 ++++++++++++++++++ .../gcp-hcp-pre-destroy-cleanup-ref.yaml | 37 +++ 4 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/OWNERS create mode 100644 ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh create mode 100644 ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.yaml diff --git a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml index 8c770d44dde11..9d47211ee872d 100644 --- a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml +++ b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml @@ -8,6 +8,7 @@ workflow: test: - ref: gcp-hcp-verify-argocd-sync post: + - ref: gcp-hcp-pre-destroy-cleanup - ref: gcp-hcp-tf-deprovision documentation: |- E2E workflow for gcp-hcp-infra. Provisions full platform infrastructure @@ -17,4 +18,4 @@ workflow: Phase breakdown: - Pre: Authenticate via WIF, provision infrastructure with terraform apply, generate kubeconfigs - Test: Validate ArgoCD application sync on region and management clusters using Ginkgo v2 tests - - Post: Clean up resources with terraform destroy (best-effort) + - Post: Clean up ArgoCD-managed GCP resources (NEGs, DNS, Gateways), then terraform destroy diff --git a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/OWNERS b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/OWNERS new file mode 100644 index 0000000000000..60325af98b50d --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/OWNERS @@ -0,0 +1,18 @@ +approvers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm +reviewers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm diff --git a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh new file mode 100644 index 0000000000000..7f55771a96497 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "=== GCP HCP Pre-Destroy Cleanup ===" +echo "" +echo "Removes GCP resources created by ArgoCD-deployed apps that block" +echo "terraform destroy: NEGs (block VPC deletion), DNS records (block" +echo "zone deletion), and Gateway API resources (create NEGs)." +echo "" + +# Authenticate with WIF credential +if [[ ! -f "${SHARED_DIR}/wif-cred.json" ]]; then + echo "WARNING: WIF credential not found, skipping cleanup" + exit 0 +fi +gcloud auth login --cred-file="${SHARED_DIR}/wif-cred.json" --quiet + +# Read cluster info from SHARED_DIR +REGION_PROJECT=$(<"${SHARED_DIR}/region-project-id") +REGION_CLUSTER_NAME=$(<"${SHARED_DIR}/region-cluster-name") +MC_PROJECT=$(<"${SHARED_DIR}/mc-project-id") +MC_CLUSTER_NAME=$(<"${SHARED_DIR}/mc-cluster-name") +REGION=${GCP_REGION:-us-central1} + +# Get project numbers for Connect Gateway +REGION_PROJECT_NUMBER=$(gcloud projects describe "${REGION_PROJECT}" --format='value(projectNumber)' 2>/dev/null || echo "") + +echo "Region: ${REGION_PROJECT} / ${REGION_CLUSTER_NAME}" +echo "MC: ${MC_PROJECT} / ${MC_CLUSTER_NAME}" +echo "" + +# Build kubeconfigs using Connect Gateway (fresh token — original may have expired) +build_kubeconfig() { + local project_number=$1 + local cluster_name=$2 + local output_path=$3 + + local endpoint="${REGION}-connectgateway.googleapis.com/v1/projects/${project_number}/locations/${REGION}/gkeMemberships/${cluster_name}" + local token + token=$(gcloud auth print-access-token) + + cat > "${output_path}" </dev/null; then + echo " ArgoCD namespace not found, skipping" + return 0 + fi + + # Scale down ArgoCD + kc "${kubeconfig}" -n argocd scale deployment --all --replicas=0 2>/dev/null || true + kc "${kubeconfig}" -n argocd scale statefulset --all --replicas=0 2>/dev/null || true + + # Delete all Applications and ApplicationSets (stops app-of-apps) + kc "${kubeconfig}" -n argocd delete applicationset --all --wait=false --timeout=30s 2>/dev/null || true + kc "${kubeconfig}" -n argocd delete application --all --wait=false --timeout=30s 2>/dev/null || true + + # Wait briefly for deletions to propagate + local elapsed=0 + while [[ ${elapsed} -lt 60 ]]; do + local count + count=$(kc "${kubeconfig}" -n argocd get applications --no-headers 2>/dev/null | wc -l || echo "0") + count=$((count + 0)) + if [[ ${count} -eq 0 ]]; then + echo " All Applications deleted" + break + fi + echo " Waiting for ${count} Application(s) to delete... (${elapsed}s)" + sleep 10 + elapsed=$((elapsed + 10)) + done +} + +# ===================================================================== +# Phase 2: Delete Gateway API resources (triggers GKE NEG cleanup) +# ===================================================================== +delete_gateway_resources() { + local kubeconfig=$1 + local label=$2 + + echo "--- [${label}] Deleting Gateway API resources ---" + + for kind in gcpbackendpolicy healthcheckpolicy httproute gateway; do + local count + count=$(kc "${kubeconfig}" get "${kind}" --all-namespaces --no-headers 2>/dev/null | wc -l || echo "0") + count=$((count + 0)) + if [[ ${count} -gt 0 ]]; then + echo " Deleting ${count} ${kind} resource(s)" + kc "${kubeconfig}" delete "${kind}" --all --all-namespaces --wait=false --timeout=30s 2>/dev/null || true + fi + done +} + +# ===================================================================== +# Phase 3: Delete DNS records from regional zones +# External-dns creates records in the tools zone that block zone deletion +# ===================================================================== +delete_dns_records() { + local project=$1 + local label=$2 + + echo "--- [${label}] Cleaning DNS records in ${project} ---" + + local zones + zones=$(gcloud dns managed-zones list --project="${project}" --format="value(name)" 2>/dev/null || echo "") + + if [[ -z "${zones}" ]]; then + echo " No DNS zones found" + return 0 + fi + + while IFS= read -r zone; do + [[ -z "${zone}" ]] && continue + echo " Zone: ${zone}" + + local records + records=$(gcloud dns record-sets list \ + --project="${project}" \ + --zone="${zone}" \ + --format="csv[no-heading](name,type)" 2>/dev/null || echo "") + + while IFS=',' read -r name type; do + [[ -z "${name}" || -z "${type}" ]] && continue + [[ "${type}" == "SOA" || "${type}" == "NS" ]] && continue + echo " Deleting ${type} record: ${name}" + gcloud dns record-sets delete "${name}" \ + --zone="${zone}" --project="${project}" --type="${type}" \ + --quiet 2>/dev/null || true + done <<< "${records}" + done <<< "${zones}" +} + +# ===================================================================== +# Phase 4: Force-delete remaining NEGs via gcloud +# GKE should clean up NEGs after Gateway deletion, but sometimes +# orphaned NEGs remain and block VPC network deletion. +# ===================================================================== +delete_negs() { + local project=$1 + local label=$2 + + echo "--- [${label}] Cleaning up NEGs in ${project} ---" + + # Zonal NEGs + local negs + negs=$(gcloud compute network-endpoint-groups list \ + --project="${project}" \ + --format="csv[no-heading](name,zone)" 2>/dev/null || echo "") + + if [[ -z "${negs}" ]]; then + echo " No NEGs found" + return 0 + fi + + while IFS=',' read -r name zone_url; do + [[ -z "${name}" || -z "${zone_url}" ]] && continue + # zone_url is a full URL — extract just the zone name + local zone + zone=$(basename "${zone_url}") + echo " Deleting NEG: ${name} (zone: ${zone})" + gcloud compute network-endpoint-groups delete "${name}" \ + --project="${project}" --zone="${zone}" \ + --quiet 2>/dev/null || true + done <<< "${negs}" +} + +# ===================================================================== +# Execute cleanup on both clusters +# ===================================================================== + +# Build fresh kubeconfigs (tokens from generate-kubeconfigs may have expired) +CLEANUP_REGION_KC="/tmp/cleanup-region-kubeconfig" +CLEANUP_MC_KC="/tmp/cleanup-mc-kubeconfig" + +if [[ -n "${REGION_PROJECT_NUMBER}" ]]; then + build_kubeconfig "${REGION_PROJECT_NUMBER}" "${REGION_CLUSTER_NAME}" "${CLEANUP_REGION_KC}" + + # MC is registered in region's fleet + build_kubeconfig "${REGION_PROJECT_NUMBER}" "${MC_CLUSTER_NAME}" "${CLEANUP_MC_KC}" +fi + +# Phase 1: Stop ArgoCD (MC first, then region) +if [[ -f "${CLEANUP_MC_KC}" ]]; then + stop_argocd "${CLEANUP_MC_KC}" "MC" || true +fi +if [[ -f "${CLEANUP_REGION_KC}" ]]; then + stop_argocd "${CLEANUP_REGION_KC}" "Region" || true +fi + +# Phase 2: Delete Gateway API resources +if [[ -f "${CLEANUP_MC_KC}" ]]; then + delete_gateway_resources "${CLEANUP_MC_KC}" "MC" || true +fi +if [[ -f "${CLEANUP_REGION_KC}" ]]; then + delete_gateway_resources "${CLEANUP_REGION_KC}" "Region" || true +fi + +# Wait for GKE controller to clean up NEGs after Gateway deletion +echo "" +echo "Waiting 60s for GKE to process Gateway/NEG deletions..." +sleep 60 + +# Phase 3: Delete DNS records from regional zones +delete_dns_records "${REGION_PROJECT}" "Region" || true + +# Phase 4: Force-delete remaining NEGs +delete_negs "${REGION_PROJECT}" "Region" || true +delete_negs "${MC_PROJECT}" "MC" || true + +# Final NEG check +echo "" +echo "=== Final NEG check ===" +echo "Region:" +gcloud compute network-endpoint-groups list --project="${REGION_PROJECT}" 2>/dev/null || echo " Unable to list" +echo "MC:" +gcloud compute network-endpoint-groups list --project="${MC_PROJECT}" 2>/dev/null || echo " Unable to list" + +echo "" +echo "=== Pre-destroy cleanup complete ===" diff --git a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.yaml b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.yaml new file mode 100644 index 0000000000000..5e12a490426fd --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.yaml @@ -0,0 +1,37 @@ +ref: + as: gcp-hcp-pre-destroy-cleanup + from_image: + namespace: ocp + name: "4.22" + tag: upi-installer + commands: gcp-hcp-pre-destroy-cleanup-commands.sh + env: + - name: GCP_REGION + default: "us-central1" + documentation: "GCP region for e2e infrastructure deployment" + resources: + requests: + cpu: 100m + memory: 256Mi + timeout: 10m0s + best_effort: true + documentation: |- + Cleans up GCP resources created by ArgoCD-deployed applications that + block terraform destroy. Runs in the post phase before tf-deprovision. + + Cleanup phases: + 1. Stop ArgoCD (scale down, delete Applications/ApplicationSets) + 2. Delete Gateway API resources (HTTPRoute, Gateway, etc.) to trigger + GKE NEG cleanup + 3. Delete DNS records from regional zones (external-dns creates records + that block zone deletion) + 4. Force-delete remaining NEGs via gcloud (block VPC network deletion) + + Does NOT delete resources that terraform destroy handles (IAM, KCC + resources, Spanner databases, etc. — these cascade with project deletion). + + Uses best_effort: true so cleanup failures don't block the job. + + Depends on: + - hypershift-gcp-wif-auth (WIF credential in SHARED_DIR) + - gcp-hcp-tf-provision (cluster names/project IDs in SHARED_DIR) From e36e3d881b1663c6fc59cee02b231dd9921c252f Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Thu, 20 Aug 2026 16:12:04 -0400 Subject: [PATCH 30/44] Fix pre-destroy-cleanup: bash arithmetic bug and connectivity checks 1. Fix 'syntax error in expression' from wc -l whitespace in arithmetic. Added safe_count() helper to strip whitespace before evaluation. 2. Add connectivity verification before attempting kubectl operations. Clusters may be unreachable if the access token expired or the cluster is already being torn down. Skip kubectl phases gracefully. 3. Improve 'Unable to list' error messages to indicate the project may be in deletion state. Related: GCP-557 --- .../gcp-hcp-pre-destroy-cleanup-commands.sh | 62 ++++++++++++++----- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh index 7f55771a96497..3ab1c52e74f2d 100644 --- a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh @@ -8,7 +8,7 @@ echo "terraform destroy: NEGs (block VPC deletion), DNS records (block" echo "zone deletion), and Gateway API resources (create NEGs)." echo "" -# Authenticate with WIF credential +# Authenticate with WIF credential (fresh token for this step) if [[ ! -f "${SHARED_DIR}/wif-cred.json" ]]; then echo "WARNING: WIF credential not found, skipping cleanup" exit 0 @@ -29,7 +29,18 @@ echo "Region: ${REGION_PROJECT} / ${REGION_CLUSTER_NAME}" echo "MC: ${MC_PROJECT} / ${MC_CLUSTER_NAME}" echo "" -# Build kubeconfigs using Connect Gateway (fresh token — original may have expired) +# Safe count: strip whitespace from wc -l output for arithmetic +safe_count() { + local result + result=$(echo "$1" | tr -d '[:space:]') + result=${result:-0} + echo "${result}" +} + +# Build kubeconfigs using Connect Gateway with a FRESH access token. +# The original kubeconfigs from generate-kubeconfigs embed a static token +# that expires after 1 hour. This step runs in the post phase, potentially +# 1+ hours after the token was generated. build_kubeconfig() { local project_number=$1 local cluster_name=$2 @@ -94,8 +105,7 @@ stop_argocd() { local elapsed=0 while [[ ${elapsed} -lt 60 ]]; do local count - count=$(kc "${kubeconfig}" -n argocd get applications --no-headers 2>/dev/null | wc -l || echo "0") - count=$((count + 0)) + count=$(safe_count "$(kc "${kubeconfig}" -n argocd get applications --no-headers 2>/dev/null | wc -l)") if [[ ${count} -eq 0 ]]; then echo " All Applications deleted" break @@ -117,8 +127,7 @@ delete_gateway_resources() { for kind in gcpbackendpolicy healthcheckpolicy httproute gateway; do local count - count=$(kc "${kubeconfig}" get "${kind}" --all-namespaces --no-headers 2>/dev/null | wc -l || echo "0") - count=$((count + 0)) + count=$(safe_count "$(kc "${kubeconfig}" get "${kind}" --all-namespaces --no-headers 2>/dev/null | wc -l)") if [[ ${count} -gt 0 ]]; then echo " Deleting ${count} ${kind} resource(s)" kc "${kubeconfig}" delete "${kind}" --all --all-namespaces --wait=false --timeout=30s 2>/dev/null || true @@ -203,7 +212,7 @@ delete_negs() { # Execute cleanup on both clusters # ===================================================================== -# Build fresh kubeconfigs (tokens from generate-kubeconfigs may have expired) +# Build fresh kubeconfigs with new access tokens CLEANUP_REGION_KC="/tmp/cleanup-region-kubeconfig" CLEANUP_MC_KC="/tmp/cleanup-mc-kubeconfig" @@ -214,26 +223,47 @@ if [[ -n "${REGION_PROJECT_NUMBER}" ]]; then build_kubeconfig "${REGION_PROJECT_NUMBER}" "${MC_CLUSTER_NAME}" "${CLEANUP_MC_KC}" fi +# Verify connectivity before proceeding +echo "Verifying cluster connectivity..." +if [[ -f "${CLEANUP_REGION_KC}" ]] && kc "${CLEANUP_REGION_KC}" get nodes --request-timeout=10s &>/dev/null; then + echo " Region cluster: connected" + REGION_CONNECTED=true +else + echo " Region cluster: unreachable (Connect Gateway or cluster may be unavailable)" + REGION_CONNECTED=false +fi + +if [[ -f "${CLEANUP_MC_KC}" ]] && kc "${CLEANUP_MC_KC}" get nodes --request-timeout=10s &>/dev/null; then + echo " MC cluster: connected" + MC_CONNECTED=true +else + echo " MC cluster: unreachable (Connect Gateway or cluster may be unavailable)" + MC_CONNECTED=false +fi +echo "" + # Phase 1: Stop ArgoCD (MC first, then region) -if [[ -f "${CLEANUP_MC_KC}" ]]; then +if [[ "${MC_CONNECTED}" == "true" ]]; then stop_argocd "${CLEANUP_MC_KC}" "MC" || true fi -if [[ -f "${CLEANUP_REGION_KC}" ]]; then +if [[ "${REGION_CONNECTED}" == "true" ]]; then stop_argocd "${CLEANUP_REGION_KC}" "Region" || true fi # Phase 2: Delete Gateway API resources -if [[ -f "${CLEANUP_MC_KC}" ]]; then +if [[ "${MC_CONNECTED}" == "true" ]]; then delete_gateway_resources "${CLEANUP_MC_KC}" "MC" || true fi -if [[ -f "${CLEANUP_REGION_KC}" ]]; then +if [[ "${REGION_CONNECTED}" == "true" ]]; then delete_gateway_resources "${CLEANUP_REGION_KC}" "Region" || true fi # Wait for GKE controller to clean up NEGs after Gateway deletion -echo "" -echo "Waiting 60s for GKE to process Gateway/NEG deletions..." -sleep 60 +if [[ "${REGION_CONNECTED}" == "true" || "${MC_CONNECTED}" == "true" ]]; then + echo "" + echo "Waiting 60s for GKE to process Gateway/NEG deletions..." + sleep 60 +fi # Phase 3: Delete DNS records from regional zones delete_dns_records "${REGION_PROJECT}" "Region" || true @@ -246,9 +276,9 @@ delete_negs "${MC_PROJECT}" "MC" || true echo "" echo "=== Final NEG check ===" echo "Region:" -gcloud compute network-endpoint-groups list --project="${REGION_PROJECT}" 2>/dev/null || echo " Unable to list" +gcloud compute network-endpoint-groups list --project="${REGION_PROJECT}" --format="table(name,zone)" 2>/dev/null || echo " (unable to list — project may be in deletion)" echo "MC:" -gcloud compute network-endpoint-groups list --project="${MC_PROJECT}" 2>/dev/null || echo " Unable to list" +gcloud compute network-endpoint-groups list --project="${MC_PROJECT}" --format="table(name,zone)" 2>/dev/null || echo " (unable to list — project may be in deletion)" echo "" echo "=== Pre-destroy cleanup complete ===" From 014224bd11cb4f5e2e140d1bb7cada4d4c92fda9 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Thu, 20 Aug 2026 17:01:06 -0400 Subject: [PATCH 31/44] Fix CI checks: add metadata.json, disable sparse checkout, add cluster assignments - Add missing gcp-hcp-pre-destroy-cleanup-ref.metadata.json - Add disable_sparse_checkout: true to ci-operator config (the Dockerfile needs the full test/ directory, not just test/Dockerfile.e2e) - Remove prowgen-generated sparse_checkout_files from job configs - Add cluster assignments (build04 postsubmit, build06 presubmit images) Related: GCP-557 --- .../openshift-online-gcp-hcp-infra-main.yaml | 1 + ...online-gcp-hcp-infra-main-postsubmits.yaml | 5 +--- ...-online-gcp-hcp-infra-main-presubmits.yaml | 13 +--------- ...-hcp-pre-destroy-cleanup-ref.metadata.json | 25 +++++++++++++++++++ 4 files changed, 28 insertions(+), 16 deletions(-) create mode 100644 ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.metadata.json diff --git a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml index 4d1cb1862aed1..e349510905e58 100644 --- a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml +++ b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml @@ -5,6 +5,7 @@ build_root: namespace: ci tag: latest use_build_cache: true +disable_sparse_checkout: true images: items: - context_dir: test diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml index ded809fb18356..530e805564ffa 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml @@ -10,8 +10,6 @@ postsubmits: oauth_token_secret: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner - sparse_checkout_files: - - test/ labels: ci.openshift.io/generator: prowgen max_concurrency: 1 @@ -71,13 +69,12 @@ postsubmits: always_run: true branches: - ^main$ + cluster: build04 decorate: true decoration_config: oauth_token_secret: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner - sparse_checkout_files: - - test/ labels: ci-operator.openshift.io/is-promotion: "true" ci.openshift.io/generator: prowgen diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml index cef5206572f55..b09ef5dd8e44c 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml @@ -12,8 +12,6 @@ presubmits: oauth_token_secret: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner - sparse_checkout_files: - - test/ labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -83,8 +81,6 @@ presubmits: oauth_token_secret: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner - sparse_checkout_files: - - test/ labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -154,8 +150,6 @@ presubmits: oauth_token_secret: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner - sparse_checkout_files: - - test/ labels: ci-operator.openshift.io/cloud: platform-gcp ci-operator.openshift.io/cloud-cluster-profile: platform-gcp @@ -237,14 +231,13 @@ presubmits: branches: - ^main$ - ^main- + cluster: build06 context: ci/prow/images decorate: true decoration_config: oauth_token_secret: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner - sparse_checkout_files: - - test/ labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -307,8 +300,6 @@ presubmits: oauth_token_secret: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner - sparse_checkout_files: - - test/ labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -395,8 +386,6 @@ presubmits: oauth_token_secret: key: oauth name: github-credentials-openshift-ci-robot-private-git-cloner - sparse_checkout_files: - - test/ labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" diff --git a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.metadata.json b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.metadata.json new file mode 100644 index 0000000000000..bf45324c1d5f3 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.metadata.json @@ -0,0 +1,25 @@ +{ + "path": "gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.yaml", + "owners": { + "approvers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ], + "reviewers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ] + } +} From 113c9c9400f737363dd71f229989d9a3c733b100 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Thu, 20 Aug 2026 17:42:42 -0400 Subject: [PATCH 32/44] Fix CI: move disable_sparse_checkout under prowgen, fix metadata trailing newline --- .../gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml | 2 +- .../gcp-hcp-pre-destroy-cleanup-ref.metadata.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml index e349510905e58..2c53814171d37 100644 --- a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml +++ b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml @@ -5,7 +5,6 @@ build_root: namespace: ci tag: latest use_build_cache: true -disable_sparse_checkout: true images: items: - context_dir: test @@ -20,6 +19,7 @@ promotion: namespace: ci tag: latest prowgen: + disable_sparse_checkout: true expose: true private: true releases: diff --git a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.metadata.json b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.metadata.json index bf45324c1d5f3..4dab6c12a73cd 100644 --- a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.metadata.json +++ b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.metadata.json @@ -22,4 +22,4 @@ "patjlm" ] } -} +} \ No newline at end of file From c35166c4481bf4224c0a6d83f23ab6f6ee72140c Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Thu, 20 Aug 2026 17:48:05 -0400 Subject: [PATCH 33/44] Fix pre-destroy-cleanup: exit gracefully when provision didn't complete If tf-provision was interrupted or never ran, the SHARED_DIR won't have region-project-id. Exit cleanly instead of failing with 'No such file or directory'. Also fixes: disable_sparse_checkout under prowgen (not top-level), metadata.json trailing newline. --- .../gcp-hcp-pre-destroy-cleanup-commands.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh index 3ab1c52e74f2d..ab35b38c06a0f 100644 --- a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh @@ -15,7 +15,13 @@ if [[ ! -f "${SHARED_DIR}/wif-cred.json" ]]; then fi gcloud auth login --cred-file="${SHARED_DIR}/wif-cred.json" --quiet -# Read cluster info from SHARED_DIR +# Read cluster info from SHARED_DIR (written by tf-provision) +# If provision didn't complete, these files won't exist — nothing to clean up. +if [[ ! -f "${SHARED_DIR}/region-project-id" ]]; then + echo "No region-project-id in SHARED_DIR — provision likely didn't complete. Nothing to clean up." + exit 0 +fi + REGION_PROJECT=$(<"${SHARED_DIR}/region-project-id") REGION_CLUSTER_NAME=$(<"${SHARED_DIR}/region-cluster-name") MC_PROJECT=$(<"${SHARED_DIR}/mc-project-id") From 4e5a6601a112086bd402b339ecdcb413aa3547d8 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Thu, 20 Aug 2026 18:04:34 -0400 Subject: [PATCH 34/44] Add missing metadata.json for generate-kubeconfigs step --- ...hcp-generate-kubeconfigs-ref.metadata.json | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.metadata.json diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.metadata.json b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.metadata.json new file mode 100644 index 0000000000000..bfaaad12b8c38 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.metadata.json @@ -0,0 +1,25 @@ +{ + "path": "gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.yaml", + "owners": { + "approvers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ], + "reviewers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ] + } +} \ No newline at end of file From 8bad6114594c680470de920fd026177a1acd5651 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Thu, 20 Aug 2026 22:46:45 -0400 Subject: [PATCH 35/44] Replace terraform destroy with force project deletion for reliable cleanup terraform destroy fails unreliably when orphaned NEGs or other resources block VPC deletion. This change replaces the pre-destroy-cleanup + tf-deprovision approach with a single cleanup-infrastructure step that: 1. Stops ArgoCD (prevents resource recreation) 2. Deletes Gateway API resources + removes finalizers 3. Force-deletes remaining NEGs (zonal and regional) 4. Deletes DNS records from regional zones 5. **Force-deletes GCP projects** (bypasses terraform destroy) 6. Clears TFC workspace state via API This approach is modeled after the Tekton cleanup-infrastructure task but adapted for Prow/TFC workflow. Project deletion cascades to all resources (NEGs, VPCs, DNS, GKE, etc.) regardless of terraform state, making cleanup reliable for ephemeral e2e environments. Projects enter PENDING_DELETE state (30-day soft delete). The e2e-render script already generates unique project IDs per run to avoid collisions. Removes: - gcp-hcp-pre-destroy-cleanup (partial cleanup, didn't solve the NEG issue) - gcp-hcp-tf-deprovision (fails when resources block terraform destroy) Adds: - gcp-hcp-cleanup-infrastructure (comprehensive cleanup via project deletion) --- .../gcp-hcp/cleanup-infrastructure/OWNERS | 18 + ...gcp-hcp-cleanup-infrastructure-commands.sh | 372 ++++++++++++++++++ ...p-cleanup-infrastructure-ref.metadata.json | 25 ++ .../gcp-hcp-cleanup-infrastructure-ref.yaml | 36 ++ .../gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml | 5 +- 5 files changed, 453 insertions(+), 3 deletions(-) create mode 100644 ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/OWNERS create mode 100644 ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh create mode 100644 ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.metadata.json create mode 100644 ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/OWNERS b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/OWNERS new file mode 100644 index 0000000000000..60325af98b50d --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/OWNERS @@ -0,0 +1,18 @@ +approvers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm +reviewers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh new file mode 100644 index 0000000000000..7f48ca820a8b2 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh @@ -0,0 +1,372 @@ +#!/usr/bin/env bash +set -euo pipefail + +LOG="${ARTIFACT_DIR}/cleanup.log" +log() { echo "$(date -u '+%Y-%m-%d %H:%M:%S UTC') | $*" | tee -a "${LOG}"; } + +log "=== GCP HCP Infrastructure Cleanup ===" +log "This script performs comprehensive cleanup modeled after the Tekton cleanup task:" +log "1. Stop ArgoCD (prevents resource recreation)" +log "2. Delete Gateway API resources (triggers NEG cleanup)" +log "3. Force-delete NEGs" +log "4. Delete DNS records" +log "5. Delete GCP projects (bypasses terraform destroy for reliability)" +log "6. Clear TFC workspace state" +log "" + +# Authenticate with WIF +if [[ ! -f "${SHARED_DIR}/wif-cred.json" ]]; then + log "ERROR: WIF credential not found" + exit 1 +fi +gcloud auth login --cred-file="${SHARED_DIR}/wif-cred.json" --quiet + +# Read infrastructure info from SHARED_DIR +if [[ ! -f "${SHARED_DIR}/region-project-id" ]]; then + log "No region-project-id in SHARED_DIR — provision didn't complete, nothing to clean up" + exit 0 +fi + +REGION_PROJECT=$(<"${SHARED_DIR}/region-project-id") +REGION_CLUSTER=$(<"${SHARED_DIR}/region-cluster-name") +MC_PROJECT=$(<"${SHARED_DIR}/mc-project-id") +MC_CLUSTER=$(<"${SHARED_DIR}/mc-cluster-name") +REGION=${GCP_REGION:-us-central1} + +# Get project numbers +REGION_PROJECT_NUMBER=$(gcloud projects describe "${REGION_PROJECT}" --format='value(projectNumber)' 2>/dev/null || echo "") +MC_PROJECT_NUMBER=$(gcloud projects describe "${MC_PROJECT}" --format='value(projectNumber)' 2>/dev/null || echo "") + +log "Infrastructure to clean up:" +log " Region: ${REGION_PROJECT} (#${REGION_PROJECT_NUMBER}) / ${REGION_CLUSTER}" +log " MC: ${MC_PROJECT} (#${MC_PROJECT_NUMBER}) / ${MC_CLUSTER}" +log " Region: ${REGION}" +log "" + +# Helper: build kubeconfig with fresh access token using Connect Gateway +build_kubeconfig() { + local project_number=$1 + local cluster_name=$2 + local output_path=$3 + + local endpoint="${REGION}-connectgateway.googleapis.com/v1/projects/${project_number}/locations/${REGION}/gkeMemberships/${cluster_name}" + local token + token=$(gcloud auth print-access-token) + + cat > "${output_path}" </dev/null; then + log " ArgoCD namespace not found, skipping" + return 0 + fi + + # Scale down ArgoCD deployments + kc "${kubeconfig}" -n argocd scale deployment --all --replicas=0 2>/dev/null || true + kc "${kubeconfig}" -n argocd scale statefulset --all --replicas=0 2>/dev/null || true + + # Delete Applications and ApplicationSets + kc "${kubeconfig}" -n argocd delete applicationset --all --wait=false --timeout=30s 2>/dev/null || true + kc "${kubeconfig}" -n argocd delete application --all --wait=false --timeout=30s 2>/dev/null || true + + log " ArgoCD stopped" +} + +# ======================================================================== +# Phase 2: Delete Gateway API resources +# ======================================================================== +delete_gateway_resources() { + local kubeconfig=$1 + local label=$2 + + log "--- [${label}] Deleting Gateway API resources ---" + + for kind in gcpbackendpolicy healthcheckpolicy httproute gateway; do + local count + count=$(kc "${kubeconfig}" get "${kind}" --all-namespaces --no-headers 2>/dev/null | wc -l || echo "0") + count=$((count + 0)) + if [[ ${count} -gt 0 ]]; then + log " Deleting ${count} ${kind} resource(s)" + kc "${kubeconfig}" delete "${kind}" --all --all-namespaces --wait=false --timeout=30s 2>/dev/null || true + fi + done + + # Remove finalizers from stuck Gateway resources + log " Removing finalizers from Gateway resources" + kc "${kubeconfig}" get gateway --all-namespaces -o json 2>/dev/null | \ + jq -r '.items[] | select(.metadata.finalizers != null) | "\(.metadata.namespace) \(.metadata.name)"' 2>/dev/null | \ + while read -r ns name; do + [[ -z "${ns}" || -z "${name}" ]] && continue + log " Patching gateway ${ns}/${name}" + kc "${kubeconfig}" patch gateway "${name}" -n "${ns}" \ + --type=json -p='[{"op": "remove", "path": "/metadata/finalizers"}]' 2>/dev/null || true + done +} + +# ======================================================================== +# Phase 3: Force-delete NEGs +# ======================================================================== +delete_negs() { + local project=$1 + local label=$2 + + log "--- [${label}] Force-deleting NEGs in ${project} ---" + + # Zonal NEGs + local zones + zones=$(gcloud compute zones list --filter="region:${REGION}" --format="value(name)" 2>/dev/null || echo "") + + for zone in ${zones}; do + local negs + negs=$(gcloud compute network-endpoint-groups list \ + --project="${project}" \ + --zones="${zone}" \ + --format="value(name)" 2>/dev/null || echo "") + + if [[ -n "${negs}" ]]; then + echo "${negs}" | while read -r neg_name; do + [[ -z "${neg_name}" ]] && continue + log " Deleting zonal NEG: ${neg_name} (zone: ${zone})" + gcloud compute network-endpoint-groups delete "${neg_name}" \ + --project="${project}" \ + --zone="${zone}" \ + --quiet 2>/dev/null || true + done + fi + done + + # Regional NEGs + local regional_negs + regional_negs=$(gcloud compute network-endpoint-groups list \ + --project="${project}" \ + --regions="${REGION}" \ + --format="value(name)" 2>/dev/null || echo "") + + if [[ -n "${regional_negs}" ]]; then + echo "${regional_negs}" | while read -r neg_name; do + [[ -z "${neg_name}" ]] && continue + log " Deleting regional NEG: ${neg_name}" + gcloud compute network-endpoint-groups delete "${neg_name}" \ + --project="${project}" \ + --region="${REGION}" \ + --quiet 2>/dev/null || true + done + fi +} + +# ======================================================================== +# Phase 4: Delete DNS records from regional zones +# ======================================================================== +delete_dns_records() { + local project=$1 + local label=$2 + + log "--- [${label}] Cleaning DNS records in ${project} ---" + + local zones + zones=$(gcloud dns managed-zones list --project="${project}" --format="value(name)" 2>/dev/null || echo "") + + if [[ -z "${zones}" ]]; then + log " No DNS zones found" + return 0 + fi + + while IFS= read -r zone; do + [[ -z "${zone}" ]] && continue + log " Zone: ${zone}" + + local records + records=$(gcloud dns record-sets list \ + --project="${project}" \ + --zone="${zone}" \ + --format="csv[no-heading](name,type)" 2>/dev/null || echo "") + + while IFS=',' read -r name type; do + [[ -z "${name}" || -z "${type}" ]] && continue + [[ "${type}" == "SOA" || "${type}" == "NS" ]] && continue + log " Deleting ${type} record: ${name}" + gcloud dns record-sets delete "${name}" \ + --zone="${zone}" --project="${project}" --type="${type}" \ + --quiet 2>/dev/null || true + done <<< "${records}" + done <<< "${zones}" +} + +# ======================================================================== +# Phase 5: Force-delete GCP projects +# ======================================================================== +delete_project() { + local project=$1 + local label=$2 + + log "--- [${label}] Force-deleting project: ${project} ---" + + if gcloud projects delete "${project}" --quiet 2>&1 | tee -a "${LOG}"; then + log " Project ${project} deletion initiated" + else + log " WARNING: Failed to delete project ${project}" + fi +} + +# ======================================================================== +# Phase 6: Clear TFC workspace state +# ======================================================================== +clear_tfc_workspace() { + log "--- Clearing TFC workspace state ---" + + # Read workspace info from SHARED_DIR + if [[ ! -f "${SHARED_DIR}/workspace-name" ]]; then + log " WARNING: No workspace-name in SHARED_DIR, skipping TFC cleanup" + return 0 + fi + + local workspace_name + workspace_name=$(<"${SHARED_DIR}/workspace-name") + + if [[ ! -f "/etc/terraform-cloud/token" ]]; then + log " WARNING: TFC token not found, skipping TFC cleanup" + return 0 + fi + + local tfc_token + tfc_token=$(<"/etc/terraform-cloud/token") + local tfc_org="hp-platform-engineering" + + log " Workspace: ${workspace_name}" + + # Get workspace ID + local workspace_id + workspace_id=$(curl -sS \ + --header "Authorization: Bearer ${tfc_token}" \ + --header "Content-Type: application/vnd.api+json" \ + "https://app.terraform.io/api/v2/organizations/${tfc_org}/workspaces/${workspace_name}" 2>/dev/null | \ + jq -r '.data.id // empty' || echo "") + + if [[ -z "${workspace_id}" ]]; then + log " WARNING: Could not find workspace ID, may already be deleted" + return 0 + fi + + # Delete workspace (deletes state) + if curl -sS -X DELETE \ + --header "Authorization: Bearer ${tfc_token}" \ + --header "Content-Type: application/vnd.api+json" \ + "https://app.terraform.io/api/v2/workspaces/${workspace_id}" 2>&1 | tee -a "${LOG}"; then + log " TFC workspace deleted: ${workspace_name}" + else + log " WARNING: Failed to delete TFC workspace (may already be deleted)" + fi +} + +# ======================================================================== +# Execute cleanup +# ======================================================================== + +# Build kubeconfigs with fresh tokens +REGION_KC="/tmp/region-kubeconfig" +MC_KC="/tmp/mc-kubeconfig" + +if [[ -n "${REGION_PROJECT_NUMBER}" ]]; then + build_kubeconfig "${REGION_PROJECT_NUMBER}" "${REGION_CLUSTER}" "${REGION_KC}" + # MC is registered in region's fleet + build_kubeconfig "${REGION_PROJECT_NUMBER}" "${MC_CLUSTER}" "${MC_KC}" +fi + +# Verify connectivity +log "Verifying cluster connectivity..." +REGION_CONNECTED=false +MC_CONNECTED=false + +if [[ -f "${REGION_KC}" ]] && kc "${REGION_KC}" get nodes --request-timeout=10s &>/dev/null; then + log " Region cluster: connected" + REGION_CONNECTED=true +else + log " Region cluster: unreachable" +fi + +if [[ -f "${MC_KC}" ]] && kc "${MC_KC}" get nodes --request-timeout=10s &>/dev/null; then + log " MC cluster: connected" + MC_CONNECTED=true +else + log " MC cluster: unreachable" +fi +log "" + +# Phase 1: Stop ArgoCD (MC first, then region) +if [[ "${MC_CONNECTED}" == "true" ]]; then + stop_argocd "${MC_KC}" "MC" || true +fi +if [[ "${REGION_CONNECTED}" == "true" ]]; then + stop_argocd "${REGION_KC}" "Region" || true +fi + +# Phase 2: Delete Gateway API resources +if [[ "${MC_CONNECTED}" == "true" ]]; then + delete_gateway_resources "${MC_KC}" "MC" || true +fi +if [[ "${REGION_CONNECTED}" == "true" ]]; then + delete_gateway_resources "${REGION_KC}" "Region" || true +fi + +# Wait for GKE Gateway controller to process deletions +if [[ "${REGION_CONNECTED}" == "true" || "${MC_CONNECTED}" == "true" ]]; then + log "" + log "Waiting 120s for GKE to process Gateway/NEG deletions..." + sleep 120 +fi + +# Phase 3: Force-delete remaining NEGs +delete_negs "${REGION_PROJECT}" "Region" || true +delete_negs "${MC_PROJECT}" "MC" || true + +# Phase 4: Delete DNS records +delete_dns_records "${REGION_PROJECT}" "Region" || true + +# Phase 5: Force-delete projects (this is the key difference from terraform destroy) +log "" +log "=== Force-deleting GCP projects ===" +log "This bypasses terraform destroy for reliability — project deletion cascades to all resources" +delete_project "${MC_PROJECT}" "MC" +delete_project "${REGION_PROJECT}" "Region" + +# Phase 6: Clear TFC workspace state +log "" +clear_tfc_workspace + +log "" +log "=== Cleanup complete ===" +log "Projects ${REGION_PROJECT} and ${MC_PROJECT} are now in PENDING_DELETE state (30-day soft delete)" +log "TFC workspace state has been cleared" diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.metadata.json b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.metadata.json new file mode 100644 index 0000000000000..9a7b65bb46bc2 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.metadata.json @@ -0,0 +1,25 @@ +{ + "path": "gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml", + "owners": { + "approvers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ], + "reviewers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml new file mode 100644 index 0000000000000..33caada37a71d --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml @@ -0,0 +1,36 @@ +ref: + as: gcp-hcp-cleanup-infrastructure + from_image: + namespace: ci + name: gcp-hcp-e2e-tests + tag: latest + commands: gcp-hcp-cleanup-infrastructure-commands.sh + resources: + requests: + cpu: 500m + memory: 512Mi + credentials: + - namespace: test-credentials + name: gcp-hcp-ci-workload-identity + mount_path: /secrets/gcp-hcp-ci + - namespace: test-credentials + name: terraform-cloud-token + mount_path: /etc/terraform-cloud + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + default: /secrets/gcp-hcp-ci/credential.json + timeout: 30m0s + documentation: |- + Performs comprehensive infrastructure cleanup for GCP HCP e2e environments. + + This step force-deletes GCP projects instead of relying on terraform destroy, + which can fail due to orphaned NEGs or other resources. The cleanup process: + + 1. Stops ArgoCD to prevent resource recreation + 2. Deletes Gateway API resources (triggers NEG cleanup) + 3. Force-deletes any remaining NEGs + 4. Deletes DNS records from regional zones + 5. Force-deletes GCP projects (cascades to all resources) + 6. Clears Terraform Cloud workspace state + + This approach is more reliable than terraform destroy for ephemeral e2e environments. diff --git a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml index 9d47211ee872d..31696fe8d2bec 100644 --- a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml +++ b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml @@ -8,8 +8,7 @@ workflow: test: - ref: gcp-hcp-verify-argocd-sync post: - - ref: gcp-hcp-pre-destroy-cleanup - - ref: gcp-hcp-tf-deprovision + - ref: gcp-hcp-cleanup-infrastructure documentation: |- E2E workflow for gcp-hcp-infra. Provisions full platform infrastructure (region + management cluster) using Terraform Cloud ephemeral workspaces, @@ -18,4 +17,4 @@ workflow: Phase breakdown: - Pre: Authenticate via WIF, provision infrastructure with terraform apply, generate kubeconfigs - Test: Validate ArgoCD application sync on region and management clusters using Ginkgo v2 tests - - Post: Clean up ArgoCD-managed GCP resources (NEGs, DNS, Gateways), then terraform destroy + - Post: Force-delete GCP projects and clear TFC workspace state (more reliable than terraform destroy) From d91f7673f181fe3ebdf499209aa0e8cf51e00b76 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Thu, 20 Aug 2026 22:53:20 -0400 Subject: [PATCH 36/44] Fix critical bugs in cleanup-infrastructure script Issues fixed: 1. **CRITICAL: Fixed project deletion exit code check** - Was checking exit code of 'tee', not 'gcloud projects delete' - Now captures output and exit code separately - Actually returns non-zero on failure 2. **Added dependency validation** - Validates jq, kubectl, gcloud exist at startup - Prevents silent failures from missing tools 3. **Fixed TFC workspace deletion exit code** - Same pattern as project deletion fix - Captures curl exit code correctly 4. **Added curl timeouts** - 30s max-time, 10s connect-timeout on TFC API calls - Prevents hanging if TFC is slow/down 5. **Better error handling for Gateway finalizer removal** - Captures kubectl output before piping to jq - Handles missing Gateway CRD gracefully 6. **Made TFC org configurable** - Uses TFC_ORGANIZATION env var with fallback - Hard-coded value now default, not mandatory 7. **Added comment about 120s wait** - Documents that it's a conservative estimate - Explains tuning rationale --- ...gcp-hcp-cleanup-infrastructure-commands.sh | 73 +++++++++++++++---- 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh index 7f48ca820a8b2..5356fe6839e5d 100644 --- a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh @@ -4,6 +4,22 @@ set -euo pipefail LOG="${ARTIFACT_DIR}/cleanup.log" log() { echo "$(date -u '+%Y-%m-%d %H:%M:%S UTC') | $*" | tee -a "${LOG}"; } +# Validate required dependencies +if ! command -v jq &>/dev/null; then + echo "ERROR: jq not found in container" >&2 + exit 1 +fi + +if ! command -v kubectl &>/dev/null; then + echo "ERROR: kubectl not found in container" >&2 + exit 1 +fi + +if ! command -v gcloud &>/dev/null; then + echo "ERROR: gcloud not found in container" >&2 + exit 1 +fi + log "=== GCP HCP Infrastructure Cleanup ===" log "This script performs comprehensive cleanup modeled after the Tekton cleanup task:" log "1. Stop ArgoCD (prevents resource recreation)" @@ -126,14 +142,18 @@ delete_gateway_resources() { # Remove finalizers from stuck Gateway resources log " Removing finalizers from Gateway resources" - kc "${kubeconfig}" get gateway --all-namespaces -o json 2>/dev/null | \ - jq -r '.items[] | select(.metadata.finalizers != null) | "\(.metadata.namespace) \(.metadata.name)"' 2>/dev/null | \ - while read -r ns name; do - [[ -z "${ns}" || -z "${name}" ]] && continue - log " Patching gateway ${ns}/${name}" - kc "${kubeconfig}" patch gateway "${name}" -n "${ns}" \ - --type=json -p='[{"op": "remove", "path": "/metadata/finalizers"}]' 2>/dev/null || true - done + local gateways_json + if gateways_json=$(kc "${kubeconfig}" get gateway --all-namespaces -o json 2>/dev/null); then + echo "${gateways_json}" | jq -r '.items[] | select(.metadata.finalizers != null) | "\(.metadata.namespace) \(.metadata.name)"' 2>/dev/null | \ + while read -r ns name; do + [[ -z "${ns}" || -z "${name}" ]] && continue + log " Patching gateway ${ns}/${name}" + kc "${kubeconfig}" patch gateway "${name}" -n "${ns}" \ + --type=json -p='[{"op": "remove", "path": "/metadata/finalizers"}]' 2>/dev/null || true + done + else + log " WARNING: Could not get Gateway resources (may not exist or cluster unreachable)" + fi } # ======================================================================== @@ -234,10 +254,19 @@ delete_project() { log "--- [${label}] Force-deleting project: ${project} ---" - if gcloud projects delete "${project}" --quiet 2>&1 | tee -a "${LOG}"; then + local output + local exit_code + output=$(gcloud projects delete "${project}" --quiet 2>&1) + exit_code=$? + + echo "${output}" | tee -a "${LOG}" + + if [[ ${exit_code} -eq 0 ]]; then log " Project ${project} deletion initiated" + return 0 else - log " WARNING: Failed to delete project ${project}" + log " ERROR: Failed to delete project ${project} (exit code: ${exit_code})" + return 1 fi } @@ -263,17 +292,19 @@ clear_tfc_workspace() { local tfc_token tfc_token=$(<"/etc/terraform-cloud/token") - local tfc_org="hp-platform-engineering" + local tfc_org="${TFC_ORGANIZATION:-hp-platform-engineering}" log " Workspace: ${workspace_name}" # Get workspace ID local workspace_id workspace_id=$(curl -sS \ + --max-time 30 \ + --connect-timeout 10 \ --header "Authorization: Bearer ${tfc_token}" \ --header "Content-Type: application/vnd.api+json" \ "https://app.terraform.io/api/v2/organizations/${tfc_org}/workspaces/${workspace_name}" 2>/dev/null | \ - jq -r '.data.id // empty' || echo "") + jq -r '.data.id // empty' 2>/dev/null || echo "") if [[ -z "${workspace_id}" ]]; then log " WARNING: Could not find workspace ID, may already be deleted" @@ -281,13 +312,22 @@ clear_tfc_workspace() { fi # Delete workspace (deletes state) - if curl -sS -X DELETE \ + local delete_output + local delete_exit + delete_output=$(curl -sS -X DELETE \ + --max-time 30 \ + --connect-timeout 10 \ --header "Authorization: Bearer ${tfc_token}" \ --header "Content-Type: application/vnd.api+json" \ - "https://app.terraform.io/api/v2/workspaces/${workspace_id}" 2>&1 | tee -a "${LOG}"; then + "https://app.terraform.io/api/v2/workspaces/${workspace_id}" 2>&1) + delete_exit=$? + + echo "${delete_output}" | tee -a "${LOG}" + + if [[ ${delete_exit} -eq 0 ]]; then log " TFC workspace deleted: ${workspace_name}" else - log " WARNING: Failed to delete TFC workspace (may already be deleted)" + log " WARNING: Failed to delete TFC workspace (exit code: ${delete_exit}, may already be deleted)" fi } @@ -342,6 +382,9 @@ if [[ "${REGION_CONNECTED}" == "true" ]]; then fi # Wait for GKE Gateway controller to process deletions +# Note: 120s is a conservative estimate. GKE typically processes Gateway deletions +# within 60s, but we add buffer time to reduce NEG orphan risk. This wait can be +# tuned based on observed cleanup times. if [[ "${REGION_CONNECTED}" == "true" || "${MC_CONNECTED}" == "true" ]]; then log "" log "Waiting 120s for GKE to process Gateway/NEG deletions..." From 811bfbeaf0381b346c472fa773fbe6c752369577 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Fri, 21 Aug 2026 08:22:21 -0400 Subject: [PATCH 37/44] Fix cleanup-infrastructure image reference Use 'from: gcp-hcp-e2e-tests' instead of 'from_image' with ci namespace. The gcp-hcp-e2e-tests image is built during the job, not imported from a registry, so we need to reference it as a pipeline image. This fixes the rehearsal failure: failed to import tag ci-gcp-hcp-e2e-tests-latest image not found in quay.io/openshift/ci --- .../gcp-hcp-cleanup-infrastructure-ref.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml index 33caada37a71d..54486f1b9734f 100644 --- a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml @@ -1,9 +1,6 @@ ref: as: gcp-hcp-cleanup-infrastructure - from_image: - namespace: ci - name: gcp-hcp-e2e-tests - tag: latest + from: gcp-hcp-e2e-tests commands: gcp-hcp-cleanup-infrastructure-commands.sh resources: requests: From a7820aa7050660550e92d35bc22a2c798fbdd5bd Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Fri, 21 Aug 2026 08:57:21 -0400 Subject: [PATCH 38/44] Fix cleanup-infrastructure credentials: use correct secret names - WIF credential comes from SHARED_DIR/wif-cred.json (no mount needed) - TFC token: use tfcloud-ci-secret in ci namespace (matches tf-provision/tf-deprovision) - Remove non-existent gcp-hcp-ci-workload-identity and terraform-cloud-token refs - Add GCP_REGION env var (matches other steps) --- .../gcp-hcp-cleanup-infrastructure-ref.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml index 54486f1b9734f..00cf70fc68829 100644 --- a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml @@ -7,15 +7,12 @@ ref: cpu: 500m memory: 512Mi credentials: - - namespace: test-credentials - name: gcp-hcp-ci-workload-identity - mount_path: /secrets/gcp-hcp-ci - - namespace: test-credentials - name: terraform-cloud-token - mount_path: /etc/terraform-cloud + - mount_path: /etc/terraform-cloud + name: tfcloud-ci-secret + namespace: ci env: - - name: GOOGLE_APPLICATION_CREDENTIALS - default: /secrets/gcp-hcp-ci/credential.json + - name: GCP_REGION + default: us-central1 timeout: 30m0s documentation: |- Performs comprehensive infrastructure cleanup for GCP HCP e2e environments. @@ -30,4 +27,7 @@ ref: 5. Force-deletes GCP projects (cascades to all resources) 6. Clears Terraform Cloud workspace state + WIF credential is read from SHARED_DIR/wif-cred.json (written by hypershift-gcp-wif-auth). + TFC token is mounted from the tfcloud-ci-secret credential. + This approach is more reliable than terraform destroy for ephemeral e2e environments. From a4a26f78b8b95d61c70dbaf1c7e1e3c1a7398be4 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Fri, 21 Aug 2026 08:58:48 -0400 Subject: [PATCH 39/44] Add best_effort and grace_period to cleanup-infrastructure step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without best_effort: true, cleanup failures would fail the entire job. Post-phase cleanup steps should not determine pass/fail — only the test phase should. Matches the pattern from tf-deprovision and pre-destroy-cleanup. --- .../gcp-hcp-cleanup-infrastructure-ref.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml index 00cf70fc68829..1b306144c8df5 100644 --- a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml @@ -12,8 +12,11 @@ ref: namespace: ci env: - name: GCP_REGION - default: us-central1 + default: "us-central1" + documentation: "GCP region for e2e infrastructure deployment" timeout: 30m0s + grace_period: 10m0s + best_effort: true documentation: |- Performs comprehensive infrastructure cleanup for GCP HCP e2e environments. @@ -30,4 +33,6 @@ ref: WIF credential is read from SHARED_DIR/wif-cred.json (written by hypershift-gcp-wif-auth). TFC token is mounted from the tfcloud-ci-secret credential. + Uses best_effort: true so cleanup failures don't block the job. + This approach is more reliable than terraform destroy for ephemeral e2e environments. From 08071d6e2f954e9007411907c90213676428a262 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Fri, 21 Aug 2026 10:09:55 -0400 Subject: [PATCH 40/44] Fix Firestore 409 import detection for TFC remote output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TFC formats error output with │ prefix characters and splits the error across multiple lines: │ Error: Error creating Database: googleapi: Error 409: Database already exists. │ with module.management_cluster.google_firestore_database.status The previous grep pattern 'google_firestore_database.*Database already exists' requires both strings on the same line, which never matches TFC output. Fix: use two separate greps to check for each string independently. --- .../tf-provision/gcp-hcp-tf-provision-commands.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh b/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh index 61ba27b0a80ff..b854bb8d43bf2 100755 --- a/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh @@ -197,8 +197,13 @@ NON_TRANSIENT_ERRORS="quota.*exceeded|forbidden|invalid.*configuration|unauthori import_orphaned_firestore() { local output="$1" - # Check for the Firestore 409 pattern - if ! echo "${output}" | grep -q "google_firestore_database.*Database already exists"; then + # Check for the Firestore 409 pattern. + # TFC remote output splits the error across multiple lines with │ prefixes, + # so we check for both strings independently rather than on a single line. + if ! echo "${output}" | grep -q "Database already exists"; then + return 1 + fi + if ! echo "${output}" | grep -q "google_firestore_database"; then return 1 fi From a38d3d3dc2ee17cd05ce274b41a20931eda732d4 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Fri, 21 Aug 2026 11:09:30 -0400 Subject: [PATCH 41/44] Fix cleanup-infrastructure: use upi-installer image (has jq/gcloud/kubectl) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gcp-hcp-e2e-tests image is a Go binary for Ginkgo tests — it doesn't include jq, gcloud, or kubectl. Use ocp/4.22:upi-installer instead, which is the same image used by pre-destroy-cleanup and generate-kubeconfigs. --- .../gcp-hcp-cleanup-infrastructure-ref.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml index 1b306144c8df5..e5d0f312bb3f9 100644 --- a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml @@ -1,6 +1,9 @@ ref: as: gcp-hcp-cleanup-infrastructure - from: gcp-hcp-e2e-tests + from_image: + namespace: ocp + name: "4.22" + tag: upi-installer commands: gcp-hcp-cleanup-infrastructure-commands.sh resources: requests: From e1ed24fcec5325c96641a2f06b6b595a50ace198 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Fri, 21 Aug 2026 13:19:49 -0400 Subject: [PATCH 42/44] TEMPORARY: Add image check step + use oc instead of kubectl 1. Add check-cleanup-image step as first pre step to validate the upi-installer image has all required binaries (jq, gcloud, curl, kubectl/oc) before spending 40 min on provision. Will fail fast in ~10 seconds if anything is missing. 2. Fix cleanup-infrastructure to use oc as kubectl fallback since upi-installer has oc but not kubectl. TODO: Remove check-cleanup-image step after validation. --- .../gcp-hcp/check-cleanup-image/OWNERS | 18 +++++++++ .../gcp-hcp-check-cleanup-image-commands.sh | 39 +++++++++++++++++++ ...-hcp-check-cleanup-image-ref.metadata.json | 25 ++++++++++++ .../gcp-hcp-check-cleanup-image-ref.yaml | 15 +++++++ ...gcp-hcp-cleanup-infrastructure-commands.sh | 28 ++++++------- .../gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml | 1 + 6 files changed, 113 insertions(+), 13 deletions(-) create mode 100644 ci-operator/step-registry/gcp-hcp/check-cleanup-image/OWNERS create mode 100644 ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-commands.sh create mode 100644 ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.metadata.json create mode 100644 ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.yaml diff --git a/ci-operator/step-registry/gcp-hcp/check-cleanup-image/OWNERS b/ci-operator/step-registry/gcp-hcp/check-cleanup-image/OWNERS new file mode 100644 index 0000000000000..60325af98b50d --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/check-cleanup-image/OWNERS @@ -0,0 +1,18 @@ +approvers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm +reviewers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm diff --git a/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-commands.sh b/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-commands.sh new file mode 100644 index 0000000000000..76b554202a167 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-commands.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "=== TEMPORARY: Verify cleanup-infrastructure image has required binaries ===" +echo "This step will be removed after validation." +echo "" + +failed=0 +for cmd in jq gcloud curl; do + if command -v "${cmd}" &>/dev/null; then + echo "✓ ${cmd}: $(command -v ${cmd})" + else + echo "✗ ${cmd}: NOT FOUND" + failed=1 + fi +done + +# Check kubectl or oc +if command -v kubectl &>/dev/null; then + echo "✓ kubectl: $(command -v kubectl)" +elif command -v oc &>/dev/null; then + echo "✓ oc (kubectl substitute): $(command -v oc)" +else + echo "✗ kubectl/oc: NEITHER FOUND" + failed=1 +fi + +echo "" +if [[ ${failed} -eq 1 ]]; then + echo "FAIL: Missing required binaries. Choose a different image." + exit 1 +fi + +echo "PASS: All required binaries present." +echo "" +echo "Versions:" +jq --version 2>/dev/null || true +gcloud version 2>/dev/null | head -3 || true +kubectl version --client 2>/dev/null | head -1 || oc version --client 2>/dev/null | head -1 || true diff --git a/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.metadata.json b/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.metadata.json new file mode 100644 index 0000000000000..b495c207b9b95 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.metadata.json @@ -0,0 +1,25 @@ +{ + "path": "gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.yaml", + "owners": { + "approvers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ], + "reviewers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.yaml b/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.yaml new file mode 100644 index 0000000000000..a2bead432d077 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.yaml @@ -0,0 +1,15 @@ +ref: + as: gcp-hcp-check-cleanup-image + from_image: + namespace: ocp + name: "4.22" + tag: upi-installer + commands: gcp-hcp-check-cleanup-image-commands.sh + resources: + requests: + cpu: 100m + memory: 128Mi + timeout: 2m0s + documentation: |- + TEMPORARY: Validates that the cleanup-infrastructure image has all required + binaries (jq, gcloud, kubectl/oc, curl). Remove after validation. diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh index 5356fe6839e5d..0e2956986be37 100644 --- a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh @@ -5,18 +5,20 @@ LOG="${ARTIFACT_DIR}/cleanup.log" log() { echo "$(date -u '+%Y-%m-%d %H:%M:%S UTC') | $*" | tee -a "${LOG}"; } # Validate required dependencies -if ! command -v jq &>/dev/null; then - echo "ERROR: jq not found in container" >&2 - exit 1 -fi - -if ! command -v kubectl &>/dev/null; then - echo "ERROR: kubectl not found in container" >&2 - exit 1 -fi +for cmd in jq gcloud curl; do + if ! command -v "${cmd}" &>/dev/null; then + echo "ERROR: ${cmd} not found in container" >&2 + exit 1 + fi +done -if ! command -v gcloud &>/dev/null; then - echo "ERROR: gcloud not found in container" >&2 +# Use oc as kubectl — upi-installer image has oc but not kubectl +if command -v kubectl &>/dev/null; then + KUBECTL=kubectl +elif command -v oc &>/dev/null; then + KUBECTL=oc +else + echo "ERROR: neither kubectl nor oc found in container" >&2 exit 1 fi @@ -89,11 +91,11 @@ users: EOF } -# Helper: kubectl wrapper +# Helper: kubectl/oc wrapper kc() { local kubeconfig=$1 shift - kubectl --kubeconfig="${kubeconfig}" "$@" + "${KUBECTL}" --kubeconfig="${kubeconfig}" "$@" } # ======================================================================== diff --git a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml index 31696fe8d2bec..1d06b6b931d90 100644 --- a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml +++ b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml @@ -2,6 +2,7 @@ workflow: as: gcp-hcp-e2e steps: pre: + - ref: gcp-hcp-check-cleanup-image - ref: hypershift-gcp-wif-auth - ref: gcp-hcp-tf-provision - ref: gcp-hcp-generate-kubeconfigs From c0b554d0c53e390af3c88e7c3ec599f6c3122072 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Fri, 21 Aug 2026 14:54:49 -0400 Subject: [PATCH 43/44] Fix TFC cleanup: use terraform CLI to clear state before workspace delete The TFC API rejects workspace deletion when it manages resources (403), and pushing empty state via API fails with lineage/lock conflicts. Fix: install terraform CLI, create minimal cloud backend config, and use 'terraform state rm' at the module level to clear all resources in ~3s. Then safe-delete the empty workspace. Tested locally against platform-e2e-be2f7703 (466 resources): - terraform state rm module.{customer_project,management_cluster,region}: 3s - safe-delete: HTTP 204 success Also handles: - Force-unlock if workspace locked from previous run - Fallback to per-resource rm if top-level data sources remain - Correct terraform version (1.15.8) to match workspace constraint --- ...gcp-hcp-cleanup-infrastructure-commands.sh | 119 ++++++++++++++++-- 1 file changed, 108 insertions(+), 11 deletions(-) diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh index 0e2956986be37..addec59ee8dd9 100644 --- a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh @@ -228,6 +228,11 @@ delete_dns_records() { while IFS= read -r zone; do [[ -z "${zone}" ]] && continue + # Skip GKE Cloud DNS Scope zones (internal cluster DNS) — deleted with the cluster + if [[ "${zone}" == gke-* ]]; then + log " Skipping GKE internal zone: ${zone}" + continue + fi log " Zone: ${zone}" local records @@ -313,23 +318,115 @@ clear_tfc_workspace() { return 0 fi - # Delete workspace (deletes state) - local delete_output - local delete_exit - delete_output=$(curl -sS -X DELETE \ + # Use terraform CLI to clear state, then safe-delete the workspace. + # The GCP projects are already deleted, so the state is stale. + # Install terraform, point it at the TFC workspace via cloud backend, + # and run 'terraform state rm' at the module level for speed (~3s for 400+ resources). + + # Install terraform (same version as .tool-versions) + local tf_version="1.15.8" + log " Installing terraform ${tf_version}..." + if ! curl -fsSL --max-time 120 \ + "https://releases.hashicorp.com/terraform/${tf_version}/terraform_${tf_version}_linux_amd64.zip" \ + -o /tmp/terraform.zip; then + log " WARNING: Failed to download terraform, skipping TFC cleanup" + return 0 + fi + if command -v unzip &>/dev/null; then + unzip -o -q /tmp/terraform.zip -d /tmp + else + python3 -c "import zipfile; zipfile.ZipFile('/tmp/terraform.zip').extractall('/tmp')" + fi + chmod +x /tmp/terraform + + # Create minimal terraform config with cloud backend + local tf_dir="/tmp/tfc-cleanup" + mkdir -p "${tf_dir}" + cat > "${tf_dir}/main.tf" < "$HOME/.terraformrc" <&1 | tee -a "${LOG}"; then + log " WARNING: terraform init failed, skipping TFC cleanup" + return 0 + fi + + # Force-unlock if the workspace is locked from a previous run + local lock_id + lock_id=$(/tmp/terraform -chdir="${tf_dir}" state list -no-color 2>&1 | \ + grep -oP 'lock ID: "\K[^"]+' || echo "") + if [[ -n "${lock_id}" ]]; then + log " Workspace locked (${lock_id}), force-unlocking..." + /tmp/terraform -chdir="${tf_dir}" force-unlock -force "${lock_id}" -no-color 2>&1 | tee -a "${LOG}" || true + fi + + # Remove all resources from state at the module level — fast bulk operation. + # E2E state has 3 top-level modules: customer_project, management_cluster, region. + # Removing at module level clears all child resources in one API call (~3 seconds). + log " Clearing all resources from state (module-level rm)..." + local resource_count + resource_count=$(/tmp/terraform -chdir="${tf_dir}" state list -no-color 2>/dev/null | wc -l | tr -d ' ') + resource_count=${resource_count:-0} + + if [[ ${resource_count} -eq 0 ]]; then + log " State is already empty" + else + log " Removing ${resource_count} resources across top-level modules..." + /tmp/terraform -chdir="${tf_dir}" state rm \ + module.customer_project \ + module.management_cluster \ + module.region \ + -no-color 2>&1 | tail -5 | tee -a "${LOG}" || true + + # Check if any resources remain (e.g. top-level data sources) + local remaining + remaining=$(/tmp/terraform -chdir="${tf_dir}" state list -no-color 2>/dev/null | wc -l | tr -d ' ') + remaining=${remaining:-0} + if [[ ${remaining} -gt 0 ]]; then + log " ${remaining} resource(s) remain, removing individually..." + /tmp/terraform -chdir="${tf_dir}" state list -no-color 2>/dev/null | \ + while IFS= read -r addr; do + [[ -z "${addr}" ]] && continue + /tmp/terraform -chdir="${tf_dir}" state rm "${addr}" -no-color 2>/dev/null || true + done + fi + fi + + # Safe-delete the workspace (should succeed with 0 resources) + log " Deleting workspace..." + local http_code + http_code=$(curl -sS -o /dev/null -w "%{http_code}" \ --max-time 30 \ --connect-timeout 10 \ --header "Authorization: Bearer ${tfc_token}" \ --header "Content-Type: application/vnd.api+json" \ - "https://app.terraform.io/api/v2/workspaces/${workspace_id}" 2>&1) - delete_exit=$? - - echo "${delete_output}" | tee -a "${LOG}" - - if [[ ${delete_exit} -eq 0 ]]; then + --request POST \ + "https://app.terraform.io/api/v2/workspaces/${workspace_id}/actions/safe-delete" 2>/dev/null || echo "000") + + if [[ "${http_code}" == "204" || "${http_code}" == "200" ]]; then log " TFC workspace deleted: ${workspace_name}" else - log " WARNING: Failed to delete TFC workspace (exit code: ${delete_exit}, may already be deleted)" + log " WARNING: Could not delete TFC workspace (HTTP ${http_code}). Manual cleanup may be needed." + log " Workspace: https://app.terraform.io/app/${tfc_org}/workspaces/${workspace_name}" fi } From 10b1ef724492aaa9a19e8957277dfb3efb4d3386 Mon Sep 17 00:00:00 2001 From: Jim D'Agostino Date: Fri, 21 Aug 2026 17:05:26 -0400 Subject: [PATCH 44/44] Remove check-cleanup-image hack + fix TFC state rm to include data sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Remove temporary check-cleanup-image step (validated upi-installer has jq, gcloud, oc, curl — no longer needed) 2. Fix TFC state rm to include data.terraform_remote_state.* sources in the bulk rm call. Previous run left 4 data sources in state, causing safe-delete to return 409. 3. Remove individual-resource fallback loop (all known addresses are now in the bulk rm call) --- .../gcp-hcp/check-cleanup-image/OWNERS | 18 --------- .../gcp-hcp-check-cleanup-image-commands.sh | 39 ------------------- ...-hcp-check-cleanup-image-ref.metadata.json | 25 ------------ .../gcp-hcp-check-cleanup-image-ref.yaml | 15 ------- ...gcp-hcp-cleanup-infrastructure-commands.sh | 29 ++++++-------- .../gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml | 1 - 6 files changed, 11 insertions(+), 116 deletions(-) delete mode 100644 ci-operator/step-registry/gcp-hcp/check-cleanup-image/OWNERS delete mode 100644 ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-commands.sh delete mode 100644 ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.metadata.json delete mode 100644 ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.yaml diff --git a/ci-operator/step-registry/gcp-hcp/check-cleanup-image/OWNERS b/ci-operator/step-registry/gcp-hcp/check-cleanup-image/OWNERS deleted file mode 100644 index 60325af98b50d..0000000000000 --- a/ci-operator/step-registry/gcp-hcp/check-cleanup-image/OWNERS +++ /dev/null @@ -1,18 +0,0 @@ -approvers: -- apahim -- cblecker -- ckandag -- cristianoveiga -- floresroger -- gbarabasz -- jimdaga -- patjlm -reviewers: -- apahim -- cblecker -- ckandag -- cristianoveiga -- floresroger -- gbarabasz -- jimdaga -- patjlm diff --git a/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-commands.sh b/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-commands.sh deleted file mode 100644 index 76b554202a167..0000000000000 --- a/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-commands.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -echo "=== TEMPORARY: Verify cleanup-infrastructure image has required binaries ===" -echo "This step will be removed after validation." -echo "" - -failed=0 -for cmd in jq gcloud curl; do - if command -v "${cmd}" &>/dev/null; then - echo "✓ ${cmd}: $(command -v ${cmd})" - else - echo "✗ ${cmd}: NOT FOUND" - failed=1 - fi -done - -# Check kubectl or oc -if command -v kubectl &>/dev/null; then - echo "✓ kubectl: $(command -v kubectl)" -elif command -v oc &>/dev/null; then - echo "✓ oc (kubectl substitute): $(command -v oc)" -else - echo "✗ kubectl/oc: NEITHER FOUND" - failed=1 -fi - -echo "" -if [[ ${failed} -eq 1 ]]; then - echo "FAIL: Missing required binaries. Choose a different image." - exit 1 -fi - -echo "PASS: All required binaries present." -echo "" -echo "Versions:" -jq --version 2>/dev/null || true -gcloud version 2>/dev/null | head -3 || true -kubectl version --client 2>/dev/null | head -1 || oc version --client 2>/dev/null | head -1 || true diff --git a/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.metadata.json b/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.metadata.json deleted file mode 100644 index b495c207b9b95..0000000000000 --- a/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.metadata.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "path": "gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.yaml", - "owners": { - "approvers": [ - "apahim", - "cblecker", - "ckandag", - "cristianoveiga", - "floresroger", - "gbarabasz", - "jimdaga", - "patjlm" - ], - "reviewers": [ - "apahim", - "cblecker", - "ckandag", - "cristianoveiga", - "floresroger", - "gbarabasz", - "jimdaga", - "patjlm" - ] - } -} \ No newline at end of file diff --git a/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.yaml b/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.yaml deleted file mode 100644 index a2bead432d077..0000000000000 --- a/ci-operator/step-registry/gcp-hcp/check-cleanup-image/gcp-hcp-check-cleanup-image-ref.yaml +++ /dev/null @@ -1,15 +0,0 @@ -ref: - as: gcp-hcp-check-cleanup-image - from_image: - namespace: ocp - name: "4.22" - tag: upi-installer - commands: gcp-hcp-check-cleanup-image-commands.sh - resources: - requests: - cpu: 100m - memory: 128Mi - timeout: 2m0s - documentation: |- - TEMPORARY: Validates that the cleanup-infrastructure image has all required - binaries (jq, gcloud, kubectl/oc, curl). Remove after validation. diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh index addec59ee8dd9..bbf336efc121d 100644 --- a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh @@ -379,10 +379,11 @@ TFRC /tmp/terraform -chdir="${tf_dir}" force-unlock -force "${lock_id}" -no-color 2>&1 | tee -a "${LOG}" || true fi - # Remove all resources from state at the module level — fast bulk operation. - # E2E state has 3 top-level modules: customer_project, management_cluster, region. - # Removing at module level clears all child resources in one API call (~3 seconds). - log " Clearing all resources from state (module-level rm)..." + # Remove all resources from state in one bulk operation. + # E2E state has 3 top-level modules plus data sources. Removing at + # module level clears child resources, then we remove any remaining + # top-level resources (data sources). + log " Clearing all resources from state..." local resource_count resource_count=$(/tmp/terraform -chdir="${tf_dir}" state list -no-color 2>/dev/null | wc -l | tr -d ' ') resource_count=${resource_count:-0} @@ -390,25 +391,17 @@ TFRC if [[ ${resource_count} -eq 0 ]]; then log " State is already empty" else - log " Removing ${resource_count} resources across top-level modules..." + log " Removing ${resource_count} resources..." + # Bulk remove: modules + data sources in one call (~3 seconds) /tmp/terraform -chdir="${tf_dir}" state rm \ module.customer_project \ module.management_cluster \ module.region \ + data.terraform_remote_state.commons \ + data.terraform_remote_state.global \ + data.terraform_remote_state.platform_ci \ + data.terraform_remote_state.service \ -no-color 2>&1 | tail -5 | tee -a "${LOG}" || true - - # Check if any resources remain (e.g. top-level data sources) - local remaining - remaining=$(/tmp/terraform -chdir="${tf_dir}" state list -no-color 2>/dev/null | wc -l | tr -d ' ') - remaining=${remaining:-0} - if [[ ${remaining} -gt 0 ]]; then - log " ${remaining} resource(s) remain, removing individually..." - /tmp/terraform -chdir="${tf_dir}" state list -no-color 2>/dev/null | \ - while IFS= read -r addr; do - [[ -z "${addr}" ]] && continue - /tmp/terraform -chdir="${tf_dir}" state rm "${addr}" -no-color 2>/dev/null || true - done - fi fi # Safe-delete the workspace (should succeed with 0 resources) diff --git a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml index 1d06b6b931d90..31696fe8d2bec 100644 --- a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml +++ b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml @@ -2,7 +2,6 @@ workflow: as: gcp-hcp-e2e steps: pre: - - ref: gcp-hcp-check-cleanup-image - ref: hypershift-gcp-wif-auth - ref: gcp-hcp-tf-provision - ref: gcp-hcp-generate-kubeconfigs