diff --git a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml index d4478cd20b2de..2c53814171d37 100644 --- a/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml +++ b/ci-operator/config/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main.yaml @@ -5,7 +5,21 @@ build_root: namespace: ci tag: latest use_build_cache: true +images: + items: + - context_dir: test + dockerfile_path: Dockerfile.e2e + to: gcp-hcp-e2e-tests +promotion: + to: + - additional_images: + gcp-hcp-e2e-tests: gcp-hcp-e2e-tests + excluded_images: + - '*' + namespace: ci + tag: latest prowgen: + disable_sparse_checkout: true expose: true private: true releases: diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml index f144d7f5eb8ac..530e805564ffa 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-postsubmits.yaml @@ -7,7 +7,9 @@ postsubmits: cluster: build03 decorate: true decoration_config: - skip_cloning: true + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner labels: ci.openshift.io/generator: prowgen max_concurrency: 1 @@ -54,15 +56,79 @@ postsubmits: readOnly: true serviceAccountName: ci-operator volumes: - - name: github-credentials-openshift-ci-robot-private-git-cloner + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner + secretName: result-aggregator + - agent: kubernetes + always_run: true + branches: + - ^main$ + cluster: build04 + decorate: true + decoration_config: + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner + labels: + ci-operator.openshift.io/is-promotion: "true" + ci.openshift.io/generator: prowgen + max_concurrency: 1 + name: branch-ci-openshift-online-gcp-hcp-infra-main-images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --image-mirror-push-secret=/etc/push-secret/.dockerconfigjson + - --oauth-token-path=/usr/local/github-credentials/oauth + - --promote + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --target=gcp-hcp-e2e-tests + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/push-secret + name: push-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher - name: pull-secret secret: secretName: registry-pull-credentials + - name: push-secret + secret: + secretName: registry-push-credentials-ci-central - name: result-aggregator secret: secretName: result-aggregator diff --git a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml index 512aca5dd1c46..b09ef5dd8e44c 100644 --- a/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml +++ b/ci-operator/jobs/openshift-online/gcp-hcp-infra/openshift-online-gcp-hcp-infra-main-presubmits.yaml @@ -9,7 +9,9 @@ presubmits: context: ci/prow/check-generated-files decorate: true decoration_config: - skip_cloning: true + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -57,9 +59,6 @@ presubmits: readOnly: true serviceAccountName: ci-operator volumes: - - name: github-credentials-openshift-ci-robot-private-git-cloner - secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher @@ -79,7 +78,9 @@ presubmits: context: ci/prow/check-orphan-modules decorate: true decoration_config: - skip_cloning: true + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -127,9 +128,6 @@ presubmits: readOnly: true serviceAccountName: ci-operator volumes: - - name: github-credentials-openshift-ci-robot-private-git-cloner - secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher @@ -149,7 +147,9 @@ presubmits: context: ci/prow/e2e-platform decorate: true decoration_config: - skip_cloning: true + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner labels: ci-operator.openshift.io/cloud: platform-gcp ci-operator.openshift.io/cloud-cluster-profile: platform-gcp @@ -216,9 +216,6 @@ presubmits: - name: ci-pull-credentials secret: secretName: ci-pull-credentials - - name: github-credentials-openshift-ci-robot-private-git-cloner - secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher @@ -229,6 +226,68 @@ presubmits: secret: secretName: result-aggregator trigger: (?m)^/test( | .* )(e2e-platform|remaining-required),?($|\s.*) + - agent: kubernetes + always_run: true + branches: + - ^main$ + - ^main- + cluster: build06 + context: ci/prow/images + decorate: true + decoration_config: + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner + labels: + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: pull-ci-openshift-online-gcp-hcp-infra-main-images + rerun_command: /test images + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --oauth-token-path=/usr/local/github-credentials/oauth + - --report-credentials-file=/etc/report/credentials + - --target=[images] + - --target=gcp-hcp-e2e-tests + command: + - ci-operator + image: quay-proxy.ci.openshift.org/openshift/ci:ci_ci-operator_latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /usr/local/github-credentials + name: github-credentials-openshift-ci-robot-private-git-cloner + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator + trigger: (?m)^/test( | .* )images,?($|\s.*) - agent: kubernetes always_run: true branches: @@ -238,7 +297,9 @@ presubmits: context: ci/prow/terraform-test decorate: true decoration_config: - skip_cloning: true + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -303,9 +364,6 @@ presubmits: - name: ci-pull-credentials secret: secretName: ci-pull-credentials - - name: github-credentials-openshift-ci-robot-private-git-cloner - secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher @@ -325,7 +383,9 @@ presubmits: context: ci/prow/terraform-validate decorate: true decoration_config: - skip_cloning: true + oauth_token_secret: + key: oauth + name: github-credentials-openshift-ci-robot-private-git-cloner labels: ci.openshift.io/generator: prowgen pj-rehearse.openshift.io/can-be-rehearsed: "true" @@ -390,9 +450,6 @@ presubmits: - name: ci-pull-credentials secret: secretName: ci-pull-credentials - - name: github-credentials-openshift-ci-robot-private-git-cloner - secret: - secretName: github-credentials-openshift-ci-robot-private-git-cloner - name: manifest-tool-local-pusher secret: secretName: manifest-tool-local-pusher diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/OWNERS b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/OWNERS new file mode 100644 index 0000000000000..60325af98b50d --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/OWNERS @@ -0,0 +1,18 @@ +approvers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm +reviewers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh new file mode 100644 index 0000000000000..bbf336efc121d --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-commands.sh @@ -0,0 +1,507 @@ +#!/usr/bin/env bash +set -euo pipefail + +LOG="${ARTIFACT_DIR}/cleanup.log" +log() { echo "$(date -u '+%Y-%m-%d %H:%M:%S UTC') | $*" | tee -a "${LOG}"; } + +# Validate required dependencies +for cmd in jq gcloud curl; do + if ! command -v "${cmd}" &>/dev/null; then + echo "ERROR: ${cmd} not found in container" >&2 + exit 1 + fi +done + +# Use oc as kubectl — upi-installer image has oc but not kubectl +if command -v kubectl &>/dev/null; then + KUBECTL=kubectl +elif command -v oc &>/dev/null; then + KUBECTL=oc +else + echo "ERROR: neither kubectl nor oc found in container" >&2 + exit 1 +fi + +log "=== GCP HCP Infrastructure Cleanup ===" +log "This script performs comprehensive cleanup modeled after the Tekton cleanup task:" +log "1. Stop ArgoCD (prevents resource recreation)" +log "2. Delete Gateway API resources (triggers NEG cleanup)" +log "3. Force-delete NEGs" +log "4. Delete DNS records" +log "5. Delete GCP projects (bypasses terraform destroy for reliability)" +log "6. Clear TFC workspace state" +log "" + +# Authenticate with WIF +if [[ ! -f "${SHARED_DIR}/wif-cred.json" ]]; then + log "ERROR: WIF credential not found" + exit 1 +fi +gcloud auth login --cred-file="${SHARED_DIR}/wif-cred.json" --quiet + +# Read infrastructure info from SHARED_DIR +if [[ ! -f "${SHARED_DIR}/region-project-id" ]]; then + log "No region-project-id in SHARED_DIR — provision didn't complete, nothing to clean up" + exit 0 +fi + +REGION_PROJECT=$(<"${SHARED_DIR}/region-project-id") +REGION_CLUSTER=$(<"${SHARED_DIR}/region-cluster-name") +MC_PROJECT=$(<"${SHARED_DIR}/mc-project-id") +MC_CLUSTER=$(<"${SHARED_DIR}/mc-cluster-name") +REGION=${GCP_REGION:-us-central1} + +# Get project numbers +REGION_PROJECT_NUMBER=$(gcloud projects describe "${REGION_PROJECT}" --format='value(projectNumber)' 2>/dev/null || echo "") +MC_PROJECT_NUMBER=$(gcloud projects describe "${MC_PROJECT}" --format='value(projectNumber)' 2>/dev/null || echo "") + +log "Infrastructure to clean up:" +log " Region: ${REGION_PROJECT} (#${REGION_PROJECT_NUMBER}) / ${REGION_CLUSTER}" +log " MC: ${MC_PROJECT} (#${MC_PROJECT_NUMBER}) / ${MC_CLUSTER}" +log " Region: ${REGION}" +log "" + +# Helper: build kubeconfig with fresh access token using Connect Gateway +build_kubeconfig() { + local project_number=$1 + local cluster_name=$2 + local output_path=$3 + + local endpoint="${REGION}-connectgateway.googleapis.com/v1/projects/${project_number}/locations/${REGION}/gkeMemberships/${cluster_name}" + local token + token=$(gcloud auth print-access-token) + + cat > "${output_path}" </dev/null; then + log " ArgoCD namespace not found, skipping" + return 0 + fi + + # Scale down ArgoCD deployments + kc "${kubeconfig}" -n argocd scale deployment --all --replicas=0 2>/dev/null || true + kc "${kubeconfig}" -n argocd scale statefulset --all --replicas=0 2>/dev/null || true + + # Delete Applications and ApplicationSets + kc "${kubeconfig}" -n argocd delete applicationset --all --wait=false --timeout=30s 2>/dev/null || true + kc "${kubeconfig}" -n argocd delete application --all --wait=false --timeout=30s 2>/dev/null || true + + log " ArgoCD stopped" +} + +# ======================================================================== +# Phase 2: Delete Gateway API resources +# ======================================================================== +delete_gateway_resources() { + local kubeconfig=$1 + local label=$2 + + log "--- [${label}] Deleting Gateway API resources ---" + + for kind in gcpbackendpolicy healthcheckpolicy httproute gateway; do + local count + count=$(kc "${kubeconfig}" get "${kind}" --all-namespaces --no-headers 2>/dev/null | wc -l || echo "0") + count=$((count + 0)) + if [[ ${count} -gt 0 ]]; then + log " Deleting ${count} ${kind} resource(s)" + kc "${kubeconfig}" delete "${kind}" --all --all-namespaces --wait=false --timeout=30s 2>/dev/null || true + fi + done + + # Remove finalizers from stuck Gateway resources + log " Removing finalizers from Gateway resources" + local gateways_json + if gateways_json=$(kc "${kubeconfig}" get gateway --all-namespaces -o json 2>/dev/null); then + echo "${gateways_json}" | jq -r '.items[] | select(.metadata.finalizers != null) | "\(.metadata.namespace) \(.metadata.name)"' 2>/dev/null | \ + while read -r ns name; do + [[ -z "${ns}" || -z "${name}" ]] && continue + log " Patching gateway ${ns}/${name}" + kc "${kubeconfig}" patch gateway "${name}" -n "${ns}" \ + --type=json -p='[{"op": "remove", "path": "/metadata/finalizers"}]' 2>/dev/null || true + done + else + log " WARNING: Could not get Gateway resources (may not exist or cluster unreachable)" + fi +} + +# ======================================================================== +# Phase 3: Force-delete NEGs +# ======================================================================== +delete_negs() { + local project=$1 + local label=$2 + + log "--- [${label}] Force-deleting NEGs in ${project} ---" + + # Zonal NEGs + local zones + zones=$(gcloud compute zones list --filter="region:${REGION}" --format="value(name)" 2>/dev/null || echo "") + + for zone in ${zones}; do + local negs + negs=$(gcloud compute network-endpoint-groups list \ + --project="${project}" \ + --zones="${zone}" \ + --format="value(name)" 2>/dev/null || echo "") + + if [[ -n "${negs}" ]]; then + echo "${negs}" | while read -r neg_name; do + [[ -z "${neg_name}" ]] && continue + log " Deleting zonal NEG: ${neg_name} (zone: ${zone})" + gcloud compute network-endpoint-groups delete "${neg_name}" \ + --project="${project}" \ + --zone="${zone}" \ + --quiet 2>/dev/null || true + done + fi + done + + # Regional NEGs + local regional_negs + regional_negs=$(gcloud compute network-endpoint-groups list \ + --project="${project}" \ + --regions="${REGION}" \ + --format="value(name)" 2>/dev/null || echo "") + + if [[ -n "${regional_negs}" ]]; then + echo "${regional_negs}" | while read -r neg_name; do + [[ -z "${neg_name}" ]] && continue + log " Deleting regional NEG: ${neg_name}" + gcloud compute network-endpoint-groups delete "${neg_name}" \ + --project="${project}" \ + --region="${REGION}" \ + --quiet 2>/dev/null || true + done + fi +} + +# ======================================================================== +# Phase 4: Delete DNS records from regional zones +# ======================================================================== +delete_dns_records() { + local project=$1 + local label=$2 + + log "--- [${label}] Cleaning DNS records in ${project} ---" + + local zones + zones=$(gcloud dns managed-zones list --project="${project}" --format="value(name)" 2>/dev/null || echo "") + + if [[ -z "${zones}" ]]; then + log " No DNS zones found" + return 0 + fi + + while IFS= read -r zone; do + [[ -z "${zone}" ]] && continue + # Skip GKE Cloud DNS Scope zones (internal cluster DNS) — deleted with the cluster + if [[ "${zone}" == gke-* ]]; then + log " Skipping GKE internal zone: ${zone}" + continue + fi + log " Zone: ${zone}" + + local records + records=$(gcloud dns record-sets list \ + --project="${project}" \ + --zone="${zone}" \ + --format="csv[no-heading](name,type)" 2>/dev/null || echo "") + + while IFS=',' read -r name type; do + [[ -z "${name}" || -z "${type}" ]] && continue + [[ "${type}" == "SOA" || "${type}" == "NS" ]] && continue + log " Deleting ${type} record: ${name}" + gcloud dns record-sets delete "${name}" \ + --zone="${zone}" --project="${project}" --type="${type}" \ + --quiet 2>/dev/null || true + done <<< "${records}" + done <<< "${zones}" +} + +# ======================================================================== +# Phase 5: Force-delete GCP projects +# ======================================================================== +delete_project() { + local project=$1 + local label=$2 + + log "--- [${label}] Force-deleting project: ${project} ---" + + local output + local exit_code + output=$(gcloud projects delete "${project}" --quiet 2>&1) + exit_code=$? + + echo "${output}" | tee -a "${LOG}" + + if [[ ${exit_code} -eq 0 ]]; then + log " Project ${project} deletion initiated" + return 0 + else + log " ERROR: Failed to delete project ${project} (exit code: ${exit_code})" + return 1 + fi +} + +# ======================================================================== +# Phase 6: Clear TFC workspace state +# ======================================================================== +clear_tfc_workspace() { + log "--- Clearing TFC workspace state ---" + + # Read workspace info from SHARED_DIR + if [[ ! -f "${SHARED_DIR}/workspace-name" ]]; then + log " WARNING: No workspace-name in SHARED_DIR, skipping TFC cleanup" + return 0 + fi + + local workspace_name + workspace_name=$(<"${SHARED_DIR}/workspace-name") + + if [[ ! -f "/etc/terraform-cloud/token" ]]; then + log " WARNING: TFC token not found, skipping TFC cleanup" + return 0 + fi + + local tfc_token + tfc_token=$(<"/etc/terraform-cloud/token") + local tfc_org="${TFC_ORGANIZATION:-hp-platform-engineering}" + + log " Workspace: ${workspace_name}" + + # Get workspace ID + local workspace_id + workspace_id=$(curl -sS \ + --max-time 30 \ + --connect-timeout 10 \ + --header "Authorization: Bearer ${tfc_token}" \ + --header "Content-Type: application/vnd.api+json" \ + "https://app.terraform.io/api/v2/organizations/${tfc_org}/workspaces/${workspace_name}" 2>/dev/null | \ + jq -r '.data.id // empty' 2>/dev/null || echo "") + + if [[ -z "${workspace_id}" ]]; then + log " WARNING: Could not find workspace ID, may already be deleted" + return 0 + fi + + # Use terraform CLI to clear state, then safe-delete the workspace. + # The GCP projects are already deleted, so the state is stale. + # Install terraform, point it at the TFC workspace via cloud backend, + # and run 'terraform state rm' at the module level for speed (~3s for 400+ resources). + + # Install terraform (same version as .tool-versions) + local tf_version="1.15.8" + log " Installing terraform ${tf_version}..." + if ! curl -fsSL --max-time 120 \ + "https://releases.hashicorp.com/terraform/${tf_version}/terraform_${tf_version}_linux_amd64.zip" \ + -o /tmp/terraform.zip; then + log " WARNING: Failed to download terraform, skipping TFC cleanup" + return 0 + fi + if command -v unzip &>/dev/null; then + unzip -o -q /tmp/terraform.zip -d /tmp + else + python3 -c "import zipfile; zipfile.ZipFile('/tmp/terraform.zip').extractall('/tmp')" + fi + chmod +x /tmp/terraform + + # Create minimal terraform config with cloud backend + local tf_dir="/tmp/tfc-cleanup" + mkdir -p "${tf_dir}" + cat > "${tf_dir}/main.tf" < "$HOME/.terraformrc" <&1 | tee -a "${LOG}"; then + log " WARNING: terraform init failed, skipping TFC cleanup" + return 0 + fi + + # Force-unlock if the workspace is locked from a previous run + local lock_id + lock_id=$(/tmp/terraform -chdir="${tf_dir}" state list -no-color 2>&1 | \ + grep -oP 'lock ID: "\K[^"]+' || echo "") + if [[ -n "${lock_id}" ]]; then + log " Workspace locked (${lock_id}), force-unlocking..." + /tmp/terraform -chdir="${tf_dir}" force-unlock -force "${lock_id}" -no-color 2>&1 | tee -a "${LOG}" || true + fi + + # Remove all resources from state in one bulk operation. + # E2E state has 3 top-level modules plus data sources. Removing at + # module level clears child resources, then we remove any remaining + # top-level resources (data sources). + log " Clearing all resources from state..." + local resource_count + resource_count=$(/tmp/terraform -chdir="${tf_dir}" state list -no-color 2>/dev/null | wc -l | tr -d ' ') + resource_count=${resource_count:-0} + + if [[ ${resource_count} -eq 0 ]]; then + log " State is already empty" + else + log " Removing ${resource_count} resources..." + # Bulk remove: modules + data sources in one call (~3 seconds) + /tmp/terraform -chdir="${tf_dir}" state rm \ + module.customer_project \ + module.management_cluster \ + module.region \ + data.terraform_remote_state.commons \ + data.terraform_remote_state.global \ + data.terraform_remote_state.platform_ci \ + data.terraform_remote_state.service \ + -no-color 2>&1 | tail -5 | tee -a "${LOG}" || true + fi + + # Safe-delete the workspace (should succeed with 0 resources) + log " Deleting workspace..." + local http_code + http_code=$(curl -sS -o /dev/null -w "%{http_code}" \ + --max-time 30 \ + --connect-timeout 10 \ + --header "Authorization: Bearer ${tfc_token}" \ + --header "Content-Type: application/vnd.api+json" \ + --request POST \ + "https://app.terraform.io/api/v2/workspaces/${workspace_id}/actions/safe-delete" 2>/dev/null || echo "000") + + if [[ "${http_code}" == "204" || "${http_code}" == "200" ]]; then + log " TFC workspace deleted: ${workspace_name}" + else + log " WARNING: Could not delete TFC workspace (HTTP ${http_code}). Manual cleanup may be needed." + log " Workspace: https://app.terraform.io/app/${tfc_org}/workspaces/${workspace_name}" + fi +} + +# ======================================================================== +# Execute cleanup +# ======================================================================== + +# Build kubeconfigs with fresh tokens +REGION_KC="/tmp/region-kubeconfig" +MC_KC="/tmp/mc-kubeconfig" + +if [[ -n "${REGION_PROJECT_NUMBER}" ]]; then + build_kubeconfig "${REGION_PROJECT_NUMBER}" "${REGION_CLUSTER}" "${REGION_KC}" + # MC is registered in region's fleet + build_kubeconfig "${REGION_PROJECT_NUMBER}" "${MC_CLUSTER}" "${MC_KC}" +fi + +# Verify connectivity +log "Verifying cluster connectivity..." +REGION_CONNECTED=false +MC_CONNECTED=false + +if [[ -f "${REGION_KC}" ]] && kc "${REGION_KC}" get nodes --request-timeout=10s &>/dev/null; then + log " Region cluster: connected" + REGION_CONNECTED=true +else + log " Region cluster: unreachable" +fi + +if [[ -f "${MC_KC}" ]] && kc "${MC_KC}" get nodes --request-timeout=10s &>/dev/null; then + log " MC cluster: connected" + MC_CONNECTED=true +else + log " MC cluster: unreachable" +fi +log "" + +# Phase 1: Stop ArgoCD (MC first, then region) +if [[ "${MC_CONNECTED}" == "true" ]]; then + stop_argocd "${MC_KC}" "MC" || true +fi +if [[ "${REGION_CONNECTED}" == "true" ]]; then + stop_argocd "${REGION_KC}" "Region" || true +fi + +# Phase 2: Delete Gateway API resources +if [[ "${MC_CONNECTED}" == "true" ]]; then + delete_gateway_resources "${MC_KC}" "MC" || true +fi +if [[ "${REGION_CONNECTED}" == "true" ]]; then + delete_gateway_resources "${REGION_KC}" "Region" || true +fi + +# Wait for GKE Gateway controller to process deletions +# Note: 120s is a conservative estimate. GKE typically processes Gateway deletions +# within 60s, but we add buffer time to reduce NEG orphan risk. This wait can be +# tuned based on observed cleanup times. +if [[ "${REGION_CONNECTED}" == "true" || "${MC_CONNECTED}" == "true" ]]; then + log "" + log "Waiting 120s for GKE to process Gateway/NEG deletions..." + sleep 120 +fi + +# Phase 3: Force-delete remaining NEGs +delete_negs "${REGION_PROJECT}" "Region" || true +delete_negs "${MC_PROJECT}" "MC" || true + +# Phase 4: Delete DNS records +delete_dns_records "${REGION_PROJECT}" "Region" || true + +# Phase 5: Force-delete projects (this is the key difference from terraform destroy) +log "" +log "=== Force-deleting GCP projects ===" +log "This bypasses terraform destroy for reliability — project deletion cascades to all resources" +delete_project "${MC_PROJECT}" "MC" +delete_project "${REGION_PROJECT}" "Region" + +# Phase 6: Clear TFC workspace state +log "" +clear_tfc_workspace + +log "" +log "=== Cleanup complete ===" +log "Projects ${REGION_PROJECT} and ${MC_PROJECT} are now in PENDING_DELETE state (30-day soft delete)" +log "TFC workspace state has been cleared" diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.metadata.json b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.metadata.json new file mode 100644 index 0000000000000..9a7b65bb46bc2 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.metadata.json @@ -0,0 +1,25 @@ +{ + "path": "gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml", + "owners": { + "approvers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ], + "reviewers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml new file mode 100644 index 0000000000000..e5d0f312bb3f9 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/cleanup-infrastructure/gcp-hcp-cleanup-infrastructure-ref.yaml @@ -0,0 +1,41 @@ +ref: + as: gcp-hcp-cleanup-infrastructure + from_image: + namespace: ocp + name: "4.22" + tag: upi-installer + commands: gcp-hcp-cleanup-infrastructure-commands.sh + resources: + requests: + cpu: 500m + memory: 512Mi + credentials: + - mount_path: /etc/terraform-cloud + name: tfcloud-ci-secret + namespace: ci + env: + - name: GCP_REGION + default: "us-central1" + documentation: "GCP region for e2e infrastructure deployment" + timeout: 30m0s + grace_period: 10m0s + best_effort: true + documentation: |- + Performs comprehensive infrastructure cleanup for GCP HCP e2e environments. + + This step force-deletes GCP projects instead of relying on terraform destroy, + which can fail due to orphaned NEGs or other resources. The cleanup process: + + 1. Stops ArgoCD to prevent resource recreation + 2. Deletes Gateway API resources (triggers NEG cleanup) + 3. Force-deletes any remaining NEGs + 4. Deletes DNS records from regional zones + 5. Force-deletes GCP projects (cascades to all resources) + 6. Clears Terraform Cloud workspace state + + WIF credential is read from SHARED_DIR/wif-cred.json (written by hypershift-gcp-wif-auth). + TFC token is mounted from the tfcloud-ci-secret credential. + + Uses best_effort: true so cleanup failures don't block the job. + + This approach is more reliable than terraform destroy for ephemeral e2e environments. diff --git a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml index a18c3c14236d4..31696fe8d2bec 100644 --- a/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml +++ b/ci-operator/step-registry/gcp-hcp/e2e/gcp-hcp-e2e-workflow.yaml @@ -4,16 +4,17 @@ workflow: pre: - ref: hypershift-gcp-wif-auth - ref: gcp-hcp-tf-provision + - ref: gcp-hcp-generate-kubeconfigs test: - ref: gcp-hcp-verify-argocd-sync post: - - ref: gcp-hcp-tf-deprovision + - ref: gcp-hcp-cleanup-infrastructure documentation: |- E2E workflow for gcp-hcp-infra. Provisions full platform infrastructure (region + management cluster) using Terraform Cloud ephemeral workspaces, runs validation tests, and cleans up resources. Phase breakdown: - - Pre: Authenticate via WIF, provision infrastructure with terraform apply - - Test: Validate outputs and infrastructure (placeholder for now) - - Post: Clean up resources with terraform destroy (best-effort) + - Pre: Authenticate via WIF, provision infrastructure with terraform apply, generate kubeconfigs + - Test: Validate ArgoCD application sync on region and management clusters using Ginkgo v2 tests + - Post: Force-delete GCP projects and clear TFC workspace state (more reliable than terraform destroy) diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/OWNERS b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/OWNERS new file mode 100644 index 0000000000000..60325af98b50d --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/OWNERS @@ -0,0 +1,18 @@ +approvers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm +reviewers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh new file mode 100755 index 0000000000000..daeb52d8af75b --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-commands.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "=== GCP HCP Generate Kubeconfigs (Connect Gateway) ===" +echo "" + +# Authenticate with WIF credential +echo "Authenticating with WIF credential..." +if [[ ! -f "${SHARED_DIR}/wif-cred.json" ]]; then + echo "ERROR: WIF credential not found at ${SHARED_DIR}/wif-cred.json" + exit 1 +fi + +gcloud auth login --cred-file="${SHARED_DIR}/wif-cred.json" --quiet + +# Read terraform outputs +REGION_PROJECT=$(<"${SHARED_DIR}/region-project-id") +REGION_CLUSTER_NAME=$(<"${SHARED_DIR}/region-cluster-name") +MC_PROJECT=$(<"${SHARED_DIR}/mc-project-id") +MC_CLUSTER_NAME=$(<"${SHARED_DIR}/mc-cluster-name") +REGION=${GCP_REGION:-us-central1} + +echo " Region Project: ${REGION_PROJECT}" +echo " Region Cluster: ${REGION_CLUSTER_NAME}" +echo " MC Project: ${MC_PROJECT}" +echo " MC Cluster: ${MC_CLUSTER_NAME}" +echo " Region: ${REGION}" +echo "" + +# Generate region cluster kubeconfig with Connect Gateway endpoint +echo "Generating region cluster kubeconfig (Connect Gateway)..." +set +x # Hide sensitive token + +# Get project number for Connect Gateway URL +REGION_PROJECT_NUMBER=$(gcloud projects describe "${REGION_PROJECT}" --format='value(projectNumber)') + +# Build Connect Gateway endpoint +# Format: https://{region}-connectgateway.googleapis.com/v1/projects/{projectNumber}/locations/{region}/gkeMemberships/{clusterName} +REGION_ENDPOINT="${REGION}-connectgateway.googleapis.com/v1/projects/${REGION_PROJECT_NUMBER}/locations/${REGION}/gkeMemberships/${REGION_CLUSTER_NAME}" + +# Get access token (valid for 1 hour - sufficient for CI jobs) +ACCESS_TOKEN=$(gcloud auth print-access-token) + +# Connect Gateway uses Google's public TLS certificates (Google Trust Services), +# NOT the cluster's self-signed CA. No certificate-authority-data needed — +# the system CA bundle validates Google's certs automatically. +cat > "${SHARED_DIR}/region-kubeconfig" << 'KUBECONFIG_EOF' +apiVersion: v1 +kind: Config +clusters: +- cluster: + server: https://REGION_ENDPOINT_PLACEHOLDER + name: region-cluster +contexts: +- context: + cluster: region-cluster + user: gcp-user + name: region-context +current-context: region-context +users: +- name: gcp-user + user: + token: ACCESS_TOKEN_PLACEHOLDER +KUBECONFIG_EOF + +# Replace placeholders +sed -i "s|REGION_ENDPOINT_PLACEHOLDER|${REGION_ENDPOINT}|g" "${SHARED_DIR}/region-kubeconfig" +sed -i "s|ACCESS_TOKEN_PLACEHOLDER|${ACCESS_TOKEN}|g" "${SHARED_DIR}/region-kubeconfig" + +echo " Region kubeconfig written (Connect Gateway)" +echo " Endpoint: https://${REGION_ENDPOINT}" +set -x + +# Generate MC cluster kubeconfig (optional - test skips if unavailable) +echo "Generating management cluster kubeconfig (Connect Gateway)..." +set +x # Hide sensitive token +if gcloud projects describe "${MC_PROJECT}" --format='value(projectNumber)' &>/dev/null; then + + # MC clusters are registered in the REGION project's fleet (not their own). + # Connect Gateway URLs must reference the project that owns the fleet membership. + MC_ENDPOINT="${REGION}-connectgateway.googleapis.com/v1/projects/${REGION_PROJECT_NUMBER}/locations/${REGION}/gkeMemberships/${MC_CLUSTER_NAME}" + + cat > "${SHARED_DIR}/mc-kubeconfig" << 'KUBECONFIG_EOF' +apiVersion: v1 +kind: Config +clusters: +- cluster: + server: https://MC_ENDPOINT_PLACEHOLDER + name: mc-cluster +contexts: +- context: + cluster: mc-cluster + user: gcp-user + name: mc-context +current-context: mc-context +users: +- name: gcp-user + user: + token: ACCESS_TOKEN_PLACEHOLDER +KUBECONFIG_EOF + + # Replace placeholders + sed -i "s|MC_ENDPOINT_PLACEHOLDER|${MC_ENDPOINT}|g" "${SHARED_DIR}/mc-kubeconfig" + sed -i "s|ACCESS_TOKEN_PLACEHOLDER|${ACCESS_TOKEN}|g" "${SHARED_DIR}/mc-kubeconfig" + + echo " MC kubeconfig written (Connect Gateway)" + echo " Endpoint: https://${MC_ENDPOINT}" +else + echo " MC cluster unavailable - tests will skip MC validation" +fi +set -x + +echo "" +echo "Kubeconfig generation completed (Connect Gateway)" diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.metadata.json b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.metadata.json new file mode 100644 index 0000000000000..bfaaad12b8c38 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.metadata.json @@ -0,0 +1,25 @@ +{ + "path": "gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.yaml", + "owners": { + "approvers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ], + "reviewers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.yaml b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.yaml new file mode 100644 index 0000000000000..1b06a88fcfd60 --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/generate-kubeconfigs/gcp-hcp-generate-kubeconfigs-ref.yaml @@ -0,0 +1,25 @@ +ref: + as: gcp-hcp-generate-kubeconfigs + from_image: + namespace: ocp + name: "4.22" + tag: upi-installer + commands: gcp-hcp-generate-kubeconfigs-commands.sh + resources: + requests: + cpu: 100m + memory: 256Mi + timeout: 5m0s + documentation: |- + Generates kubeconfigs with embedded access tokens for region and management clusters. + + Uses gcloud to fetch cluster CA and endpoint, then creates static kubeconfigs + with embedded tokens (valid for ~60 minutes, sufficient for CI jobs). + + Depends on: + - hypershift-gcp-wif-auth (must run first) + - gcp-hcp-tf-provision (must write cluster names to SHARED_DIR) + + Outputs: + - ${SHARED_DIR}/region-kubeconfig: Region cluster kubeconfig + - ${SHARED_DIR}/mc-kubeconfig: MC cluster kubeconfig (optional) diff --git a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/OWNERS b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/OWNERS new file mode 100644 index 0000000000000..60325af98b50d --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/OWNERS @@ -0,0 +1,18 @@ +approvers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm +reviewers: +- apahim +- cblecker +- ckandag +- cristianoveiga +- floresroger +- gbarabasz +- jimdaga +- patjlm diff --git a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh new file mode 100644 index 0000000000000..ab35b38c06a0f --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-commands.sh @@ -0,0 +1,290 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "=== GCP HCP Pre-Destroy Cleanup ===" +echo "" +echo "Removes GCP resources created by ArgoCD-deployed apps that block" +echo "terraform destroy: NEGs (block VPC deletion), DNS records (block" +echo "zone deletion), and Gateway API resources (create NEGs)." +echo "" + +# Authenticate with WIF credential (fresh token for this step) +if [[ ! -f "${SHARED_DIR}/wif-cred.json" ]]; then + echo "WARNING: WIF credential not found, skipping cleanup" + exit 0 +fi +gcloud auth login --cred-file="${SHARED_DIR}/wif-cred.json" --quiet + +# Read cluster info from SHARED_DIR (written by tf-provision) +# If provision didn't complete, these files won't exist — nothing to clean up. +if [[ ! -f "${SHARED_DIR}/region-project-id" ]]; then + echo "No region-project-id in SHARED_DIR — provision likely didn't complete. Nothing to clean up." + exit 0 +fi + +REGION_PROJECT=$(<"${SHARED_DIR}/region-project-id") +REGION_CLUSTER_NAME=$(<"${SHARED_DIR}/region-cluster-name") +MC_PROJECT=$(<"${SHARED_DIR}/mc-project-id") +MC_CLUSTER_NAME=$(<"${SHARED_DIR}/mc-cluster-name") +REGION=${GCP_REGION:-us-central1} + +# Get project numbers for Connect Gateway +REGION_PROJECT_NUMBER=$(gcloud projects describe "${REGION_PROJECT}" --format='value(projectNumber)' 2>/dev/null || echo "") + +echo "Region: ${REGION_PROJECT} / ${REGION_CLUSTER_NAME}" +echo "MC: ${MC_PROJECT} / ${MC_CLUSTER_NAME}" +echo "" + +# Safe count: strip whitespace from wc -l output for arithmetic +safe_count() { + local result + result=$(echo "$1" | tr -d '[:space:]') + result=${result:-0} + echo "${result}" +} + +# Build kubeconfigs using Connect Gateway with a FRESH access token. +# The original kubeconfigs from generate-kubeconfigs embed a static token +# that expires after 1 hour. This step runs in the post phase, potentially +# 1+ hours after the token was generated. +build_kubeconfig() { + local project_number=$1 + local cluster_name=$2 + local output_path=$3 + + local endpoint="${REGION}-connectgateway.googleapis.com/v1/projects/${project_number}/locations/${REGION}/gkeMemberships/${cluster_name}" + local token + token=$(gcloud auth print-access-token) + + cat > "${output_path}" </dev/null; then + echo " ArgoCD namespace not found, skipping" + return 0 + fi + + # Scale down ArgoCD + kc "${kubeconfig}" -n argocd scale deployment --all --replicas=0 2>/dev/null || true + kc "${kubeconfig}" -n argocd scale statefulset --all --replicas=0 2>/dev/null || true + + # Delete all Applications and ApplicationSets (stops app-of-apps) + kc "${kubeconfig}" -n argocd delete applicationset --all --wait=false --timeout=30s 2>/dev/null || true + kc "${kubeconfig}" -n argocd delete application --all --wait=false --timeout=30s 2>/dev/null || true + + # Wait briefly for deletions to propagate + local elapsed=0 + while [[ ${elapsed} -lt 60 ]]; do + local count + count=$(safe_count "$(kc "${kubeconfig}" -n argocd get applications --no-headers 2>/dev/null | wc -l)") + if [[ ${count} -eq 0 ]]; then + echo " All Applications deleted" + break + fi + echo " Waiting for ${count} Application(s) to delete... (${elapsed}s)" + sleep 10 + elapsed=$((elapsed + 10)) + done +} + +# ===================================================================== +# Phase 2: Delete Gateway API resources (triggers GKE NEG cleanup) +# ===================================================================== +delete_gateway_resources() { + local kubeconfig=$1 + local label=$2 + + echo "--- [${label}] Deleting Gateway API resources ---" + + for kind in gcpbackendpolicy healthcheckpolicy httproute gateway; do + local count + count=$(safe_count "$(kc "${kubeconfig}" get "${kind}" --all-namespaces --no-headers 2>/dev/null | wc -l)") + if [[ ${count} -gt 0 ]]; then + echo " Deleting ${count} ${kind} resource(s)" + kc "${kubeconfig}" delete "${kind}" --all --all-namespaces --wait=false --timeout=30s 2>/dev/null || true + fi + done +} + +# ===================================================================== +# Phase 3: Delete DNS records from regional zones +# External-dns creates records in the tools zone that block zone deletion +# ===================================================================== +delete_dns_records() { + local project=$1 + local label=$2 + + echo "--- [${label}] Cleaning DNS records in ${project} ---" + + local zones + zones=$(gcloud dns managed-zones list --project="${project}" --format="value(name)" 2>/dev/null || echo "") + + if [[ -z "${zones}" ]]; then + echo " No DNS zones found" + return 0 + fi + + while IFS= read -r zone; do + [[ -z "${zone}" ]] && continue + echo " Zone: ${zone}" + + local records + records=$(gcloud dns record-sets list \ + --project="${project}" \ + --zone="${zone}" \ + --format="csv[no-heading](name,type)" 2>/dev/null || echo "") + + while IFS=',' read -r name type; do + [[ -z "${name}" || -z "${type}" ]] && continue + [[ "${type}" == "SOA" || "${type}" == "NS" ]] && continue + echo " Deleting ${type} record: ${name}" + gcloud dns record-sets delete "${name}" \ + --zone="${zone}" --project="${project}" --type="${type}" \ + --quiet 2>/dev/null || true + done <<< "${records}" + done <<< "${zones}" +} + +# ===================================================================== +# Phase 4: Force-delete remaining NEGs via gcloud +# GKE should clean up NEGs after Gateway deletion, but sometimes +# orphaned NEGs remain and block VPC network deletion. +# ===================================================================== +delete_negs() { + local project=$1 + local label=$2 + + echo "--- [${label}] Cleaning up NEGs in ${project} ---" + + # Zonal NEGs + local negs + negs=$(gcloud compute network-endpoint-groups list \ + --project="${project}" \ + --format="csv[no-heading](name,zone)" 2>/dev/null || echo "") + + if [[ -z "${negs}" ]]; then + echo " No NEGs found" + return 0 + fi + + while IFS=',' read -r name zone_url; do + [[ -z "${name}" || -z "${zone_url}" ]] && continue + # zone_url is a full URL — extract just the zone name + local zone + zone=$(basename "${zone_url}") + echo " Deleting NEG: ${name} (zone: ${zone})" + gcloud compute network-endpoint-groups delete "${name}" \ + --project="${project}" --zone="${zone}" \ + --quiet 2>/dev/null || true + done <<< "${negs}" +} + +# ===================================================================== +# Execute cleanup on both clusters +# ===================================================================== + +# Build fresh kubeconfigs with new access tokens +CLEANUP_REGION_KC="/tmp/cleanup-region-kubeconfig" +CLEANUP_MC_KC="/tmp/cleanup-mc-kubeconfig" + +if [[ -n "${REGION_PROJECT_NUMBER}" ]]; then + build_kubeconfig "${REGION_PROJECT_NUMBER}" "${REGION_CLUSTER_NAME}" "${CLEANUP_REGION_KC}" + + # MC is registered in region's fleet + build_kubeconfig "${REGION_PROJECT_NUMBER}" "${MC_CLUSTER_NAME}" "${CLEANUP_MC_KC}" +fi + +# Verify connectivity before proceeding +echo "Verifying cluster connectivity..." +if [[ -f "${CLEANUP_REGION_KC}" ]] && kc "${CLEANUP_REGION_KC}" get nodes --request-timeout=10s &>/dev/null; then + echo " Region cluster: connected" + REGION_CONNECTED=true +else + echo " Region cluster: unreachable (Connect Gateway or cluster may be unavailable)" + REGION_CONNECTED=false +fi + +if [[ -f "${CLEANUP_MC_KC}" ]] && kc "${CLEANUP_MC_KC}" get nodes --request-timeout=10s &>/dev/null; then + echo " MC cluster: connected" + MC_CONNECTED=true +else + echo " MC cluster: unreachable (Connect Gateway or cluster may be unavailable)" + MC_CONNECTED=false +fi +echo "" + +# Phase 1: Stop ArgoCD (MC first, then region) +if [[ "${MC_CONNECTED}" == "true" ]]; then + stop_argocd "${CLEANUP_MC_KC}" "MC" || true +fi +if [[ "${REGION_CONNECTED}" == "true" ]]; then + stop_argocd "${CLEANUP_REGION_KC}" "Region" || true +fi + +# Phase 2: Delete Gateway API resources +if [[ "${MC_CONNECTED}" == "true" ]]; then + delete_gateway_resources "${CLEANUP_MC_KC}" "MC" || true +fi +if [[ "${REGION_CONNECTED}" == "true" ]]; then + delete_gateway_resources "${CLEANUP_REGION_KC}" "Region" || true +fi + +# Wait for GKE controller to clean up NEGs after Gateway deletion +if [[ "${REGION_CONNECTED}" == "true" || "${MC_CONNECTED}" == "true" ]]; then + echo "" + echo "Waiting 60s for GKE to process Gateway/NEG deletions..." + sleep 60 +fi + +# Phase 3: Delete DNS records from regional zones +delete_dns_records "${REGION_PROJECT}" "Region" || true + +# Phase 4: Force-delete remaining NEGs +delete_negs "${REGION_PROJECT}" "Region" || true +delete_negs "${MC_PROJECT}" "MC" || true + +# Final NEG check +echo "" +echo "=== Final NEG check ===" +echo "Region:" +gcloud compute network-endpoint-groups list --project="${REGION_PROJECT}" --format="table(name,zone)" 2>/dev/null || echo " (unable to list — project may be in deletion)" +echo "MC:" +gcloud compute network-endpoint-groups list --project="${MC_PROJECT}" --format="table(name,zone)" 2>/dev/null || echo " (unable to list — project may be in deletion)" + +echo "" +echo "=== Pre-destroy cleanup complete ===" diff --git a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.metadata.json b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.metadata.json new file mode 100644 index 0000000000000..4dab6c12a73cd --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.metadata.json @@ -0,0 +1,25 @@ +{ + "path": "gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.yaml", + "owners": { + "approvers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ], + "reviewers": [ + "apahim", + "cblecker", + "ckandag", + "cristianoveiga", + "floresroger", + "gbarabasz", + "jimdaga", + "patjlm" + ] + } +} \ No newline at end of file diff --git a/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.yaml b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.yaml new file mode 100644 index 0000000000000..5e12a490426fd --- /dev/null +++ b/ci-operator/step-registry/gcp-hcp/pre-destroy-cleanup/gcp-hcp-pre-destroy-cleanup-ref.yaml @@ -0,0 +1,37 @@ +ref: + as: gcp-hcp-pre-destroy-cleanup + from_image: + namespace: ocp + name: "4.22" + tag: upi-installer + commands: gcp-hcp-pre-destroy-cleanup-commands.sh + env: + - name: GCP_REGION + default: "us-central1" + documentation: "GCP region for e2e infrastructure deployment" + resources: + requests: + cpu: 100m + memory: 256Mi + timeout: 10m0s + best_effort: true + documentation: |- + Cleans up GCP resources created by ArgoCD-deployed applications that + block terraform destroy. Runs in the post phase before tf-deprovision. + + Cleanup phases: + 1. Stop ArgoCD (scale down, delete Applications/ApplicationSets) + 2. Delete Gateway API resources (HTTPRoute, Gateway, etc.) to trigger + GKE NEG cleanup + 3. Delete DNS records from regional zones (external-dns creates records + that block zone deletion) + 4. Force-delete remaining NEGs via gcloud (block VPC network deletion) + + Does NOT delete resources that terraform destroy handles (IAM, KCC + resources, Spanner databases, etc. — these cascade with project deletion). + + Uses best_effort: true so cleanup failures don't block the job. + + Depends on: + - hypershift-gcp-wif-auth (WIF credential in SHARED_DIR) + - gcp-hcp-tf-provision (cluster names/project IDs in SHARED_DIR) diff --git a/ci-operator/step-registry/gcp-hcp/tf-deprovision/gcp-hcp-tf-deprovision-commands.sh b/ci-operator/step-registry/gcp-hcp/tf-deprovision/gcp-hcp-tf-deprovision-commands.sh index 47f7abcc89cb7..a08ac50e0d44f 100755 --- a/ci-operator/step-registry/gcp-hcp/tf-deprovision/gcp-hcp-tf-deprovision-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/tf-deprovision/gcp-hcp-tf-deprovision-commands.sh @@ -117,8 +117,12 @@ destroy_wait=30 while (( destroy_attempt <= MAX_DESTROY_ATTEMPTS )); do log "DESTROY ATTEMPT: ${destroy_attempt}/${MAX_DESTROY_ATTEMPTS}" - destroy_output=$(terraform destroy -auto-approve -no-color 2>&1) - destroy_exit=$? + # Capture terraform output and exit code without triggering errexit + if destroy_output=$(terraform destroy -auto-approve -no-color 2>&1); then + destroy_exit=0 + else + destroy_exit=$? + fi echo "${destroy_output}" | tee -a "${LOG}" if [[ ${destroy_exit} -eq 0 ]]; then diff --git a/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh b/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh index 1da3e047bf4e1..b854bb8d43bf2 100755 --- a/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/tf-provision/gcp-hcp-tf-provision-commands.sh @@ -30,7 +30,7 @@ tfc_api_call() { local attempt=1 while (( attempt <= max_retries )); do - if output=$(curl -sf "$@" 2>&1); then + if output=$(curl -sf --connect-timeout 10 --max-time 60 "$@" 2>&1); then echo "${output}" return 0 fi @@ -187,6 +187,71 @@ log "TFC workspace: https://app.terraform.io/app/${TFC_ORG}/workspaces/${WORKSPA # Errors that retrying cannot fix — fail fast instead of wasting time NON_TRANSIENT_ERRORS="quota.*exceeded|forbidden|invalid.*configuration|unauthorized" +# Workaround for hashicorp/terraform-provider-google#22533: +# google_firestore_database has a read-after-write consistency bug on newly +# created projects. The provider creates the database, but the immediate GET +# returns 404, so the resource is dropped from state. The next apply attempt +# then fails with 409 "already exists" because the database is in GCP but not +# in state. Fix: detect the 409, extract the project ID, and import the +# orphaned resources before retrying. +import_orphaned_firestore() { + local output="$1" + + # Check for the Firestore 409 pattern. + # TFC remote output splits the error across multiple lines with │ prefixes, + # so we check for both strings independently rather than on a single line. + if ! echo "${output}" | grep -q "Database already exists"; then + return 1 + fi + if ! echo "${output}" | grep -q "google_firestore_database"; then + return 1 + fi + + log "Detected Firestore provider bug (hashicorp/terraform-provider-google#22533)" + log "Database exists in GCP but not in state — attempting import..." + + # Extract MC project ID from terraform state (the project resource is + # created before Firestore, so it should be in state) + local mc_project + mc_project=$(terraform output -json 2>/dev/null | jq -r '.management_cluster.value.project_id // empty' 2>/dev/null || echo "") + + if [[ -z "${mc_project}" ]]; then + log "WARNING: Could not extract MC project ID from outputs, trying state..." + mc_project=$(terraform show -json 2>/dev/null | \ + jq -r '.. | objects | select(.address? == "module.management_cluster.module.project.google_project.main") | .values.project_id // empty' 2>/dev/null || echo "") + fi + + if [[ -z "${mc_project}" ]]; then + log "ERROR: Could not determine MC project ID for import" + return 1 + fi + + log "MC Project ID: ${mc_project}" + + local imported=0 + for db_name in specs status; do + local address="module.management_cluster.google_firestore_database.${db_name}" + local import_id="projects/${mc_project}/databases/${db_name}" + + # Only import if the error mentions this specific database + if echo "${output}" | grep -q "google_firestore_database.${db_name}"; then + log "Importing ${address} <- ${import_id}" + if terraform import -no-color "${address}" "${import_id}" 2>&1 | tee -a "${LOG}"; then + log "Successfully imported ${db_name} database" + imported=$((imported + 1)) + else + log "WARNING: Failed to import ${db_name} database" + fi + fi + done + + if [[ ${imported} -gt 0 ]]; then + log "Imported ${imported} orphaned Firestore database(s)" + return 0 + fi + return 1 +} + MAX_APPLY_ATTEMPTS=5 apply_attempt=1 apply_wait=30 @@ -194,8 +259,12 @@ apply_wait=30 while (( apply_attempt <= MAX_APPLY_ATTEMPTS )); do log "APPLY ATTEMPT: ${apply_attempt}/${MAX_APPLY_ATTEMPTS}" - apply_output=$(terraform apply -auto-approve -no-color 2>&1) - apply_exit=$? + # Capture terraform output and exit code without triggering errexit + if apply_output=$(terraform apply -auto-approve -no-color 2>&1); then + apply_exit=0 + else + apply_exit=$? + fi echo "${apply_output}" | tee -a "${LOG}" if [[ ${apply_exit} -eq 0 ]]; then @@ -213,6 +282,12 @@ while (( apply_attempt <= MAX_APPLY_ATTEMPTS )); do exit 1 fi + # Handle Firestore provider bug: import orphaned databases before retry + if import_orphaned_firestore "${apply_output}"; then + log "Firestore import succeeded — retrying apply immediately" + apply_wait=10 # Short wait after import + fi + if (( apply_attempt < MAX_APPLY_ATTEMPTS )); then log "Transient failure — waiting ${apply_wait}s before retry..." log "This is common due to GCP eventual consistency (IAM propagation, API enablement)" @@ -247,6 +322,7 @@ fi # Write individual outputs to SHARED_DIR for downstream steps jq -r '.region.value.project_id // empty' /tmp/tf-outputs.json > "${SHARED_DIR}/region-project-id" +jq -r '.region.value.cluster_name // empty' /tmp/tf-outputs.json > "${SHARED_DIR}/region-cluster-name" jq -r '.management_cluster.value.project_id // empty' /tmp/tf-outputs.json > "${SHARED_DIR}/mc-project-id" jq -r '.management_cluster.value.cluster_name // empty' /tmp/tf-outputs.json > "${SHARED_DIR}/mc-cluster-name" jq -r '.management_cluster.value.cluster_endpoint // empty' /tmp/tf-outputs.json > "${SHARED_DIR}/mc-cluster-endpoint" @@ -256,7 +332,7 @@ echo "${WORKSPACE_NAME}" > "${SHARED_DIR}/workspace-name" echo "${RUN_ID}" > "${SHARED_DIR}/run-id" # Validate critical outputs were written -for output_file in region-project-id mc-project-id mc-cluster-name mc-cluster-endpoint workspace-name run-id; do +for output_file in region-project-id region-cluster-name mc-project-id mc-cluster-name mc-cluster-endpoint workspace-name run-id; do if [[ ! -s "${SHARED_DIR}/${output_file}" ]]; then log "ERROR: Output file ${output_file} is empty or missing" exit 1 diff --git a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh index 863e37b504240..8f1288497fd97 100755 --- a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh +++ b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-commands.sh @@ -1,29 +1,39 @@ #!/usr/bin/env bash set -euo pipefail -echo "=== GCP HCP E2E Test Placeholder ===" +echo "=== GCP HCP ArgoCD Sync Validation ===" echo "" -echo "Validating provision outputs..." -# Check all expected outputs exist and are non-empty -REQUIRED_OUTPUTS="region-project-id mc-project-id mc-cluster-name mc-cluster-endpoint workspace-name run-id" +# Verify kubeconfigs exist (generated by gcp-hcp-generate-kubeconfigs step) +if [[ ! -f "${SHARED_DIR}/region-kubeconfig" ]]; then + echo "ERROR: Region kubeconfig not found at ${SHARED_DIR}/region-kubeconfig" + echo "The gcp-hcp-generate-kubeconfigs step must run before this step" + exit 1 +fi -for output in ${REQUIRED_OUTPUTS}; do - if [[ ! -s "${SHARED_DIR}/${output}" ]]; then - echo "ERROR: Missing or empty output: ${output}" - exit 1 - fi - echo " OK ${output}: $(<"${SHARED_DIR}/${output}")" -done +# Set environment variables for Ginkgo test +export REGION_KUBECONFIG="${SHARED_DIR}/region-kubeconfig" + +# MC kubeconfig is optional +if [[ -f "${SHARED_DIR}/mc-kubeconfig" ]]; then + export MC_KUBECONFIG="${SHARED_DIR}/mc-kubeconfig" + echo "Using MC kubeconfig from ${SHARED_DIR}/mc-kubeconfig" +else + echo "MC kubeconfig not found - tests will skip management cluster validation" +fi + +# Optional: Override sync timeouts (defaults to 30m in test code) +# export REGION_SYNC_TIMEOUT="30m" +# export MC_SYNC_TIMEOUT="30m" + +# Run Ginkgo v2 test binary +echo "Running ArgoCD sync validation tests..." +/usr/bin/test-e2e \ + --ginkgo.v \ + --ginkgo.no-color \ + --ginkgo.timeout=85m \ + --ginkgo.junit-report="${ARTIFACT_DIR}/junit_argocd_sync.xml" \ + --ginkgo.label-filter="argocd" echo "" -echo "Infrastructure Details:" -echo " Region Project: $(<"${SHARED_DIR}/region-project-id")" -echo " MC Project: $(<"${SHARED_DIR}/mc-project-id")" -echo " MC Cluster: $(<"${SHARED_DIR}/mc-cluster-name")" -echo " Workspace: $(<"${SHARED_DIR}/workspace-name")" -echo "" -echo "All provision outputs validated successfully" -echo "" -echo "NOTE: This is a placeholder. Real ArgoCD sync validation" -echo " will be implemented in a future story." +echo "✓ ArgoCD sync validation completed successfully" diff --git a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml index f27861752811c..d42fc83129ee6 100644 --- a/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml +++ b/ci-operator/step-registry/gcp-hcp/verify-argocd-sync/gcp-hcp-verify-argocd-sync-ref.yaml @@ -1,15 +1,29 @@ ref: as: gcp-hcp-verify-argocd-sync - from_image: - namespace: ocp - name: "4.22" - tag: cli + from: gcp-hcp-e2e-tests commands: gcp-hcp-verify-argocd-sync-commands.sh resources: requests: - cpu: 100m - memory: 256Mi - timeout: 5m0s + cpu: 500m + memory: 1Gi + timeout: 90m0s documentation: |- - Placeholder test step that validates provision outputs are available. - Real ArgoCD sync validation will be added in a future story. + Validates that ArgoCD applications have synced successfully on the region + and management clusters after infrastructure provisioning. + + Uses Ginkgo v2 tests to: + - Wait for ArgoCD namespace to be bootstrapped by Fleet Config Sync + - Wait for ArgoCD server deployment to become ready + - Poll all ArgoCD Application CRs for Synced status + - Auto-retry stuck applications every 2 minutes + - Report detailed status and failures + + Depends on: + - gcp-hcp-generate-kubeconfigs (must run first to generate kubeconfigs) + + Environment variables (optional): + - REGION_SYNC_TIMEOUT: Timeout for region cluster sync (default: 30m) + - MC_SYNC_TIMEOUT: Timeout for MC cluster sync (default: 30m) + + Outputs: + - ${ARTIFACT_DIR}/junit_argocd_sync.xml: JUnit test results for Prow