From 1f0b9072bb34334134ed5029171ac3f513ecea69 Mon Sep 17 00:00:00 2001 From: Alessandro Rossi Date: Tue, 14 Jul 2026 16:52:41 +0200 Subject: [PATCH 1/5] feat(hypershift-install): add OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE parameter OVERRIDE_IMAGE_HYPERSHIFT_OPERATOR (ci-operator's env var mechanism) has a race condition: both the override and the base_images step write to the same pipeline ImageStream tag concurrently. Since IST Create is an upsert, the base_images step consistently wins because it finishes last (resolveOfficialImport makes API calls, the override does not). The override is silently discarded. --dependency-override-param would work, but Gangway only supports env vars in the payload, not CLI arguments. This commit adds OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE as a step parameter that bypasses the pipeline ImageStream entirely. When set, the script overrides both the operator image (--hypershift-image) and the hcp CLI binary, extracting it from the same image to avoid version skew. Also refactors the existing HO_MULTI and INSTALL_FROM_LATEST branches into a shared extract_hcp_cli() function and adds WARNING log lines to all override paths for better observability. Ref: CNTRLPLANE-3434 Co-Authored-By: Claude Opus 4.6 --- .../install/hypershift-install-commands.sh | 33 +++++++++++++------ .../install/hypershift-install-ref.yaml | 3 ++ 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh b/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh index b0a103090a25f..d7f4cf309e913 100755 --- a/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh +++ b/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh @@ -6,20 +6,33 @@ EXTRA_ARGS="" HCP_CLI="bin/hypershift" OPERATOR_IMAGE=$HYPERSHIFT_RELEASE_LATEST -if [[ $HO_MULTI == "true" ]]; then - OPERATOR_IMAGE="quay.io/acm-d/rhtap-hypershift-operator:latest" +extract_hcp_cli() { + local image=$1 oc extract secret/pull-secret -n openshift-config --to=/tmp --confirm - mkdir /tmp/hs-cli - oc image extract quay.io/acm-d/rhtap-hypershift-operator:latest --path /usr/bin/hypershift:/tmp/hs-cli --registry-config=/tmp/.dockerconfigjson --filter-by-os="linux/amd64" + mkdir -p /tmp/hs-cli + oc image extract "${image}" --path /usr/bin/hypershift:/tmp/hs-cli --registry-config=/tmp/.dockerconfigjson --filter-by-os="linux/amd64" chmod +x /tmp/hs-cli/hypershift HCP_CLI="/tmp/hs-cli/hypershift" -elif [[ $INSTALL_FROM_LATEST == "true" ]]; then +} + +if [[ -n "${OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE:-}" ]]; then + echo "WARNING: Overriding OPERATOR_IMAGE" + echo " Default: ${HYPERSHIFT_RELEASE_LATEST}" + echo " Override: ${OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE}" + OPERATOR_IMAGE="${OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE}" + extract_hcp_cli "${OPERATOR_IMAGE}" +elif [[ "${HO_MULTI}" == "true" ]]; then + # extract_hcp_cli uses --filter-by-os=linux/amd64 because the step container is always amd64 + echo "WARNING: Using ACM downstream multi-arch operator image (CI pipeline builds amd64 only)" + echo " Default: ${HYPERSHIFT_RELEASE_LATEST}" + echo " Override: quay.io/acm-d/rhtap-hypershift-operator:latest" + OPERATOR_IMAGE="quay.io/acm-d/rhtap-hypershift-operator:latest" + extract_hcp_cli "${OPERATOR_IMAGE}" +elif [[ "${INSTALL_FROM_LATEST}" == "true" ]]; then # We should use the hypershift cli from the HYPERSHIFT_RELEASE_LATEST - oc extract secret/pull-secret -n openshift-config --to=/tmp --confirm - mkdir /tmp/hs-cli - oc image extract $HYPERSHIFT_RELEASE_LATEST --path /usr/bin/hypershift:/tmp/hs-cli --registry-config=/tmp/.dockerconfigjson --filter-by-os="linux/amd64" - chmod +x /tmp/hs-cli/hypershift - HCP_CLI="/tmp/hs-cli/hypershift" + echo "WARNING: Extracting hcp CLI from dependency image instead of step container" + echo " Image: ${OPERATOR_IMAGE}" + extract_hcp_cli "${OPERATOR_IMAGE}" fi if [ "${TECH_PREVIEW_NO_UPGRADE}" = "true" ]; then diff --git a/ci-operator/step-registry/hypershift/install/hypershift-install-ref.yaml b/ci-operator/step-registry/hypershift/install/hypershift-install-ref.yaml index 19f5ca79879ee..c2ba772456fd7 100644 --- a/ci-operator/step-registry/hypershift/install/hypershift-install-ref.yaml +++ b/ci-operator/step-registry/hypershift/install/hypershift-install-ref.yaml @@ -59,6 +59,9 @@ ref: - name: AZURE_PLS_RESOURCE_GROUP default: "" documentation: "Azure resource group of the management cluster where Private Link Services reside. Falls back to SHARED_DIR/azure_pls_resource_group if empty." + - name: OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE + default: "" + documentation: "If set, overrides both the HyperShift Operator image and the hcp CLI used during install, bypassing the pipeline ImageStream dependency resolution." commands: hypershift-install-commands.sh credentials: - mount_path: /etc/hypershift-pool-aws-credentials From 5c8d562004635fd6878f2c3d9d15575772d959b5 Mon Sep 17 00:00:00 2001 From: Alessandro Rossi Date: Tue, 14 Jul 2026 17:25:59 +0200 Subject: [PATCH 2/5] fix(hypershift-install): use mktemp for CLI extraction and warn on conflicting overrides Address CodeRabbit review feedback: 1. Replace hardcoded /tmp/hs-cli with mktemp -d to avoid predictable temporary paths in the extract_hcp_cli function (CWE-377). 2. Add precedence warning when multiple image override flags are set simultaneously (OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE, HO_MULTI, INSTALL_FROM_LATEST). Logs the priority order so operators can understand which override takes effect without inspecting the script. Co-Authored-By: Claude Opus 4.6 --- .../install/hypershift-install-commands.sh | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh b/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh index d7f4cf309e913..1ecec2e70a129 100755 --- a/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh +++ b/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh @@ -8,13 +8,26 @@ OPERATOR_IMAGE=$HYPERSHIFT_RELEASE_LATEST extract_hcp_cli() { local image=$1 - oc extract secret/pull-secret -n openshift-config --to=/tmp --confirm - mkdir -p /tmp/hs-cli - oc image extract "${image}" --path /usr/bin/hypershift:/tmp/hs-cli --registry-config=/tmp/.dockerconfigjson --filter-by-os="linux/amd64" - chmod +x /tmp/hs-cli/hypershift - HCP_CLI="/tmp/hs-cli/hypershift" + local cli_dir + cli_dir="$(mktemp -d "${TMPDIR:-/tmp}/hs-cli.XXXXXX")" + oc extract secret/pull-secret -n openshift-config --to="${cli_dir}" --confirm + oc image extract "${image}" --path "/usr/bin/hypershift:${cli_dir}" --registry-config="${cli_dir}/.dockerconfigjson" --filter-by-os="linux/amd64" + chmod +x "${cli_dir}/hypershift" + HCP_CLI="${cli_dir}/hypershift" } +OVERRIDE_COUNT=0 +[[ -n "${OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE:-}" ]] && ((OVERRIDE_COUNT++)) +[[ "${HO_MULTI}" == "true" ]] && ((OVERRIDE_COUNT++)) +[[ "${INSTALL_FROM_LATEST}" == "true" ]] && ((OVERRIDE_COUNT++)) + +if [[ ${OVERRIDE_COUNT} -gt 1 ]]; then + echo "WARNING: Multiple image override flags are set. Precedence (highest to lowest):" + echo " 1. OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE" + echo " 2. HO_MULTI" + echo " 3. INSTALL_FROM_LATEST" +fi + if [[ -n "${OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE:-}" ]]; then echo "WARNING: Overriding OPERATOR_IMAGE" echo " Default: ${HYPERSHIFT_RELEASE_LATEST}" From cf017dabaeb5ea441d8c67fe22f571e1744c323b Mon Sep 17 00:00:00 2001 From: Alessandro Rossi Date: Wed, 15 Jul 2026 09:54:42 +0200 Subject: [PATCH 3/5] fix(hypershift-install): use if/then and arithmetic assignment for OVERRIDE_COUNT The previous `[[ ]] && ((OVERRIDE_COUNT++))` pattern caused the script to exit under `set -eux` on bash 4.x/5.x (RHEL CI containers). `((expr))` returns exit code 1 when the expression evaluates to 0. With post-increment (`OVERRIDE_COUNT++`), the first increment from 0 evaluates to 0 (pre-increment value), triggering `set -e` termination. Replace with explicit `if/then/fi` blocks (exempt from `set -e`) and `OVERRIDE_COUNT=$((OVERRIDE_COUNT + 1))` (assignment, always exit 0). Confirmed failure in rehearsal job: rehearse-81877-pull-ci-openshift-hypershift-main-e2e-aws-upgrade-hypershift-operator Co-Authored-By: Claude Opus 4.6 --- .../install/hypershift-install-commands.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh b/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh index 1ecec2e70a129..b5dcaa0bd60dd 100755 --- a/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh +++ b/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh @@ -17,9 +17,15 @@ extract_hcp_cli() { } OVERRIDE_COUNT=0 -[[ -n "${OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE:-}" ]] && ((OVERRIDE_COUNT++)) -[[ "${HO_MULTI}" == "true" ]] && ((OVERRIDE_COUNT++)) -[[ "${INSTALL_FROM_LATEST}" == "true" ]] && ((OVERRIDE_COUNT++)) +if [[ -n "${OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE:-}" ]]; then + OVERRIDE_COUNT=$((OVERRIDE_COUNT + 1)) +fi +if [[ "${HO_MULTI}" == "true" ]]; then + OVERRIDE_COUNT=$((OVERRIDE_COUNT + 1)) +fi +if [[ "${INSTALL_FROM_LATEST}" == "true" ]]; then + OVERRIDE_COUNT=$((OVERRIDE_COUNT + 1)) +fi if [[ ${OVERRIDE_COUNT} -gt 1 ]]; then echo "WARNING: Multiple image override flags are set. Precedence (highest to lowest):" From 8cc6a4107f57fcc05d6d4f97c1228725ef928806 Mon Sep 17 00:00:00 2001 From: Alessandro Rossi Date: Wed, 15 Jul 2026 11:15:56 +0200 Subject: [PATCH 4/5] fix(hypershift-install): add --confirm to oc image extract in extract_hcp_cli The mktemp-based temp directory (introduced for CWE-377) is no longer empty when oc image extract runs, because oc extract already wrote .dockerconfigjson to it. Without --confirm, oc image extract refuses to write to a non-empty directory. Co-Authored-By: Claude Opus 4.6 --- .../hypershift/install/hypershift-install-commands.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh b/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh index b5dcaa0bd60dd..7499cec6453cc 100755 --- a/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh +++ b/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh @@ -11,7 +11,7 @@ extract_hcp_cli() { local cli_dir cli_dir="$(mktemp -d "${TMPDIR:-/tmp}/hs-cli.XXXXXX")" oc extract secret/pull-secret -n openshift-config --to="${cli_dir}" --confirm - oc image extract "${image}" --path "/usr/bin/hypershift:${cli_dir}" --registry-config="${cli_dir}/.dockerconfigjson" --filter-by-os="linux/amd64" + oc image extract "${image}" --path "/usr/bin/hypershift:${cli_dir}" --registry-config="${cli_dir}/.dockerconfigjson" --filter-by-os="linux/amd64" --confirm chmod +x "${cli_dir}/hypershift" HCP_CLI="${cli_dir}/hypershift" } From 379def8de4bc3b850973d0a3fda0e09f9d8bfc3c Mon Sep 17 00:00:00 2001 From: Alessandro Rossi Date: Thu, 16 Jul 2026 14:13:20 +0200 Subject: [PATCH 5/5] feat(hypershift-install): add Gangway transport variable for HO release controller Gangway does not strip the MULTISTAGE_PARAM_OVERRIDE_ prefix when injecting env vars into step pods. Add the prefixed parameter MULTISTAGE_PARAM_OVERRIDE_OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE to the ref YAML and copy its value into OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE at runtime, following the same pattern used by hypershift-mce-install. This enables the HO release controller to override the operator image via Gangway without hitting the ci-operator ImageStream race condition (OVERRIDE_IMAGE_* mechanism). Co-Authored-By: Claude Opus 4.6 --- .../hypershift/install/hypershift-install-commands.sh | 6 ++++++ .../hypershift/install/hypershift-install-ref.yaml | 3 +++ 2 files changed, 9 insertions(+) diff --git a/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh b/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh index 7499cec6453cc..938cf3c15ac65 100755 --- a/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh +++ b/ci-operator/step-registry/hypershift/install/hypershift-install-commands.sh @@ -16,6 +16,12 @@ extract_hcp_cli() { HCP_CLI="${cli_dir}/hypershift" } +# Gangway does not strip the MULTISTAGE_PARAM_OVERRIDE_ prefix, so the HO release +# controller passes the image via the prefixed variable and we copy it here. +if [[ -n "${MULTISTAGE_PARAM_OVERRIDE_OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE:-}" ]]; then + OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE="${MULTISTAGE_PARAM_OVERRIDE_OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE}" +fi + OVERRIDE_COUNT=0 if [[ -n "${OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE:-}" ]]; then OVERRIDE_COUNT=$((OVERRIDE_COUNT + 1)) diff --git a/ci-operator/step-registry/hypershift/install/hypershift-install-ref.yaml b/ci-operator/step-registry/hypershift/install/hypershift-install-ref.yaml index c2ba772456fd7..3858ec043659f 100644 --- a/ci-operator/step-registry/hypershift/install/hypershift-install-ref.yaml +++ b/ci-operator/step-registry/hypershift/install/hypershift-install-ref.yaml @@ -62,6 +62,9 @@ ref: - name: OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE default: "" documentation: "If set, overrides both the HyperShift Operator image and the hcp CLI used during install, bypassing the pipeline ImageStream dependency resolution." + - name: MULTISTAGE_PARAM_OVERRIDE_OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE + default: "" + documentation: "Gangway transport variable. Gangway cannot set step parameters directly, so this prefixed variable is used as a transport mechanism. The script copies its value into OVERRIDE_HYPERSHIFT_OPERATOR_IMAGE at runtime." commands: hypershift-install-commands.sh credentials: - mount_path: /etc/hypershift-pool-aws-credentials