Add AI analysis to medik8s jobs

ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__4.22.yaml

@@ -3,6 +3,10 @@ base_images:
     name: ubi
     namespace: ocp
     tag: "9"
+  claude-ai-helpers:
+    name: claude-ai-helpers
+    namespace: ci
+    tag: latest
   operator-sdk:
     name: "4.17"
     namespace: origin
@@ -51,6 +55,9 @@ tests:
       BASE_DOMAIN: ocp-ci.medik8s-ci.devcluster.openshift.com
       INSTALL_NAMESPACE: nhc-install
       OPERATOR_RELEASED_VERSION: 0.12.0
+      TEST_STEPS: e2e-test
+    post:
+    - ref: medik8s-analyze-e2e-failure
     test:
     - as: e2e-install
       cli: latest

ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__4.23.yaml

@@ -3,6 +3,10 @@ base_images:
     name: ubi
     namespace: ocp
     tag: "9"
+  claude-ai-helpers:
+    name: claude-ai-helpers
+    namespace: ci
+    tag: latest
   operator-sdk:
     name: "4.17"
     namespace: origin
@@ -51,6 +55,9 @@ tests:
       BASE_DOMAIN: ocp-ci.medik8s-ci.devcluster.openshift.com
       INSTALL_NAMESPACE: nhc-install
       OPERATOR_RELEASED_VERSION: 0.12.0
+      TEST_STEPS: e2e-test
+    post:
+    - ref: medik8s-analyze-e2e-failure
     test:
     - as: e2e-install
       cli: latest

ci-operator/config/medik8s/node-healthcheck-operator/medik8s-node-healthcheck-operator-main__5.0.yaml

@@ -3,6 +3,10 @@ base_images:
     name: ubi
     namespace: ocp
     tag: "9"
+  claude-ai-helpers:
+    name: claude-ai-helpers
+    namespace: ci
+    tag: latest
   operator-sdk:
     name: "4.17"
     namespace: origin
@@ -51,6 +55,9 @@ tests:
       BASE_DOMAIN: ocp-ci.medik8s-ci.devcluster.openshift.com
       INSTALL_NAMESPACE: nhc-install
       OPERATOR_RELEASED_VERSION: 0.12.0
+      TEST_STEPS: e2e-test
+    post:
+    - ref: medik8s-analyze-e2e-failure
     test:
     - as: e2e-install
       cli: latest

ci-operator/step-registry/medik8s/analyze-e2e-failure/OWNERS

@@ -0,0 +1,4 @@
+approvers:
+  - medik8s-admins
+reviewers:
+  - medik8s-admins

ci-operator/step-registry/medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-commands.sh

@@ -0,0 +1,143 @@
+#!/bin/bash
+set -euo pipefail
+
+echo "=== Medik8s E2E Failure Analyzer ==="
+
+# ---------------------------------------------------------------------------
+# 1. Construct GCS base path and wait for test step artifacts
+# ---------------------------------------------------------------------------
+JOB_NAME="${JOB_NAME:-unknown}"
+BUILD_ID="${BUILD_ID:-unknown}"
+JOB_TYPE="${JOB_TYPE:-}"
+PULL_NUMBER="${PULL_NUMBER:-}"
+REPO_OWNER="${REPO_OWNER:-}"
+REPO_NAME="${REPO_NAME:-}"
+
+if [[ "$JOB_TYPE" == "presubmit" ]] && [[ -n "$PULL_NUMBER" ]]; then
+  GCS_BUCKET_PATH="pr-logs/pull/${REPO_OWNER}_${REPO_NAME}/${PULL_NUMBER}/${JOB_NAME}/${BUILD_ID}"
+else
+  GCS_BUCKET_PATH="logs/${JOB_NAME}/${BUILD_ID}"
+fi
+
+GCSWEB_BASE="https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs/test-platform-results"
+PROW_JOB_URL="${GCSWEB_BASE}/${GCS_BUCKET_PATH}"
+ARTIFACTS_BASE="${GCSWEB_BASE}/${GCS_BUCKET_PATH}/artifacts/${TEST_NAME}"
+
+echo "Waiting for test step artifacts in GCS..."
+FAILURE_DETECTED=false
+FAILED_STEP=""
+MAX_WAIT=600
+POLL_INTERVAL=15
+WAITED=0
+
+while [[ $WAITED -lt $MAX_WAIT ]]; do
+  for STEP_NAME in $TEST_STEPS; do
+    FINISHED_JSON=$(curl -sL "${ARTIFACTS_BASE}/${STEP_NAME}/finished.json" 2>/dev/null || true)
+    if echo "$FINISHED_JSON" | jq -e '.passed == false' &>/dev/null; then
+      echo "Detected test failure in ${STEP_NAME}/finished.json (waited ${WAITED}s)"
+      FAILURE_DETECTED=true
+      FAILED_STEP="$STEP_NAME"
+      break 2
+    elif echo "$FINISHED_JSON" | jq -e '.passed == true' &>/dev/null; then
+      echo "Test step ${STEP_NAME} passed — skipping analysis."
+      exit 0
+    fi
+  done
+
+  echo "  Waiting for artifacts... (${WAITED}s/${MAX_WAIT}s)"
+  sleep "$POLL_INTERVAL"
+  WAITED=$((WAITED + POLL_INTERVAL))
+done
+
+if [[ "$FAILURE_DETECTED" == "false" ]]; then
+  echo "Timed out waiting for test step artifacts after ${WAITED}s — skipping analysis."
+  exit 0
+fi
+
+# ---------------------------------------------------------------------------
+# 2. Verify Claude Code CLI
+# ---------------------------------------------------------------------------
+if ! command -v claude &>/dev/null; then
+  echo "ERROR: Claude Code CLI not found — skipping analysis"
+  exit 0
+fi
+
+echo "Claude Code CLI: $(claude --version 2>/dev/null || echo 'unknown')"
+
+# ---------------------------------------------------------------------------
+# 3. Run Claude with the pre-installed ai-helpers skill
+# ---------------------------------------------------------------------------
+echo "Prow job URL: $PROW_JOB_URL"
+echo "Failed step: $FAILED_STEP"
+
+SYSTEM_PROMPT="IMPORTANT CI CONTEXT:
+- You are running inside the CI job itself as a post-step.
+- This step's artifact directory is: ${ARTIFACT_DIR}
+- Other steps' artifacts (build-log, JUnit, intervals) are available via GCS at: ${PROW_JOB_URL}
+- You have network access to download artifacts from GCS using curl.
+- Write the final analysis report to: ${ARTIFACT_DIR}/failure-analysis.md
+- Use --fast mode (do NOT use AskUserQuestion).
+- Do NOT prompt for JIRA export — just write the markdown analysis.
+
+MEDIK8S CONTEXT:
+- Medik8s operators handle automated node remediation in OpenShift/Kubernetes clusters.
+- Operators: self-node-remediation (SNR), node-healthcheck-operator (NHC), fence-agents-remediation (FAR), machine-deletion-remediation (MDR), node-maintenance-operator (NMO), storage-based-remediation (SBR).
+- E2E tests use Ginkgo v2 + Gomega framework.
+- Tests run on ephemeral OCP clusters provisioned via IPI-AWS."
+
+echo ""
+echo "Running Claude with /ci:prow-job-analysis skill..."
+echo ""
+
+set +e
+timeout 1200 claude -p "/ci:prow-job-analysis ${PROW_JOB_URL} --fast" \
+  --append-system-prompt "$SYSTEM_PROMPT" \
+  --allowedTools "Bash Read Write Edit Grep Glob WebFetch Skill" \
+  --max-turns 100 \
+  --model "$CLAUDE_MODEL" \
+  --verbose \
+  --output-format stream-json \
+  2> "${ARTIFACT_DIR}/claude-failure-analysis.log" \
+  | tee "${ARTIFACT_DIR}/claude-failure-analysis.json"
+CLAUDE_EXIT=$?
+set -e
+
+if [[ "$CLAUDE_EXIT" -eq 124 ]]; then
+  echo "Claude timed out — report may be incomplete"
+fi
+
+# ---------------------------------------------------------------------------
+# 4. Extract token usage
+# ---------------------------------------------------------------------------
+
+TOKENS_JSON=$(grep '"type":"result"' "${ARTIFACT_DIR}/claude-failure-analysis.json" 2>/dev/null \
+  | head -1 \
+  | jq '{
+      total_cost_usd: (.total_cost_usd // 0),
+      duration_ms: (.duration_ms // 0),
+      num_turns: (.num_turns // 0),
+      input_tokens: (.usage.input_tokens // 0),
+      output_tokens: (.usage.output_tokens // 0),
+      cache_read_input_tokens: (.usage.cache_read_input_tokens // 0),
+      cache_creation_input_tokens: (.usage.cache_creation_input_tokens // 0)
+    }' 2>/dev/null \
+  || echo '{"total_cost_usd":0,"duration_ms":0,"num_turns":0,"input_tokens":0,"output_tokens":0,"cache_read_input_tokens":0,"cache_creation_input_tokens":0}')
+
+DURATION_MS=$(echo "$TOKENS_JSON" | jq -r '.duration_ms // 0')
+DURATION_S=$(printf '%.0f' "$(echo "${DURATION_MS} / 1000" | bc -l 2>/dev/null)" 2>/dev/null || echo 0)
+NUM_TURNS=$(echo "$TOKENS_JSON" | jq -r '.num_turns // 0')
+
+jq -j 'select(.type == "assistant") | .message.content[]? | select(.type == "text") | .text // empty' \
+  "${ARTIFACT_DIR}/claude-failure-analysis.json" \
+  > "${ARTIFACT_DIR}/claude-failure-analysis-text.txt" 2>/dev/null || true
+
+echo "$TOKENS_JSON" > "${SHARED_DIR}/claude-failure-analysis-tokens.json" 2>/dev/null || true
+
+echo ""
+echo "=== Failure Analysis Complete ==="
+echo "Claude exit code: $CLAUDE_EXIT"
+echo "Duration: ${DURATION_S}s"
+echo "Turns: ${NUM_TURNS}"
+echo "Analysis: ${ARTIFACT_DIR}/failure-analysis.md"
+
+exit 0

ci-operator/step-registry/medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-ref.metadata.json

@@ -0,0 +1,11 @@
+{
+	"path": "medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-ref.yaml",
+	"owners": {
+		"approvers": [
+			"medik8s-admins"
+		],
+		"reviewers": [
+			"medik8s-admins"
+		]
+	}
+}

ci-operator/step-registry/medik8s/analyze-e2e-failure/medik8s-analyze-e2e-failure-ref.yaml

@@ -0,0 +1,51 @@
+ref:
+  as: medik8s-analyze-e2e-failure
+  from: claude-ai-helpers
+  best_effort: true
+  commands: medik8s-analyze-e2e-failure-commands.sh
+  timeout: 30m0s
+  grace_period: 30s
+  env:
+  - name: CLAUDE_CODE_USE_VERTEX
+    default: "1"
+    documentation: |-
+      Enable Vertex AI for Claude Code.
+  - name: CLOUD_ML_REGION
+    default: "global"
+    documentation: |-
+      Google Cloud region for Vertex AI.
+  - name: ANTHROPIC_VERTEX_PROJECT_ID
+    default: "itpc-gcp-hybrid-pe-eng-claude"
+    documentation: |-
+      Google Cloud project ID for Vertex AI authentication.
+  - name: GOOGLE_APPLICATION_CREDENTIALS
+    default: "/var/run/claude-code-service-account/token"
+    documentation: |-
+      Path to the Google Cloud service account JSON key file for Vertex AI authentication.
+  - name: CLAUDE_MODEL
+    default: "claude-opus-4-6"
+    documentation: |-
+      Claude model to use for test failure analysis.
+  - name: TEST_NAME
+    default: "openshift-e2e"
+    documentation: |-
+      The test name (as: field) in the ci-operator config. Used to construct
+      the GCS artifact path. Override if your test uses a different name.
+  - name: TEST_STEPS
+    default: "test-command"
+    documentation: |-
+      Space-separated list of test step names to check for finished.json.
+      These are the inner step names (as: fields) within the test definition.
+  resources:
+    requests:
+      cpu: 100m
+      memory: 256Mi
+  credentials:
+  - namespace: test-credentials
+    name: sa-claude-openshift-ci
+    mount_path: /var/run/claude-code-service-account
+  documentation: |-
+    Post-step that uses Claude to analyze e2e test failures for medik8s operators.
+    Only runs when test failures are detected (polls GCS for finished.json).
+    On success (no test failures), exits early with no cost.
+    Produces a markdown analysis report in ARTIFACT_DIR.