From 1fe073df788748a1040b6087ce6eca17cdbad59d Mon Sep 17 00:00:00 2001 From: mctl-deploy Date: Thu, 4 Jun 2026 00:56:50 +0200 Subject: [PATCH] feat(argo): auto-GC orphaned workflow workdir PVCs Two layers so failed workflows stop stranding their `workdir` PVCs (and the backing Hetzner volumes): B) workflowDefaults.spec.volumeClaimGC.strategy=OnWorkflowCompletion in the argo-workflows Helm values. Argo's default OnWorkflowSuccess retains the PVC on failure; ttlStrategy then GCs the Workflow, orphaning the PVC forever. C) argo-pvc-janitor CronJob (hourly) as a backstop: deletes a `*-workdir` PVC only when no pod mounts it AND its owning Workflow is gone or terminal (Succeeded/Failed/Error). Running/pending workflows are kept, so it never races a live workflow. Uses alpine/k8s (not the retired bitnami/kubectl). WHY: several failed mctl-agents issue-poll runs left 10Gi workdir PVCs stranded (observed + cleaned manually this week). B fixes the cause; C catches strays and pre-existing orphans. --- .../argo-workflows/config/pvc-janitor.yaml | 115 ++++++++++++++++++ .../templates/core-infra/argo-workflows.yaml | 8 ++ 2 files changed, 123 insertions(+) create mode 100644 platform-gitops/argo-workflows/config/pvc-janitor.yaml diff --git a/platform-gitops/argo-workflows/config/pvc-janitor.yaml b/platform-gitops/argo-workflows/config/pvc-janitor.yaml new file mode 100644 index 000000000..53c297bef --- /dev/null +++ b/platform-gitops/argo-workflows/config/pvc-janitor.yaml @@ -0,0 +1,115 @@ +# Backstop janitor for orphaned per-workflow `workdir` PVCs. +# +# Primary defence is workflowDefaults.spec.volumeClaimGC.strategy=OnWorkflowCompletion +# (set in the argo-workflows Helm values), which deletes a workflow's +# volumeClaimTemplate PVC when it finishes, success or failure. This CronJob is +# defence-in-depth: it catches PVCs that slip through (e.g. a Workflow object +# force-deleted or TTL-GC'd before its PVC was reclaimed, or pre-existing +# orphans from before volumeClaimGC was enabled). +# +# A `-workdir` PVC is deleted only when BOTH hold: +# - no Pod currently mounts it, AND +# - its owning Workflow is gone, or in a terminal phase (Succeeded/Failed/Error). +# A running/pending workflow (incl. a just-submitted one with empty phase) is +# kept, so this never races a live workflow. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: argo-pvc-janitor + namespace: argo-workflows +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: argo-pvc-janitor + namespace: argo-workflows +rules: + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "delete"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list"] + - apiGroups: ["argoproj.io"] + resources: ["workflows"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: argo-pvc-janitor + namespace: argo-workflows +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: argo-pvc-janitor +subjects: + - kind: ServiceAccount + name: argo-pvc-janitor + namespace: argo-workflows +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: argo-pvc-janitor + namespace: argo-workflows +spec: + schedule: "17 * * * *" # hourly, offset to avoid top-of-hour contention + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 1 + activeDeadlineSeconds: 300 + ttlSecondsAfterFinished: 600 # don't let finished janitor jobs pile up + template: + spec: + serviceAccountName: argo-pvc-janitor + restartPolicy: Never + # alpine/k8s ships kubectl and is multi-pull-reliable. Do NOT use + # bitnami/kubectl — those Docker Hub tags were retired and silently + # wedge on ImagePullBackOff (see vault-auto-unseal incident). + containers: + - name: janitor + image: alpine/k8s:1.33.12 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 200m + memory: 128Mi + command: + - /bin/sh + - -c + - | + set -eu + NS=argo-workflows + echo "=== argo workdir PVC janitor ($(date -u +%FT%TZ)) ===" + # PVCs currently mounted by some pod — never touch these. + INUSE=$(kubectl get pods -n "$NS" \ + -o jsonpath='{range .items[*]}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{"\n"}{end}{end}' \ + 2>/dev/null | sort -u) + for pvc in $(kubectl get pvc -n "$NS" -o name 2>/dev/null | sed 's#.*/##' | grep -- '-workdir$' || true); do + if printf '%s\n' "$INUSE" | grep -qx "$pvc"; then + echo "keep $pvc (in use by a pod)"; continue + fi + wf="${pvc%-workdir}" + if ! kubectl get workflow -n "$NS" "$wf" >/dev/null 2>&1; then + echo "DELETE $pvc (owning workflow '$wf' gone — orphan)" + kubectl delete pvc -n "$NS" "$pvc" --wait=false || true + continue + fi + phase=$(kubectl get workflow -n "$NS" "$wf" -o jsonpath='{.status.phase}' 2>/dev/null || echo "") + case "$phase" in + Succeeded|Failed|Error) + echo "DELETE $pvc (workflow '$wf' phase=$phase, no pod)" + kubectl delete pvc -n "$NS" "$pvc" --wait=false || true + ;; + *) + echo "keep $pvc (workflow '$wf' phase=${phase:-})" + ;; + esac + done + echo "=== done ===" diff --git a/platform-gitops/bootstrap/templates/core-infra/argo-workflows.yaml b/platform-gitops/bootstrap/templates/core-infra/argo-workflows.yaml index ed98c21e9..d2600e445 100644 --- a/platform-gitops/bootstrap/templates/core-infra/argo-workflows.yaml +++ b/platform-gitops/bootstrap/templates/core-infra/argo-workflows.yaml @@ -120,6 +120,14 @@ spec: secondsAfterFailure: 86400 podGC: strategy: OnWorkflowCompletion + # Delete volumeClaimTemplate PVCs (the per-workflow `workdir`) on + # completion regardless of outcome. Argo's default is + # OnWorkflowSuccess, which RETAINS the PVC when a workflow fails; + # ttlStrategy then GCs the Workflow object, orphaning the PVC (and + # its backing Hetzner volume) permanently. This left several failed + # mctl-agents issue-poll workdir PVCs stranded (10Gi each). + volumeClaimGC: + strategy: OnWorkflowCompletion artifactRepository: archiveLogs: true