diff --git a/.github/scripts/microvm-ci-release_test.sh b/.github/scripts/microvm-ci-release_test.sh index 4fdcb9b825..732a092807 100644 --- a/.github/scripts/microvm-ci-release_test.sh +++ b/.github/scripts/microvm-ci-release_test.sh @@ -133,15 +133,18 @@ import sys workflow = open(sys.argv[1], encoding="utf-8").read().splitlines() jobs = {} +job_lines = {} current = None for line in workflow: match = re.fullmatch(r" ([A-Za-z0-9_-]+):", line) if match: current = match.group(1) jobs[current] = [] + job_lines[current] = [] continue if current is None: continue + job_lines[current].append(line) match = re.fullmatch(r" needs: (.+)", line) if not match: continue @@ -160,10 +163,27 @@ def reaches_validation(job, seen=None): seen.add(job) return any(reaches_validation(dep, seen) for dep in jobs.get(job, [])) +for job, lines in job_lines.items(): + for producer in re.findall(r"needs\.([A-Za-z0-9_-]+)\.outputs\.", "\n".join(lines)): + if producer not in jobs[job]: + raise SystemExit(f"job {job} reads outputs without directly needing {producer}") + +validation = "\n".join(job_lines["validate-release-ref"]) +assert ' needs: guard' in validation +assert ' ref: ${{ github.sha }}' in validation +assert ' run: test "$(git rev-parse HEAD)" = "${GITHUB_SHA}"' in validation +assert ''' - name: Assert guard and immutable run commits agree + env: + RELEASE_COMMIT: ${{ needs.guard.outputs.commit }} + run: test "$(git rev-parse HEAD)" = "${RELEASE_COMMIT}"''' in validation + for job in sorted(name for name in jobs if name == "publish" or name.startswith("publish-")): if not reaches_validation(job): raise SystemExit(f"publishing job {job} bypasses validate-release-ref") PY +# Exercise the real guard and validator agreement step with divergent on-main +# commits; counting checkout producers alone cannot establish revision authority. +sh "$repo_root/.github/scripts/release-tag-guard_test.sh" # GoReleaser uploads to the release created above without replacing its notes. require 'mode: keep-existing' "$goreleaser" @@ -197,13 +217,15 @@ test "$(printf '%s\n' "$assemble_bundle" | grep -Fc 'signing_ref="${{ needs.vali require 'release.yml@${signing_ref}' "$release" checkout_count=$(grep -c 'uses: actions/checkout@' "$release") bound_checkout_count=$(grep -c 'ref: ${{ github.sha }}' "$release") -version_checkout_count=$(grep -c 'ref: ${{ env.VERSION }}' "$release") +verified_commit_checkout_count=$(grep -c 'ref: .*needs.guard.outputs.commit' "$release") +trusted_guard_checkout_count=$(grep -c 'ref: refs/heads/main' "$release") head_assertion_count=$(grep -c 'run: test "$(git rev-parse HEAD)" = "${GITHUB_SHA}"' "$release") -# The guard checkout validates the requested release tag is on main, and the CLI -# publisher checks out that same tag so GoReleaser can inspect tag history; every -# other release checkout remains bound to the workflow SHA and immediately asserts it. -test "$version_checkout_count" -eq 2 -test "$checkout_count" -eq "$((bound_checkout_count + version_checkout_count))" +# The guard implementation is checked out once from protected main. Jobs that need +# tag history use the guard's verified commit; all other publishing checkouts are +# bound to the workflow SHA and immediately assert it. +test "$trusted_guard_checkout_count" -eq 1 +test "$verified_commit_checkout_count" -eq 3 +test "$checkout_count" -eq "$((bound_checkout_count + verified_commit_checkout_count + trusted_guard_checkout_count))" test "$bound_checkout_count" -eq "$head_assertion_count" if "$validate_release_ref" v1.2.3 refs/heads/main >/dev/null 2>&1; then echo 'branch-dispatched release tag input was accepted' >&2 diff --git a/.github/scripts/release-tag-guard.sh b/.github/scripts/release-tag-guard.sh new file mode 100755 index 0000000000..a9f5b33b15 --- /dev/null +++ b/.github/scripts/release-tag-guard.sh @@ -0,0 +1,67 @@ +#!/bin/sh +set -eu + +: "${VERSION:?VERSION is required}" +: "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" + +if ! printf '%s\n' "$VERSION" | grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'; then + echo "release tag must match vMAJOR.MINOR.PATCH" >&2 + exit 1 +fi + +tag_ref="refs/tags/$VERSION" +if ! git show-ref --verify --quiet "$tag_ref"; then + echo "release tag does not exist: $VERSION" >&2 + exit 1 +fi +commit=$(git rev-parse --verify "$tag_ref^{commit}") +if ! printf '%s\n' "$commit" | grep -Eq '^[0-9a-f]{40}$'; then + echo "release tag did not peel to a commit" >&2 + exit 1 +fi + +case "${GITHUB_EVENT_NAME:-}" in + push|workflow_dispatch) + if [ "${GITHUB_REF:-}" != "$tag_ref" ]; then + echo "event ref does not match the release tag" >&2 + exit 1 + fi + ;; + *) + echo "unsupported release event: ${GITHUB_EVENT_NAME:-unset}" >&2 + exit 1 + ;; +esac +case "${GITHUB_SHA:-}" in + ''|*[!0-9a-f]*) echo "event commit is not a full SHA" >&2; exit 1 ;; +esac +if [ "${#GITHUB_SHA}" -ne 40 ]; then + echo "event commit is not a full SHA" >&2 + exit 1 +fi +# Preserve annotated-tag peeling, but never resolve authority from a mutable ref. +event_commit=$(git rev-parse --verify "${GITHUB_SHA}^{commit}" 2>/dev/null) || { + echo "event commit did not peel to a commit" >&2 + exit 1 +} +if [ "$event_commit" != "$commit" ]; then + echo "event commit does not match the release tag" >&2 + exit 1 +fi +if ! git merge-base --is-ancestor "$commit" origin/main; then + echo "release tag commit is not on origin/main" >&2 + exit 1 +fi + +git checkout --detach "$commit" >/dev/null 2>&1 +head=$(git rev-parse --verify HEAD) +if [ "$head" != "$commit" ]; then + echo "checked-out commit does not match the verified release tag" >&2 + exit 1 +fi + +{ + echo "version=$VERSION" + echo "commit=$commit" +} >>"$GITHUB_OUTPUT" +printf '%s is verified at %s\n' "$VERSION" "$commit" diff --git a/.github/scripts/release-tag-guard_test.sh b/.github/scripts/release-tag-guard_test.sh new file mode 100755 index 0000000000..92e670ec95 --- /dev/null +++ b/.github/scripts/release-tag-guard_test.sh @@ -0,0 +1,120 @@ +#!/bin/sh +set -eu + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +script="$script_dir/release-tag-guard.sh" +work=${TEST_TMPDIR:-.scratch}/release-tag-guard-$$ +mkdir -p "$work" +work=$(CDPATH= cd -- "$work" && pwd -P) + +repo="$work/repo" +mkdir "$repo" +repo=$(CDPATH= cd -- "$repo" && pwd -P) +git -C "$repo" init -q -b main +git -C "$repo" config user.name test +git -C "$repo" config user.email test@example.com +git -C "$repo" config commit.gpgSign false +git -C "$repo" config tag.gpgSign false +printf 'release\n' >"$repo/file" +git -C "$repo" add file +git -C "$repo" commit -qm release +release_commit=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" tag v1.2.3 +printf 'main\n' >>"$repo/file" +git -C "$repo" commit -qam main +main=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" remote add origin "$repo" +git -C "$repo" fetch -q origin main:refs/remotes/origin/main + +run_ok() { + output="$work/output" + : >"$output" + (cd "$repo" && VERSION="$1" GITHUB_EVENT_NAME="$2" GITHUB_REF="$3" GITHUB_SHA="$4" GITHUB_OUTPUT="$output" "$script") >/dev/null + grep -Fx "version=$1" "$output" >/dev/null + grep -Fx "commit=$5" "$output" >/dev/null + [ "$(git -C "$repo" rev-parse HEAD)" = "$5" ] +} +run_fail() { + output="$work/output" + : >"$output" + if (cd "$repo" && VERSION="$1" GITHUB_EVENT_NAME="$2" GITHUB_REF="$3" GITHUB_SHA="$4" GITHUB_OUTPUT="$output" "$script") >/dev/null 2>&1; then + echo "unexpectedly accepted: $1" >&2 + exit 1 + fi + [ ! -s "$output" ] +} + +# Dispatch must select the same immutable tag ref as its input. +run_ok v1.2.3 workflow_dispatch refs/tags/v1.2.3 "$release_commit" "$release_commit" +run_fail v1.2.3 workflow_dispatch refs/heads/main "$main" +run_fail v1.2.3 workflow_dispatch refs/tags/v1.2.4 "$release_commit" +run_fail main workflow_dispatch refs/tags/main "$main" +run_fail refs/heads/main workflow_dispatch refs/tags/refs/heads/main "$main" +marker="$work/pwned" +run_fail "v1.2.3; touch $marker" workflow_dispatch refs/heads/main "$main" +[ ! -e "$marker" ] +run_fail v01.2.3 workflow_dispatch refs/heads/main "$main" +run_fail v9.9.9 workflow_dispatch refs/heads/main "$main" +run_fail v1.2.3 push refs/heads/main "$release_commit" +run_fail v1.2.3 push refs/tags/v1.2.3 0000000000000000000000000000000000000000 +run_ok v1.2.3 push refs/tags/v1.2.3 "$release_commit" "$release_commit" + +# An annotated tag push reports its tag-object SHA; both sides must peel it. +git -C "$repo" tag -a v1.2.4 -m annotated "$release_commit" +annotated_tag=$(git -C "$repo" rev-parse v1.2.4) +run_ok v1.2.4 push refs/tags/v1.2.4 "$annotated_tag" "$release_commit" +run_ok v1.2.4 push refs/tags/v1.2.4 "$release_commit" "$release_commit" +run_ok v1.2.4 workflow_dispatch refs/tags/v1.2.4 "$release_commit" "$release_commit" + +# Both commits are on main: moving the same tag after the event was captured +# must fail before checkout/output, even though ref-name and ancestry checks pass. +git -C "$repo" tag -f v1.2.3 "$main" >/dev/null +for event in push workflow_dispatch; do + run_fail v1.2.3 "$event" refs/tags/v1.2.3 "$release_commit" + run_fail v1.2.4 "$event" refs/tags/v1.2.4 "$main" + run_ok v1.2.3 "$event" refs/tags/v1.2.3 "$main" "$main" + for invalid_sha in '' HEAD refs/tags/v1.2.3 "${main%?}" "${main}0" \ + 0000000000000000000000000000000000000000 "$(printf '%s\nHEAD' "$main")"; do + run_fail v1.2.3 "$event" refs/tags/v1.2.3 "$invalid_sha" + done +done +# An annotated event object must not hide a moved-tag mismatch either. +git -C "$repo" tag -f -a v1.2.4 -m moved "$main" >/dev/null +run_fail v1.2.4 push refs/tags/v1.2.4 "$annotated_tag" +run_fail v1.2.4 workflow_dispatch refs/tags/v1.2.4 "$release_commit" + +workflow="$script_dir/../workflows/release.yml" +# Execute the real validator agreement step, not a duplicate test-only check. +agreement=$(awk '/^ - name: Assert guard and immutable run commits agree$/{on=1; next} on && /^ run: /{sub(/^ run: /, ""); print; exit}' "$workflow") +[ -n "$agreement" ] +(cd "$repo" && RELEASE_COMMIT="$main" sh -eu -c "$agreement") +if (cd "$repo" && RELEASE_COMMIT="$release_commit" sh -eu -c "$agreement"); then + echo 'validator accepted a different on-main guard commit' >&2 + exit 1 +fi +if (cd "$repo" && RELEASE_COMMIT='' sh -eu -c "$agreement"); then + echo 'validator accepted an empty guard commit' >&2 + exit 1 +fi + +git -C "$repo" switch -q --detach "$release_commit" +printf 'branch\n' >>"$repo/file" +git -C "$repo" commit -qam branch +off_main=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" tag v2.0.0 +run_fail v2.0.0 workflow_dispatch refs/tags/v2.0.0 "$off_main" +run_fail v2.0.0 push refs/tags/v2.0.0 "$off_main" +run_fail v1.2.3 schedule refs/tags/v1.2.3 "$release_commit" + +workflow="$script_dir/../workflows/release.yml" +# The privileged graph consumes only the commit emitted by a guard implementation +# checked out from protected main; dispatch must select the requested tag ref. +grep -F "if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.ref == format('refs/tags/{0}', inputs.tag))" "$workflow" >/dev/null +grep -F 'ref: refs/heads/main' "$workflow" >/dev/null +if grep -F 'org.opencontainers.image.revision=${{ github.sha }}' "$workflow" >/dev/null; then + echo "release image revision still uses the event workflow SHA" >&2 + exit 1 +fi +[ "$(grep -Fc 'org.opencontainers.image.revision=${{ needs.guard.outputs.commit }}' "$workflow")" -eq 5 ] + +printf 'release tag guard tests passed\n' diff --git a/.github/workflows/e2e-live.yml b/.github/workflows/e2e-live.yml index 9d1a019bfb..ebcb67bd21 100644 --- a/.github/workflows/e2e-live.yml +++ b/.github/workflows/e2e-live.yml @@ -18,9 +18,9 @@ # even when labelled; see ci.yml's header for the pwn-request rationale. # - The job `if:` also requires github.repository == 'stacklok/mecatl', so a # fork's own scheduled/dispatched runs never reference the secret. -# - A guard step checks OPENROUTER_API_KEY availability and skips the live -# run cleanly (notice, green) when it is absent — the workflow never -# hard-fails just because the secret isn't configured. +# - Ordinary live jobs skip cleanly when OPENROUTER_API_KEY is absent. The +# manual-only native qualification instead fails closed on a missing key. +# It requires a successful same-run production qualification before staging. # # All third-party actions are SHA-pinned (with a # vX.Y.Z comment) so a # re-pointed tag from a compromised maintainer cannot change what runs here. @@ -31,6 +31,15 @@ on: # Nightly, off-peak UTC. Odd minute to avoid the top-of-hour thundering herd. - cron: '17 3 * * *' workflow_dispatch: + inputs: + native_execution: + description: Run only native execution production + OpenRouter qualification + type: boolean + default: false + expected_sha: + description: Optional exact reviewed commit SHA (native execution only) + type: string + default: '' pull_request: types: [opened, synchronize, reopened, labeled] @@ -58,6 +67,7 @@ jobs: # Gate 2: PRs run only when a maintainer opted in via the `e2e-live` label. if: >- github.repository == 'stacklok/mecatl' && + !(github.event_name == 'workflow_dispatch' && inputs.native_execution) && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'e2e-live')) # One live run at a time, globally: overlapping dispatches/labelled pushes @@ -177,6 +187,7 @@ jobs: # and PRs run only when a maintainer opted in via the `e2e-live` label. if: >- github.repository == 'stacklok/mecatl' && + !(github.event_name == 'workflow_dispatch' && inputs.native_execution) && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'e2e-live')) # Separate group from `e2e-live` so the two live jobs run in parallel; they @@ -258,3 +269,206 @@ jobs: path: k8s-e2e-live-output.log retention-days: 7 if-no-files-found: warn + + native-execution-live: + name: Native execution (production + OpenRouter) + if: >- + github.repository == 'stacklok/mecatl' && + github.event_name == 'workflow_dispatch' && inputs.native_execution + runs-on: ubuntu-24.04 + timeout-minutes: 100 + concurrency: + group: e2e-live-native-execution + cancel-in-progress: false + env: + NATIVE_CI_DIR: ${{ github.workspace }}/.scratch/native-live-${{ github.run_id }}-${{ github.run_attempt }} + steps: + # Step ceilings sum to 98m: setup 9, production 50, credential 1, + # live 25, cleanup 10, reporting 3. The job retains 2m overhead. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Verify reviewed commit and reserve private CI directory + timeout-minutes: 1 + id: prepare + env: + EXPECTED_SHA: ${{ inputs.expected_sha }} + EVENT_SHA: ${{ github.sha }} + run: | + actual=$(git rev-parse HEAD) + test "$actual" = "$EVENT_SHA" + if [ -n "$EXPECTED_SHA" ] && [ "$actual" != "$EXPECTED_SHA" ]; then + echo '::error::Reviewed SHA does not match dispatch SHA' + exit 1 + fi + printf '## Native execution qualification\n\nCommit: %s\n' "$actual" >> "$GITHUB_STEP_SUMMARY" + umask 077 + mkdir -p .scratch + mkdir -m 700 "$NATIVE_CI_DIR" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + timeout-minutes: 2 + with: + go-version-file: go.mod + cache: false + - uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 + timeout-minutes: 1 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + timeout-minutes: 1 + with: + version: v3.18.4 + - name: Install pinned Kind and ko + timeout-minutes: 3 + run: | + go install sigs.k8s.io/kind@v0.33.0 + go install github.com/google/ko@v0.18.1 + command -v kubectl jq docker + # No provider secret here. run.sh installs pinned Calico and retains the + # owned cluster; the success dependency, not ownership.profile, admits live. + - name: Qualify production execution + timeout-minutes: 50 + id: production + env: + CONTAINER_ENGINE: docker + MECATL_EXECUTION_QUAL_CI: "0" + run: MECATL_EXECUTION_QUAL_OUTPUT="$GITHUB_OUTPUT" timeout --kill-after=15s 49m task e2e:k8s:execution:production + - name: Stage private provider credential + timeout-minutes: 1 + id: credential + if: success() && steps.production.outcome == 'success' + env: + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + run: | + set +x + if [ -z "$OPENROUTER_API_KEY" ]; then + echo '::error::OPENROUTER_API_KEY is required for native qualification' + exit 1 + fi + umask 077 + set -C + printf '%s' "$OPENROUTER_API_KEY" > "$NATIVE_CI_DIR/provider-key" + unset OPENROUTER_API_KEY + - name: Qualify native OpenRouter coding + timeout-minutes: 25 + id: live + if: success() && steps.production.outcome == 'success' && steps.credential.outcome == 'success' + env: + MECATL_EXECUTION_QUAL_STATE: ${{ steps.production.outputs.state }} + NATIVE_CLUSTER: ${{ steps.production.outputs.cluster }} + run: | + test -n "$NATIVE_CLUSTER" + grep -Fx -- "cluster=$NATIVE_CLUSTER" "$MECATL_EXECUTION_QUAL_STATE/ownership" >/dev/null + MECATL_EXECUTION_CREDENTIAL_FILE="$NATIVE_CI_DIR/provider-key" \ + timeout --kill-after=15s 24m ./deploy/mecatl-execution-kind/live.sh + # Credential removal is independent of cluster availability. Fail closed + # on ownership drift; runner disposal is a last resort, not cleanup proof. + - name: Clean up exact owned resources + timeout-minutes: 10 + id: cleanup + if: always() && steps.prepare.outcome == 'success' + env: + PRODUCTION_OUTCOME: ${{ steps.production.outcome }} + LIVE_OUTCOME: ${{ steps.live.outcome }} + MECATL_EXECUTION_QUAL_STATE: ${{ steps.production.outputs.state }} + NATIVE_CLUSTER: ${{ steps.production.outputs.cluster }} + run: | + key="$NATIVE_CI_DIR/provider-key" + if [ -e "$key" ] || [ -L "$key" ]; then + test -f "$key" && test ! -L "$key" + rm -- "$key" + fi + if [ "$PRODUCTION_OUTCOME" = skipped ]; then exit 0; fi + trap 'echo "::error::Owned cleanup incomplete; hosted-runner disposal is the fallback, not cleanup proof"' ERR + state="$MECATL_EXECUTION_QUAL_STATE" + test -n "$state" && test -n "$NATIVE_CLUSTER" + test ! -L "$state" + state=$(cd -- "$state" && pwd -P) + case "$state" in "$GITHUB_WORKSPACE/.scratch/k8s-execution/"*) ;; *) exit 1 ;; esac + ownership="$state/ownership" + test -f "$ownership" && test ! -L "$ownership" + owned_value() { + awk -F= -v key="$1" '$1 == key {value=substr($0,length(key)+2); count++} END {if(count != 1 || value == "") exit 1; print value}' "$ownership" + } + cluster=$(owned_value cluster) + test "$cluster" = "$NATIVE_CLUSTER" + case "$cluster" in mecatl-execution-qual-*) ;; *) exit 1 ;; esac + case "$cluster" in *[!a-z0-9-]*) exit 1 ;; esac + test "${#cluster}" -le 63 + test "$(owned_value owner)" = "$USER" + test "$(owned_value runtime)" = docker + test "$(owned_value profile)" = production + test "$(owned_value namespace)" = execution-qualification + context="kind-$cluster" + test "$(owned_value context)" = "$context" + kubeconfig="$state/kubeconfig" + test "$(owned_value kubeconfig)" = "$kubeconfig" + test -f "$kubeconfig" && test ! -L "$kubeconfig" + test "$(stat -c '%a' "$kubeconfig")" = 600 + test "$(kubectl --kubeconfig "$kubeconfig" --context "$context" config current-context)" = "$context" + test "$(docker inspect "${cluster}-control-plane" --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}')" = "$cluster" + if [ "$PRODUCTION_OUTCOME" != success ] || [ "${LIVE_OUTCOME:-}" = failure ] || [ "${LIVE_OUTCOME:-}" = cancelled ]; then + # Keep pre-restoration real-process evidence distinct from any fallback + # collected after Helm has restored the mock process. + diagnostic_status=missing + evidence="$state/live-diagnostics.jsonl" + if [ -f "$evidence" ] && [ ! -L "$evidence" ] && [ "$(stat -c '%s' "$evidence")" -le 1048576 ] && \ + cp -- "$evidence" "$NATIVE_CI_DIR/live-diagnostics.jsonl"; then + if [ -f "$state/live-diagnostics.status" ] && [ ! -L "$state/live-diagnostics.status" ]; then + IFS= read -r diagnostic_status < "$state/live-diagnostics.status" || diagnostic_status=incomplete + fi + fi + case "$diagnostic_status" in complete|incomplete|missing) ;; *) diagnostic_status=incomplete ;; esac + printf 'pre_restore=%s\n' "$diagnostic_status" > "$NATIVE_CI_DIR/diagnostics-status.txt" || echo '::warning::Diagnostic status unavailable' + if [ "$diagnostic_status" != complete ]; then + if timeout --kill-after=5s 45s sh ./deploy/mecatl-execution-kind/collect-failure.sh \ + "$kubeconfig" "$context" "$NATIVE_CI_DIR/production-diagnostics.jsonl"; then + printf 'fallback=complete\n' >> "$NATIVE_CI_DIR/diagnostics-status.txt" || echo '::warning::Diagnostic status unavailable' + else + printf 'fallback=incomplete\n' >> "$NATIVE_CI_DIR/diagnostics-status.txt" || echo '::warning::Diagnostic status unavailable' + echo '::warning::Bounded production diagnostics incomplete' + fi + fi + fi + kind delete cluster --name "$cluster" + clusters=$(kind get clusters) + if printf '%s\n' "$clusters" | grep -Fx -- "$cluster"; then exit 1; fi + # This file is emitted by the typed live test, not a transcript. Bound + # its upload independently and never collect the surrounding state. + summary="$state/live-summary.json" + if [ -e "$summary" ]; then + test -f "$summary" && test ! -L "$summary" + test "$(stat -c '%s' "$summary")" -le 16384 + cp -- "$summary" "$NATIVE_CI_DIR/live-summary.json" + fi + - name: Record sanitized qualification status + timeout-minutes: 1 + if: always() && steps.prepare.outcome == 'success' + env: + PRODUCTION_OUTCOME: ${{ steps.production.outcome }} + CREDENTIAL_OUTCOME: ${{ steps.credential.outcome }} + LIVE_OUTCOME: ${{ steps.live.outcome }} + CLEANUP_OUTCOME: ${{ steps.cleanup.outcome }} + run: | + printf 'production=%s\ncredential=%s\nlive=%s\ncleanup=%s\n' \ + "$PRODUCTION_OUTCOME" "$CREDENTIAL_OUTCOME" "$LIVE_OUTCOME" "$CLEANUP_OUTCOME" \ + > "$NATIVE_CI_DIR/qualification-status.txt" + cat "$NATIVE_CI_DIR/qualification-status.txt" >> "$GITHUB_STEP_SUMMARY" + - name: Upload only bounded native qualification evidence + timeout-minutes: 2 + if: always() && steps.prepare.outcome == 'success' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-execution-live-${{ github.run_id }}-${{ github.run_attempt }} + path: | + ${{ env.NATIVE_CI_DIR }}/live-summary.json + ${{ env.NATIVE_CI_DIR }}/qualification-status.txt + ${{ env.NATIVE_CI_DIR }}/production-diagnostics.jsonl + ${{ env.NATIVE_CI_DIR }}/live-diagnostics.jsonl + ${{ env.NATIVE_CI_DIR }}/diagnostics-status.txt + include-hidden-files: true + retention-days: 7 + if-no-files-found: error diff --git a/.github/workflows/k8s-e2e.yml b/.github/workflows/k8s-e2e.yml index c610233e43..9134f1c4f8 100644 --- a/.github/workflows/k8s-e2e.yml +++ b/.github/workflows/k8s-e2e.yml @@ -37,6 +37,17 @@ on: - internal/adapter/server/grpc.go - internal/adapter/server/http.go - deploy/helm/mecak8s/** + - deploy/helm/mecatl-execution/** + - cmd/mecatl-execution-provider/** + - cmd/mecatl-executor/** + - internal/adapter/executioncontroller/** + - internal/adapter/executionclient/** + - internal/executionenv/** + - contracts/proto/mecatl/execution/** + - e2e/k8s_execution/** + - deploy/mecatl-execution-kind/** + - build/execution-provider/Dockerfile + - build/execution-workload/Dockerfile - e2e/k8s/** - go.mod - go.sum @@ -54,6 +65,17 @@ on: - internal/adapter/server/grpc.go - internal/adapter/server/http.go - deploy/helm/mecak8s/** + - deploy/helm/mecatl-execution/** + - cmd/mecatl-execution-provider/** + - cmd/mecatl-executor/** + - internal/adapter/executioncontroller/** + - internal/adapter/executionclient/** + - internal/executionenv/** + - contracts/proto/mecatl/execution/** + - e2e/k8s_execution/** + - deploy/mecatl-execution-kind/** + - build/execution-provider/Dockerfile + - build/execution-workload/Dockerfile - e2e/k8s/** - go.mod - go.sum @@ -123,9 +145,9 @@ jobs: # kind: pinned version, installed via `go install` (no third-party action — # the house idiom for go-installable tools, matching actionlint/govulncheck - # in ci.yml). v0.32.0 is the current release. + # in ci.yml). v0.33.0 is the current release. - name: Install kind - run: go install sigs.k8s.io/kind@v0.32.0 + run: go install sigs.k8s.io/kind@v0.33.0 # ko: pinned version, same `go install` idiom. v0.18.1 builds the mecak8s # image from .ko.yaml onto the distroless base. KO_DOCKER_REPO=ko.local below @@ -166,3 +188,44 @@ jobs: path: k8s-e2e-output.log retention-days: 7 if-no-files-found: warn + + execution-production-e2e: + name: execution provider production profile (kind) + runs-on: ubuntu-24.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: go.mod + cache: true + cache-dependency-path: | + go.sum + engine/go.sum + - uses: go-task/setup-task@a00fbb05ce67b35648be3c78cbc9fd85354c757e # v2.2.0 + with: + version: 3.x + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: + version: v3.18.4 + - name: Install pinned Kind and ko + run: | + go install sigs.k8s.io/kind@v0.33.0 + go install github.com/google/ko@v0.18.1 + - name: Run production execution qualification + shell: bash + env: + CONTAINER_ENGINE: docker + MECATL_EXECUTION_QUAL_CI: "1" + run: task e2e:k8s:execution:production + - name: Upload bounded failure artifact + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: execution-e2e-artifacts-${{ github.run_id }} + path: .scratch/k8s-execution/*/production-failure-artifact.txt + retention-days: 7 + if-no-files-found: warn diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fa78c9278c..4a7271ea54 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,7 +49,8 @@ jobs: # KEEPS its own inline copy as well (like the pin gate it repeats): it is the one job that # writes outside this repository, so it re-verifies rather than inheriting. guard: - name: Ensure the release tag is on main + if: github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && github.ref == format('refs/tags/{0}', inputs.tag)) + name: Verify immutable release tag runs-on: ubuntu-24.04 # 10, not 5: the v0.0.35 release was LOST because an unfiltered depth-0 clone # took 5m18s and tripped a 5-minute cap, skipping all six publish jobs. The @@ -58,6 +59,9 @@ jobs: timeout-minutes: 10 permissions: contents: read + outputs: + version: ${{ steps.verify.outputs.version }} + commit: ${{ steps.verify.outputs.commit }} env: # On workflow_dispatch github.ref_name is the default branch, so prefer the # explicit input; on a tag push github.ref_name is the tag. @@ -66,26 +70,15 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - # 0 so origin/main is present to compare against. NOT `git fetch origin main`: - # persist-credentials is false, so a fetch has no credentials on this repo. fetch-depth: 0 - # This job reads NO file contents — only `git merge-base` and `git rev-parse` - # against the commit graph. An unfiltered depth-0 clone here cost ~5 minutes - # (306 MB, 184 branches, 123 tags) in front of every publish job, since all - # six `needs: guard`. blob:none fetches the graph without the blobs. filter: blob:none - ref: ${{ env.VERSION }} + # The guard implementation is trusted only from protected main. The tag + # and dispatch input remain data verified by that implementation. + ref: refs/heads/main - # HEAD, not $GITHUB_SHA: on workflow_dispatch $GITHUB_SHA is the branch head, while - # HEAD is the tag commit actually checked out above. - - name: Ensure the release tag is on main - run: | - set -euo pipefail - if ! git merge-base --is-ancestor HEAD origin/main; then - echo "::error::release tags must be cut from a commit on main; $(git rev-parse HEAD) is not on main" - exit 1 - fi - echo "$(git rev-parse HEAD) is on main" + - name: Verify and check out the immutable release tag + id: verify + run: .github/scripts/release-tag-guard.sh validate-release-ref: needs: guard @@ -117,10 +110,14 @@ jobs: ref: ${{ github.sha }} - name: Assert checkout is the immutable run commit run: test "$(git rev-parse HEAD)" = "${GITHUB_SHA}" + - name: Assert guard and immutable run commits agree + env: + RELEASE_COMMIT: ${{ needs.guard.outputs.commit }} + run: test "$(git rev-parse HEAD)" = "${RELEASE_COMMIT}" publish: name: Build, publish, sign, attest - needs: validate-release-ref + needs: [guard, validate-release-ref] runs-on: ubuntu-24.04 timeout-minutes: 20 permissions: @@ -134,7 +131,7 @@ jobs: KO_DOCKER_REPO: ghcr.io/${{ github.repository }} # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -214,7 +211,7 @@ jobs: --image-refs=image-refs.txt \ --image-label=org.opencontainers.image.source=https://github.com/${{ github.repository }} \ --image-label=org.opencontainers.image.title=mecated \ - --image-label=org.opencontainers.image.revision=${{ github.sha }} \ + --image-label=org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} \ --image-label=org.opencontainers.image.vendor=Stacklok \ ./cmd/mecated # The digest ref is the same image regardless of tag; take the first. @@ -660,7 +657,7 @@ jobs: # support and identifies it with the org.stacklok.broodbox.agent OCI label. publish-mecatui: name: Build, publish, sign, attest mecatui - needs: publish-microvm + needs: [guard, publish-microvm] runs-on: ubuntu-24.04 timeout-minutes: 20 permissions: @@ -675,7 +672,7 @@ jobs: KO_DOCKER_REPO: ghcr.io/${{ github.repository }}/mecatui # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -770,7 +767,7 @@ jobs: --image-label=org.stacklok.broodbox.agent=/var/run/ko/agent.yaml \ --image-label=org.opencontainers.image.source=https://github.com/${{ github.repository }} \ --image-label=org.opencontainers.image.title=mecatui \ - --image-label=org.opencontainers.image.revision=${{ github.sha }} \ + --image-label=org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} \ --image-label=org.opencontainers.image.vendor=Stacklok \ ./cmd/mecatui # The digest ref is the same image regardless of tag; take the first. @@ -837,7 +834,7 @@ jobs: publish-mecak8s: name: Build, publish, sign, attest mecak8s - needs: validate-release-ref + needs: [guard, validate-release-ref] runs-on: ubuntu-24.04 timeout-minutes: 20 permissions: @@ -852,7 +849,7 @@ jobs: KO_DOCKER_REPO: ghcr.io/${{ github.repository }}/mecak8s # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -931,7 +928,7 @@ jobs: --image-refs=image-refs.txt \ --image-label=org.opencontainers.image.source=https://github.com/${{ github.repository }} \ --image-label=org.opencontainers.image.title=mecak8s \ - --image-label=org.opencontainers.image.revision=${{ github.sha }} \ + --image-label=org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} \ --image-label=org.opencontainers.image.vendor=Stacklok \ ./cmd/mecak8s # The digest ref is the same image regardless of tag; take the first. @@ -996,6 +993,128 @@ jobs: # silently failing at it. create-storage-record: false + publish-execution-images: + needs: [guard, validate-release-ref] + name: Build, publish, sign, attest execution ${{ matrix.name }} + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + packages: write + id-token: write + attestations: write + strategy: + fail-fast: false + matrix: + include: + - {name: provider, dockerfile: build/execution-provider/Dockerfile, image: mecatl-execution-provider} + - {name: workload, dockerfile: build/execution-workload/Dockerfile, image: mecatl-execution-workload} + env: + VERSION: ${{ needs.guard.outputs.version }} + IMAGE: ghcr.io/${{ github.repository }}/${{ matrix.image }} + GO_IMAGE: ${{ vars.EXECUTION_GO_IMAGE }} + PROVIDER_RUNTIME_IMAGE: ${{ vars.EXECUTION_PROVIDER_RUNTIME_IMAGE }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: {persist-credentials: false, fetch-depth: 1, ref: "${{ needs.guard.outputs.commit }}"} + - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + with: + version: v0.30.1 + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v4.0.0 + with: {registry: ghcr.io, username: "${{ github.actor }}", password: "${{ secrets.GITHUB_TOKEN }}"} + - name: Validate immutable base inputs + run: | + case "$GO_IMAGE" in *@sha256:*) ;; *) echo "EXECUTION_GO_IMAGE must be digest-pinned" >&2; exit 1;; esac + if [ "${{ matrix.name }}" = provider ]; then + case "$PROVIDER_RUNTIME_IMAGE" in *@sha256:*) ;; *) echo "EXECUTION_PROVIDER_RUNTIME_IMAGE must be digest-pinned" >&2; exit 1;; esac + fi + - name: Build and push image + id: build + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 + with: + context: . + file: ${{ matrix.dockerfile }} + platforms: linux/amd64,linux/arm64 + push: true + provenance: false + sbom: false + build-args: | + GO_IMAGE=${{ env.GO_IMAGE }} + RUNTIME_IMAGE=${{ matrix.name == 'provider' && env.PROVIDER_RUNTIME_IMAGE || env.GO_IMAGE }} + tags: | + ${{ env.IMAGE }}:${{ env.VERSION }} + ${{ env.IMAGE }}:latest + labels: | + org.opencontainers.image.source=https://github.com/${{ github.repository }} + org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} + - name: Sign image + run: cosign sign --yes "${IMAGE}@${{ steps.build.outputs.digest }}" + - name: Generate SBOM + uses: anchore/sbom-action@3ad7283483fc7af8ff2b4ea19663c2d5ca935e26 # v0.24.2 + with: {image: "${{ env.IMAGE }}@${{ steps.build.outputs.digest }}", format: spdx-json, output-file: sbom.spdx.json, upload-artifact: false} + - name: Attest SBOM + run: cosign attest --yes --predicate sbom.spdx.json --type spdxjson "${IMAGE}@${{ steps.build.outputs.digest }}" + - name: Attest build provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ env.IMAGE }} + subject-digest: ${{ steps.build.outputs.digest }} + push-to-registry: true + create-storage-record: false + + publish-execution-chart: + needs: [guard, validate-release-ref, publish-execution-images] + name: Package, publish, sign, attest execution Helm chart + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: read + packages: write + id-token: write + attestations: write + env: + VERSION: ${{ needs.guard.outputs.version }} + OCI_REGISTRY: ghcr.io/${{ github.repository }}/charts + CHART: mecatl-execution + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: {persist-credentials: false, fetch-depth: 1, ref: "${{ needs.guard.outputs.commit }}"} + - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 + with: {version: v3.18.4} + - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2 + - uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v4.0.0 + with: {registry: ghcr.io, username: "${{ github.actor }}", password: "${{ secrets.GITHUB_TOKEN }}"} + - name: Package chart + id: package + run: | + set -euo pipefail + chart_version="${VERSION#v}" + mkdir -p dist + helm package deploy/helm/mecatl-execution --version "$chart_version" --app-version "$VERSION" --destination dist + echo "chart_version=$chart_version" >> "$GITHUB_OUTPUT" + - name: Push chart + id: push + run: | + set -euo pipefail + helm push "dist/${CHART}-${{ steps.package.outputs.chart_version }}.tgz" "oci://${OCI_REGISTRY}" > push-output.txt 2>&1 + cat push-output.txt + digest="$(awk '/^Digest:/ {print $2}' push-output.txt)" + case "$digest" in sha256:*) ;; *) echo "could not parse chart digest" >&2; exit 1;; esac + echo "digest=$digest" >> "$GITHUB_OUTPUT" + - name: Sign chart + run: cosign sign --yes "${OCI_REGISTRY}/${CHART}@${{ steps.push.outputs.digest }}" + # The chart has no chart dependencies or bundled binaries. Its packaged + # templates and metadata are the artifact, so image SBOMs cover executable + # contents while this immutable chart digest receives provenance directly. + - name: Attest chart provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-name: ${{ env.OCI_REGISTRY }}/${{ env.CHART }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true + create-storage-record: false + # Not a Go binary, so this doesn't go through ko like the jobs above — a # plain docker/build-push-action build of the existing dev/Compose # Dockerfile. The bot's TypeScript source still lives under @@ -1003,7 +1122,7 @@ jobs: # ../sdk/typescript/examples/slack-bot/README.md); only this container # image is released, from the same v* tag as mecated/mecatui/mecak8s. publish-slack-bot: - needs: validate-release-ref + needs: [guard, validate-release-ref] name: Build, publish, sign, attest Slack bot example image runs-on: ubuntu-24.04 timeout-minutes: 20 @@ -1016,7 +1135,7 @@ jobs: IMAGE: ghcr.io/${{ github.repository }}/slack-bot # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -1053,7 +1172,7 @@ jobs: labels: | org.opencontainers.image.source=https://github.com/${{ github.repository }} org.opencontainers.image.title=slack-bot - org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.revision=${{ needs.guard.outputs.commit }} org.opencontainers.image.vendor=Stacklok # Sign/SBOM/attest the pushed manifest list BY DIGEST, matching the @@ -1249,7 +1368,7 @@ jobs: CHART_PATH: deploy/helm/mecak8s # On workflow_dispatch github.ref_name is the default branch, so prefer # the explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -1400,7 +1519,7 @@ jobs: env: # On workflow_dispatch github.ref_name is the default branch, so prefer the # explicit input; on a tag push github.ref_name is the tag. - VERSION: ${{ inputs.tag || github.ref_name }} + VERSION: ${{ needs.guard.outputs.version }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -1421,7 +1540,7 @@ jobs: # credentials for a lazy blob fetch, so do not add a step here that reads a # HISTORICAL file's contents. filter: blob:none - ref: ${{ env.VERSION }} + ref: ${{ needs.guard.outputs.commit }} # Eight cross-compiles (2 binaries x 4 platforms) of this dependency graph # exhaust the runner's disk: the first v0.0.30 attempt died 18 minutes in diff --git a/Taskfile.yml b/Taskfile.yml index 55593dd775..a8b578cbbf 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -75,6 +75,9 @@ tasks: # mecak8s: the storage-free, k8s-native agent binary (ADR 0048; a thin peer # of mecated composing app.Build with Redis store + k8s lease + drain gate). - go build -ldflags "{{.BUILD_LDFLAGS}}" -o bin/mecak8s ./cmd/mecak8s + # Optional native Kubernetes execution provider and its fixed workload helper. + - go build -ldflags "{{.BUILD_LDFLAGS}}" -o bin/mecatl-execution-provider ./cmd/mecatl-execution-provider + - go build -ldflags "{{.BUILD_LDFLAGS}}" -o bin/mecatl-executor ./cmd/mecatl-executor - go build -ldflags "{{.BUILD_LDFLAGS}}" -o bin/mecatui ./cmd/mecatui # engine/ is its own Go module (consumed via the committed go.work). Building # it here keeps the importable-core module compiling alongside the binaries. @@ -665,6 +668,50 @@ tasks: esac GOWORK=off go test -tags kind_e2e -count=1 -timeout 30m ./e2e/k8s/... + e2e:k8s:execution: + desc: | + Run the focused mock-only Kind qualification for the optional Kubernetes + execution provider. It uses a unique cluster and explicit kubeconfig under + .scratch, builds/loads immutable local images, and never reads a provider + credential. Docker is the default; set CONTAINER_ENGINE=podman for rootless + Podman. The owned cluster is intentionally retained; the command prints the + exact human-confirmed cleanup command. NOT part of task test. + preconditions: + - sh: command -v kind >/dev/null 2>&1 + msg: "kind is required" + - sh: | + runtime="${CONTAINER_ENGINE:-docker}" + case "$runtime" in docker|podman) command -v "$runtime" >/dev/null 2>&1 ;; *) exit 1 ;; esac + msg: "CONTAINER_ENGINE must be docker or podman and available on PATH" + cmds: + - deploy/mecatl-execution-kind/run.sh + + e2e:k8s:execution:production: + desc: | + Run the deterministic production-network and lifecycle execution-provider + qualification. CI sets MECATL_EXECUTION_QUAL_CI=1 so its uniquely owned Kind + cluster is removed automatically; local runs retain it for inspection and + print the human-confirmed cleanup command. No model credential is used. + cmds: + - MECATL_EXECUTION_QUAL_PROFILE=production task e2e:k8s:execution + + e2e:k8s:execution:live: + desc: | + EXPLICIT OPT-IN: run mock qualification followed by one bounded real-provider + coding smoke on the approved retained Kind cluster. Requires absolute + MECATL_EXECUTION_CREDENTIAL_FILE and MECATL_EXECUTION_QUAL_STATE paths. A + trusted loader stages a run-scoped Secret without exposing its value, the + test independently verifies files and go test through typed gRPC, then the + mock profile is restored and only that run-scoped Secret is removed. NOT + part of task test. + preconditions: + - sh: test -n "${MECATL_EXECUTION_CREDENTIAL_FILE:-}" + msg: "MECATL_EXECUTION_CREDENTIAL_FILE is required" + - sh: test -n "${MECATL_EXECUTION_QUAL_STATE:-}" + msg: "MECATL_EXECUTION_QUAL_STATE is required" + cmds: + - deploy/mecatl-execution-kind/live.sh + e2e:k8s:podman: desc: Run e2e:k8s with rootless Podman (requires the k8s-file Podman log driver) cmds: @@ -1048,6 +1095,7 @@ tasks: - bash .github/scripts/ensure-perf-dashboard-noindex_test.sh - bash .github/scripts/deslop-workflow_test.sh - bash .github/scripts/microvm-ci-release_test.sh + - sh .github/scripts/release-tag-guard_test.sh lint:action-templates: desc: | diff --git a/build/execution-provider/Dockerfile b/build/execution-provider/Dockerfile new file mode 100644 index 0000000000..7c3574b94d --- /dev/null +++ b/build/execution-provider/Dockerfile @@ -0,0 +1,16 @@ +# Both bases are mandatory digest references supplied by the release workflow. +ARG GO_IMAGE +ARG RUNTIME_IMAGE +FROM ${GO_IMAGE} AS build +ARG GO_IMAGE +ARG RUNTIME_IMAGE +WORKDIR /src +COPY . . +RUN case "${GO_IMAGE}" in *@sha256:*) ;; *) echo "GO_IMAGE must be digest-pinned" >&2; exit 1;; esac \ + && case "${RUNTIME_IMAGE}" in *@sha256:*) ;; *) echo "RUNTIME_IMAGE must be digest-pinned" >&2; exit 1;; esac \ + && CGO_ENABLED=0 go build -trimpath -buildvcs=false -o /out/mecatl-execution-provider ./cmd/mecatl-execution-provider + +FROM ${RUNTIME_IMAGE} +COPY --from=build --chown=65532:65532 /out/mecatl-execution-provider /mecatl-execution-provider +USER 65532:65532 +ENTRYPOINT ["/mecatl-execution-provider"] diff --git a/build/execution-workload/Dockerfile b/build/execution-workload/Dockerfile new file mode 100644 index 0000000000..1654cf2acb --- /dev/null +++ b/build/execution-workload/Dockerfile @@ -0,0 +1,17 @@ +# GO_IMAGE is deliberately mandatory and must include a digest, for example: +# docker build --build-arg GO_IMAGE=golang:1.27@sha256: ... +ARG GO_IMAGE +FROM ${GO_IMAGE} AS build +ARG GO_IMAGE +WORKDIR /src +COPY . . +RUN case "${GO_IMAGE}" in *@sha256:*) ;; *) echo "GO_IMAGE must be digest-pinned" >&2; exit 1;; esac \ + && go build -trimpath -buildvcs=false -o /out/mecatl-executor ./cmd/mecatl-executor + +FROM ${GO_IMAGE} +COPY --from=build --chown=65532:65532 /out/mecatl-executor /mecatl-executor +RUN mkdir -p /workspace /tmp && chown 65532:65532 /workspace /tmp +USER 65532:65532 +WORKDIR /workspace +ENV HOME=/workspace TMPDIR=/tmp GOTMPDIR=/tmp +ENTRYPOINT ["/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"] diff --git a/cmd/mecak8s/execution_startup_test.go b/cmd/mecak8s/execution_startup_test.go new file mode 100644 index 0000000000..06c2812538 --- /dev/null +++ b/cmd/mecak8s/execution_startup_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" +) + +func TestDisabledExecutionStartupDoesNotContactExecutionOrKubernetes(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "config")) + t.Setenv("XDG_STATE_HOME", filepath.Join(home, "state")) + var requests atomic.Int32 + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer endpoint.Close() + kubeconfig := filepath.Join(home, "kubeconfig") + contents := "apiVersion: v1\nkind: Config\nclusters:\n- name: poison\n cluster:\n server: " + endpoint.URL + "\ncontexts:\n- name: poison\n context:\n cluster: poison\n user: offline\ncurrent-context: poison\nusers:\n- name: offline\n user: {}\n" + if err := os.WriteFile(kubeconfig, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("KUBECONFIG", kubeconfig) + oldArgs, oldLogger := os.Args, slog.Default() + t.Cleanup(func() { os.Args = oldArgs; slog.SetDefault(oldLogger) }) + // Exercise run(), including actual app.Build, stopping at the first listener. + // Poison TLS paths would fail before serve if the disabled client were built. + os.Args = []string{"mecak8s", "--mock", "--posture=strict", "--no-soul", "--no-user-model", "--no-scheduler", "--session-lease-k8s-namespace=", "--grpc-addr=invalid-address", "--execution-enabled=false", "--execution-endpoint=" + strings.TrimPrefix(endpoint.URL, "http://"), "--execution-profile=unused", "--execution-tls-ca=" + filepath.Join(home, "absent-ca"), "--execution-tls-cert=" + filepath.Join(home, "absent-cert"), "--execution-tls-key=" + filepath.Join(home, "absent-key")} + err := run() + if err == nil || !strings.Contains(err.Error(), "invalid-address") { + t.Fatalf("did not reach listener after composition: %v", err) + } + if requests.Load() != 0 { + t.Fatalf("disabled startup made %d API requests", requests.Load()) + } +} + +func TestExecutionPreflightRejectsIncompleteAndIncompatibleCLI(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + required := []string{"--execution-endpoint=127.0.0.1:1", "--execution-profile=go", "--execution-tls-ca=unused", "--execution-tls-cert=unused", "--execution-tls-key=unused"} + base := []string{"--mock", "--execution-enabled", "--oidc-issuer=https://issuer.example", "--oidc-audience=mecatl"} + if _, err := parseFlags(append(append([]string{}, base...), required...)); err != nil { + t.Fatalf("complete configuration rejected: %v", err) + } + for missing := range required { + args := append([]string{}, base...) + for i, flag := range required { + if i != missing { + args = append(args, flag) + } + } + if _, err := parseFlags(args); err == nil || !strings.Contains(err.Error(), "--execution-enabled requires") { + t.Fatalf("missing %s: %v", required[missing], err) + } + } + for _, extra := range []string{"--workspace=unused", "--redis-filesystem", "--enable-parallel", "--enable-teams"} { + args := append(append(append([]string{}, base...), required...), extra) + if _, err := parseFlags(args); err == nil { + t.Fatalf("incompatible %s accepted", extra) + } + } +} diff --git a/cmd/mecak8s/flags.go b/cmd/mecak8s/flags.go index 125f0b06b3..3f2c0adc87 100644 --- a/cmd/mecak8s/flags.go +++ b/cmd/mecak8s/flags.go @@ -120,6 +120,12 @@ type config struct { httpShutdownTimeout time.Duration closeTimeout time.Duration workspace string + executionEnabled bool + executionEndpoint string + executionProfile string + executionTLSCA string + executionTLSCert string + executionTLSKey string model string defaultProvider string defaultModel string @@ -352,6 +358,12 @@ func parseFlags(argv []string) (config, error) { positiveDurationFlag(fs, &cfg.httpShutdownTimeout, "http-shutdown-timeout", defaultHTTPShutdownTimeout, "Maximum time for graceful HTTP and metrics shutdown") positiveDurationFlag(fs, &cfg.closeTimeout, "close-timeout", defaultCloseTimeout, "Maximum time for final application cleanup") fs.StringVar(&cfg.workspace, "workspace", "", "Shared agent workspace root, such as a mounted PVC. Empty gives every session a shell-less, file-less workspace; clients cannot choose another root") + fs.BoolVar(&cfg.executionEnabled, "execution-enabled", false, "Use an independently deployed Kubernetes execution provider for default sessions") + fs.StringVar(&cfg.executionEndpoint, "execution-endpoint", "", "Host:port endpoint of the mTLS gRPC execution provider. Requires --execution-enabled") + fs.StringVar(&cfg.executionProfile, "execution-profile", "", "Operator-configured execution provider profile") + fs.StringVar(&cfg.executionTLSCA, "execution-tls-ca", "", "Mounted CA bundle used only by the execution client") + fs.StringVar(&cfg.executionTLSCert, "execution-tls-cert", "", "Mounted execution-provider mTLS client certificate") + fs.StringVar(&cfg.executionTLSKey, "execution-tls-key", "", "Mounted execution-provider mTLS client private key") fs.StringVar(&cfg.model, "model", "", "Model identifier sent to the provider. Empty uses the provider default") fs.StringVar(&cfg.defaultProvider, "default-provider", "", "Deployment-wide default provider ID, such as openai, openrouter, or anthropic. Invalid values prevent startup") fs.StringVar(&cfg.defaultModel, "default-model", "", "Deployment-wide default model ID for the default provider. Invalid values prevent startup") @@ -612,6 +624,21 @@ func parseFlags(argv []string) (config, error) { if cfg.workspace != "" && (!filepath.IsAbs(cfg.workspace) || filepath.Clean(cfg.workspace) != cfg.workspace) { return config{}, fmt.Errorf("--workspace %q must be a clean absolute path (a mounted filesystem root); leave it empty for a file-less deployment", cfg.workspace) } + if cfg.executionEnabled { + if cfg.executionEndpoint == "" || cfg.executionProfile == "" || cfg.executionTLSCA == "" || cfg.executionTLSCert == "" || cfg.executionTLSKey == "" { + return config{}, errors.New("--execution-enabled requires --execution-endpoint, --execution-profile, --execution-tls-ca, --execution-tls-cert, and --execution-tls-key") + } + if !cfg.oidc.Enabled() { + return config{}, errors.New("--execution-enabled requires OIDC caller ownership enforcement") + } + if cfg.workspace != "" || cfg.redisFilesystem { + return config{}, errors.New("--execution-enabled conflicts with --workspace and --redis-filesystem") + } + if cfg.enableParallel || cfg.enableTeams { + return config{}, errors.New("remote execution does not support --enable-parallel or --enable-teams") + } + cfg.noScheduler = true + } if cfg.redisFilesystem && cfg.workspace != "" { return config{}, errors.New("--redis-filesystem and --workspace are mutually exclusive") } @@ -659,6 +686,7 @@ func appConfig(cfg config, diag port.Diagnostics, obs observability) app.Config mcpAuthorityDefault = mcpauthority.Global } out := app.Config{ + RemoteExecution: cfg.executionEnabled, Workspace: cfg.workspace, Model: cfg.model, DefaultProvider: cfg.defaultProvider, diff --git a/cmd/mecak8s/main.go b/cmd/mecak8s/main.go index d1bcc0c430..aa994c75a1 100644 --- a/cmd/mecak8s/main.go +++ b/cmd/mecak8s/main.go @@ -10,6 +10,8 @@ import ( "os" "time" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/internal/adapter/executionclient" "github.com/stacklok/mecatl/internal/adapter/mockscript" "github.com/stacklok/mecatl/internal/adapter/slogdiag" "github.com/stacklok/mecatl/internal/app" @@ -67,6 +69,11 @@ func run() error { // port.Diagnostics (ban-guarded). Mirrors cmd/mecated. slog.SetDefault(logger) diag := slogdiag.NewFromLogger(logger) + // Native debug qualification consumes structured diagnostics only. Keep the + // ordinary daemon/third-party logger unchanged; no new operator flag is needed. + if cfg.executionEnabled && cfg.logLevel == slog.LevelDebug { + diag = slogdiag.New(os.Stderr, true, port.LevelDebug) + } cfg.diagnostics = diag ctx, stop := signalCtx() @@ -82,6 +89,26 @@ func run() error { } composition := appConfig(cfg, diag, obs) + if cfg.executionEnabled { + tlsConfig, tlsErr := executionclient.LoadTLSConfig(executionclient.TLSFiles{CA: cfg.executionTLSCA, Cert: cfg.executionTLSCert, Key: cfg.executionTLSKey}) + if tlsErr != nil { + flushTelemetry(os.Stderr, obs, cfg.otlpShutdownTimeout) + return tlsErr + } + client, clientErr := executionclient.New(cfg.executionEndpoint, tlsConfig) + if clientErr != nil { + flushTelemetry(os.Stderr, obs, cfg.otlpShutdownTimeout) + return clientErr + } + defer client.Close() + placement, placementErr := executionclient.NewProvider(client, cfg.executionProfile) + if placementErr != nil { + flushTelemetry(os.Stderr, obs, cfg.otlpShutdownTimeout) + return placementErr + } + composition.PlacementProvider = placement + composition.PlacementScope = "remote-execution" + } built, err := app.Build(ctx, composition) if err != nil { flushTelemetry(os.Stderr, obs, cfg.otlpShutdownTimeout) diff --git a/cmd/mecatl-execution-provider/main.go b/cmd/mecatl-execution-provider/main.go new file mode 100644 index 0000000000..e703fc9fd1 --- /dev/null +++ b/cmd/mecatl-execution-provider/main.go @@ -0,0 +1,163 @@ +// Command mecatl-execution-provider runs the authenticated Kubernetes execution provider. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/keepalive" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/adapter/slogdiag" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func main() { + if err := run(); err != nil { + slog.Error("execution provider stopped", "error", err) + os.Exit(1) + } +} +func run() error { //nolint:gocyclo // Startup validation and owned-resource shutdown stay in one composition root. + var addr, healthAddr, namespace, profilesPath, manifestPath, keyDirectory, authorityConfigMap string + var reloadInterval time.Duration + var maxConcurrentStreams, maxConcurrentRPCs, maxConcurrentRPCsPerClient int + flag.StringVar(&addr, "listen", ":8443", "gRPC listen address") + flag.StringVar(&namespace, "namespace", "", "managed Kubernetes namespace") + flag.StringVar(&profilesPath, "profiles", "/etc/mecatl-execution/profiles.yaml", "strict operator profile file") + flag.StringVar(&healthAddr, "health-listen", ":8081", "operational HTTP health listen address; empty disables") + flag.StringVar(&manifestPath, "grant-keyring-manifest", "/etc/mecatl-execution/security/manifest.json", "versioned security manifest") + flag.StringVar(&keyDirectory, "grant-key-directory", "/etc/mecatl-execution/security", "projected security key and TLS directory") + flag.StringVar(&authorityConfigMap, "security-authority-configmap", "", "provider-owned security generation high-water ConfigMap") + flag.DurationVar(&reloadInterval, "security-reload-interval", 2*time.Second, "security material reload interval") + flag.IntVar(&maxConcurrentStreams, "max-concurrent-streams", 64, "maximum concurrent HTTP/2 streams per connection") + flag.IntVar(&maxConcurrentRPCs, "max-concurrent-rpcs", 128, "maximum active provider RPCs") + flag.IntVar(&maxConcurrentRPCsPerClient, "max-concurrent-rpcs-per-client", 32, "maximum active provider RPCs per authorized client") + flag.Parse() + if namespace == "" || authorityConfigMap == "" || reloadInterval <= 0 || reloadInterval > time.Minute || maxConcurrentStreams < 1 || maxConcurrentStreams > 1024 || maxConcurrentRPCs < 1 || maxConcurrentRPCs > 4096 || maxConcurrentRPCsPerClient < 1 || maxConcurrentRPCsPerClient > maxConcurrentRPCs { + return errors.New("required identity, security reload interval, or RPC concurrency bounds are invalid") + } + profiles, err := executioncontroller.LoadProfiles(profilesPath) + if err != nil { + return err + } + cfg, err := rest.InClusterConfig() + if err != nil { + return fmt.Errorf("build in-cluster Kubernetes config: %w", err) + } + kube, err := kubernetes.NewForConfig(cfg) + if err != nil { + return err + } + dyn, err := dynamic.NewForConfig(cfg) + if err != nil { + return err + } + security := executioncontroller.NewSecurityManager(manifestPath, keyDirectory, namespace, authorityConfigMap, kube) + if err := security.Reload(context.Background()); err != nil { + return fmt.Errorf("load security material: %w", err) + } + podexec := executioncontroller.NewPodExecutor(cfg, kube, namespace) + store := executioncontroller.NewStore(dyn, namespace, profiles, podexec).WithKubeClient(kube) + reconciler := executioncontroller.NewReconciler(dyn, kube, namespace, profiles) + handler := executioncontroller.NewHandler(executioncontroller.HandlerConfig{Security: security, Ready: func() bool { return reconciler.Ready() && security.Ready() }, Diagnostics: slogdiag.New(os.Stderr, true, port.LevelInfo)}, store) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + go security.Run(ctx, reloadInterval) + if err := reconciler.Initialize(ctx); err != nil { + return fmt.Errorf("initialize controller: %w", err) + } + controllerErr := make(chan error, 1) + go func() { controllerErr <- reconciler.Run(ctx) }() + ln, err := net.Listen("tcp", addr) + if err != nil { + return err + } + limiter, err := executioncontroller.NewRPCLimiter(security, maxConcurrentRPCs, maxConcurrentRPCsPerClient) + if err != nil { + return err + } + server := grpc.NewServer( + grpc.Creds(credentials.NewTLS(security.TLSConfig())), + grpc.ChainUnaryInterceptor(limiter.UnaryInterceptor), + grpc.MaxRecvMsgSize(executionenv.MaxMessageBytes), + grpc.MaxSendMsgSize(executionenv.MaxMessageBytes), + grpc.MaxConcurrentStreams(uint32(maxConcurrentStreams)), //nolint:gosec // validated to 1..1024 above. + grpc.KeepaliveParams(keepalive.ServerParameters{MaxConnectionAge: 30 * time.Minute, MaxConnectionAgeGrace: 2 * time.Minute, Time: 2 * time.Minute, Timeout: 20 * time.Second}), + grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{MinTime: 30 * time.Second, PermitWithoutStream: false}), + ) + executionv1.RegisterExecutionProviderServiceServer(server, handler) + var healthServer *http.Server + if healthAddr != "" { + mux := http.NewServeMux() + mux.HandleFunc("GET /live", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + mux.HandleFunc("GET /ready", func(w http.ResponseWriter, r *http.Request) { + if !reconciler.Ready() { + http.Error(w, "not ready: "+reconciler.ReadinessReason(), http.StatusServiceUnavailable) + return + } + readyCtx, stop := context.WithTimeout(r.Context(), 2*time.Second) + defer stop() + if !security.CheckReady(readyCtx) { + http.Error(w, "not ready: security-authority-or-expiry", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusNoContent) + }) + healthServer = &http.Server{Addr: healthAddr, Handler: mux, ReadHeaderTimeout: 2 * time.Second, IdleTimeout: 30 * time.Second} + go func() { + if err := healthServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + cancel() + } + }() + } + defer func() { + if healthServer != nil { + shutdown, stop := context.WithTimeout(context.Background(), 3*time.Second) + defer stop() + _ = healthServer.Shutdown(shutdown) + } + }() + serveErr := make(chan error, 1) + go func() { serveErr <- server.Serve(ln) }() + select { + case err := <-controllerErr: + cancel() + server.GracefulStop() + if err != nil { + return fmt.Errorf("controller: %w", err) + } + return nil + case err := <-serveErr: + cancel() + if err != nil { + return err + } + return nil + case <-ctx.Done(): + done := make(chan struct{}) + go func() { server.GracefulStop(); close(done) }() + select { + case <-done: + case <-time.After(10 * time.Second): + server.Stop() + } + return nil + } +} diff --git a/cmd/mecatl-executor/main.go b/cmd/mecatl-executor/main.go new file mode 100644 index 0000000000..34c594783e --- /dev/null +++ b/cmd/mecatl-executor/main.go @@ -0,0 +1,81 @@ +// Command mecatl-executor runs the fixed credential-free workload helper. +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/signal" + "runtime" + "syscall" + "time" + + "github.com/stacklok/mecatl/internal/executionenv" + "github.com/stacklok/mecatl/internal/executionexecutor" +) + +const workspaceRoot = "/workspace" + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + if err := run(ctx, os.Stdin, os.Stdout); err != nil { + _, _ = fmt.Fprintln(os.Stderr, "mecatl-executor: protocol output failed") + os.Exit(1) + } +} + +func run(parent context.Context, in io.Reader, out io.Writer) error { + var resp executionenv.ExecutorResponse + var opErr error + if runtime.GOOS != "linux" { + opErr = errors.New("mecatl-executor requires Linux at runtime") + } else if err := executionexecutor.EnableCommandExecution(); err != nil { + opErr = err + } else { + reader := bufio.NewReader(&io.LimitedReader{R: in, N: executionenv.MaxJSONBody + 1}) + body, err := reader.ReadBytes('\n') + if err != nil && !errors.Is(err, io.EOF) { + opErr = err + } else if len(body) > executionenv.MaxJSONBody { + opErr = &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "executor request exceeds limit"} + } else { + var q executionenv.ExecutorRequest + if err := executionenv.DecodeStrict(body, &q); err != nil { + opErr = err + } else { + x, err := executionexecutor.New(workspaceRoot, executionexecutor.Limits{CommandTimeout: 30 * time.Minute}) + if err != nil { + opErr = err + } else { + defer func() { _ = x.Close() }() + opCtx, cancel := context.WithCancel(parent) + if q.Operation == executionenv.OpCommandStart { + go func() { + var one [1]byte + _, _ = reader.Read(one[:]) + cancel() + }() + } + resp, opErr = x.Execute(opCtx, q) + cancel() + } + } + } + } + envelope := executionenv.ExecutorEnvelope{} + if opErr != nil { + var pe *executionenv.Error + if !errors.As(opErr, &pe) { + pe = &executionenv.Error{Code: executionenv.CodeInternal, Message: "executor operation failed"} + } + envelope.Error = pe + } else { + envelope.Response = &resp + } + return json.NewEncoder(out).Encode(envelope) +} diff --git a/cmd/mecatui/client/client.go b/cmd/mecatui/client/client.go index 45680906ea..a3f535bb6d 100644 --- a/cmd/mecatui/client/client.go +++ b/cmd/mecatui/client/client.go @@ -611,6 +611,31 @@ func containsBoundedCode(s, code string) bool { func isDigitByte(b byte) bool { return b >= '0' && b <= '9' } +// SafeStartupRunEntryErrorTitle returns a non-empty closed title only when the +// status can be presented without weakening masked session absence. +func SafeStartupRunEntryErrorTitle(err error) string { + switch status.Code(err) { + case codes.Unavailable, codes.DeadlineExceeded, codes.ResourceExhausted, codes.FailedPrecondition, codes.Aborted, codes.AlreadyExists: + return "turn not started" + default: + return "" + } +} + +// SafeStartupRunEntryError maps a pre-session.init Converse failure to closed, +// user-actionable text. Absence and authorization failures deliberately share the +// generic message so unknown and foreign session IDs remain indistinguishable. +func SafeStartupRunEntryError(err error) string { + switch status.Code(err) { + case codes.Unavailable, codes.DeadlineExceeded, codes.ResourceExhausted: + return "The service is temporarily unavailable. Retry this turn." + case codes.FailedPrecondition, codes.Aborted, codes.AlreadyExists: + return "This chat is not ready for a new turn. Retry after its current operation finishes." + default: + return "This conversation could not be loaded. You cannot continue this session." + } +} + // TransientStreamErr classifies a Converse stream Recv error for presentation and // compatibility only. A stream error has no semantic commit fact, so the TUI never // uses this signal to authorize automatic replay or queue draining. The gRPC status @@ -681,11 +706,13 @@ func (c *Client) openConverse(ctx context.Context) (*Stream, error) { // ModeDefaultString is the canonical CLI/UI spelling for default permission mode. const ModeDefaultString = "default" +const modePlanString = "plan" + // ModeFromString maps a CLI mode string to the proto enum. Unknown/empty maps to // UNSPECIFIED (the server defaults that to DEFAULT). func ModeFromString(s string) mecatlv1.PermissionMode { switch s { - case "plan": + case modePlanString: return mecatlv1.PermissionMode_PERMISSION_MODE_PLAN case "accept-edits", "acceptEdits", "accept_edits", "accept edits": return mecatlv1.PermissionMode_PERMISSION_MODE_ACCEPT_EDITS @@ -701,7 +728,7 @@ func ModeFromString(s string) mecatlv1.PermissionMode { func ModeString(m mecatlv1.PermissionMode) string { switch m { case mecatlv1.PermissionMode_PERMISSION_MODE_PLAN: - return "plan" + return modePlanString case mecatlv1.PermissionMode_PERMISSION_MODE_ACCEPT_EDITS: return "accept-edits" case mecatlv1.PermissionMode_PERMISSION_MODE_DEFAULT, mecatlv1.PermissionMode_PERMISSION_MODE_UNSPECIFIED: @@ -715,8 +742,8 @@ func ModeString(m mecatlv1.PermissionMode) string { func NextMode(mode string) string { switch ModeString(ModeFromString(mode)) { case ModeDefaultString: - return "plan" - case "plan": + return modePlanString + case modePlanString: return "accept-edits" default: return ModeDefaultString diff --git a/cmd/mecatui/client/msgs.go b/cmd/mecatui/client/msgs.go index 7e0ecb0db8..48c5e9bca5 100644 --- a/cmd/mecatui/client/msgs.go +++ b/cmd/mecatui/client/msgs.go @@ -156,6 +156,7 @@ type PermissionAskMsg struct { Reason string ExpectedRunID string Guardrail *GuardrailApprovalScope + Recovery bool } // GuardrailApprovalScope distinguishes an outbound action approval from release diff --git a/cmd/mecatui/client/pendingapproval.go b/cmd/mecatui/client/pendingapproval.go new file mode 100644 index 0000000000..14faa6a139 --- /dev/null +++ b/cmd/mecatui/client/pendingapproval.go @@ -0,0 +1,464 @@ +package client + +import ( + "context" + "encoding/json" + "errors" + "io" + "sync" + + tea "charm.land/bubbletea/v2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" +) + +// PendingApproval is the exact ordinary ask correlation recovered from a +// complete durable replay. Cursor remains scoped to the unfiltered watch that +// issued it; continuation filtering is therefore performed locally by RunID. +type PendingApproval struct { + SessionID string + RunID string + AskID string + Tool string + Args string + Reason string + Cursor string +} + +// PendingApprovalFailure is a closed, proto-free refusal category. +type PendingApprovalFailure string + +// Pending approval failure categories are closed and safe for presentation code. +const ( + PendingApprovalNone PendingApprovalFailure = "none" + PendingApprovalIncomplete PendingApprovalFailure = "incomplete_replay" + PendingApprovalMalformed PendingApprovalFailure = "malformed_correlation" + PendingApprovalUnsupportedAsk PendingApprovalFailure = "unsupported_ask" + PendingApprovalWatchUnsupported PendingApprovalFailure = "watch_unsupported" + PendingApprovalControlsUnsupported PendingApprovalFailure = "controls_unsupported" + PendingApprovalNotRecoverable PendingApprovalFailure = "not_recoverable" + PendingApprovalCorrelation PendingApprovalFailure = "correlation_mismatch" + PendingApprovalUnavailable PendingApprovalFailure = "unavailable" +) + +type pendingApprovalError struct { + kind PendingApprovalFailure + text string +} + +func (e *pendingApprovalError) Error() string { return e.text } + +// IsPendingApprovalFailure reports a closed recovery refusal without exposing a +// server error or tool arguments to presentation code. +func IsPendingApprovalFailure(err error, kind PendingApprovalFailure) bool { + var target *pendingApprovalError + return errors.As(err, &target) && target.kind == kind +} + +func pendingApprovalFailure(kind PendingApprovalFailure) error { + text := "pending approval recovery is unavailable" + switch kind { + case PendingApprovalNone: + text = "no recoverable pending ordinary approval was found" + case PendingApprovalIncomplete: + text = "pending approval recovery requires a complete gap-free event replay" + case PendingApprovalMalformed: + text = "pending approval recovery found invalid event correlation" + case PendingApprovalUnsupportedAsk: + text = "this pending approval type is not supported by recovery" + case PendingApprovalWatchUnsupported: + text = "pending approval recovery requires durable event watch support" + case PendingApprovalControlsUnsupported: + text = "pending approval recovery controls are not supported" + case PendingApprovalNotRecoverable: + text = "pending approval is no longer recoverable because its current state changed" + case PendingApprovalCorrelation: + text = "pending approval recovery correlation changed" + } + return &pendingApprovalError{kind: kind, text: text} +} + +func pendingApprovalControlFailure(err error) error { + switch status.Code(err) { + case codes.Aborted: + return pendingApprovalFailure(PendingApprovalCorrelation) + case codes.FailedPrecondition: + return pendingApprovalFailure(PendingApprovalNotRecoverable) + case codes.Unimplemented: + return pendingApprovalFailure(PendingApprovalControlsUnsupported) + default: + return pendingApprovalFailure(PendingApprovalUnavailable) + } +} + +func pendingApprovalWatchFailure(err error) error { + if status.Code(err) == codes.Unimplemented { + return pendingApprovalFailure(PendingApprovalWatchUnsupported) + } + return pendingApprovalFailure(PendingApprovalUnavailable) +} + +type pendingApprovalWatchStream interface { + Recv() (*mecatlv1.WatchSessionEventsResponse, error) +} + +// PendingApprovalEventKind is the narrow recovery lifecycle vocabulary. +type PendingApprovalEventKind string + +// Pending approval event kinds describe the recovery lifecycle projection. +const ( + PendingApprovalEventBoundary PendingApprovalEventKind = "boundary" + PendingApprovalEventAsk PendingApprovalEventKind = "ask" + PendingApprovalEventResolved PendingApprovalEventKind = "resolved" + PendingApprovalEventRetracted PendingApprovalEventKind = "retracted" + PendingApprovalEventTerminal PendingApprovalEventKind = "terminal" + PendingApprovalEventOther PendingApprovalEventKind = "other" +) + +// PendingApprovalEvent is one safe projection from the exact-run continuation. +type PendingApprovalEvent struct { + Kind PendingApprovalEventKind + Approval *PendingApproval + Message tea.Msg +} + +// PendingApprovalWatch owns the context of one durable subscription. Call Close +// on every path; Recv does not start a background goroutine. +type PendingApprovalWatch struct { + stream pendingApprovalWatchStream + cancel context.CancelFunc + closeOnce sync.Once + sessionID string + runID string +} + +// Close cancels the in-flight Recv and releases the server-side follower. +func (w *PendingApprovalWatch) Close() { + if w == nil { + return + } + w.closeOnce.Do(w.cancel) +} + +func (c *Client) openPendingApprovalWatch(ctx context.Context, sessionID, cursor string) (*PendingApprovalWatch, error) { + if sessionID == "" { + return nil, pendingApprovalFailure(PendingApprovalMalformed) + } + watchCtx, cancel := context.WithCancel(ctx) + stream, err := c.svc.WatchSessionEvents(withSessionAffinity(watchCtx, sessionID), &mecatlv1.WatchSessionEventsRequest{ + SessionId: sessionID, + Cursor: cursor, + // cursor was issued by an unfiltered discovery watch. Changing its wire + // run_id scope would silently skip records; exact-run filtering is local. + }) + if err != nil { + cancel() + return nil, pendingApprovalWatchFailure(err) + } + return &PendingApprovalWatch{stream: stream, cancel: cancel, sessionID: sessionID}, nil +} + +// WatchPendingApprovalRun resumes from the discovery cursor before a verdict is +// sent and projects only the recovered run. The caller owns Close. +func (c *Client) WatchPendingApprovalRun(ctx context.Context, approval PendingApproval) (*PendingApprovalWatch, error) { + if approval.SessionID == "" || approval.RunID == "" || approval.AskID == "" || approval.Cursor == "" { + return nil, pendingApprovalFailure(PendingApprovalMalformed) + } + watch, err := c.openPendingApprovalWatch(ctx, approval.SessionID, approval.Cursor) + if err != nil { + return nil, err + } + watch.runID = approval.RunID + return watch, nil +} + +// Recv returns the next event for the exact recovered run. Events from other +// runs are consumed but never adopted. Gap and unknown watch phases fail closed. +func (w *PendingApprovalWatch) Recv() (PendingApprovalEvent, error) { + for { + frame, err := w.stream.Recv() + if err != nil { + w.Close() + if errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled { + return PendingApprovalEvent{}, context.Canceled + } + if errors.Is(err, io.EOF) { + return PendingApprovalEvent{}, pendingApprovalFailure(PendingApprovalIncomplete) + } + return PendingApprovalEvent{}, pendingApprovalWatchFailure(err) + } + if frame.GetPhase() == "gap" || (frame.GetPhase() != "replay" && frame.GetPhase() != "live") { + w.Close() + return PendingApprovalEvent{}, pendingApprovalFailure(PendingApprovalIncomplete) + } + ev := frame.GetEvent() + if ev == nil { + if frame.GetPhase() != "live" || frame.GetCursor() == "" { + w.Close() + return PendingApprovalEvent{}, pendingApprovalFailure(PendingApprovalMalformed) + } + return PendingApprovalEvent{Kind: PendingApprovalEventBoundary}, nil + } + if frame.GetCursor() == "" { + w.Close() + return PendingApprovalEvent{}, pendingApprovalFailure(PendingApprovalMalformed) + } + if ev.GetRunId() == "" { + if ev.GetType() == "session.title" && ev.GetTitle() != nil { + continue + } + w.Close() + return PendingApprovalEvent{}, pendingApprovalFailure(PendingApprovalMalformed) + } + if ev.GetRunId() != w.runID { + continue + } + projected, err := projectPendingApprovalEvent(w.sessionID, frame.GetCursor(), ev) + if err != nil { + w.Close() + return PendingApprovalEvent{}, err + } + return projected, nil + } +} + +func projectPendingApprovalEvent(sessionID, cursor string, ev *mecatlv1.Event) (PendingApprovalEvent, error) { + out := PendingApprovalEvent{Kind: PendingApprovalEventOther} + switch ev.GetType() { + case "permission.ask": + approval, supported, err := approvalFromEvent(sessionID, cursor, ev) + if err != nil { + return PendingApprovalEvent{}, err + } + if !supported { + return PendingApprovalEvent{}, pendingApprovalFailure(PendingApprovalUnsupportedAsk) + } + out.Kind, out.Approval = PendingApprovalEventAsk, &approval + case "approval": + if !validApprovalEvent(ev) { + return PendingApprovalEvent{}, pendingApprovalFailure(PendingApprovalMalformed) + } + out.Kind = PendingApprovalEventResolved + case "permission.retract": + if ev.GetRunId() == "" || ev.GetAsk() == nil || ev.GetAsk().GetAskId() == "" { + return PendingApprovalEvent{}, pendingApprovalFailure(PendingApprovalMalformed) + } + out.Kind = PendingApprovalEventRetracted + case "result": + if ev.GetRunId() == "" || ev.GetResult() == nil || !validResultStop(ev.GetResult().GetStop()) { + return PendingApprovalEvent{}, pendingApprovalFailure(PendingApprovalMalformed) + } + out.Kind = PendingApprovalEventTerminal + } + out.Message = EventToMsg(ev) + return out, nil +} + +func approvalFromEvent(sessionID, cursor string, ev *mecatlv1.Event) (PendingApproval, bool, error) { + ask := ev.GetAsk() + if ev.GetRunId() == "" || ask == nil || ask.GetAskId() == "" || ask.GetTool() == "" || !validToolArgs(ask.GetArgs()) { + return PendingApproval{}, false, pendingApprovalFailure(PendingApprovalMalformed) + } + approval := PendingApproval{ + SessionID: sessionID, + RunID: ev.GetRunId(), + AskID: ask.GetAskId(), + Tool: ask.GetTool(), + Args: ask.GetArgs(), + Reason: ask.GetReason(), + Cursor: cursor, + } + return approval, ask.GetGuardrail() == nil && ask.GetTool() != "PresentPlan", nil +} + +func validToolArgs(args string) bool { + var object map[string]json.RawMessage + return json.Unmarshal([]byte(args), &object) == nil && object != nil +} + +func validResultStop(stop string) bool { + switch stop { + case "end_turn", "max_turns", "max_tool_calls", "max_consecutive_failures", "cancelled", "error", "no_progress", "budget", "timeout", "structured_output", "plan_approved", "plan_iterate": + return true + default: + return false + } +} + +func validApprovalEvent(ev *mecatlv1.Event) bool { + if ev.GetRunId() == "" || ev.GetApproval() == nil || ev.GetApproval().GetAskId() == "" { + return false + } + switch ev.GetApproval().GetVerdict() { + case mecatlv1.ApprovalVerdict_APPROVAL_VERDICT_DENY, + mecatlv1.ApprovalVerdict_APPROVAL_VERDICT_ALLOW_ONCE, + mecatlv1.ApprovalVerdict_APPROVAL_VERDICT_ALLOW_ALWAYS: + default: + return false + } + switch ev.GetApproval().GetOrigin() { + case "permission", "hook_guardrail", "plan": + return true + default: + return false + } +} + +// DiscoverPendingApproval drains one watch from the beginning through its exact +// replay/live boundary, closes it, and returns the sole unresolved ordinary ask. +func (c *Client) DiscoverPendingApproval(ctx context.Context, sessionID string) (PendingApproval, error) { + watch, err := c.openPendingApprovalWatch(ctx, sessionID, "") + if err != nil { + return PendingApproval{}, err + } + defer watch.Close() + + pending := make(map[string]PendingApproval) + unsupported := make(map[string]bool) + for { + frame, recvErr := watch.stream.Recv() + if recvErr != nil { + if errors.Is(recvErr, io.EOF) { + return PendingApproval{}, pendingApprovalFailure(PendingApprovalIncomplete) + } + return PendingApproval{}, pendingApprovalWatchFailure(recvErr) + } + phase := frame.GetPhase() + if phase == "gap" || (phase != "replay" && phase != "live") { + return PendingApproval{}, pendingApprovalFailure(PendingApprovalIncomplete) + } + ev := frame.GetEvent() + if ev == nil { + if phase != "live" { + return PendingApproval{}, pendingApprovalFailure(PendingApprovalMalformed) + } + if frame.GetCursor() == "" { + return PendingApproval{}, pendingApprovalFailure(PendingApprovalMalformed) + } + if len(pending) != 1 { + return PendingApproval{}, pendingApprovalFailure(PendingApprovalNone) + } + for key, approval := range pending { + if unsupported[key] { + return PendingApproval{}, pendingApprovalFailure(PendingApprovalUnsupportedAsk) + } + approval.Cursor = frame.GetCursor() + return approval, nil + } + } + if phase != "replay" || frame.GetCursor() == "" { + return PendingApproval{}, pendingApprovalFailure(PendingApprovalIncomplete) + } + if err := foldPendingApprovalEvent(sessionID, frame.GetCursor(), ev, pending, unsupported); err != nil { + return PendingApproval{}, err + } + } +} + +func pendingApprovalKey(runID, askID string) string { + return runID + "\x00" + askID +} + +func removePendingApproval(key string, pending map[string]PendingApproval, unsupported map[string]bool) error { + if _, exists := pending[key]; !exists { + return pendingApprovalFailure(PendingApprovalMalformed) + } + delete(pending, key) + delete(unsupported, key) + return nil +} + +func clearPendingApprovalRun(runID string, pending map[string]PendingApproval, unsupported map[string]bool) { + prefix := runID + "\x00" + for key := range pending { + if len(key) >= len(prefix) && key[:len(prefix)] == prefix { + delete(pending, key) + delete(unsupported, key) + } + } +} + +func hasPendingApprovalRun(runID string, pending map[string]PendingApproval) bool { + prefix := runID + "\x00" + for key := range pending { + if runID != "" && len(key) >= len(prefix) && key[:len(prefix)] == prefix { + return true + } + } + return false +} + +func foldPendingApprovalEvent(sessionID, cursor string, ev *mecatlv1.Event, pending map[string]PendingApproval, unsupported map[string]bool) error { + switch ev.GetType() { + case "permission.ask": + approval, supported, err := approvalFromEvent(sessionID, cursor, ev) + if err != nil { + return err + } + key := pendingApprovalKey(approval.RunID, approval.AskID) + if _, exists := pending[key]; exists { + return pendingApprovalFailure(PendingApprovalMalformed) + } + pending[key], unsupported[key] = approval, !supported + case "approval": + if !validApprovalEvent(ev) { + return pendingApprovalFailure(PendingApprovalMalformed) + } + return removePendingApproval(pendingApprovalKey(ev.GetRunId(), ev.GetApproval().GetAskId()), pending, unsupported) + case "permission.retract": + if ev.GetRunId() == "" || ev.GetAsk() == nil || ev.GetAsk().GetAskId() == "" { + return pendingApprovalFailure(PendingApprovalMalformed) + } + return removePendingApproval(pendingApprovalKey(ev.GetRunId(), ev.GetAsk().GetAskId()), pending, unsupported) + case "result": + if ev.GetRunId() == "" || ev.GetResult() == nil || !validResultStop(ev.GetResult().GetStop()) { + return pendingApprovalFailure(PendingApprovalMalformed) + } + clearPendingApprovalRun(ev.GetRunId(), pending, unsupported) + default: + if EventToMsg(ev) == nil && hasPendingApprovalRun(ev.GetRunId(), pending) { + return pendingApprovalFailure(PendingApprovalIncomplete) + } + } + return nil +} + +// ResolvePendingApproval submits only Allow Once or Deny and requires the wire +// acknowledgement to echo the exact run and ask. +func (c *Client) ResolvePendingApproval(ctx context.Context, approval PendingApproval, verdict Verdict) error { + if approval.SessionID == "" || approval.RunID == "" || approval.AskID == "" || (verdict != VerdictAllowOnce && verdict != VerdictDeny) { + return pendingApprovalFailure(PendingApprovalMalformed) + } + wireVerdict := mecatlv1.ApprovalVerdict_APPROVAL_VERDICT_ALLOW_ONCE + if verdict == VerdictDeny { + wireVerdict = mecatlv1.ApprovalVerdict_APPROVAL_VERDICT_DENY + } + ack, err := c.svc.ResolveRunAsk(withSessionAffinity(ctx, approval.SessionID), &mecatlv1.ResolveRunAskRequest{ + SessionId: approval.SessionID, ExpectedRunId: approval.RunID, AskId: approval.AskID, Verdict: wireVerdict, + }) + if err != nil { + return pendingApprovalControlFailure(err) + } + if ack.GetRunId() != approval.RunID || ack.GetAskId() != approval.AskID { + return pendingApprovalFailure(PendingApprovalCorrelation) + } + return nil +} + +// CancelPendingRun explicitly cancels only the exact recovered run and checks +// the correlated acknowledgement. +func (c *Client) CancelPendingRun(ctx context.Context, approval PendingApproval) error { + if approval.SessionID == "" || approval.RunID == "" { + return pendingApprovalFailure(PendingApprovalMalformed) + } + ack, err := c.svc.CancelRun(withSessionAffinity(ctx, approval.SessionID), &mecatlv1.CancelRunRequest{SessionId: approval.SessionID, ExpectedRunId: approval.RunID}) + if err != nil { + return pendingApprovalControlFailure(err) + } + if ack.GetRunId() != approval.RunID { + return pendingApprovalFailure(PendingApprovalCorrelation) + } + return nil +} diff --git a/cmd/mecatui/client/pendingapproval_test.go b/cmd/mecatui/client/pendingapproval_test.go new file mode 100644 index 0000000000..b36e92f34e --- /dev/null +++ b/cmd/mecatui/client/pendingapproval_test.go @@ -0,0 +1,469 @@ +package client + +import ( + "context" + "net" + "reflect" + "sync" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" + "github.com/stacklok/mecatl/contracts/sessionaffinity" +) + +type pendingApprovalWireServer struct { + mecatlv1.UnimplementedHarnessServiceServer + mu sync.Mutex + frames []*mecatlv1.WatchSessionEventsResponse + watchErr error + watchReqs []*mecatlv1.WatchSessionEventsRequest + affinities [][]string + resolveReq *mecatlv1.ResolveRunAskRequest + resolveResp *mecatlv1.ResolveRunAskResponse + resolveErr error + resolveAffinity []string + cancelReq *mecatlv1.CancelRunRequest + cancelResp *mecatlv1.CancelRunResponse + cancelErr error + cancelAffinity []string + watchDone chan struct{} +} + +func (s *pendingApprovalWireServer) WatchSessionEvents(req *mecatlv1.WatchSessionEventsRequest, stream grpc.ServerStreamingServer[mecatlv1.WatchSessionEventsResponse]) error { + md, _ := metadata.FromIncomingContext(stream.Context()) + s.mu.Lock() + s.watchReqs = append(s.watchReqs, req) + s.affinities = append(s.affinities, append([]string(nil), md.Get(sessionaffinity.HeaderName)...)) + frames, watchErr, done := s.frames, s.watchErr, s.watchDone + s.mu.Unlock() + if watchErr != nil { + return watchErr + } + for _, frame := range frames { + if err := stream.Send(frame); err != nil { + return err + } + } + if done != nil { + <-stream.Context().Done() + close(done) + } + return nil +} + +func (s *pendingApprovalWireServer) ResolveRunAsk(ctx context.Context, req *mecatlv1.ResolveRunAskRequest) (*mecatlv1.ResolveRunAskResponse, error) { + md, _ := metadata.FromIncomingContext(ctx) + s.mu.Lock() + defer s.mu.Unlock() + s.resolveReq = req + s.resolveAffinity = append([]string(nil), md.Get(sessionaffinity.HeaderName)...) + if s.resolveErr != nil { + return nil, s.resolveErr + } + if s.resolveResp != nil { + return s.resolveResp, nil + } + return &mecatlv1.ResolveRunAskResponse{RunId: req.GetExpectedRunId(), AskId: req.GetAskId()}, nil +} + +func (s *pendingApprovalWireServer) CancelRun(ctx context.Context, req *mecatlv1.CancelRunRequest) (*mecatlv1.CancelRunResponse, error) { + md, _ := metadata.FromIncomingContext(ctx) + s.mu.Lock() + defer s.mu.Unlock() + s.cancelReq = req + s.cancelAffinity = append([]string(nil), md.Get(sessionaffinity.HeaderName)...) + if s.cancelErr != nil { + return nil, s.cancelErr + } + if s.cancelResp != nil { + return s.cancelResp, nil + } + return &mecatlv1.CancelRunResponse{RunId: req.GetExpectedRunId()}, nil +} + +func newPendingApprovalWireClient(t *testing.T, service *pendingApprovalWireServer) *Client { + t.Helper() + listener := bufconn.Listen(1 << 20) + server := grpc.NewServer() + mecatlv1.RegisterHarnessServiceServer(server, service) + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + t.Cleanup(func() { _ = listener.Close() }) + conn, err := grpc.NewClient("passthrough:///pending-approval", grpc.WithTransportCredentials(insecure.NewCredentials()), grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return listener.DialContext(ctx) + })) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + return &Client{conn: conn, svc: mecatlv1.NewHarnessServiceClient(conn)} +} + +func watchFrame(cursor, phase string, event *mecatlv1.Event) *mecatlv1.WatchSessionEventsResponse { + return &mecatlv1.WatchSessionEventsResponse{Cursor: cursor, Phase: phase, Event: event} +} + +func ordinaryAsk(runID, askID string) *mecatlv1.Event { + return &mecatlv1.Event{Type: "permission.ask", RunId: runID, Ask: &mecatlv1.PermissionAsk{AskId: askID, Tool: "Shell", Args: `{"command":"go test ./..."}`, Reason: "protected"}} +} + +func TestDiscoverPendingApprovalAcceptsOnlyTrailingOrdinaryAsk(t *testing.T) { + service := &pendingApprovalWireServer{frames: []*mecatlv1.WatchSessionEventsResponse{ + watchFrame("c1", "replay", ordinaryAsk("old-run", "old-ask")), + watchFrame("c2", "replay", &mecatlv1.Event{Type: "result", RunId: "old-run", Result: &mecatlv1.Result{Stop: "end_turn"}}), + watchFrame("c3", "replay", ordinaryAsk("current-run", "current-ask")), + watchFrame("c3", "live", nil), + }} + candidate, err := newPendingApprovalWireClient(t, service).DiscoverPendingApproval(t.Context(), "session-1") + if err != nil { + t.Fatal(err) + } + want := PendingApproval{SessionID: "session-1", RunID: "current-run", AskID: "current-ask", Tool: "Shell", Args: `{"command":"go test ./..."}`, Reason: "protected", Cursor: "c3"} + if !reflect.DeepEqual(candidate, want) { + t.Fatalf("candidate = %#v, want %#v", candidate, want) + } + service.mu.Lock() + defer service.mu.Unlock() + if got := service.affinities; !reflect.DeepEqual(got, [][]string{{"session-1"}}) { + t.Fatalf("watch affinity = %#v", got) + } +} + +func TestDiscoverPendingApprovalAcceptsRecoveryLifecycleBeforeLaterAsk(t *testing.T) { + approval := &mecatlv1.Event{Type: "approval", RunId: "run-1", Approval: &mecatlv1.Approval{AskId: "ask-1", Verdict: mecatlv1.ApprovalVerdict_APPROVAL_VERDICT_ALLOW_ONCE, Origin: "permission"}} + tests := []struct { + name string + frames []*mecatlv1.WatchSessionEventsResponse + want PendingApproval + }{ + { + name: "later ask in recovered run", + frames: []*mecatlv1.WatchSessionEventsResponse{ + watchFrame("c1", "replay", ordinaryAsk("run-1", "ask-1")), + watchFrame("c2", "replay", &mecatlv1.Event{Type: "session.init", RunId: "run-1"}), + watchFrame("c3", "replay", approval), + watchFrame("c4", "replay", ordinaryAsk("run-1", "ask-2")), + watchFrame("c4", "live", nil), + }, + want: PendingApproval{SessionID: "session-1", RunID: "run-1", AskID: "ask-2", Tool: "Shell", Args: `{"command":"go test ./..."}`, Reason: "protected", Cursor: "c4"}, + }, + { + name: "later run ask after recovered run", + frames: []*mecatlv1.WatchSessionEventsResponse{ + watchFrame("c1", "replay", ordinaryAsk("run-1", "ask-1")), + watchFrame("c2", "replay", &mecatlv1.Event{Type: "session.init", RunId: "run-1"}), + watchFrame("c3", "replay", approval), + watchFrame("c4", "replay", &mecatlv1.Event{Type: "result", RunId: "run-1", Result: &mecatlv1.Result{Stop: "end_turn"}}), + watchFrame("c5", "replay", ordinaryAsk("run-2", "ask-2")), + watchFrame("c5", "live", nil), + }, + want: PendingApproval{SessionID: "session-1", RunID: "run-2", AskID: "ask-2", Tool: "Shell", Args: `{"command":"go test ./..."}`, Reason: "protected", Cursor: "c5"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := newPendingApprovalWireClient(t, &pendingApprovalWireServer{frames: tt.frames}).DiscoverPendingApproval(t.Context(), "session-1") + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, tt.want) { + t.Fatalf("candidate = %#v, want %#v", got, tt.want) + } + }) + } +} + +func TestDiscoverPendingApprovalRefusesUnsafeReplay(t *testing.T) { + tests := []struct { + name string + frames []*mecatlv1.WatchSessionEventsResponse + kind PendingApprovalFailure + }{ + {"gap", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "gap", nil)}, PendingApprovalIncomplete}, + {"unknown phase", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "future", ordinaryAsk("run", "ask"))}, PendingApprovalIncomplete}, + {"empty live cursor", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "replay", ordinaryAsk("run", "ask")), watchFrame("", "live", nil)}, PendingApprovalMalformed}, + {"missing run id", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "replay", ordinaryAsk("", "ask"))}, PendingApprovalMalformed}, + {"missing ask id", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "replay", ordinaryAsk("run", ""))}, PendingApprovalMalformed}, + {"malformed args", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "replay", &mecatlv1.Event{Type: "permission.ask", RunId: "run", Ask: &mecatlv1.PermissionAsk{AskId: "ask", Tool: "Shell", Args: "{"}})}, PendingApprovalMalformed}, + {"unknown event after ask", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "replay", ordinaryAsk("run", "ask")), watchFrame("c2", "replay", &mecatlv1.Event{Type: "future.terminal", RunId: "run"})}, PendingApprovalIncomplete}, + {"unknown result stop", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "replay", ordinaryAsk("run", "ask")), watchFrame("c2", "replay", &mecatlv1.Event{Type: "result", RunId: "run", Result: &mecatlv1.Result{Stop: "future_stop"}})}, PendingApprovalMalformed}, + {"plan ask", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "replay", &mecatlv1.Event{Type: "permission.ask", RunId: "run", Ask: &mecatlv1.PermissionAsk{AskId: "ask", Tool: "PresentPlan", Args: `{}`}}), watchFrame("c1", "live", nil)}, PendingApprovalUnsupportedAsk}, + {"scoped ask", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "replay", &mecatlv1.Event{Type: "permission.ask", RunId: "run", Ask: &mecatlv1.PermissionAsk{AskId: "ask", Tool: "Shell", Args: `{}`, Guardrail: &mecatlv1.GuardrailApprovalScope{}}}), watchFrame("c1", "live", nil)}, PendingApprovalUnsupportedAsk}, + {"resolved ask", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "replay", ordinaryAsk("run", "ask")), watchFrame("c2", "replay", &mecatlv1.Event{Type: "approval", RunId: "run", Approval: &mecatlv1.Approval{AskId: "ask", Verdict: mecatlv1.ApprovalVerdict_APPROVAL_VERDICT_DENY, Origin: "permission"}}), watchFrame("c2", "live", nil)}, PendingApprovalNone}, + {"retracted ask", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "replay", ordinaryAsk("run", "ask")), watchFrame("c2", "replay", &mecatlv1.Event{Type: "permission.retract", RunId: "run", Ask: &mecatlv1.PermissionAsk{AskId: "ask"}}), watchFrame("c2", "live", nil)}, PendingApprovalNone}, + {"terminal run", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c1", "replay", ordinaryAsk("run", "ask")), watchFrame("c2", "replay", &mecatlv1.Event{Type: "result", RunId: "run", Result: &mecatlv1.Result{Stop: "cancelled"}}), watchFrame("c2", "live", nil)}, PendingApprovalNone}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := newPendingApprovalWireClient(t, &pendingApprovalWireServer{frames: tt.frames}).DiscoverPendingApproval(t.Context(), "session-1") + if !IsPendingApprovalFailure(err, tt.kind) { + t.Fatalf("error = %v, want kind %q", err, tt.kind) + } + }) + } +} + +func TestDiscoverPendingApprovalRefusesUnsupportedWatchWithoutFallback(t *testing.T) { + service := &pendingApprovalWireServer{watchErr: status.Error(codes.Unimplemented, "sensitive backend detail")} + _, err := newPendingApprovalWireClient(t, service).DiscoverPendingApproval(t.Context(), "session-1") + if !IsPendingApprovalFailure(err, PendingApprovalWatchUnsupported) || err.Error() != "pending approval recovery requires durable event watch support" { + t.Fatalf("error = %q", err) + } +} + +func TestWatchPendingApprovalRunRequiresCompleteCorrelation(t *testing.T) { + client := newPendingApprovalWireClient(t, &pendingApprovalWireServer{}) + for name, approval := range map[string]PendingApproval{ + "session": {RunID: "run", AskID: "ask", Cursor: "c1"}, + "run": {SessionID: "session", AskID: "ask", Cursor: "c1"}, + "ask": {SessionID: "session", RunID: "run", Cursor: "c1"}, + "cursor": {SessionID: "session", RunID: "run", AskID: "ask"}, + } { + t.Run(name, func(t *testing.T) { + if _, err := client.WatchPendingApprovalRun(t.Context(), approval); !IsPendingApprovalFailure(err, PendingApprovalMalformed) { + t.Fatalf("error = %v", err) + } + }) + } +} + +func TestPendingApprovalContinuationFailsClosed(t *testing.T) { + tests := []struct { + name string + frames []*mecatlv1.WatchSessionEventsResponse + kind PendingApprovalFailure + }{ + {"gap", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c2", "gap", nil)}, PendingApprovalIncomplete}, + {"unknown phase", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c2", "future", ordinaryAsk("run-1", "ask-2"))}, PendingApprovalIncomplete}, + {"unknown stop", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c2", "live", &mecatlv1.Event{Type: "result", RunId: "run-1", Result: &mecatlv1.Result{Stop: "future_stop"}})}, PendingApprovalMalformed}, + {"missing cursor", []*mecatlv1.WatchSessionEventsResponse{watchFrame("", "live", &mecatlv1.Event{Type: "request.manifest", RunId: "run-1"})}, PendingApprovalMalformed}, + {"missing run", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c2", "live", ordinaryAsk("", "ask-2"))}, PendingApprovalMalformed}, + {"missing ask", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c2", "live", ordinaryAsk("run-1", ""))}, PendingApprovalMalformed}, + {"malformed approval", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c2", "live", &mecatlv1.Event{Type: "approval", RunId: "run-1"})}, PendingApprovalMalformed}, + {"malformed retraction", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c2", "live", &mecatlv1.Event{Type: "permission.retract", RunId: "run-1"})}, PendingApprovalMalformed}, + {"missing result", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c2", "live", &mecatlv1.Event{Type: "result", RunId: "run-1"})}, PendingApprovalMalformed}, + {"plan ask", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c2", "live", &mecatlv1.Event{Type: "permission.ask", RunId: "run-1", Ask: &mecatlv1.PermissionAsk{AskId: "ask-2", Tool: "PresentPlan", Args: `{}`}})}, PendingApprovalUnsupportedAsk}, + {"guardrail ask", []*mecatlv1.WatchSessionEventsResponse{watchFrame("c2", "live", &mecatlv1.Event{Type: "permission.ask", RunId: "run-1", Ask: &mecatlv1.PermissionAsk{AskId: "ask-2", Tool: "Shell", Args: `{}`, Guardrail: &mecatlv1.GuardrailApprovalScope{}}})}, PendingApprovalUnsupportedAsk}, + {"premature eof", nil, PendingApprovalIncomplete}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + watch, err := newPendingApprovalWireClient(t, &pendingApprovalWireServer{frames: tt.frames}).WatchPendingApprovalRun(t.Context(), PendingApproval{SessionID: "session-1", RunID: "run-1", AskID: "ask-1", Cursor: "c1"}) + if err != nil { + t.Fatal(err) + } + defer watch.Close() + if _, err = watch.Recv(); !IsPendingApprovalFailure(err, tt.kind) { + t.Fatalf("error = %v, want kind %q", err, tt.kind) + } + }) + } +} + +func TestPendingApprovalContinuationKeepsCanonicalTelemetryProjection(t *testing.T) { + for _, kind := range []string{"request.manifest", "network.attempt", "future.telemetry"} { + t.Run(kind, func(t *testing.T) { + telemetry := &mecatlv1.Event{Type: kind, RunId: "run-1"} + if EventToMsg(telemetry) != nil { + t.Fatal("fixture must exercise canonical nil/skip projection") + } + service := &pendingApprovalWireServer{frames: []*mecatlv1.WatchSessionEventsResponse{ + watchFrame("c2", "live", telemetry), + watchFrame("c3", "live", &mecatlv1.Event{Type: "result", RunId: "run-1", Result: &mecatlv1.Result{Stop: "end_turn"}}), + }} + watch, err := newPendingApprovalWireClient(t, service).WatchPendingApprovalRun(t.Context(), PendingApproval{SessionID: "session-1", RunID: "run-1", AskID: "ask-1", Cursor: "c1"}) + if err != nil { + t.Fatal(err) + } + defer watch.Close() + event, err := watch.Recv() + if err != nil || event.Kind != PendingApprovalEventOther || event.Message != nil { + t.Fatalf("telemetry projection = %+v, %v", event, err) + } + event, err = watch.Recv() + if err != nil || event.Kind != PendingApprovalEventTerminal { + t.Fatalf("terminal projection = %+v, %v", event, err) + } + }) + } +} + +func TestPendingApprovalContinuationMasksStreamError(t *testing.T) { + watch, err := newPendingApprovalWireClient(t, &pendingApprovalWireServer{watchErr: status.Error(codes.Unavailable, "private backend detail")}).WatchPendingApprovalRun(t.Context(), PendingApproval{SessionID: "session-1", RunID: "run-1", AskID: "ask-1", Cursor: "c1"}) + if err != nil { + t.Fatal(err) + } + defer watch.Close() + if _, err = watch.Recv(); !IsPendingApprovalFailure(err, PendingApprovalUnavailable) || err.Error() == "private backend detail" { + t.Fatalf("error = %v", err) + } +} + +func TestPendingApprovalContinuationFiltersForeignRuns(t *testing.T) { + service := &pendingApprovalWireServer{frames: []*mecatlv1.WatchSessionEventsResponse{ + watchFrame("c2", "live", ordinaryAsk("other-run", "other-ask")), + watchFrame("c3", "live", ordinaryAsk("run-1", "ask-2")), + }} + watch, err := newPendingApprovalWireClient(t, service).WatchPendingApprovalRun(t.Context(), PendingApproval{SessionID: "session-1", RunID: "run-1", AskID: "ask-1", Cursor: "c1"}) + if err != nil { + t.Fatal(err) + } + defer watch.Close() + event, err := watch.Recv() + if err != nil || event.Approval == nil || event.Approval.AskID != "ask-2" { + t.Fatalf("event = %#v, err = %v", event, err) + } +} + +func TestPendingApprovalContinuationSkipsRunlessSessionTitle(t *testing.T) { + service := &pendingApprovalWireServer{frames: []*mecatlv1.WatchSessionEventsResponse{ + watchFrame("c2", "replay", &mecatlv1.Event{Type: "session.title", Title: &mecatlv1.SessionTitle{Title: "Recovered session"}}), + watchFrame("c2", "live", nil), + watchFrame("c3", "live", ordinaryAsk("other-run", "other-ask")), + watchFrame("c4", "live", ordinaryAsk("run-1", "ask-2")), + }} + watch, err := newPendingApprovalWireClient(t, service).WatchPendingApprovalRun(t.Context(), PendingApproval{SessionID: "session-1", RunID: "run-1", AskID: "ask-1", Cursor: "c1"}) + if err != nil { + t.Fatal(err) + } + defer watch.Close() + boundary, err := watch.Recv() + if err != nil || boundary.Kind != PendingApprovalEventBoundary { + t.Fatalf("boundary after runless title = %#v, err = %v", boundary, err) + } + event, err := watch.Recv() + if err != nil || event.Kind != PendingApprovalEventAsk || event.Approval == nil || event.Approval.RunID != "run-1" || event.Approval.AskID != "ask-2" { + t.Fatalf("recovered-run event = %#v, err = %v", event, err) + } +} + +func TestPendingApprovalContinuationUsesRetainedCursorAndCloses(t *testing.T) { + done := make(chan struct{}) + service := &pendingApprovalWireServer{frames: []*mecatlv1.WatchSessionEventsResponse{watchFrame("c4", "live", ordinaryAsk("run-1", "ask-2"))}, watchDone: done} + client := newPendingApprovalWireClient(t, service) + watch, err := client.WatchPendingApprovalRun(t.Context(), PendingApproval{SessionID: "session-1", RunID: "run-1", AskID: "ask-1", Cursor: "c3"}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(watch.Close) + event, err := watch.Recv() + askMsg, projected := event.Message.(PermissionAskMsg) + if err != nil || event.Kind != PendingApprovalEventAsk || event.Approval == nil || event.Approval.AskID != "ask-2" || !projected || askMsg.AskID != "ask-2" { + t.Fatalf("event = %#v, err = %v", event, err) + } + watch.Close() + waitCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + select { + case <-done: + case <-waitCtx.Done(): + t.Fatalf("closing the watch did not release its server read loop: %v", waitCtx.Err()) + } + service.mu.Lock() + defer service.mu.Unlock() + if len(service.watchReqs) != 1 || service.watchReqs[0].GetCursor() != "c3" || service.watchReqs[0].GetRunId() != "" { + t.Fatalf("watch request = %#v; retained unfiltered cursor must keep its wire scope", service.watchReqs) + } +} + +func TestResolvePendingApprovalChecksAckAndPayload(t *testing.T) { + service := &pendingApprovalWireServer{} + client := newPendingApprovalWireClient(t, service) + candidate := PendingApproval{SessionID: "session-1", RunID: "run-1", AskID: "ask-1"} + if err := client.ResolvePendingApproval(t.Context(), candidate, VerdictDeny); err != nil { + t.Fatal(err) + } + service.mu.Lock() + req := service.resolveReq + affinity := append([]string(nil), service.resolveAffinity...) + service.resolveResp = &mecatlv1.ResolveRunAskResponse{RunId: "other-run", AskId: "ask-1"} + service.mu.Unlock() + if req.GetSessionId() != "session-1" || req.GetExpectedRunId() != "run-1" || req.GetAskId() != "ask-1" || req.GetVerdict() != mecatlv1.ApprovalVerdict_APPROVAL_VERDICT_DENY { + t.Fatalf("resolve request = %#v", req) + } + if !reflect.DeepEqual(affinity, []string{"session-1"}) { + t.Fatalf("resolve affinity = %#v", affinity) + } + if err := client.ResolvePendingApproval(t.Context(), candidate, VerdictAllowOnce); !IsPendingApprovalFailure(err, PendingApprovalCorrelation) { + t.Fatalf("ack mismatch error = %v", err) + } +} + +func TestPendingApprovalControlsClassifyClosedRPCFailures(t *testing.T) { + tests := []struct { + name string + code codes.Code + kind PendingApprovalFailure + }{ + {"stale run", codes.Aborted, PendingApprovalCorrelation}, + {"state changed", codes.FailedPrecondition, PendingApprovalNotRecoverable}, + {"unsupported", codes.Unimplemented, PendingApprovalControlsUnsupported}, + {"transport", codes.Unavailable, PendingApprovalUnavailable}, + {"deadline", codes.DeadlineExceeded, PendingApprovalUnavailable}, + {"masked absence", codes.NotFound, PendingApprovalUnavailable}, + {"masked denial", codes.PermissionDenied, PendingApprovalUnavailable}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + serverErr := status.Error(tt.code, "private backend detail") + service := &pendingApprovalWireServer{resolveErr: serverErr, cancelErr: serverErr} + client := newPendingApprovalWireClient(t, service) + approval := PendingApproval{SessionID: "session-1", RunID: "run-1", AskID: "ask-1"} + for name, err := range map[string]error{ + "resolve": client.ResolvePendingApproval(t.Context(), approval, VerdictDeny), + "cancel": client.CancelPendingRun(t.Context(), approval), + } { + if !IsPendingApprovalFailure(err, tt.kind) || err.Error() == "private backend detail" { + t.Fatalf("%s error = %v, want kind %q", name, err, tt.kind) + } + } + }) + } +} + +func TestPendingApprovalControlsRejectMalformedInputsBeforeRPC(t *testing.T) { + service := &pendingApprovalWireServer{} + client := newPendingApprovalWireClient(t, service) + if err := client.ResolvePendingApproval(t.Context(), PendingApproval{SessionID: "session", RunID: "run"}, VerdictDeny); !IsPendingApprovalFailure(err, PendingApprovalMalformed) { + t.Fatalf("resolve error = %v", err) + } + if err := client.CancelPendingRun(t.Context(), PendingApproval{SessionID: "session"}); !IsPendingApprovalFailure(err, PendingApprovalMalformed) { + t.Fatalf("cancel error = %v", err) + } + service.mu.Lock() + defer service.mu.Unlock() + if service.resolveReq != nil || service.cancelReq != nil { + t.Fatalf("malformed controls reached RPC: resolve=%#v cancel=%#v", service.resolveReq, service.cancelReq) + } +} + +func TestCancelPendingRunChecksAckPayloadAndAffinity(t *testing.T) { + service := &pendingApprovalWireServer{} + client := newPendingApprovalWireClient(t, service) + candidate := PendingApproval{SessionID: "session-1", RunID: "run-1"} + if err := client.CancelPendingRun(t.Context(), candidate); err != nil { + t.Fatal(err) + } + service.mu.Lock() + req := service.cancelReq + affinity := append([]string(nil), service.cancelAffinity...) + service.cancelResp = &mecatlv1.CancelRunResponse{RunId: "other-run"} + service.mu.Unlock() + if req.GetSessionId() != "session-1" || req.GetExpectedRunId() != "run-1" { + t.Fatalf("cancel request = %#v", req) + } + if !reflect.DeepEqual(affinity, []string{"session-1"}) { + t.Fatalf("cancel affinity = %#v", affinity) + } + if err := client.CancelPendingRun(t.Context(), candidate); !IsPendingApprovalFailure(err, PendingApprovalCorrelation) { + t.Fatalf("ack mismatch error = %v", err) + } +} diff --git a/cmd/mecatui/client/transcript.go b/cmd/mecatui/client/transcript.go index 0217caf845..4cc08446f6 100644 --- a/cmd/mecatui/client/transcript.go +++ b/cmd/mecatui/client/transcript.go @@ -34,6 +34,7 @@ type ResumeSelection struct { Row SessionListItem Transcript SessionTranscript Snapshot SessionSnapshot + Pending *PendingApproval } // GetSessionTranscript fetches the authoritative snapshot-derived transcript. diff --git a/cmd/mecatui/config.go b/cmd/mecatui/config.go index 0a38465344..a890d69554 100644 --- a/cmd/mecatui/config.go +++ b/cmd/mecatui/config.go @@ -387,11 +387,11 @@ func parseTransportFlags(mode transportMode, out io.Writer, args []string, brows cfg.debugMCP = append(cfg.debugMCP, value) return nil }) - fs.StringVar(&cfg.resumeID, "resume", "", "continue the main chat with SESSION_ID (conflicts with --resume-latest)") - fs.BoolVar(&cfg.resumeLatest, "resume-latest", false, "continue the most recent resumable main chat, or start a new chat if none is available (conflicts with --resume)") - fs.StringVar(&cfg.prompt, "prompt", "", "submit TEXT when the session is ready; the TUI remains open for follow-ups") + fs.StringVar(&cfg.resumeID, "resume", "", "continue a stored main chat or supported ordinary pending approval with SESSION_ID (conflicts with --resume-latest)") + fs.BoolVar(&cfg.resumeLatest, "resume-latest", false, "continue the most recent resumable main chat, excluding pending approvals, or start a new chat if none is available (conflicts with --resume)") + fs.StringVar(&cfg.prompt, "prompt", "", "submit TEXT when the session is ready; kept as an unsent draft during pending-approval recovery; the TUI remains open for follow-ups") fs.StringVar(&cfg.prompt, "p", "", "short form of --prompt") - fs.StringVar(&cfg.promptFile, "prompt-file", "", "submit the contents of FILE when the session is ready; appended after --prompt when both are set") + fs.StringVar(&cfg.promptFile, "prompt-file", "", "submit the contents of FILE when the session is ready; appended after --prompt; kept as an unsent draft during pending-approval recovery") fs.StringVar(&cfg.theme, "theme", "", "theme name (default: aztec)") fs.StringVar(&cfg.themeDir, "theme-dir", "", "extra directory of *.json themes to load") fs.StringVar(&cfg.authToken, "auth-token", "", "bearer token for an external server (or MECATL_AUTH_TOKEN)") diff --git a/cmd/mecatui/main.go b/cmd/mecatui/main.go index 0a7e6af25b..625504cae2 100644 --- a/cmd/mecatui/main.go +++ b/cmd/mecatui/main.go @@ -169,6 +169,12 @@ type restartTransport struct { TLSCAFile string } +type pendingApprovalClient struct{ *client.Client } + +func (c pendingApprovalClient) WatchPendingApprovalRun(ctx context.Context, approval client.PendingApproval) (ui.PendingApprovalWatch, error) { + return c.Client.WatchPendingApprovalRun(ctx, approval) +} + type runOptions struct { connectOpen bool connectError string @@ -282,15 +288,15 @@ func runWithOptions(argv []string, options runOptions) error { return err } - resumeCfg := cfg + var resume *client.ResumeSelection + uiWorkspace := cfg.workspace if options.connectResumeSessionID != "" { // This candidate originated from an interrupted auth stream, not an - // explicit --resume. GetSession/transcript remain the server's ownership - // proof; only a terminal turn boundary can be adopted automatically. - resumeCfg.resumeID = options.connectResumeSessionID - resumeCfg.resumeLatest = false + // explicit --resume. It never opens pending-approval recovery. + resume, err = loadExactStartupResume(ctx, cl, options.connectResumeSessionID, false) + } else { + resume, uiWorkspace, err = startupResumeConfig(ctx, cl, cfg) } - resume, uiWorkspace, err := startupResumeConfig(ctx, cl, resumeCfg) if options.connectResumeSessionID != "" { // An auth-recovery candidate is opportunistic. Only a verified terminal // boundary is adopted; every other state and every ambiguous verification @@ -351,6 +357,7 @@ func runWithOptions(argv []string, options runOptions) error { Transcript: cl, Replayer: cl, LiveStream: cl, + PendingApprovals: pendingApprovalClient{cl}, SelectionStore: store, Learning: learningSettingsForConfig(cfg), Connect: savedConnectController{}, diff --git a/cmd/mecatui/startup_resume.go b/cmd/mecatui/startup_resume.go index 1230fd252a..a56690ae3f 100644 --- a/cmd/mecatui/startup_resume.go +++ b/cmd/mecatui/startup_resume.go @@ -17,6 +17,10 @@ type startupResumeSource interface { client.SessionGetter } +type pendingApprovalRecoverySource interface { + DiscoverPendingApproval(context.Context, string) (client.PendingApproval, error) +} + const startupReasonNotFound = client.CapabilityReasonUnknown type startupResumeError struct { @@ -58,7 +62,7 @@ func resolveStartupResume(ctx context.Context, source startupResumeSource, exact return nil, nil } if exactID != "" { - return loadExactStartupResume(ctx, source, exactID) + return loadExactStartupResume(ctx, source, exactID, true) } rows, err := source.ListSessions(ctx) if err != nil { @@ -92,7 +96,7 @@ func resolveStartupResume(ctx context.Context, source startupResumeSource, exact return nil, &startupResumeError{Reason: startupReasonNotFound, noEligibleChat: true, text: "no eligible resumable chat was found; use --resume with an exact ID or start a new chat"} } -func loadExactStartupResume(ctx context.Context, source startupResumeSource, id string) (*client.ResumeSelection, error) { +func loadExactStartupResume(ctx context.Context, source startupResumeSource, id string, recoverPending bool) (*client.ResumeSelection, error) { snapshot, err := source.GetSession(ctx, id) if err != nil { reason := client.CapabilityReasonUnknown @@ -124,6 +128,22 @@ func loadExactStartupResume(ctx context.Context, source startupResumeSource, id row.ReasonCode = client.CapabilityReasonInspectOnlyKind } else if snapshot.State == "awaiting" { row.ReasonCode = client.CapabilityReasonAwaitingApproval + if !recoverPending { + return nil, &startupResumeError{Reason: row.ReasonCode, text: capabilityStartupGuidance(row.ReasonCode)} + } + recovery, ok := source.(pendingApprovalRecoverySource) + if !ok { + return nil, pendingApprovalStartupError(nil) + } + pending, discoverErr := recovery.DiscoverPendingApproval(ctx, id) + if discoverErr != nil { + return nil, pendingApprovalStartupError(discoverErr) + } + current, currentErr := source.GetSession(ctx, id) + if currentErr != nil || current.State != "awaiting" { + return nil, pendingApprovalStartupError(currentErr) + } + return &client.ResumeSelection{Row: row, Transcript: transcript, Snapshot: current, Pending: &pending}, nil } if !startupResumeEligible(row, false) { return nil, &startupResumeError{Reason: row.ReasonCode, text: capabilityStartupGuidance(row.ReasonCode)} @@ -131,6 +151,16 @@ func loadExactStartupResume(ctx context.Context, source startupResumeSource, id return &client.ResumeSelection{Row: row, Transcript: transcript, Snapshot: snapshot}, nil } +func pendingApprovalStartupError(err error) error { + text := "that chat is awaiting approval, but recovery could not be verified; retry or leave the approval pending" + if client.IsPendingApprovalFailure(err, client.PendingApprovalWatchUnsupported) { + text = "that chat is awaiting approval; this server does not support durable pending-approval recovery" + } else if client.IsPendingApprovalFailure(err, client.PendingApprovalUnsupportedAsk) { + text = "that chat requires a dedicated plan or guardrail approval flow; ordinary approval recovery cannot resolve it" + } + return &startupResumeError{Reason: client.CapabilityReasonAwaitingApproval, text: text, cause: err} +} + func startupResumeEligible(row client.SessionListItem, latest bool) bool { if row.Kind != client.SessionKindMain || !row.Capabilities.PublicChat { return false diff --git a/cmd/mecatui/startup_resume_e2e_test.go b/cmd/mecatui/startup_resume_e2e_test.go index 66f84449e4..74bb45040b 100644 --- a/cmd/mecatui/startup_resume_e2e_test.go +++ b/cmd/mecatui/startup_resume_e2e_test.go @@ -5,7 +5,10 @@ import ( "errors" "io" "net" + "os" + "path/filepath" "strings" + "sync" "testing" "time" @@ -18,6 +21,8 @@ import ( "github.com/stacklok/mecatl/cmd/mecatui/theme" "github.com/stacklok/mecatl/cmd/mecatui/ui" mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/session" "github.com/stacklok/mecatl/internal/adapter/server" "github.com/stacklok/mecatl/internal/app" ) @@ -462,3 +467,226 @@ func seedStartupResumeSession(ctx context.Context, t *testing.T, target, _ strin } } } + +func TestPendingApprovalStartupRecovery(t *testing.T) { + for _, tc := range []struct { + name string + verdict client.Verdict + wantRun bool + finalTurn mockllm.Turn + chained bool + }{ + {name: "allow-once", verdict: client.VerdictAllowOnce, wantRun: true, finalTurn: mockllm.TextTurn("continuation complete")}, + {name: "allow-recovery-history", verdict: client.VerdictAllowOnce, wantRun: true, chained: true}, + {name: "allow-empty-model", verdict: client.VerdictAllowOnce, wantRun: true, finalTurn: mockllm.EmptyTurn()}, + {name: "allow-error-model", verdict: client.VerdictAllowOnce, wantRun: true, finalTurn: mockllm.ErrorTurn(errors.New("bounded provider failure"))}, + {name: "deny", verdict: client.VerdictDeny, finalTurn: mockllm.TextTurn("must not be called")}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + workspace, storeDir := t.TempDir(), t.TempDir() + settings := filepath.Join(t.TempDir(), "operator.yaml") + if err := os.WriteFile(settings, []byte("permissions:\n ask: [Write]\n"), 0o600); err != nil { + t.Fatalf("write permissions: %v", err) + } + turns := []mockllm.Turn{mockllm.ToolCallTurn(session.NewToolCall("write-1", "Write", []byte(`{"path":"approved.txt","content":"executed"}`)))} + if tc.chained { + turns = append(turns, + mockllm.ToolCallTurn(session.NewToolCall("write-2", "Write", []byte(`{"path":"second.txt","content":"executed"}`))), + mockllm.TextTurn("continuation complete"), + mockllm.ToolCallTurn(session.NewToolCall("write-3", "Write", []byte(`{"path":"third.txt","content":"pending"}`))), + ) + } else { + turns = append(turns, tc.finalTurn) + } + provider := mockllm.New(turns...) + cfg := app.Config{ + Workspace: workspace, StoreDir: storeDir, Model: "mock-model", MockProvider: provider, + Shell: "/bin/sh", Compaction: "heuristic", Tokenizer: "heuristic", PermissionConfigs: []string{settings}, + } + + target, _, cleanupFirst := startPendingRecoveryServer(t, cfg) + conn, err := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatalf("dial first server: %v", err) + } + t.Cleanup(func() { _ = conn.Close() }) + svc := mecatlv1.NewHarnessServiceClient(conn) + created, err := svc.CreateSession(ctx, &mecatlv1.CreateSessionRequest{}) + if err != nil { + t.Fatalf("create session: %v", err) + } + stream, err := svc.Converse(ctx) + if err != nil { + t.Fatalf("open converse: %v", err) + } + if err := stream.Send(&mecatlv1.ConverseRequest{Kind: &mecatlv1.ConverseRequest_Prompt{Prompt: &mecatlv1.Prompt{SessionId: created.GetSessionId(), Text: "write the file"}}}); err != nil { + t.Fatalf("send prompt: %v", err) + } + var originalRun, originalAsk string + for originalAsk == "" { + response, recvErr := stream.Recv() + if recvErr != nil { + t.Fatalf("receive initial ask: %v", recvErr) + } + event := response.GetEvent() + if event.GetType() == "permission.ask" { + originalRun, originalAsk = event.GetRunId(), event.GetAsk().GetAskId() + } + } + probe, err := client.Dial(client.DialConfig{Server: target}) + if err != nil { + t.Fatalf("dial persistence probe: %v", err) + } + t.Cleanup(func() { _ = probe.Close() }) + persisted, err := probe.DiscoverPendingApproval(ctx, created.GetSessionId()) + if err != nil || persisted.RunID != originalRun || persisted.AskID != originalAsk { + t.Fatalf("persist initial ask before restart: candidate=%+v err=%v", persisted, err) + } + persistenceWatch, err := probe.WatchPendingApprovalRun(ctx, persisted) + if err != nil { + t.Fatalf("open pre-restart persistence barrier: %v", err) + } + boundary, err := persistenceWatch.Recv() + persistenceWatch.Close() + if err != nil || boundary.Kind != client.PendingApprovalEventBoundary { + t.Fatalf("pre-restart persistence barrier: event=%+v err=%v", boundary, err) + } + _ = probe.Close() + + cleanupFirst() + _ = conn.Close() + + target, _, cleanupSecond := startPendingRecoveryServer(t, cfg) + defer cleanupSecond() + cl, err := client.Dial(client.DialConfig{Server: target}) + if err != nil { + t.Fatalf("dial restarted server: %v", err) + } + defer func() { _ = cl.Close() }() + + selection, err := resolveStartupResume(ctx, cl, created.GetSessionId(), false) + if err != nil { + t.Fatalf("exact pending resume: %v (cause: %v)", err, errors.Unwrap(err)) + } + if selection.Pending == nil || selection.Pending.RunID != originalRun || selection.Pending.AskID != originalAsk { + t.Fatalf("recovered correlation = %+v, want run %q ask %q", selection.Pending, originalRun, originalAsk) + } + if _, err := os.Stat(filepath.Join(workspace, "approved.txt")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("tool ran before owner verdict: %v", err) + } + + watch, err := cl.WatchPendingApprovalRun(ctx, *selection.Pending) + if err != nil { + t.Fatalf("open exact continuation watch: %v", err) + } + defer watch.Close() + boundary, err = watch.Recv() + if err != nil || boundary.Kind != client.PendingApprovalEventBoundary { + t.Fatalf("establish replay/live boundary: event=%+v err=%v", boundary, err) + } + if err := cl.ResolvePendingApproval(ctx, *selection.Pending, tc.verdict); err != nil { + t.Fatalf("resolve pending approval: %v", err) + } + var sawToolResult bool + for { + event, recvErr := watch.Recv() + if recvErr != nil { + t.Fatalf("watch exact continuation: %v", recvErr) + } + if _, ok := event.Message.(client.ToolResultMsg); ok { + sawToolResult = true + } + if tc.chained && event.Kind == client.PendingApprovalEventAsk { + later, discoverErr := cl.DiscoverPendingApproval(ctx, created.GetSessionId()) + if discoverErr != nil { + t.Fatalf("rediscover later same-run ask: %v", discoverErr) + } + if later.RunID != originalRun || later.AskID == originalAsk { + t.Fatalf("later same-run correlation = %+v, original run=%q ask=%q", later, originalRun, originalAsk) + } + if resolveErr := cl.ResolvePendingApproval(ctx, later, client.VerdictAllowOnce); resolveErr != nil { + t.Fatalf("resolve later same-run ask: %v", resolveErr) + } + } + if event.Kind == client.PendingApprovalEventTerminal { + break + } + } + _, statErr := os.Stat(filepath.Join(workspace, "approved.txt")) + if tc.wantRun { + if statErr != nil || !sawToolResult { + t.Fatalf("allow-once did not continue original tool: stat=%v tool-result=%v", statErr, sawToolResult) + } + } else if !errors.Is(statErr, os.ErrNotExist) { + t.Fatalf("deny executed original tool: stat=%v", statErr) + } + if tc.chained { + followConn, dialErr := grpc.NewClient(target, grpc.WithTransportCredentials(insecure.NewCredentials())) + if dialErr != nil { + t.Fatalf("dial later run: %v", dialErr) + } + defer func() { _ = followConn.Close() }() + follow, converseErr := mecatlv1.NewHarnessServiceClient(followConn).Converse(ctx) + if converseErr != nil { + t.Fatalf("open later run: %v", converseErr) + } + if sendErr := follow.Send(&mecatlv1.ConverseRequest{Kind: &mecatlv1.ConverseRequest_Prompt{Prompt: &mecatlv1.Prompt{SessionId: created.GetSessionId(), Text: "write another file"}}}); sendErr != nil { + t.Fatalf("send later prompt: %v", sendErr) + } + var laterRun string + for laterRun == "" { + response, recvErr := follow.Recv() + if recvErr != nil { + t.Fatalf("receive later-run ask: %v", recvErr) + } + if event := response.GetEvent(); event.GetType() == "permission.ask" { + laterRun = event.GetRunId() + } + } + rediscovered, discoverErr := cl.DiscoverPendingApproval(ctx, created.GetSessionId()) + if discoverErr != nil { + t.Fatalf("rediscover later-run ask: %v", discoverErr) + } + if rediscovered.RunID != laterRun || rediscovered.RunID == originalRun { + t.Fatalf("later-run correlation = %+v, live run=%q original=%q", rediscovered, laterRun, originalRun) + } + } + }) + } +} + +func startPendingRecoveryServer(t *testing.T, cfg app.Config) (string, *app.Built, func()) { + t.Helper() + built, err := buildIsolated(t, t.Context(), cfg) + if err != nil { + t.Fatalf("build pending recovery server: %v", err) + } + var ( + cleanupOnce sync.Once + lis net.Listener + grpcServer *grpc.Server + ) + cleanup := func() { + cleanupOnce.Do(func() { + built.Close() + if grpcServer != nil { + grpcServer.Stop() + } + if lis != nil { + _ = lis.Close() + } + }) + } + t.Cleanup(cleanup) + lis, err = net.Listen("tcp", "127.0.0.1:0") + if err != nil { + cleanup() + t.Fatalf("listen pending recovery server: %v", err) + } + grpcServer = grpc.NewServer() + mecatlv1.RegisterHarnessServiceServer(grpcServer, server.NewHarnessServer(built.Service)) + go func() { _ = grpcServer.Serve(lis) }() + return lis.Addr().String(), built, cleanup +} diff --git a/cmd/mecatui/startup_resume_test.go b/cmd/mecatui/startup_resume_test.go index a10cdc13d9..8c7b410828 100644 --- a/cmd/mecatui/startup_resume_test.go +++ b/cmd/mecatui/startup_resume_test.go @@ -74,6 +74,65 @@ func TestResumeLatestLoaderPreservesStatusSnapshot(t *testing.T) { } } +type fakePendingStartupResumeSource struct { + *fakeStartupResumeSource + pending client.PendingApproval + discoverErr error + afterDiscover *client.SessionSnapshot + discoverCalls int +} + +func (f *fakePendingStartupResumeSource) DiscoverPendingApproval(context.Context, string) (client.PendingApproval, error) { + f.discoverCalls++ + if f.afterDiscover != nil { + f.snapshots[f.pending.SessionID] = *f.afterDiscover + } + return f.pending, f.discoverErr +} + +func TestNativePendingApprovalStartupSelection(t *testing.T) { + base := func() *fakePendingStartupResumeSource { + return &fakePendingStartupResumeSource{ + fakeStartupResumeSource: &fakeStartupResumeSource{ + transcripts: map[string]client.SessionTranscript{"s": {SessionID: "s", Complete: true, Kind: client.SessionKindMain}}, + snapshots: map[string]client.SessionSnapshot{"s": {State: "awaiting"}}, + }, + pending: client.PendingApproval{SessionID: "s", RunID: "run", AskID: "ask", Tool: "Write", Args: `{}`}, + } + } + + t.Run("exact awaiting ordinary ask", func(t *testing.T) { + source := base() + selection, err := resolveStartupResume(t.Context(), source, "s", false) + if err != nil || selection.Pending == nil || selection.Pending.RunID != "run" { + t.Fatalf("selection = %+v, %v", selection, err) + } + }) + t.Run("foreign snapshot denied", func(t *testing.T) { + source := base() + source.snapshotErrs = map[string]error{"s": errors.New("foreign owner at /private/path")} + selection, err := resolveStartupResume(t.Context(), source, "s", false) + if err == nil || selection != nil || source.discoverCalls != 0 || strings.Contains(err.Error(), "foreign owner") || strings.Contains(err.Error(), "/private/path") { + t.Fatalf("foreign selection = %+v, calls=%d, err=%v", selection, source.discoverCalls, err) + } + }) + t.Run("current state changed", func(t *testing.T) { + source := base() + changed := client.SessionSnapshot{State: "completed"} + source.afterDiscover = &changed + if selection, err := resolveStartupResume(t.Context(), source, "s", false); err == nil || selection != nil { + t.Fatalf("changed state selection = %+v, %v", selection, err) + } + }) + t.Run("watch unsupported", func(t *testing.T) { + source := base() + source.discoverErr = errors.New("unsupported") + if selection, err := resolveStartupResume(t.Context(), source, "s", false); err == nil || selection != nil || strings.Contains(err.Error(), "unsupported") { + t.Fatalf("unsupported selection = %+v, %v", selection, err) + } + }) +} + // TestSessionContinuityUX_Scenario6_FlagGrammar pins startup resume flag parsing. func TestSessionContinuityUX_Scenario6_FlagGrammar(t *testing.T) { for _, mode := range []transportMode{modeLocal, modeConnect} { diff --git a/cmd/mecatui/ui/approval.go b/cmd/mecatui/ui/approval.go index 8e180782ca..e059ff33fe 100644 --- a/cmd/mecatui/ui/approval.go +++ b/cmd/mecatui/ui/approval.go @@ -52,6 +52,9 @@ func approvalSurfaceFor(m *Model) *approvalSurface { // current stream before returning the command, preserving the existing resolved // correlation and nil-stream behavior without exposing transport to the surface. func (m *Model) approvalSendCmd(askID string, verdict client.Verdict, guardrail *client.GuardrailApprovalScope, expectedRunID string) tea.Cmd { + if m.pendingRecovery != nil && askID == m.pendingRecovery.approval.AskID { + return m.resolvePendingApprovalCmd(verdict) + } if authorizationStream := m.authorization.controlStream; authorizationStream != nil && m.authorization.runningControlGen == m.authorization.controlGen { authorizationStream.MarkApprovalResolved(askID) return func() tea.Msg { diff --git a/cmd/mecatui/ui/approval_surface.go b/cmd/mecatui/ui/approval_surface.go index ac7b602d8e..6bacd0859f 100644 --- a/cmd/mecatui/ui/approval_surface.go +++ b/cmd/mecatui/ui/approval_surface.go @@ -268,7 +268,7 @@ func (s *approvalSurface) applyPermissionAsk(msg client.PermissionAskMsg, open b if s.known(msg.AskID) { return false } - offerAlways := !isChildAsk(msg.AskID, s.sessionID) && !s.isDebugMCPMutationAsk(msg) + offerAlways := !msg.Recovery && !isChildAsk(msg.AskID, s.sessionID) && !s.isDebugMCPMutationAsk(msg) if msg.Guardrail != nil { offerAlways = msg.Guardrail.Kind == "action" && msg.Guardrail.RepeatAvailable } diff --git a/cmd/mecatui/ui/conversation.go b/cmd/mecatui/ui/conversation.go index 3a593c9b57..f881814f20 100644 --- a/cmd/mecatui/ui/conversation.go +++ b/cmd/mecatui/ui/conversation.go @@ -348,6 +348,14 @@ func (c *conversation) addTool(id, name, args string) { c.scrollback.Tools().Add(scrollback.ToolCall{ID: id, Name: name, Arguments: args}) } +// reconcileUnresolvedTool updates the snapshot card for a replayed call. Recovery +// scopes callers to unresolved calls from the adopted transcript, so reused IDs in +// later turns still append normally. +func (c *conversation) reconcileUnresolvedTool(id, name, args string) bool { + return c.scrollback.Tools().ReconcileUnresolved(scrollback.ToolCall{ID: id, Name: name, Arguments: args}) +} + +// resolveTool marks the tool block matching callID as resolved with its result. func (c *conversation) resolveTool(callID, body string, isErr bool, blocks ...client.ContentBlock) bool { return c.scrollback.Tools().Resolve(callID, scrollback.ToolResult{Body: body, IsError: isErr, Artifacts: artifacts(blocks)}) } diff --git a/cmd/mecatui/ui/internal/scrollback/tool.go b/cmd/mecatui/ui/internal/scrollback/tool.go index 7d7f48d3a8..28b7b191dc 100644 --- a/cmd/mecatui/ui/internal/scrollback/tool.go +++ b/cmd/mecatui/ui/internal/scrollback/tool.go @@ -61,6 +61,26 @@ func (t ToolCards) Add(call ToolCall) BlockID { return id } +// ReconcileUnresolved updates the pending card indexed by call.ID. It returns +// false for an unknown, resolved, or non-tool card. +func (t ToolCards) ReconcileUnresolved(call ToolCall) bool { + c := t.conversation + i, ok := c.call(call.ID) + if !ok { + return false + } + payload, ok := c.cards[i].payload.(ToolCardSnapshot) + if !ok || payload.Resolved { + return false + } + updated := cloneCall(call) + if reflect.DeepEqual(payload.Call, updated) { + return true + } + payload.Call = updated + return c.replace(i, payload) +} + // Resolve records result as the terminal result for the call indexed by callID. // It returns false for an unknown call or a conflicting replay. An identical // replay succeeds without changing the card or its revision. diff --git a/cmd/mecatui/ui/model.go b/cmd/mecatui/ui/model.go index 912f4f5bd8..b4635416ab 100644 --- a/cmd/mecatui/ui/model.go +++ b/cmd/mecatui/ui/model.go @@ -19,6 +19,7 @@ import ( customization "github.com/stacklok/mecatl/cmd/mecatui/customization" "github.com/stacklok/mecatl/cmd/mecatui/internal/terminaltext" "github.com/stacklok/mecatl/cmd/mecatui/theme" + "github.com/stacklok/mecatl/cmd/mecatui/ui/internal/scrollback" "github.com/stacklok/mecatl/cmd/mecatui/ui/platform" "github.com/stacklok/mecatl/cmd/mecatui/ui/prompttextarea" "github.com/stacklok/mecatl/cmd/mecatui/ui/welcome" @@ -184,6 +185,9 @@ type Deps struct { // tests. nil disables the live bridge (the ui still renders deliveries via the // replay on a session switch/reload, just not live). *Client satisfies it. LiveStream client.LiveStreamer + // PendingApprovals owns exact-run watch and control operations for an exact + // startup recovery. It is unused for ordinary continuation. + PendingApprovals PendingApprovalController // MCPAuthorization is the distinct browser authorization surface. It never // shares the permission-approval stream or controls. MCPAuthorization client.MCPAuthorizationController @@ -677,8 +681,10 @@ type Model struct { // is later; the field carries the one migrated surface. A surface's state is // created at Open and lives ONLY inside this interface field — never a // pre-declared tombstone field (surface.go). - modal surface - pendingApproval *approvalResolvedIntent + modal surface + pendingApproval *approvalResolvedIntent + pendingRecovery *pendingApprovalRecovery + pendingRecoveryGeneration uint64 // modelSwitchRequestToken correlates the asynchronous create-and-hydrate handoff. // A stale result must not replace a session selected by a later lifecycle action. modelSwitchRequestToken uint64 @@ -1157,6 +1163,30 @@ func New(deps Deps) Model { m.startupAdopted = true m.restartedThisRun = true m.statusMsg = "continuing chat " + terminaltext.Sanitize(resume.Row.Title) + " — type to add a turn" + if resume.Pending != nil { + m.pendingRecoveryGeneration++ + recoveryCtx, recoveryCancel := context.WithCancel(deps.Ctx) + snapshotCalls := make(map[string]struct{}) + for i := 0; i < m.conv.scrollback.Len(); i++ { + toolCard, ok := m.conv.scrollback.SnapshotAt(i).Payload.(scrollback.ToolCardSnapshot) + if ok && !toolCard.Resolved { + snapshotCalls[toolCard.Call.ID] = struct{}{} + } + } + m.pendingRecovery = &pendingApprovalRecovery{ + approval: *resume.Pending, ctx: recoveryCtx, cancel: recoveryCancel, + generation: m.pendingRecoveryGeneration, snapshotCalls: snapshotCalls, + } + m.phase = phaseConnecting + m.pendingInitialPrompt = "" + if deps.InitialPrompt != "" { + m.prompt.Rewrite(deps.InitialPrompt) + m.statusMsg = "initial prompt kept as a draft while approval recovery is pending" + } else { + m.statusMsg = "verifying pending approval recovery" + } + m.prompt.Blur() + } m.refreshView() } return m @@ -1191,6 +1221,7 @@ func (m Model) resetSession() Model { } func (m Model) resetSessionDerived() Model { + (&m).retirePendingApprovalRecovery() m.admissionSubmission = nil m = m.resetDocumentProjection() m.usage = client.Usage{} @@ -1356,6 +1387,9 @@ func (m Model) startupCmd() tea.Cmd { return tea.Batch(cmds...) } if m.deps.Resume != nil { + if m.pendingRecovery != nil { + return tea.Batch(m.sp.Tick, m.openPendingApprovalWatchCmd(), m.statusLineWaitCmd()) + } return tea.Batch(m.sp.Tick, func() tea.Msg { return startupResumeReadyMsg{} }, m.statusLineWaitCmd()) } if m.deps.Models != nil { diff --git a/cmd/mecatui/ui/pending_approval.go b/cmd/mecatui/ui/pending_approval.go new file mode 100644 index 0000000000..076b520c0d --- /dev/null +++ b/cmd/mecatui/ui/pending_approval.go @@ -0,0 +1,297 @@ +package ui + +import ( + "context" + + tea "charm.land/bubbletea/v2" + + "github.com/stacklok/mecatl/cmd/mecatui/client" +) + +// PendingApprovalWatch is the caller-owned exact-run receive side. +type PendingApprovalWatch interface { + Recv() (client.PendingApprovalEvent, error) + Close() +} + +// PendingApprovalController is the narrow exact-run recovery transport. +type PendingApprovalController interface { + WatchPendingApprovalRun(context.Context, client.PendingApproval) (PendingApprovalWatch, error) + ResolvePendingApproval(context.Context, client.PendingApproval, client.Verdict) error + CancelPendingRun(context.Context, client.PendingApproval) error +} + +type pendingApprovalRecovery struct { + approval client.PendingApproval + watch PendingApprovalWatch + ctx context.Context + cancel context.CancelFunc + generation uint64 + snapshotCalls map[string]struct{} + resolved bool + resolving bool + cancelled bool +} + +type pendingApprovalWatchOpenedMsg struct { + generation uint64 + watch PendingApprovalWatch + event client.PendingApprovalEvent + err error +} + +type pendingApprovalWatchMsg struct { + generation uint64 + event client.PendingApprovalEvent + err error +} + +type pendingApprovalResolvedMsg struct { + generation uint64 + approval client.PendingApproval + err error +} +type pendingApprovalCancelledMsg struct { + generation uint64 + approval client.PendingApproval + err error +} + +func samePendingApproval(a, b client.PendingApproval) bool { + return a.SessionID == b.SessionID && a.RunID == b.RunID && a.AskID == b.AskID +} + +func (m Model) openPendingApprovalWatchCmd() tea.Cmd { + recovery := m.pendingRecovery + controller := m.deps.PendingApprovals + return func() tea.Msg { + if recovery == nil || controller == nil { + return pendingApprovalWatchOpenedMsg{err: context.Canceled} + } + watch, err := controller.WatchPendingApprovalRun(recovery.ctx, recovery.approval) + if err != nil { + return pendingApprovalWatchOpenedMsg{generation: recovery.generation, err: err} + } + if recovery.ctx.Err() != nil { + watch.Close() + return pendingApprovalWatchOpenedMsg{generation: recovery.generation, err: context.Canceled} + } + event, err := watch.Recv() + if err != nil || recovery.ctx.Err() != nil { + watch.Close() + if err == nil { + err = context.Canceled + } + } + return pendingApprovalWatchOpenedMsg{generation: recovery.generation, watch: watch, event: event, err: err} + } +} + +func pendingApprovalRecvCmd(generation uint64, watch PendingApprovalWatch) tea.Cmd { + return func() tea.Msg { + if watch == nil { + return pendingApprovalWatchMsg{generation: generation, err: context.Canceled} + } + event, err := watch.Recv() + return pendingApprovalWatchMsg{generation: generation, event: event, err: err} + } +} + +func (m *Model) resolvePendingApprovalCmd(verdict client.Verdict) tea.Cmd { + if m.pendingRecovery == nil || m.pendingRecovery.resolving || m.pendingRecovery.resolved { + return nil + } + m.pendingRecovery.resolving = true + controller := m.deps.PendingApprovals + recovery := m.pendingRecovery + approval := recovery.approval + return func() tea.Msg { + if controller == nil { + return pendingApprovalResolvedMsg{generation: recovery.generation, approval: approval, err: context.Canceled} + } + return pendingApprovalResolvedMsg{generation: recovery.generation, approval: approval, err: controller.ResolvePendingApproval(recovery.ctx, approval, verdict)} + } +} + +func (m *Model) cancelPendingApprovalCmd() tea.Cmd { + if m.pendingRecovery == nil || !m.pendingRecovery.resolved || m.pendingRecovery.cancelled { + return nil + } + m.pendingRecovery.cancelled = true + controller := m.deps.PendingApprovals + recovery := m.pendingRecovery + approval := recovery.approval + return func() tea.Msg { + if controller == nil { + return pendingApprovalCancelledMsg{generation: recovery.generation, approval: approval, err: context.Canceled} + } + return pendingApprovalCancelledMsg{generation: recovery.generation, approval: approval, err: controller.CancelPendingRun(recovery.ctx, approval)} + } +} + +func (m *Model) retirePendingApprovalRecovery() { + if m.pendingRecovery == nil { + return + } + m.pendingRecovery.cancel() + if m.pendingRecovery.watch != nil { + m.pendingRecovery.watch.Close() + } + m.pendingRecovery = nil +} + +func (m Model) refusePendingApprovalRecovery(text string) Model { + (&m).retirePendingApprovalRecovery() + m.closeModal() + m.phase = phaseFatal + m.statusMsg = m.deps.Theme.Style("errorText").Render(text) + m.conv.addError(text) + m.prompt.Blur() + m.refreshView() + return m +} + +func (m Model) updatePendingApproval(msg tea.Msg) (tea.Model, tea.Cmd, bool) { + if m.pendingRecovery == nil { + return m.consumeRetiredPendingApprovalMessage(msg) + } + + switch msg := msg.(type) { + case pendingApprovalWatchOpenedMsg: + return m.updatePendingApprovalWatchOpened(msg) + case pendingApprovalResolvedMsg: + return m.updatePendingApprovalResolved(msg) + case pendingApprovalCancelledMsg: + return m.updatePendingApprovalCancelled(msg) + case pendingApprovalWatchMsg: + return m.updatePendingApprovalWatch(msg) + default: + return m, nil, false + } +} + +// Recovery messages are always consumed. A late open owns a watch that the +// retired generation can no longer publish, so close it here. +func (m Model) consumeRetiredPendingApprovalMessage(msg tea.Msg) (tea.Model, tea.Cmd, bool) { + switch msg := msg.(type) { + case pendingApprovalWatchOpenedMsg: + if msg.watch != nil { + msg.watch.Close() + } + return m, nil, true + case pendingApprovalWatchMsg, pendingApprovalResolvedMsg, pendingApprovalCancelledMsg: + return m, nil, true + default: + return m, nil, false + } +} + +func (m Model) updatePendingApprovalWatchOpened(msg pendingApprovalWatchOpenedMsg) (tea.Model, tea.Cmd, bool) { + recovery := m.pendingRecovery + if msg.generation != recovery.generation { + if msg.watch != nil { + msg.watch.Close() + } + return m, nil, true + } + if msg.err != nil || msg.watch == nil || msg.event.Kind != client.PendingApprovalEventBoundary { + if msg.watch != nil { + msg.watch.Close() + } + return m.refusePendingApprovalRecovery("pending approval recovery could not establish a gap-free watch; leave the approval pending and retry"), nil, true + } + recovery.watch = msg.watch + m.phase = phaseRunning + ask := recovery.approval + mm, cmd := m.applyPermissionAsk(client.PermissionAskMsg{RunID: ask.RunID, AskID: ask.AskID, Tool: ask.Tool, Args: ask.Args, Reason: ask.Reason, ExpectedRunID: ask.RunID, Recovery: true}) + m = mm.(Model) + return m, tea.Batch(cmd, pendingApprovalRecvCmd(recovery.generation, msg.watch)), true +} + +func (m Model) updatePendingApprovalResolved(msg pendingApprovalResolvedMsg) (tea.Model, tea.Cmd, bool) { + recovery := m.pendingRecovery + if msg.generation != recovery.generation || !samePendingApproval(msg.approval, recovery.approval) { + return m, nil, true + } + recovery.resolving = false + if msg.err != nil { + return m.refusePendingApprovalRecovery("the pending approval changed or could not be acknowledged; no verdict will be resent"), nil, true + } + recovery.resolved = true + return m, nil, true +} + +func (m Model) updatePendingApprovalCancelled(msg pendingApprovalCancelledMsg) (tea.Model, tea.Cmd, bool) { + recovery := m.pendingRecovery + if msg.generation != recovery.generation || !samePendingApproval(msg.approval, recovery.approval) { + return m, nil, true + } + if msg.err != nil { + return m.refusePendingApprovalRecovery("the recovered run could not be cancelled; check the session before retrying"), nil, true + } + (&m).retirePendingApprovalRecovery() + return m, tea.Quit, true +} + +func (m Model) updatePendingApprovalWatch(msg pendingApprovalWatchMsg) (tea.Model, tea.Cmd, bool) { + recovery := m.pendingRecovery + if msg.generation != recovery.generation { + return m, nil, true + } + if msg.err != nil { + text := "the pending approval watch ended before a terminal result; check the session before retrying" + if !recovery.resolving && !recovery.resolved { + text = "the pending approval watch ended before your choice; leave the approval pending and retry" + } + return m.refusePendingApprovalRecovery(text), nil, true + } + if msg.event.Kind == client.PendingApprovalEventResolved && !recovery.resolved && !recovery.resolving { + return m.refusePendingApprovalRecovery("the pending approval changed before your choice; no verdict was sent"), nil, true + } + if msg.event.Kind == client.PendingApprovalEventRetracted { + text := "the pending approval was withdrawn; no verdict was sent" + if recovery.resolving || recovery.resolved { + text = "the recovered approval was withdrawn after your choice; check the session before retrying" + } + return m.refusePendingApprovalRecovery(text), nil, true + } + if msg.event.Kind == client.PendingApprovalEventAsk && msg.event.Approval != nil { + ask := *msg.event.Approval + recovery.approval = ask + recovery.resolved = false + recovery.resolving = false + recovery.cancelled = false + mm, cmd := m.applyPermissionAsk(client.PermissionAskMsg{RunID: ask.RunID, AskID: ask.AskID, Tool: ask.Tool, Args: ask.Args, Reason: ask.Reason, ExpectedRunID: ask.RunID, Recovery: true}) + m = mm.(Model) + return m, tea.Batch(cmd, pendingApprovalRecvCmd(recovery.generation, recovery.watch)), true + } + // After an explicit choice, follow EventToMsg's nil/skip convention for + // incidental telemetry (for example request.manifest). Before a choice, + // an unknown event still makes the pending permission state uncertain. + if msg.event.Kind == client.PendingApprovalEventOther && msg.event.Message == nil && !recovery.resolving && !recovery.resolved { + return m.refusePendingApprovalRecovery("the pending approval watch returned an unknown event; no control was sent"), nil, true + } + eventCmd := m.applyPendingApprovalEventMessage(msg.event.Message, recovery) + if msg.event.Kind == client.PendingApprovalEventTerminal { + (&m).retirePendingApprovalRecovery() + return m, eventCmd, true + } + return m, tea.Batch(eventCmd, pendingApprovalRecvCmd(recovery.generation, recovery.watch)), true +} + +func (m *Model) applyPendingApprovalEventMessage(message tea.Msg, recovery *pendingApprovalRecovery) tea.Cmd { + if message == nil { + return nil + } + if call, ok := message.(client.ToolCallMsg); ok { + if _, replay := recovery.snapshotCalls[call.ID]; replay { + delete(recovery.snapshotCalls, call.ID) + if m.conv.reconcileUnresolvedTool(call.ID, call.Name, call.Args) { + return nil + } + } + } + mm, cmd := m.updateStreamEvent(message) + *m = mm.(Model) + return cmd +} diff --git a/cmd/mecatui/ui/pending_approval_lifecycle_test.go b/cmd/mecatui/ui/pending_approval_lifecycle_test.go new file mode 100644 index 0000000000..9ec0565a14 --- /dev/null +++ b/cmd/mecatui/ui/pending_approval_lifecycle_test.go @@ -0,0 +1,365 @@ +package ui + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + tea "charm.land/bubbletea/v2" + + "github.com/stacklok/mecatl/cmd/mecatui/client" + "github.com/stacklok/mecatl/cmd/mecatui/theme" + "github.com/stacklok/mecatl/cmd/mecatui/ui/internal/scrollback" +) + +func receivePendingTest[T any](t *testing.T, ch <-chan T) T { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), scaleWait(5*time.Second)) + defer cancel() + select { + case value := <-ch: + return value + case <-ctx.Done(): + t.Fatalf("pending recovery test barrier: %v", ctx.Err()) + var zero T + return zero + } +} + +func TestPendingApprovalRecoveryLateControlCannotOverwriteLaterAsk(t *testing.T) { + started, release := make(chan struct{}, 1), make(chan struct{}) + controller := &pendingApprovalFixtureController{ + watch: &pendingApprovalFixtureWatch{}, resolveStarted: started, resolveRelease: release, + } + m := pendingRecoveryModel(t, controller, "") + old := m.pendingRecovery.approval + + next, choose := m.Update(tea.KeyPressMsg{Code: 'a', Text: "a"}) + m = next.(Model) + result := make(chan tea.Msg, 1) + var execute func(tea.Cmd) + execute = func(cmd tea.Cmd) { + if cmd == nil { + return + } + switch msg := cmd().(type) { + case tea.BatchMsg: + for _, child := range msg { + execute(child) + } + case pendingApprovalResolvedMsg: + result <- msg + } + } + go execute(choose) + receivePendingTest(t, started) + var releaseOnce sync.Once + releaseControl := func() { releaseOnce.Do(func() { close(release) }) } + defer releaseControl() + + // The first choice removes the approval surface before its acknowledgement. + // Repeating the same key while that RPC is blocked must not submit it again. + next, repeated := m.Update(tea.KeyPressMsg{Code: 'a', Text: "a"}) + m = next.(Model) + repeatedDone := make(chan struct{}) + go func() { + execute(repeated) + close(repeatedDone) + }() + waitCtx, cancelWait := context.WithTimeout(context.Background(), scaleWait(5*time.Second)) + defer cancelWait() + select { + case <-repeatedDone: + case <-started: + t.Fatal("repeated approval input started a second RPC") + case <-waitCtx.Done(): + t.Fatalf("repeated approval input did not settle: %v", waitCtx.Err()) + } + if calls, _, _ := controller.counts(); calls != 1 { + t.Fatalf("resolve calls while acknowledgement blocked = %d, want one", calls) + } + + later := old + later.AskID, later.Tool, later.Cursor = "ask-later", "Shell", "cursor-later" + next, _ = m.Update(pendingApprovalWatchMsg{generation: m.pendingRecovery.generation, event: client.PendingApprovalEvent{ + Kind: client.PendingApprovalEventAsk, Approval: &later, + }}) + m = next.(Model) + releaseControl() + next, _ = m.Update(receivePendingTest(t, result)) + m = next.(Model) + if m.pendingRecovery == nil || m.pendingRecovery.approval.AskID != later.AskID || m.pendingRecovery.resolved { + t.Fatalf("stale old control changed later ask: %+v", m.pendingRecovery) + } + if calls, _, _ := controller.counts(); calls != 1 { + t.Fatalf("resolve calls = %d, want one", calls) + } + controller.mu.Lock() + got := append([]client.PendingApproval(nil), controller.resolvedFor...) + controller.mu.Unlock() + if len(got) != 1 || !samePendingApproval(got[0], old) { + t.Fatalf("resolved identities = %+v, want old %+v", got, old) + } +} + +func TestPendingApprovalRecoveryQuitDuringBlockedResolve(t *testing.T) { + for _, direct := range []bool{false, true} { + name := "double-ctrl-c" + if direct { + name = "quit-now" + } + t.Run(name, func(t *testing.T) { + controller := &pendingApprovalFixtureController{ + watch: &pendingApprovalFixtureWatch{}, + resolveStarted: make(chan struct{}, 1), resolveRelease: make(chan struct{}), + } + const draft = "keep this unsent recovery prompt" + m := pendingRecoveryModel(t, controller, draft) + recovery := m.pendingRecovery + t.Cleanup(recovery.cancel) + t.Cleanup(controller.watch.Close) + next, choose := m.Update(tea.KeyPressMsg{Code: 'a', Text: "a"}) + m = next.(Model) + result := make(chan pendingApprovalResolvedMsg, 1) + var execute func(tea.Cmd) + execute = func(cmd tea.Cmd) { + if cmd == nil { + return + } + switch msg := cmd().(type) { + case tea.BatchMsg: + for _, child := range msg { + execute(child) + } + case pendingApprovalResolvedMsg: + result <- msg + } + } + go execute(choose) + receivePendingTest(t, controller.resolveStarted) + var quit tea.Cmd + if direct { + next, quit = m.quitNow() + } else { + next, _ = m.Update(tea.KeyPressMsg{Code: 'c', Mod: tea.ModCtrl}) + m = next.(Model) + if !m.quitArmed || m.prompt.Value() != draft || recovery.ctx.Err() != nil || controller.watch.isClosed() { + t.Fatal("first ctrl+c must preserve the draft and arm exit without cancelling the owner choice") + } + next, quit = m.Update(tea.KeyPressMsg{Code: 'c', Mod: tea.ModCtrl}) + } + m = next.(Model) + if quit == nil { + t.Fatal("blocked Resolve prevented exit") + } + msg := quit() + if _, ok := msg.(tea.QuitMsg); !ok { + t.Fatalf("exit command = %T, want tea.QuitMsg", msg) + } + if m.pendingRecovery != nil || !controller.watch.isClosed() { + t.Fatal("exit did not retire recovery and close watch") + } + ack := receivePendingTest(t, result) + if !errors.Is(ack.err, context.Canceled) { + t.Fatalf("blocked Resolve did not observe owned context cancellation: %v", ack.err) + } + // Either a cancelled RPC or a late successful acknowledgement is stale. + for _, err := range []error{ack.err, nil} { + ack.err = err + next, cmd := m.Update(ack) + if cmd != nil || next.(Model).pendingRecovery != nil || next.(Model).phase != m.phase { + t.Fatal("late acknowledgement reactivated retired recovery") + } + } + if resolves, cancels, verdict := controller.counts(); resolves != 1 || cancels != 0 || verdict != client.VerdictAllowOnce { + t.Fatalf("exit controls: resolve=%d cancel=%d verdict=%v", resolves, cancels, verdict) + } + }) + } +} + +func TestPendingApprovalRecoveryFailureRetiresQueuedMessages(t *testing.T) { + watch := &pendingApprovalFixtureWatch{} + controller := &pendingApprovalFixtureController{watch: watch, resolveErr: errors.New("rejected")} + m := pendingRecoveryModel(t, controller, "") + generation := m.pendingRecovery.generation + + next, cmd := m.Update(tea.KeyPressMsg{Code: 'd', Text: "d"}) + m = applyPendingCommand(t, next.(Model), cmd) + if m.pendingRecovery != nil || m.phase != phaseFatal || !watch.isClosed() { + t.Fatalf("failed control did not retire recovery: recovery=%+v phase=%v closed=%t", m.pendingRecovery, m.phase, watch.isClosed()) + } + queued := []tea.Msg{ + pendingApprovalWatchMsg{generation: generation, event: client.PendingApprovalEvent{Kind: client.PendingApprovalEventOther, Message: client.ToolProgressMsg{Text: "late"}}}, + pendingApprovalWatchMsg{generation: generation, event: client.PendingApprovalEvent{Kind: client.PendingApprovalEventAsk, Approval: &client.PendingApproval{AskID: "late"}}}, + pendingApprovalWatchMsg{generation: generation, event: client.PendingApprovalEvent{Kind: client.PendingApprovalEventTerminal, Message: client.ResultMsg{Stop: "end_turn"}}}, + } + for _, msg := range queued { + next, cmd = m.Update(msg) + m = next.(Model) + if cmd != nil || m.pendingRecovery != nil || m.phase != phaseFatal { + t.Fatalf("late message %T rearmed recovery: recovery=%+v phase=%v cmd=%v", msg, m.pendingRecovery, m.phase, cmd != nil) + } + } + if calls, _, _ := controller.counts(); calls != 1 { + t.Fatalf("failed verdict resent %d times", calls) + } +} + +func TestPendingApprovalRecoveryQuitBeforeOpenClosesLateWatch(t *testing.T) { + started, release := make(chan struct{}, 1), make(chan struct{}) + watch := &pendingApprovalFixtureWatch{} + controller := &pendingApprovalFixtureController{watch: watch, watchStarted: started, watchRelease: release} + approval := client.PendingApproval{SessionID: "session", RunID: "run", AskID: "ask", Cursor: "cursor"} + m := newTestModelFromDeps(Deps{ + Theme: theme.New("aztec", theme.AztecPalette()), Ctx: t.Context(), PendingApprovals: controller, + Resume: &client.ResumeSelection{Row: client.SessionListItem{ID: "session", Kind: client.SessionKindMain}, Transcript: client.SessionTranscript{SessionID: "session", Complete: true}, Pending: &approval}, + NoAltScreen: true, + }) + cmd := m.openPendingApprovalWatchCmd() + opened := make(chan tea.Msg, 1) + go func() { opened <- cmd() }() + receivePendingTest(t, started) + var releaseOnce sync.Once + releaseOpen := func() { releaseOnce.Do(func() { close(release) }) } + defer releaseOpen() + next, quit := m.Update(tea.KeyPressMsg{Code: tea.KeyEsc}) + m = next.(Model) + if _, ok := quit().(tea.QuitMsg); !ok { + t.Fatalf("quit command = %T", quit()) + } + releaseOpen() + msg := receivePendingTest(t, opened) + if !watch.isClosed() { + t.Fatal("watch returned after cancellation was not closed by opener") + } + next, _ = m.Update(msg) + m = next.(Model) + if m.pendingRecovery != nil { + t.Fatal("late open republished retired recovery") + } +} + +func TestPendingApprovalRecoveryReconcilesSnapshotToolCalls(t *testing.T) { + approval := client.PendingApproval{SessionID: "session", RunID: "run", AskID: "ask", Cursor: "cursor"} + transcript := client.SessionTranscript{SessionID: "session", Complete: true, Messages: []client.ConversationMessage{{ + Role: "assistant", Text: "working", ToolCalls: []client.ConvToolCall{ + {ID: "call-1", Name: "Write", Args: `{"path":"old"}`}, + {ID: "call-2", Name: "Shell", Args: `{"command":"queued"}`}, + }, + }}} + controller := &pendingApprovalFixtureController{watch: &pendingApprovalFixtureWatch{}} + m := newTestModelFromDeps(Deps{ + Theme: theme.New("aztec", theme.AztecPalette()), Ctx: t.Context(), PendingApprovals: controller, + Resume: &client.ResumeSelection{Row: client.SessionListItem{ID: "session", Kind: client.SessionKindMain}, Transcript: transcript, Pending: &approval}, NoAltScreen: true, + }) + m = applyAll(m, tea.WindowSizeMsg{Width: 100, Height: 30}) + opened := m.openPendingApprovalWatchCmd()() + next, _ := m.Update(opened) + m = next.(Model) + generation := m.pendingRecovery.generation + + next, _ = m.Update(pendingApprovalWatchMsg{generation: generation, event: client.PendingApprovalEvent{ + Kind: client.PendingApprovalEventOther, Message: client.ToolCallMsg{ID: "call-1", Name: "Write", Args: `{"path":"actual"}`}, + }}) + m = next.(Model) + var tools []scrollback.ToolCardSnapshot + for i := 0; i < m.conv.scrollback.Len(); i++ { + if toolCard, ok := m.conv.scrollback.SnapshotAt(i).Payload.(scrollback.ToolCardSnapshot); ok { + tools = append(tools, toolCard) + } + } + if len(tools) != 2 || tools[0].Call.Arguments != `{"path":"actual"}` || tools[1].Call.ID != "call-2" { + t.Fatalf("reconciled tools = %+v", tools) + } + + // Once the recovered occurrence is consumed, a later legitimate reuse is a new card. + next, _ = m.Update(pendingApprovalWatchMsg{generation: generation, event: client.PendingApprovalEvent{ + Kind: client.PendingApprovalEventOther, Message: client.ToolCallMsg{ID: "call-1", Name: "Read", Args: `{"path":"later"}`}, + }}) + m = next.(Model) + tools = tools[:0] + for i := 0; i < m.conv.scrollback.Len(); i++ { + if toolCard, ok := m.conv.scrollback.SnapshotAt(i).Payload.(scrollback.ToolCardSnapshot); ok { + tools = append(tools, toolCard) + } + } + if len(tools) != 3 || tools[2].Call.Name != "Read" { + t.Fatalf("later reused call identity was globally deduplicated: %+v", tools) + } +} + +func TestPendingApprovalRecoverySkipsUnmappedTelemetryOnlyAfterChoice(t *testing.T) { + for _, state := range []string{"before-choice", "resolving", "resolved", "later-ask"} { + t.Run(state, func(t *testing.T) { + controller := &pendingApprovalFixtureController{watch: &pendingApprovalFixtureWatch{}} + m := pendingRecoveryModel(t, controller, "draft only") + t.Cleanup(m.pendingRecovery.cancel) + t.Cleanup(controller.watch.Close) + if state != "before-choice" { + next, cmd := m.Update(tea.KeyPressMsg{Code: 'a', Text: "a"}) + m = next.(Model) + if state != "resolving" { + m = applyPendingCommand(t, m, cmd) + } + } + if state == "later-ask" { + later := m.pendingRecovery.approval + later.AskID = "later" + next, _ := m.Update(pendingApprovalWatchMsg{generation: m.pendingRecovery.generation, event: client.PendingApprovalEvent{ + Kind: client.PendingApprovalEventAsk, Approval: &later, + }}) + m = next.(Model) + } + beforePhase := m.phase + resolves, cancels, _ := controller.counts() + next, cmd := m.Update(pendingApprovalWatchMsg{generation: m.pendingRecovery.generation, event: client.PendingApprovalEvent{ + Kind: client.PendingApprovalEventOther, + }}) + m = next.(Model) + if state == "before-choice" || state == "later-ask" { + if m.phase != phaseFatal || m.pendingRecovery != nil || !controller.watch.isClosed() || cmd != nil { + t.Fatal("uncertain pending permission state did not fail closed") + } + } else { + if m.phase != beforePhase || m.pendingRecovery == nil || controller.watch.isClosed() || cmd == nil { + t.Fatal("incidental telemetry changed run state or failed to rearm watch") + } + next, _ = m.Update(pendingApprovalWatchMsg{generation: m.pendingRecovery.generation, event: client.PendingApprovalEvent{ + Kind: client.PendingApprovalEventTerminal, Message: client.ResultMsg{Stop: "end_turn"}, + }}) + m = next.(Model) + if m.phase != phaseIdle || m.pendingRecovery != nil || !controller.watch.isClosed() { + t.Fatal("terminal continuation did not settle recovery") + } + } + if gotResolves, gotCancels, _ := controller.counts(); gotResolves != resolves || gotCancels != cancels || m.prompt.Value() != "draft only" { + t.Fatal("telemetry changed controls or draft") + } + }) + } +} + +func TestPendingApprovalRecoveryMalformedOpenReleasesWatch(t *testing.T) { + for _, result := range []pendingApprovalWatchResult{ + {event: client.PendingApprovalEvent{}}, + {event: client.PendingApprovalEvent{Kind: client.PendingApprovalEventAsk}}, + {err: errors.New("stream failed")}, + } { + watch := &pendingApprovalFixtureWatch{events: make(chan pendingApprovalWatchResult, 1), done: make(chan struct{})} + watch.events <- result + controller := &pendingApprovalFixtureController{watch: watch} + approval := client.PendingApproval{SessionID: "session", RunID: "run", AskID: "ask", Cursor: "cursor"} + m := newTestModelFromDeps(Deps{ + Theme: theme.New("aztec", theme.AztecPalette()), Ctx: t.Context(), PendingApprovals: controller, + Resume: &client.ResumeSelection{Row: client.SessionListItem{ID: "session"}, Transcript: client.SessionTranscript{Complete: true}, Pending: &approval}, + }) + next, _ := m.Update(m.openPendingApprovalWatchCmd()()) + m = next.(Model) + if !watch.isClosed() || m.pendingRecovery != nil || m.phase != phaseFatal { + t.Fatalf("malformed open leaked: result=%+v closed=%t recovery=%+v phase=%v", result, watch.isClosed(), m.pendingRecovery, m.phase) + } + } +} diff --git a/cmd/mecatui/ui/pending_approval_test.go b/cmd/mecatui/ui/pending_approval_test.go new file mode 100644 index 0000000000..d589015c22 --- /dev/null +++ b/cmd/mecatui/ui/pending_approval_test.go @@ -0,0 +1,283 @@ +package ui + +import ( + "context" + "sync" + "testing" + "time" + + tea "charm.land/bubbletea/v2" + "github.com/charmbracelet/x/exp/teatest/v2" + + "github.com/stacklok/mecatl/cmd/mecatui/client" + "github.com/stacklok/mecatl/cmd/mecatui/theme" +) + +type pendingApprovalWatchResult struct { + event client.PendingApprovalEvent + err error +} + +type pendingApprovalFixtureWatch struct { + mu sync.Mutex + closed bool + boundarySent bool + once sync.Once + events chan pendingApprovalWatchResult + done chan struct{} +} + +func (w *pendingApprovalFixtureWatch) Recv() (client.PendingApprovalEvent, error) { + if w.events == nil { + w.mu.Lock() + defer w.mu.Unlock() + if !w.boundarySent { + w.boundarySent = true + return client.PendingApprovalEvent{Kind: client.PendingApprovalEventBoundary}, nil + } + return client.PendingApprovalEvent{}, context.Canceled + } + select { + case result := <-w.events: + return result.event, result.err + case <-w.done: + return client.PendingApprovalEvent{}, context.Canceled + } +} +func (w *pendingApprovalFixtureWatch) Close() { + w.once.Do(func() { + w.mu.Lock() + w.closed = true + w.mu.Unlock() + if w.done != nil { + close(w.done) + } + }) +} +func (w *pendingApprovalFixtureWatch) isClosed() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.closed +} + +type pendingApprovalFixtureController struct { + mu sync.Mutex + watch *pendingApprovalFixtureWatch + watchCalls int + resolveCalls int + cancelCalls int + verdict client.Verdict + resolvedFor []client.PendingApproval + cancelledFor []client.PendingApproval + resolved chan struct{} + resolveStarted chan struct{} + resolveRelease chan struct{} + resolveErr error + watchStarted chan struct{} + watchRelease chan struct{} +} + +func (f *pendingApprovalFixtureController) WatchPendingApprovalRun(context.Context, client.PendingApproval) (PendingApprovalWatch, error) { + f.mu.Lock() + f.watchCalls++ + started, release, watch := f.watchStarted, f.watchRelease, f.watch + f.mu.Unlock() + if started != nil { + started <- struct{}{} + } + if release != nil { + <-release + } + return watch, nil +} +func (f *pendingApprovalFixtureController) ResolvePendingApproval(ctx context.Context, approval client.PendingApproval, verdict client.Verdict) error { + f.mu.Lock() + f.resolveCalls++ + f.verdict = verdict + f.resolvedFor = append(f.resolvedFor, approval) + started, release, resolved, err := f.resolveStarted, f.resolveRelease, f.resolved, f.resolveErr + f.mu.Unlock() + if started != nil { + started <- struct{}{} + } + if release != nil { + select { + case <-release: + case <-ctx.Done(): + return ctx.Err() + } + } + if resolved != nil { + resolved <- struct{}{} + } + return err +} +func (f *pendingApprovalFixtureController) CancelPendingRun(_ context.Context, approval client.PendingApproval) error { + f.mu.Lock() + defer f.mu.Unlock() + f.cancelCalls++ + f.cancelledFor = append(f.cancelledFor, approval) + return nil +} + +func (f *pendingApprovalFixtureController) counts() (int, int, client.Verdict) { + f.mu.Lock() + defer f.mu.Unlock() + return f.resolveCalls, f.cancelCalls, f.verdict +} + +func pendingRecoveryModel(t *testing.T, controller *pendingApprovalFixtureController, prompt string) Model { + t.Helper() + approval := client.PendingApproval{SessionID: "session", RunID: "run", AskID: "ask", Tool: "Write", Args: `{"path":"x"}`, Reason: "write?", Cursor: "cursor"} + selection := &client.ResumeSelection{ + Row: client.SessionListItem{ID: "session", Kind: client.SessionKindMain}, + Transcript: client.SessionTranscript{SessionID: "session", Complete: true, Kind: client.SessionKindMain}, + Snapshot: client.SessionSnapshot{State: "awaiting"}, Pending: &approval, + } + m := newTestModelFromDeps(Deps{ + Theme: theme.New("aztec", theme.AztecPalette()), Ctx: t.Context(), Resume: selection, + PendingApprovals: controller, InitialPrompt: prompt, NoAltScreen: true, + }) + m = applyAll(m, tea.WindowSizeMsg{Width: 100, Height: 30}) + opened := m.openPendingApprovalWatchCmd()() + next, _ := m.Update(opened) + return next.(Model) +} + +func applyPendingCommand(t *testing.T, m Model, cmd tea.Cmd) Model { + t.Helper() + var messages []tea.Msg + var collect func(tea.Cmd, int) + collect = func(next tea.Cmd, depth int) { + if next == nil || depth > 4 { + return + } + msg := next() + if batch, ok := msg.(tea.BatchMsg); ok { + for _, child := range batch { + collect(child, depth+1) + } + return + } + switch msg.(type) { + case pendingApprovalResolvedMsg, pendingApprovalCancelledMsg: + messages = append(messages, msg) + } + } + collect(cmd, 0) + for _, msg := range messages { + next, _ := m.Update(msg) + m = next.(Model) + } + return m +} + +func TestPendingApprovalRecoveryUIExplicitControls(t *testing.T) { + controller := &pendingApprovalFixtureController{watch: &pendingApprovalFixtureWatch{}} + m := pendingRecoveryModel(t, controller, "keep this draft") + if controller.resolveCalls != 0 || controller.cancelCalls != 0 || m.prompt.Value() != "keep this draft" { + t.Fatalf("startup sent control or lost draft: resolve=%d cancel=%d draft=%q", controller.resolveCalls, controller.cancelCalls, m.prompt.Value()) + } + surface := approvalSurfaceOf(t, m) + if surface.ask.offerAlways { + t.Fatal("recovered approval offered Allow Always") + } + + next, cmd := m.Update(tea.KeyPressMsg{Code: 'a', Text: "a"}) + m = applyPendingCommand(t, next.(Model), cmd) + if controller.resolveCalls != 1 || controller.verdict != client.VerdictAllowOnce { + t.Fatalf("allow controls = %d, %v", controller.resolveCalls, controller.verdict) + } + // The modal is gone, so a repeated key cannot submit the verdict twice. + next, cmd = m.Update(tea.KeyPressMsg{Code: 'a', Text: "a"}) + m = applyPendingCommand(t, next.(Model), cmd) + if controller.resolveCalls != 1 { + t.Fatalf("repeated verdict calls = %d, want 1", controller.resolveCalls) + } + + next, cmd = m.Update(tea.KeyPressMsg{Code: 'c', Mod: tea.ModCtrl}) + m = applyPendingCommand(t, next.(Model), cmd) + if controller.cancelCalls != 1 { + t.Fatalf("ctrl+c cancel calls = %d, want 1", controller.cancelCalls) + } + controller.mu.Lock() + cancelled := append([]client.PendingApproval(nil), controller.cancelledFor...) + controller.mu.Unlock() + if len(cancelled) != 1 || cancelled[0].SessionID != "session" || cancelled[0].RunID != "run" || cancelled[0].AskID != "ask" { + t.Fatalf("cancel identities = %+v", cancelled) + } +} + +func TestPendingApprovalRecoveryTeatestPhase(t *testing.T) { + watch := &pendingApprovalFixtureWatch{events: make(chan pendingApprovalWatchResult, 2), done: make(chan struct{})} + watch.events <- pendingApprovalWatchResult{event: client.PendingApprovalEvent{Kind: client.PendingApprovalEventBoundary}} + controller := &pendingApprovalFixtureController{watch: watch, resolved: make(chan struct{}, 1)} + converser, converseService := newResumeBufConverser(t) + approval := client.PendingApproval{SessionID: "session", RunID: "run", AskID: "ask", Tool: "Write", Args: `{"path":"x"}`, Cursor: "cursor"} + progress := newProgress() + model := newTestModelFromDeps(Deps{ + Theme: theme.New("aztec", theme.AztecPalette()), Ctx: t.Context(), PendingApprovals: controller, Conv: converser, + Resume: &client.ResumeSelection{ + Row: client.SessionListItem{ID: "session", Kind: client.SessionKindMain}, + Transcript: client.SessionTranscript{SessionID: "session", Complete: true, Kind: client.SessionKindMain}, + Snapshot: client.SessionSnapshot{State: "awaiting"}, Pending: &approval, + }, + InitialPrompt: "draft only", NoAltScreen: true, onPhase: progress.record, + }) + tm := teatest.NewTestModel(t, model, teatest.WithInitialTermSize(100, 30)) + progress.wait(t, phaseAwaitingApproval, 5*time.Second) + if resolves, cancels, _ := controller.counts(); resolves != 0 || cancels != 0 { + t.Fatalf("startup controls = resolve %d cancel %d", resolves, cancels) + } + if _, prompts := converseService.snapshot(); len(prompts) != 0 { + t.Fatalf("approval recovery automatically prompted: %q", prompts) + } + + tm.Send(tea.KeyPressMsg{Code: 'd', Text: "d"}) + select { + case <-controller.resolved: + case <-time.After(5 * time.Second): + t.Fatal("deny control was not delivered") + } + progress.wait(t, phaseRunning, 5*time.Second) + if resolves, _, verdict := controller.counts(); resolves != 1 || verdict != client.VerdictDeny { + t.Fatalf("deny controls = %d, %v", resolves, verdict) + } + watch.events <- pendingApprovalWatchResult{event: client.PendingApprovalEvent{ + Kind: client.PendingApprovalEventTerminal, Message: client.ResultMsg{Stop: "end_turn"}, + }} + progress.wait(t, phaseIdle, 5*time.Second) + if _, prompts := converseService.snapshot(); len(prompts) != 0 { + t.Fatalf("verdict automatically prompted: %q", prompts) + } + tm.Send(tea.KeyPressMsg{Code: 'u', Mod: tea.ModCtrl}) + tm.Type("manual follow-up") + tm.Send(tea.KeyPressMsg{Code: tea.KeyEnter}) + progress.waitRunComplete(t, 2, 5*time.Second) + if _, prompts := converseService.snapshot(); len(prompts) != 1 || prompts[0] != "manual follow-up" { + t.Fatalf("manual prompt control = %q", prompts) + } + if err := tm.Quit(); err != nil { + t.Fatalf("quit teatest: %v", err) + } + tm.WaitFinished(t, teatest.WithFinalTimeout(scaleWait(3*time.Second))) + final := tm.FinalModel(t).(Model) + if final.prompt.Value() != "" || !watch.isClosed() { + t.Fatalf("terminal recovery draft=%q watch closed=%t", final.prompt.Value(), watch.isClosed()) + } +} + +func TestPendingApprovalRecoveryEscLeavesAskUnchanged(t *testing.T) { + controller := &pendingApprovalFixtureController{watch: &pendingApprovalFixtureWatch{}} + m := pendingRecoveryModel(t, controller, "") + _, cmd := m.Update(tea.KeyPressMsg{Code: tea.KeyEsc}) + if cmd == nil { + t.Fatal("esc did not quit pending recovery") + } + if _, ok := cmd().(tea.QuitMsg); !ok { + t.Fatalf("esc command = %T, want tea.QuitMsg", cmd()) + } + if controller.resolveCalls != 0 || controller.cancelCalls != 0 || !controller.watch.isClosed() { + t.Fatalf("esc changed ask or leaked watch: resolve=%d cancel=%d closed=%t", controller.resolveCalls, controller.cancelCalls, controller.watch.isClosed()) + } +} diff --git a/cmd/mecatui/ui/render_cache_test.go b/cmd/mecatui/ui/render_cache_test.go index 28bb6d3a23..92e6233b5c 100644 --- a/cmd/mecatui/ui/render_cache_test.go +++ b/cmd/mecatui/ui/render_cache_test.go @@ -105,6 +105,9 @@ var oracleSteps = []struct { {"endReasoningStream", func(c *conversation) { c.endReasoningStream() }}, {"addTurnStat", func(c *conversation) { c.addTurnStat("turn 1 · ↑1.2k ↓300 · 2.1s") }}, {"addTool", func(c *conversation) { c.addTool("call-1", "Read", `{"path":"main.go"}`) }}, + {"reconcileUnresolvedTool", func(c *conversation) { + c.reconcileUnresolvedTool("call-1", "Read", `{"path":"actual.go"}`) + }}, {"addNotice", func(c *conversation) { c.addNotice("context compacted") }}, {"retractLatestNotice", func(c *conversation) { c.addNotice("provisional approval") diff --git a/cmd/mecatui/ui/session_continuity_scenario6_test.go b/cmd/mecatui/ui/session_continuity_scenario6_test.go index 404f933734..687e918211 100644 --- a/cmd/mecatui/ui/session_continuity_scenario6_test.go +++ b/cmd/mecatui/ui/session_continuity_scenario6_test.go @@ -7,6 +7,8 @@ import ( "testing" tea "charm.land/bubbletea/v2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/stacklok/mecatl/cmd/mecatui/client" ) @@ -97,7 +99,7 @@ func TestStartupRunEntryFailureRebuildsDocumentProjection(t *testing.T) { {Role: "assistant", Text: "new tool", ToolCalls: []client.ConvToolCall{{ID: "call-1", Name: "Bash", Args: `{"command":"printf new"}`}}}, {Role: "tool", ToolResult: &client.ConvToolResult{CallID: "call-1", Content: "new result", IsError: true}}, } - m = m.failStartupRunEntry() + m = m.failStartupRunEntry(nil) content := stripANSIstr(m.vp.GetContent()) if !strings.Contains(content, "new request") || !strings.Contains(content, "new result") || strings.Contains(content, "old result") { @@ -120,6 +122,36 @@ func TestStartupRunEntryFailureRebuildsDocumentProjection(t *testing.T) { } } +func TestStartupRunEntryFailureClassifiesSafeStatus(t *testing.T) { + for _, tc := range []struct { + name, want string + code codes.Code + masked bool + }{ + {name: "temporary service failure", code: codes.Unavailable, want: "The service is temporarily unavailable. Retry this turn."}, + {name: "run-entry conflict", code: codes.FailedPrecondition, want: "This chat is not ready for a new turn. Retry after its current operation finishes."}, + {name: "masked absence", code: codes.NotFound, want: "This conversation could not be loaded. You cannot continue this session.", masked: true}, + } { + t.Run(tc.name, func(t *testing.T) { + conv := &fakeConv{recv: &fakeRecver{}, send: &fakeSender{}} + m := startupResumeUI(t, conv, "continue", "completed") + m = applyAll(m, startupResumeReadyMsg{}) + m = applyAll(m, client.StreamErrMsg{Err: status.Error(tc.code, "private server detail / secret-path")}) + + view := stripANSIstr(m.View().Content) + if !strings.Contains(view, tc.want) { + t.Fatalf("startup failure view missing safe classification %q:\n%s", tc.want, view) + } + if strings.Contains(view, "private server detail") || strings.Contains(view, "secret-path") { + t.Fatalf("startup failure exposed server detail:\n%s", view) + } + if gotMasked := strings.Contains(view, "conversation unavailable"); gotMasked != tc.masked { + t.Fatalf("generic unavailable classification = %t, want %t:\n%s", gotMasked, tc.masked, view) + } + }) + } +} + func TestADR_0108_FirstPromptRevalidatesAtomically(t *testing.T) { newFailedModel := func() (Model, *fakeConv, *fakeSender) { sender := &fakeSender{} diff --git a/cmd/mecatui/ui/sessions_surface.go b/cmd/mecatui/ui/sessions_surface.go index ea7a88f0c0..891b677621 100644 --- a/cmd/mecatui/ui/sessions_surface.go +++ b/cmd/mecatui/ui/sessions_surface.go @@ -1575,8 +1575,16 @@ func renderSessionsTranscript(th theme.Theme, st sessionsState, _ string, vpCont return b.String() } if st.loadErr != nil { - b.WriteString(th.Style("title").Render("conversation unavailable") + "\n\n") - b.WriteString(th.Style("errorText").Render("This conversation could not be loaded. You cannot continue this session.")) + title := "conversation unavailable" + if safe, ok := st.loadErr.(interface{ SafeTranscriptTitle() string }); ok && safe.SafeTranscriptTitle() != "" { + title = safe.SafeTranscriptTitle() + } + b.WriteString(th.Style("title").Render(title) + "\n\n") + message := "This conversation could not be loaded. You cannot continue this session." + if safe, ok := st.loadErr.(interface{ SafeTranscriptText() string }); ok { + message = safe.SafeTranscriptText() + } + b.WriteString(th.Style("errorText").Render(message)) b.WriteString("\n" + th.Style("muted").Render("r: Retry "+hk.closeOnly+": Back")) return b.String() } diff --git a/cmd/mecatui/ui/startup_resume_grpc_test.go b/cmd/mecatui/ui/startup_resume_grpc_test.go new file mode 100644 index 0000000000..28aef43c99 --- /dev/null +++ b/cmd/mecatui/ui/startup_resume_grpc_test.go @@ -0,0 +1,186 @@ +package ui + +import ( + "context" + "net" + "sync" + "testing" + "time" + + tea "charm.land/bubbletea/v2" + "github.com/charmbracelet/x/exp/teatest/v2" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + + "github.com/stacklok/mecatl/cmd/mecatui/client" + "github.com/stacklok/mecatl/cmd/mecatui/theme" + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" +) + +type resumeBufServer struct { + mecatlv1.UnimplementedHarnessServiceServer + + mu sync.Mutex + order []string + prompts []string + failFirst bool + calls int +} + +func (s *resumeBufServer) Converse(stream grpc.BidiStreamingServer[mecatlv1.ConverseRequest, mecatlv1.ConverseResponse]) error { + s.record("recv-before-send") + request, err := stream.Recv() + if err != nil { + return err + } + prompt := request.GetPrompt() + s.mu.Lock() + s.order = append(s.order, "prompt-received") + s.prompts = append(s.prompts, prompt.GetText()) + s.calls++ + fail := s.failFirst && s.calls == 1 + s.mu.Unlock() + if fail { + return status.Error(codes.FailedPrecondition, "private lease owner and /private/path") + } + for _, response := range simpleRunScript("resume") { + if err := stream.Send(response); err != nil { + return err + } + } + return nil +} + +func (s *resumeBufServer) record(event string) { + s.mu.Lock() + defer s.mu.Unlock() + s.order = append(s.order, event) +} + +func (s *resumeBufServer) snapshot() ([]string, []string) { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.order...), append([]string(nil), s.prompts...) +} + +type resumeGRPCConverser struct{ svc mecatlv1.HarnessServiceClient } + +func (c resumeGRPCConverser) OpenConverse(ctx context.Context) (*client.Stream, error) { + stream, err := c.svc.Converse(ctx) + if err != nil { + return nil, err + } + return client.NewStream(stream, stream), nil +} + +func (c resumeGRPCConverser) OpenConverseForSession(ctx context.Context, _ string) (*client.Stream, error) { + return c.OpenConverse(ctx) +} + +func newResumeBufConverser(t *testing.T) (resumeGRPCConverser, *resumeBufServer) { + t.Helper() + listener := bufconn.Listen(1 << 20) + server := grpc.NewServer() + service := &resumeBufServer{} + mecatlv1.RegisterHarnessServiceServer(server, service) + go func() { _ = server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + _ = listener.Close() + }) + conn, err := grpc.NewClient("passthrough:///resume-bufconn", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { return listener.Dial() }), + ) + if err != nil { + t.Fatalf("new bufconn client: %v", err) + } + t.Cleanup(func() { _ = conn.Close() }) + return resumeGRPCConverser{svc: mecatlv1.NewHarnessServiceClient(conn)}, service +} + +func TestCompletedResumeFirstStatusFailureThenManualSubmissionWorks(t *testing.T) { + const prompt = "continue after one-shot conflict" + converser, service := newResumeBufConverser(t) + service.failFirst = true + progress := newProgress() + model := newTestModelFromDeps(Deps{ + Conv: converser, Theme: theme.New("aztec", theme.AztecPalette()), Ctx: context.Background(), + Workspace: "/workspace", Resume: startupSelection("completed-session", "completed"), InitialPrompt: prompt, + NoAltScreen: true, emojiCapable: func() bool { return false }, kittyCapable: func() bool { return false }, + scrollKeysMarking: func() string { return "pgup/pgdn" }, onPhase: progress.record, + }) + tm := teatest.NewTestModel(t, model, teatest.WithInitialTermSize(100, 30)) + progress.wait(t, phaseReplay, 5*time.Second) + + // Back returns to the same adopted chat. Re-enter the prompt as an ordinary + // operator submission; the second stream succeeds without any transport reorder. + tm.Send(tea.KeyPressMsg{Code: tea.KeyEsc}) + tm.Send(tea.KeyPressMsg{Code: 'u', Mod: tea.ModCtrl}) + tm.Type(prompt) + tm.Send(tea.KeyPressMsg{Code: tea.KeyEnter}) + progress.waitRunComplete(t, 2, 5*time.Second) + + _, prompts := service.snapshot() + if len(prompts) != 2 || prompts[0] != prompt || prompts[1] != prompt { + t.Fatalf("automatic/manual prompts = %q, want two identical submissions", prompts) + } + + tm.Send(tea.KeyPressMsg{Code: 'c', Mod: tea.ModCtrl}) + tm.Send(tea.KeyPressMsg{Code: 'c', Mod: tea.ModCtrl}) + tm.WaitFinished(t, teatest.WithFinalTimeout(scaleWait(3*time.Second))) + final := tm.FinalModel(t).(Model) + if final.phase != phaseIdle || final.sessionID != "completed-session" || final.startupRunEntryFailed { + t.Fatalf("final retry state: phase=%v session=%q failed=%t", final.phase, final.sessionID, final.startupRunEntryFailed) + } +} + +func TestCompletedResumeAutomaticAndManualUseSameGRPCPath(t *testing.T) { + const prompt = "continue the completed chat" + for _, automatic := range []bool{true, false} { + name := "manual" + if automatic { + name = "automatic" + } + t.Run(name, func(t *testing.T) { + converser, service := newResumeBufConverser(t) + progress := newProgress() + initial := "" + if automatic { + initial = prompt + } + model := newTestModelFromDeps(Deps{ + Conv: converser, Theme: theme.New("aztec", theme.AztecPalette()), Ctx: context.Background(), + Workspace: "/workspace", Resume: startupSelection("completed-session", "completed"), InitialPrompt: initial, + NoAltScreen: true, emojiCapable: func() bool { return false }, kittyCapable: func() bool { return false }, + scrollKeysMarking: func() string { return "pgup/pgdn" }, onPhase: progress.record, + }) + tm := teatest.NewTestModel(t, model, teatest.WithInitialTermSize(100, 30)) + if !automatic { + progress.wait(t, phaseIdle, 3*time.Second) + tm.Type(prompt) + tm.Send(tea.KeyPressMsg{Code: tea.KeyEnter}) + } + progress.waitRunComplete(t, 1, 5*time.Second) + + order, prompts := service.snapshot() + if len(prompts) != 1 || prompts[0] != prompt { + t.Fatalf("server prompts = %q, want [%q]", prompts, prompt) + } + if len(order) < 2 || order[0] != "recv-before-send" || order[1] != "prompt-received" { + t.Fatalf("first-frame order = %q, want Recv waiting before prompt Send", order) + } + + tm.Send(tea.KeyPressMsg{Code: 'c', Mod: tea.ModCtrl}) + tm.Send(tea.KeyPressMsg{Code: 'c', Mod: tea.ModCtrl}) + tm.WaitFinished(t, teatest.WithFinalTimeout(scaleWait(3*time.Second))) + final := tm.FinalModel(t).(Model) + if final.phase != phaseIdle || final.sessionID != "completed-session" || final.startupRunEntryFailed { + t.Fatalf("final resume state: phase=%v session=%q failed=%t", final.phase, final.sessionID, final.startupRunEntryFailed) + } + }) + } +} diff --git a/cmd/mecatui/ui/update.go b/cmd/mecatui/ui/update.go index 69abb4a43e..a8f0b499d1 100644 --- a/cmd/mecatui/ui/update.go +++ b/cmd/mecatui/ui/update.go @@ -457,6 +457,9 @@ func (m Model) update(msg tea.Msg) (tea.Model, tea.Cmd) { // overlays accrue (each helper returns handled=false for a non-matching msg, so // at most one consumes). func (m Model) dispatchNonInputMsg(msg tea.Msg) (tea.Model, tea.Cmd) { + if mm, cmd, handled := m.updatePendingApproval(msg); handled { + return mm, cmd + } // The open modal surface routes every non-key/wheel Msg through m.modal // BEFORE the Model's generic reducer; handled=false falls through to the // rest of the chain. @@ -1019,7 +1022,7 @@ func (m Model) updateLifecycle(msg tea.Msg) (tea.Model, tea.Cmd, bool) { return mm, cmd, true } if m.startupFirstPromptPending { - m = m.failStartupRunEntry() + m = m.failStartupRunEntry(msg.Err) return m, nil, true } if m.failedStepRetryRun && !m.failedStepRetryAuthoritative { @@ -2224,30 +2227,8 @@ func (m Model) onKey(msg tea.KeyPressMsg) (tea.Model, tea.Cmd) { m.doubleEscapeReleased = false } - // Global lifecycle controls retain precedence over every overlay. - if key.Matches(msg, m.keys.Quit) { - return m.onQuitKey() - } - - // ctrl+d is the unix EOF-habit quit (issue #504), an INDEPENDENT second guard. - // It sits right after Quit and BEFORE the textarea/phase routing so the chord is - // intercepted everywhere — but only when the prompt textarea is EMPTY. On a - // populated prompt we deliberately do NOT consume it, so the chord falls through - // to the textarea's DeleteCharacterForward (its default bubble binding) — never a - // surprise quit mid-draft. Handled even when an overlay/modal owns the keyboard - // (the textarea is blurred and empty then, so the empty gate passes). - if key.Matches(msg, m.keys.QuitD) && strings.TrimSpace(m.prompt.Value()) == "" { - return m.onQuitDKey() - } - - // ctrl+z suspends the whole TUI process to the shell (issue #504). Handled - // BEFORE the phase/overlay routing so it works in EVERY state — idle, mid-run, - // the permission modal (the ask stays pending and durable). onSuspend writes the - // leave-behind notice to stderr, then returns tea.Suspend. Suspending does NOT - // stop the embedded mecated or an in-flight run — they keep working and the UI - // re-syncs on resume. - if key.Matches(msg, m.keys.Suspend) { - return m.onSuspend() + if mm, cmd, handled := m.onGlobalLifecycleKey(msg); handled { + return mm, cmd } // Help owns the remaining keys while open: its documented navigation and close @@ -2318,6 +2299,33 @@ func (m Model) onKey(msg tea.KeyPressMsg) (tea.Model, tea.Cmd) { return m.dispatchPhaseKey(msg) } +func (m Model) onGlobalLifecycleKey(msg tea.KeyPressMsg) (tea.Model, tea.Cmd, bool) { + // Global lifecycle controls retain precedence over every overlay. + if key.Matches(msg, m.keys.Quit) { + mm, cmd := m.onQuitKey() + return mm, cmd, true + } + // A populated prompt sends ctrl+d through to DeleteCharacterForward rather + // than allowing the independent EOF-habit quit guard to consume it. + if key.Matches(msg, m.keys.QuitD) && strings.TrimSpace(m.prompt.Value()) == "" { + mm, cmd := m.onQuitDKey() + return mm, cmd, true + } + if key.Matches(msg, m.keys.Suspend) { + mm, cmd := m.onSuspend() + return mm, cmd, true + } + return m.onPendingApprovalRecoveryKey(msg) +} + +func (m Model) onPendingApprovalRecoveryKey(msg tea.KeyPressMsg) (tea.Model, tea.Cmd, bool) { + if m.pendingRecovery == nil || m.pendingRecovery.resolving || m.pendingRecovery.resolved || !key.Matches(msg, m.keys.Close) { + return m, nil, false + } + (&m).retirePendingApprovalRecovery() + return m, tea.Quit, true +} + // onHelpKey handles the help overlay's complete keyboard contract. It runs before // phase routing, so navigation never reaches the conversation and every other key // remains swallowed. @@ -2459,7 +2467,7 @@ func (m Model) dispatchSurfaceKey(msg tea.KeyPressMsg) (tea.Model, tea.Cmd, bool // A clear command composed while running stays actionable if a permission ask // opens before Enter is pressed. Route it ahead of the approval surface: // ClearSession cancels the ask and must never become Allow/Deny/Learn. - if clearCommandSubmitted(msg, m.keys.Submit, m.prompt.Value()) { + if m.pendingRecovery == nil && clearCommandSubmitted(msg, m.keys.Submit, m.prompt.Value()) { return m.dispatchBareBuiltin(m.prompt.Value()) } cmd, handled, closed := m.modal.HandleKey(msg) @@ -2673,6 +2681,7 @@ func (m Model) retryPendingModeCmd() tea.Cmd { // shows the hint, and schedules the timed disarm. See onKey's doc for the rationale. func (m Model) quitNow() (tea.Model, tea.Cmd) { m.admissionSubmission = nil + (&m).retirePendingApprovalRecovery() if m.cancelRun != nil { m.cancelRun() } @@ -2682,15 +2691,21 @@ func (m Model) quitNow() (tea.Model, tea.Cmd) { // onQuitKey implements the guarded ctrl+c exit behavior. The immediate exit is // shared with /quit so both paths cancel an active run before Bubble Tea exits. func (m Model) onQuitKey() (tea.Model, tea.Cmd) { + if m.pendingRecovery != nil && m.pendingRecovery.resolved && !m.pendingRecovery.cancelled { + cmd := (&m).cancelPendingApprovalCmd() + m.statusMsg = "cancelling recovered run…" + m.refreshView() + return m, cmd + } // Already armed → a second ctrl+c within the window: quit now. The fatal // (dead-connection) screen also exits on a single press — there is no input to // clear and no run to protect, so the guard would only add friction. if m.quitArmed || m.phase == phaseFatal { return m.quitNow() } - // First ctrl+c with staged input: clear the input (mirrors esc's clear-the-line) - // and do NOT arm — a single press to wipe a draft is expected. - if strings.TrimSpace(m.prompt.Value()) != "" { + // Ordinary staged input clears on the first press. An in-flight recovery + // choice instead keeps its draft and arms exit with the uncertainty warning. + if strings.TrimSpace(m.prompt.Value()) != "" && (m.pendingRecovery == nil || !m.pendingRecovery.resolving) { m.prompt.Reset() m.pendingPromptMedia = client.MediaResult{} return m.afterInputEdit(nil) @@ -2700,6 +2715,9 @@ func (m Model) onQuitKey() (tea.Model, tea.Cmd) { m.quitArmed = true m.quitArmGen++ m.statusMsg = quitHintFor(m.keys.Quit) + if m.pendingRecovery != nil && m.pendingRecovery.resolving { + m.statusMsg += " — choice submitted; exiting cannot undo it, outcome may be unknown" + } m.refreshView() return m, m.quitDisarmCmd(m.quitArmGen) } @@ -3472,16 +3490,23 @@ func (m Model) afterInputEdit(cmd tea.Cmd) (tea.Model, tea.Cmd) { return mm, tea.Batch(cmd, fetch) } -var errStartupRunEntry = &sessionTranscriptError{"the chat could not be attached for a new turn"} +type startupRunEntryError struct{ title, text string } + +func (e *startupRunEntryError) Error() string { return e.text } +func (e *startupRunEntryError) SafeTranscriptText() string { return e.text } +func (e *startupRunEntryError) SafeTranscriptTitle() string { return e.title } -func (m Model) failStartupRunEntry() Model { +func (m Model) failStartupRunEntry(err error) Model { m = m.endRun("") m = m.resetDocumentProjection() m.conv = conversationFromTranscript(m.deps.Resume.Transcript.Messages) m.modal = &sessionsState{ - selected: m.deps.Resume.Row, - inspect: true, - loadErr: errStartupRunEntry, + selected: m.deps.Resume.Row, + inspect: true, + loadErr: &startupRunEntryError{ + title: client.SafeStartupRunEntryErrorTitle(err), + text: client.SafeStartupRunEntryError(err), + }, view: sessionsTranscript, deps: (&m).surfaceDeps(), activeSessionID: m.sessionID, diff --git a/contracts/gen/go/mecatl/execution/v1/execution.pb.go b/contracts/gen/go/mecatl/execution/v1/execution.pb.go new file mode 100644 index 0000000000..83f59d3ff6 --- /dev/null +++ b/contracts/gen/go/mecatl/execution/v1/execution.pb.go @@ -0,0 +1,3684 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: LicenseRef-Stacklok-Proprietary + +// Private, mutually authenticated execution-provider protocol. This is not part +// of the public Harness API. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.4 +// protoc (unknown) +// source: mecatl/execution/v1/execution.proto + +package executionv1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + emptypb "google.golang.org/protobuf/types/known/emptypb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type FileOperation int32 + +const ( + FileOperation_FILE_OPERATION_UNSPECIFIED FileOperation = 0 + FileOperation_FILE_OPERATION_READ FileOperation = 1 + FileOperation_FILE_OPERATION_RESOLVE_AUTHORITY FileOperation = 2 + FileOperation_FILE_OPERATION_STAT FileOperation = 3 + FileOperation_FILE_OPERATION_CREATE FileOperation = 4 + FileOperation_FILE_OPERATION_REPLACE FileOperation = 5 + FileOperation_FILE_OPERATION_LIST FileOperation = 6 + FileOperation_FILE_OPERATION_REMOVE FileOperation = 7 + FileOperation_FILE_OPERATION_RENAME FileOperation = 8 + FileOperation_FILE_OPERATION_COPY FileOperation = 9 + FileOperation_FILE_OPERATION_GLOB FileOperation = 10 + FileOperation_FILE_OPERATION_GREP FileOperation = 11 +) + +// Enum value maps for FileOperation. +var ( + FileOperation_name = map[int32]string{ + 0: "FILE_OPERATION_UNSPECIFIED", + 1: "FILE_OPERATION_READ", + 2: "FILE_OPERATION_RESOLVE_AUTHORITY", + 3: "FILE_OPERATION_STAT", + 4: "FILE_OPERATION_CREATE", + 5: "FILE_OPERATION_REPLACE", + 6: "FILE_OPERATION_LIST", + 7: "FILE_OPERATION_REMOVE", + 8: "FILE_OPERATION_RENAME", + 9: "FILE_OPERATION_COPY", + 10: "FILE_OPERATION_GLOB", + 11: "FILE_OPERATION_GREP", + } + FileOperation_value = map[string]int32{ + "FILE_OPERATION_UNSPECIFIED": 0, + "FILE_OPERATION_READ": 1, + "FILE_OPERATION_RESOLVE_AUTHORITY": 2, + "FILE_OPERATION_STAT": 3, + "FILE_OPERATION_CREATE": 4, + "FILE_OPERATION_REPLACE": 5, + "FILE_OPERATION_LIST": 6, + "FILE_OPERATION_REMOVE": 7, + "FILE_OPERATION_RENAME": 8, + "FILE_OPERATION_COPY": 9, + "FILE_OPERATION_GLOB": 10, + "FILE_OPERATION_GREP": 11, + } +) + +func (x FileOperation) Enum() *FileOperation { + p := new(FileOperation) + *p = x + return p +} + +func (x FileOperation) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FileOperation) Descriptor() protoreflect.EnumDescriptor { + return file_mecatl_execution_v1_execution_proto_enumTypes[0].Descriptor() +} + +func (FileOperation) Type() protoreflect.EnumType { + return &file_mecatl_execution_v1_execution_proto_enumTypes[0] +} + +func (x FileOperation) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use FileOperation.Descriptor instead. +func (FileOperation) EnumDescriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{0} +} + +type CommandState int32 + +const ( + CommandState_COMMAND_STATE_UNSPECIFIED CommandState = 0 + CommandState_COMMAND_STATE_RUNNING CommandState = 1 + CommandState_COMMAND_STATE_SUCCEEDED CommandState = 2 + CommandState_COMMAND_STATE_FAILED CommandState = 3 + CommandState_COMMAND_STATE_CANCELLED CommandState = 4 + CommandState_COMMAND_STATE_FENCE_UNKNOWN CommandState = 5 +) + +// Enum value maps for CommandState. +var ( + CommandState_name = map[int32]string{ + 0: "COMMAND_STATE_UNSPECIFIED", + 1: "COMMAND_STATE_RUNNING", + 2: "COMMAND_STATE_SUCCEEDED", + 3: "COMMAND_STATE_FAILED", + 4: "COMMAND_STATE_CANCELLED", + 5: "COMMAND_STATE_FENCE_UNKNOWN", + } + CommandState_value = map[string]int32{ + "COMMAND_STATE_UNSPECIFIED": 0, + "COMMAND_STATE_RUNNING": 1, + "COMMAND_STATE_SUCCEEDED": 2, + "COMMAND_STATE_FAILED": 3, + "COMMAND_STATE_CANCELLED": 4, + "COMMAND_STATE_FENCE_UNKNOWN": 5, + } +) + +func (x CommandState) Enum() *CommandState { + p := new(CommandState) + *p = x + return p +} + +func (x CommandState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (CommandState) Descriptor() protoreflect.EnumDescriptor { + return file_mecatl_execution_v1_execution_proto_enumTypes[1].Descriptor() +} + +func (CommandState) Type() protoreflect.EnumType { + return &file_mecatl_execution_v1_execution_proto_enumTypes[1] +} + +func (x CommandState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use CommandState.Descriptor instead. +func (CommandState) EnumDescriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{1} +} + +type EnvironmentRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Revision string `protobuf:"bytes,2,opt,name=revision,proto3" json:"revision,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnvironmentRef) Reset() { + *x = EnvironmentRef{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnvironmentRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnvironmentRef) ProtoMessage() {} + +func (x *EnvironmentRef) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnvironmentRef.ProtoReflect.Descriptor instead. +func (*EnvironmentRef) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{0} +} + +func (x *EnvironmentRef) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *EnvironmentRef) GetRevision() string { + if x != nil { + return x.Revision + } + return "" +} + +type Owner struct { + state protoimpl.MessageState `protogen:"open.v1"` + Issuer string `protobuf:"bytes,1,opt,name=issuer,proto3" json:"issuer,omitempty"` + Subject string `protobuf:"bytes,2,opt,name=subject,proto3" json:"subject,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Owner) Reset() { + *x = Owner{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Owner) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Owner) ProtoMessage() {} + +func (x *Owner) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Owner.ProtoReflect.Descriptor instead. +func (*Owner) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{1} +} + +func (x *Owner) GetIssuer() string { + if x != nil { + return x.Issuer + } + return "" +} + +func (x *Owner) GetSubject() string { + if x != nil { + return x.Subject + } + return "" +} + +type RequestContext struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + Epoch uint64 `protobuf:"varint,4,opt,name=epoch,proto3" json:"epoch,omitempty"` + Grant string `protobuf:"bytes,5,opt,name=grant,proto3" json:"grant,omitempty"` + RunId string `protobuf:"bytes,6,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ClaimId string `protobuf:"bytes,7,opt,name=claim_id,json=claimId,proto3" json:"claim_id,omitempty"` + GrantGeneration uint64 `protobuf:"varint,8,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestContext) Reset() { + *x = RequestContext{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestContext) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestContext) ProtoMessage() {} + +func (x *RequestContext) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestContext.ProtoReflect.Descriptor instead. +func (*RequestContext) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{2} +} + +func (x *RequestContext) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RequestContext) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RequestContext) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *RequestContext) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *RequestContext) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *RequestContext) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *RequestContext) GetClaimId() string { + if x != nil { + return x.ClaimId + } + return "" +} + +func (x *RequestContext) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type ValidateProfileRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Profile string `protobuf:"bytes,1,opt,name=profile,proto3" json:"profile,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ValidateProfileRequest) Reset() { + *x = ValidateProfileRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ValidateProfileRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ValidateProfileRequest) ProtoMessage() {} + +func (x *ValidateProfileRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ValidateProfileRequest.ProtoReflect.Descriptor instead. +func (*ValidateProfileRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{3} +} + +func (x *ValidateProfileRequest) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +type ValidateProfileResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Profile string `protobuf:"bytes,1,opt,name=profile,proto3" json:"profile,omitempty"` + Digest string `protobuf:"bytes,2,opt,name=digest,proto3" json:"digest,omitempty"` + Capabilities []string `protobuf:"bytes,3,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + MaxFileBytes int64 `protobuf:"varint,4,opt,name=max_file_bytes,json=maxFileBytes,proto3" json:"max_file_bytes,omitempty"` + MaxCommandBytes int64 `protobuf:"varint,5,opt,name=max_command_bytes,json=maxCommandBytes,proto3" json:"max_command_bytes,omitempty"` + MaxCommandDurationMillis int64 `protobuf:"varint,6,opt,name=max_command_duration_millis,json=maxCommandDurationMillis,proto3" json:"max_command_duration_millis,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ValidateProfileResponse) Reset() { + *x = ValidateProfileResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ValidateProfileResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ValidateProfileResponse) ProtoMessage() {} + +func (x *ValidateProfileResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ValidateProfileResponse.ProtoReflect.Descriptor instead. +func (*ValidateProfileResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{4} +} + +func (x *ValidateProfileResponse) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +func (x *ValidateProfileResponse) GetDigest() string { + if x != nil { + return x.Digest + } + return "" +} + +func (x *ValidateProfileResponse) GetCapabilities() []string { + if x != nil { + return x.Capabilities + } + return nil +} + +func (x *ValidateProfileResponse) GetMaxFileBytes() int64 { + if x != nil { + return x.MaxFileBytes + } + return 0 +} + +func (x *ValidateProfileResponse) GetMaxCommandBytes() int64 { + if x != nil { + return x.MaxCommandBytes + } + return 0 +} + +func (x *ValidateProfileResponse) GetMaxCommandDurationMillis() int64 { + if x != nil { + return x.MaxCommandDurationMillis + } + return 0 +} + +type EnsureEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + BindingId string `protobuf:"bytes,1,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + Profile string `protobuf:"bytes,2,opt,name=profile,proto3" json:"profile,omitempty"` + Owner *Owner `protobuf:"bytes,3,opt,name=owner,proto3" json:"owner,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnsureEnvironmentRequest) Reset() { + *x = EnsureEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnsureEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnsureEnvironmentRequest) ProtoMessage() {} + +func (x *EnsureEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnsureEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*EnsureEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{5} +} + +func (x *EnsureEnvironmentRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *EnsureEnvironmentRequest) GetProfile() string { + if x != nil { + return x.Profile + } + return "" +} + +func (x *EnsureEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *EnsureEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type EnsureEnvironmentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` + Ready bool `protobuf:"varint,3,opt,name=ready,proto3" json:"ready,omitempty"` + Grant string `protobuf:"bytes,4,opt,name=grant,proto3" json:"grant,omitempty"` + GrantExpiresAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=grant_expires_at,json=grantExpiresAt,proto3" json:"grant_expires_at,omitempty"` + GrantGeneration uint64 `protobuf:"varint,6,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnsureEnvironmentResponse) Reset() { + *x = EnsureEnvironmentResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnsureEnvironmentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnsureEnvironmentResponse) ProtoMessage() {} + +func (x *EnsureEnvironmentResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnsureEnvironmentResponse.ProtoReflect.Descriptor instead. +func (*EnsureEnvironmentResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{6} +} + +func (x *EnsureEnvironmentResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *EnsureEnvironmentResponse) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *EnsureEnvironmentResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *EnsureEnvironmentResponse) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *EnsureEnvironmentResponse) GetGrantExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.GrantExpiresAt + } + return nil +} + +func (x *EnsureEnvironmentResponse) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type AttachEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + Purpose string `protobuf:"bytes,2,opt,name=purpose,proto3" json:"purpose,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttachEnvironmentRequest) Reset() { + *x = AttachEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttachEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttachEnvironmentRequest) ProtoMessage() {} + +func (x *AttachEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttachEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*AttachEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{7} +} + +func (x *AttachEnvironmentRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *AttachEnvironmentRequest) GetPurpose() string { + if x != nil { + return x.Purpose + } + return "" +} + +type AttachEnvironmentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Epoch uint64 `protobuf:"varint,2,opt,name=epoch,proto3" json:"epoch,omitempty"` + Ready bool `protobuf:"varint,3,opt,name=ready,proto3" json:"ready,omitempty"` + Grant string `protobuf:"bytes,4,opt,name=grant,proto3" json:"grant,omitempty"` + GrantExpiresAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=grant_expires_at,json=grantExpiresAt,proto3" json:"grant_expires_at,omitempty"` + GrantGeneration uint64 `protobuf:"varint,6,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AttachEnvironmentResponse) Reset() { + *x = AttachEnvironmentResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AttachEnvironmentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AttachEnvironmentResponse) ProtoMessage() {} + +func (x *AttachEnvironmentResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AttachEnvironmentResponse.ProtoReflect.Descriptor instead. +func (*AttachEnvironmentResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{8} +} + +func (x *AttachEnvironmentResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *AttachEnvironmentResponse) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *AttachEnvironmentResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *AttachEnvironmentResponse) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *AttachEnvironmentResponse) GetGrantExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.GrantExpiresAt + } + return nil +} + +func (x *AttachEnvironmentResponse) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type AcquireRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + RunId string `protobuf:"bytes,4,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + OperationId string `protobuf:"bytes,5,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + TtlMillis int64 `protobuf:"varint,6,opt,name=ttl_millis,json=ttlMillis,proto3" json:"ttl_millis,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AcquireRunRequest) Reset() { + *x = AcquireRunRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AcquireRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AcquireRunRequest) ProtoMessage() {} + +func (x *AcquireRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AcquireRunRequest.ProtoReflect.Descriptor instead. +func (*AcquireRunRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{9} +} + +func (x *AcquireRunRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *AcquireRunRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *AcquireRunRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *AcquireRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *AcquireRunRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +func (x *AcquireRunRequest) GetTtlMillis() int64 { + if x != nil { + return x.TtlMillis + } + return 0 +} + +type RenewRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + RunId string `protobuf:"bytes,4,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ClaimId string `protobuf:"bytes,5,opt,name=claim_id,json=claimId,proto3" json:"claim_id,omitempty"` + Epoch uint64 `protobuf:"varint,6,opt,name=epoch,proto3" json:"epoch,omitempty"` + OperationId string `protobuf:"bytes,7,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + TtlMillis int64 `protobuf:"varint,8,opt,name=ttl_millis,json=ttlMillis,proto3" json:"ttl_millis,omitempty"` + GrantGeneration uint64 `protobuf:"varint,9,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RenewRunRequest) Reset() { + *x = RenewRunRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RenewRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RenewRunRequest) ProtoMessage() {} + +func (x *RenewRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RenewRunRequest.ProtoReflect.Descriptor instead. +func (*RenewRunRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{10} +} + +func (x *RenewRunRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RenewRunRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RenewRunRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *RenewRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *RenewRunRequest) GetClaimId() string { + if x != nil { + return x.ClaimId + } + return "" +} + +func (x *RenewRunRequest) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *RenewRunRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +func (x *RenewRunRequest) GetTtlMillis() int64 { + if x != nil { + return x.TtlMillis + } + return 0 +} + +func (x *RenewRunRequest) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type ReleaseRunRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + RunId string `protobuf:"bytes,4,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ClaimId string `protobuf:"bytes,5,opt,name=claim_id,json=claimId,proto3" json:"claim_id,omitempty"` + Epoch uint64 `protobuf:"varint,6,opt,name=epoch,proto3" json:"epoch,omitempty"` + OperationId string `protobuf:"bytes,7,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + GrantGeneration uint64 `protobuf:"varint,8,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReleaseRunRequest) Reset() { + *x = ReleaseRunRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReleaseRunRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReleaseRunRequest) ProtoMessage() {} + +func (x *ReleaseRunRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReleaseRunRequest.ProtoReflect.Descriptor instead. +func (*ReleaseRunRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{11} +} + +func (x *ReleaseRunRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReleaseRunRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ReleaseRunRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *ReleaseRunRequest) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *ReleaseRunRequest) GetClaimId() string { + if x != nil { + return x.ClaimId + } + return "" +} + +func (x *ReleaseRunRequest) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *ReleaseRunRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +func (x *ReleaseRunRequest) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type RunClaimResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + BindingId string `protobuf:"bytes,2,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + RunId string `protobuf:"bytes,3,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` + ClaimId string `protobuf:"bytes,4,opt,name=claim_id,json=claimId,proto3" json:"claim_id,omitempty"` + Epoch uint64 `protobuf:"varint,5,opt,name=epoch,proto3" json:"epoch,omitempty"` + GrantGeneration uint64 `protobuf:"varint,6,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + Grant string `protobuf:"bytes,7,opt,name=grant,proto3" json:"grant,omitempty"` + ExpiresAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RunClaimResponse) Reset() { + *x = RunClaimResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RunClaimResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunClaimResponse) ProtoMessage() {} + +func (x *RunClaimResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RunClaimResponse.ProtoReflect.Descriptor instead. +func (*RunClaimResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{12} +} + +func (x *RunClaimResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RunClaimResponse) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *RunClaimResponse) GetRunId() string { + if x != nil { + return x.RunId + } + return "" +} + +func (x *RunClaimResponse) GetClaimId() string { + if x != nil { + return x.ClaimId + } + return "" +} + +func (x *RunClaimResponse) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *RunClaimResponse) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +func (x *RunClaimResponse) GetGrant() string { + if x != nil { + return x.Grant + } + return "" +} + +func (x *RunClaimResponse) GetExpiresAt() *timestamppb.Timestamp { + if x != nil { + return x.ExpiresAt + } + return nil +} + +type ReferenceMutationRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + BindingId string `protobuf:"bytes,3,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReferenceMutationRequest) Reset() { + *x = ReferenceMutationRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReferenceMutationRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReferenceMutationRequest) ProtoMessage() {} + +func (x *ReferenceMutationRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReferenceMutationRequest.ProtoReflect.Descriptor instead. +func (*ReferenceMutationRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{13} +} + +func (x *ReferenceMutationRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReferenceMutationRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ReferenceMutationRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *ReferenceMutationRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type ReserveSuccessorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + SourceBindingId string `protobuf:"bytes,3,opt,name=source_binding_id,json=sourceBindingId,proto3" json:"source_binding_id,omitempty"` + DestinationBindingId string `protobuf:"bytes,4,opt,name=destination_binding_id,json=destinationBindingId,proto3" json:"destination_binding_id,omitempty"` + OperationId string `protobuf:"bytes,5,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReserveSuccessorRequest) Reset() { + *x = ReserveSuccessorRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReserveSuccessorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReserveSuccessorRequest) ProtoMessage() {} + +func (x *ReserveSuccessorRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReserveSuccessorRequest.ProtoReflect.Descriptor instead. +func (*ReserveSuccessorRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{14} +} + +func (x *ReserveSuccessorRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReserveSuccessorRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ReserveSuccessorRequest) GetSourceBindingId() string { + if x != nil { + return x.SourceBindingId + } + return "" +} + +func (x *ReserveSuccessorRequest) GetDestinationBindingId() string { + if x != nil { + return x.DestinationBindingId + } + return "" +} + +func (x *ReserveSuccessorRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type ReferenceReservationResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReferenceReservationResponse) Reset() { + *x = ReferenceReservationResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReferenceReservationResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReferenceReservationResponse) ProtoMessage() {} + +func (x *ReferenceReservationResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReferenceReservationResponse.ProtoReflect.Descriptor instead. +func (*ReferenceReservationResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{15} +} + +func (x *ReferenceReservationResponse) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +type ListReferenceIntentsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Owner *Owner `protobuf:"bytes,1,opt,name=owner,proto3" json:"owner,omitempty"` + Limit int32 `protobuf:"varint,2,opt,name=limit,proto3" json:"limit,omitempty"` + Environment *EnvironmentRef `protobuf:"bytes,3,opt,name=environment,proto3" json:"environment,omitempty"` + BindingId string `protobuf:"bytes,4,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListReferenceIntentsRequest) Reset() { + *x = ListReferenceIntentsRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListReferenceIntentsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListReferenceIntentsRequest) ProtoMessage() {} + +func (x *ListReferenceIntentsRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListReferenceIntentsRequest.ProtoReflect.Descriptor instead. +func (*ListReferenceIntentsRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{16} +} + +func (x *ListReferenceIntentsRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ListReferenceIntentsRequest) GetLimit() int32 { + if x != nil { + return x.Limit + } + return 0 +} + +func (x *ListReferenceIntentsRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ListReferenceIntentsRequest) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +type ReferenceIntent struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + BindingId string `protobuf:"bytes,2,opt,name=binding_id,json=bindingId,proto3" json:"binding_id,omitempty"` + State string `protobuf:"bytes,3,opt,name=state,proto3" json:"state,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + SourceBindingId string `protobuf:"bytes,5,opt,name=source_binding_id,json=sourceBindingId,proto3" json:"source_binding_id,omitempty"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + Owner *Owner `protobuf:"bytes,7,opt,name=owner,proto3" json:"owner,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReferenceIntent) Reset() { + *x = ReferenceIntent{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReferenceIntent) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReferenceIntent) ProtoMessage() {} + +func (x *ReferenceIntent) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReferenceIntent.ProtoReflect.Descriptor instead. +func (*ReferenceIntent) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{17} +} + +func (x *ReferenceIntent) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReferenceIntent) GetBindingId() string { + if x != nil { + return x.BindingId + } + return "" +} + +func (x *ReferenceIntent) GetState() string { + if x != nil { + return x.State + } + return "" +} + +func (x *ReferenceIntent) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +func (x *ReferenceIntent) GetSourceBindingId() string { + if x != nil { + return x.SourceBindingId + } + return "" +} + +func (x *ReferenceIntent) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +func (x *ReferenceIntent) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +type ListReferenceIntentsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Intents []*ReferenceIntent `protobuf:"bytes,1,rep,name=intents,proto3" json:"intents,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListReferenceIntentsResponse) Reset() { + *x = ListReferenceIntentsResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListReferenceIntentsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListReferenceIntentsResponse) ProtoMessage() {} + +func (x *ListReferenceIntentsResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListReferenceIntentsResponse.ProtoReflect.Descriptor instead. +func (*ListReferenceIntentsResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{18} +} + +func (x *ListReferenceIntentsResponse) GetIntents() []*ReferenceIntent { + if x != nil { + return x.Intents + } + return nil +} + +type ReleaseReferenceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReleaseReferenceRequest) Reset() { + *x = ReleaseReferenceRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReleaseReferenceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReleaseReferenceRequest) ProtoMessage() {} + +func (x *ReleaseReferenceRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReleaseReferenceRequest.ProtoReflect.Descriptor instead. +func (*ReleaseReferenceRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{19} +} + +func (x *ReleaseReferenceRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +type RetireEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedExecutionEpoch uint64 `protobuf:"varint,3,opt,name=expected_execution_epoch,json=expectedExecutionEpoch,proto3" json:"expected_execution_epoch,omitempty"` + ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RetireEnvironmentRequest) Reset() { + *x = RetireEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RetireEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RetireEnvironmentRequest) ProtoMessage() {} + +func (x *RetireEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RetireEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*RetireEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{20} +} + +func (x *RetireEnvironmentRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RetireEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RetireEnvironmentRequest) GetExpectedExecutionEpoch() uint64 { + if x != nil { + return x.ExpectedExecutionEpoch + } + return 0 +} + +func (x *RetireEnvironmentRequest) GetExpectedPodUid() string { + if x != nil { + return x.ExpectedPodUid + } + return "" +} + +func (x *RetireEnvironmentRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid + } + return "" +} + +func (x *RetireEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type ReplaceExecutorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedExecutionEpoch uint64 `protobuf:"varint,3,opt,name=expected_execution_epoch,json=expectedExecutionEpoch,proto3" json:"expected_execution_epoch,omitempty"` + ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ReplaceExecutorRequest) Reset() { + *x = ReplaceExecutorRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ReplaceExecutorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReplaceExecutorRequest) ProtoMessage() {} + +func (x *ReplaceExecutorRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReplaceExecutorRequest.ProtoReflect.Descriptor instead. +func (*ReplaceExecutorRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{21} +} + +func (x *ReplaceExecutorRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *ReplaceExecutorRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *ReplaceExecutorRequest) GetExpectedExecutionEpoch() uint64 { + if x != nil { + return x.ExpectedExecutionEpoch + } + return 0 +} + +func (x *ReplaceExecutorRequest) GetExpectedPodUid() string { + if x != nil { + return x.ExpectedPodUid + } + return "" +} + +func (x *ReplaceExecutorRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid + } + return "" +} + +func (x *ReplaceExecutorRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type RecoverEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedExecutionEpoch uint64 `protobuf:"varint,3,opt,name=expected_execution_epoch,json=expectedExecutionEpoch,proto3" json:"expected_execution_epoch,omitempty"` + ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RecoverEnvironmentRequest) Reset() { + *x = RecoverEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RecoverEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RecoverEnvironmentRequest) ProtoMessage() {} + +func (x *RecoverEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RecoverEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*RecoverEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{22} +} + +func (x *RecoverEnvironmentRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RecoverEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RecoverEnvironmentRequest) GetExpectedExecutionEpoch() uint64 { + if x != nil { + return x.ExpectedExecutionEpoch + } + return 0 +} + +func (x *RecoverEnvironmentRequest) GetExpectedPodUid() string { + if x != nil { + return x.ExpectedPodUid + } + return "" +} + +func (x *RecoverEnvironmentRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid + } + return "" +} + +func (x *RecoverEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type DeleteRetiredEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,3,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteRetiredEnvironmentRequest) Reset() { + *x = DeleteRetiredEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteRetiredEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteRetiredEnvironmentRequest) ProtoMessage() {} + +func (x *DeleteRetiredEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteRetiredEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*DeleteRetiredEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{23} +} + +func (x *DeleteRetiredEnvironmentRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *DeleteRetiredEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *DeleteRetiredEnvironmentRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid + } + return "" +} + +func (x *DeleteRetiredEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type RevokeEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedGrantGeneration uint64 `protobuf:"varint,3,opt,name=expected_grant_generation,json=expectedGrantGeneration,proto3" json:"expected_grant_generation,omitempty"` + OperationId string `protobuf:"bytes,4,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RevokeEnvironmentRequest) Reset() { + *x = RevokeEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RevokeEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RevokeEnvironmentRequest) ProtoMessage() {} + +func (x *RevokeEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RevokeEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*RevokeEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{24} +} + +func (x *RevokeEnvironmentRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *RevokeEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *RevokeEnvironmentRequest) GetExpectedGrantGeneration() uint64 { + if x != nil { + return x.ExpectedGrantGeneration + } + return 0 +} + +func (x *RevokeEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type RevokeEnvironmentResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + GrantGeneration uint64 `protobuf:"varint,1,opt,name=grant_generation,json=grantGeneration,proto3" json:"grant_generation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RevokeEnvironmentResponse) Reset() { + *x = RevokeEnvironmentResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RevokeEnvironmentResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RevokeEnvironmentResponse) ProtoMessage() {} + +func (x *RevokeEnvironmentResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RevokeEnvironmentResponse.ProtoReflect.Descriptor instead. +func (*RevokeEnvironmentResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{25} +} + +func (x *RevokeEnvironmentResponse) GetGrantGeneration() uint64 { + if x != nil { + return x.GrantGeneration + } + return 0 +} + +type MigrateEnvironmentRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Environment *EnvironmentRef `protobuf:"bytes,1,opt,name=environment,proto3" json:"environment,omitempty"` + Owner *Owner `protobuf:"bytes,2,opt,name=owner,proto3" json:"owner,omitempty"` + ExpectedSchemaVersion *uint32 `protobuf:"varint,3,opt,name=expected_schema_version,json=expectedSchemaVersion,proto3,oneof" json:"expected_schema_version,omitempty"` + ExpectedPodUid string `protobuf:"bytes,4,opt,name=expected_pod_uid,json=expectedPodUid,proto3" json:"expected_pod_uid,omitempty"` + ExpectedPvcUid string `protobuf:"bytes,5,opt,name=expected_pvc_uid,json=expectedPvcUid,proto3" json:"expected_pvc_uid,omitempty"` + OperationId string `protobuf:"bytes,6,opt,name=operation_id,json=operationId,proto3" json:"operation_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MigrateEnvironmentRequest) Reset() { + *x = MigrateEnvironmentRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MigrateEnvironmentRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MigrateEnvironmentRequest) ProtoMessage() {} + +func (x *MigrateEnvironmentRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MigrateEnvironmentRequest.ProtoReflect.Descriptor instead. +func (*MigrateEnvironmentRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{26} +} + +func (x *MigrateEnvironmentRequest) GetEnvironment() *EnvironmentRef { + if x != nil { + return x.Environment + } + return nil +} + +func (x *MigrateEnvironmentRequest) GetOwner() *Owner { + if x != nil { + return x.Owner + } + return nil +} + +func (x *MigrateEnvironmentRequest) GetExpectedSchemaVersion() uint32 { + if x != nil && x.ExpectedSchemaVersion != nil { + return *x.ExpectedSchemaVersion + } + return 0 +} + +func (x *MigrateEnvironmentRequest) GetExpectedPodUid() string { + if x != nil { + return x.ExpectedPodUid + } + return "" +} + +func (x *MigrateEnvironmentRequest) GetExpectedPvcUid() string { + if x != nil { + return x.ExpectedPvcUid + } + return "" +} + +func (x *MigrateEnvironmentRequest) GetOperationId() string { + if x != nil { + return x.OperationId + } + return "" +} + +type FileRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + Operation FileOperation `protobuf:"varint,2,opt,name=operation,proto3,enum=mecatl.execution.v1.FileOperation" json:"operation,omitempty"` + Path string `protobuf:"bytes,3,opt,name=path,proto3" json:"path,omitempty"` + Destination string `protobuf:"bytes,4,opt,name=destination,proto3" json:"destination,omitempty"` + Pattern string `protobuf:"bytes,5,opt,name=pattern,proto3" json:"pattern,omitempty"` + Data []byte `protobuf:"bytes,6,opt,name=data,proto3" json:"data,omitempty"` + Version []byte `protobuf:"bytes,7,opt,name=version,proto3" json:"version,omitempty"` + Limit int32 `protobuf:"varint,8,opt,name=limit,proto3" json:"limit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileRequest) Reset() { + *x = FileRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileRequest) ProtoMessage() {} + +func (x *FileRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileRequest.ProtoReflect.Descriptor instead. +func (*FileRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{27} +} + +func (x *FileRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *FileRequest) GetOperation() FileOperation { + if x != nil { + return x.Operation + } + return FileOperation_FILE_OPERATION_UNSPECIFIED +} + +func (x *FileRequest) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *FileRequest) GetDestination() string { + if x != nil { + return x.Destination + } + return "" +} + +func (x *FileRequest) GetPattern() string { + if x != nil { + return x.Pattern + } + return "" +} + +func (x *FileRequest) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *FileRequest) GetVersion() []byte { + if x != nil { + return x.Version + } + return nil +} + +func (x *FileRequest) GetLimit() int32 { + if x != nil { + return x.Limit + } + return 0 +} + +type FileInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Size int64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + Mode uint32 `protobuf:"varint,3,opt,name=mode,proto3" json:"mode,omitempty"` + ModTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=mod_time,json=modTime,proto3" json:"mod_time,omitempty"` + IsDir bool `protobuf:"varint,5,opt,name=is_dir,json=isDir,proto3" json:"is_dir,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileInfo) Reset() { + *x = FileInfo{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileInfo) ProtoMessage() {} + +func (x *FileInfo) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[28] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileInfo.ProtoReflect.Descriptor instead. +func (*FileInfo) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{28} +} + +func (x *FileInfo) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *FileInfo) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *FileInfo) GetMode() uint32 { + if x != nil { + return x.Mode + } + return 0 +} + +func (x *FileInfo) GetModTime() *timestamppb.Timestamp { + if x != nil { + return x.ModTime + } + return nil +} + +func (x *FileInfo) GetIsDir() bool { + if x != nil { + return x.IsDir + } + return false +} + +type GrepMatch struct { + state protoimpl.MessageState `protogen:"open.v1"` + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` + Line int32 `protobuf:"varint,2,opt,name=line,proto3" json:"line,omitempty"` + Text string `protobuf:"bytes,3,opt,name=text,proto3" json:"text,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GrepMatch) Reset() { + *x = GrepMatch{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GrepMatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GrepMatch) ProtoMessage() {} + +func (x *GrepMatch) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[29] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GrepMatch.ProtoReflect.Descriptor instead. +func (*GrepMatch) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{29} +} + +func (x *GrepMatch) GetPath() string { + if x != nil { + return x.Path + } + return "" +} + +func (x *GrepMatch) GetLine() int32 { + if x != nil { + return x.Line + } + return 0 +} + +func (x *GrepMatch) GetText() string { + if x != nil { + return x.Text + } + return "" +} + +type FileResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + Version []byte `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Info *FileInfo `protobuf:"bytes,3,opt,name=info,proto3" json:"info,omitempty"` + Entries []*FileInfo `protobuf:"bytes,4,rep,name=entries,proto3" json:"entries,omitempty"` + Paths []string `protobuf:"bytes,5,rep,name=paths,proto3" json:"paths,omitempty"` + Matches []*GrepMatch `protobuf:"bytes,6,rep,name=matches,proto3" json:"matches,omitempty"` + AuthorityTarget string `protobuf:"bytes,7,opt,name=authority_target,json=authorityTarget,proto3" json:"authority_target,omitempty"` + AuthorityWorkspace string `protobuf:"bytes,8,opt,name=authority_workspace,json=authorityWorkspace,proto3" json:"authority_workspace,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileResponse) Reset() { + *x = FileResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileResponse) ProtoMessage() {} + +func (x *FileResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[30] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileResponse.ProtoReflect.Descriptor instead. +func (*FileResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{30} +} + +func (x *FileResponse) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *FileResponse) GetVersion() []byte { + if x != nil { + return x.Version + } + return nil +} + +func (x *FileResponse) GetInfo() *FileInfo { + if x != nil { + return x.Info + } + return nil +} + +func (x *FileResponse) GetEntries() []*FileInfo { + if x != nil { + return x.Entries + } + return nil +} + +func (x *FileResponse) GetPaths() []string { + if x != nil { + return x.Paths + } + return nil +} + +func (x *FileResponse) GetMatches() []*GrepMatch { + if x != nil { + return x.Matches + } + return nil +} + +func (x *FileResponse) GetAuthorityTarget() string { + if x != nil { + return x.AuthorityTarget + } + return "" +} + +func (x *FileResponse) GetAuthorityWorkspace() string { + if x != nil { + return x.AuthorityWorkspace + } + return "" +} + +type CommandStartRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + Command string `protobuf:"bytes,2,opt,name=command,proto3" json:"command,omitempty"` + TimeoutMillis int64 `protobuf:"varint,3,opt,name=timeout_millis,json=timeoutMillis,proto3" json:"timeout_millis,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommandStartRequest) Reset() { + *x = CommandStartRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommandStartRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommandStartRequest) ProtoMessage() {} + +func (x *CommandStartRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[31] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommandStartRequest.ProtoReflect.Descriptor instead. +func (*CommandStartRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{31} +} + +func (x *CommandStartRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *CommandStartRequest) GetCommand() string { + if x != nil { + return x.Command + } + return "" +} + +func (x *CommandStartRequest) GetTimeoutMillis() int64 { + if x != nil { + return x.TimeoutMillis + } + return 0 +} + +type CommandStartResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"` + State CommandState `protobuf:"varint,2,opt,name=state,proto3,enum=mecatl.execution.v1.CommandState" json:"state,omitempty"` + Result *CommandStatusResponse `protobuf:"bytes,3,opt,name=result,proto3" json:"result,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommandStartResponse) Reset() { + *x = CommandStartResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommandStartResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommandStartResponse) ProtoMessage() {} + +func (x *CommandStartResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[32] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommandStartResponse.ProtoReflect.Descriptor instead. +func (*CommandStartResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{32} +} + +func (x *CommandStartResponse) GetCommandId() string { + if x != nil { + return x.CommandId + } + return "" +} + +func (x *CommandStartResponse) GetState() CommandState { + if x != nil { + return x.State + } + return CommandState_COMMAND_STATE_UNSPECIFIED +} + +func (x *CommandStartResponse) GetResult() *CommandStatusResponse { + if x != nil { + return x.Result + } + return nil +} + +type CommandQueryRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Context *RequestContext `protobuf:"bytes,1,opt,name=context,proto3" json:"context,omitempty"` + CommandId string `protobuf:"bytes,2,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"` + Offset int64 `protobuf:"varint,3,opt,name=offset,proto3" json:"offset,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommandQueryRequest) Reset() { + *x = CommandQueryRequest{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommandQueryRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommandQueryRequest) ProtoMessage() {} + +func (x *CommandQueryRequest) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[33] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommandQueryRequest.ProtoReflect.Descriptor instead. +func (*CommandQueryRequest) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{33} +} + +func (x *CommandQueryRequest) GetContext() *RequestContext { + if x != nil { + return x.Context + } + return nil +} + +func (x *CommandQueryRequest) GetCommandId() string { + if x != nil { + return x.CommandId + } + return "" +} + +func (x *CommandQueryRequest) GetOffset() int64 { + if x != nil { + return x.Offset + } + return 0 +} + +type CommandStatusResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + CommandId string `protobuf:"bytes,1,opt,name=command_id,json=commandId,proto3" json:"command_id,omitempty"` + State CommandState `protobuf:"varint,2,opt,name=state,proto3,enum=mecatl.execution.v1.CommandState" json:"state,omitempty"` + ExitCode int32 `protobuf:"varint,3,opt,name=exit_code,json=exitCode,proto3" json:"exit_code,omitempty"` + Stdout []byte `protobuf:"bytes,4,opt,name=stdout,proto3" json:"stdout,omitempty"` + Stderr []byte `protobuf:"bytes,5,opt,name=stderr,proto3" json:"stderr,omitempty"` + NextOffset int64 `protobuf:"varint,6,opt,name=next_offset,json=nextOffset,proto3" json:"next_offset,omitempty"` + Truncated bool `protobuf:"varint,7,opt,name=truncated,proto3" json:"truncated,omitempty"` + TerminalReceipt string `protobuf:"bytes,8,opt,name=terminal_receipt,json=terminalReceipt,proto3" json:"terminal_receipt,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CommandStatusResponse) Reset() { + *x = CommandStatusResponse{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CommandStatusResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CommandStatusResponse) ProtoMessage() {} + +func (x *CommandStatusResponse) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[34] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CommandStatusResponse.ProtoReflect.Descriptor instead. +func (*CommandStatusResponse) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{34} +} + +func (x *CommandStatusResponse) GetCommandId() string { + if x != nil { + return x.CommandId + } + return "" +} + +func (x *CommandStatusResponse) GetState() CommandState { + if x != nil { + return x.State + } + return CommandState_COMMAND_STATE_UNSPECIFIED +} + +func (x *CommandStatusResponse) GetExitCode() int32 { + if x != nil { + return x.ExitCode + } + return 0 +} + +func (x *CommandStatusResponse) GetStdout() []byte { + if x != nil { + return x.Stdout + } + return nil +} + +func (x *CommandStatusResponse) GetStderr() []byte { + if x != nil { + return x.Stderr + } + return nil +} + +func (x *CommandStatusResponse) GetNextOffset() int64 { + if x != nil { + return x.NextOffset + } + return 0 +} + +func (x *CommandStatusResponse) GetTruncated() bool { + if x != nil { + return x.Truncated + } + return false +} + +func (x *CommandStatusResponse) GetTerminalReceipt() string { + if x != nil { + return x.TerminalReceipt + } + return "" +} + +// ErrorDetail is a stable machine-readable classification. Status messages are +// intentionally generic so backend/Kubernetes details never cross the boundary. +type ErrorDetail struct { + state protoimpl.MessageState `protogen:"open.v1"` + Code string `protobuf:"bytes,1,opt,name=code,proto3" json:"code,omitempty"` + Retryable bool `protobuf:"varint,2,opt,name=retryable,proto3" json:"retryable,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ErrorDetail) Reset() { + *x = ErrorDetail{} + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ErrorDetail) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ErrorDetail) ProtoMessage() {} + +func (x *ErrorDetail) ProtoReflect() protoreflect.Message { + mi := &file_mecatl_execution_v1_execution_proto_msgTypes[35] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ErrorDetail.ProtoReflect.Descriptor instead. +func (*ErrorDetail) Descriptor() ([]byte, []int) { + return file_mecatl_execution_v1_execution_proto_rawDescGZIP(), []int{35} +} + +func (x *ErrorDetail) GetCode() string { + if x != nil { + return x.Code + } + return "" +} + +func (x *ErrorDetail) GetRetryable() bool { + if x != nil { + return x.Retryable + } + return false +} + +var File_mecatl_execution_v1_execution_proto protoreflect.FileDescriptor + +var file_mecatl_execution_v1_execution_proto_rawDesc = string([]byte{ + 0x0a, 0x23, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2f, 0x76, 0x31, 0x2f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x13, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, + 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x3c, 0x0a, 0x0e, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, + 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x72, 0x65, + 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x39, 0x0a, 0x05, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x12, + 0x16, 0x0a, 0x06, 0x69, 0x73, 0x73, 0x75, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x69, 0x73, 0x73, 0x75, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, + 0x63, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x73, 0x75, 0x62, 0x6a, 0x65, 0x63, + 0x74, 0x22, 0xb1, 0x02, 0x0a, 0x0e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, + 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, + 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, + 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, + 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x12, + 0x19, 0x0a, 0x08, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x07, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x49, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, + 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x32, 0x0a, 0x16, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x22, 0x80, 0x02, 0x0a, 0x17, 0x56, 0x61, + 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x12, + 0x16, 0x0a, 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x06, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x63, 0x61, 0x70, 0x61, 0x62, + 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x63, + 0x61, 0x70, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x24, 0x0a, 0x0e, 0x6d, + 0x61, 0x78, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0c, 0x6d, 0x61, 0x78, 0x46, 0x69, 0x6c, 0x65, 0x42, 0x79, 0x74, 0x65, + 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x5f, 0x62, 0x79, 0x74, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0f, 0x6d, 0x61, + 0x78, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73, 0x12, 0x3d, 0x0a, + 0x1b, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x64, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x18, 0x6d, 0x61, 0x78, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x44, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, 0xa8, 0x01, 0x0a, + 0x18, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x66, + 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x66, 0x69, + 0x6c, 0x65, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x95, 0x02, 0x0a, 0x19, 0x45, 0x6e, 0x73, 0x75, + 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, + 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, + 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, + 0x63, 0x68, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x44, + 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, + 0x61, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0e, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x45, 0x78, 0x70, 0x69, 0x72, + 0x65, 0x73, 0x41, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, + 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0x73, 0x0a, 0x18, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x75, + 0x72, 0x70, 0x6f, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x75, 0x72, + 0x70, 0x6f, 0x73, 0x65, 0x22, 0x95, 0x02, 0x0a, 0x19, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, + 0x63, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, + 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, + 0x72, 0x65, 0x61, 0x64, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x44, 0x0a, 0x10, 0x67, + 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x61, 0x74, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x0e, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, + 0x74, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, + 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x84, 0x02, 0x0a, + 0x11, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, + 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, + 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, + 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, + 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x74, 0x6c, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, + 0x69, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x74, 0x6c, 0x4d, 0x69, 0x6c, + 0x6c, 0x69, 0x73, 0x22, 0xde, 0x02, 0x0a, 0x0f, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, + 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, + 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, + 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x5f, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, 0x61, 0x69, 0x6d, 0x49, + 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x74, + 0x6c, 0x5f, 0x6d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, + 0x74, 0x74, 0x6c, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, 0x61, + 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xc1, 0x02, 0x0a, 0x11, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, + 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, + 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, + 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, + 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6c, 0x61, + 0x69, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, 0x61, + 0x69, 0x6d, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, 0x29, 0x0a, + 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0xbc, 0x02, 0x0a, 0x10, 0x52, 0x75, 0x6e, + 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, + 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, + 0x67, 0x49, 0x64, 0x12, 0x15, 0x0a, 0x06, 0x72, 0x75, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x75, 0x6e, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x6c, + 0x61, 0x69, 0x6d, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6c, + 0x61, 0x69, 0x6d, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x29, 0x0a, 0x10, 0x67, + 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x12, 0x39, 0x0a, 0x0a, + 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x78, + 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, 0x22, 0xd5, 0x01, 0x0a, 0x18, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, + 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, + 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x0a, + 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, + 0x97, 0x02, 0x0a, 0x17, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, + 0x77, 0x6e, 0x65, 0x72, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, + 0x12, 0x34, 0x0a, 0x16, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x14, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x69, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x65, 0x0a, 0x1c, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x22, 0xcb, 0x01, 0x0a, 0x1b, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, + 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, + 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, + 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x22, 0xc9, + 0x02, 0x0a, 0x0f, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, + 0x6e, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x62, 0x69, 0x6e, + 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x62, + 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x21, + 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, + 0x64, 0x12, 0x2a, 0x0a, 0x11, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x5f, 0x62, 0x69, 0x6e, 0x64, + 0x69, 0x6e, 0x67, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x42, 0x69, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x49, 0x64, 0x12, 0x39, 0x0a, + 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, + 0x72, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, + 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x22, 0x5e, 0x0a, 0x1c, 0x4c, 0x69, + 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x07, 0x69, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x52, 0x07, 0x69, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x58, 0x0a, 0x17, 0x52, 0x65, + 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x22, 0xc4, 0x02, 0x0a, 0x18, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, + 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, + 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x78, + 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, + 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, + 0x70, 0x6f, 0x63, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, + 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, + 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, + 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, + 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xc2, 0x02, 0x0a, 0x16, + 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, + 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, + 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, + 0x38, 0x0a, 0x18, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x70, 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x16, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x70, 0x6f, 0x63, 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, + 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, + 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, + 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, + 0x22, 0xc5, 0x02, 0x0a, 0x19, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, + 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, + 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x38, 0x0a, 0x18, 0x65, 0x78, 0x70, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x65, 0x70, + 0x6f, 0x63, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x16, 0x65, 0x78, 0x70, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x45, 0x70, 0x6f, 0x63, + 0x68, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x6f, + 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, 0x10, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, + 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0xe7, 0x01, 0x0a, 0x1f, 0x44, 0x65, 0x6c, + 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, 0x0b, + 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, + 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, 0x05, + 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, + 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x49, 0x64, 0x22, 0xf2, 0x01, 0x0a, 0x18, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x45, 0x0a, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, + 0x72, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x3a, 0x0a, 0x19, 0x65, 0x78, 0x70, 0x65, + 0x63, 0x74, 0x65, 0x64, 0x5f, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x04, 0x52, 0x17, 0x65, 0x78, 0x70, + 0x65, 0x63, 0x74, 0x65, 0x64, 0x47, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, 0x70, 0x65, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x22, 0x46, 0x0a, 0x19, 0x52, 0x65, 0x76, 0x6f, 0x6b, + 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x67, 0x72, 0x61, 0x6e, 0x74, 0x5f, 0x67, 0x65, + 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0f, + 0x67, 0x72, 0x61, 0x6e, 0x74, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0xe4, 0x02, 0x0a, 0x19, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x45, 0x0a, + 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x66, 0x52, 0x0b, 0x65, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x52, + 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x12, 0x3b, 0x0a, 0x17, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, + 0x65, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x00, 0x52, 0x15, 0x65, 0x78, 0x70, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x88, 0x01, 0x01, 0x12, 0x28, 0x0a, 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, + 0x70, 0x6f, 0x64, 0x5f, 0x75, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x50, 0x6f, 0x64, 0x55, 0x69, 0x64, 0x12, 0x28, 0x0a, + 0x10, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x70, 0x76, 0x63, 0x5f, 0x75, 0x69, + 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x65, 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, + 0x64, 0x50, 0x76, 0x63, 0x55, 0x69, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x6f, 0x70, 0x65, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6f, + 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x42, 0x1a, 0x0a, 0x18, 0x5f, 0x65, + 0x78, 0x70, 0x65, 0x63, 0x74, 0x65, 0x64, 0x5f, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x5f, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xa2, 0x02, 0x0a, 0x0b, 0x46, 0x69, 0x6c, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, + 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x40, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x22, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, + 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x6f, 0x70, + 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x20, 0x0a, 0x0b, 0x64, + 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x64, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x18, 0x0a, + 0x07, 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, + 0x70, 0x61, 0x74, 0x74, 0x65, 0x72, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x18, + 0x06, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x12, 0x18, 0x0a, 0x07, 0x76, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x76, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x08, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0x94, 0x01, 0x0a, 0x08, + 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, + 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, + 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x35, 0x0a, 0x08, 0x6d, 0x6f, 0x64, 0x5f, 0x74, 0x69, 0x6d, 0x65, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x52, 0x07, 0x6d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x15, 0x0a, 0x06, 0x69, + 0x73, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x69, 0x73, 0x44, + 0x69, 0x72, 0x22, 0x47, 0x0a, 0x09, 0x47, 0x72, 0x65, 0x70, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x12, + 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, + 0x61, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x04, 0x6c, 0x69, 0x6e, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x65, 0x78, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x65, 0x78, 0x74, 0x22, 0xd4, 0x02, 0x0a, 0x0c, + 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x12, 0x0a, 0x04, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, + 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x31, 0x0a, 0x04, 0x69, 0x6e, + 0x66, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, + 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x69, 0x6e, 0x66, 0x6f, 0x12, 0x37, 0x0a, + 0x07, 0x65, 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x07, 0x65, + 0x6e, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x18, + 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x70, 0x61, 0x74, 0x68, 0x73, 0x12, 0x38, 0x0a, 0x07, + 0x6d, 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x72, 0x65, 0x70, 0x4d, 0x61, 0x74, 0x63, 0x68, 0x52, 0x07, 0x6d, + 0x61, 0x74, 0x63, 0x68, 0x65, 0x73, 0x12, 0x29, 0x0a, 0x10, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, + 0x69, 0x74, 0x79, 0x5f, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0f, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x54, 0x61, 0x72, 0x67, 0x65, + 0x74, 0x12, 0x2f, 0x0a, 0x13, 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x5f, 0x77, + 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, + 0x61, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x74, 0x79, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x22, 0x95, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, + 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x6d, + 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x74, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x4d, 0x69, 0x6c, 0x6c, 0x69, 0x73, 0x22, 0xb2, 0x01, 0x0a, 0x14, 0x43, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x5f, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, + 0x49, 0x64, 0x12, 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, 0x42, 0x0a, 0x06, 0x72, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, + 0x8b, 0x01, 0x0a, 0x13, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3d, 0x0a, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, + 0x78, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x23, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, + 0x64, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x22, 0xa6, 0x02, + 0x0a, 0x15, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6f, 0x6d, 0x6d, 0x61, + 0x6e, 0x64, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x49, 0x64, 0x12, 0x37, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x73, 0x74, 0x61, 0x74, 0x65, 0x12, + 0x1b, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, + 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x74, + 0x64, 0x6f, 0x75, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, 0x12, 0x1f, 0x0a, 0x0b, + 0x6e, 0x65, 0x78, 0x74, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x0a, 0x6e, 0x65, 0x78, 0x74, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x1c, 0x0a, + 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x09, 0x74, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x29, 0x0a, 0x10, 0x74, + 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x5f, 0x72, 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x6c, 0x52, + 0x65, 0x63, 0x65, 0x69, 0x70, 0x74, 0x22, 0x3f, 0x0a, 0x0b, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x44, + 0x65, 0x74, 0x61, 0x69, 0x6c, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x6f, 0x64, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x74, + 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, + 0x74, 0x72, 0x79, 0x61, 0x62, 0x6c, 0x65, 0x2a, 0xd8, 0x02, 0x0a, 0x0d, 0x46, 0x69, 0x6c, 0x65, + 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x1a, 0x46, 0x49, 0x4c, + 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, + 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x00, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, + 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x41, 0x44, + 0x10, 0x01, 0x12, 0x24, 0x0a, 0x20, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, + 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x53, 0x4f, 0x4c, 0x56, 0x45, 0x5f, 0x41, 0x55, 0x54, + 0x48, 0x4f, 0x52, 0x49, 0x54, 0x59, 0x10, 0x02, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, + 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x10, + 0x03, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x43, 0x52, 0x45, 0x41, 0x54, 0x45, 0x10, 0x04, 0x12, 0x1a, 0x0a, 0x16, + 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, + 0x45, 0x50, 0x4c, 0x41, 0x43, 0x45, 0x10, 0x05, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, + 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x4c, 0x49, 0x53, 0x54, 0x10, + 0x06, 0x12, 0x19, 0x0a, 0x15, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, + 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x45, 0x4d, 0x4f, 0x56, 0x45, 0x10, 0x07, 0x12, 0x19, 0x0a, 0x15, + 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, + 0x45, 0x4e, 0x41, 0x4d, 0x45, 0x10, 0x08, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, + 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x43, 0x4f, 0x50, 0x59, 0x10, 0x09, + 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, + 0x4f, 0x4e, 0x5f, 0x47, 0x4c, 0x4f, 0x42, 0x10, 0x0a, 0x12, 0x17, 0x0a, 0x13, 0x46, 0x49, 0x4c, + 0x45, 0x5f, 0x4f, 0x50, 0x45, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x47, 0x52, 0x45, 0x50, + 0x10, 0x0b, 0x2a, 0xbd, 0x01, 0x0a, 0x0c, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x74, 0x65, 0x12, 0x1d, 0x0a, 0x19, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, + 0x54, 0x41, 0x54, 0x45, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, + 0x41, 0x54, 0x45, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x1b, 0x0a, + 0x17, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x53, + 0x55, 0x43, 0x43, 0x45, 0x45, 0x44, 0x45, 0x44, 0x10, 0x02, 0x12, 0x18, 0x0a, 0x14, 0x43, 0x4f, + 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x46, 0x41, 0x49, 0x4c, + 0x45, 0x44, 0x10, 0x03, 0x12, 0x1b, 0x0a, 0x17, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, + 0x53, 0x54, 0x41, 0x54, 0x45, 0x5f, 0x43, 0x41, 0x4e, 0x43, 0x45, 0x4c, 0x4c, 0x45, 0x44, 0x10, + 0x04, 0x12, 0x1f, 0x0a, 0x1b, 0x43, 0x4f, 0x4d, 0x4d, 0x41, 0x4e, 0x44, 0x5f, 0x53, 0x54, 0x41, + 0x54, 0x45, 0x5f, 0x46, 0x45, 0x4e, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, + 0x10, 0x05, 0x32, 0xe4, 0x12, 0x0a, 0x18, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x50, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, + 0x6c, 0x0a, 0x0f, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, + 0x6c, 0x65, 0x12, 0x2b, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x65, 0x50, 0x72, 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x65, 0x50, 0x72, + 0x6f, 0x66, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x72, 0x0a, + 0x11, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, + 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, + 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x72, 0x0a, 0x11, 0x41, 0x74, 0x74, 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, + 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x74, 0x74, + 0x61, 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x74, 0x74, 0x61, + 0x63, 0x68, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x5b, 0x0a, 0x0a, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, 0x65, + 0x52, 0x75, 0x6e, 0x12, 0x26, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x71, 0x75, 0x69, 0x72, + 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6c, 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x57, 0x0a, 0x08, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, 0x12, 0x24, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6e, 0x65, 0x77, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6c, + 0x61, 0x69, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x0a, 0x52, + 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x75, 0x6e, 0x12, 0x26, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x75, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x58, 0x0a, 0x0f, 0x43, 0x6f, 0x6d, + 0x6d, 0x69, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2d, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x12, 0x57, 0x0a, 0x0e, 0x41, 0x62, 0x6f, 0x72, 0x74, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, + 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x73, 0x0a, 0x10, + 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, + 0x12, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x53, 0x75, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, + 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, + 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, + 0x73, 0x65, 0x72, 0x76, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x5f, 0x0a, 0x16, 0x50, 0x72, 0x65, 0x70, 0x61, 0x72, 0x65, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x12, 0x5f, 0x0a, 0x16, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x72, 0x6d, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x12, 0x5e, 0x0a, 0x15, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x2d, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x4d, 0x75, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, + 0x70, 0x74, 0x79, 0x12, 0x7b, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x30, 0x2e, 0x6d, 0x65, + 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, + 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x49, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x31, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, + 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x58, 0x0a, 0x10, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, + 0x65, 0x6e, 0x63, 0x65, 0x12, 0x2c, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, + 0x73, 0x65, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5a, 0x0a, 0x11, 0x52, 0x65, + 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, + 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x56, 0x0a, 0x0f, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x12, 0x2b, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x6f, 0x72, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5c, + 0x0a, 0x12, 0x52, 0x65, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, + 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x63, 0x6f, 0x76, + 0x65, 0x72, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x68, 0x0a, 0x18, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, + 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x34, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x74, 0x69, 0x72, 0x65, 0x64, 0x45, 0x6e, 0x76, 0x69, + 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x5c, 0x0a, 0x12, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, + 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2e, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x4d, 0x69, 0x67, 0x72, 0x61, 0x74, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, + 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x12, 0x72, 0x0a, 0x11, 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, + 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x2d, 0x2e, 0x6d, 0x65, 0x63, 0x61, + 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x76, 0x6f, 0x6b, 0x65, 0x45, 0x6e, 0x76, 0x69, 0x72, 0x6f, 0x6e, 0x6d, 0x65, 0x6e, 0x74, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4c, 0x0a, 0x05, 0x46, 0x69, 0x6c, 0x65, + 0x73, 0x12, 0x20, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x63, 0x0a, 0x0c, 0x53, 0x74, 0x61, 0x72, 0x74, 0x43, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x29, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, + 0x61, 0x72, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, + 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x2e, 0x6d, + 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, + 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, + 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, + 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x12, 0x65, 0x0a, 0x0d, 0x43, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x43, 0x6f, 0x6d, 0x6d, + 0x61, 0x6e, 0x64, 0x12, 0x28, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, + 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, + 0x64, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, + 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x2e, 0x76, 0x31, 0x2e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0xe4, 0x01, 0x0a, 0x17, 0x63, 0x6f, + 0x6d, 0x2e, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2e, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x2e, 0x76, 0x31, 0x42, 0x0e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x4b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x6c, 0x6f, 0x6b, 0x2f, 0x6d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2f, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x61, 0x63, 0x74, 0x73, 0x2f, 0x67, 0x65, + 0x6e, 0x2f, 0x67, 0x6f, 0x2f, 0x6d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x2f, 0x65, 0x78, 0x65, 0x63, + 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x76, 0x31, 0x3b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x76, 0x31, 0xa2, 0x02, 0x03, 0x4d, 0x45, 0x58, 0xaa, 0x02, 0x13, 0x4d, 0x65, 0x63, + 0x61, 0x74, 0x6c, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x56, 0x31, + 0xca, 0x02, 0x13, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x5c, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, + 0x69, 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0xe2, 0x02, 0x1f, 0x4d, 0x65, 0x63, 0x61, 0x74, 0x6c, 0x5c, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5c, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, + 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x15, 0x4d, 0x65, 0x63, 0x61, 0x74, + 0x6c, 0x3a, 0x3a, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x3a, 0x3a, 0x56, 0x31, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +}) + +var ( + file_mecatl_execution_v1_execution_proto_rawDescOnce sync.Once + file_mecatl_execution_v1_execution_proto_rawDescData []byte +) + +func file_mecatl_execution_v1_execution_proto_rawDescGZIP() []byte { + file_mecatl_execution_v1_execution_proto_rawDescOnce.Do(func() { + file_mecatl_execution_v1_execution_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_mecatl_execution_v1_execution_proto_rawDesc), len(file_mecatl_execution_v1_execution_proto_rawDesc))) + }) + return file_mecatl_execution_v1_execution_proto_rawDescData +} + +var file_mecatl_execution_v1_execution_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_mecatl_execution_v1_execution_proto_msgTypes = make([]protoimpl.MessageInfo, 36) +var file_mecatl_execution_v1_execution_proto_goTypes = []any{ + (FileOperation)(0), // 0: mecatl.execution.v1.FileOperation + (CommandState)(0), // 1: mecatl.execution.v1.CommandState + (*EnvironmentRef)(nil), // 2: mecatl.execution.v1.EnvironmentRef + (*Owner)(nil), // 3: mecatl.execution.v1.Owner + (*RequestContext)(nil), // 4: mecatl.execution.v1.RequestContext + (*ValidateProfileRequest)(nil), // 5: mecatl.execution.v1.ValidateProfileRequest + (*ValidateProfileResponse)(nil), // 6: mecatl.execution.v1.ValidateProfileResponse + (*EnsureEnvironmentRequest)(nil), // 7: mecatl.execution.v1.EnsureEnvironmentRequest + (*EnsureEnvironmentResponse)(nil), // 8: mecatl.execution.v1.EnsureEnvironmentResponse + (*AttachEnvironmentRequest)(nil), // 9: mecatl.execution.v1.AttachEnvironmentRequest + (*AttachEnvironmentResponse)(nil), // 10: mecatl.execution.v1.AttachEnvironmentResponse + (*AcquireRunRequest)(nil), // 11: mecatl.execution.v1.AcquireRunRequest + (*RenewRunRequest)(nil), // 12: mecatl.execution.v1.RenewRunRequest + (*ReleaseRunRequest)(nil), // 13: mecatl.execution.v1.ReleaseRunRequest + (*RunClaimResponse)(nil), // 14: mecatl.execution.v1.RunClaimResponse + (*ReferenceMutationRequest)(nil), // 15: mecatl.execution.v1.ReferenceMutationRequest + (*ReserveSuccessorRequest)(nil), // 16: mecatl.execution.v1.ReserveSuccessorRequest + (*ReferenceReservationResponse)(nil), // 17: mecatl.execution.v1.ReferenceReservationResponse + (*ListReferenceIntentsRequest)(nil), // 18: mecatl.execution.v1.ListReferenceIntentsRequest + (*ReferenceIntent)(nil), // 19: mecatl.execution.v1.ReferenceIntent + (*ListReferenceIntentsResponse)(nil), // 20: mecatl.execution.v1.ListReferenceIntentsResponse + (*ReleaseReferenceRequest)(nil), // 21: mecatl.execution.v1.ReleaseReferenceRequest + (*RetireEnvironmentRequest)(nil), // 22: mecatl.execution.v1.RetireEnvironmentRequest + (*ReplaceExecutorRequest)(nil), // 23: mecatl.execution.v1.ReplaceExecutorRequest + (*RecoverEnvironmentRequest)(nil), // 24: mecatl.execution.v1.RecoverEnvironmentRequest + (*DeleteRetiredEnvironmentRequest)(nil), // 25: mecatl.execution.v1.DeleteRetiredEnvironmentRequest + (*RevokeEnvironmentRequest)(nil), // 26: mecatl.execution.v1.RevokeEnvironmentRequest + (*RevokeEnvironmentResponse)(nil), // 27: mecatl.execution.v1.RevokeEnvironmentResponse + (*MigrateEnvironmentRequest)(nil), // 28: mecatl.execution.v1.MigrateEnvironmentRequest + (*FileRequest)(nil), // 29: mecatl.execution.v1.FileRequest + (*FileInfo)(nil), // 30: mecatl.execution.v1.FileInfo + (*GrepMatch)(nil), // 31: mecatl.execution.v1.GrepMatch + (*FileResponse)(nil), // 32: mecatl.execution.v1.FileResponse + (*CommandStartRequest)(nil), // 33: mecatl.execution.v1.CommandStartRequest + (*CommandStartResponse)(nil), // 34: mecatl.execution.v1.CommandStartResponse + (*CommandQueryRequest)(nil), // 35: mecatl.execution.v1.CommandQueryRequest + (*CommandStatusResponse)(nil), // 36: mecatl.execution.v1.CommandStatusResponse + (*ErrorDetail)(nil), // 37: mecatl.execution.v1.ErrorDetail + (*timestamppb.Timestamp)(nil), // 38: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 39: google.protobuf.Empty +} +var file_mecatl_execution_v1_execution_proto_depIdxs = []int32{ + 2, // 0: mecatl.execution.v1.RequestContext.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 1: mecatl.execution.v1.RequestContext.owner:type_name -> mecatl.execution.v1.Owner + 3, // 2: mecatl.execution.v1.EnsureEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 3: mecatl.execution.v1.EnsureEnvironmentResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 38, // 4: mecatl.execution.v1.EnsureEnvironmentResponse.grant_expires_at:type_name -> google.protobuf.Timestamp + 4, // 5: mecatl.execution.v1.AttachEnvironmentRequest.context:type_name -> mecatl.execution.v1.RequestContext + 2, // 6: mecatl.execution.v1.AttachEnvironmentResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 38, // 7: mecatl.execution.v1.AttachEnvironmentResponse.grant_expires_at:type_name -> google.protobuf.Timestamp + 2, // 8: mecatl.execution.v1.AcquireRunRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 9: mecatl.execution.v1.AcquireRunRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 10: mecatl.execution.v1.RenewRunRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 11: mecatl.execution.v1.RenewRunRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 12: mecatl.execution.v1.ReleaseRunRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 13: mecatl.execution.v1.ReleaseRunRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 14: mecatl.execution.v1.RunClaimResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 38, // 15: mecatl.execution.v1.RunClaimResponse.expires_at:type_name -> google.protobuf.Timestamp + 2, // 16: mecatl.execution.v1.ReferenceMutationRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 17: mecatl.execution.v1.ReferenceMutationRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 18: mecatl.execution.v1.ReserveSuccessorRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 19: mecatl.execution.v1.ReserveSuccessorRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 20: mecatl.execution.v1.ReferenceReservationResponse.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 21: mecatl.execution.v1.ListReferenceIntentsRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 22: mecatl.execution.v1.ListReferenceIntentsRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 2, // 23: mecatl.execution.v1.ReferenceIntent.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 38, // 24: mecatl.execution.v1.ReferenceIntent.created_at:type_name -> google.protobuf.Timestamp + 3, // 25: mecatl.execution.v1.ReferenceIntent.owner:type_name -> mecatl.execution.v1.Owner + 19, // 26: mecatl.execution.v1.ListReferenceIntentsResponse.intents:type_name -> mecatl.execution.v1.ReferenceIntent + 4, // 27: mecatl.execution.v1.ReleaseReferenceRequest.context:type_name -> mecatl.execution.v1.RequestContext + 2, // 28: mecatl.execution.v1.RetireEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 29: mecatl.execution.v1.RetireEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 30: mecatl.execution.v1.ReplaceExecutorRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 31: mecatl.execution.v1.ReplaceExecutorRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 32: mecatl.execution.v1.RecoverEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 33: mecatl.execution.v1.RecoverEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 34: mecatl.execution.v1.DeleteRetiredEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 35: mecatl.execution.v1.DeleteRetiredEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 36: mecatl.execution.v1.RevokeEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 37: mecatl.execution.v1.RevokeEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 2, // 38: mecatl.execution.v1.MigrateEnvironmentRequest.environment:type_name -> mecatl.execution.v1.EnvironmentRef + 3, // 39: mecatl.execution.v1.MigrateEnvironmentRequest.owner:type_name -> mecatl.execution.v1.Owner + 4, // 40: mecatl.execution.v1.FileRequest.context:type_name -> mecatl.execution.v1.RequestContext + 0, // 41: mecatl.execution.v1.FileRequest.operation:type_name -> mecatl.execution.v1.FileOperation + 38, // 42: mecatl.execution.v1.FileInfo.mod_time:type_name -> google.protobuf.Timestamp + 30, // 43: mecatl.execution.v1.FileResponse.info:type_name -> mecatl.execution.v1.FileInfo + 30, // 44: mecatl.execution.v1.FileResponse.entries:type_name -> mecatl.execution.v1.FileInfo + 31, // 45: mecatl.execution.v1.FileResponse.matches:type_name -> mecatl.execution.v1.GrepMatch + 4, // 46: mecatl.execution.v1.CommandStartRequest.context:type_name -> mecatl.execution.v1.RequestContext + 1, // 47: mecatl.execution.v1.CommandStartResponse.state:type_name -> mecatl.execution.v1.CommandState + 36, // 48: mecatl.execution.v1.CommandStartResponse.result:type_name -> mecatl.execution.v1.CommandStatusResponse + 4, // 49: mecatl.execution.v1.CommandQueryRequest.context:type_name -> mecatl.execution.v1.RequestContext + 1, // 50: mecatl.execution.v1.CommandStatusResponse.state:type_name -> mecatl.execution.v1.CommandState + 5, // 51: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:input_type -> mecatl.execution.v1.ValidateProfileRequest + 7, // 52: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:input_type -> mecatl.execution.v1.EnsureEnvironmentRequest + 9, // 53: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:input_type -> mecatl.execution.v1.AttachEnvironmentRequest + 11, // 54: mecatl.execution.v1.ExecutionProviderService.AcquireRun:input_type -> mecatl.execution.v1.AcquireRunRequest + 12, // 55: mecatl.execution.v1.ExecutionProviderService.RenewRun:input_type -> mecatl.execution.v1.RenewRunRequest + 13, // 56: mecatl.execution.v1.ExecutionProviderService.ReleaseRun:input_type -> mecatl.execution.v1.ReleaseRunRequest + 15, // 57: mecatl.execution.v1.ExecutionProviderService.CommitReference:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 15, // 58: mecatl.execution.v1.ExecutionProviderService.AbortReference:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 16, // 59: mecatl.execution.v1.ExecutionProviderService.ReserveSuccessor:input_type -> mecatl.execution.v1.ReserveSuccessorRequest + 15, // 60: mecatl.execution.v1.ExecutionProviderService.PrepareReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 15, // 61: mecatl.execution.v1.ExecutionProviderService.ConfirmReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 15, // 62: mecatl.execution.v1.ExecutionProviderService.CancelReferenceDelete:input_type -> mecatl.execution.v1.ReferenceMutationRequest + 18, // 63: mecatl.execution.v1.ExecutionProviderService.ListReferenceIntents:input_type -> mecatl.execution.v1.ListReferenceIntentsRequest + 21, // 64: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:input_type -> mecatl.execution.v1.ReleaseReferenceRequest + 22, // 65: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:input_type -> mecatl.execution.v1.RetireEnvironmentRequest + 23, // 66: mecatl.execution.v1.ExecutionProviderService.ReplaceExecutor:input_type -> mecatl.execution.v1.ReplaceExecutorRequest + 24, // 67: mecatl.execution.v1.ExecutionProviderService.RecoverEnvironment:input_type -> mecatl.execution.v1.RecoverEnvironmentRequest + 25, // 68: mecatl.execution.v1.ExecutionProviderService.DeleteRetiredEnvironment:input_type -> mecatl.execution.v1.DeleteRetiredEnvironmentRequest + 28, // 69: mecatl.execution.v1.ExecutionProviderService.MigrateEnvironment:input_type -> mecatl.execution.v1.MigrateEnvironmentRequest + 26, // 70: mecatl.execution.v1.ExecutionProviderService.RevokeEnvironment:input_type -> mecatl.execution.v1.RevokeEnvironmentRequest + 29, // 71: mecatl.execution.v1.ExecutionProviderService.Files:input_type -> mecatl.execution.v1.FileRequest + 33, // 72: mecatl.execution.v1.ExecutionProviderService.StartCommand:input_type -> mecatl.execution.v1.CommandStartRequest + 35, // 73: mecatl.execution.v1.ExecutionProviderService.CommandStatus:input_type -> mecatl.execution.v1.CommandQueryRequest + 35, // 74: mecatl.execution.v1.ExecutionProviderService.CancelCommand:input_type -> mecatl.execution.v1.CommandQueryRequest + 6, // 75: mecatl.execution.v1.ExecutionProviderService.ValidateProfile:output_type -> mecatl.execution.v1.ValidateProfileResponse + 8, // 76: mecatl.execution.v1.ExecutionProviderService.EnsureEnvironment:output_type -> mecatl.execution.v1.EnsureEnvironmentResponse + 10, // 77: mecatl.execution.v1.ExecutionProviderService.AttachEnvironment:output_type -> mecatl.execution.v1.AttachEnvironmentResponse + 14, // 78: mecatl.execution.v1.ExecutionProviderService.AcquireRun:output_type -> mecatl.execution.v1.RunClaimResponse + 14, // 79: mecatl.execution.v1.ExecutionProviderService.RenewRun:output_type -> mecatl.execution.v1.RunClaimResponse + 39, // 80: mecatl.execution.v1.ExecutionProviderService.ReleaseRun:output_type -> google.protobuf.Empty + 39, // 81: mecatl.execution.v1.ExecutionProviderService.CommitReference:output_type -> google.protobuf.Empty + 39, // 82: mecatl.execution.v1.ExecutionProviderService.AbortReference:output_type -> google.protobuf.Empty + 17, // 83: mecatl.execution.v1.ExecutionProviderService.ReserveSuccessor:output_type -> mecatl.execution.v1.ReferenceReservationResponse + 39, // 84: mecatl.execution.v1.ExecutionProviderService.PrepareReferenceDelete:output_type -> google.protobuf.Empty + 39, // 85: mecatl.execution.v1.ExecutionProviderService.ConfirmReferenceDelete:output_type -> google.protobuf.Empty + 39, // 86: mecatl.execution.v1.ExecutionProviderService.CancelReferenceDelete:output_type -> google.protobuf.Empty + 20, // 87: mecatl.execution.v1.ExecutionProviderService.ListReferenceIntents:output_type -> mecatl.execution.v1.ListReferenceIntentsResponse + 39, // 88: mecatl.execution.v1.ExecutionProviderService.ReleaseReference:output_type -> google.protobuf.Empty + 39, // 89: mecatl.execution.v1.ExecutionProviderService.RetireEnvironment:output_type -> google.protobuf.Empty + 39, // 90: mecatl.execution.v1.ExecutionProviderService.ReplaceExecutor:output_type -> google.protobuf.Empty + 39, // 91: mecatl.execution.v1.ExecutionProviderService.RecoverEnvironment:output_type -> google.protobuf.Empty + 39, // 92: mecatl.execution.v1.ExecutionProviderService.DeleteRetiredEnvironment:output_type -> google.protobuf.Empty + 39, // 93: mecatl.execution.v1.ExecutionProviderService.MigrateEnvironment:output_type -> google.protobuf.Empty + 27, // 94: mecatl.execution.v1.ExecutionProviderService.RevokeEnvironment:output_type -> mecatl.execution.v1.RevokeEnvironmentResponse + 32, // 95: mecatl.execution.v1.ExecutionProviderService.Files:output_type -> mecatl.execution.v1.FileResponse + 34, // 96: mecatl.execution.v1.ExecutionProviderService.StartCommand:output_type -> mecatl.execution.v1.CommandStartResponse + 36, // 97: mecatl.execution.v1.ExecutionProviderService.CommandStatus:output_type -> mecatl.execution.v1.CommandStatusResponse + 36, // 98: mecatl.execution.v1.ExecutionProviderService.CancelCommand:output_type -> mecatl.execution.v1.CommandStatusResponse + 75, // [75:99] is the sub-list for method output_type + 51, // [51:75] is the sub-list for method input_type + 51, // [51:51] is the sub-list for extension type_name + 51, // [51:51] is the sub-list for extension extendee + 0, // [0:51] is the sub-list for field type_name +} + +func init() { file_mecatl_execution_v1_execution_proto_init() } +func file_mecatl_execution_v1_execution_proto_init() { + if File_mecatl_execution_v1_execution_proto != nil { + return + } + file_mecatl_execution_v1_execution_proto_msgTypes[26].OneofWrappers = []any{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_mecatl_execution_v1_execution_proto_rawDesc), len(file_mecatl_execution_v1_execution_proto_rawDesc)), + NumEnums: 2, + NumMessages: 36, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_mecatl_execution_v1_execution_proto_goTypes, + DependencyIndexes: file_mecatl_execution_v1_execution_proto_depIdxs, + EnumInfos: file_mecatl_execution_v1_execution_proto_enumTypes, + MessageInfos: file_mecatl_execution_v1_execution_proto_msgTypes, + }.Build() + File_mecatl_execution_v1_execution_proto = out.File + file_mecatl_execution_v1_execution_proto_goTypes = nil + file_mecatl_execution_v1_execution_proto_depIdxs = nil +} diff --git a/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go b/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go new file mode 100644 index 0000000000..abfd835935 --- /dev/null +++ b/contracts/gen/go/mecatl/execution/v1/execution_grpc.pb.go @@ -0,0 +1,1003 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: LicenseRef-Stacklok-Proprietary + +// Private, mutually authenticated execution-provider protocol. This is not part +// of the public Harness API. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc (unknown) +// source: mecatl/execution/v1/execution.proto + +package executionv1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + emptypb "google.golang.org/protobuf/types/known/emptypb" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + ExecutionProviderService_ValidateProfile_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ValidateProfile" + ExecutionProviderService_EnsureEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/EnsureEnvironment" + ExecutionProviderService_AttachEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/AttachEnvironment" + ExecutionProviderService_AcquireRun_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/AcquireRun" + ExecutionProviderService_RenewRun_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RenewRun" + ExecutionProviderService_ReleaseRun_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReleaseRun" + ExecutionProviderService_CommitReference_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CommitReference" + ExecutionProviderService_AbortReference_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/AbortReference" + ExecutionProviderService_ReserveSuccessor_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReserveSuccessor" + ExecutionProviderService_PrepareReferenceDelete_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/PrepareReferenceDelete" + ExecutionProviderService_ConfirmReferenceDelete_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ConfirmReferenceDelete" + ExecutionProviderService_CancelReferenceDelete_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CancelReferenceDelete" + ExecutionProviderService_ListReferenceIntents_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ListReferenceIntents" + ExecutionProviderService_ReleaseReference_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReleaseReference" + ExecutionProviderService_RetireEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RetireEnvironment" + ExecutionProviderService_ReplaceExecutor_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/ReplaceExecutor" + ExecutionProviderService_RecoverEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RecoverEnvironment" + ExecutionProviderService_DeleteRetiredEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/DeleteRetiredEnvironment" + ExecutionProviderService_MigrateEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/MigrateEnvironment" + ExecutionProviderService_RevokeEnvironment_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/RevokeEnvironment" + ExecutionProviderService_Files_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/Files" + ExecutionProviderService_StartCommand_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/StartCommand" + ExecutionProviderService_CommandStatus_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CommandStatus" + ExecutionProviderService_CancelCommand_FullMethodName = "/mecatl.execution.v1.ExecutionProviderService/CancelCommand" +) + +// ExecutionProviderServiceClient is the client API for ExecutionProviderService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type ExecutionProviderServiceClient interface { + ValidateProfile(ctx context.Context, in *ValidateProfileRequest, opts ...grpc.CallOption) (*ValidateProfileResponse, error) + EnsureEnvironment(ctx context.Context, in *EnsureEnvironmentRequest, opts ...grpc.CallOption) (*EnsureEnvironmentResponse, error) + AttachEnvironment(ctx context.Context, in *AttachEnvironmentRequest, opts ...grpc.CallOption) (*AttachEnvironmentResponse, error) + AcquireRun(ctx context.Context, in *AcquireRunRequest, opts ...grpc.CallOption) (*RunClaimResponse, error) + RenewRun(ctx context.Context, in *RenewRunRequest, opts ...grpc.CallOption) (*RunClaimResponse, error) + ReleaseRun(ctx context.Context, in *ReleaseRunRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + CommitReference(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + AbortReference(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + ReserveSuccessor(ctx context.Context, in *ReserveSuccessorRequest, opts ...grpc.CallOption) (*ReferenceReservationResponse, error) + PrepareReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + ConfirmReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + CancelReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + ListReferenceIntents(ctx context.Context, in *ListReferenceIntentsRequest, opts ...grpc.CallOption) (*ListReferenceIntentsResponse, error) + ReleaseReference(ctx context.Context, in *ReleaseReferenceRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + RetireEnvironment(ctx context.Context, in *RetireEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + ReplaceExecutor(ctx context.Context, in *ReplaceExecutorRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + RecoverEnvironment(ctx context.Context, in *RecoverEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + DeleteRetiredEnvironment(ctx context.Context, in *DeleteRetiredEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + MigrateEnvironment(ctx context.Context, in *MigrateEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + RevokeEnvironment(ctx context.Context, in *RevokeEnvironmentRequest, opts ...grpc.CallOption) (*RevokeEnvironmentResponse, error) + Files(ctx context.Context, in *FileRequest, opts ...grpc.CallOption) (*FileResponse, error) + StartCommand(ctx context.Context, in *CommandStartRequest, opts ...grpc.CallOption) (*CommandStartResponse, error) + CommandStatus(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) + CancelCommand(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) +} + +type executionProviderServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewExecutionProviderServiceClient(cc grpc.ClientConnInterface) ExecutionProviderServiceClient { + return &executionProviderServiceClient{cc} +} + +func (c *executionProviderServiceClient) ValidateProfile(ctx context.Context, in *ValidateProfileRequest, opts ...grpc.CallOption) (*ValidateProfileResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ValidateProfileResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_ValidateProfile_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) EnsureEnvironment(ctx context.Context, in *EnsureEnvironmentRequest, opts ...grpc.CallOption) (*EnsureEnvironmentResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(EnsureEnvironmentResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_EnsureEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) AttachEnvironment(ctx context.Context, in *AttachEnvironmentRequest, opts ...grpc.CallOption) (*AttachEnvironmentResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(AttachEnvironmentResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_AttachEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) AcquireRun(ctx context.Context, in *AcquireRunRequest, opts ...grpc.CallOption) (*RunClaimResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RunClaimResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_AcquireRun_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) RenewRun(ctx context.Context, in *RenewRunRequest, opts ...grpc.CallOption) (*RunClaimResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RunClaimResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_RenewRun_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ReleaseRun(ctx context.Context, in *ReleaseRunRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_ReleaseRun_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) CommitReference(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_CommitReference_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) AbortReference(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_AbortReference_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ReserveSuccessor(ctx context.Context, in *ReserveSuccessorRequest, opts ...grpc.CallOption) (*ReferenceReservationResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ReferenceReservationResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_ReserveSuccessor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) PrepareReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_PrepareReferenceDelete_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ConfirmReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_ConfirmReferenceDelete_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) CancelReferenceDelete(ctx context.Context, in *ReferenceMutationRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_CancelReferenceDelete_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ListReferenceIntents(ctx context.Context, in *ListReferenceIntentsRequest, opts ...grpc.CallOption) (*ListReferenceIntentsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListReferenceIntentsResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_ListReferenceIntents_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ReleaseReference(ctx context.Context, in *ReleaseReferenceRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_ReleaseReference_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) RetireEnvironment(ctx context.Context, in *RetireEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_RetireEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) ReplaceExecutor(ctx context.Context, in *ReplaceExecutorRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_ReplaceExecutor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) RecoverEnvironment(ctx context.Context, in *RecoverEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_RecoverEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) DeleteRetiredEnvironment(ctx context.Context, in *DeleteRetiredEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_DeleteRetiredEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) MigrateEnvironment(ctx context.Context, in *MigrateEnvironmentRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, ExecutionProviderService_MigrateEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) RevokeEnvironment(ctx context.Context, in *RevokeEnvironmentRequest, opts ...grpc.CallOption) (*RevokeEnvironmentResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RevokeEnvironmentResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_RevokeEnvironment_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) Files(ctx context.Context, in *FileRequest, opts ...grpc.CallOption) (*FileResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(FileResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_Files_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) StartCommand(ctx context.Context, in *CommandStartRequest, opts ...grpc.CallOption) (*CommandStartResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CommandStartResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_StartCommand_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) CommandStatus(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CommandStatusResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_CommandStatus_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *executionProviderServiceClient) CancelCommand(ctx context.Context, in *CommandQueryRequest, opts ...grpc.CallOption) (*CommandStatusResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CommandStatusResponse) + err := c.cc.Invoke(ctx, ExecutionProviderService_CancelCommand_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ExecutionProviderServiceServer is the server API for ExecutionProviderService service. +// All implementations must embed UnimplementedExecutionProviderServiceServer +// for forward compatibility. +type ExecutionProviderServiceServer interface { + ValidateProfile(context.Context, *ValidateProfileRequest) (*ValidateProfileResponse, error) + EnsureEnvironment(context.Context, *EnsureEnvironmentRequest) (*EnsureEnvironmentResponse, error) + AttachEnvironment(context.Context, *AttachEnvironmentRequest) (*AttachEnvironmentResponse, error) + AcquireRun(context.Context, *AcquireRunRequest) (*RunClaimResponse, error) + RenewRun(context.Context, *RenewRunRequest) (*RunClaimResponse, error) + ReleaseRun(context.Context, *ReleaseRunRequest) (*emptypb.Empty, error) + CommitReference(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + AbortReference(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + ReserveSuccessor(context.Context, *ReserveSuccessorRequest) (*ReferenceReservationResponse, error) + PrepareReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + ConfirmReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + CancelReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) + ListReferenceIntents(context.Context, *ListReferenceIntentsRequest) (*ListReferenceIntentsResponse, error) + ReleaseReference(context.Context, *ReleaseReferenceRequest) (*emptypb.Empty, error) + RetireEnvironment(context.Context, *RetireEnvironmentRequest) (*emptypb.Empty, error) + ReplaceExecutor(context.Context, *ReplaceExecutorRequest) (*emptypb.Empty, error) + RecoverEnvironment(context.Context, *RecoverEnvironmentRequest) (*emptypb.Empty, error) + DeleteRetiredEnvironment(context.Context, *DeleteRetiredEnvironmentRequest) (*emptypb.Empty, error) + MigrateEnvironment(context.Context, *MigrateEnvironmentRequest) (*emptypb.Empty, error) + RevokeEnvironment(context.Context, *RevokeEnvironmentRequest) (*RevokeEnvironmentResponse, error) + Files(context.Context, *FileRequest) (*FileResponse, error) + StartCommand(context.Context, *CommandStartRequest) (*CommandStartResponse, error) + CommandStatus(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) + CancelCommand(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) + mustEmbedUnimplementedExecutionProviderServiceServer() +} + +// UnimplementedExecutionProviderServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedExecutionProviderServiceServer struct{} + +func (UnimplementedExecutionProviderServiceServer) ValidateProfile(context.Context, *ValidateProfileRequest) (*ValidateProfileResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ValidateProfile not implemented") +} +func (UnimplementedExecutionProviderServiceServer) EnsureEnvironment(context.Context, *EnsureEnvironmentRequest) (*EnsureEnvironmentResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method EnsureEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) AttachEnvironment(context.Context, *AttachEnvironmentRequest) (*AttachEnvironmentResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AttachEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) AcquireRun(context.Context, *AcquireRunRequest) (*RunClaimResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method AcquireRun not implemented") +} +func (UnimplementedExecutionProviderServiceServer) RenewRun(context.Context, *RenewRunRequest) (*RunClaimResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RenewRun not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ReleaseRun(context.Context, *ReleaseRunRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReleaseRun not implemented") +} +func (UnimplementedExecutionProviderServiceServer) CommitReference(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method CommitReference not implemented") +} +func (UnimplementedExecutionProviderServiceServer) AbortReference(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method AbortReference not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ReserveSuccessor(context.Context, *ReserveSuccessorRequest) (*ReferenceReservationResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReserveSuccessor not implemented") +} +func (UnimplementedExecutionProviderServiceServer) PrepareReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method PrepareReferenceDelete not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ConfirmReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ConfirmReferenceDelete not implemented") +} +func (UnimplementedExecutionProviderServiceServer) CancelReferenceDelete(context.Context, *ReferenceMutationRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method CancelReferenceDelete not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ListReferenceIntents(context.Context, *ListReferenceIntentsRequest) (*ListReferenceIntentsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method ListReferenceIntents not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ReleaseReference(context.Context, *ReleaseReferenceRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReleaseReference not implemented") +} +func (UnimplementedExecutionProviderServiceServer) RetireEnvironment(context.Context, *RetireEnvironmentRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method RetireEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) ReplaceExecutor(context.Context, *ReplaceExecutorRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method ReplaceExecutor not implemented") +} +func (UnimplementedExecutionProviderServiceServer) RecoverEnvironment(context.Context, *RecoverEnvironmentRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method RecoverEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) DeleteRetiredEnvironment(context.Context, *DeleteRetiredEnvironmentRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteRetiredEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) MigrateEnvironment(context.Context, *MigrateEnvironmentRequest) (*emptypb.Empty, error) { + return nil, status.Errorf(codes.Unimplemented, "method MigrateEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) RevokeEnvironment(context.Context, *RevokeEnvironmentRequest) (*RevokeEnvironmentResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RevokeEnvironment not implemented") +} +func (UnimplementedExecutionProviderServiceServer) Files(context.Context, *FileRequest) (*FileResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Files not implemented") +} +func (UnimplementedExecutionProviderServiceServer) StartCommand(context.Context, *CommandStartRequest) (*CommandStartResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method StartCommand not implemented") +} +func (UnimplementedExecutionProviderServiceServer) CommandStatus(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CommandStatus not implemented") +} +func (UnimplementedExecutionProviderServiceServer) CancelCommand(context.Context, *CommandQueryRequest) (*CommandStatusResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method CancelCommand not implemented") +} +func (UnimplementedExecutionProviderServiceServer) mustEmbedUnimplementedExecutionProviderServiceServer() { +} +func (UnimplementedExecutionProviderServiceServer) testEmbeddedByValue() {} + +// UnsafeExecutionProviderServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ExecutionProviderServiceServer will +// result in compilation errors. +type UnsafeExecutionProviderServiceServer interface { + mustEmbedUnimplementedExecutionProviderServiceServer() +} + +func RegisterExecutionProviderServiceServer(s grpc.ServiceRegistrar, srv ExecutionProviderServiceServer) { + // If the following call pancis, it indicates UnimplementedExecutionProviderServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&ExecutionProviderService_ServiceDesc, srv) +} + +func _ExecutionProviderService_ValidateProfile_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ValidateProfileRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ValidateProfile(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ValidateProfile_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ValidateProfile(ctx, req.(*ValidateProfileRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_EnsureEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(EnsureEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).EnsureEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_EnsureEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).EnsureEnvironment(ctx, req.(*EnsureEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_AttachEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AttachEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).AttachEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_AttachEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).AttachEnvironment(ctx, req.(*AttachEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_AcquireRun_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AcquireRunRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).AcquireRun(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_AcquireRun_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).AcquireRun(ctx, req.(*AcquireRunRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_RenewRun_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RenewRunRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).RenewRun(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_RenewRun_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).RenewRun(ctx, req.(*RenewRunRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ReleaseRun_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReleaseRunRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ReleaseRun(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ReleaseRun_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ReleaseRun(ctx, req.(*ReleaseRunRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_CommitReference_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).CommitReference(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_CommitReference_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).CommitReference(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_AbortReference_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).AbortReference(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_AbortReference_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).AbortReference(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ReserveSuccessor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReserveSuccessorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ReserveSuccessor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ReserveSuccessor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ReserveSuccessor(ctx, req.(*ReserveSuccessorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_PrepareReferenceDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).PrepareReferenceDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_PrepareReferenceDelete_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).PrepareReferenceDelete(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ConfirmReferenceDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ConfirmReferenceDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ConfirmReferenceDelete_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ConfirmReferenceDelete(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_CancelReferenceDelete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReferenceMutationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).CancelReferenceDelete(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_CancelReferenceDelete_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).CancelReferenceDelete(ctx, req.(*ReferenceMutationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ListReferenceIntents_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListReferenceIntentsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ListReferenceIntents(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ListReferenceIntents_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ListReferenceIntents(ctx, req.(*ListReferenceIntentsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ReleaseReference_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReleaseReferenceRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ReleaseReference(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ReleaseReference_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ReleaseReference(ctx, req.(*ReleaseReferenceRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_RetireEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RetireEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).RetireEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_RetireEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).RetireEnvironment(ctx, req.(*RetireEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_ReplaceExecutor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ReplaceExecutorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).ReplaceExecutor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_ReplaceExecutor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).ReplaceExecutor(ctx, req.(*ReplaceExecutorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_RecoverEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RecoverEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).RecoverEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_RecoverEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).RecoverEnvironment(ctx, req.(*RecoverEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_DeleteRetiredEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteRetiredEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).DeleteRetiredEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_DeleteRetiredEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).DeleteRetiredEnvironment(ctx, req.(*DeleteRetiredEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_MigrateEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(MigrateEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).MigrateEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_MigrateEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).MigrateEnvironment(ctx, req.(*MigrateEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_RevokeEnvironment_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RevokeEnvironmentRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).RevokeEnvironment(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_RevokeEnvironment_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).RevokeEnvironment(ctx, req.(*RevokeEnvironmentRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_Files_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FileRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).Files(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_Files_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).Files(ctx, req.(*FileRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_StartCommand_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CommandStartRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).StartCommand(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_StartCommand_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).StartCommand(ctx, req.(*CommandStartRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_CommandStatus_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CommandQueryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).CommandStatus(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_CommandStatus_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).CommandStatus(ctx, req.(*CommandQueryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _ExecutionProviderService_CancelCommand_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CommandQueryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ExecutionProviderServiceServer).CancelCommand(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: ExecutionProviderService_CancelCommand_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ExecutionProviderServiceServer).CancelCommand(ctx, req.(*CommandQueryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// ExecutionProviderService_ServiceDesc is the grpc.ServiceDesc for ExecutionProviderService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var ExecutionProviderService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "mecatl.execution.v1.ExecutionProviderService", + HandlerType: (*ExecutionProviderServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "ValidateProfile", + Handler: _ExecutionProviderService_ValidateProfile_Handler, + }, + { + MethodName: "EnsureEnvironment", + Handler: _ExecutionProviderService_EnsureEnvironment_Handler, + }, + { + MethodName: "AttachEnvironment", + Handler: _ExecutionProviderService_AttachEnvironment_Handler, + }, + { + MethodName: "AcquireRun", + Handler: _ExecutionProviderService_AcquireRun_Handler, + }, + { + MethodName: "RenewRun", + Handler: _ExecutionProviderService_RenewRun_Handler, + }, + { + MethodName: "ReleaseRun", + Handler: _ExecutionProviderService_ReleaseRun_Handler, + }, + { + MethodName: "CommitReference", + Handler: _ExecutionProviderService_CommitReference_Handler, + }, + { + MethodName: "AbortReference", + Handler: _ExecutionProviderService_AbortReference_Handler, + }, + { + MethodName: "ReserveSuccessor", + Handler: _ExecutionProviderService_ReserveSuccessor_Handler, + }, + { + MethodName: "PrepareReferenceDelete", + Handler: _ExecutionProviderService_PrepareReferenceDelete_Handler, + }, + { + MethodName: "ConfirmReferenceDelete", + Handler: _ExecutionProviderService_ConfirmReferenceDelete_Handler, + }, + { + MethodName: "CancelReferenceDelete", + Handler: _ExecutionProviderService_CancelReferenceDelete_Handler, + }, + { + MethodName: "ListReferenceIntents", + Handler: _ExecutionProviderService_ListReferenceIntents_Handler, + }, + { + MethodName: "ReleaseReference", + Handler: _ExecutionProviderService_ReleaseReference_Handler, + }, + { + MethodName: "RetireEnvironment", + Handler: _ExecutionProviderService_RetireEnvironment_Handler, + }, + { + MethodName: "ReplaceExecutor", + Handler: _ExecutionProviderService_ReplaceExecutor_Handler, + }, + { + MethodName: "RecoverEnvironment", + Handler: _ExecutionProviderService_RecoverEnvironment_Handler, + }, + { + MethodName: "DeleteRetiredEnvironment", + Handler: _ExecutionProviderService_DeleteRetiredEnvironment_Handler, + }, + { + MethodName: "MigrateEnvironment", + Handler: _ExecutionProviderService_MigrateEnvironment_Handler, + }, + { + MethodName: "RevokeEnvironment", + Handler: _ExecutionProviderService_RevokeEnvironment_Handler, + }, + { + MethodName: "Files", + Handler: _ExecutionProviderService_Files_Handler, + }, + { + MethodName: "StartCommand", + Handler: _ExecutionProviderService_StartCommand_Handler, + }, + { + MethodName: "CommandStatus", + Handler: _ExecutionProviderService_CommandStatus_Handler, + }, + { + MethodName: "CancelCommand", + Handler: _ExecutionProviderService_CancelCommand_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "mecatl/execution/v1/execution.proto", +} diff --git a/contracts/gen/go/mecatl/v1/harness_grpc.pb.go b/contracts/gen/go/mecatl/v1/harness_grpc.pb.go index 6abdd27113..4575f6f144 100644 --- a/contracts/gen/go/mecatl/v1/harness_grpc.pb.go +++ b/contracts/gen/go/mecatl/v1/harness_grpc.pb.go @@ -233,11 +233,13 @@ type HarnessServiceClient interface { // model, its effective read-only tool scope, permission mode, and UX color. // Derived from the snapshot taken at startup; it performs no live discovery. ListAgents(ctx context.Context, in *ListAgentsRequest, opts ...grpc.CallOption) (*ListAgentsResponse, error) - // ListCommands returns slash commands discovered for one owned session's exact - // server-bound placement. It authorizes and reattaches that session before - // discovery and accepts no workspace/root input. It powers the client's command - // palette; command expansion remains a run-path concern. A no-FS session or a - // server with no command expander returns an empty list. + // ListCommands returns slash commands from the configured, resolved sources + // for one authorized, owned session. It runs independently of execution and + // accepts no client workspace/root input. A source that needs execution files + // acquires only that session's exact backend. It powers the client's command + // palette; command expansion remains a run-path concern. A server with no + // configured command sources returns an empty list; a no-FS session can retain + // independently configured sources. ListCommands(ctx context.Context, in *ListCommandsRequest, opts ...grpc.CallOption) (*ListCommandsResponse, error) // ListWorktrees discovers eligible alternatives for one owned source session's // exactly reattached placement. Results carry bounded display metadata and an @@ -1467,11 +1469,13 @@ type HarnessServiceServer interface { // model, its effective read-only tool scope, permission mode, and UX color. // Derived from the snapshot taken at startup; it performs no live discovery. ListAgents(context.Context, *ListAgentsRequest) (*ListAgentsResponse, error) - // ListCommands returns slash commands discovered for one owned session's exact - // server-bound placement. It authorizes and reattaches that session before - // discovery and accepts no workspace/root input. It powers the client's command - // palette; command expansion remains a run-path concern. A no-FS session or a - // server with no command expander returns an empty list. + // ListCommands returns slash commands from the configured, resolved sources + // for one authorized, owned session. It runs independently of execution and + // accepts no client workspace/root input. A source that needs execution files + // acquires only that session's exact backend. It powers the client's command + // palette; command expansion remains a run-path concern. A server with no + // configured command sources returns an empty list; a no-FS session can retain + // independently configured sources. ListCommands(context.Context, *ListCommandsRequest) (*ListCommandsResponse, error) // ListWorktrees discovers eligible alternatives for one owned source session's // exactly reattached placement. Results carry bounded display metadata and an diff --git a/contracts/proto/mecatl/execution/v1/execution.proto b/contracts/proto/mecatl/execution/v1/execution.proto new file mode 100644 index 0000000000..9b3b369092 --- /dev/null +++ b/contracts/proto/mecatl/execution/v1/execution.proto @@ -0,0 +1,266 @@ +// SPDX-FileCopyrightText: Copyright 2026 Stacklok, Inc. +// SPDX-License-Identifier: LicenseRef-Stacklok-Proprietary + +// Private, mutually authenticated execution-provider protocol. This is not part +// of the public Harness API. +syntax = "proto3"; + +package mecatl.execution.v1; + +import "google/protobuf/empty.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1;executionv1"; + +service ExecutionProviderService { + rpc ValidateProfile(ValidateProfileRequest) returns (ValidateProfileResponse); + rpc EnsureEnvironment(EnsureEnvironmentRequest) returns (EnsureEnvironmentResponse); + rpc AttachEnvironment(AttachEnvironmentRequest) returns (AttachEnvironmentResponse); + rpc AcquireRun(AcquireRunRequest) returns (RunClaimResponse); + rpc RenewRun(RenewRunRequest) returns (RunClaimResponse); + rpc ReleaseRun(ReleaseRunRequest) returns (google.protobuf.Empty); + rpc CommitReference(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc AbortReference(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc ReserveSuccessor(ReserveSuccessorRequest) returns (ReferenceReservationResponse); + rpc PrepareReferenceDelete(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc ConfirmReferenceDelete(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc CancelReferenceDelete(ReferenceMutationRequest) returns (google.protobuf.Empty); + rpc ListReferenceIntents(ListReferenceIntentsRequest) returns (ListReferenceIntentsResponse); + rpc ReleaseReference(ReleaseReferenceRequest) returns (google.protobuf.Empty); + rpc RetireEnvironment(RetireEnvironmentRequest) returns (google.protobuf.Empty); + rpc ReplaceExecutor(ReplaceExecutorRequest) returns (google.protobuf.Empty); + rpc RecoverEnvironment(RecoverEnvironmentRequest) returns (google.protobuf.Empty); + rpc DeleteRetiredEnvironment(DeleteRetiredEnvironmentRequest) returns (google.protobuf.Empty); + rpc MigrateEnvironment(MigrateEnvironmentRequest) returns (google.protobuf.Empty); + rpc RevokeEnvironment(RevokeEnvironmentRequest) returns (RevokeEnvironmentResponse); + rpc Files(FileRequest) returns (FileResponse); + rpc StartCommand(CommandStartRequest) returns (CommandStartResponse); + rpc CommandStatus(CommandQueryRequest) returns (CommandStatusResponse); + rpc CancelCommand(CommandQueryRequest) returns (CommandStatusResponse); +} + +message EnvironmentRef { string id = 1; string revision = 2; } +message Owner { string issuer = 1; string subject = 2; } +message RequestContext { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + uint64 epoch = 4; + string grant = 5; + string run_id = 6; + string claim_id = 7; + uint64 grant_generation = 8; +} + +message ValidateProfileRequest { string profile = 1; } +message ValidateProfileResponse { + string profile = 1; + string digest = 2; + repeated string capabilities = 3; + int64 max_file_bytes = 4; + int64 max_command_bytes = 5; + int64 max_command_duration_millis = 6; +} +message EnsureEnvironmentRequest { string binding_id = 1; string profile = 2; Owner owner = 3; string operation_id = 4; } +message EnsureEnvironmentResponse { + EnvironmentRef environment = 1; + uint64 epoch = 2; + bool ready = 3; + string grant = 4; + google.protobuf.Timestamp grant_expires_at = 5; + uint64 grant_generation = 6; +} +message AttachEnvironmentRequest { RequestContext context = 1; string purpose = 2; } +message AttachEnvironmentResponse { + EnvironmentRef environment = 1; + uint64 epoch = 2; + bool ready = 3; + string grant = 4; + google.protobuf.Timestamp grant_expires_at = 5; + uint64 grant_generation = 6; +} +message AcquireRunRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + string run_id = 4; + string operation_id = 5; + int64 ttl_millis = 6; +} +message RenewRunRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + string run_id = 4; + string claim_id = 5; + uint64 epoch = 6; + string operation_id = 7; + int64 ttl_millis = 8; + uint64 grant_generation = 9; +} +message ReleaseRunRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + string run_id = 4; + string claim_id = 5; + uint64 epoch = 6; + string operation_id = 7; + uint64 grant_generation = 8; +} +message RunClaimResponse { + EnvironmentRef environment = 1; + string binding_id = 2; + string run_id = 3; + string claim_id = 4; + uint64 epoch = 5; + uint64 grant_generation = 6; + string grant = 7; + google.protobuf.Timestamp expires_at = 8; +} +message ReferenceMutationRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string binding_id = 3; + string operation_id = 4; +} +message ReserveSuccessorRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string source_binding_id = 3; + string destination_binding_id = 4; + string operation_id = 5; +} +message ReferenceReservationResponse { EnvironmentRef environment = 1; } +message ListReferenceIntentsRequest { + Owner owner = 1; + int32 limit = 2; + EnvironmentRef environment = 3; + string binding_id = 4; +} +message ReferenceIntent { + EnvironmentRef environment = 1; + string binding_id = 2; + string state = 3; + string operation_id = 4; + string source_binding_id = 5; + google.protobuf.Timestamp created_at = 6; + Owner owner = 7; +} +message ListReferenceIntentsResponse { repeated ReferenceIntent intents = 1; } +message ReleaseReferenceRequest { RequestContext context = 1; } +message RetireEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint64 expected_execution_epoch = 3; + string expected_pod_uid = 4; + string expected_pvc_uid = 5; + string operation_id = 6; +} +message ReplaceExecutorRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint64 expected_execution_epoch = 3; + string expected_pod_uid = 4; + string expected_pvc_uid = 5; + string operation_id = 6; +} +message RecoverEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint64 expected_execution_epoch = 3; + string expected_pod_uid = 4; + string expected_pvc_uid = 5; + string operation_id = 6; +} +message DeleteRetiredEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + string expected_pvc_uid = 3; + string operation_id = 4; +} +message RevokeEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + uint64 expected_grant_generation = 3; + string operation_id = 4; +} +message RevokeEnvironmentResponse { uint64 grant_generation = 1; } + +message MigrateEnvironmentRequest { + EnvironmentRef environment = 1; + Owner owner = 2; + optional uint32 expected_schema_version = 3; + string expected_pod_uid = 4; + string expected_pvc_uid = 5; + string operation_id = 6; +} + +enum FileOperation { + FILE_OPERATION_UNSPECIFIED = 0; + FILE_OPERATION_READ = 1; + FILE_OPERATION_RESOLVE_AUTHORITY = 2; + FILE_OPERATION_STAT = 3; + FILE_OPERATION_CREATE = 4; + FILE_OPERATION_REPLACE = 5; + FILE_OPERATION_LIST = 6; + FILE_OPERATION_REMOVE = 7; + FILE_OPERATION_RENAME = 8; + FILE_OPERATION_COPY = 9; + FILE_OPERATION_GLOB = 10; + FILE_OPERATION_GREP = 11; +} +message FileRequest { + RequestContext context = 1; + FileOperation operation = 2; + string path = 3; + string destination = 4; + string pattern = 5; + bytes data = 6; + bytes version = 7; + int32 limit = 8; +} +message FileInfo { + string name = 1; + int64 size = 2; + uint32 mode = 3; + google.protobuf.Timestamp mod_time = 4; + bool is_dir = 5; +} +message GrepMatch { string path = 1; int32 line = 2; string text = 3; } +message FileResponse { + bytes data = 1; + bytes version = 2; + FileInfo info = 3; + repeated FileInfo entries = 4; + repeated string paths = 5; + repeated GrepMatch matches = 6; + string authority_target = 7; + string authority_workspace = 8; +} + +enum CommandState { + COMMAND_STATE_UNSPECIFIED = 0; + COMMAND_STATE_RUNNING = 1; + COMMAND_STATE_SUCCEEDED = 2; + COMMAND_STATE_FAILED = 3; + COMMAND_STATE_CANCELLED = 4; + COMMAND_STATE_FENCE_UNKNOWN = 5; +} +message CommandStartRequest { RequestContext context = 1; string command = 2; int64 timeout_millis = 3; } +message CommandStartResponse { string command_id = 1; CommandState state = 2; CommandStatusResponse result = 3; } +message CommandQueryRequest { RequestContext context = 1; string command_id = 2; int64 offset = 3; } +message CommandStatusResponse { + string command_id = 1; + CommandState state = 2; + int32 exit_code = 3; + bytes stdout = 4; + bytes stderr = 5; + int64 next_offset = 6; + bool truncated = 7; + string terminal_receipt = 8; +} + +// ErrorDetail is a stable machine-readable classification. Status messages are +// intentionally generic so backend/Kubernetes details never cross the boundary. +message ErrorDetail { string code = 1; bool retryable = 2; } diff --git a/contracts/proto/mecatl/v1/harness.proto b/contracts/proto/mecatl/v1/harness.proto index 8e76c25463..c134fd3f5c 100644 --- a/contracts/proto/mecatl/v1/harness.proto +++ b/contracts/proto/mecatl/v1/harness.proto @@ -164,11 +164,13 @@ service HarnessService { // Derived from the snapshot taken at startup; it performs no live discovery. rpc ListAgents(ListAgentsRequest) returns (ListAgentsResponse); - // ListCommands returns slash commands discovered for one owned session's exact - // server-bound placement. It authorizes and reattaches that session before - // discovery and accepts no workspace/root input. It powers the client's command - // palette; command expansion remains a run-path concern. A no-FS session or a - // server with no command expander returns an empty list. + // ListCommands returns slash commands from the configured, resolved sources + // for one authorized, owned session. It runs independently of execution and + // accepts no client workspace/root input. A source that needs execution files + // acquires only that session's exact backend. It powers the client's command + // palette; command expansion remains a run-path concern. A server with no + // configured command sources returns an empty list; a no-FS session can retain + // independently configured sources. rpc ListCommands(ListCommandsRequest) returns (ListCommandsResponse); // ListWorktrees discovers eligible alternatives for one owned source session's diff --git a/deploy/helm/mecak8s/templates/deployment.yaml b/deploy/helm/mecak8s/templates/deployment.yaml index 9fd32c5019..78a15d6edf 100644 --- a/deploy/helm/mecak8s/templates/deployment.yaml +++ b/deploy/helm/mecak8s/templates/deployment.yaml @@ -130,6 +130,14 @@ spec: {{- if .Values.workspace }} - {{ printf "--workspace=%s" .Values.workspace | quote }} {{- end }} + {{- if .Values.execution.enabled }} + - --execution-enabled + - {{ printf "--execution-endpoint=%s" .Values.execution.endpoint | quote }} + - {{ printf "--execution-profile=%s" .Values.execution.profile | quote }} + - {{ printf "--execution-tls-ca=/var/run/secrets/execution/%s" .Values.execution.caKey | quote }} + - {{ printf "--execution-tls-cert=/var/run/secrets/execution/%s" .Values.execution.certKey | quote }} + - {{ printf "--execution-tls-key=/var/run/secrets/execution/%s" .Values.execution.keyKey | quote }} + {{- end }} {{- if .Values.mockProvider }} - --mock {{- end }} @@ -294,6 +302,9 @@ spec: {{- if .Values.oidc.caSecret }} - {name: oidc-ca, mountPath: /var/run/secrets/oidc-ca, readOnly: true} {{- end }} + {{- if .Values.execution.enabled }} + - {name: execution-mtls, mountPath: /var/run/secrets/execution, readOnly: true} + {{- end }} {{- if .Values.learning.store.tls.caSecret }} - {name: learning-store-ca, mountPath: /var/run/secrets/learning-store-ca, readOnly: true} {{- end }} @@ -342,6 +353,16 @@ spec: items: - {key: {{ .Values.oidc.caKey | quote }}, path: {{ .Values.oidc.caKey | quote }}} {{- end }} + {{- if .Values.execution.enabled }} + - name: execution-mtls + secret: + secretName: {{ .Values.execution.tlsSecret }} + defaultMode: 0440 + items: + - {key: {{ .Values.execution.caKey | quote }}, path: {{ .Values.execution.caKey | quote }}} + - {key: {{ .Values.execution.certKey | quote }}, path: {{ .Values.execution.certKey | quote }}} + - {key: {{ .Values.execution.keyKey | quote }}, path: {{ .Values.execution.keyKey | quote }}} + {{- end }} {{- if .Values.learning.store.tls.caSecret }} - name: learning-store-ca secret: diff --git a/deploy/helm/mecak8s/values.schema.json b/deploy/helm/mecak8s/values.schema.json index b17f2cd3f0..ab7daa5a15 100644 --- a/deploy/helm/mecak8s/values.schema.json +++ b/deploy/helm/mecak8s/values.schema.json @@ -116,6 +116,20 @@ } }, "workspace": {"type": "string"}, + "execution": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "endpoint", "profile", "tlsSecret", "caKey", "certKey", "keyKey"], + "properties": { + "enabled": {"type": "boolean"}, + "endpoint": {"type": "string", "anyOf": [{"const": ""}, {"pattern": "^(?:[A-Za-z0-9.-]+|\\[[0-9A-Fa-f:]+\\]):[0-9]+$"}]}, + "profile": {"type": "string"}, + "tlsSecret": {"type": "string"}, + "caKey": {"type": "string", "minLength": 1}, + "certKey": {"type": "string", "minLength": 1}, + "keyKey": {"type": "string", "minLength": 1} + } + }, "imagePullSecrets": {"type": "array", "items": {"type": "object", "additionalProperties": false, "required": ["name"], "properties": {"name": {"type": "string", "minLength": 1}}}}, "mcp": { "type": "object", @@ -371,6 +385,17 @@ "podAnnotations": {"type": "object"} }, "allOf": [ + { + "if": {"properties": {"execution": {"properties": {"enabled": {"const": true}}}}}, + "then": { + "properties": { + "workspace": {"maxLength": 0}, + "execution": {"properties": {"endpoint": {"minLength": 1}, "profile": {"minLength": 1}, "tlsSecret": {"minLength": 1}}}, + "redis": {"properties": {"filesystem": {"properties": {"enabled": {"const": false}}}}}, + "oidc": {"properties": {"enabled": {"const": true}}} + } + } + }, { "if": {"properties": {"redis": {"properties": {"filesystem": {"properties": {"enabled": {"const": true}}}}}}}, "then": {"properties": {"workspace": {"maxLength": 0}}} diff --git a/deploy/helm/mecak8s/values.yaml b/deploy/helm/mecak8s/values.yaml index b8a5519afb..52d08b62a8 100644 --- a/deploy/helm/mecak8s/values.yaml +++ b/deploy/helm/mecak8s/values.yaml @@ -160,6 +160,17 @@ podDisruptionBudget: # the volume yourself; the chart only passes the flag. workspace: "" +# Optional client for the separately installed execution-provider service. The +# provider chart is not a dependency of this chart. +execution: + enabled: false + endpoint: "" + profile: "" + tlsSecret: "" + caKey: ca.crt + certKey: tls.crt + keyKey: tls.key + redis: # A production release never creates Redis. Supply the managed endpoint and, # when the service needs a private CA or ACL credentials, a Secret holding diff --git a/deploy/helm/mecatl-execution/Chart.yaml b/deploy/helm/mecatl-execution/Chart.yaml new file mode 100644 index 0000000000..7d35df551f --- /dev/null +++ b/deploy/helm/mecatl-execution/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: mecatl-execution +version: 0.1.0-draft +appVersion: dev +description: Optional native Kubernetes execution provider (draft) +type: application diff --git a/deploy/helm/mecatl-execution/chart_test.go b/deploy/helm/mecatl-execution/chart_test.go new file mode 100644 index 0000000000..a3da6bdbad --- /dev/null +++ b/deploy/helm/mecatl-execution/chart_test.go @@ -0,0 +1,78 @@ +package chart_test + +import ( + "os" + "strings" + "testing" +) + +func TestChartRetainsCRDAndDoesNotGrantSecretAPI(t *testing.T) { + crd, err := os.ReadFile("crds/executionenvironment.yaml") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(crd), "helm.sh/resource-policy: keep") { + t.Fatal("CRD is not retained on uninstall") + } + rbac, err := os.ReadFile("templates/rbac.yaml") + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(rbac), "secrets") { + t.Fatal("provider RBAC must have no Secret access") + } + for _, required := range []string{"kind: ClusterRole", `resources: ["runtimeclasses"]`, `resources: ["storageclasses"]`, "resourceNames:", `verbs: ["get"]`} { + if !strings.Contains(string(rbac), required) { + t.Fatalf("profile preflight RBAC missing %q", required) + } + } + if strings.Contains(string(rbac), `resources: ["*"]`) { + t.Fatal("cluster-scoped profile preflight RBAC must not grant wildcard resources") + } +} +func TestChartHasNoDeletionHook(t *testing.T) { + entries, err := os.ReadDir("templates") + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + b, err := os.ReadFile("templates/" + entry.Name()) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(b), "helm.sh/hook") || strings.Contains(string(b), "PersistentVolumeClaim\nmetadata") { + t.Fatalf("unsafe lifecycle resource in %s", entry.Name()) + } + } +} + +func TestProductionHardeningIsFailClosed(t *testing.T) { + provider, err := os.ReadFile("templates/provider.yaml") + if err != nil { + t.Fatal(err) + } + network, err := os.ReadFile("templates/network-policy.yaml") + if err != nil { + t.Fatal(err) + } + values, err := os.ReadFile("values.schema.json") + if err != nil { + t.Fatal(err) + } + for _, required := range []string{"runAsNonRoot: true", "seccompProfile", "capabilities: {drop: [\"ALL\"]}", "readOnlyRootFilesystem: true", "httpGet: {path: /ready", "security-authority-configmap", "projected:", "maxUnavailable: 0"} { + if !strings.Contains(string(provider), required) { + t.Fatalf("provider hardening missing %q", required) + } + } + for _, required := range []string{"workload-default-deny", "ingress: []", "egress: []", "ResourceQuota", "LimitRange", "count/executionenvironments.execution.mecatl.dev", "requests.ephemeral-storage", "limits.ephemeral-storage"} { + if !strings.Contains(string(network), required) { + t.Fatalf("network/resource hardening missing %q", required) + } + } + if strings.Contains(string(network), "0.0.0.0/0") || !strings.Contains(string(values), `"minimum":2`) { + t.Fatal("chart permits an unsafe replica or egress default") + } +} diff --git a/deploy/helm/mecatl-execution/crds/executionenvironment.yaml b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml new file mode 100644 index 0000000000..f31fc8340c --- /dev/null +++ b/deploy/helm/mecatl-execution/crds/executionenvironment.yaml @@ -0,0 +1,213 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: executionenvironments.execution.mecatl.dev + annotations: + helm.sh/resource-policy: keep +spec: + group: execution.mecatl.dev + scope: Namespaced + names: + plural: executionenvironments + singular: executionenvironment + kind: ExecutionEnvironment + shortNames: [execenv] + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + spec: + type: object + x-kubernetes-validations: + - rule: "!has(oldSelf.allocationID) || self.allocationID == oldSelf.allocationID" + message: allocationID is immutable + - rule: "!has(oldSelf.revision) || self.revision == oldSelf.revision" + message: revision is immutable + - rule: "!has(oldSelf.ownerHash) || self.ownerHash == oldSelf.ownerHash" + message: ownerHash is immutable + - rule: "!has(oldSelf.ownerIssuer) || self.ownerIssuer == oldSelf.ownerIssuer" + message: ownerIssuer is immutable + - rule: "!has(oldSelf.ownerSubject) || self.ownerSubject == oldSelf.ownerSubject" + message: ownerSubject is immutable + - rule: "!has(oldSelf.clientHash) || self.clientHash == oldSelf.clientHash" + message: clientHash is immutable + - rule: "!has(oldSelf.bindingID) || self.bindingID == oldSelf.bindingID" + message: bindingID is immutable + - rule: "!has(oldSelf.requestFingerprint) || self.requestFingerprint == oldSelf.requestFingerprint" + message: requestFingerprint is immutable + - rule: "!has(oldSelf.profileDigest) || self.profileDigest == oldSelf.profileDigest" + message: profileDigest is immutable + - rule: "!has(oldSelf.profile) || self.profile == oldSelf.profile" + message: profile is immutable + - rule: "!has(oldSelf.image) || self.image == oldSelf.image" + message: image is immutable + - rule: "!has(oldSelf.storageClass) || self.storageClass == oldSelf.storageClass" + message: storageClass is immutable + - rule: "!has(oldSelf.storageSize) || self.storageSize == oldSelf.storageSize" + message: storageSize is immutable + - rule: "!has(oldSelf.resources) || self.resources == oldSelf.resources" + message: resources are immutable + required: [allocationID, revision, ownerHash, clientHash, bindingID, requestFingerprint, profile, profileDigest, image, storageClass, storageSize, resources, desired] + properties: + schemaVersion: {type: integer, format: int64, enum: [0, 1, 2]} + allocationID: {type: string, minLength: 1, maxLength: 63} + revision: {type: string, minLength: 1, maxLength: 64} + ownerHash: {type: string, pattern: "^[0-9a-f]{64}$"} + ownerIssuer: {type: string, minLength: 1, maxLength: 1024} + ownerSubject: {type: string, minLength: 1, maxLength: 1024} + clientHash: {type: string, pattern: "^[0-9a-f]{64}$"} + bindingID: {type: string, minLength: 1, maxLength: 253} + requestFingerprint: {type: string, pattern: "^[0-9a-f]{64}$"} + profile: {type: string, minLength: 1, maxLength: 63} + profileDigest: {type: string, pattern: "^sha256:[0-9a-f]{64}$"} + image: {type: string, pattern: "@sha256:[0-9a-fA-F]{64}$"} + storageClass: {type: string, minLength: 1} + storageSize: {type: string, minLength: 1} + resources: + type: object + required: [cpuRequest, memoryRequest, cpuLimit, memoryLimit] + properties: + cpuRequest: {type: string, minLength: 1} + memoryRequest: {type: string, minLength: 1} + cpuLimit: {type: string, minLength: 1} + memoryLimit: {type: string, minLength: 1} + desired: {type: string, enum: [Active, Retiring]} + status: + type: object + properties: + schemaVersion: {type: integer, format: int64, enum: [0, 1, 2]} + observedGeneration: {type: integer, format: int64} + epoch: {type: integer, format: int64, minimum: 1} + fenceState: {type: string, enum: [Healthy, FenceUnknown]} + grantGeneration: {type: integer, format: int64, minimum: 1} + revocationReceipts: + type: array + maxItems: 32 + items: + type: object + required: [operationID, fingerprint, expectedGrantGeneration, grantGeneration] + properties: + operationID: {type: string, minLength: 1, maxLength: 1024} + fingerprint: {type: string, pattern: "^[0-9a-f]{64}$"} + expectedGrantGeneration: {type: integer, format: int64, minimum: 1} + grantGeneration: {type: integer, format: int64, minimum: 1} + references: + type: array + maxItems: 64 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: [bindingID] + items: + type: object + required: [bindingID, state, operationID, createdAt] + properties: + bindingID: {type: string, minLength: 1, maxLength: 253} + state: {type: string, enum: [PendingCreate, Published, PendingDelete]} + operationID: {type: string, minLength: 1, maxLength: 1024} + sourceBindingID: {type: string, maxLength: 253} + createdAt: {type: string, format: date-time} + pvc: + type: object + properties: + name: {type: string} + uid: {type: string} + pod: + type: object + properties: + name: {type: string} + uid: {type: string} + activeRun: + type: object + required: [bindingID, runID, claimID, operationID, ownerHash, clientHash, epoch, grantGeneration, expiresAt] + properties: + bindingID: {type: string, maxLength: 253} + runID: {type: string, maxLength: 1024} + claimID: {type: string, maxLength: 1024} + operationID: {type: string, maxLength: 1024} + ownerHash: {type: string, pattern: "^[0-9a-f]{64}$"} + clientHash: {type: string, pattern: "^[0-9a-f]{64}$"} + epoch: {type: integer, format: int64, minimum: 1} + grantGeneration: {type: integer, format: int64, minimum: 1} + expiresAt: {type: string, format: date-time} + renewReceipts: + type: array + maxItems: 32 + items: + type: object + required: [operationID, fingerprint, expiresAt] + properties: + operationID: {type: string, minLength: 1, maxLength: 1024} + fingerprint: {type: string, pattern: "^[0-9a-f]{64}$"} + expiresAt: {type: string, format: date-time} + activeOperation: + type: object + required: [id, operation, startedAt, claimID, runID, epoch, holderID, renewedAt, expiresAt] + properties: + id: {type: string, minLength: 1, maxLength: 1024} + operation: {type: string, minLength: 1, maxLength: 128} + startedAt: {type: string, format: date-time} + claimID: {type: string, maxLength: 1024} + runID: {type: string, maxLength: 1024} + epoch: {type: integer, format: int64, minimum: 1} + holderID: {type: string, minLength: 1, maxLength: 1024} + renewedAt: {type: string, format: date-time} + expiresAt: {type: string, format: date-time} + lifecycleOperation: + type: object + required: [id, type, phase, createdAt] + properties: + id: {type: string, minLength: 1, maxLength: 1024} + type: {type: string, enum: [ReplaceExecutor, RetireEnvironment, DeleteRetiredEnvironment]} + phase: {type: string, enum: [Quiescing, WaitingForTermination, RemovingPodFinalizer, WaitingForPodDeletion, CreatingReplacement, DeletingPVC, ReleasingSlot]} + expectedEpoch: {type: integer, format: int64, minimum: 1} + expectedPodUID: {type: string, maxLength: 128} + expectedPVCUID: {type: string, minLength: 1, maxLength: 128} + createdAt: {type: string, format: date-time} + terminationProof: + type: object + required: [operationID, podUID, pvcUID, epoch, podPhase, observedAt] + properties: + operationID: {type: string, minLength: 1, maxLength: 1024} + podUID: {type: string, minLength: 1, maxLength: 128} + pvcUID: {type: string, minLength: 1, maxLength: 128} + epoch: {type: integer, format: int64, minimum: 1} + podPhase: {type: string, enum: [Succeeded, Failed]} + observedAt: {type: string, format: date-time} + lastReplacement: + type: object + required: [operationID, previousPodUID, replacementPodUID, pvcUID, previousEpoch, replacementEpoch] + properties: + operationID: {type: string, minLength: 1, maxLength: 1024} + previousPodUID: {type: string, minLength: 1, maxLength: 128} + replacementPodUID: {type: string, minLength: 1, maxLength: 128} + pvcUID: {type: string, minLength: 1, maxLength: 128} + previousEpoch: {type: integer, format: int64, minimum: 1} + replacementEpoch: {type: integer, format: int64, minimum: 2} + migrationOperation: + type: object + required: [id, fromSchema, expectedPodUID, expectedPVCUID] + properties: + id: {type: string, minLength: 1, maxLength: 1024} + fromSchema: {type: integer, format: int64, enum: [0, 1]} + expectedPodUID: {type: string, minLength: 1, maxLength: 128} + expectedPVCUID: {type: string, minLength: 1, maxLength: 128} + lastMigrationOperationID: {type: string, maxLength: 1024} + lastMigrationFromSchema: {type: integer, format: int64, enum: [0, 1]} + conditions: + type: array + items: + type: object + required: [type, status, reason, message, observedGeneration, lastTransitionTime] + properties: + type: {type: string} + status: {type: string, enum: ["True", "False", Unknown]} + reason: {type: string} + message: {type: string, maxLength: 1024} + observedGeneration: {type: integer, format: int64} + lastTransitionTime: {type: string, format: date-time} diff --git a/deploy/helm/mecatl-execution/lifetime_test.go b/deploy/helm/mecatl-execution/lifetime_test.go new file mode 100644 index 0000000000..fa4d0b2c5f --- /dev/null +++ b/deploy/helm/mecatl-execution/lifetime_test.go @@ -0,0 +1,268 @@ +package chart_test + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/yaml" +) + +// Exercise Helm's actual lookup/render path, with a loopback API containing only +// synthetic nonsecret objects. No ambient kubeconfig or cluster is consulted. +func TestChartRetainedLifetime(t *testing.T) { + fresh, err := renderLifetime(t, nil) + if err != nil { + t.Fatal(err) + } + retained := map[string]map[string]any{} + for _, obj := range fresh { + u := &unstructured.Unstructured{Object: obj} + if u.GetAnnotations()["helm.sh/resource-policy"] == "keep" { + retained[u.GetKind()+"/"+u.GetName()] = obj + } + } + if len(retained) != 6 { + t.Fatalf("retained resources=%d, want four ConfigMaps and two workload policies", len(retained)) + } + authority := "ConfigMap/test-mecatl-execution-security-authority" + capacity := "ConfigMap/mecatl-execution-profile-allocations" + retained[authority]["data"] = map[string]any{"state.json": `{"generation":7,"digest":"synthetic","fingerprints":{"k1:1":"synthetic"}}`} + retained[capacity]["data"] = map[string]any{"profile-go": `["allocation-uid"]`} + retained["ExecutionEnvironment/exec-test"] = map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "exec-test", "namespace": "ns"}} + for _, upgrade := range []bool{false, true} { + t.Run(fmt.Sprintf("reuse-upgrade-%t", upgrade), func(t *testing.T) { + var args []string + if upgrade { + args = append(args, "--is-upgrade") + } + rendered, err := renderLifetime(t, retained, args...) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{authority, capacity} { + if !reflect.DeepEqual(rendered[key]["data"], retained[key]["data"]) { + t.Fatalf("%s data was reset", key) + } + } + for key, obj := range retained { + if strings.HasPrefix(key, "NetworkPolicy/") && !reflect.DeepEqual(rendered[key]["spec"], obj["spec"]) { + t.Fatalf("%s changed confinement", key) + } + } + }) + } + tests := []struct { + name string + mutate func(map[string]map[string]any) + args []string + want string + }{ + {name: "active provider deployment", mutate: func(m map[string]map[string]any) { + m["Deployment/test-mecatl-execution"] = map[string]any{"apiVersion": "apps/v1", "kind": "Deployment", "metadata": map[string]any{"name": "test-mecatl-execution", "namespace": "ns"}, "spec": map[string]any{"replicas": 2}} + }, want: "quiesce the execution provider"}, + {name: "provider pod still terminating", mutate: func(m map[string]map[string]any) { + m["Pod/provider"] = map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": map[string]any{"name": "provider", "namespace": "ns", "deletionTimestamp": "2026-09-21T00:00:00Z", "labels": map[string]any{"app.kubernetes.io/name": "test-mecatl-execution"}}} + }, want: "wait for all execution provider Pods"}, + {name: "reserved capacity without CR", mutate: func(m map[string]map[string]any) { + delete(m, "ExecutionEnvironment/exec-test") + delete(m, authority) + }, want: "bootstrap is forbidden"}, + {name: "surviving pod without CR", mutate: func(m map[string]map[string]any) { + delete(m, "ExecutionEnvironment/exec-test") + delete(m, authority) + delete(m, capacity) + m["Pod/executor"] = map[string]any{"apiVersion": "v1", "kind": "Pod", "metadata": map[string]any{"name": "executor", "namespace": "ns", "labels": map[string]any{"execution.mecatl.dev/environment": "exec-test"}}} + }, want: "bootstrap is forbidden"}, + {name: "surviving PVC without CR", mutate: func(m map[string]map[string]any) { + delete(m, "ExecutionEnvironment/exec-test") + delete(m, authority) + delete(m, capacity) + m["PersistentVolumeClaim/workspace"] = map[string]any{"apiVersion": "v1", "kind": "PersistentVolumeClaim", "metadata": map[string]any{"name": "workspace", "namespace": "ns", "labels": map[string]any{"execution.mecatl.dev/environment": "exec-test"}}} + }, want: "bootstrap is forbidden"}, + {name: "missing lifetime configuration", mutate: func(m map[string]map[string]any) { + delete(m["ConfigMap/test-mecatl-execution-profiles"]["data"].(map[string]any), "lifetime.json") + }, want: "missing lifetime.json history"}, + {name: "missing authority", mutate: func(m map[string]map[string]any) { delete(m, authority) }, want: "bootstrap is forbidden"}, + {name: "empty authority", mutate: func(m map[string]map[string]any) { m[authority]["data"] = map[string]any{} }, want: "bootstrap is forbidden"}, + {name: "missing state", mutate: func(m map[string]map[string]any) { m[authority]["data"] = map[string]any{"other": "value"} }, want: "state.json"}, + {name: "missing capacity", mutate: func(m map[string]map[string]any) { delete(m, capacity) }, want: "bootstrap is forbidden"}, + {name: "missing deny", mutate: func(m map[string]map[string]any) { + delete(m, "NetworkPolicy/test-mecatl-execution-workload-default-deny") + }, want: "existing workload NetworkPolicies"}, + {name: "foreign release", mutate: func(m map[string]map[string]any) { + u := &unstructured.Unstructured{Object: m[capacity]} + a := u.GetAnnotations() + a["meta.helm.sh/release-name"] = "foreign" + u.SetAnnotations(a) + }, want: "foreign or ambiguous"}, + {name: "foreign namespace", mutate: func(m map[string]map[string]any) { + u := &unstructured.Unstructured{Object: m[authority]} + a := u.GetAnnotations() + a["meta.helm.sh/release-namespace"] = "foreign" + u.SetAnnotations(a) + }, want: "foreign or ambiguous"}, + {name: "unmanaged", mutate: func(m map[string]map[string]any) { + u := &unstructured.Unstructured{Object: m[authority]} + u.SetLabels(nil) + }, want: "foreign or ambiguous"}, + {name: "removed profile policy", args: []string{"--set-json", "networkPolicy.workloadProfiles={}"}, want: "configuration is incompatible"}, + {name: "changed egress", args: []string{"--set", "networkPolicy.workloadProfiles.go.egress[0].cidr=10.3.0.0/16"}, want: "configuration is incompatible"}, + {name: "changed profile", args: []string{"--set", "profiles.go.maxEnvironments=30"}, want: "configuration is incompatible"}, + {name: "changed fullname", args: []string{"--set", "fullnameOverride=other"}, want: "bootstrap is forbidden"}, + {name: "missing profiles with orphan policies", mutate: func(m map[string]map[string]any) { + delete(m, "ExecutionEnvironment/exec-test") + delete(m, "ConfigMap/test-mecatl-execution-profiles") + delete(m, capacity) + }, want: "original release profiles"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + objects := make(map[string]map[string]any, len(retained)) + for key, obj := range retained { + objects[key] = (&unstructured.Unstructured{Object: obj}).DeepCopy().Object + } + if tt.mutate != nil { + tt.mutate(objects) + } + _, err := renderLifetime(t, objects, tt.args...) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Fatalf("render error=%v, want %q", err, tt.want) + } + }) + } +} + +func renderLifetime(t *testing.T, objects map[string]map[string]any, extra ...string) (map[string]map[string]any, error) { + t.Helper() + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is required for real chart rendering") + } + resources := map[string][]map[string]any{} + for gv, kinds := range map[string][]string{ + "v1": {"ConfigMap", "Pod", "PersistentVolumeClaim", "ResourceQuota", "LimitRange", "ServiceAccount", "Service"}, + "execution.mecatl.dev/v1alpha1": {"ExecutionEnvironment"}, + "networking.k8s.io/v1": {"NetworkPolicy"}, + "apps/v1": {"Deployment"}, + "policy/v1": {"PodDisruptionBudget"}, + "rbac.authorization.k8s.io/v1": {"Role", "RoleBinding", "ClusterRole", "ClusterRoleBinding"}, + } { + for _, kind := range kinds { + plural := strings.ToLower(kind) + "s" + if kind == "NetworkPolicy" { + plural = "networkpolicies" + } + resources[gv] = append(resources[gv], map[string]any{"name": plural, "kind": kind, "namespaced": !strings.HasPrefix(kind, "Cluster")}) + } + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + var response any + switch r.URL.Path { + case "/version": + response = map[string]any{"major": "1", "minor": "35", "gitVersion": "v1.35.0"} + case "/api": + response = map[string]any{"kind": "APIVersions", "apiVersion": "v1", "versions": []string{"v1"}} + case "/apis": + groups := []any{} + for _, group := range []string{"execution.mecatl.dev", "networking.k8s.io", "apps", "policy", "rbac.authorization.k8s.io"} { + version := "v1" + if group == "execution.mecatl.dev" { + version = "v1alpha1" + } + gv := map[string]any{"groupVersion": group + "/" + version, "version": version} + groups = append(groups, map[string]any{"name": group, "versions": []any{gv}, "preferredVersion": gv}) + } + response = map[string]any{"kind": "APIGroupList", "apiVersion": "v1", "groups": groups} + default: + for gv, rs := range resources { + prefix := "/apis/" + gv + if gv == "v1" { + prefix = "/api/v1" + } + if r.URL.Path == prefix { + response = map[string]any{"kind": "APIResourceList", "apiVersion": "v1", "groupVersion": gv, "resources": rs} + break + } + for _, resource := range rs { + base := prefix + "/namespaces/ns/" + resource["name"].(string) + if r.URL.Path == base { + items := []any{} + for _, obj := range objects { + if obj["kind"] == resource["kind"] { + items = append(items, obj) + } + } + response = map[string]any{"kind": resource["kind"].(string) + "List", "apiVersion": gv, "items": items} + } else if strings.HasPrefix(r.URL.Path, base+"/") { + if obj := objects[resource["kind"].(string)+"/"+strings.TrimPrefix(r.URL.Path, base+"/")]; obj != nil { + response = obj + } + } + } + } + } + if response == nil { + w.WriteHeader(http.StatusNotFound) + response = map[string]any{"kind": "Status", "apiVersion": "v1", "status": "Failure", "reason": "NotFound", "code": 404} + } + if err := json.NewEncoder(w).Encode(response); err != nil { + t.Error(err) + } + })) + defer server.Close() + // Repo-local scratch: synthetic kubeconfig has no credentials. + if err := os.MkdirAll("../../../.scratch", 0o700); err != nil { + t.Fatal(err) + } + dir, err := os.MkdirTemp("../../../.scratch", "chart-lifetime-") + if err != nil { + t.Fatal(err) + } + config := filepath.Join(dir, "kubeconfig") + body := fmt.Sprintf("apiVersion: v1\nkind: Config\nclusters:\n- name: offline\n cluster:\n server: %s\ncontexts:\n- name: offline\n context:\n cluster: offline\n namespace: ns\ncurrent-context: offline\n", server.URL) + if err := os.WriteFile(config, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + args := []string{"template", "test", ".", "--dry-run=server", "--disable-openapi-validation", "--kubeconfig", config, "--kube-context", "offline", "--namespace", "ns", "-f", "../../mecatl-execution-kind/execution-values.yaml", "--set-string", "provider.image=example.invalid/provider@sha256:" + strings.Repeat("a", 64), "--set", "provider.securitySecretName=synthetic", "--set-literal", "provider.securityManifest={}", "--set-json", `networkPolicy.workloadProfiles={"go":{"egress":[{"cidr":"10.2.0.0/16","ports":[{"port":8080}]}]}}`} + cmd := exec.CommandContext(ctx, "helm", append(args, extra...)...) + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + dir, "KUBECONFIG=" + config} + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("helm rendering: %w: %s", err, stderr.String()) + } + result := map[string]map[string]any{} + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(out), 4096) + for { + var obj map[string]any + err := decoder.Decode(&obj) + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if len(obj) == 0 { + continue + } + u := &unstructured.Unstructured{Object: obj} + result[u.GetKind()+"/"+u.GetName()] = obj + } + return result, nil +} diff --git a/deploy/helm/mecatl-execution/templates/_helpers.tpl b/deploy/helm/mecatl-execution/templates/_helpers.tpl new file mode 100644 index 0000000000..bee4eefeb5 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/_helpers.tpl @@ -0,0 +1,26 @@ +{{- define "mecatl-execution.fullname" -}} +{{- default (printf "%s-mecatl-execution" .Release.Name) .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* Retained resources are adoptable only by their original Helm identity. */}} +{{- define "mecatl-execution.retainedMetadata" -}} +{{- $live := lookup .apiVersion .kind .root.Release.Namespace .name -}} +{{- if $live -}} +{{- if or $live.metadata.deletionTimestamp (ne (dig "app.kubernetes.io/managed-by" "" ($live.metadata.labels | default dict)) "Helm") (ne (dig "meta.helm.sh/release-name" "" ($live.metadata.annotations | default dict)) .root.Release.Name) (ne (dig "meta.helm.sh/release-namespace" "" ($live.metadata.annotations | default dict)) .root.Release.Namespace) -}} +{{- fail (printf "retained %s %s has foreign or ambiguous Helm ownership" .kind .name) -}} +{{- end -}} +{{- end -}} +name: {{ .name }} +namespace: {{ .root.Release.Namespace }} +labels: + app.kubernetes.io/managed-by: Helm +annotations: + helm.sh/resource-policy: keep + meta.helm.sh/release-name: {{ .root.Release.Name | quote }} + meta.helm.sh/release-namespace: {{ .root.Release.Namespace | quote }} +{{- end -}} + +{{/* Frozen workload identity prevents orphan allow policies from widening egress. */}} +{{- define "mecatl-execution.lifetimeConfig" -}} +{{- dict "profiles" .Values.profiles "networkPolicy" .Values.networkPolicy "securitySecretName" .Values.provider.securitySecretName | toJson -}} +{{- end -}} diff --git a/deploy/helm/mecatl-execution/templates/lifetime-check.yaml b/deploy/helm/mecatl-execution/templates/lifetime-check.yaml new file mode 100644 index 0000000000..592a7cdff3 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/lifetime-check.yaml @@ -0,0 +1,59 @@ +{{- $root := . -}} +{{- $name := include "mecatl-execution.fullname" . -}} +{{- $provider := lookup "apps/v1" "Deployment" .Release.Namespace $name -}} +{{- if gt (int (dig "spec" "replicas" 0 $provider)) 0 -}} +{{- fail "quiesce the execution provider (scale to zero and wait for its Pods to disappear) before upgrading" -}} +{{- end -}} +{{- $profiles := lookup "v1" "ConfigMap" .Release.Namespace (printf "%s-profiles" $name) -}} +{{- if $profiles -}} +{{- if not (dig "lifetime.json" "" ($profiles.data | default dict)) -}} +{{- fail "retained execution profiles are missing lifetime.json history; automatic adoption is unsupported, recover trusted retained history before upgrading" -}} +{{- end -}} +{{- if ne (dig "lifetime.json" "" ($profiles.data | default dict)) (include "mecatl-execution.lifetimeConfig" .) -}} +{{- fail "retained execution profiles/network configuration is incompatible; preserve the original lifetime configuration" -}} +{{- end -}} +{{- end -}} +{{/* A reserved slot can precede CR publication; no CR is not bootstrap proof. */}} +{{- $capacity := lookup "v1" "ConfigMap" .Release.Namespace "mecatl-execution-profile-allocations" -}} +{{- $retained := not (empty $capacity.data) -}} +{{- range (lookup "execution.mecatl.dev/v1alpha1" "ExecutionEnvironment" .Release.Namespace "").items -}} +{{- $retained = true -}} +{{- end -}} +{{- range $kind := list "Pod" "PersistentVolumeClaim" -}} +{{- range (lookup "v1" $kind $root.Release.Namespace "").items -}} +{{- if and (eq $kind "Pod") (eq (dig "app.kubernetes.io/name" "" (.metadata.labels | default dict)) $name) -}} +{{- fail "wait for all execution provider Pods to disappear before upgrading" -}} +{{- end -}} +{{- if hasKey (.metadata.labels | default dict) "execution.mecatl.dev/environment" -}} +{{- $retained = true -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- if $retained -}} +{{- range $cm := list (printf "%s-profiles" $name) (printf "%s-security-manifest" $name) (printf "%s-security-authority" $name) "mecatl-execution-profile-allocations" -}} +{{- $live := lookup "v1" "ConfigMap" $root.Release.Namespace $cm -}} +{{- if not ($live.data | default dict) -}} +{{- fail (printf "retained execution allocations require nonempty %s; authority/capacity bootstrap is forbidden" $cm) -}} +{{- end -}} +{{- end -}} +{{- $authority := lookup "v1" "ConfigMap" .Release.Namespace (printf "%s-security-authority" $name) -}} +{{- if not (dig "state.json" "" ($authority.data | default dict)) -}} +{{- fail "retained execution allocations require security authority state.json; bootstrap is forbidden" -}} +{{- end -}} +{{- $policies := list (printf "%s-workload-default-deny" $name) -}} +{{- range $profile, $_ := .Values.networkPolicy.workloadProfiles -}} +{{- $policies = append $policies (printf "%s-profile-%s" $name (sha256sum $profile | trunc 16)) -}} +{{- end -}} +{{- range $policy := $policies -}} +{{- if not (lookup "networking.k8s.io/v1" "NetworkPolicy" $root.Release.Namespace $policy) -}} +{{- fail "retained execution allocations require their existing workload NetworkPolicies" -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- range (lookup "networking.k8s.io/v1" "NetworkPolicy" .Release.Namespace "").items -}} +{{- if or (hasKey (.spec.podSelector.matchLabels | default dict) "execution.mecatl.dev/profile") (hasSuffix "-workload-default-deny" .metadata.name) -}} +{{- if not $profiles -}} +{{- fail "retained workload policies require the original release profiles ConfigMap" -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/deploy/helm/mecatl-execution/templates/network-policy.yaml b/deploy/helm/mecatl-execution/templates/network-policy.yaml new file mode 100644 index 0000000000..43f77f268b --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/network-policy.yaml @@ -0,0 +1,101 @@ +{{- if .Values.networkPolicy.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "mecatl-execution.fullname" . }}-provider +spec: + podSelector: + matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + policyTypes: [Ingress, Egress] + ingress: + - from: +{{- range .Values.provider.clientIngressSelectors }} + - namespaceSelector: + matchLabels: +{{ toYaml .namespaceLabels | indent 12 }} + podSelector: + matchLabels: +{{ toYaml .podLabels | indent 12 }} +{{- end }} + ports: + - {protocol: TCP, port: {{ .Values.service.port }}} + egress: +{{- range .Values.provider.apiServerCIDRs }} + - to: [{ipBlock: {cidr: {{ . | quote }}}}] + ports: +{{- range $.Values.networkPolicy.apiServerPorts }} + - {protocol: TCP, port: {{ . }}} +{{- end }} +{{- end }} +{{- range .Values.provider.dnsCIDRs }} + - to: [{ipBlock: {cidr: {{ . | quote }}}}] + ports: +{{- range $.Values.networkPolicy.dnsPorts }} + - {protocol: UDP, port: {{ . }}} + - {protocol: TCP, port: {{ . }}} +{{- end }} +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + {{- include "mecatl-execution.retainedMetadata" (dict "root" . "apiVersion" "networking.k8s.io/v1" "kind" "NetworkPolicy" "name" (printf "%s-workload-default-deny" (include "mecatl-execution.fullname" .))) | nindent 2 }} +spec: + podSelector: + matchExpressions: + - {key: execution.mecatl.dev/environment, operator: Exists} + policyTypes: [Ingress, Egress] + ingress: [] + egress: [] +{{- range $name, $profile := .Values.networkPolicy.workloadProfiles }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + {{- include "mecatl-execution.retainedMetadata" (dict "root" $ "apiVersion" "networking.k8s.io/v1" "kind" "NetworkPolicy" "name" (printf "%s-profile-%s" (include "mecatl-execution.fullname" $) (sha256sum $name | trunc 16))) | nindent 2 }} +spec: + podSelector: + matchLabels: + execution.mecatl.dev/profile: {{ sha256sum $name | trunc 16 }} + policyTypes: [Egress] + egress: +{{- range $profile.egress }} + - to: [{ipBlock: {cidr: {{ .cidr | quote }}}}] + ports: +{{- range .ports }} + - {protocol: {{ default "TCP" .protocol }}, port: {{ .port }}} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +--- +{{- if .Values.resourceGovernance.enabled }} +apiVersion: v1 +kind: ResourceQuota +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + hard: + pods: {{ .Values.resourceGovernance.pods | quote }} + persistentvolumeclaims: {{ .Values.resourceGovernance.persistentVolumeClaims | quote }} + count/executionenvironments.execution.mecatl.dev: {{ .Values.resourceGovernance.executionEnvironments | quote }} + requests.cpu: {{ .Values.resourceGovernance.requestsCPU | quote }} + requests.memory: {{ .Values.resourceGovernance.requestsMemory | quote }} + requests.storage: {{ .Values.resourceGovernance.requestsStorage | quote }} + requests.ephemeral-storage: {{ .Values.resourceGovernance.requestsEphemeralStorage | quote }} + limits.cpu: {{ .Values.resourceGovernance.limitsCPU | quote }} + limits.memory: {{ .Values.resourceGovernance.limitsMemory | quote }} + limits.ephemeral-storage: {{ .Values.resourceGovernance.limitsEphemeralStorage | quote }} +--- +apiVersion: v1 +kind: LimitRange +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + limits: + - type: Container + defaultRequest: {cpu: 50m, memory: 64Mi, ephemeral-storage: 32Mi} + default: {cpu: "1", memory: 1Gi, ephemeral-storage: 1Gi} + - type: PersistentVolumeClaim + max: {storage: {{ .Values.resourceGovernance.requestsStorage | quote }}} +{{- end }} diff --git a/deploy/helm/mecatl-execution/templates/pdb.yaml b/deploy/helm/mecatl-execution/templates/pdb.yaml new file mode 100644 index 0000000000..43ff755726 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/pdb.yaml @@ -0,0 +1,10 @@ +{{- if gt (int .Values.provider.replicas) 1 }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + minAvailable: 1 + selector: + matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} +{{- end }} diff --git a/deploy/helm/mecatl-execution/templates/provider.yaml b/deploy/helm/mecatl-execution/templates/provider.yaml new file mode 100644 index 0000000000..f17f92ee45 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/provider.yaml @@ -0,0 +1,113 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + {{- include "mecatl-execution.retainedMetadata" (dict "root" . "apiVersion" "v1" "kind" "ConfigMap" "name" (printf "%s-profiles" (include "mecatl-execution.fullname" .))) | nindent 2 }} +data: + lifetime.json: {{ include "mecatl-execution.lifetimeConfig" . | quote }} + profiles.yaml: | + profiles: +{{ toYaml .Values.profiles | indent 6 }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + {{- include "mecatl-execution.retainedMetadata" (dict "root" . "apiVersion" "v1" "kind" "ConfigMap" "name" (printf "%s-security-manifest" (include "mecatl-execution.fullname" .))) | nindent 2 }} +data: + manifest.json: | +{{ required "provider.securityManifest is required" .Values.provider.securityManifest | indent 4 }} +--- +{{- range $name := list (printf "%s-security-authority" (include "mecatl-execution.fullname" .)) "mecatl-execution-profile-allocations" }} +apiVersion: v1 +kind: ConfigMap +metadata: + {{- include "mecatl-execution.retainedMetadata" (dict "root" $ "apiVersion" "v1" "kind" "ConfigMap" "name" $name) | nindent 2 }} +data: {{ (lookup "v1" "ConfigMap" $.Release.Namespace $name).data | default dict | toJson }} +--- +{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + replicas: {{ .Values.provider.replicas }} + strategy: + type: RollingUpdate + rollingUpdate: {maxUnavailable: 0, maxSurge: 1} + selector: + matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + template: + metadata: + labels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + spec: + serviceAccountName: {{ include "mecatl-execution.fullname" . }} + automountServiceAccountToken: true + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + seccompProfile: {type: RuntimeDefault} + containers: + - name: provider + image: {{ required "provider.image must be digest-pinned" .Values.provider.image | quote }} + imagePullPolicy: {{ .Values.provider.imagePullPolicy }} + args: + - --namespace={{ .Release.Namespace }} + - --profiles=/etc/mecatl-execution/profiles.yaml + - --grant-keyring-manifest=/etc/mecatl-execution/security/manifest.json + - --grant-key-directory=/etc/mecatl-execution/security + - --security-authority-configmap={{ include "mecatl-execution.fullname" . }}-security-authority + - --security-reload-interval={{ .Values.provider.securityReloadInterval }} + - --max-concurrent-streams={{ .Values.provider.maxConcurrentStreams }} + - --max-concurrent-rpcs={{ .Values.provider.maxConcurrentRPCs }} + - --max-concurrent-rpcs-per-client={{ .Values.provider.maxConcurrentRPCsPerClient }} + - --health-listen=:{{ .Values.provider.healthPort }} + ports: + - {name: grpc, containerPort: 8443} + - {name: health, containerPort: {{ .Values.provider.healthPort }}} + startupProbe: + httpGet: {path: /ready, port: health} + failureThreshold: 30 + periodSeconds: 2 + readinessProbe: + httpGet: {path: /ready, port: health} + periodSeconds: 5 + livenessProbe: + httpGet: {path: /live, port: health} + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: ["ALL"]} + resources: +{{ toYaml .Values.provider.resources | indent 10 }} + volumeMounts: + - {name: profiles, mountPath: /etc/mecatl-execution/profiles.yaml, subPath: profiles.yaml, readOnly: true} + - {name: security, mountPath: /etc/mecatl-execution/security, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: profiles + configMap: {name: {{ include "mecatl-execution.fullname" . }}-profiles} + - name: security + projected: + sources: + - configMap: + name: {{ include "mecatl-execution.fullname" . }}-security-manifest + - secret: + name: {{ required "provider.securitySecretName is required" .Values.provider.securitySecretName }} + - name: tmp + emptyDir: {sizeLimit: 64Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mecatl-execution.fullname" . }} +spec: + selector: {app.kubernetes.io/name: {{ include "mecatl-execution.fullname" . }}} + ports: + - {name: grpc, port: {{ .Values.service.port }}, targetPort: grpc} diff --git a/deploy/helm/mecatl-execution/templates/rbac.yaml b/deploy/helm/mecatl-execution/templates/rbac.yaml new file mode 100644 index 0000000000..ef3dab3635 --- /dev/null +++ b/deploy/helm/mecatl-execution/templates/rbac.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "mecatl-execution.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "mecatl-execution.fullname" . }} +rules: +- apiGroups: ["execution.mecatl.dev"] + resources: ["executionenvironments"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["execution.mecatl.dev"] + resources: ["executionenvironments/status"] + verbs: ["get", "update", "patch"] +- apiGroups: [""] + resources: ["configmaps"] + resourceNames: + - {{ printf "%s-security-authority" (include "mecatl-execution.fullname" .) | quote }} + - "mecatl-execution-profile-allocations" + verbs: ["get", "update"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "mecatl-execution.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "mecatl-execution.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "mecatl-execution.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ printf "%s-%s-profile-preflight" .Release.Namespace (include "mecatl-execution.fullname" .) }} +rules: +- apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + resourceNames: +{{- range $name, $profile := .Values.profiles }} + - {{ $profile.runtimeClassName | quote }} +{{- end }} + verbs: ["get"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + resourceNames: +{{- range $name, $profile := .Values.profiles }} + - {{ $profile.storageClass | quote }} +{{- end }} + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ printf "%s-%s-profile-preflight" .Release.Namespace (include "mecatl-execution.fullname" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "mecatl-execution.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ printf "%s-%s-profile-preflight" .Release.Namespace (include "mecatl-execution.fullname" .) }} diff --git a/deploy/helm/mecatl-execution/values.schema.json b/deploy/helm/mecatl-execution/values.schema.json new file mode 100644 index 0000000000..fc37d2ea8a --- /dev/null +++ b/deploy/helm/mecatl-execution/values.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "additionalProperties": false, + "required": ["provider", "service", "profiles", "networkPolicy", "resourceGovernance"], + "properties": { + "fullnameOverride": {"type":"string"}, + "provider": { + "type":"object", "additionalProperties":false, + "required":["image","replicas","securitySecretName","securityManifest","maxConcurrentStreams","maxConcurrentRPCs","maxConcurrentRPCsPerClient","apiServerCIDRs","dnsCIDRs","resources"], + "properties": { + "image":{"type":"string","pattern":"@sha256:[0-9a-fA-F]{64}$"}, + "imagePullPolicy":{"enum":["Always","IfNotPresent","Never"]}, + "replicas":{"type":"integer","minimum":2,"maximum":32}, + "securitySecretName":{"type":"string","minLength":1}, + "securityManifest":{"type":"string","minLength":2}, + "securityReloadInterval":{"type":"string","pattern":"^[1-9][0-9]*(ms|s|m)$"}, + "maxConcurrentStreams":{"type":"integer","minimum":1,"maximum":1024}, + "maxConcurrentRPCs":{"type":"integer","minimum":1,"maximum":4096}, + "maxConcurrentRPCsPerClient":{"type":"integer","minimum":1,"maximum":4096}, + "healthPort":{"type":"integer","minimum":1,"maximum":65535}, + "clientIngressSelectors":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["namespaceLabels","podLabels"],"properties":{"namespaceLabels":{"type":"object","minProperties":1,"additionalProperties":{"type":"string"}},"podLabels":{"type":"object","minProperties":1,"additionalProperties":{"type":"string"}}}}}, + "apiServerCIDRs":{"type":"array","minItems":1,"uniqueItems":true,"items":{"type":"string","pattern":"^[0-9a-fA-F:.]+/[0-9]+$"}}, + "dnsCIDRs":{"type":"array","minItems":1,"uniqueItems":true,"items":{"type":"string","pattern":"^[0-9a-fA-F:.]+/[0-9]+$"}}, + "resources":{"type":"object","required":["requests","limits"],"properties":{"requests":{"type":"object","required":["cpu","memory","ephemeral-storage"]},"limits":{"type":"object","required":["cpu","memory","ephemeral-storage"]}}} + } + }, + "service":{"type":"object","additionalProperties":false,"required":["port"],"properties":{"port":{"type":"integer","minimum":1,"maximum":65535}}}, + "networkPolicy":{"type":"object","additionalProperties":false,"required":["enabled","dnsPorts","apiServerPorts","workloadProfiles"],"properties":{"enabled":{"const":true},"dnsPorts":{"type":"array","minItems":1,"items":{"type":"integer","minimum":1,"maximum":65535}},"apiServerPorts":{"type":"array","minItems":1,"items":{"type":"integer","minimum":1,"maximum":65535}},"workloadProfiles":{"type":"object","additionalProperties":{"type":"object","additionalProperties":false,"required":["egress"],"properties":{"egress":{"type":"array","items":{"type":"object","additionalProperties":false,"required":["cidr","ports"],"properties":{"cidr":{"type":"string","pattern":"^[0-9a-fA-F:.]+/[0-9]+$"},"ports":{"type":"array","minItems":1,"items":{"type":"object","additionalProperties":false,"required":["port"],"properties":{"port":{"type":"integer","minimum":1,"maximum":65535},"protocol":{"enum":["TCP","UDP"]}}}}}}}}}}}}, + "resourceGovernance":{"type":"object","additionalProperties":false,"required":["enabled","pods","persistentVolumeClaims","executionEnvironments","requestsCPU","requestsMemory","requestsStorage","requestsEphemeralStorage","limitsCPU","limitsMemory","limitsEphemeralStorage"],"properties":{"enabled":{"const":true},"pods":{"type":"string","minLength":1},"persistentVolumeClaims":{"type":"string","minLength":1},"executionEnvironments":{"type":"string","minLength":1},"requestsCPU":{"type":"string","minLength":1},"requestsMemory":{"type":"string","minLength":1},"requestsStorage":{"type":"string","minLength":1},"requestsEphemeralStorage":{"type":"string","minLength":1},"limitsCPU":{"type":"string","minLength":1},"limitsMemory":{"type":"string","minLength":1},"limitsEphemeralStorage":{"type":"string","minLength":1}}}, + "profiles":{"type":"object","minProperties":1,"additionalProperties":{"type":"object","additionalProperties":false,"required":["image","storageClass","storageSize","cpuRequest","memoryRequest","cpuLimit","memoryLimit","ephemeralStorageRequest","ephemeralStorageLimit","tmpSizeLimit","runtimeClassName","maxFileBytes","maxCommandBytes","maxCommandDuration","maxEnvironments"],"properties":{"image":{"type":"string","pattern":"@sha256:[0-9a-fA-F]{64}$"},"storageClass":{"type":"string","minLength":1},"storageSize":{"type":"string","minLength":1},"cpuRequest":{"type":"string","minLength":1},"memoryRequest":{"type":"string","minLength":1},"cpuLimit":{"type":"string","minLength":1},"memoryLimit":{"type":"string","minLength":1},"ephemeralStorageRequest":{"type":"string","minLength":1},"ephemeralStorageLimit":{"type":"string","minLength":1},"tmpSizeLimit":{"type":"string","minLength":1},"runtimeClassName":{"type":"string","minLength":1},"maxFileBytes":{"type":"integer","minimum":1,"maximum":5242880},"maxCommandBytes":{"type":"integer","minimum":1,"maximum":1048576},"maxCommandDuration":{"type":"string","pattern":"^[1-9][0-9]*(ms|s|m)$"},"maxEnvironments":{"type":"integer","minimum":1,"maximum":10000}}}} + } +} diff --git a/deploy/helm/mecatl-execution/values.yaml b/deploy/helm/mecatl-execution/values.yaml new file mode 100644 index 0000000000..d97077f19f --- /dev/null +++ b/deploy/helm/mecatl-execution/values.yaml @@ -0,0 +1,58 @@ +fullnameOverride: "" +provider: + image: "" + imagePullPolicy: IfNotPresent + replicas: 2 + securitySecretName: "" + securityManifest: "" + securityReloadInterval: 2s + maxConcurrentStreams: 64 + maxConcurrentRPCs: 128 + maxConcurrentRPCsPerClient: 32 + healthPort: 8081 + clientIngressSelectors: [] + apiServerCIDRs: [] + dnsCIDRs: [] + resources: + requests: {cpu: 100m, memory: 128Mi, ephemeral-storage: 64Mi} + limits: {cpu: "1", memory: 512Mi, ephemeral-storage: 256Mi} + # Deprecated single-key/TLS settings are intentionally unsupported. +service: + port: 8443 +networkPolicy: + enabled: true + dnsPorts: [53] + apiServerPorts: [443] + workloadProfiles: {} +resourceGovernance: + enabled: true + pods: "200" + persistentVolumeClaims: "100" + executionEnvironments: "100" + requestsCPU: "20" + requestsMemory: 40Gi + requestsStorage: 500Gi + requestsEphemeralStorage: 40Gi + limitsCPU: "40" + limitsMemory: 80Gi + limitsEphemeralStorage: 80Gi +profiles: {} + +# Example profile (all fields are required): +# profiles: +# go: +# image: registry.example/workload@sha256: +# storageClass: standard +# storageSize: 2Gi +# cpuRequest: 100m +# memoryRequest: 128Mi +# cpuLimit: "1" +# memoryLimit: 1Gi +# ephemeralStorageRequest: 64Mi +# ephemeralStorageLimit: 1Gi +# tmpSizeLimit: 256Mi +# runtimeClassName: gvisor +# maxFileBytes: 5242880 +# maxCommandBytes: 1048576 +# maxCommandDuration: 5m +# maxEnvironments: 100 diff --git a/deploy/mecatl-execution-kind/ci_test.go b/deploy/mecatl-execution-kind/ci_test.go new file mode 100644 index 0000000000..c2509f3a69 --- /dev/null +++ b/deploy/mecatl-execution-kind/ci_test.go @@ -0,0 +1,284 @@ +package executionkind_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/goccy/go-yaml" +) + +type workflowStep struct { + ID string `yaml:"id"` + Timeout int `yaml:"timeout-minutes"` + Run string `yaml:"run"` + If string `yaml:"if"` + Env map[string]string `yaml:"env"` + With map[string]any `yaml:"with"` +} + +func nativeSteps(t *testing.T) map[string]workflowStep { + t.Helper() + data, err := os.ReadFile("../../.github/workflows/e2e-live.yml") + if err != nil { + t.Fatal(err) + } + var workflow struct { + Jobs map[string]struct { + If string `yaml:"if"` + Steps []workflowStep `yaml:"steps"` + Timeout int `yaml:"timeout-minutes"` + } `yaml:"jobs"` + } + if err := yaml.Unmarshal(data, &workflow); err != nil { + t.Fatal(err) + } + job := workflow.Jobs["native-execution-live"] + if strings.Join(strings.Fields(job.If), " ") != "github.repository == 'stacklok/mecatl' && github.event_name == 'workflow_dispatch' && inputs.native_execution" || job.Timeout != 100 { + t.Fatal("native job must remain explicitly dispatched, repo-gated, and bounded") + } + steps := make(map[string]workflowStep) + total := 0 + for _, step := range job.Steps { + if step.Timeout <= 0 { + t.Fatal("every native step needs an explicit ceiling") + } + total += step.Timeout + if step.ID != "" { + steps[step.ID] = step + } + if strings.Contains(step.Run, "${{") { + t.Fatal("workflow expressions must cross into shell through env") + } + if step.With["cache"] == true { + t.Fatal("native credential job must not save a cache") + } + } + if total > job.Timeout-2 || steps["cleanup"].Timeout < 10 { + t.Fatal("stage ceilings must reserve cleanup and job overhead") + } + for _, id := range []string{"live", "cleanup"} { + step := steps[id] + if step.Env["MECATL_EXECUTION_QUAL_STATE"] != "${{ steps.production.outputs.state }}" || step.Env["NATIVE_CLUSTER"] != "${{ steps.production.outputs.cluster }}" || strings.Contains(step.Run, "/current") { + t.Fatal("downstream ownership must use production outputs, never current") + } + } + if steps["production"].Env["MECATL_EXECUTION_QUAL_CI"] != "0" || len(steps["credential"].Env) != 1 || steps["credential"].Env["OPENROUTER_API_KEY"] != "${{ secrets.OPENROUTER_API_KEY }}" { + t.Fatal("production retention or step-scoped credential wiring drifted") + } + if steps["credential"].If != "success() && steps.production.outcome == 'success'" || steps["live"].If != "success() && steps.production.outcome == 'success' && steps.credential.outcome == 'success'" || steps["cleanup"].If != "always() && steps.prepare.outcome == 'success'" { + t.Fatal("qualification must depend on actual production success and always clean up") + } + return steps +} + +func writeFixture(t *testing.T, path, body string, mode os.FileMode) { + t.Helper() + if err := os.WriteFile(path, []byte(body), mode); err != nil { + t.Fatal(err) + } +} + +func runStep(t *testing.T, root, script string, env ...string) ([]byte, error) { + t.Helper() + cmd := exec.CommandContext(t.Context(), "bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", script) + cmd.Dir = root + cmd.Env = append([]string{"PATH=" + os.Getenv("PATH"), "HOME=" + root}, env...) + return cmd.CombinedOutput() +} + +func TestNativeWorkflowCommitAndCredentialBoundary(t *testing.T) { + steps := nativeSteps(t) + for _, expected := range []string{"", "reviewed", "wrong", "$(touch injected)"} { + t.Run("sha="+expected, func(t *testing.T) { + root := t.TempDir() + bin := filepath.Join(root, "bin") + if err := os.Mkdir(bin, 0o700); err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(bin, "git"), "#!/bin/sh\nprintf 'reviewed\\n'\n", 0o700) + dir := filepath.Join(root, ".scratch", "ci") + out, err := runStep(t, root, steps["prepare"].Run, "PATH="+bin+":"+os.Getenv("PATH"), "NATIVE_CI_DIR="+dir, "EVENT_SHA=reviewed", "EXPECTED_SHA="+expected, "GITHUB_STEP_SUMMARY="+filepath.Join(root, "summary")) + wantOK := expected == "" || expected == "reviewed" + if (err == nil) != wantOK { + t.Fatalf("SHA gate: %v: %s", err, out) + } + if _, err := os.Stat(filepath.Join(root, "injected")); !os.IsNotExist(err) { + t.Fatal("expected SHA was executed") + } + if !wantOK { + return + } + info, err := os.Stat(dir) + if err != nil || info.Mode().Perm() != 0o700 { + t.Fatal("CI directory is not private") + } + if _, err := runStep(t, root, steps["credential"].Run, "NATIVE_CI_DIR="+dir, "OPENROUTER_API_KEY="); err == nil { + t.Fatal("missing credential skipped successfully") + } + const fake = "synthetic-offline-key" + out, err = runStep(t, root, steps["credential"].Run+"\ntest -z \"${OPENROUTER_API_KEY+x}\"", "NATIVE_CI_DIR="+dir, "OPENROUTER_API_KEY="+fake) + if err != nil || strings.Contains(string(out), fake) { + t.Fatal("credential staging failed or disclosed synthetic credential") + } + info, err = os.Stat(filepath.Join(dir, "provider-key")) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatal("credential file is not private") + } + if _, err := runStep(t, root, steps["credential"].Run, "NATIVE_CI_DIR="+dir, "OPENROUTER_API_KEY=replacement"); err == nil { + t.Fatal("credential staging overwrote existing file") + } + data, err := os.ReadFile(filepath.Join(dir, "provider-key")) + if err != nil || string(data) != fake { + t.Fatal("existing synthetic credential changed") + } + }) + } +} + +func TestNativeWorkflowCleanupOwnership(t *testing.T) { + steps := nativeSteps(t) + for _, fault := range []string{"", "owner", "context", "label", "duplicate", "symlink", "delete", "list", "switched-pointer", "captured-cluster", "missing-kubeconfig", "missing-ownership", "missing-output", "partial-with-kubeconfig", "collector-error", "collector-timeout", "collector-error-delete", "collector-timeout-delete", "live-failed", "live-failed-preserved"} { + t.Run("fault="+fault, func(t *testing.T) { + collectorFails := strings.HasPrefix(fault, "collector-") + root := t.TempDir() + state := filepath.Join(root, ".scratch", "k8s-execution", "owned") + dir := filepath.Join(root, ".scratch", "ci") + bin := filepath.Join(root, "bin") + for _, path := range []string{state, dir, bin} { + if err := os.MkdirAll(path, 0o700); err != nil { + t.Fatal(err) + } + } + const cluster = "mecatl-execution-qual-offline" + context := "kind-" + cluster + owner := "fixture" + if fault == "owner" { + owner = "someone-else" + } + if fault == "context" { + context = "ambient" + } + kubeconfig := filepath.Join(state, "kubeconfig") + if fault != "missing-kubeconfig" { + writeFixture(t, kubeconfig, "synthetic", 0o600) + } + ownership := "cluster=" + cluster + "\ncontext=" + context + "\nowner=" + owner + "\nruntime=docker\nprofile=production\nnamespace=execution-qualification\nkubeconfig=" + kubeconfig + "\n" + if fault == "duplicate" { + ownership += "owner=fixture\n" + } + if fault != "missing-ownership" { + writeFixture(t, filepath.Join(state, "ownership"), ownership, 0o600) + } + pointer := filepath.Join(filepath.Dir(state), "current") + writeFixture(t, pointer, state+"\n", 0o600) + if fault == "switched-pointer" { + foreign := filepath.Join(filepath.Dir(state), "foreign") + if err := os.Mkdir(foreign, 0o700); err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(foreign, "ownership"), strings.ReplaceAll(ownership, cluster, "mecatl-execution-qual-foreign"), 0o600) + writeFixture(t, pointer, foreign+"\n", 0o600) + } + capturedState, capturedCluster := state, cluster + if fault == "symlink" { + capturedState = filepath.Join(filepath.Dir(state), "alias") + if err := os.Symlink(state, capturedState); err != nil { + t.Fatal(err) + } + } + if fault == "captured-cluster" { + capturedCluster = "mecatl-execution-qual-other" + } + if fault == "missing-output" { + capturedState, capturedCluster = "", "" + } + writeFixture(t, filepath.Join(dir, "provider-key"), "synthetic", 0o600) + writeFixture(t, filepath.Join(state, "live-summary.json"), "{}\n", 0o600) + writeFixture(t, filepath.Join(bin, "kubectl"), "#!/bin/sh\nprintf '%s\\n' 'kind-"+cluster+"'\n", 0o700) + writeFixture(t, filepath.Join(bin, "docker"), "#!/bin/sh\nif [ \"$FAULT\" = label ]; then exit 1; fi\nprintf '%s\\n' '"+cluster+"'\n", 0o700) + writeFixture(t, filepath.Join(bin, "kind"), "#!/bin/sh\nif [ \"$1\" = delete ]; then\n test \"$*\" = 'delete cluster --name "+cluster+"' || exit 1\n printf '%s\\n' \"$*\" >> \"$MARKER.attempt\"\n case \"$FAULT\" in delete|collector-*-delete) exit 1 ;; esac\n printf deleted > \"$MARKER\"\nelse\n test \"$FAULT\" != list || exit 1\nfi\n", 0o700) + marker := filepath.Join(root, "deleted") + if fault == "delete" || fault == "list" || fault == "partial-with-kubeconfig" || collectorFails || fault == "live-failed" { + scripts := filepath.Join(root, "deploy/mecatl-execution-kind") + if err := os.MkdirAll(scripts, 0o700); err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(scripts, "collect-failure.sh"), "#!/bin/sh\ntest ! -e \"$MARKER.attempt\" || exit 1\nprintf collected > \"$MARKER.collection\"\ncase \"$FAULT\" in collector-error*) exit 17 ;; collector-timeout*) exit 124 ;; esac\nprintf '{\"kind\":\"collection\"}\\n' > \"$3\"\n", 0o700) + } + outcome := "success" + if strings.HasPrefix(fault, "missing-") || fault == "partial-with-kubeconfig" || fault == "delete" || fault == "list" || collectorFails { + outcome = "failure" + } + liveOutcome := "success" + if strings.HasPrefix(fault, "live-failed") { + liveOutcome = "failure" + } + if fault == "live-failed-preserved" { + writeFixture(t, filepath.Join(state, "live-diagnostics.jsonl"), "{\"kind\":\"create_stage\",\"stage\":\"bind_ensure\"}\n", 0o600) + writeFixture(t, filepath.Join(state, "live-diagnostics.status"), "complete\n", 0o600) + } + out, err := runStep(t, root, steps["cleanup"].Run, "PATH="+bin+":"+os.Getenv("PATH"), "NATIVE_CI_DIR="+dir, "GITHUB_WORKSPACE="+root, "USER=fixture", "PRODUCTION_OUTCOME="+outcome, "LIVE_OUTCOME="+liveOutcome, "MECATL_EXECUTION_QUAL_STATE="+capturedState, "NATIVE_CLUSTER="+capturedCluster, "FAULT="+fault, "MARKER="+marker) + wantOK := fault == "" || fault == "switched-pointer" || fault == "partial-with-kubeconfig" || fault == "collector-error" || fault == "collector-timeout" || strings.HasPrefix(fault, "live-failed") + if fault == "live-failed-preserved" { + data, err := os.ReadFile(filepath.Join(dir, "live-diagnostics.jsonl")) + if err != nil || !strings.Contains(string(data), "bind_ensure") { + t.Fatal("pre-restoration real evidence lost") + } + if _, err := os.Stat(filepath.Join(dir, "production-diagnostics.jsonl")); !os.IsNotExist(err) { + t.Fatal("mock fallback should not replace complete real evidence") + } + } + if collectorFails { + if !strings.Contains(string(out), "::warning::Bounded production diagnostics incomplete") { + t.Fatal("collector failure must warn") + } + if data, err := os.ReadFile(marker + ".collection"); err != nil || string(data) != "collected" { + t.Fatal("collector must run before deletion") + } + if data, err := os.ReadFile(marker + ".attempt"); err != nil || string(data) != "delete cluster --name "+cluster+"\n" { + t.Fatal("collector failure must not skip exact owned-cluster deletion") + } + } + if (err == nil) != wantOK { + t.Fatalf("cleanup result: %v: %s", err, out) + } + if _, err := os.Stat(filepath.Join(dir, "provider-key")); !os.IsNotExist(err) { + t.Fatal("CI credential was not removed independently") + } + if fault == "delete" || fault == "list" || fault == "partial-with-kubeconfig" || fault == "live-failed" { + if data, err := os.ReadFile(filepath.Join(dir, "production-diagnostics.jsonl")); err != nil || !strings.Contains(string(data), "collection") { + t.Fatal("failure evidence must be captured before cleanup, even when cleanup fails") + } + } + _, deleted := os.Stat(marker) + if (deleted == nil) != (wantOK || fault == "list") { + t.Fatal("wrong cluster deletion decision") + } + }) + } +} + +func TestLiveImageDigestUsesRuntimeSpecificInspect(t *testing.T) { + data, err := os.ReadFile("live.sh") + if err != nil { + t.Fatal(err) + } + start := strings.Index(string(data), "if [ \"$runtime\" = podman ]; then\n podman image exists") + end := strings.Index(string(data), "\nworkload_tag=") + if start < 0 || end < start { + t.Fatal("image build preparation missing") + } + for _, runtime := range []string{"docker", "podman"} { + t.Run(runtime, func(t *testing.T) { + root := t.TempDir() + writeFixture(t, filepath.Join(root, runtime), "#!/bin/sh\ncase \"$*\" in\n 'image exists docker.io/library/golang:1.27'|'image inspect docker.io/library/golang:1.27') exit 0 ;;\n 'image inspect docker.io/library/golang:1.27 --format {{.Digest}}') test \"$RUNTIME\" = podman || exit 1; printf 'sha256:fake' ;;\n 'image inspect docker.io/library/golang:1.27 --format {{index .RepoDigests 0}}') test \"$RUNTIME\" = docker || exit 1; printf 'golang@sha256:fake' ;;\n *) exit 1 ;;\nesac\n", 0o700) + out, err := runStep(t, root, string(data[start:end])+"\ntest \"$go_image\" = docker.io/library/golang@sha256:fake", "PATH="+root+":"+os.Getenv("PATH"), "runtime="+runtime, "RUNTIME="+runtime) + if err != nil { + t.Fatalf("digest selection: %v: %s", err, out) + } + }) + } +} diff --git a/deploy/mecatl-execution-kind/collect-failure.sh b/deploy/mecatl-execution-kind/collect-failure.sh new file mode 100644 index 0000000000..f464ef1da0 --- /dev/null +++ b/deploy/mecatl-execution-kind/collect-failure.sh @@ -0,0 +1,36 @@ +#!/bin/sh +set -eu +# Called only with the owned fixture's explicit kubeconfig/context. Each request +# is bounded; the caller also bounds the whole collector (including jq). +[ "$#" -eq 3 ] || exit 2 +kubeconfig=$1 +context=$2 +output=$3 +filters=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P) +umask 077 +set -C +exec 3> "$output" +failed=0 +# Collect logs first so a short termination grace still retains create progress. +# Drain every bounded response: no head/SIGPIPE and no raw log file, even locally. +# Fixed workloads in the already ownership-checked fixture namespace only. +for deployment in mecak8s mecatl-execution; do + if ! { + if ! kubectl --kubeconfig "$kubeconfig" --context "$context" --request-timeout=5s logs "deployment/$deployment" -n execution-qualification --all-containers=true --tail=1000 --limit-bytes=262144 2>/dev/null; then + printf '\n{"collector_unavailable":true}\n' + fi + } | jq -Rnc --arg source "$deployment" -f "$filters/create-stages.jq" >&3 2>/dev/null; then + failed=1 + fi +done +if ! { + for resource in executionenvironments pods persistentvolumeclaims resourcequotas events; do + if ! kubectl --kubeconfig "$kubeconfig" --context "$context" --request-timeout=5s get "$resource" -n execution-qualification -o json 2>/dev/null; then + printf '{"unavailable":true}\n' + fi + done +} | jq -cs -f "$filters/failure-evidence.jq" >&3 2>/dev/null; then + failed=1 + printf '{"kind":"collection","unavailable":true}\n' >&3 +fi +exit "$failed" diff --git a/deploy/mecatl-execution-kind/create-stages.jq b/deploy/mecatl-execution-kind/create-stages.jq new file mode 100644 index 0000000000..a696668ee3 --- /dev/null +++ b/deploy/mecatl-execution-kind/create-stages.jq @@ -0,0 +1,16 @@ +# No timestamps, text, stack, endpoint, owner, ref, grant or arbitrary keys survive. +def member($values): . as $v | ($values | index($v)) != null; +def count: type == "number" and . >= 0 and . <= 1000000000000 and floor == .; +[inputs | fromjson? | select(type == "object")] as $rows | +($rows | map(select(.msg == "remote create stage" and .level == "DEBUG") | + select(.stage | member(["http_handler","session_id_probe","bind_ensure","attach_poll","attach_poll_end","engine_factory","session_persist","reference_commit","http_response"])) | + select(.reason | member(["begin","ok","error","cancelled","deadline","returned","pending","ready","not_ready_retryable","not_ready_nonretryable","remote_retryable","remote_terminal","transport_error"])) | + select((.elapsed_ms | count) and (.calls | count)) | + select(.session | type == "string" and test("^([a-f0-9]{32})?$"))) | .[-512:][] | + {kind:"create_stage", source:$source, stage, reason, elapsed_ms, calls, session}), +($rows | map(select(.msg == "execution backend failure" and .level == "WARN") | + select(.operation == "delete_retired") | + select(.reason | member(["invalid_argument","unauthenticated","permission_denied","not_found","already_exists","conflict","version_mismatch","directory_not_empty","not_ready","fence_unknown","resource_exhausted","internal","invalid","timeout","rate_limited","forbidden","unauthorized","unavailable","cancelled","other"]))) | .[-128:][] | + {kind:"backend_failure", source:$source, operation, reason}), +{kind:"log_collection", source:$source, unavailable:any($rows[]; .collector_unavailable == true)}, +(if any($rows[]; .collector_unavailable == true) then "" | halt_error(1) else empty end) diff --git a/deploy/mecatl-execution-kind/execution-values.yaml b/deploy/mecatl-execution-kind/execution-values.yaml new file mode 100644 index 0000000000..a6ed5a8bc9 --- /dev/null +++ b/deploy/mecatl-execution-kind/execution-values.yaml @@ -0,0 +1,59 @@ +provider: + clientIngressSelectors: + - namespaceLabels: + kubernetes.io/metadata.name: execution-qualification + podLabels: + app.kubernetes.io/name: mecak8s + apiServerCIDRs: [172.16.0.0/12] + dnsCIDRs: [10.96.0.10/32] +profiles: + go: + image: placeholder.invalid/workload@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + storageClass: standard + storageSize: 1Gi + cpuRequest: 50m + memoryRequest: 64Mi + cpuLimit: "1" + memoryLimit: 512Mi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: qualification-runc + maxFileBytes: 5242880 + maxCommandBytes: 1048576 + maxCommandDuration: 5m + maxEnvironments: 20 + quota-cas: + image: placeholder.invalid/workload@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + storageClass: standard + storageSize: 1Gi + cpuRequest: 50m + memoryRequest: 64Mi + cpuLimit: "1" + memoryLimit: 512Mi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: qualification-runc + maxFileBytes: 5242880 + maxCommandBytes: 1048576 + maxCommandDuration: 5m + maxEnvironments: 1 + quota-kube: + image: placeholder.invalid/workload@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + storageClass: standard + storageSize: 1Gi + cpuRequest: 50m + memoryRequest: 64Mi + cpuLimit: "1" + memoryLimit: 512Mi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: qualification-runc + maxFileBytes: 5242880 + maxCommandBytes: 1048576 + maxCommandDuration: 5m + maxEnvironments: 4 +networkPolicy: + workloadProfiles: {} diff --git a/deploy/mecatl-execution-kind/failure-evidence.jq b/deploy/mecatl-execution-kind/failure-evidence.jq new file mode 100644 index 0000000000..1f6924b5cf --- /dev/null +++ b/deploy/mecatl-execution-kind/failure-evidence.jq @@ -0,0 +1,33 @@ +# Only closed vocabulary, booleans and counts may leave the fixture. Names, +# messages, UIDs, spec data, lease identities and timestamps stay in memory. +def known($allowed): . as $v | if ($allowed | index($v)) != null then $v else "other" end; +def finalizers: + [.metadata.finalizers[]? | select(. == "execution.mecatl.dev/retain-workspace" or . == "execution.mecatl.dev/verify-termination" or . == "kubernetes.io/pvc-protection")] | unique; +def quota_keys: + ["pods", "persistentvolumeclaims", "count/executionenvironments.execution.mecatl.dev", "requests.cpu", "requests.memory", "requests.storage", "requests.ephemeral-storage", "limits.cpu", "limits.memory", "limits.ephemeral-storage"]; +[.[] | .items[]?] as $items | +($items | map(select(.kind == "Pod"))) as $pods | +($items | map(select(.kind == "PersistentVolumeClaim"))) as $pvcs | +($items | map(select(.kind == "ExecutionEnvironment"))) as $envs | +{kind:"collection", unavailable:([.[] | select(.unavailable == true)] | length), environments:($envs|length), pods:($pods|length), pvcs:($pvcs|length)}, +($envs[:32][] | . as $env | + ([$pods[] | select(.metadata.name == $env.status.pod.name)][0]) as $pod | + ([$pvcs[] | select(.metadata.name == $env.status.pvc.name)][0]) as $pvc | + {kind:"environment", deleting:(.metadata.deletionTimestamp != null), finalizers:finalizers, + lifecycle:(.status.lifecycleOperation.type | known(["DeleteRetiredEnvironment","RetireEnvironment","ReplaceExecutor"])), + phase:(.status.lifecycleOperation.phase | known(["DeletingPVC","ReleasingSlot","Quiescing","WaitingForTermination","RemovingPodFinalizer","WaitingForPodDeletion","CreatingReplacement"])), + pod_present:($pod != null), pod_uid_matches:($pod != null and .status.pod.uid != null and .status.pod.uid == $pod.metadata.uid), + pvc_present:($pvc != null), pvc_uid_matches:($pvc != null and .status.pvc.uid != null and .status.pvc.uid == $pvc.metadata.uid), + operation_pvc_uid_matches:(.status.lifecycleOperation.expectedPVCUID != null and .status.lifecycleOperation.expectedPVCUID == .status.pvc.uid), + operation_pod_uid_matches:(.status.lifecycleOperation.expectedPodUID != null and .status.lifecycleOperation.expectedPodUID == .status.pod.uid), + lease_present:(.status.activeOperation.expiresAt != null), + lease_expired:((.status.activeOperation.expiresAt // "" | sub("\\.[0-9]+Z$"; "Z") | try fromdateiso8601 catch null) as $expiry | if $expiry == null then null else $expiry <= now end), + conditions:[.status.conditions[:16][]? | {type:(.type | known(["Ready","Retired","ExecutorTerminated","DeletionBlocked"])), status:(.status | known(["True","False","Unknown"])), reason:(.reason | known(["FenceUnknown","ExactLifecycleRequired","WorkspaceRetained","TerminalPodProof","AwaitingTerminalExecutor","Retained","ReplacementStarting","ReplacementReady","PVCUnavailable","ExecutorUnavailable","Reconciled","InvalidProfile","IncompatibleSchema"]))}]}), +($pods[:64][] | {kind:"pod", deleting:(.metadata.deletionTimestamp != null), finalizers:finalizers, + phase:(.status.phase | known(["Pending","Running","Succeeded","Failed","Unknown"])), + containers:([.status.containerStatuses[]?, .status.initContainerStatuses[]?, .status.ephemeralContainerStatuses[]?][:16] | map({terminated:(.state.terminated != null), reason:((.state.terminated.reason // .state.waiting.reason) | known(["Completed","Error","OOMKilled","ContainerStatusUnknown","CrashLoopBackOff","ImagePullBackOff","ErrImagePull","ContainerCreating"]))}))}), +($pvcs[:64][] | {kind:"pvc", deleting:(.metadata.deletionTimestamp != null), finalizers:finalizers}), +($items | map(select(.kind == "ResourceQuota")) | .[:32][] | . as $q | + {kind:"quota", missing_or_mismatched_keys:[quota_keys[] | . as $key | select($q.spec.hard[$key] != null and ($q.status.hard[$key] != $q.spec.hard[$key] or $q.status.used[$key] == null))]}), +($items | map(select(.kind == "Event") | .reason | known(["FailedScheduling","FailedMount","FailedAttachVolume","FailedBinding","ProvisioningFailed","FailedCreate","Failed","BackOff","Killing","Scheduled","Pulled","Created","Started"])) | group_by(.)[] | {kind:"event", reason:.[0], count:length}), +(if any(.[]; .unavailable == true) then "" | halt_error(1) else empty end) diff --git a/deploy/mecatl-execution-kind/failure_evidence_test.go b/deploy/mecatl-execution-kind/failure_evidence_test.go new file mode 100644 index 0000000000..f040e0bf42 --- /dev/null +++ b/deploy/mecatl-execution-kind/failure_evidence_test.go @@ -0,0 +1,164 @@ +package executionkind_test + +import ( + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestFailureEvidenceRejectsProducerText(t *testing.T) { + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("jq unavailable") + } + // Even a syntactically valid metadata name or reason is not an output grant. + const hostile = "PRIVATE-URL-TOKEN-COMMAND" + fixture := `{"items":[{"kind":"ExecutionEnvironment","metadata":{"name":"` + hostile + `","deletionTimestamp":"private","finalizers":["` + hostile + `","execution.mecatl.dev/retain-workspace"]},"status":{"pod":{"name":"p","uid":"expected"},"pvc":{"name":"v","uid":"expected"},"activeOperation":{"id":"` + hostile + `","expiresAt":"2000-01-01T00:00:00.123Z"},"lifecycleOperation":{"type":"DeleteRetiredEnvironment","phase":"ReleasingSlot"},"conditions":[{"type":"Ready","status":"False","reason":"FenceUnknown","message":"` + hostile + `"}]}},{"kind":"Pod","metadata":{"name":"p","uid":"foreign","finalizers":["execution.mecatl.dev/verify-termination"]},"status":{"phase":"Failed","containerStatuses":[{"name":"` + hostile + `","state":{"terminated":{"reason":"Error","message":"` + hostile + `"}}}]}},{"kind":"PersistentVolumeClaim","metadata":{"name":"v","uid":"expected","finalizers":["kubernetes.io/pvc-protection"]}},{"kind":"ResourceQuota","spec":{"hard":{"pods":"10","requests.cpu":"10","` + hostile + `":"10"}},"status":{"hard":{"pods":"10","requests.cpu":"20"},"used":{"requests.cpu":"0"}}},{"kind":"Event","reason":"` + hostile + `","message":"` + hostile + `"}]}` + cmd := exec.CommandContext(t.Context(), "jq", "-cs", "-f", "failure-evidence.jq") + cmd.Stdin = strings.NewReader(fixture) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("filter: %v: %s", err, out) + } + if strings.Contains(string(out), hostile) || len(out) > 16384 { + t.Fatal("untrusted evidence escaped") + } + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + var v map[string]any + if err := json.Unmarshal([]byte(line), &v); err != nil { + t.Fatal(err) + } + if v["kind"] == "environment" { + if v["pod_uid_matches"] != false || v["pvc_uid_matches"] != true || v["lease_expired"] != true || v["deleting"] != true || len(v["finalizers"].([]any)) != 1 { + t.Fatalf("lost diagnostic facts: %s", line) + } + } + if v["kind"] == "quota" && len(v["missing_or_mismatched_keys"].([]any)) != 2 { + t.Fatalf("lost quota mismatch: %s", line) + } + } +} + +func TestCreateStageProjectionRejectsRawAndPrivateData(t *testing.T) { + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("jq unavailable") + } + const sentinel = "PRIVATE-URL-TOKEN-OWNER-REF-STACK" + valid := `{"msg":"remote create stage","level":"DEBUG","stage":"attach_poll","reason":"not_ready_nonretryable","elapsed_ms":480000,"calls":4800,"session":"0123456789abcdef0123456789abcdef","error":"` + sentinel + `","grant":"` + sentinel + `"}` + backend := `{"msg":"execution backend failure","level":"WARN","operation":"delete_retired","reason":"invalid","error":"` + sentinel + `","resource":"` + sentinel + `"}` + input := sentinel + "\n" + `{"msg":"` + sentinel + `"}` + "\n" + valid + "\n" + backend + "\n" + for _, field := range []string{"stage", "reason", "session", "elapsed_ms", "calls"} { + var row map[string]any + if err := json.Unmarshal([]byte(valid), &row); err != nil { + t.Fatal(err) + } + row[field] = sentinel + bad, err := json.Marshal(row) + if err != nil { + t.Fatal(err) + } + input += string(bad) + "\n" + } + cmd := exec.CommandContext(t.Context(), "jq", "-Rnc", "--arg", "source", "mecak8s", "-f", "create-stages.jq") + cmd.Stdin = strings.NewReader(input) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("projection failed: %v", err) + } + if strings.Contains(string(out), sentinel) || strings.Contains(string(out), "grant") || strings.Count(string(out), `"kind":"create_stage"`) != 1 || strings.Count(string(out), `"kind":"backend_failure"`) != 1 || !strings.Contains(string(out), `"operation":"delete_retired","reason":"invalid"`) || !strings.Contains(string(out), "not_ready_nonretryable") { + t.Fatal("projection leaked or lost discriminating state") + } + cmd = exec.CommandContext(t.Context(), "jq", "-Rnc", "--arg", "source", "mecak8s", "-f", "create-stages.jq") + churn := valid + "\n" + strings.ReplaceAll(valid, "not_ready_nonretryable", "not_ready_retryable") + "\n" + terminal := strings.ReplaceAll(valid, "not_ready_nonretryable", "error") + input = strings.Repeat(churn, 750) + strings.ReplaceAll(terminal, "attach_poll", "attach_poll_end") + "\n" + strings.ReplaceAll(terminal, "attach_poll", "engine_factory") + "\n" + input += sentinel + "\n" + strings.ReplaceAll(valid, "attach_poll", sentinel) + "\n" + cmd.Stdin = strings.NewReader(input) + out, err = cmd.CombinedOutput() + if err != nil || strings.Count(string(out), `"kind":"create_stage"`) != 512 || len(out) > 262144 { + t.Fatal("projection did not drain and bound the input") + } + if !strings.Contains(string(out), `"stage":"attach_poll_end","reason":"error"`) || !strings.Contains(string(out), `"stage":"engine_factory","reason":"error"`) { + t.Fatal("projection lost latest terminal stages") + } + if strings.Contains(string(out), sentinel) || strings.Contains(string(out), "grant") { + t.Fatal("bounded projection leaked raw or private data") + } +} + +func TestFailureCollectorBoundsAndContext(t *testing.T) { + if _, err := exec.LookPath("jq"); err != nil { + t.Skip("jq unavailable") + } + root := t.TempDir() + bin := filepath.Join(root, "bin") + if err := os.Mkdir(bin, 0o700); err != nil { + t.Fatal(err) + } + writeFixture(t, filepath.Join(bin, "kubectl"), `#!/bin/sh +set -eu +test "$1 $2 $3 $4 $5" = '--kubeconfig synthetic --context kind-owned --request-timeout=5s' +if [ "$6" = logs ]; then + test "${8} ${9} ${10} ${11} ${12}" = '-n execution-qualification --all-containers=true --tail=1000 --limit-bytes=262144' + case "$7" in deployment/mecak8s|deployment/mecatl-execution) ;; *) exit 99 ;; esac + printf 'PRIVATE-RAW-STACK\n{"msg":"remote create stage","level":"DEBUG","stage":"bind_ensure","reason":"begin","calls":0,"elapsed_ms":1,"session":"","error":"PRIVATE-ERROR"}\n' + if [ "${LOG_FAILURE:-}" = 1 ]; then printf 'PRIVATE-LOG-ERROR' >&2; exit 17; fi + exit 0 +fi +test "$6 ${8} ${9} ${10} ${11}" = 'get -n execution-qualification -o json' +if [ -n "${EVIDENCE_FIXTURE:-}" ]; then cat "$EVIDENCE_FIXTURE"; exit; fi +case "$7" in +pods) printf 'PRIVATE-ERROR' >&2; exit 1 ;; +*) printf '{"items":[{"kind":"Event","reason":"FailedMount","message":"PRIVATE-DATA"}]}' ;; +esac +`, 0o700) + outPath := filepath.Join(root, "evidence") + script, err := filepath.Abs("collect-failure.sh") + if err != nil { + t.Fatal(err) + } + out, err := runStep(t, root, "sh \"$COLLECTOR\" synthetic kind-owned \"$OUT\"", "COLLECTOR="+script, "OUT="+outPath, "PATH="+bin+":"+os.Getenv("PATH")) + if err == nil { + t.Fatal("collector hid failed resource API") + } + data, err := os.ReadFile(outPath) + if err != nil || len(data) > 16384 || len(data) == 0 || strings.Contains(string(data), "PRIVATE") || strings.Contains(string(out), "PRIVATE") { + t.Fatalf("unsafe/missing evidence: %v", err) + } + if !strings.Contains(string(data), `"unavailable":1`) || strings.Count(string(data), `"kind":"create_stage"`) != 2 { + t.Fatal("API failure or projected workload stages not recorded") + } + if _, err := runStep(t, root, "sh \"$COLLECTOR\" synthetic kind-owned \"$OUT\"", "COLLECTOR="+script, "OUT="+outPath, "PATH="+bin+":"+os.Getenv("PATH")); err == nil { + t.Fatal("collector overwrote earlier real-process evidence") + } + preserved, err := os.ReadFile(outPath) + if err != nil || string(preserved) != string(data) { + t.Fatal("earlier evidence changed") + } + // A large fleet must retain late quota evidence, not exhaust a summary-sized + // cap on environment/Pod rows before reaching the quota and event sections. + container := `{"state":{"terminated":{"reason":"Completed"}}}` + pod := `{"kind":"Pod","status":{"phase":"Failed","containerStatuses":[` + strings.TrimSuffix(strings.Repeat(container+",", 16), ",") + `]}}` + fixture := filepath.Join(root, "fleet.json") + writeFixture(t, fixture, `{"items":[`+strings.Repeat(pod+",", 64)+`{"kind":"ResourceQuota","spec":{"hard":{"pods":"1"}}}]}`, 0o600) + outPath = filepath.Join(root, "fleet-evidence") + out, err = runStep(t, root, "sh \"$COLLECTOR\" synthetic kind-owned \"$OUT\"", "COLLECTOR="+script, "OUT="+outPath, "EVIDENCE_FIXTURE="+fixture, "PATH="+bin+":"+os.Getenv("PATH")) + if err != nil { + t.Fatalf("fleet collector: %v: %s", err, out) + } + data, err = os.ReadFile(outPath) + if err != nil || len(data) > 1<<20 || !strings.Contains(string(data), `"kind":"quota"`) { + t.Fatal("fleet evidence lost quota or exceeded artifact cap") + } + outPath = filepath.Join(root, "log-failure-evidence") + out, err = runStep(t, root, "sh \"$COLLECTOR\" synthetic kind-owned \"$OUT\"", "COLLECTOR="+script, "OUT="+outPath, "EVIDENCE_FIXTURE="+fixture, "LOG_FAILURE=1", "PATH="+bin+":"+os.Getenv("PATH")) + if err == nil || strings.Contains(string(out), "PRIVATE") { + t.Fatal("log API failure hidden or leaked") + } + data, err = os.ReadFile(outPath) + if err != nil || strings.Contains(string(data), "PRIVATE") || strings.Count(string(data), `"unavailable":true`) != 2 || !strings.Contains(string(data), `"kind":"quota"`) { + t.Fatal("log API failure lost safe partial evidence or skipped resources") + } +} diff --git a/deploy/mecatl-execution-kind/images.sh b/deploy/mecatl-execution-kind/images.sh new file mode 100644 index 0000000000..7f9859f582 --- /dev/null +++ b/deploy/mecatl-execution-kind/images.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# The alias name is not proof of its target: compare containerd's canonical +# descriptor before reusing it. Never overwrite a conflicting retained alias. +pin_loaded() { + tagged=$1 + images=$("$runtime" exec "${cluster}-control-plane" ctr -n k8s.io images ls) || return 1 + digest=$(printf '%s\n' "$images" | awk -v ref="$tagged" '$1 == ref {print $3; exit}') + printf '%s\n' "$digest" | grep -Eq '^sha256:[0-9a-f]{64}$' || { echo "loaded image digest unavailable or invalid" >&2; return 1; } + pinned="${tagged%:*}@${digest}" + existing=$(printf '%s\n' "$images" | awk -v ref="$pinned" '$1 == ref {print $3; exit}') + if [ -n "$existing" ]; then + [ "$existing" = "$digest" ] || { echo "loaded image alias conflicts with canonical digest" >&2; return 1; } + else + "$runtime" exec "${cluster}-control-plane" ctr -n k8s.io images tag "$tagged" "$pinned" >/dev/null || return 1 + fi + printf '%s\n' "$pinned" +} diff --git a/deploy/mecatl-execution-kind/live.sh b/deploy/mecatl-execution-kind/live.sh new file mode 100755 index 0000000000..36c9791cbb --- /dev/null +++ b/deploy/mecatl-execution-kind/live.sh @@ -0,0 +1,190 @@ +#!/bin/sh +set -eu + +# Explicit opt-in live qualification against one already-owned Kind cluster. +# Re-exec with only non-secret runtime selectors plus the credential FILE path. +if [ "${MECATL_EXECUTION_LIVE_CLEAN_ENV:-}" != 1 ]; then + exec env -i HOME="$HOME" USER="${USER:-user}" PATH="$PATH" XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" \ + DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-}" CONTAINER_HOST="${CONTAINER_HOST:-}" DOCKER_HOST="${DOCKER_HOST:-}" \ + TOOLBOX_PATH="${TOOLBOX_PATH:-}" MECATL_EXECUTION_DEV_TOOLBOX="${MECATL_EXECUTION_DEV_TOOLBOX:-}" MECATL_EXECUTION_K8S_TOOLBOX="${MECATL_EXECUTION_K8S_TOOLBOX:-}" \ + MECATL_EXECUTION_CREDENTIAL_FILE="${MECATL_EXECUTION_CREDENTIAL_FILE:-}" MECATL_EXECUTION_QUAL_STATE="${MECATL_EXECUTION_QUAL_STATE:-}" \ + MECATL_EXECUTION_LIVE_CLEAN_ENV=1 "$0" "$@" +fi +[ "$#" -eq 0 ] || { echo "usage: MECATL_EXECUTION_CREDENTIAL_FILE=/absolute/file MECATL_EXECUTION_QUAL_STATE=/owned/state $0" >&2; exit 2; } +[ -n "$MECATL_EXECUTION_CREDENTIAL_FILE" ] || { echo "MECATL_EXECUTION_CREDENTIAL_FILE is required" >&2; exit 2; } +[ -n "$MECATL_EXECUTION_QUAL_STATE" ] || { echo "MECATL_EXECUTION_QUAL_STATE is required" >&2; exit 2; } + +root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd -P) +cd "$root" +owned_root="$root/.scratch/k8s-execution" +case "${MECATL_EXECUTION_DEV_TOOLBOX:-}" in ''|dev) ;; *) echo "MECATL_EXECUTION_DEV_TOOLBOX must be empty or dev" >&2; exit 2 ;; esac +case "${MECATL_EXECUTION_K8S_TOOLBOX:-}" in ''|sre) ;; *) echo "MECATL_EXECUTION_K8S_TOOLBOX must be empty or sre" >&2; exit 2 ;; esac +state=$(CDPATH= cd -- "$MECATL_EXECUTION_QUAL_STATE" && pwd -P) || { echo "qualification state unavailable" >&2; exit 1; } +case "$state" in + "$owned_root"/*) ;; + *) echo "refusing qualification state outside owned root" >&2; exit 1 ;; +esac +ownership="$state/ownership" +[ -f "$ownership" ] && [ ! -L "$ownership" ] || { echo "owned qualification record unavailable" >&2; exit 1; } +owned_value() { + awk -F= -v key="$1" ' + $1 == key { value=substr($0, length(key)+2); count++ } + END { if (count != 1 || value == "") exit 1; print value } + ' "$ownership" +} +optional_owned_value() { + awk -F= -v key="$1" ' + $1 == key { value=substr($0, length(key)+2); count++ } + END { if (count > 1) exit 1; if (count == 1) print value } + ' "$ownership" +} +cluster=$(owned_value cluster) || { echo "owned cluster identity unavailable" >&2; exit 1; } +case "$cluster" in + [a-z0-9]|[a-z0-9]*[a-z0-9]) ;; + *) echo "owned cluster identity invalid" >&2; exit 1 ;; +esac +case "$cluster" in *[!a-z0-9-]*) echo "owned cluster identity invalid" >&2; exit 1 ;; esac +[ "${#cluster}" -le 63 ] || { echo "owned cluster identity invalid" >&2; exit 1; } +owned_namespace=$(owned_value namespace) || { echo "owned namespace unavailable" >&2; exit 1; } +[ "$owned_namespace" = execution-qualification ] || { echo "owned namespace mismatch" >&2; exit 1; } +recorded_owner=$(owned_value owner) || { echo "owned user identity unavailable" >&2; exit 1; } +[ "$recorded_owner" = "${USER:-user}" ] || { echo "owned user identity mismatch" >&2; exit 1; } +kubeconfig="$state/kubeconfig" +recorded_kubeconfig=$(owned_value kubeconfig) || { echo "owned kubeconfig identity unavailable" >&2; exit 1; } +[ "$recorded_kubeconfig" = "$kubeconfig" ] || { echo "owned kubeconfig mismatch" >&2; exit 1; } +[ -f "$kubeconfig" ] && [ ! -L "$kubeconfig" ] && [ "$(stat -c '%a' "$kubeconfig")" = 600 ] || { echo "owned kubeconfig must be a private regular file" >&2; exit 1; } +context=$(optional_owned_value context) || { echo "owned context identity invalid" >&2; exit 1; } +[ -n "$context" ] || context="kind-$cluster" +[ "$context" = "kind-$cluster" ] || { echo "owned context mismatch" >&2; exit 1; } +runtime=$(owned_value runtime) || { echo "owned container runtime unavailable" >&2; exit 1; } +[ "$runtime" = docker ] || [ "$runtime" = podman ] || { echo "owned container runtime invalid" >&2; exit 1; } +profile=$(owned_value profile) || { echo "owned qualification profile unavailable" >&2; exit 1; } +[ "$profile" = production ] || { echo "live qualification requires a completed production-state fixture" >&2; exit 1; } +label=$($runtime inspect "${cluster}-control-plane" --format '{{index .Config.Labels "io.x-k8s.kind.cluster"}}') +[ "$label" = "$cluster" ] || { echo "owned Kind container label mismatch" >&2; exit 1; } + +if [ "$runtime" = podman ]; then export KIND_EXPERIMENTAL_PROVIDER=podman; else unset KIND_EXPERIMENTAL_PROVIDER; fi +export KUBECONFIG=$kubeconfig +export REGISTRY_AUTH_FILE="$state/registry-auth.json" +dev() { + if [ -n "$MECATL_EXECUTION_DEV_TOOLBOX" ]; then toolbox run -c "$MECATL_EXECUTION_DEV_TOOLBOX" "$@"; else "$@"; fi +} +kube() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" kubectl --kubeconfig "$kubeconfig" --context "$context" "$@"; else kubectl --kubeconfig "$kubeconfig" --context "$context" "$@"; fi +} +helm_kube() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" helm --kubeconfig "$kubeconfig" --kube-context "$context" "$@"; else helm --kubeconfig "$kubeconfig" --kube-context "$context" "$@"; fi +} +[ "$(kube config current-context)" = "$context" ] || { echo "owned kube context mismatch" >&2; exit 1; } +build_ko() { package=$1 repo=$2; dev env KIND_EXPERIMENTAL_PROVIDER="${KIND_EXPERIMENTAL_PROVIDER:-}" KO_DOCKER_REPO="$repo" ko build --local --bare "$package" | tail -n 1; } +. "$root/deploy/mecatl-execution-kind/images.sh" +load_image() { + tagged=$1 + archive="$state/images/live-$(printf '%s' "$tagged" | sha256sum | cut -c1-16).tar" + $runtime save "$tagged" -o "$archive" >/dev/null + kind load image-archive "$archive" --name "$cluster" + pin_loaded "$tagged" +} + +provider_tag=$(build_ko ./cmd/mecatl-execution-provider ko.local/mecatl-execution-provider) +agent_tag=$(build_ko ./cmd/mecak8s ko.local/mecak8s) +if [ "$runtime" = podman ]; then + podman image exists docker.io/library/golang:1.27 || podman pull docker.io/library/golang:1.27 >/dev/null + go_digest=$(podman image inspect docker.io/library/golang:1.27 --format '{{.Digest}}') +else + docker image inspect docker.io/library/golang:1.27 >/dev/null 2>&1 || docker pull docker.io/library/golang:1.27 >/dev/null + go_ref=$(docker image inspect docker.io/library/golang:1.27 --format '{{index .RepoDigests 0}}') + go_digest=${go_ref#*@} +fi +go_image="docker.io/library/golang@${go_digest}" +workload_tag=localhost/mecatl-execution-workload:e2e +$runtime build --build-arg GO_IMAGE="$go_image" -f "$root/build/execution-workload/Dockerfile" -t "$workload_tag" "$root" >/dev/null +provider_image=$(load_image "$provider_tag") +agent_image=$(load_image "$agent_tag") +workload_image=$(load_image "$workload_tag") +printf 'provider=%s\nagent=%s\nworkload=%s\ngo_base=%s\n' "$provider_image" "$agent_image" "$workload_image" "$go_image" >"$state/images/live-proof" + +# Consume the already-qualified storage and synthetic security state. Preserve +# Kind's local-path helper configuration; the non-root workload is not its helper. +# Live mode never regenerates or replaces the execution Secret/keyring, or reads it back. +# A real provider credential is staged only after this deterministic rerun passes. +kube -n execution-qualification create configmap execution-mock --from-file=mock-script.json="$root/deploy/mecatl-execution-kind/mock-script.json" --dry-run=client -o yaml | kube apply -f - +kube -n execution-qualification rollout status deployment/mecatl-execution --timeout=240s +helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ + --set-string image.repository="${agent_image%@*}" --set-string image.tag= --set-string image.digest="${agent_image#*@}" --wait --timeout=5m +kube -n execution-qualification rollout restart deployment/mecak8s +kube -n execution-qualification rollout status deployment/mecak8s --timeout=240s +dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_QUAL_STATE="$state" \ + go test -tags kind_execution_e2e -run '^TestKindExecutionQualification$' -count=1 -timeout=15m ./e2e/k8s_execution + +secret="mecak8s-live-$(date -u +%Y%m%d%H%M%S)-$$" +receipt="$state/live-secret-$secret.receipt.json" +restore_needed=0 +restore() { + status=$? + trap - EXIT INT TERM + cleanup_failed=0 + mock_restored=0 + secret_cleaned=0 + if [ "$status" -ne 0 ] && [ "$restore_needed" -eq 1 ]; then + # Ownership was verified before staging. Capture the real process before + # Helm replaces it, independently of Secret cleanup and mock restoration. + set -- sh "$root/deploy/mecatl-execution-kind/collect-failure.sh" "$kubeconfig" "$context" "$state/live-diagnostics.jsonl" + if [ -n "${MECATL_EXECUTION_K8S_TOOLBOX:-}" ]; then + set -- toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" "$@" + fi + if timeout --kill-after=5s 45s "$@"; then + printf 'complete\n' > "$state/live-diagnostics.status" || echo 'warning: diagnostic status unavailable' >&2 + else + printf 'incomplete\n' > "$state/live-diagnostics.status" || echo 'warning: diagnostic status unavailable' >&2 + echo 'warning: bounded pre-restoration diagnostics incomplete' >&2 + fi + fi + if [ "$restore_needed" -eq 1 ]; then + if helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ + --set-string image.repository="${agent_image%@*}" --set-string image.tag= --set-string image.digest="${agent_image#*@}" --wait --timeout=5m >/dev/null 2>&1; then + mock_restored=1 + else + echo "mock profile restoration failed; cluster retained for inspection" >&2 + cleanup_failed=1 + fi + fi + if [ -f "$receipt" ]; then + if dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" \ + go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/credentialloader delete "$kubeconfig" "$context" "$secret" "$receipt"; then + secret_cleaned=1 + else + echo "UID-pinned run-scoped provider Secret cleanup pending; cluster retained for inspection" >&2 + cleanup_failed=1 + fi + else + secret_cleaned=1 + fi + if [ "$mock_restored" -eq 1 ] && [ "$secret_cleaned" -eq 1 ]; then + echo "cleanup verification passed: mock harness restored; run-scoped Secret receipt cleared" + fi + if [ "$status" -eq 0 ] && [ "$cleanup_failed" -ne 0 ]; then + status=1 + fi + exit "$status" +} +trap restore EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_CREDENTIAL_FILE="$MECATL_EXECUTION_CREDENTIAL_FILE" \ + go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/credentialloader stage "$kubeconfig" "$context" "$secret" "$receipt" +restore_needed=1 +helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ + --set-string image.repository="${agent_image%@*}" --set-string image.tag= --set-string image.digest="${agent_image#*@}" \ + --set mockProvider=false --set security.allowUnsafeRealProvider=true --set defaultProvider=openrouter --set-string model=anthropic/claude-haiku-4.5 --set maxRunTokens=32000 \ + --set-json 'extraArgs=["--log-level=debug","--no-soul","--no-user-model","--permissions-conventional=false","--agents-conventional=false"]' \ + --set extraVolumes=null --set extraVolumeMounts=null \ + --set-string extraEnv[0].name=OPENROUTER_API_KEY --set-string extraEnv[0].valueFrom.secretKeyRef.name="$secret" --set-string extraEnv[0].valueFrom.secretKeyRef.key=OPENROUTER_API_KEY \ + --wait --timeout=5m +kube -n execution-qualification rollout status deployment/mecak8s --timeout=240s +dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_QUAL_STATE="$state" MECATL_EXECUTION_LIVE=1 MECATL_EXECUTION_LIVE_SECRET="$secret" MECATL_EXECUTION_LIVE_CLUSTER="$cluster" \ + go test -tags kind_execution_e2e -run '^TestKindExecutionLiveQualification$' -count=1 -timeout=10m ./e2e/k8s_execution + +echo "live qualification passed; sanitized summary: $state/live-summary.json" +echo "qualification cluster retained: $cluster" diff --git a/deploy/mecatl-execution-kind/local-path.yaml b/deploy/mecatl-execution-kind/local-path.yaml new file mode 100644 index 0000000000..bb7d4991fb --- /dev/null +++ b/deploy/mecatl-execution-kind/local-path.yaml @@ -0,0 +1,33 @@ +# Recovery fixture for a Kind local-path provisioner whose helper image was +# changed during an interrupted qualification setup. Fresh clusters do not need +# this manifest. The workload image supplies /bin/sh, mkdir, and chown. +apiVersion: v1 +kind: ConfigMap +metadata: + name: local-path-config + namespace: local-path-storage +data: + config.json: |- + {"nodePathMap":[{"node":"DEFAULT_PATH_FOR_NON_LISTED_NODES","paths":["/var/local-path-provisioner"]}]} + setup: |- + #!/bin/sh + set -eu + mkdir -m 0770 -p "$VOL_DIR" + chown 65532:65532 "$VOL_DIR" + teardown: |- + #!/bin/sh + echo "retained by execution qualification; no automatic deletion" + helperPod.yaml: |- + apiVersion: v1 + kind: Pod + metadata: + name: helper-pod + spec: + securityContext: {runAsUser: 0, runAsGroup: 0} + priorityClassName: system-node-critical + tolerations: + - operator: Exists + containers: + - name: helper-pod + image: WORKLOAD_IMAGE + imagePullPolicy: IfNotPresent diff --git a/deploy/mecatl-execution-kind/mecak8s-values.yaml b/deploy/mecatl-execution-kind/mecak8s-values.yaml new file mode 100644 index 0000000000..49c04703ef --- /dev/null +++ b/deploy/mecatl-execution-kind/mecak8s-values.yaml @@ -0,0 +1,38 @@ +fullnameOverride: mecak8s +replicaCount: 1 +mockProvider: true +redis: + endpoint: redis:6379 + credentialsSecret: "" + local: + enabled: true +execution: + enabled: true + endpoint: mecatl-execution.execution-qualification.svc.cluster.local:8443 + profile: go + tlsSecret: execution-client-tls + caKey: ca.crt + certKey: tls.crt + keyKey: tls.key +oidc: + enabled: true + issuer: https://oidc-issuer.execution-qualification.svc.cluster.local:8443 + audience: mecatl + allowPrivateHTTPSIssuer: true + caSecret: execution-client-tls + caKey: ca.crt +extraArgs: +- --log-level=debug +- --mock-script=/var/run/mecatl-execution/mock-script.json +- --no-soul +- --no-user-model +- --permissions-conventional=false +- --agents-conventional=false +extraVolumeMounts: +- name: execution-mock + mountPath: /var/run/mecatl-execution + readOnly: true +extraVolumes: +- name: execution-mock + configMap: + name: execution-mock diff --git a/deploy/mecatl-execution-kind/mock-script-reattach.json b/deploy/mecatl-execution-kind/mock-script-reattach.json new file mode 100644 index 0000000000..104f83ecf3 --- /dev/null +++ b/deploy/mecatl-execution-kind/mock-script-reattach.json @@ -0,0 +1,7 @@ +{ + "turns": [ + {"tool_calls":[{"id":"reattach-read","name":"Read","args":{"path":"proof.txt"}}]}, + {"tool_calls":[{"id":"reattach-shell","name":"Shell","args":{"command":"go test ./...","timeout_ms":120000}}]}, + {"text":"REMOTE_EXECUTION_REATTACH_COMPLETE"} + ] +} diff --git a/deploy/mecatl-execution-kind/mock-script.json b/deploy/mecatl-execution-kind/mock-script.json new file mode 100644 index 0000000000..16e59416b7 --- /dev/null +++ b/deploy/mecatl-execution-kind/mock-script.json @@ -0,0 +1,16 @@ +{ + "turns": [ + {"tool_calls":[{"id":"write-proof","name":"Write","args":{"path":"proof.txt","content":"alpha\n"}}]}, + {"tool_calls":[{"id":"read-before-edit","name":"Read","args":{"path":"proof.txt"}}]}, + {"tool_calls":[{"id":"edit-proof","name":"Edit","args":{"path":"proof.txt","old_string":"alpha","new_string":"beta"}}]}, + {"tool_calls":[{"id":"copy-proof","name":"Copy","args":{"source":"proof.txt","destination":"copy.txt"}}]}, + {"tool_calls":[{"id":"move-proof","name":"Move","args":{"source":"copy.txt","destination":"moved.txt"}}]}, + {"tool_calls":[{"id":"glob-proof","name":"Glob","args":{"pattern":"*.txt"}},{"id":"grep-proof","name":"Grep","args":{"pattern":"beta","path":"*.txt"}}]}, + {"tool_calls":[{"id":"write-mod","name":"Write","args":{"path":"go.mod","content":"module example.com/qualification\n\ngo 1.27\n"}}]}, + {"tool_calls":[{"id":"write-test","name":"Write","args":{"path":"remote_test.go","content":"package qualification\n\nimport (\"os\"; \"testing\")\n\nfunc TestRemoteWorkspace(t *testing.T) { b, err := os.ReadFile(\"proof.txt\"); if err != nil { t.Fatal(err) }; if string(b) != \"beta\\n\" { t.Fatalf(\"proof = %q\", b) } }\n"}}]}, + {"tool_calls":[{"id":"shell-go-test","name":"Shell","args":{"command":"go test ./...","timeout_ms":120000}}]}, + {"tool_calls":[{"id":"remove-moved","name":"Remove","args":{"path":"moved.txt"}}]}, + {"tool_calls":[{"id":"read-final","name":"Read","args":{"path":"proof.txt"}}]}, + {"text":"REMOTE_EXECUTION_QUALIFICATION_COMPLETE"} + ] +} diff --git a/deploy/mecatl-execution-kind/oidc.yaml b/deploy/mecatl-execution-kind/oidc.yaml new file mode 100644 index 0000000000..5f9775c216 --- /dev/null +++ b/deploy/mecatl-execution-kind/oidc.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oidc-issuer + namespace: execution-qualification +spec: + replicas: 1 + selector: + matchLabels: {app.kubernetes.io/name: oidc-issuer} + template: + metadata: + labels: {app.kubernetes.io/name: oidc-issuer} + spec: + automountServiceAccountToken: false + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + seccompProfile: {type: RuntimeDefault} + containers: + - name: issuer + image: OIDC_IMAGE + imagePullPolicy: IfNotPresent + args: + - --tls-cert=/identity/tls.crt + - --tls-key=/identity/tls.key + - --signing-key=/identity/jwt-key.pem + - --issuer=https://oidc-issuer.execution-qualification.svc.cluster.local:8443 + - --audience=mecatl + ports: [{name: https, containerPort: 8443}] + readinessProbe: {httpGet: {scheme: HTTPS, path: /healthz, port: https}, periodSeconds: 2} + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: {drop: ["ALL"]} + resources: + requests: {cpu: 10m, memory: 32Mi} + limits: {cpu: 200m, memory: 128Mi} + volumeMounts: + - {name: identity, mountPath: /identity, readOnly: true} + - {name: tmp, mountPath: /tmp} + volumes: + - name: identity + secret: {secretName: oidc-fixture-identity, defaultMode: 0440} + - name: tmp + emptyDir: {sizeLimit: 16Mi} +--- +apiVersion: v1 +kind: Service +metadata: + name: oidc-issuer + namespace: execution-qualification +spec: + selector: {app.kubernetes.io/name: oidc-issuer} + ports: [{name: https, port: 8443, targetPort: https}] diff --git a/deploy/mecatl-execution-kind/repair_test.go b/deploy/mecatl-execution-kind/repair_test.go new file mode 100644 index 0000000000..565ba14c90 --- /dev/null +++ b/deploy/mecatl-execution-kind/repair_test.go @@ -0,0 +1,322 @@ +package executionkind_test + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func scriptRange(t *testing.T, file, start, end string) string { + t.Helper() + data, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + body := string(data) + i, j := strings.Index(body, start), strings.Index(body, end) + if i < 0 || j <= i { + t.Fatalf("missing script range in %s", file) + } + return body[i:j] +} + +func TestProductionHelmSkipsOnlyTheManuallyInstalledLegacyCRD(t *testing.T) { + deploy := scriptRange(t, "run.sh", "set -- upgrade --install mecatl-execution", "\nif [ \"${MECATL_EXECUTION_QUAL_PROFILE:-development}\" = production ]; then\n # Establish") + for _, profile := range []string{"development", "production"} { + t.Run(profile, func(t *testing.T) { + root := t.TempDir() + marker := filepath.Join(root, "helm-args") + out, err := runStep(t, root, ` +helm_kube() { printf '%s\n' "$@" > "$MARKER"; } +`+deploy, "root="+root, "state="+root, "runtime_values="+filepath.Join(root, "runtime-values.yaml"), "provider_image=provider", "workload_image=workload", "MECATL_EXECUTION_QUAL_PROFILE="+profile, "MARKER="+marker) + if err != nil { + t.Fatalf("deploy fixture: %v: %s", err, out) + } + args, err := os.ReadFile(marker) + if err != nil { + t.Fatal(err) + } + hasSkip := strings.Contains("\n"+string(args), "\n--skip-crds\n") + if hasSkip != (profile == "production") { + t.Fatalf("profile %s --skip-crds=%v, want production only; args=%s", profile, hasSkip, args) + } + if strings.Contains("\n"+string(args), "\n--server-side=false\n") { + t.Fatal("initial fixture install must preserve Helm 4 default apply mode for the migration proof") + } + }) + } +} + +func TestProductionThenLiveImageAlias(t *testing.T) { + production := scriptRange(t, "run.sh", "for item in", "printf 'provider=%s") + live := scriptRange(t, "live.sh", ". \"$root/deploy/mecatl-execution-kind/images.sh\"", "\nprovider_tag=$(build_ko") + helper, err := os.ReadFile("images.sh") + if err != nil { + t.Fatal(err) + } + for _, runtime := range []string{"docker", "podman"} { + for _, conflict := range []bool{false, true} { + t.Run(runtime+"/conflict="+map[bool]string{false: "false", true: "true"}[conflict], func(t *testing.T) { + root := t.TempDir() + for _, dir := range []string{"bin", "images", "deploy/mecatl-execution-kind"} { + if err := os.MkdirAll(filepath.Join(root, dir), 0o700); err != nil { + t.Fatal(err) + } + } + writeFixture(t, filepath.Join(root, "deploy/mecatl-execution-kind/images.sh"), string(helper), 0o600) + digest := "sha256:" + strings.Repeat("a", 64) + tag := "ko.local/provider:head" + alias := "ko.local/provider@" + digest + db := filepath.Join(root, "db") + var sourceRows strings.Builder + for _, name := range []string{"provider", "agent", "oidc", "netprobe", "workload"} { + sourceRows.WriteString("ko.local/" + name + ":head type " + digest + "\n") + } + writeFixture(t, db, sourceRows.String(), 0o600) + writeFixture(t, filepath.Join(root, "bin", runtime), `#!/bin/sh +set -eu +case "$1" in +save) : > "$4" ;; +exec) + test "$2" = owned-control-plane + shift 5 + test "$1" = images + case "$2" in + ls) cat "$DB" ;; + tag) + test "$#" = 4 + if awk -v ref="$4" '$1 == ref {found=1} END {exit !found}' "$DB"; then + echo 'already exists' >&2; exit 1 + fi + digest=$(awk -v ref="$3" '$1 == ref {print $3}' "$DB") + printf '%s type %s\n' "$4" "$digest" >> "$DB" + printf 'tag\n' >> "$TAGS" + ;; + *) exit 1 ;; + esac ;; +*) exit 1 ;; +esac +`, 0o700) + writeFixture(t, filepath.Join(root, "bin/kind"), "#!/bin/sh\nset -eu\ntest \"$1 $2\" = 'load image-archive'\ntest -f \"$3\"\ntest \"$4 $5\" = '--name owned'\n", 0o700) + env := []string{"PATH=" + filepath.Join(root, "bin") + ":" + os.Getenv("PATH"), "root=" + root, "state=" + root, "runtime=" + runtime, "cluster=owned", "DB=" + db, "TAGS=" + filepath.Join(root, "tags")} + for _, name := range []string{"provider", "agent", "oidc", "netprobe", "workload"} { + env = append(env, name+"_tag=ko.local/"+name+":head") + } + out, err := runStep(t, root, production, env...) + if err != nil { + t.Fatalf("fresh production load: %v: %s", err, out) + } + if conflict { + writeFixture(t, db, tag+" type "+digest+"\n"+alias+" type sha256:"+strings.Repeat("b", 64)+"\n", 0o600) + } + out, err = runStep(t, root, live+"\nload_image \"$provider_tag\"", env...) + if (err == nil) == conflict { + t.Fatalf("retained live load: %v: %s", err, out) + } + if !conflict && strings.TrimSpace(string(out)) != alias { + t.Fatalf("wrong alias: %s", out) + } + tags, err := os.ReadFile(filepath.Join(root, "tags")) + if err != nil || string(tags) != strings.Repeat("tag\n", 5) { + t.Fatalf("must create each alias once, never overwrite/re-tag: %q, %v", tags, err) + } + }) + } + } +} + +func TestLivePreservesQualifiedStorageAndHelmDigestThroughRestoration(t *testing.T) { + live := scriptRange(t, "live.sh", "printf 'provider=%s", "\necho \"live qualification passed") + root := t.TempDir() + if err := os.Mkdir(filepath.Join(root, "images"), 0o700); err != nil { + t.Fatal(err) + } + marker := filepath.Join(root, "helm-calls") + kubeMarker := filepath.Join(root, "kube-calls") + devMarker := filepath.Join(root, "dev-calls") + digest := "sha256:" + strings.Repeat("a", 64) + out, err := runStep(t, root, ` +set -u +kube() { + case "$*" in + "-n execution-qualification create configmap execution-mock --from-file=mock-script.json=$root/deploy/mecatl-execution-kind/mock-script.json --dry-run=client -o yaml") + printf 'mock-config\n' >> "$KUBE_MARKER" + printf 'synthetic mock config\n' ;; + 'apply -f -') + payload=$(cat) + test "$payload" = 'synthetic mock config' || { echo 'forbidden applied manifest' >&2; return 1; } ;; + '-n execution-qualification rollout status deployment/mecatl-execution --timeout=240s') + printf 'provider-ready\n' >> "$KUBE_MARKER" ;; + '-n execution-qualification rollout restart deployment/mecak8s') + printf 'agent-restart\n' >> "$KUBE_MARKER" ;; + '-n execution-qualification rollout status deployment/mecak8s --timeout=240s') + printf 'agent-ready\n' >> "$KUBE_MARKER" ;; + *) echo "forbidden Kubernetes operation: $*" >&2; return 1 ;; + esac +} +kube_jq() { jq "$@"; } +dev() { + case "$*" in + *'go test -tags kind_execution_e2e -run ^TestKindExecutionQualification$ '*) printf 'mock-test\n' >> "$DEV_MARKER" ;; + *'go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/credentialloader stage '*) printf 'stage\n' >> "$DEV_MARKER" ;; + *'go test -tags kind_execution_e2e -run ^TestKindExecutionLiveQualification$ '*) printf 'live-test\n' >> "$DEV_MARKER" ;; + *) echo "unexpected dev operation: $*" >&2; return 1 ;; + esac +} +helm_kube() { + test "$1 $2 $3" = 'upgrade --install mecak8s' || return 1 + # Model a nonempty inherited tag; apply the actual CLI overrides in order. + tag=e2e digest= repository= profile=mock + while [ "$#" -gt 0 ]; do + case "$1" in + --set-string|--set) + shift + case "$1" in + image.tag=*) tag=${1#*=} ;; + image.digest=*) digest=${1#*=} ;; + image.repository=*) repository=${1#*=} ;; + mockProvider=false) profile=live ;; + esac ;; + esac + shift + done + test -z "$tag" && test "$repository@$digest" = "$agent_image" || return 1 + printf '%s\n' "$profile" >> "$MARKER" +} +`+live, "root="+root, "state="+root, "MARKER="+marker, "KUBE_MARKER="+kubeMarker, "DEV_MARKER="+devMarker, + "agent_image=ko.local/mecak8s@"+digest, "provider_image=synthetic", "workload_image=synthetic", "go_image=synthetic", + "kubeconfig=synthetic", "context=synthetic", + "cluster=owned", "MECATL_EXECUTION_CREDENTIAL_FILE=unused") + if err != nil { + t.Fatalf("live must preserve qualified storage and use digest-only Helm images: %v: %s", err, out) + } + calls, err := os.ReadFile(marker) + if err != nil || string(calls) != "mock\nlive\nmock\n" { + t.Fatalf("expected digest-only mock setup, live upgrade, and mock restoration: %q: %v", calls, err) + } + calls, err = os.ReadFile(kubeMarker) + if err != nil || string(calls) != "mock-config\nprovider-ready\nagent-restart\nagent-ready\nagent-ready\n" { + t.Fatalf("expected only mock configuration and readiness operations: %q: %v", calls, err) + } + calls, err = os.ReadFile(devMarker) + if err != nil || string(calls) != "mock-test\nstage\nlive-test\n" { + t.Fatalf("expected mock qualification before credential staging and live qualification: %q: %v", calls, err) + } +} + +func TestLiveSignalPreservesFailureAndAttemptsCleanup(t *testing.T) { + restore := scriptRange(t, "live.sh", "restore() {", "\ndev env -i HOME=\"$HOME\" PATH=\"$PATH\" KUBECONFIG=\"$kubeconfig\" MECATL_KUBE_CONTEXT=\"$context\" MECATL_EXECUTION_CREDENTIAL_FILE=") + for _, signal := range []struct { + name string + code string + command string + }{{"INT", "130", "kill -INT $$"}, {"TERM", "143", "kill -TERM $$"}, {"live-failed", "1", "exit 1"}, {"collector-failed", "1", "exit 1"}} { + t.Run(signal.name, func(t *testing.T) { + root := t.TempDir() + scripts := filepath.Join(root, "deploy/mecatl-execution-kind") + if err := os.MkdirAll(scripts, 0o700); err != nil { + t.Fatal(err) + } + collector := "#!/bin/sh\nset -eu\ntest \"$1 $2\" = 'synthetic kind-owned'\nprintf 'collect\\n' >> \"$MARKER\"\nprintf '{\"kind\":\"collection\"}\\n' > \"$3\"\n" + if signal.name == "collector-failed" { + collector += "exit 17\n" + } + writeFixture(t, filepath.Join(scripts, "collect-failure.sh"), collector, 0o700) + writeFixture(t, filepath.Join(root, "receipt"), "synthetic", 0o600) + writeFixture(t, filepath.Join(root, "signal.sh"), "#!/bin/sh\nset -eu\n"+` +helm_kube() { printf 'restore\n' >> "$MARKER"; } +dev() { printf 'delete\n' >> "$MARKER"; } +restore_needed=1 +`+restore+"\n"+signal.command+"\nexit 99\n", 0o700) + out, err := runStep(t, root, "if sh ./signal.sh; then exit 1; else test \"$?\" -eq "+signal.code+"; fi", "root="+root, "state="+root, "MARKER="+filepath.Join(root, "cleanup"), "receipt="+filepath.Join(root, "receipt"), "agent_image=synthetic@sha256:fake", "kubeconfig=synthetic", "context=kind-owned", "secret=synthetic") + if err != nil { + t.Fatalf("signal was reported as success: %v: %s", err, out) + } + calls, err := os.ReadFile(filepath.Join(root, "cleanup")) + if err != nil || string(calls) != "collect\nrestore\ndelete\n" { + t.Fatalf("evidence must precede restoration and UID cleanup, even on collection failure: %q: %v", calls, err) + } + status, err := os.ReadFile(filepath.Join(root, "live-diagnostics.status")) + want := "complete\n" + if signal.name == "collector-failed" { + want = "incomplete\n" + } + if err != nil || string(status) != want { + t.Fatal("diagnostic failure status lost") + } + }) + } +} + +func TestProductionPublishesOwnershipBeforePartialCreation(t *testing.T) { + data, err := os.ReadFile("run.sh") + if err != nil { + t.Fatal(err) + } + for _, fault := range []string{"partial", "discovery", "collision"} { + t.Run(fault, func(t *testing.T) { + root := t.TempDir() + scripts := filepath.Join(root, "deploy/mecatl-execution-kind") + bin := filepath.Join(root, "bin") + for _, dir := range []string{scripts, bin} { + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Fatal(err) + } + } + writeFixture(t, filepath.Join(scripts, "run.sh"), string(data), 0o700) + writeFixture(t, filepath.Join(bin, "docker"), "#!/bin/sh\nexit 99\n", 0o700) + writeFixture(t, filepath.Join(root, "fault"), fault+"\n", 0o600) + writeFixture(t, filepath.Join(bin, "kind"), `#!/bin/sh +set -eu +IFS= read -r fault < "$HOME/fault" +case "$1" in +get) + case "$fault" in + discovery) exit 1 ;; + collision) awk -F= '$1 == "cluster" {print $2}' .scratch/k8s-execution/*/ownership ;; + esac ;; +create) + test -s "$MECATL_EXECUTION_QUAL_OUTPUT" + printf 'creation attempted\n' > "$HOME/attempted" + exit 1 ;; +*) exit 99 ;; +esac +`, 0o700) + output := filepath.Join(root, "output") + marker := filepath.Join(root, "attempted") + out, err := runStep(t, root, "./deploy/mecatl-execution-kind/run.sh", "PATH="+bin+":"+os.Getenv("PATH"), "CONTAINER_ENGINE=docker", "MECATL_EXECUTION_QUAL_PROFILE=production", "MECATL_EXECUTION_QUAL_OUTPUT="+output) + if err == nil { + t.Fatalf("partial production unexpectedly succeeded: %s", out) + } + outputs, err := os.ReadFile(output) + if fault != "partial" { + if !os.IsNotExist(err) { + t.Fatal("ownership published before collision/discovery gate") + } + if _, err := os.Stat(marker); !os.IsNotExist(err) { + t.Fatal("creation attempted after failed preflight") + } + return + } + if err != nil { + t.Fatal(err) + } + lines := strings.Split(strings.TrimSpace(string(outputs)), "\n") + if len(lines) != 2 || !strings.HasPrefix(lines[0], "state=") || !strings.HasPrefix(lines[1], "cluster=mecatl-execution-qual-") { + t.Fatalf("invalid ownership outputs: %q", outputs) + } + state := strings.TrimPrefix(lines[0], "state=") + ownership, err := os.ReadFile(filepath.Join(state, "ownership")) + if err != nil || !strings.Contains(string(ownership), lines[1]+"\n") { + t.Fatal("output does not bind the initial ownership record") + } + if _, err := os.Stat(marker); err != nil { + t.Fatal("fixture did not reach partial cluster creation") + } + if _, err := os.Stat(filepath.Join(state, "kubeconfig")); !os.IsNotExist(err) { + t.Fatal("partial fixture unexpectedly has a kubeconfig") + } + }) + } +} diff --git a/deploy/mecatl-execution-kind/run.sh b/deploy/mecatl-execution-kind/run.sh new file mode 100755 index 0000000000..1780a6cfac --- /dev/null +++ b/deploy/mecatl-execution-kind/run.sh @@ -0,0 +1,353 @@ +#!/bin/sh +set -eu + +# Re-exec once with a deliberately small environment. In particular, no +# provider/API credential or ambient kube context can reach any subprocess. +if [ "${MECATL_EXECUTION_QUAL_CLEAN_ENV:-}" != 1 ]; then + exec env -i \ + HOME="$HOME" USER="${USER:-user}" PATH="$PATH" XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" \ + DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-}" CONTAINER_HOST="${CONTAINER_HOST:-}" DOCKER_HOST="${DOCKER_HOST:-}" \ + CONTAINER_ENGINE="${CONTAINER_ENGINE:-}" MECATL_EXECUTION_QUAL_CI="${MECATL_EXECUTION_QUAL_CI:-}" MECATL_EXECUTION_QUAL_PROFILE="${MECATL_EXECUTION_QUAL_PROFILE:-}" \ + TOOLBOX_PATH="${TOOLBOX_PATH:-}" MECATL_EXECUTION_DEV_TOOLBOX="${MECATL_EXECUTION_DEV_TOOLBOX:-}" MECATL_EXECUTION_K8S_TOOLBOX="${MECATL_EXECUTION_K8S_TOOLBOX:-}" \ + MECATL_EXECUTION_QUAL_OUTPUT="${MECATL_EXECUTION_QUAL_OUTPUT:-}" \ + MECATL_EXECUTION_QUAL_CLEAN_ENV=1 "$0" "$@" +fi + +[ "$#" -eq 0 ] || { echo "usage: $0" >&2; exit 2; } +case "${MECATL_EXECUTION_DEV_TOOLBOX:-}" in ''|dev) ;; *) echo "MECATL_EXECUTION_DEV_TOOLBOX must be empty or dev" >&2; exit 2 ;; esac +case "${MECATL_EXECUTION_K8S_TOOLBOX:-}" in ''|sre) ;; *) echo "MECATL_EXECUTION_K8S_TOOLBOX must be empty or sre" >&2; exit 2 ;; esac +runtime=${CONTAINER_ENGINE:-docker} +case "$runtime" in docker|podman) command -v "$runtime" >/dev/null 2>&1 ;; *) echo "CONTAINER_ENGINE must be docker or podman" >&2; exit 2 ;; esac +root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd -P) +cd "$root" +run_id=$(date -u +%Y%m%d%H%M%S)-$$ +cluster="mecatl-execution-qual-$run_id" +context="kind-$cluster" +state="$root/.scratch/k8s-execution/$run_id" +kubeconfig="$state/kubeconfig" +mkdir -p "$root/.scratch/k8s-execution" +umask 077 +mkdir "$state" +mkdir "$state/pki" "$state/images" +printf 'cluster=%s\ncontext=%s\nkubeconfig=%s\nnamespace=execution-qualification\nowner=%s\nruntime=%s\nprofile=%s\ncreated_at=%s\n' \ + "$cluster" "$context" "$kubeconfig" "${USER:-user}" "$runtime" "${MECATL_EXECUTION_QUAL_PROFILE:-development}" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" >"$state/ownership" + +# Refuse collisions; never delete or reuse any cluster. +if [ "$runtime" = podman ]; then + export KIND_EXPERIMENTAL_PROVIDER=podman +else + unset KIND_EXPERIMENTAL_PROVIDER +fi +clusters=$(kind get clusters) || { echo "qualification cluster discovery failed" >&2; exit 1; } +if printf '%s\n' "$clusters" | grep -Fx "$cluster" >/dev/null; then + echo "qualification cluster already exists: $cluster" >&2 + exit 1 +fi +# Publish the exact owned identity before creation can partially fail. Downstream +# CI steps consume these outputs, never the mutable local convenience pointer. +if [ -n "${MECATL_EXECUTION_QUAL_OUTPUT:-}" ]; then + printf 'state=%s\ncluster=%s\n' "$state" "$cluster" >>"$MECATL_EXECUTION_QUAL_OUTPUT" +fi +printf '%s\n' "$state" >"$root/.scratch/k8s-execution/current" + +if [ "${MECATL_EXECUTION_QUAL_CI:-}" = 1 ]; then + cleanup_cluster() { + status=$? + artifact="$state/production-failure-artifact.txt" + if [ "$status" -ne 0 ]; then + timeout --kill-after=5s 60s sh "$root/deploy/mecatl-execution-kind/collect-failure.sh" \ + "$kubeconfig" "$context" "$artifact" || printf 'qualification_diagnostics_incomplete\n' >&2 + fi + kind delete cluster --name "$cluster" + return "$status" + } + trap cleanup_cluster EXIT +fi +export KUBECONFIG="$kubeconfig" +printf '{}\n' >"$state/registry-auth.json" +chmod 600 "$state/registry-auth.json" +export REGISTRY_AUTH_FILE="$state/registry-auth.json" + +kind_config= +if [ "${MECATL_EXECUTION_QUAL_PROFILE:-development}" = production ]; then + kind_config="$state/kind.yaml" + cat >"$kind_config" <<'EOF' +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +networking: + disableDefaultCNI: true + podSubnet: 192.168.0.0/16 + serviceSubnet: 10.96.0.0/12 +EOF +fi +if [ -n "$kind_config" ]; then + kind create cluster --name "$cluster" --image kindest/node:v1.35.8@sha256:07b2536e30b803ed61d1677a79df6115f798ce64c80f9e22f6ed45afd09323c0 --config "$kind_config" --kubeconfig "$kubeconfig" +else + kind create cluster --name "$cluster" --image kindest/node:v1.35.8@sha256:07b2536e30b803ed61d1677a79df6115f798ce64c80f9e22f6ed45afd09323c0 --kubeconfig "$kubeconfig" +fi + +dev() { + if [ -n "$MECATL_EXECUTION_DEV_TOOLBOX" ]; then + (cd "$root" && toolbox run -c "$MECATL_EXECUTION_DEV_TOOLBOX" "$@") + else + (cd "$root" && "$@") + fi +} +kube() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then + toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" kubectl --kubeconfig "$kubeconfig" --context "$context" "$@" + else + kubectl --kubeconfig "$kubeconfig" --context "$context" "$@" + fi +} +helm_kube() { + if [ -n "$MECATL_EXECUTION_K8S_TOOLBOX" ]; then + toolbox run -c "$MECATL_EXECUTION_K8S_TOOLBOX" helm --kubeconfig "$kubeconfig" --kube-context "$context" "$@" + else + helm --kubeconfig "$kubeconfig" --kube-context "$context" "$@" + fi +} + +if [ "${MECATL_EXECUTION_QUAL_PROFILE:-development}" = production ]; then + cni_manifest="$state/calico-v3.32.2.yaml" + curl --fail --location --proto '=https' --tlsv1.2 \ + https://raw.githubusercontent.com/projectcalico/calico/v3.32.2/manifests/calico.yaml \ + --output "$cni_manifest" + printf '%s %s\n' a8c828a06a87c629a282ebbc424895b77f3a030251993e41ea400a743675bb02 "$cni_manifest" | sha256sum -c - + sed -i \ + -e 's|quay.io/calico/cni:v3.32.2|quay.io/calico/cni@sha256:0ef740bc587f25565905adf1d1f61a7faff0d571c449c6bdd789feed743d3ef7|g' \ + -e 's|quay.io/calico/kube-controllers:v3.32.2|quay.io/calico/kube-controllers@sha256:7870b67ebb13fabc3005252b44fe6e78b21635649bd3072b80afa1684b6565d0|g' \ + -e 's|quay.io/calico/node:v3.32.2|quay.io/calico/node@sha256:99b03fe91e8bfbcb153ae65ef4b701b24ce541ffdd74ff314eb041096008f7fd|g' \ + "$cni_manifest" + kube apply -f "$cni_manifest" + kube -n kube-system rollout status daemonset/calico-node --timeout=5m + kube -n kube-system rollout status deployment/calico-kube-controllers --timeout=5m + kube wait --for=condition=Ready node --all --timeout=3m +fi + +# Synthetic keys are generated by the fixture only and never printed. Resume +# keeps the original PKI so mounted trust and running identities cannot drift. +if [ ! -s "$state/pki/ca.crt" ]; then + dev go run -tags kind_execution_e2e ./e2e/k8s_execution/fixture/pki "$state/pki" +fi + +# Build the three static Go images in the development toolbox. ko emits a +# content tag; after loading, the fixture resolves the node's actual manifest +# digest and creates a matching digest alias for Kubernetes. +build_ko() { + package=$1 repo=$2 + dev env KO_DOCKER_REPO="$repo" ko build --local --bare "$package" | tail -n 1 +} +provider_tag=$(build_ko ./cmd/mecatl-execution-provider ko.local/mecatl-execution-provider) +agent_tag=$(build_ko ./cmd/mecak8s ko.local/mecak8s) +oidc_tag=$(dev env GOFLAGS=-tags=kind_execution_e2e KO_DOCKER_REPO=ko.local/mecatl-oidc-fixture ko build --local --bare ./e2e/k8s_execution/fixture/oidcissuer | tail -n 1) +netprobe_tag=$(dev env GOFLAGS=-tags=kind_execution_e2e KO_DOCKER_REPO=ko.local/mecatl-netprobe-fixture ko build --local --bare ./e2e/k8s_execution/fixture/netprobe | tail -n 1) + +# Resolve the public Go base from the local OCI store, then feed its observed +# digest to the production workload recipe without weakening its pin check. +if [ "$runtime" = podman ]; then + podman image exists docker.io/library/golang:1.27 || podman pull docker.io/library/golang:1.27 >/dev/null + go_digest=$(podman image inspect docker.io/library/golang:1.27 --format '{{.Digest}}') +else + docker image inspect docker.io/library/golang:1.27 >/dev/null 2>&1 || docker pull docker.io/library/golang:1.27 >/dev/null + go_ref=$(docker image inspect docker.io/library/golang:1.27 --format '{{index .RepoDigests 0}}') + go_digest=${go_ref#*@} +fi +go_image="docker.io/library/golang@${go_digest}" +workload_tag="localhost/mecatl-execution-workload:e2e" +"$runtime" build --build-arg GO_IMAGE="$go_image" -f "$root/build/execution-workload/Dockerfile" -t "$workload_tag" "$root" >/dev/null + +for item in "$provider_tag" "$agent_tag" "$oidc_tag" "$netprobe_tag" "$workload_tag"; do + archive="$state/images/$(printf '%s' "$item" | sha256sum | cut -c1-16).tar" + "$runtime" save "$item" -o "$archive" >/dev/null + kind load image-archive "$archive" --name "$cluster" +done +. "$root/deploy/mecatl-execution-kind/images.sh" +provider_image=$(pin_loaded "$provider_tag") +agent_image=$(pin_loaded "$agent_tag") +oidc_image=$(pin_loaded "$oidc_tag") +netprobe_image=$(pin_loaded "$netprobe_tag") +workload_image=$(pin_loaded "$workload_tag") +printf 'provider=%s\nagent=%s\noidc=%s\nnetprobe=%s\nworkload=%s\ngo_base=%s\n' "$provider_image" "$agent_image" "$oidc_image" "$netprobe_image" "$workload_image" "$go_image" >"$state/images/proof" + +kube create namespace execution-qualification --dry-run=client -o yaml | kube apply -f - +kube label namespace execution-qualification pod-security.kubernetes.io/enforce=restricted pod-security.kubernetes.io/audit=restricted pod-security.kubernetes.io/warn=restricted --overwrite + +# Kind v0.33 installs its pinned local-path provisioner and default "standard" +# StorageClass as part of cluster creation. The fixture RuntimeClass names Kind's +# existing runc handler so provider startup can exercise the production preflight. +kube apply -f "$root/deploy/mecatl-execution-kind/runtimeclass.yaml" + +if [ "${MECATL_EXECUTION_QUAL_PROFILE:-development}" = production ]; then + cat >"$state/network-fixtures.yaml" <"$legacy_profiles" <&2; exit 1 ;; + :*|*:) echo "missing discovered network-policy endpoint" >&2; exit 1 ;; + esac + runtime_values="$state/production-runtime-values.yaml" + cat >"$runtime_values" < /workspace/migration-sentinel' + # Helm does not upgrade existing CRDs. Preserve the stored legacy references + # until the explicit CRD upgrade; the first production test then migrates them. + kube apply -f "$root/deploy/helm/mecatl-execution/crds/executionenvironment.yaml" + kube wait --for=condition=Established crd/executionenvironments.execution.mecatl.dev --timeout=60s + kube -n execution-qualification scale deployment/mecatl-execution --replicas=2 + kube -n execution-qualification rollout status deployment/mecatl-execution --timeout=4m +fi + +kube -n execution-qualification create configmap execution-mock --from-file=mock-script.json="$root/deploy/mecatl-execution-kind/mock-script.json" --dry-run=client -o yaml | kube apply -f - +helm_kube upgrade --install mecak8s "$root/deploy/helm/mecak8s" \ + --namespace execution-qualification -f "$root/deploy/mecatl-execution-kind/mecak8s-values.yaml" \ + --set-string image.repository="${agent_image%@*}" --set-string image.digest="${agent_image#*@}" --set-string image.tag= \ + --wait --timeout=5m +kube -n execution-qualification rollout restart deployment/mecak8s +kube -n execution-qualification rollout status deployment/mecak8s --timeout=240s + +dev env -i HOME="$HOME" PATH="$PATH" KUBECONFIG="$kubeconfig" MECATL_KUBE_CONTEXT="$context" MECATL_EXECUTION_QUAL_STATE="$state" MECATL_EXECUTION_QUAL_PROFILE="${MECATL_EXECUTION_QUAL_PROFILE:-development}" \ + go test -tags kind_execution_e2e -count=1 -timeout=45m ./e2e/k8s_execution + +if [ "${MECATL_EXECUTION_QUAL_CI:-}" = 1 ]; then + echo "qualification cluster is CI-owned and will be removed: $cluster" +else + echo "qualification cluster retained: $cluster" + echo "kubeconfig: $kubeconfig" + echo "cleanup (requires human confirmation): CONTAINER_ENGINE=$runtime kind delete cluster --name $cluster" +fi diff --git a/deploy/mecatl-execution-kind/runtimeclass.yaml b/deploy/mecatl-execution-kind/runtimeclass.yaml new file mode 100644 index 0000000000..80524fc6c3 --- /dev/null +++ b/deploy/mecatl-execution-kind/runtimeclass.yaml @@ -0,0 +1,5 @@ +apiVersion: node.k8s.io/v1 +kind: RuntimeClass +metadata: + name: qualification-runc +handler: runc diff --git a/docs/acceptance/microvm-execution-environments.md b/docs/acceptance/microvm-execution-environments.md index 985e520978..92c2b953c1 100644 --- a/docs/acceptance/microvm-execution-environments.md +++ b/docs/acceptance/microvm-execution-environments.md @@ -2,15 +2,15 @@ **Contract:** human-reviewed/v2 **Work classification:** Architectural — this amendment adds connection-owned, daemon-local MicroVM acquisitions across local harness processes while preserving the independently approved Environment and HarnessContext contracts. -**Decision record:** [ADR 0367](../adr/0367-microvm-execution-environments.md) +**Decision record:** [ADR 0368](../adr/0368-microvm-execution-environments.md) **Phase:** Repository-scoped local microVM execution, including the experimental Darwin arm64 path **Status:** in-progress **Qualification:** Linux amd64 source-build validation includes the real KVM private-v4 journey and a live GPT-5.6 Sol multi-instance coding workflow through HTTP/SSE. This is not released-artifact qualification. Darwin arm64 remains experimental and awaits physical HVF and release qualification. **Evidence:** On 2026-09-25, `task e2e:microvm` passed at `3d55a8cfa600546fd7c03a177dd2293a1b54aa50`, using a real Linux KVM guest and the deterministic mock provider. A separate OpenRouter `openai/gpt-5.6-sol` journey completed ten outer runs and one isolated Subagent execution: coding with Read/Edit/Write/Shell, guest UID 65532/Wolfi and boundary checks, distinct placements sharing a repository VM, cross-instance read-only scheduling, peer shutdown, harness resume, and linked-worktree isolation. Its final saved-session cold resume passed with host `9a1c2cd05abca6f101a2b29e467baeda3babdb80` and the unchanged compatible runtime bundle from `3d55a8cfa600546fd7c03a177dd2293a1b54aa50`. Exact placement and guest changes survived. The final cold driver returned nonzero only because an extra greater-than-15-second timing assertion was not met: HTTP 200 arrived after 12.986 seconds and the model completed `end_turn`. Long-acquisition regression coverage is deterministic, not a claim that this successful live acquisition exceeded 15 seconds. One scheduled PresentPlan attempt was denied in headless mode; the read-only fire subsequently completed. No credentials, private refs, sockets, or host paths are retained here. These source-build runs use locally verified development artifacts, not a published signed release; no physical Apple Silicon/HVF journey was performed. **Issue:** [stacklok/mecatl#526](https://github.com/stacklok/mecatl/issues/526) -**Delivery:** Split. PR 580 is the sole delivery. The operator explicitly authorized direct same-PR amendment of this plan and unmerged ADR 0367 for connection-owned acquisitions, overriding the separate amendment-PR/superseding-ADR route. The amendment is committed separately before implementation; tests, security review, and human merge authority remain required. -**ADR:** [ADR 0367](../adr/0367-microvm-execution-environments.md), with the proposed Darwin decision in [ADR 0365](../adr/0365-microvm-darwin-xattr-ownership.md) +**Delivery:** Split. PR 580 is the sole delivery. The operator explicitly authorized direct same-PR amendment of this plan and unmerged ADR 0368 for connection-owned acquisitions, overriding the separate amendment-PR/superseding-ADR route. The amendment is committed separately before implementation; tests, security review, and human merge authority remain required. +**ADR:** [ADR 0368](../adr/0368-microvm-execution-environments.md), with the proposed Darwin decision in [ADR 0365](../adr/0365-microvm-darwin-xattr-ownership.md) **Accumulator branch:** `acc/microvm-execution-environments` The MVP uses one mutable microVM and one rootfs for one local operator and one canonical @@ -38,7 +38,7 @@ all earlier superseded redesign tasks. - [x] Local multi-instance baseline. — Decision: Multiple mecatui/embedded-mecated instances on one repository are normal coding use; no one-harness-per-daemon restriction is acceptable. - [x] Connection-owned acquisitions. — Decision: The directing human approved the reviewed connection-owned, daemon-local acquisition proposal on 2026-09-25. One retained authenticated Unix socket per independently acquired binding provides process-death cleanup without heartbeat, TTL, database, or new configuration. The exact private v4 exchange below governs release, operation pins, rollback and restart. - [x] Cleanup and compatibility. — Decision: Ambiguous outcomes retain durable worktrees, live ownership/pins block destructive deletion, and private v4 rejects v3 rather than allowing ref-global detach. A compatibility mismatch does not automatically replace a daemon serving other local harnesses. -- [x] Direct same-PR amendment. — Decision: The directing human authorized this plan's AC5.2 replacement and AC5.7–AC5.10 additions, the narrow in-place attachment-lifetime amendment to unmerged ADR 0367, and ADR 0027's maintained inventories. This explicitly overrides the separate amendment-PR/superseding-ADR route; all other criteria and the shared HarnessContext contract remain intact. Verbatim approval and source are recorded in the existing run's `run.md`. +- [x] Direct same-PR amendment. — Decision: The directing human authorized this plan's AC5.2 replacement and AC5.7–AC5.10 additions, the narrow in-place attachment-lifetime amendment to unmerged ADR 0368, and ADR 0027's maintained inventories. This explicitly overrides the separate amendment-PR/superseding-ADR route; all other criteria and the shared HarnessContext contract remain intact. Verbatim approval and source are recorded in the existing run's `run.md`. ## Interface contract @@ -68,7 +68,7 @@ The added `acquisition_id` is 32 lowercase hexadecimal characters from 16 crypto Successful `create`/`resolve`/`fork` publish their acquisition in the daemon before replying. The client returns a binding only after validating the response and atomically transferring the socket from attempt-context cancellation to the returned owner's lifetime. Cancellation winning that transfer runs the safely attributable cleanup below; later completion/cancellation of the creation request cannot close a transferred binding. A resolve retry opens a fresh socket and gets a fresh owner; no request-ID deduplication store is required. Create/fork are not blindly retried after an unknown outcome. **Response validation is separate from rollback authority:** -- A fully validated new create/fork result failing before session publication retains exact rollback authority: close attempt-owned source borrows, then perform bounded ownership-consuming deletion. Do not reduce ADR 0367's known unpublished-failure cleanup to detach-only. +- A fully validated new create/fork result failing before session publication retains exact rollback authority: close attempt-owned source borrows, then perform bounded ownership-consuming deletion. Do not reduce ADR 0368's known unpublished-failure cleanup to detach-only. - Malformed non-authority metadata (for example Created profile/root/egress fields) does not erase safe attribution. For create, preserve the existing check at `internal/adapter/microvm/client.go:244-255`: expected owner, the unpredictable fresh request placement/session ID, and a complete canonical logical ref/generation. A matching well-formed AcquisitionID additionally permits terminal deletion on its originating socket; the daemon verifies that ID and tuple are precisely the new result created by that connection. If create's existing exact attribution is safe but its ID is absent/invalid, close the socket and attempt the existing bounded exact-ref rollback under the new zero-owner/zero-pin deletion gate. Cleanup refusal/error is reported as retained or uncertain, never as successful deletion. - Fork labels are reusable: parent/owner/label similarity alone never authorizes deleting a returned sibling. Fork rollback requires a complete validated child tuple, distinct from the parent, and a well-formed AcquisitionID whose originating connection the daemon verifies created that exact child. A malformed result lacking that proof is not destructively cleaned up. - Unknown, untrustworthy, truncated, or ambiguous transport outcomes without the above safe attribution close the connection and retain durable state. Resolve failure only releases its borrow; it never deletes the pre-existing placement. A syntactically valid ID alone grants no deletion authority. Preserve existing actionable retained/unknown-cleanup diagnostics and clean-delete/dirty-retention results; no extra cleanup service is introduced. @@ -108,7 +108,7 @@ registry or new manager orchestration. ### Scenario 1 — ordinary deployment-default selection converges on a ready local backend This completed scenario follows -[ADR 0367 — readiness](../adr/0367-microvm-execution-environments.md#5-make-readiness-part-of-ordinary-use). +[ADR 0368 — readiness](../adr/0368-microvm-execution-environments.md#5-make-readiness-part-of-ordinary-use). **Acceptance:** @@ -125,7 +125,7 @@ This completed scenario follows ### Scenario 2 — canonical repository identity owns one durable VM generation -This follows [ADR 0367 — singleton lifecycle](../adr/0367-microvm-execution-environments.md#2-use-one-durable-vmrootfs-record-per-operator-and-canonical-git-repository). +This follows [ADR 0368 — singleton lifecycle](../adr/0368-microvm-execution-environments.md#2-use-one-durable-vmrootfs-record-per-operator-and-canonical-git-repository). The daemon uses `(operator, canonical Git common directory)` as the only repository VM key. This scenario establishes lifecycle identity and the rootfs singleton; it does not add @@ -160,7 +160,7 @@ repository guest. Filesystem and exec remain bound to one `EnvironmentRef` and o ### Scenario 4 — immutable Brood bytes and one explicit repository rootfs -This follows [ADR 0367 — direct Brood consumption](../adr/0367-microvm-execution-environments.md#4-consume-brood-directly-and-provide-an-explicit-linux-guest). +This follows [ADR 0368 — direct Brood consumption](../adr/0368-microvm-execution-environments.md#4-consume-brood-directly-and-provide-an-explicit-linux-guest). The artifact and static guest-contract work in tasks 60–61 remains complete. The remaining MVP step attaches that primitive to the repository generation. @@ -227,7 +227,7 @@ and [shared source-lifetime criteria](harness-context.md) remain unchanged. ### Scenario 6 — platform ownership and useful networking are honest -This follows [ADR 0367's Linux guest decision](../adr/0367-microvm-execution-environments.md#4-consume-brood-directly-and-provide-an-explicit-linux-guest) and the proposed [Darwin xattr decision](../adr/0365-microvm-darwin-xattr-ownership.md#prepare-fixed-guest-ownership-with-virtiofs-xattrs). +This follows [ADR 0368's Linux guest decision](../adr/0368-microvm-execution-environments.md#4-consume-brood-directly-and-provide-an-explicit-linux-guest) and the proposed [Darwin xattr decision](../adr/0365-microvm-darwin-xattr-ownership.md#prepare-fixed-guest-ownership-with-virtiofs-xattrs). Linux amd64 keeps its qualified namespace mechanism. Darwin arm64 keeps the same fixed guest identity through strict VirtioFS ownership preparation, but remains experimental until Scenario 8. The default network is useful rather than presented as containment; tightening remains optional and fail-closed when selected. diff --git a/docs/adr/0027-cloud-native.md b/docs/adr/0027-cloud-native.md index 69ab1805d1..7027cf9be2 100644 --- a/docs/adr/0027-cloud-native.md +++ b/docs/adr/0027-cloud-native.md @@ -570,7 +570,7 @@ zero ref is stamped from the first resolved live Environment on the next save (n sweep). The re-audit verdict is CLEAN: no restart-losable state beyond what the snapshot already holds, no new outlives-a-call resource. -**MicroVM lifecycle re-audit (List 1 / List 2 - ADR 0367).** The production +**MicroVM lifecycle re-audit (List 1 / List 2 - ADR 0368).** The production backend adds the durable repository singleton record and the VM/process/socket/rootfs resource family plus logical attachment records (List 1 rows 73 and 79-80). The singleton record and attachment metadata are fidelity sources in List 2 rows 47 and 50. Harness-local environment @@ -609,7 +609,7 @@ not include a per-session resource quota controller. The repository registry ser forks remain distinct worktrees inside that one generation. Deployment-level capacity policy is outside this repository-only lifecycle. -**MicroVM operations re-audit (List 1 / List 2 - ADR 0367).** The +**MicroVM operations re-audit (List 1 / List 2 - ADR 0368).** The process-scoped `OperationsObserver` adds one bounded in-memory metric snapshot and a generation-keyed `egressSources` map (List 1 row 76). The map owns each active hosted-network denial source and its last sampled counter until generation teardown removes it. It has no @@ -856,6 +856,15 @@ durable artifact survives and is reloaded), or **lost** (gone, possibly leaking) | 76 | Contextual guardrail current-root trajectory ledger | delegation-root `agent.Run` (`reviewRoot`) | root run shared with its workers | dies after child drain/run termination; exact duplicates are skipped. Admission checks 256 facts / 128,000 retained bytes before append; exhaustion sticks `TrajectoryComplete=false` and retains only bounded authorization-class metadata when it fits | **reset-by-design**: no cross-run/session/restart authority; missing trajectory forces a capacity-dependent assessment unresolved rather than acceptable | `engine/agent/actionreview.go` (`reviewRoot`, `record`, `snapshot`); `engine/agent/loop.go` (`startRun`) | | 77 | Contextual guardrail exact repeat grant registry and HMAC key | `app.Build` (`modelhook.WaiverHolder`) | process, partitioned by session | `SessionCleared` calls `ClearSession`; process shutdown destroys key and map. Admission is bounded to 1,024 sessions and 256 digests/session; full capacity makes repeat unavailable while Run once remains | **reset-by-design, fail-safe**: grants and random key never persist; restart or session clear requires a fresh contextual review/human approval | `internal/adapter/modelhook/waiver.go` (`WaiverHolder`, `CanArm`, `ClearSession`); `internal/app/build.go` (`SessionCleared`) | | 78 | Plan approval receipt registry | `app.Build` (`planApprovalReceipts`) | process, at most one exact `{ref, session, call, target mode}` receipt per session and 1,024 sessions total; absent when the checker is unconfigured or disabled | consumed or discarded once after the next accepted execution prompt; both `OnCloseSession` and `SessionCleared` remove the session entry; process shutdown drops the map | **reset-by-design, fail-safe**: restart never reconstructs plan approval authority from snapshots, events, or prose | `internal/app/plan_approval_receipts.go` (`planApprovalReceipts`, `RecordPlanApproval`, `ConsumePlanApproval`, `ClearPlanApprovals`); `internal/app/build.go` (`OnCloseSession`, `SessionCleared`) | +| 81 | Native-execution reference-intent reconciler ticker and worker | `app.Build` | process | `Built.Close` cancels and joins the bounded worker before closing the provider client; each pass handles at most 64 intents | reconstructible; unresolved intents remain durable in `ExecutionEnvironment.status.references` and are retried after restart | `internal/app/reference_intent_reconcile.go` (`startReferenceIntentReconcile`); `internal/adapter/server/service.go` (`ReconcileReferenceIntents`) | +| 82 | Native-execution provider gRPC connection and bounded RPC limiter identity counters | `executionclient.Provider` / execution-provider process | process / active RPC | provider close releases the connection; limiter reservations release at RPC return and idle per-client entries are removed | reconstructible from operator endpoint and mTLS files; limiter counters reset by design | `internal/adapter/executionclient/client.go` (`Client`, `Provider`, `Close`); `internal/adapter/executioncontroller/security_limit.go` (`RPCLimiter`) | +| 83 | Native-execution active run and operation renewal handles | `server.Service.runState` owns the admitted run handle and renewal lifecycle; `executionclient.Provider` and `executioncontroller.Store` hold the corresponding native authority | run / operation | `ResolveRunAsk` reacquires native authority for a persisted ask before resuming. `runState` renews it while the run and event relay drain; post-drain teardown joins renewal and releases only the matching durable holder. Cancelled or late acquisition releases its handle without publishing it. Uncertain expiry is retained and fenced | persisted in `ExecutionEnvironment.status.activeRun` and `status.activeOperation`; restart reacquires only after exact session/run/ask/owner validation and never adopts an unknown holder | `internal/adapter/server/service.go` (`runState`, `acquireExecution`, `startExecutionRenewal`, `teardownExecution`, `ResolveRunAsk`); `internal/adapter/executionclient/client.go` (`Provider`); `internal/adapter/executioncontroller/store.go` (`Store`) | +| 84 | Native-execution controller informers, queue, and worker | execution-provider `Reconciler` | process | provider context stops informers and worker; `Run` shuts down the queue | reconstructible from Kubernetes objects after profile-resource preflight and cache synchronization | `internal/adapter/executioncontroller/controller.go` (`Reconciler`, `Initialize`, `Run`) | +| 85 | Security authority immutable snapshot, reload ticker, and durable high-water ConfigMap | execution-provider `SecurityManager` | process snapshot / cluster-durable authority | context stops reload; candidates publish atomically, invalid candidates clear readiness, and no secret bytes enter the ConfigMap | snapshot reconstructible from projected files; generation, canonical digest, and public fingerprints persist in the authority ConfigMap, retained across default Helm uninstall and reused byte-for-byte by a quiesced same-identity reinstall | `internal/adapter/executioncontroller/security.go` (`SecurityManager`, `Reload`, `Run`); `deploy/helm/mecatl-execution/templates/provider.yaml` | +| 86 | Profile-allocation slot CAS ledger | execution-provider `Store` | namespace, per profile | exact final CR deletion releases a slot; conflicts retry with resource-version CAS and invalid state fails closed | persisted in the `mecatl-execution-profile-allocations` ConfigMap and reconciled against retained CRs; default Helm uninstall retains the ledger and same-identity reinstall preserves its data | `internal/adapter/executioncontroller/store.go` (`reserveProfileSlot`, `releaseProfileSlot`); `deploy/helm/mecatl-execution/templates/provider.yaml` | +| 87 | Bounded revocation receipt ledger and lifecycle/migration proof state | `ExecutionEnvironment` status | environment | at most 32 revocation receipts are retained; lifecycle and migration records clear only after exact terminal/UID proof and successful phase completion | persisted in CR status; retries use operation ID and fingerprint, while evicted receipt IDs remain rejected | `internal/adapter/executioncontroller/store_revoke.go` (`RevokeEnvironment`); `internal/adapter/executioncontroller/admin_lifecycle.go`; `internal/adapter/executioncontroller/controller_lifecycle.go` | +| 88 | Native-execution workload NetworkPolicies and retained profile/security configuration | Helm release, namespace-scoped | retained allocation lifetime | default uninstall retains default-deny/profile policies and both configuration ConfigMaps; no deletion hook | persisted; `profiles` ConfigMap `lifetime.json` pins profiles, network configuration, and security Secret name. Same-release/namespace ownership checks permit adoption; incompatible configuration or missing ledgers with retained allocations fails closed. Operator-owned Secret material is retained outside Helm | `deploy/helm/mecatl-execution/templates/network-policy.yaml`; `deploy/helm/mecatl-execution/templates/provider.yaml`; `deploy/helm/mecatl-execution/templates/lifetime-check.yaml` | +| 89 | Mecatui pending-approval recovery watch, generation, and volatile cursor | `ui.Model` for one exact `--resume` selection | client process / one exact session and run | cancellation while opening retires the generation; a late-opened watch is closed instead of published. The UI closes the current watch on terminal result, watch failure, stale or refused control, pre-verdict escape or quit, and process teardown. An accepted verdict keeps the watch until the exact run terminates, while Ctrl+C uses `CancelRun` before cleanup | **reattached by the client under fresh authorization**: startup replays the durable event log from the beginning, requires one gap-free live boundary and one trailing unresolved ordinary ask, then keeps the cursor only in memory. Generation retirement discards late events. No client verdict, cursor, or recovery decision is persisted or automatically replayed | `cmd/mecatui/ui/pending_approval.go` (`pendingApprovalRecovery`, `retirePendingApprovalRecovery`); `cmd/mecatui/client/pendingapproval.go` (`DiscoverPendingApproval`, `WatchPendingApprovalRun`) | | 58 | Title coordinator (two workers, 64-item queue, dedupe set, and retry timers) | `server.Service` / `app.Build` | process | `Service.Close` cancels and joins workers before session dependencies close; queue admission is non-blocking and bounded | reset-by-design; durable session attempt records make a crash-interrupted claim terminal rather than rebilling | `internal/adapter/server/title_coordinator.go` (`titleCoordinator`) | | 59 | Provider OIDC bearer sources and runtime registry | `internal/cliconfig.NativeEndpointLoader` / `NativeEndpointRuntime` | process, one per configured OIDC provider | `NativeEndpointLoader.Close` closes every loader-owned runtime; each runtime closes its serving Store handles | reconstructible from explicit operator provider configuration and an exact protected record; access tokens reset and are refreshed from that record | `internal/cliconfig/native_endpoint.go` (`NativeEndpointLoader`, `NativeEndpointRuntime`, `Source`, `Close`) | | 60 | Caller OIDC JWKS validator cache | `authn/oidc.Validator` | process | validator `Close` stops its refresh work during command-root shutdown | reset/refetch: a restart fetches current signing keys; this authenticates callers only and is separate from provider issuer/gateway trust | `authn/oidc/oidc.go` (`Validator`, `Close`) | @@ -935,10 +944,10 @@ durable artifact survives and is reloaded), or **lost** (gone, possibly leaking) | 68 | mecak8s plaintext drain-only listener | `cmd/mecak8s` (`serve`) | process | bound before serving; `boundedShutdown` shuts down its `http.Server`, and deferred listener close releases it on every later startup failure | **reconstructible/reset-by-design**: listener state is process-local and recreated from `--drain-addr` (default `0.0.0.0:8082`); no session state exists | `cmd/mecak8s/serve.go` (`drainHTTPMux`, `serve`, `boundedShutdown`); ADR 0290 | | 69 | Worktree selector HMAC key (ADR 0291) | `app.Build` | process lifetime | no goroutine, registry, map, cache, or external handle; the random key dies with Build/process and needs no explicit cleanup | **reset-by-design**: the key and selectors are not persisted. Restart invalidates every selector, clients relist worktrees, and the server re-enumerates current choices for constant-time matching. There is no selector registry/map to reattach | `internal/app/build.go` (`Build`); `internal/adapter/server/worktree_selector.go` (`WorktreeSelectorIssuer`) | | 70 | Managed temporary command/job reaper worker, root GC lock/completion record, and per-lease locks (ADR 0281) | `app.Build` / `managedtemp.Namespace` | process worker; durable private managed root | `Built.Close` cancels the worker and waits only `shutdown_reap_timeout`; each startup/periodic sweep is bounded by `reap_timeout`. The root lock is non-blocking, lease locks are acquired non-blockingly per candidate, and only validated unlocked leases past TTL are removed. System or unsupported modes construct no worker and make no mutation | **mixed**: the ticker/worker reset on restart; owner-only completion record and lease manifests persist. A crashed holder releases advisory locks, so a later interval-gated Build sweep can reclaim eligible residue; incomplete/cancelled scans publish no completion record. No List 2 row: this is adapter-owned cleanup coordination, not session/run state | `internal/app/managed_temp_worker.go` (`startManagedTempWorker`); `internal/adapter/managedtemp/reaper_unix.go` (`Sweep`) | -| 73 | Repository-scoped local microVM placement: stable generation, replaceable VM/runner boot, private endpoint, retained rootfs, admitted launch artifacts, shared Unix daemon service flock, launch receipts/locks, Darwin launch-owner supervisor and liveness pipe, and registry flock (ADR 0367; proposed ADR 0365) | `mecatl-microvmd`; one `RepositoryVMRecord` per `(operator, canonical Git common directory)` in `RepositoryVMRegistry` | placement/rootfs/artifacts outlive sessions, daemon processes, and host boots; boot runtime is replaceable; the service flock spans one daemon process; registry locks are operation scoped; a launch attempt's supervisor, pipe descriptors, and inherited flock copies span that runner | first admission retains verified immutable artifacts, writes a provisioning intent with a unique host-only staging name, materializes there, atomically publishes one rootfs without overwrite, and commits ready after authenticated health. A retry rebuilds only the recorded never-published staging transaction or recognizes the already-published rootfs. The daemon acquires its stable owner-only service flock before socket cleanup or repository runtime mutation and retains it until exit. Recovery validates retained state and policy, reconciles every stable-repository launch attempt, rotates boot authority, and starts a new boot without changing the placement generation. On Darwin, daemon-pipe EOF asks the supervisor to TERM/KILL and reap its direct runner; normal shutdown closes and joins. A runner surviving supervisor death retains the attempt flock and blocks replacement pending operator recovery. Graceful shutdown stops owned VM/network resources but retains durable files | **persisted/reconstructible**: versioned registry, immutable artifact identities, rootfs, and launch intent/receipt/lock files survive restart. Linux exact launch reconciliation retains boot ID, process start, executable and lock-fd identity around pidfd open. Darwin reconciliation waits on the inherited flock and never signals a stored PID. Pending or uncertain ownership blocks replacement; no rootfs rematerialization or command replay occurs. The Darwin supervisor, pipe, and fd copies are process-only and are not reconstructed for a historical launch. See List 2 row 47 | `environment/microvm/repository_lifecycle.go` (`RepositoryVMRegistry`, `RepositoryVMRecord`, `Ensure`); `environment/microvm/repository_rootfs.go` (`repositoryRootFSMaterializer`); `environment/microvm/cmd/mecatl-microvmd/daemon_ownership_unix.go` (`acquireDaemonOwnership`); `environment/microvm/launch_ownership.go` (`LaunchOwnership`); `environment/microvm/launch_ownership_linux.go` (`Reconcile`); `environment/microvm/launch_ownership_darwin.go` (`ownDirectChild`) | -| 75 | repository-logical microVM child attachments and process-local fork/merge state (ADR 0367) | the authoritative `mecatl-microvmd` `RepositoryAttachmentManager` and `Daemon`; the root microVM client routes by the parent `EnvironmentRef.Kind` | each child attachment owns one logical worktree, exact repository generation binding, and guest registration; the active map holds fork parentage and base, and one process-local mutex serializes logical merges | Fork captures an immutable Git tree containing the parent's exact dirty state, creates a separate logical worktree in the same repository VM, and validates the complete returned owner/session/environment/ref/generation tuple. Merge checks process-local parentage, discovers all child changes, checks the parent against the fork base, validates the patch, immediately rechecks, and applies once. Conflict preserves the child. Cleanup uses a cancellation-detached bounded RPC and deletes only an exactly identified child attachment | **persisted/reset split**: attachment records persist the repository generation, exact binding, and worktree metadata. Active handles, fork parentage/base, and merge serialization reset with the daemon process. Exact reattachment rebuilds handles only against the retained repository generation; merge after process restart is unavailable. Crash-durable fork parentage and cross-process merge coordination remain deferred. See List 2 row 49 | `environment/microvm/repository_attachment.go` (`RepositoryAttachmentManager`, `Fork`, `Merge`, `register`); `environment/microvm/operational.go` (`captureForkBase`, `mergeRepositoryWorktrees`); `environment/microvm/daemon.go` (`Daemon`); `internal/adapter/microvm/client.go` (`Fork`) | -| 76 | Local microVM operations metrics and active egress-source snapshot (ADR 0367) | one `microvm.OperationsObserver` per daemon composition | process; fixed closed-dimension counters plus a generation-keyed map of active egress-denial sources and their last samples | no goroutine, fd, or timer. Generation registration adds or replaces one source after sampling its predecessor; snapshot samples each source; teardown samples and removes it. Producers may pass command/destination text only to methods that discard it immediately; diagnostics emit fixed messages and bounded dimensions. The observer and remaining map entries die with the daemon | **reset-by-design**: counters, source handles, and sample baselines are process observations, not session/run state. Durable repository and attachment truth remains in List 1 rows 73 and 80 / List 2 rows 47 and 50. The per-invocation Doctor adds no outlives-a-call resource | `environment/microvm/operations.go` (`OperationsObserver`, `trackEgressDenials`, `untrackEgressDenials`, `Snapshot`, `Doctor`) | -| 77 | Multiplexed microVM guest connection reader plus authentication/replay state (ADR 0367 data plane) | host `control.Client` owns one response-demultiplexing reader; generic guest `guestagent.Server` owns a bounded `CapabilityVerifier`, while `RepositoryServer` owns boot-authenticated registration sequence/incarnation state | live environment connection / guest-agent process | `guestagent.Services.Close` closes the sole stream and unblocks the reader. Generic environments retain at most 4096 consumed capability nonces and fail closed. Repository channels instead mutually authenticate the current boot, serialize lifecycle mutations with a monotonic sequence, retain one exact response for lost-acknowledgement replay, bind data claims to the current random registration incarnation, and cap persistent data at 16 slots plus one reserved control connection. A matching acknowledgement updates the named host registration incarnation even when another caller drains it; transport loss retains the exact pending request, while decoded semantic failure advances the sequence without applying success state. No host fallback exists | **derived/reset-safe**: generic reattachment mints a fresh capability; repository boot recovery rotates independently provisioned boot authority and reconstructs sequence/incarnation state. Earlier-boot proofs fail before either replay mechanism resets. No List 2 row | `environment/microvm/control/multiplex.go` (`Client`, `ServeMultiplex`); `environment/microvm/control/capability.go` (`CapabilityVerifier`); `environment/microvm/guestagent/guestagent.go` (`Server`, `Connect`); `environment/microvm/guestagent/repository.go` (`RepositoryServer`); `environment/microvm/repository_runtime.go` (`mutateLocked`) | +| 73 | Repository-scoped local microVM placement: stable generation, replaceable VM/runner boot, private endpoint, retained rootfs, admitted launch artifacts, shared Unix daemon service flock, launch receipts/locks, Darwin launch-owner supervisor and liveness pipe, and registry flock (ADR 0368; proposed ADR 0365) | `mecatl-microvmd`; one `RepositoryVMRecord` per `(operator, canonical Git common directory)` in `RepositoryVMRegistry` | placement/rootfs/artifacts outlive sessions, daemon processes, and host boots; boot runtime is replaceable; the service flock spans one daemon process; registry locks are operation scoped; a launch attempt's supervisor, pipe descriptors, and inherited flock copies span that runner | first admission retains verified immutable artifacts, writes a provisioning intent with a unique host-only staging name, materializes there, atomically publishes one rootfs without overwrite, and commits ready after authenticated health. A retry rebuilds only the recorded never-published staging transaction or recognizes the already-published rootfs. The daemon acquires its stable owner-only service flock before socket cleanup or repository runtime mutation and retains it until exit. Recovery validates retained state and policy, reconciles every stable-repository launch attempt, rotates boot authority, and starts a new boot without changing the placement generation. On Darwin, daemon-pipe EOF asks the supervisor to TERM/KILL and reap its direct runner; normal shutdown closes and joins. A runner surviving supervisor death retains the attempt flock and blocks replacement pending operator recovery. Graceful shutdown stops owned VM/network resources but retains durable files | **persisted/reconstructible**: versioned registry, immutable artifact identities, rootfs, and launch intent/receipt/lock files survive restart. Linux exact launch reconciliation retains boot ID, process start, executable and lock-fd identity around pidfd open. Darwin reconciliation waits on the inherited flock and never signals a stored PID. Pending or uncertain ownership blocks replacement; no rootfs rematerialization or command replay occurs. The Darwin supervisor, pipe, and fd copies are process-only and are not reconstructed for a historical launch. See List 2 row 47 | `environment/microvm/repository_lifecycle.go` (`RepositoryVMRegistry`, `RepositoryVMRecord`, `Ensure`); `environment/microvm/repository_rootfs.go` (`repositoryRootFSMaterializer`); `environment/microvm/cmd/mecatl-microvmd/daemon_ownership_unix.go` (`acquireDaemonOwnership`); `environment/microvm/launch_ownership.go` (`LaunchOwnership`); `environment/microvm/launch_ownership_linux.go` (`Reconcile`); `environment/microvm/launch_ownership_darwin.go` (`ownDirectChild`) | +| 75 | repository-logical microVM child attachments and process-local fork/merge state (ADR 0368) | the authoritative `mecatl-microvmd` `RepositoryAttachmentManager` and `Daemon`; the root microVM client routes by the parent `EnvironmentRef.Kind` | each child attachment owns one logical worktree, exact repository generation binding, and guest registration; the active map holds fork parentage and base, and one process-local mutex serializes logical merges | Fork captures an immutable Git tree containing the parent's exact dirty state, creates a separate logical worktree in the same repository VM, and validates the complete returned owner/session/environment/ref/generation tuple. Merge checks process-local parentage, discovers all child changes, checks the parent against the fork base, validates the patch, immediately rechecks, and applies once. Conflict preserves the child. Cleanup uses a cancellation-detached bounded RPC and deletes only an exactly identified child attachment | **persisted/reset split**: attachment records persist the repository generation, exact binding, and worktree metadata. Active handles, fork parentage/base, and merge serialization reset with the daemon process. Exact reattachment rebuilds handles only against the retained repository generation; merge after process restart is unavailable. Crash-durable fork parentage and cross-process merge coordination remain deferred. See List 2 row 49 | `environment/microvm/repository_attachment.go` (`RepositoryAttachmentManager`, `Fork`, `Merge`, `register`); `environment/microvm/operational.go` (`captureForkBase`, `mergeRepositoryWorktrees`); `environment/microvm/daemon.go` (`Daemon`); `internal/adapter/microvm/client.go` (`Fork`) | +| 76 | Local microVM operations metrics and active egress-source snapshot (ADR 0368) | one `microvm.OperationsObserver` per daemon composition | process; fixed closed-dimension counters plus a generation-keyed map of active egress-denial sources and their last samples | no goroutine, fd, or timer. Generation registration adds or replaces one source after sampling its predecessor; snapshot samples each source; teardown samples and removes it. Producers may pass command/destination text only to methods that discard it immediately; diagnostics emit fixed messages and bounded dimensions. The observer and remaining map entries die with the daemon | **reset-by-design**: counters, source handles, and sample baselines are process observations, not session/run state. Durable repository and attachment truth remains in List 1 rows 73 and 80 / List 2 rows 47 and 50. The per-invocation Doctor adds no outlives-a-call resource | `environment/microvm/operations.go` (`OperationsObserver`, `trackEgressDenials`, `untrackEgressDenials`, `Snapshot`, `Doctor`) | +| 77 | Multiplexed microVM guest connection reader plus authentication/replay state (ADR 0368 data plane) | host `control.Client` owns one response-demultiplexing reader; generic guest `guestagent.Server` owns a bounded `CapabilityVerifier`, while `RepositoryServer` owns boot-authenticated registration sequence/incarnation state | live environment connection / guest-agent process | `guestagent.Services.Close` closes the sole stream and unblocks the reader. Generic environments retain at most 4096 consumed capability nonces and fail closed. Repository channels instead mutually authenticate the current boot, serialize lifecycle mutations with a monotonic sequence, retain one exact response for lost-acknowledgement replay, bind data claims to the current random registration incarnation, and cap persistent data at 16 slots plus one reserved control connection. A matching acknowledgement updates the named host registration incarnation even when another caller drains it; transport loss retains the exact pending request, while decoded semantic failure advances the sequence without applying success state. No host fallback exists | **derived/reset-safe**: generic reattachment mints a fresh capability; repository boot recovery rotates independently provisioned boot authority and reconstructs sequence/incarnation state. Earlier-boot proofs fail before either replay mechanism resets. No List 2 row | `environment/microvm/control/multiplex.go` (`Client`, `ServeMultiplex`); `environment/microvm/control/capability.go` (`CapabilityVerifier`); `environment/microvm/guestagent/guestagent.go` (`Server`, `Connect`); `environment/microvm/guestagent/repository.go` (`RepositoryServer`); `environment/microvm/repository_runtime.go` (`mutateLocked`) | | 78 | Local microVM inventory-continuation signing key | one random 256-bit HMAC key owned by the authoritative `microvmd` `Daemon` | daemon process | no goroutine, fd, timer, collection, or explicit cleanup; the fixed-size key dies with the daemon. It authenticates bounded owner/cursor tokens and is never projected or persisted | **reset-by-design**: a daemon restart invalidates outstanding status continuations; the operator restarts pagination from the first deterministic owner-scoped page. Tokens are navigation state, not session/run state, so no List 2 row | `environment/microvm/daemon.go` (`Daemon`, `encodeInventoryCursor`, `decodeInventoryCursor`) | | 79 | Repository-scoped microVM boot and logical-registration runtime | production `mecatl-microvmd` owns one `RepositoryComposition`; `RepositoryRuntime` owns current boot handles, capability issuers, authority copies, hosted network, and Unix listeners, while `RepositoryServer` owns the guest registration table | one replaceable boot under a stable repository placement; logical-environment lifetime for each registration and data stream | every connection proves the current boot authority over an unpredictable owner/repository/VM/boot-generation/purpose challenge before protocol payload. Boot-specific registrations use bounded monotonic mutation sequence and random incarnation, with exact lost-response retry. Graceful daemon shutdown stops VM/network resources under a bound and retains rootfs/worktrees/artifacts. On crash, launch ownership reconciliation prevents a second writer before replacement | **persisted/reconstructed split**: `RepositoryVMRegistry` persists stable placement plus current boot intent and immutable artifact/policy identities. A daemon restart reconstructs hosted networking and a new boot only after exact old-runner reconciliation, rotates authority, then recreates logical registrations from stable attachment refs. In-memory registrations, sequence state, listeners, and network handles reset. See List 1 row 80/List 2 row 50 for durable attachment inventory | `environment/microvm/repository_lifecycle.go` (`RepositoryVMRegistry`); `environment/microvm/repository_runtime.go` (`RepositoryRuntime`); `environment/microvm/repository_composition.go` (`RepositoryComposition`); `environment/microvm/guestagent/repository.go` (`RepositoryServer`); `environment/microvm/guestagent/repository_channel.go` (`AuthenticateHostRepositoryChannel`) | | 80 | Repository logical-attachment operator inventory | one private versioned flat record `attachments/.json` sibling to the guest-exported `logical/` tree, plus an ignored private `attachments/.staging/` publication namespace, loaded into `RepositoryAttachmentManager.records` | logical attachment and retained-dirty recovery lifetime; the map is one daemon process cache | registration fsyncs owner/session/ref/stable placement generation, repository identity, confined worktree metadata, and deletion-retained state to a staging file, then atomically renames and fsyncs the committed-record directory. Clean close/delete atomically unlinks the flat record and fsyncs that directory, so interruption cannot expose an empty committed child directory. The document stores no rotating boot copy; live workspace/runner handles remain process-local | **persist-in-store**: a fresh manager ignores only validated private staging residue, scans every committed flat record, and loudly rejects corruption, symlinks, inconsistent identities, legacy guest-exported metadata, or the obsolete unmerged directory layout while preserving data. It resolves the registry's current boot record and reconstructs bounded status plus exact logical delete without booting. Ordinary exact reattachment may recover the runtime. Pagination signing tokens reset per row 78. See List 2 row 50 | `environment/microvm/repository_attachment_store.go`; `environment/microvm/repository_attachment.go` (`RepositoryAttachmentManager`) | @@ -1058,6 +1067,19 @@ rather than papered over. | 77 | Harness-context process snapshots, per-session source-binding generations, and selected execution-file borrows | `internal/app.Built` owns process-scoped sources; `internal/app.harnessCommandResolver` owns session bindings for all five kinds, their in-flight references, and each selected source's returned cleanup | process plus canonical session ID and owner/profile; an execution-file borrow is anchored to the exact authorized placement for that generation | `Retire` rejects new borrows of that generation and defers cleanup until its final engine or child reference releases. Explicit owner-authorized reload can activate a fresh generation; old releases cannot evict it. Session teardown and unpublished-create failure retire bindings. `Built.Close` rejects activation and defers process cleanup until outstanding references drain. Failed binding creation releases only that attempt's source borrows and remains retryable; each execution-file cleanup releases its own exact placement borrow without closing the execution owner's binding | **reset-by-design**: sessions and schedules rebind current operator policy and exact placement under current source authorization after restart, without a durable context identity, migration, or legacy-state rejection. See List 2 row 49 | `internal/app/harness_context.go` (`harnessCommandResolver`); `internal/app/harness_context_binding.go` (`cacheProcessHarness`, `bindSessionSources`); `internal/app/harness_context_generation.go` (`harnessGeneration`); `internal/app/build.go` (`sessionContextFactory`); `internal/adapter/server/placement.go` (`executionWorkspaceAcquirer`); `internal/adapter/server/placement_discovery.go` (`ListCommandsForSession`) | **Harness-context re-audit (List 1 / List 2 — ADR 0359).** List 1 row 77 inventories process snapshots, session generations, in-flight accounting, and selected exact-placement source borrows. List 2 row 49 explicitly records their reset-by-design treatment; no durable harness-context reference is added. +| 76 | Experimental execution-provider bounded mTLS gRPC server and Kubernetes REST transports | `cmd/mecatl-execution-provider` | provider process | process cancellation gracefully stops the gRPC server (then hard-stops after the shutdown bound); Kubernetes clients own no separate application goroutine | **derive** from explicit flags, mounted mTLS material, in-cluster configuration, and the live Kubernetes API. Disabled `mecak8s` composition creates none of these resources | `cmd/mecatl-execution-provider/main.go`; `internal/adapter/executioncontroller/handler.go` (`Handler`) | +| 77 | ExecutionEnvironment informer, rate-limited workqueue, and controller worker | `executioncontroller.Reconciler` | provider process, namespace-scoped | the Run context stops the informer and worker; `Run` shuts down the queue before return | **derive** by listing and watching the external Kubernetes CRs after restart. Startup marks any persisted active operation `FenceUnknown` before admitting mutations | `internal/adapter/executioncontroller/controller.go` (`Reconciler`, `Initialize`, `Run`) | +| 78 | ExecutionEnvironment CR, retained PVC, executor Pod, and one status-held active-operation claim | Kubernetes API and storage provider; the execution controller reconciles them | logical remote environment | explicit retirement removes a proven-terminal Pod but retains the PVC. CR deletion remains finalizer-blocked until retirement and quiescence; this candidate has no PVC-delete path | **external Kubernetes state**: CR status, PVC data, and current Pod references survive provider or `mecak8s` process restart according to cluster/storage durability. An interrupted active-operation claim becomes fail-closed `FenceUnknown`, not reconstructed authority | `internal/adapter/executioncontroller/controller.go` (`Reconcile`, `reconcileRetiring`, `reconcileDeletion`); `internal/adapter/executioncontroller/store.go` (`execute`) | +| 79 | In-memory grant signer/verifier configuration, short-lived signed grants, and revocation maps | execution-provider `Handler`; each remote `grantContext` caches one grant | provider process / bound session | grants expire; client reattachment refreshes them. Signing and verifier values leave process memory at shutdown. Full durable key rotation and revocation are not implemented | **reset-by-design / derive** from mounted signing configuration and exact external environment state. A restart does not preserve in-memory nonce/key revocation, so this draft is not production rotation or revocation | `internal/adapter/executioncontroller/handler.go` (`GrantSigner`, `HandlerConfig`, `sign`); `internal/executionenv/grant.go`; `internal/adapter/executionclient/client.go` (`grantContext`) | +| 80 | Mecak8s execution-provider bounded mTLS gRPC client connection and per-binding remote Workspace/runner | `executionclient.Client` and the Service placement registry | process transport / session environment | `Client.Close` closes its gRPC connection; session engines and environments follow existing Service teardown. Each authority lookup and file/command call has caller or adapter bounds | **derive** by exact reattachment from the persisted `session.EnvironmentRef` and owner-bound provider state. Missing, changed, or fenced environments fail closed with no local fallback | `internal/adapter/executionclient/client.go` (`Client`, `Provider`, `workspace`, `runner`) | + +**Experimental native-execution re-audit (List 1 / List 2).** List 1 rows +76–80 inventory the provider server and transports, controller informer and queue, +external Kubernetes runtime state, grant state, and `mecak8s` client transport. +List 2 row 48 records the split between the persisted exact environment reference, +external CR/PVC state, and process-local operation/grant state. The implementation +is draft: run-wide ownership, complete successor reference lifecycle, controlled +replacement/PVC deletion, and durable grant-key rotation remain unresolved. **Mecatui agent-lifecycle-hook re-audit (List 1 / List 2).** The host-editor lifecycle emitter adds ONE outlives-a-call resource, List 1 row 75: the ordered @@ -1280,6 +1302,11 @@ repository VM records and durable worktree inventory remain the restart truth. | 50 | Contextual current-root trajectory | root-run `agent.reviewRoot.facts` plus completeness/byte accounting | the run and worker correlation ledger disappears | **reset-by-design**: its authority is deliberately one delegation-root run; no cross-session or restart inference is permitted | ADR 0363 | | 51 | Contextual exact repeat grants | Build-owned `modelhook.WaiverHolder` map and random HMAC key | every digest and the key disappear | **reset-by-design, fail-safe**: a restarted process/session cannot match the old grant and requires fresh review/human approval; it is not permission learning or approval replay | ADR 0363 | | 52 | Plan approval receipt | Build-owned `app.planApprovalReceipts`, exact ref/session/call/target mode | every unconsumed receipt disappears | **reset-by-design, fail-safe**: restart cannot reconstruct action authority; only a new real plan callback can mint another receipt | ADR 0363 | +| 48 | Native-execution pending reference intents | `ExecutionEnvironment.status.references` structured PendingCreate/PendingDelete records | process-local reconciler state resets; complete environment, owner, binding, source binding, operation ID, state, and creation time remain in the CR | **persist-in-CR**: the next bounded reference-intent pass commits, confirms, or cancels from durable state; unknown intent remains retained | native execution | +| 49 | Native-execution run/operation ownership and renewal | `ExecutionEnvironment.status.activeRun` and `status.activeOperation` | renewal goroutines and handles disappear; holder, claim/run IDs, epoch, generation, expiry, and operation fingerprint remain | **persist-in-CR and fail closed**: expiry becomes `FenceUnknown`; another replica never clears unknown ownership by timeout alone | native execution | +| 50 | Security authority and grant revocation | projected security files, authority high-water ConfigMap, and CR `grantGeneration` plus bounded revocation receipts | immutable in-memory snapshot and limiter counts reset; durable generation/digest/fingerprints, environment generation, and receipts remain | **persist authority metadata and derive snapshot**: reload all secret bytes atomically; drift, rollback, expiry, or unavailable ledger keeps readiness false | native execution | +| 51 | Native-execution allocation, lifecycle proof, and schema migration | profile-allocation ConfigMap plus `ExecutionEnvironment` spec/status, including runtime UIDs, termination proof, lifecycle operation, and migration operation | controller queues reset; retained CR/PVC and every operation phase remain observable | **persist-in-Kubernetes**: cache sync and exact UID/CAS checks resume convergence; malformed or unknown schema is retained for explicit administration | native execution | +| 52 | Native-execution Helm lifetime configuration, isolation, and ledgers | retained workload NetworkPolicies, profile/security-manifest ConfigMaps, authority high-water ConfigMap, and capacity ConfigMap | provider absence leaves enforcement and data in Kubernetes; Helm release history may disappear but original ownership metadata remains | **persist-in-Kubernetes and explicitly adopt**: same release/namespace and unchanged lifetime configuration only; live lookup renders actual ledger data while writers are quiesced. Missing history with retained allocations, foreign ownership, and incompatible configuration fail closed; no bootstrap reset. CRDs upgrade manually before compatible provider/schema migration | native execution N2; List 1 rows 85, 86, 88 | | 36 | Title coordinator queue/dedupe and in-flight title call | `server.titleCoordinator` | process | `Service.Close` cancels and joins; pending queued work is discarded | reset-by-design: durable incomplete attempts are reloaded as `interrupted` and exhausted, so no uncertain provider call is retried | title coordinator (SHIPPED) | | 37 | Native endpoint access and refresh tokens | protected `mecatl/provider-oidc/v1` record; transient access material in `nativeBearerSource` | access tokens reset; the encrypted exact-identity record reloads refresh state only | persist-in-record / derive: no OAuth material enters a session snapshot, event, diagnostic, or RPC; an unavailable, corrupt, or identity-drifted record is not-enrolled | ADR 0329 | | 38 | Native endpoint issuer/gateway clients, keyring/store handles, and transaction locks | `NativeEndpointRuntime` and `TransactionLocker` | handles and locks are closed/released; a restart has no live runtime state | explicitly reattach/reset: Build or local lifecycle reopens them only from explicit configuration; issuer/gateway trust stays separate and the next lifecycle operation reacquires its lock | ADR 0329 | @@ -1330,9 +1357,11 @@ repository VM records and durable worktree inventory remain the restart truth. | 47 | Prompt-free exact-ask handoff and acknowledgement-only resumed-run lifecycle (ADR 0347, ADR 0366; List 1 row 74) | the existing session snapshot persists `StateAwaiting`, exact `RunID`, `PendingAsk` with origin, mode, and `EnvironmentRef`; the accepting Service owns only reconstructed runs/contexts/relays and the process-local plan-continuation reservation described in List 1 | a restart or replica handoff loses local execution and join bookkeeping but retains the last authoritative awaiting or terminal snapshot. An accepting replica acquires the distributed lease, reloads after acquisition, and revalidates exact run, awaiting state, ask ID, required origin, and placement before `ResumeApproval`; a stale pre-lease snapshot cannot replay a verdict. A plan approval accepted before process loss but not yet followed by its fresh proceed run remains uncertain, never proof of execution | **persist existing snapshot, reconstruct under fresh lease**: no new durable field is introduced. Before shutdown cancellation reaches a detached continuation, an already-persisted awaiting point is marked for preservation and the relay is joined boundedly; otherwise terminal cancellation is persisted. Plan verdicts use the dedicated exact control, and an allow starts a fresh run only after `StopPlanApproved` | 0347, 0366 | | 48 | Mecatui rejected admission submission (ADR 0360; List 1 row 76) | one private `admissionSubmission` on `ui.Model`, keyed to session ID and stream generation | a client restart loses the prepared text/media, editable staging, counters, and recovery controls; the adopted server transcript remains unchanged because admission rejected before execution | **reset-by-design**: retain only in the current client until SessionInit, explicit recovery/disposal, session replacement/exit, or unrelated terminal error. Retry is explicit and byte-identical; Back transfers staging with confirmation before replacing a newer draft. No snapshot, event, or transport schema changes | 0360 | | 49 | Harness-context policy, source snapshots, and binding generations (ADR 0359; List 1 row 77) | Build-local registrations and process snapshots; owner/profile-bound session generations with reference counts and retirement state | restart loses the selected source objects and lifecycle bookkeeping, not session history or execution placement; existing sessions and schedules resolve current configured sources using their authoritative owner/profile and current source authorization | **reset-by-design**: current deployment composition is authoritative on rebind. No `HarnessContextRef`, source path, policy snapshot, migration, or legacy-session rejection is persisted. Configuration changes can change resumed context | 0359 | +| 47 | Prompt-free ordinary-ask handoff and acknowledgement-only resumed-run lifecycle (ADR 0347; List 1 row 74) | the existing session snapshot persists `StateAwaiting`, exact `RunID`, `PendingAsk`, mode, and `EnvironmentRef`; the accepting Service owns only the reconstructed run/context/relay described in List 1 | a restart or replica handoff loses local execution and join bookkeeping but retains the last authoritative awaiting or terminal snapshot. An accepting replica acquires the distributed lease, reloads after acquisition, and revalidates exact run, awaiting state, ask ID, non-plan origin, and placement before `ResumeApproval`; a stale pre-lease snapshot cannot replay allow-once work | **persist existing snapshot, reconstruct under fresh lease**: no new durable field is introduced. Before shutdown cancellation reaches a detached continuation, an already-persisted awaiting point is marked for preservation and the relay is joined boundedly; otherwise terminal cancellation is persisted so accepted work is not ambiguously replayed. Plan asks remain on their dedicated resolution path | 0347 | +| 48 | Experimental native Kubernetes environment identity, references, grants, and active operation | the session snapshot persists the exact `EnvironmentRef`; the Kubernetes `ExecutionEnvironment` CR/status and retained PVC hold provider-side identity, references, operation state, and workspace data; grants and client transports are process memory | restart reloads the session reference and exactly reattaches through the provider to the external CR. The controller derives its queue from an informer list/watch. A provider restart that observes an active operation clears the claim only by marking the environment `FenceUnknown`; it does not infer completion or takeover. Short-lived grants and in-memory revocation state reset | **persist-in-snapshot + external Kubernetes state + derive/reset-by-design**: derive the live Workspace/runner from exact reattachment; derive controller work from CRs; reset transports, queue state, and expired grants. Complete successor reference release, run-wide ownership, automated fenced recovery, controlled replacement/PVC deletion, and durable multi-key rotation are not implemented | experimental native execution; List 1 rows 76–88 | -| 47 | Repository-scoped local microVM placement and boot record (ADR 0367; proposed ADR 0365; List 1 row 73) | `environment/microvm.RepositoryVMRegistry`: canonical owner/repository key, stable placement generation, private rootfs, retained artifact/policy identities, and replaceable boot intent | restart reloads the stable placement. A healthy same-daemon boot is reused; otherwise platform-specific exact launch ownership reconciles the old runner before authority rotation and a new boot. Linux uses pidfd-backed exact signaling. Darwin waits on the inherited attempt flock, never signals a stored PID, and requires operator recovery if a historical runner remains locked after its supervisor dies. Provisioning retries use the same rootfs and artifacts. Missing, malformed, incompatible, pending, or uncertain state fails without replacement or deletion | **persist-in-store/reconstruct boot**: one owner/repository-confined, flock-serialized versioned JSON record plus retained artifact trees and launch-attempt records. Registry and authority commits use fsync and atomic rename. Concurrent recovery converges under the same key lock; session and schedule refs retain the stable generation. Darwin's supervisor, daemon-liveness pipe, and flock descriptor copies are process resources inventoried in List 1 row 73, not persisted session/run state | 0350, proposed 0351 | -| 49 | Repository-logical child identity, immutable exact-tree fork base, and process-local merge synchronization (ADR 0367; List 1 row 75) | child binding and repository/worktree metadata persist in the private attachment record; the active `RepositoryAttachmentManager` map alone holds parent ref and fork base, while the daemon owns one merge mutex | restart loses active guest handles, fork parentage/base, and mutex state. Persisted attachment identity remains available for exact status, delete, and reattachment against the retained repository generation, but does not authorize merge | **persist/reset split**: exact attachment identity survives, while fork relationship and merge authority reset by design. A restarted process cannot merge the child. Crash-durable fork parentage and cross-process merge coordination remain deferred. Cancellation cleanup is detached and bounded; malformed fork responses never authorize automatic deletion because the returned tuple does not prove request-scoped creation. The caller receives guidance to inspect status and explicitly delete the intended attachment. | ADR 0367 | +| 47 | Repository-scoped local microVM placement and boot record (ADR 0368; proposed ADR 0365; List 1 row 73) | `environment/microvm.RepositoryVMRegistry`: canonical owner/repository key, stable placement generation, private rootfs, retained artifact/policy identities, and replaceable boot intent | restart reloads the stable placement. A healthy same-daemon boot is reused; otherwise platform-specific exact launch ownership reconciles the old runner before authority rotation and a new boot. Linux uses pidfd-backed exact signaling. Darwin waits on the inherited attempt flock, never signals a stored PID, and requires operator recovery if a historical runner remains locked after its supervisor dies. Provisioning retries use the same rootfs and artifacts. Missing, malformed, incompatible, pending, or uncertain state fails without replacement or deletion | **persist-in-store/reconstruct boot**: one owner/repository-confined, flock-serialized versioned JSON record plus retained artifact trees and launch-attempt records. Registry and authority commits use fsync and atomic rename. Concurrent recovery converges under the same key lock; session and schedule refs retain the stable generation. Darwin's supervisor, daemon-liveness pipe, and flock descriptor copies are process resources inventoried in List 1 row 73, not persisted session/run state | 0350, proposed 0351 | +| 49 | Repository-logical child identity, immutable exact-tree fork base, and process-local merge synchronization (ADR 0368; List 1 row 75) | child binding and repository/worktree metadata persist in the private attachment record; the active `RepositoryAttachmentManager` map alone holds parent ref and fork base, while the daemon owns one merge mutex | restart loses active guest handles, fork parentage/base, and mutex state. Persisted attachment identity remains available for exact status, delete, and reattachment against the retained repository generation, but does not authorize merge | **persist/reset split**: exact attachment identity survives, while fork relationship and merge authority reset by design. A restarted process cannot merge the child. Crash-durable fork parentage and cross-process merge coordination remain deferred. Cancellation cleanup is detached and bounded; malformed fork responses never authorize automatic deletion because the returned tuple does not prove request-scoped creation. The caller receives guidance to inspect status and explicitly delete the intended attachment. | ADR 0368 | | 50 | Repository logical-attachment inventory and dirty-retention decision (List 1 row 80) | host-only private versioned metadata stores exact owner/session/ref/stable generation, repository identity, source/worktree/metadata/branch paths, and deletion-retained state; the manager map is derived | restart drops live guest registrations and Workspace/runner handles but reloads status/delete identity without booting. The document resolves the registry's current boot instead of persisting a stale boot copy. A dirty retained worktree remains available for exact recovery or cleanup | **persist-in-store**: fsync + atomic rename + directory fsync under the host-only `attachments/` namespace outside the guest export. Startup rejects legacy exported metadata, malformed, symlinked, duplicate, or path-inconsistent records. Exact delete does not require guest reattachment; repository-VM deletion and a retention reconciler remain deferred | Repository inventory correction | Two ledger observations worth stating in prose: diff --git a/docs/adr/0365-microvm-darwin-xattr-ownership.md b/docs/adr/0365-microvm-darwin-xattr-ownership.md index 5e428f9ee8..aac49f23bb 100644 --- a/docs/adr/0365-microvm-darwin-xattr-ownership.md +++ b/docs/adr/0365-microvm-darwin-xattr-ownership.md @@ -3,12 +3,12 @@ - Status: Proposed - Date: 2026-09-23 - Scope: Darwin arm64 VirtioFS ownership preparation and exact launch ownership -- Supersedes on acceptance: ADR 0367 section 4 only for the macOS platform deferral; the Linux amd64 mechanism remains authoritative +- Supersedes on acceptance: ADR 0368 section 4 only for the macOS platform deferral; the Linux amd64 mechanism remains authoritative - Superseded by: none ## Context -ADR 0367 fixes the guest workload identity at UID/GID `65532:65532`. Linux maps +ADR 0368 fixes the guest workload identity at UID/GID `65532:65532`. Linux maps that identity to the daemon account with an unprivileged user namespace. Darwin arm64 has no equivalent namespace mapping in this integration, but go-microvm v0.0.41 can project a prepared VirtioFS ownership xattr as the fixed guest @@ -123,6 +123,6 @@ uses the inherited flock instead. ## See also -- [ADR 0367](./0367-microvm-execution-environments.md) +- [ADR 0368](./0368-microvm-execution-environments.md) - [MicroVM execution-environment acceptance plan](../acceptance/microvm-execution-environments.md) - [MicroVM architecture](../architecture/microvm-environments.md) diff --git a/docs/adr/0367-microvm-execution-environments.md b/docs/adr/0368-microvm-execution-environments.md similarity index 99% rename from docs/adr/0367-microvm-execution-environments.md rename to docs/adr/0368-microvm-execution-environments.md index 4a4c45ce18..40c84c8edf 100644 --- a/docs/adr/0367-microvm-execution-environments.md +++ b/docs/adr/0368-microvm-execution-environments.md @@ -1,4 +1,4 @@ -# ADR 0367 — Repository-scoped local microVM execution environments +# ADR 0368 — Repository-scoped local microVM execution environments - Status: Accepted - Date: 2026-08-19 diff --git a/docs/adr/README.md b/docs/adr/README.md index 18a73c53ec..26a077daaa 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -159,8 +159,8 @@ Documentation/citation conventions are in [`docs/design/README.md`](../design/RE - [0208 — Execution environments and version-aware file mutation](./0208-execution-environment.md) *(runtime-seam deferral superseded by 0211; version protocol authoritative)* - [0211 — Execution-environment runtime seam](./0211-execution-environment-runtime-seam.md) *(supersedes 0208 decisions 1–3; phase-3 persistence deferral superseded by 0214)* - [0214 — Execution-environment persistence and reattachment](./0214-environment-persistence.md) *(supersedes 0211 decision 6 only)* -- [0367 — Local microVM execution environments and developer-only release activation](./0367-microvm-execution-environments.md) -- [0365 — Prepare Darwin VirtioFS ownership and supervise the direct runner](./0365-microvm-darwin-xattr-ownership.md) *(proposed; partially supersedes ADR 0367's macOS platform deferral if accepted; Linux amd64 remains authoritative)* +- [0368 — Local microVM execution environments and developer-only release activation](./0368-microvm-execution-environments.md) +- [0365 — Prepare Darwin VirtioFS ownership and supervise the direct runner](./0365-microvm-darwin-xattr-ownership.md) *(proposed; partially supersedes ADR 0368's macOS platform deferral if accepted; Linux amd64 remains authoritative)* ### Agents, teams & delegation - [0283 — Managed delegation-fork lifecycle](./0283-managed-delegation-fork-lifecycle.md) *(proposed; depends on 0281)* diff --git a/docs/architecture.md b/docs/architecture.md index 5b5ae25ecb..883f433715 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1467,7 +1467,7 @@ idempotent readiness immediately before each actual default MicroVM provision at startup and no-FS creation do not run readiness or allocate a validation attachment. The live `microvm-local` support boundary is the signed Linux-amd64 `mecatui` release binary: ordinary source builds have no authenticated release defaults and fail closed. For source -development only, [ADR 0367](adr/0367-microvm-execution-environments.md#6-keep-source-build-release-activation-developer-only) defines a +development only, [ADR 0368](adr/0368-microvm-execution-environments.md#6-keep-source-build-release-activation-developer-only) defines a separately tagged `microvm_dev` mecated and embedded-local mecatui binaries whose development activation requires explicit acknowledgement and a strict owner-only local release descriptor. Untagged and published @@ -1491,7 +1491,7 @@ exact logical attachment across fires. Before the first claim, deleting an indep placed schedule cleans its attachment while preserving dirty state. After the first claim, the attachment is retained for historical and resumable fire sessions when the schedule is deleted. See the [microVM architecture](architecture/microvm-environments.md) and -[operator guide](usage/microvm-environments.md) (ADR 0367). +[operator guide](usage/microvm-environments.md) (ADR 0368). Discovery is source-session scoped. `ListCommands(session_id)` and `ListWorktrees(session_id)` first authorize the owner and exactly reattach that source. diff --git a/docs/architecture/microvm-environments.md b/docs/architecture/microvm-environments.md index d52796f3c4..c1354ff265 100644 --- a/docs/architecture/microvm-environments.md +++ b/docs/architecture/microvm-environments.md @@ -289,4 +289,4 @@ schedules borrow their exact logical attachment, while independent schedules all attachment and persist a trusted ownership bit. Every fire reattaches that ref. Delete disables first, preserves a claimed/running record for scheduler recovery, and removes only the owned logical attachment; dirty state remains exact-reattachable and the repository VM/rootfs and -siblings are never deleted. See [ADR 0367](../adr/0367-microvm-execution-environments.md). +siblings are never deleted. See [ADR 0368](../adr/0368-microvm-execution-environments.md). diff --git a/docs/tui.md b/docs/tui.md index d5f9e1dae5..46bad72f72 100644 --- a/docs/tui.md +++ b/docs/tui.md @@ -493,18 +493,25 @@ mecatui connect 127.0.0.1:8080 --resume-latest Adoption does not create a temporary session: mecatui loads and displays the stored transcript, placement label, mode, model, and capabilities, then targets the same opaque ID. -`--mode` describes only a newly created session; an adopted chat keeps its stored values. Scheduled runs, child runs, unknown legacy rows, chats -awaiting approval, active chats, and rows without a complete authoritative transcript -are not eligible. Exact `--resume` reports why its row cannot be continued; `--resume-latest` -skips ineligible or unreadable rows and tries the next one. A genuine inventory-list -failure still surfaces rather than being masked as "start new". - -The read-only startup lookup does not reopen, recover, abandon, or acquire a lease. -Those checks remain atomic at the ordinary run-entry funnel when the first new prompt -is sent. If that attachment fails, the stored transcript remains visible and no -fallback session is created: press `r` to retry the preserved prompt or `esc` to go -Back and edit it. Combining a resume selector with `--prompt` or `--prompt-file` -adopts the transcript first and then submits the seed exactly once as the next turn. +`--mode` describes only a newly created session; an adopted chat keeps its stored values. +Scheduled runs, child runs, unknown legacy rows, active chats, and rows without a complete +authoritative transcript are not eligible for ordinary continuation. Exact `--resume` can +instead open the guarded recovery path for an owned main chat with one supported pending +ordinary approval. `--resume-latest` always excludes chats awaiting approval, skips other +ineligible or unreadable rows, and tries the next one. Exact `--resume` reports why a row +cannot be continued or safely recovered. A genuine inventory-list failure still surfaces +rather than being masked as "start new". + +The ordinary continuation lookup does not reopen, recover, abandon, or acquire a +lease. Those checks remain atomic at the ordinary run-entry funnel when the first +new prompt is sent. Exact pending-approval recovery instead verifies a durable +watch and waits for the owner's choice; see +[Recover a pending approval](https://github.com/stacklok/mecatl/blob/main/user-docs/mecatui/sessions.md#recover-a-pending-approval). +If ordinary attachment fails, the stored transcript remains visible and no +fallback session is created: press `r` to retry the preserved prompt or `esc` to +go Back and edit it. Combining a resume selector with `--prompt` or +`--prompt-file` submits the seed exactly once after an ordinary continuation; +during pending-approval recovery it remains an unsent draft. On an ordinary clean exit, after the terminal has left the alternate screen and cleanup has completed, mecatui writes exactly one handoff line to **stderr**: @@ -527,7 +534,9 @@ decoded value to `--resume` later. `-p`/`--prompt` (or `--prompt-file` for a longer body) launches the session with a seed prompt auto-submitted as the FIRST turn — the equivalent of typing the prompt and pressing enter the moment the session is ready. The TUI then stays -interactive for follow-ups; this is NOT a print-and-exit one-shot. +interactive for follow-ups; this is NOT a print-and-exit one-shot. During +pending-approval recovery, the seed remains an unsent composer draft instead; +it is not automatically submitted after the approval or recovered run finishes. ```sh mecatui -p "Summarize the failing tests in this repo" @@ -545,9 +554,9 @@ a short directive with a longer brief. The seed fires ONCE: a `/models` restart |---|---|---| | `--workspace` | cwd | **embedded only:** trusted operator configuration for the hosted server's default root; never sent in CreateSession and rejected by every `connect` form | | `--mode` | `default` | permission posture for a new session: `default` \| `plan` \| `accept-edits`; an adopted chat keeps its stored mode | -| `--resume` | – | continue the owned main chat with this exact opaque session ID; loads its authoritative transcript without creating a throwaway session; mutually exclusive with `--resume-latest` | -| `--resume-latest` | off | continue the newest eligible owned main chat with an available authoritative transcript; excludes active, awaiting, scheduled, child, and unknown sessions; when none is eligible, start a new chat; mutually exclusive with `--resume` | -| `-p` / `--prompt` | – | seed prompt auto-submitted once the first session is ready (the CLI task to launch with). The TUI stays interactive for follow-ups; this is NOT a one-shot. Both `--prompt` and `--prompt-file` may be given (literal first, joined by a blank line). Fires ONCE — a `/models` restart or `/clear` never re-submits it | +| `--resume` | – | continue the owned stored main chat with this exact opaque session ID, or recover its supported pending ordinary approval; mutually exclusive with `--resume-latest` | +| `--resume-latest` | off | continue the newest eligible owned main chat with an available authoritative transcript; excludes active, pending-approval, scheduled, child, and unknown sessions; when none is eligible, start a new chat; mutually exclusive with `--resume` | +| `-p` / `--prompt` | – | seed prompt auto-submitted once the first session is ready; kept as an unsent draft during pending-approval recovery. The TUI stays interactive for follow-ups; this is NOT a one-shot. Both `--prompt` and `--prompt-file` may be given (literal first, joined by a blank line). Fires ONCE — a `/models` restart or `/clear` never re-submits it | | `--prompt-file` | – | path to a file whose contents are the seed prompt body. Read at startup (fail-fast on unreadable). Joined after `--prompt` when both are given. Same once-only semantics as `--prompt` | | `--theme` | `aztec` | theme name (also `MECATUI_THEME`); giving either pins the theme and disables the light/dark auto-detect below | | `--theme-dir` | – | extra directory of `*.json` themes to load | diff --git a/e2e/k8s_execution/aa_production_migration_test.go b/e2e/k8s_execution/aa_production_migration_test.go new file mode 100644 index 0000000000..86ccae5a5f --- /dev/null +++ b/e2e/k8s_execution/aa_production_migration_test.go @@ -0,0 +1,9 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import "testing" + +func TestKindExecutionProductionCompatiblePrototypeMigration(t *testing.T) { + runKindExecutionProductionCompatiblePrototypeMigration(t) +} diff --git a/e2e/k8s_execution/ab_production_admin_test.go b/e2e/k8s_execution/ab_production_admin_test.go new file mode 100644 index 0000000000..3932ec14fe --- /dev/null +++ b/e2e/k8s_execution/ab_production_admin_test.go @@ -0,0 +1,70 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestKindExecutionProductionScopedAdministrator(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + qualifyDistinctAdministrator(t, ctx, state, kubeconfig, "initial") +} + +func qualifyDistinctAdministrator(t *testing.T, ctx context.Context, state, kubeconfig, phase string) { + t.Helper() + creator, forward := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, creator, "admin-"+phase) + clients := map[string]*executionclient.Client{} + for _, name := range []string{"operations", "wrong-scope", "intruder"} { + client, err := executionclient.New(forward.addr, loadTLS(t, filepath.Join(state, "pki"), name, "mecatl-execution.execution-qualification.svc.cluster.local")) + if err != nil { + t.Fatal("create synthetic admin client") + } + t.Cleanup(func() { client.Close() }) + clients[name] = client + } + rc, release := acquireRun(t, ctx, creator, owner, binding, attached, "admin-data-"+phase) + if _, err := creator.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "admin-sentinel", Data: []byte("creator-only\n")}); err != nil { + t.Fatal("creator data-plane positive:", remoteErrorCode(err)) + } + admin := clients["operations"] + if _, err := admin.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); !isRemoteCode(err, executionenv.CodeNotFound) { + t.Fatal("scoped administrator acquired creator attach privilege:", remoteErrorCode(err)) + } + if _, err := admin.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: "admin-sentinel"}); !isRemoteCode(err, executionenv.CodePermissionDenied) { + t.Fatal("scoped administrator used creator file grant:", remoteErrorCode(err)) + } + if _, err := admin.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "true"}); !isRemoteCode(err, executionenv.CodePermissionDenied) { + t.Fatal("scoped administrator used creator command grant:", remoteErrorCode(err)) + } + release() + before := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + request := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: before.Epoch, ExpectedPodUID: before.PodUID, ExpectedPVCUID: before.PVCUID, OperationID: "admin-replace-" + phase} + for name, code := range map[string]executionenv.ErrorCode{"wrong-scope": executionenv.CodeNotFound, "intruder": executionenv.CodePermissionDenied} { + if err := clients[name].ReplaceExecutor(ctx, request); !isRemoteCode(err, code) { + t.Fatalf("%s admin refusal: %s", name, remoteErrorCode(err)) + } + } + if after := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID); after != before { + t.Fatal("denied admin calls changed runtime state") + } + if err := admin.ReplaceExecutor(ctx, request); err != nil { + t.Fatal("distinct scoped administrator replacement:", remoteErrorCode(err)) + } + after := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Ready && s.PodUID != before.PodUID }) + if after.PVCUID != before.PVCUID || after.Epoch <= before.Epoch { + t.Fatal("scoped replacement failed exact PVC/epoch reconciliation") + } + reattached := waitReady(t, ctx, creator, owner, binding, attached.Environment) + rc, release = acquireRun(t, ctx, creator, owner, binding, reattached, "admin-verify-"+phase) + waitFileContent(t, ctx, creator, rc, "admin-sentinel", "creator-only\n", "scoped replacement") + release() + retireSyntheticEnvironment(t, ctx, creator, kubeconfig, owner, binding, attached.Environment) +} diff --git a/e2e/k8s_execution/file_tools_test.go b/e2e/k8s_execution/file_tools_test.go new file mode 100644 index 0000000000..65b88f56cb --- /dev/null +++ b/e2e/k8s_execution/file_tools_test.go @@ -0,0 +1,61 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/stacklok/mecatl/engine/adapter/fstools" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func qualifyRemoteFileTools(t *testing.T, ctx context.Context, client *executionclient.Client, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) { + t.Helper() + provider, err := executionclient.NewProvider(client, "go") + if err != nil { + t.Fatal(err) + } + handle, err := provider.AcquireRun(ctx, server.ExecutionRunRequest{Ref: session.EnvironmentRef{Kind: "kubernetes", ID: ref.ID, Revision: ref.Revision}, Principal: &session.Principal{Issuer: owner.Issuer, Subject: owner.Subject, GrantType: session.GrantTypeUser}, BindingID: session.SessionID(binding), RunID: "file-tool-matrix"}) + if err != nil { + t.Fatal(err) + } + defer func() { + if err := handle.Release(ctx); err != nil { + t.Error(err) + } + }() + env := handle.Environment() + call := func(body tool.Tool, args string, wantError bool, contains string) { + t.Helper() + result, err := body.Execute(ctx, session.ToolCall{ID: "matrix", Name: body.Spec().Name, Args: json.RawMessage(args)}, env) + if err != nil || result.IsError != wantError || !strings.Contains(result.Content, contains) { + t.Fatalf("%s: result=%+v err=%v", body.Spec().Name, result, err) + } + } + if _, err := env.Workspace().CreateFile(ctx, "matrix/source.go", []byte("package matrix\n// before\n")); err != nil { + t.Fatal(err) + } + call(fstools.EditTool{}, `{"path":"matrix/source.go","old_string":"before","new_string":"after"}`, true, "") + call(fstools.WriteTool{}, `{"path":"matrix/source.go","content":"clobber"}`, true, "") + call(fstools.CopyTool{}, `{"source":"matrix/source.go","destination":"matrix/copied.go"}`, false, "") + call(fstools.MoveTool{}, `{"source":"matrix/copied.go","destination":"matrix/moved.go"}`, false, "") + call(fstools.RemoveTool{}, `{"path":"matrix/moved.go"}`, false, "") + call(fstools.ReadTool{}, `{"path":"matrix/source.go"}`, false, "before") + call(fstools.EditTool{}, `{"path":"matrix/source.go","old_string":"before","new_string":"after"}`, false, "") + call(fstools.WriteTool{}, `{"path":"matrix/source.go","content":"package matrix\n// final\n"}`, false, "") + call(fstools.WriteTool{}, `{"path":"matrix/new.go","content":"package matrix\n"}`, false, "") + call(fstools.ListDirTool{}, `{"path":"matrix"}`, false, "source.go") + call(fstools.GrepTool{}, `{"path":"matrix/*.go","pattern":"final"}`, false, "final") + call(fstools.GlobTool{}, `{"pattern":"matrix/*.go"}`, false, "source.go") + data, err := env.Workspace().Read(ctx, "matrix/source.go") + if err != nil || string(data) != "package matrix\n// final\n" { + t.Fatalf("persisted matrix=%q err=%v", data, err) + } +} diff --git a/e2e/k8s_execution/fixture/credentialloader/main.go b/e2e/k8s_execution/fixture/credentialloader/main.go new file mode 100644 index 0000000000..7f69f1882e --- /dev/null +++ b/e2e/k8s_execution/fixture/credentialloader/main.go @@ -0,0 +1,279 @@ +//go:build kind_execution_e2e + +// Command credentialloader is the only qualification component permitted to read +// a real provider credential. It creates a new, narrowly labelled Secret and +// never reads Secret data back from Kubernetes. +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + utilvalidation "k8s.io/apimachinery/pkg/util/validation" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + namespace = "execution-qualification" + secretKey = "OPENROUTER_API_KEY" + maxCredentialBytes = 16 << 10 + maxReceiptBytes = 4 << 10 +) + +type cleanupReceipt struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + UID types.UID `json:"uid"` + Context string `json:"context"` + Kubeconfig string `json:"kubeconfig"` +} + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, errorMessage(err)) + os.Exit(1) + } +} + +func errorMessage(err error) string { + return "credential loader: " + classify(err) +} + +func run(args []string) error { + if len(args) != 5 { + return errors.New("usage") + } + mode, kubeconfig, contextName, name, receiptPath := args[0], args[1], args[2], args[3], args[4] + if (mode != "stage" && mode != "delete") || !strings.HasPrefix(name, "mecak8s-live-") || len(utilvalidation.IsDNS1123Subdomain(name)) != 0 { + return errors.New("invalid arguments") + } + kubeconfig, receiptPath, err := validatePrivatePaths(kubeconfig, receiptPath, name) + if err != nil { + return err + } + + // Resolve kube configuration (including any auth plugin) and prove API access + // before the credential file is opened. + cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, + &clientcmd.ConfigOverrides{CurrentContext: contextName}, + ).ClientConfig() + if err != nil { + return fmt.Errorf("kube configuration: %w", err) + } + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return fmt.Errorf("kube client: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if _, err = client.Discovery().ServerVersion(); err != nil { + return fmt.Errorf("kube access: %w", err) + } + + secrets := client.CoreV1().Secrets(namespace) + if mode == "delete" { + if err := deleteStagedSecret(ctx, secrets, receiptPath, kubeconfig, contextName, name); err != nil { + return err + } + fmt.Println(`{"status":"deleted"}`) + return nil + } + path := os.Getenv("MECATL_EXECUTION_CREDENTIAL_FILE") + credential, err := loadCredential(path) + if err != nil { + return err + } + defer clear(credential) + if err := stageSecret(ctx, secrets, receiptPath, kubeconfig, contextName, name, credential); err != nil { + return err + } + fmt.Println(`{"status":"staged","provider":"openrouter"}`) + return nil +} + +func stageSecret(ctx context.Context, secrets v1.SecretInterface, receiptPath, kubeconfig, contextName, name string, credential []byte) error { + created, err := secrets.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace, Labels: map[string]string{"app.kubernetes.io/managed-by": "mecatl-execution-live-qualification"}}, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{secretKey: credential}, + }, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("secret stage: %w", err) + } + if created == nil || created.UID == "" { + return errors.New("secret staged without UID; cleanup pending") + } + receipt := cleanupReceipt{Name: name, Namespace: namespace, UID: created.UID, Context: contextName, Kubeconfig: kubeconfig} + if err := writeReceipt(receiptPath, receipt); err != nil { + uid := receipt.UID + _ = secrets.Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + return errors.New("cleanup receipt creation failed; cleanup status ambiguous") + } + return nil +} + +func deleteStagedSecret(ctx context.Context, secrets v1.SecretInterface, receiptPath, kubeconfig, contextName, name string) error { + receipt, err := readReceipt(receiptPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("cleanup receipt: %w", err) + } + if receipt.Name != name || receipt.Namespace != namespace || receipt.Context != contextName || receipt.Kubeconfig != kubeconfig { + return errors.New("cleanup receipt identity mismatch") + } + if receipt.UID == "" { + return errors.New("secret staged without UID; cleanup pending") + } + uid := receipt.UID + err = secrets.Delete(ctx, name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("secret cleanup: %w", err) + } + if err := os.Remove(receiptPath); err != nil && !errors.Is(err, os.ErrNotExist) { + return errors.New("cleanup receipt removal failed") + } + return nil +} + +func validatePrivatePaths(kubeconfig, receiptPath, name string) (string, string, error) { + if !filepath.IsAbs(kubeconfig) || !filepath.IsAbs(receiptPath) { + return "", "", errors.New("state paths must be absolute") + } + kubeconfig, err := filepath.EvalSymlinks(kubeconfig) + if err != nil { + return "", "", errors.New("kubeconfig unavailable") + } + kubeInfo, err := os.Stat(kubeconfig) + if err != nil || !kubeInfo.Mode().IsRegular() || kubeInfo.Mode().Perm()&0o077 != 0 { + return "", "", errors.New("kubeconfig must be a private regular file") + } + stateDir, err := filepath.EvalSymlinks(filepath.Dir(receiptPath)) + if err != nil || stateDir != filepath.Dir(kubeconfig) { + return "", "", errors.New("receipt path must be in kubeconfig state directory") + } + info, err := os.Stat(stateDir) + if err != nil || !info.IsDir() || info.Mode().Perm()&0o077 != 0 { + return "", "", errors.New("state directory must be private") + } + base := filepath.Base(receiptPath) + if base != "live-secret-"+name+".receipt.json" || filepath.Clean(receiptPath) != filepath.Join(stateDir, base) { + return "", "", errors.New("invalid receipt path") + } + return kubeconfig, filepath.Join(stateDir, base), nil +} + +func writeReceipt(path string, receipt cleanupReceipt) error { + blob, err := json.Marshal(receipt) + if err != nil { + return err + } + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + ok := false + defer func() { + _ = f.Close() + if !ok { + _ = os.Remove(path) + } + }() + if _, err := f.Write(append(blob, '\n')); err != nil { + return err + } + if err := f.Sync(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + ok = true + return nil +} + +func readReceipt(path string) (cleanupReceipt, error) { + var receipt cleanupReceipt + f, err := os.Open(path) + if err != nil { + return receipt, err + } + defer f.Close() + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm() != 0o600 || info.Size() > maxReceiptBytes { + return receipt, errors.New("receipt must be a bounded private regular file") + } + dec := json.NewDecoder(io.LimitReader(f, maxReceiptBytes+1)) + dec.DisallowUnknownFields() + if err := dec.Decode(&receipt); err != nil { + return receipt, errors.New("invalid cleanup receipt") + } + var extra any + if err := dec.Decode(&extra); !errors.Is(err, io.EOF) { + return receipt, errors.New("invalid cleanup receipt") + } + return receipt, nil +} + +func loadCredential(path string) ([]byte, error) { + if path == "" || !filepath.IsAbs(path) { + return nil, errors.New("credential file must be an absolute path") + } + f, err := os.Open(path) + if err != nil { + return nil, errors.New("credential file unavailable") + } + defer f.Close() + info, err := f.Stat() + if err != nil || !info.Mode().IsRegular() { + return nil, errors.New("credential file must be regular") + } + if info.Mode().Perm()&0o077 != 0 { + return nil, errors.New("credential file permissions must exclude group and others") + } + b, err := io.ReadAll(io.LimitReader(f, maxCredentialBytes+1)) + if err != nil { + return nil, errors.New("credential file read failed") + } + defer clear(b) + if len(b) > maxCredentialBytes { + return nil, errors.New("credential file exceeds limit") + } + trimmed := bytes.TrimSpace(b) + if len(trimmed) == 0 { + return nil, errors.New("credential file is empty") + } + return append([]byte(nil), trimmed...), nil +} + +func classify(err error) string { + s := err.Error() + for _, allowed := range []string{ + "usage", "invalid arguments", "state paths must be absolute", "kubeconfig unavailable", "kubeconfig must be a private regular file", + "receipt path must be in kubeconfig state directory", "state directory must be private", "invalid receipt path", + "credential file must be an absolute path", "credential file unavailable", "credential file must be regular", + "credential file permissions must exclude group and others", "credential file read failed", "credential file exceeds limit", "credential file is empty", + "cleanup receipt creation failed; cleanup status ambiguous", "secret staged without UID; cleanup pending", "cleanup receipt identity mismatch", + } { + if s == allowed { + return allowed + } + } + return "operation failed (details redacted)" +} diff --git a/e2e/k8s_execution/fixture/credentialloader/main_test.go b/e2e/k8s_execution/fixture/credentialloader/main_test.go new file mode 100644 index 0000000000..3a49282812 --- /dev/null +++ b/e2e/k8s_execution/fixture/credentialloader/main_test.go @@ -0,0 +1,188 @@ +//go:build kind_execution_e2e + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" +) + +func TestLoadCredentialAndRedaction(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "credential") + sentinel := "synthetic-provider-secret-never-print" + if err := os.WriteFile(path, []byte(" "+sentinel+"\n"), 0o600); err != nil { + t.Fatal(err) + } + got, err := loadCredential(path) + if err != nil { + t.Fatal(err) + } + if string(got) != sentinel { + t.Fatal("credential was not trimmed") + } + clear(got) + if out := classify(errors.New("request rejected: " + sentinel)); strings.Contains(out, sentinel) || out != "operation failed (details redacted)" { + t.Fatalf("classification was not strictly redacted: %q", out) + } + if stderr := errorMessage(errors.New("provider body: " + sentinel)); strings.Contains(stderr, sentinel) || stderr != "credential loader: operation failed (details redacted)" { + t.Fatalf("stderr was not strictly redacted: %q", stderr) + } +} + +func TestLoadCredentialRejectsBroadPermissions(t *testing.T) { + path := filepath.Join(t.TempDir(), "credential") + if err := os.WriteFile(path, []byte("synthetic"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := loadCredential(path); err == nil { + t.Fatal("broad permissions accepted") + } +} + +func TestDeletePinsReceiptUIDAndLeavesReplacement(t *testing.T) { + state, receipt, kubeconfig := privateState(t) + _ = state + const name = "mecak8s-live-replacement-test" + originalUID := types.UID("original-synthetic-uid") + client := fake.NewSimpleClientset() + client.PrependReactor("create", "secrets", func(action ktesting.Action) (bool, runtime.Object, error) { + created := action.(ktesting.CreateAction).GetObject().(*corev1.Secret).DeepCopy() + created.UID = originalUID + return true, created, nil + }) + if err := stageSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", name, []byte("synthetic-sentinel")); err != nil { + t.Fatal(err) + } + + replacementUID := types.UID("replacement-synthetic-uid") + client.PrependReactor("delete", "secrets", func(action ktesting.Action) (bool, runtime.Object, error) { + deleteAction := action.(ktesting.DeleteAction) + preconditions := deleteAction.GetDeleteOptions().Preconditions + if preconditions == nil || preconditions.UID == nil || *preconditions.UID != originalUID { + t.Fatal("cleanup did not pin the staged UID") + } + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "secrets"}, name, errors.New("UID precondition does not match")) + }) + if err := deleteStagedSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", name); err == nil { + t.Fatal("replacement UID mismatch was accepted") + } + if _, err := os.Stat(receipt); err != nil { + t.Fatal("receipt was removed despite UID mismatch") + } + _ = replacementUID // The simulated replacement identity must never be targeted. + assertNoSecretReads(t, client.Actions()) +} + +func TestDeleteTreatsNotFoundAsCleaned(t *testing.T) { + _, receipt, kubeconfig := privateState(t) + const name = "mecak8s-live-not-found-test" + uid := types.UID("staged-synthetic-uid") + if err := writeReceipt(receipt, cleanupReceipt{Name: name, Namespace: namespace, UID: uid, Context: "kind-owned", Kubeconfig: kubeconfig}); err != nil { + t.Fatal(err) + } + client := fake.NewSimpleClientset() + client.PrependReactor("delete", "secrets", func(action ktesting.Action) (bool, runtime.Object, error) { + preconditions := action.(ktesting.DeleteAction).GetDeleteOptions().Preconditions + if preconditions == nil || preconditions.UID == nil || *preconditions.UID != uid { + t.Fatal("cleanup did not pin the staged UID") + } + return true, nil, apierrors.NewNotFound(schema.GroupResource{Resource: "secrets"}, name) + }) + if err := deleteStagedSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", name); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(receipt); !errors.Is(err, os.ErrNotExist) { + t.Fatal("successful cleanup retained its receipt") + } + assertNoSecretReads(t, client.Actions()) +} + +func TestReceiptIsPrivateAndCreateOnly(t *testing.T) { + path := filepath.Join(t.TempDir(), "receipt") + receipt := cleanupReceipt{Name: "mecak8s-live-receipt-test", Namespace: namespace, UID: "synthetic-uid", Context: "kind-owned", Kubeconfig: "/private/kubeconfig"} + if err := writeReceipt(path, receipt); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil || info.Mode().Perm() != 0o600 { + t.Fatal("receipt was not created with mode 0600") + } + if err := writeReceipt(path, receipt); err == nil { + t.Fatal("receipt overwrite was accepted") + } +} + +func TestStageFailureDoesNotDeleteOrWriteReceipt(t *testing.T) { + _, receipt, kubeconfig := privateState(t) + client := fake.NewSimpleClientset() + client.PrependReactor("create", "secrets", func(ktesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewAlreadyExists(schema.GroupResource{Resource: "secrets"}, "mecak8s-live-stage-failure") + }) + err := stageSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", "mecak8s-live-stage-failure", []byte("synthetic-sentinel")) + if err == nil { + t.Fatal("stage failure was accepted") + } + if _, err := os.Stat(receipt); !errors.Is(err, os.ErrNotExist) { + t.Fatal("stage failure wrote a cleanup receipt") + } + for _, action := range client.Actions() { + if action.GetVerb() == "delete" { + t.Fatal("stage failure attempted deletion") + } + } + assertNoSecretReads(t, client.Actions()) +} + +func TestMissingUIDDoesNotWriteReceiptOrDeleteByName(t *testing.T) { + _, receipt, kubeconfig := privateState(t) + client := fake.NewSimpleClientset(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "unrelated", Namespace: namespace}}) + name := "mecak8s-live-ambiguous-test" + if err := stageSecret(context.Background(), client.CoreV1().Secrets(namespace), receipt, kubeconfig, "kind-owned", name, []byte("synthetic-sentinel")); err == nil || err.Error() != "secret staged without UID; cleanup pending" { + t.Fatalf("missing UID result = %v", err) + } + if _, err := os.Stat(receipt); !errors.Is(err, os.ErrNotExist) { + t.Fatal("missing UID wrote an invalid cleanup receipt") + } + for _, action := range client.Actions() { + if action.GetVerb() == "delete" { + t.Fatal("missing UID triggered name-only deletion") + } + } + assertNoSecretReads(t, client.Actions()) +} + +func privateState(t *testing.T) (string, string, string) { + t.Helper() + state := t.TempDir() + if err := os.Chmod(state, 0o700); err != nil { + t.Fatal(err) + } + kubeconfig := filepath.Join(state, "kubeconfig") + if err := os.WriteFile(kubeconfig, []byte("synthetic"), 0o600); err != nil { + t.Fatal(err) + } + return state, filepath.Join(state, "live-secret-test.receipt.json"), kubeconfig +} + +func assertNoSecretReads(t *testing.T, actions []ktesting.Action) { + t.Helper() + for _, action := range actions { + if action.GetResource().Resource == "secrets" && (action.GetVerb() == "get" || action.GetVerb() == "list") { + t.Fatalf("unexpected Secret %s action", action.GetVerb()) + } + } +} diff --git a/e2e/k8s_execution/fixture/legacyfixture/crd.yaml b/e2e/k8s_execution/fixture/legacyfixture/crd.yaml new file mode 100644 index 0000000000..a8fe3be98e --- /dev/null +++ b/e2e/k8s_execution/fixture/legacyfixture/crd.yaml @@ -0,0 +1,34 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: executionenvironments.execution.mecatl.dev +spec: + group: execution.mecatl.dev + scope: Namespaced + names: + plural: executionenvironments + singular: executionenvironment + kind: ExecutionEnvironment + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + required: [spec] + properties: + spec: + type: object + x-kubernetes-preserve-unknown-fields: true + status: + type: object + x-kubernetes-preserve-unknown-fields: true + properties: + references: + type: array + maxItems: 64 + items: + type: string diff --git a/e2e/k8s_execution/fixture/legacyfixture/main.go b/e2e/k8s_execution/fixture/legacyfixture/main.go new file mode 100644 index 0000000000..45833ecf9c --- /dev/null +++ b/e2e/k8s_execution/fixture/legacyfixture/main.go @@ -0,0 +1,52 @@ +//go:build kind_execution_e2e + +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" +) + +func main() { + if len(os.Args) != 4 { + fmt.Fprintln(os.Stderr, "usage: legacyfixture NAMESPACE PROFILES OUTPUT") + os.Exit(2) + } + config, err := clientcmd.BuildConfigFromFlags("", os.Getenv("KUBECONFIG")) + if err != nil { + fail(err) + } + d, err := dynamic.NewForConfig(config) + if err != nil { + fail(err) + } + kube, err := kubernetes.NewForConfig(config) + if err != nil { + fail(err) + } + seed, err := executioncontroller.SeedLegacyMigrationFixture(context.Background(), d, kube, os.Args[1], os.Args[2]) + if err != nil { + fail(err) + } + encoded, err := json.Marshal(seed) + if err != nil { + fail(err) + } + encoded = append(encoded, '\n') + if err := os.WriteFile(os.Args[3], encoded, 0o600); err != nil { + fail(err) + } +} + +func fail(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/e2e/k8s_execution/fixture/netprobe/main.go b/e2e/k8s_execution/fixture/netprobe/main.go new file mode 100644 index 0000000000..fae09de4cb --- /dev/null +++ b/e2e/k8s_execution/fixture/netprobe/main.go @@ -0,0 +1,37 @@ +//go:build kind_execution_e2e + +package main + +import ( + "flag" + "fmt" + "net" + "net/http" + "os" + "time" +) + +func main() { + listen := flag.String("listen", "", "HTTP listen address") + target := flag.String("target", "", "TCP endpoint to probe") + timeout := flag.Duration("timeout", 3*time.Second, "dial timeout") + flag.Parse() + if *listen != "" { + srv := &http.Server{Addr: *listen, Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }), ReadHeaderTimeout: 2 * time.Second} + if err := srv.ListenAndServe(); err != nil { + fmt.Fprintln(os.Stderr, "listen failed") + os.Exit(1) + } + } + if *target == "" { + fmt.Fprintln(os.Stderr, "target is required") + os.Exit(2) + } + conn, err := net.DialTimeout("tcp", *target, *timeout) + if err != nil { + fmt.Fprintln(os.Stderr, "connect denied or unavailable") + os.Exit(42) + } + _ = conn.Close() + fmt.Println("connected") +} diff --git a/e2e/k8s_execution/fixture/oidcissuer/main.go b/e2e/k8s_execution/fixture/oidcissuer/main.go new file mode 100644 index 0000000000..ddd1c3ce8c --- /dev/null +++ b/e2e/k8s_execution/fixture/oidcissuer/main.go @@ -0,0 +1,91 @@ +//go:build kind_execution_e2e + +// Command oidcissuer is a synthetic OIDC issuer for the offline Kind qualification. +package main + +import ( + "crypto/rsa" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "flag" + "fmt" + "log" + "math/big" + "net/http" + "os" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +func main() { + var certFile, keyFile, signingFile, issuer, audience string + flag.StringVar(&certFile, "tls-cert", "", "TLS certificate") + flag.StringVar(&keyFile, "tls-key", "", "TLS key") + flag.StringVar(&signingFile, "signing-key", "", "RSA PKCS8 signing key") + flag.StringVar(&issuer, "issuer", "", "exact issuer URL") + flag.StringVar(&audience, "audience", "mecatl", "token audience") + flag.Parse() + if certFile == "" || keyFile == "" || signingFile == "" || issuer == "" { + log.Fatal("all identity flags are required") + } + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + must(err) + raw, err := os.ReadFile(signingFile) + must(err) + block, _ := pem.Decode(raw) + if block == nil { + log.Fatal("invalid signing key") + } + parsed, err := x509.ParsePKCS8PrivateKey(block.Bytes) + must(err) + key, ok := parsed.(*rsa.PrivateKey) + if !ok { + log.Fatal("signing key is not RSA") + } + kidRaw := sha256.Sum256(x509.MarshalPKCS1PublicKey(&key.PublicKey)) + kid := base64.RawURLEncoding.EncodeToString(kidRaw[:12]) + mux := http.NewServeMux() + mux.HandleFunc("GET /.well-known/openid-configuration", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, map[string]any{"issuer": issuer, "jwks_uri": issuer + "/keys", "id_token_signing_alg_values_supported": []string{"RS256"}}) + }) + mux.HandleFunc("GET /keys", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, map[string]any{"keys": []any{map[string]any{"kty": "RSA", "use": "sig", "alg": "RS256", "kid": kid, "n": base64.RawURLEncoding.EncodeToString(key.N.Bytes()), "e": base64.RawURLEncoding.EncodeToString(big.NewInt(int64(key.E)).Bytes())}}}) + }) + mux.HandleFunc("GET /token", func(w http.ResponseWriter, r *http.Request) { + sub := r.URL.Query().Get("sub") + if sub != "alice" && sub != "bob" { + http.Error(w, "unknown fixture subject", http.StatusBadRequest) + return + } + now := time.Now().UTC() + tok := jwt.NewWithClaims(jwt.SigningMethodRS256, jwt.MapClaims{"iss": issuer, "aud": audience, "sub": sub, "iat": now.Unix(), "exp": now.Add(10 * time.Minute).Unix()}) + tok.Header["kid"] = kid + signed, err := tok.SignedString(key) + if err != nil { + http.Error(w, "signing failed", 500) + return + } + writeJSON(w, map[string]string{"id_token": signed}) + }) + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + server := &http.Server{Addr: ":8443", Handler: mux, ReadHeaderTimeout: 5 * time.Second, TLSConfig: &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{cert}}} + ln, err := tls.Listen("tcp", server.Addr, server.TLSConfig) + must(err) + must(server.Serve(ln)) +} +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(v); err != nil { + log.Print("response encoding failed") + } +} +func must(err error) { + if err != nil { + log.Fatal(fmt.Errorf("fixture startup: %w", err)) + } +} diff --git a/e2e/k8s_execution/fixture/pki/main.go b/e2e/k8s_execution/fixture/pki/main.go new file mode 100644 index 0000000000..3a2c2205e7 --- /dev/null +++ b/e2e/k8s_execution/fixture/pki/main.go @@ -0,0 +1,107 @@ +//go:build kind_execution_e2e + +// Command pki generates synthetic, short-lived qualification identities. +package main + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/json" + "encoding/pem" + "math/big" + "net/url" + "os" + "path/filepath" + "time" +) + +func main() { + if len(os.Args) != 2 { + panic("usage: pki OUTPUT_DIRECTORY") + } + dir := os.Args[1] + if err := os.MkdirAll(dir, 0o700); err != nil { + panic(err) + } + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + must(err) + now := time.Now().UTC() + ca := &x509.Certificate{SerialNumber: serial(), Subject: pkix.Name{CommonName: "mecatl execution qualification CA"}, NotBefore: now.Add(-time.Minute), NotAfter: now.Add(6 * time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) + must(err) + write(filepath.Join(dir, "ca.crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), 0o600) + must(issue(dir, "provider", ca, caKey, []string{"mecatl-execution", "mecatl-execution.execution-qualification.svc", "mecatl-execution.execution-qualification.svc.cluster.local"}, "", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth})) + must(issue(dir, "mecak8s", ca, caKey, nil, "spiffe://mecatl.test/client/mecak8s", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})) + must(issue(dir, "intruder", ca, caKey, nil, "spiffe://mecatl.test/client/intruder", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})) + for _, name := range []string{"operations", "wrong-scope"} { + must(issue(dir, name, ca, caKey, nil, "spiffe://mecatl.test/client/"+name, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})) + } + grantPub, grantKey, err := ed25519.GenerateKey(rand.Reader) + must(err) + grantDER, err := x509.MarshalPKCS8PrivateKey(grantKey) + must(err) + write(filepath.Join(dir, "grant-key.pem"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: grantDER}), 0o600) + fingerprint := sha256.Sum256(grantPub) + manifest, err := json.Marshal(map[string]any{ + "version": 1, "generation": 1, "issuer": "https://mecatl.execution.test", "audience": "mecatl-execution", "activeKeyID": "k1", "grantTTL": "1m", "clockSkew": "5s", + "keys": []any{map[string]any{"id": "k1", "version": 1, "file": "grant-k1.pem", "publicKeySHA256": hex.EncodeToString(fingerprint[:]), "activateAt": now.Add(-time.Minute).Format(time.RFC3339), "verifyUntil": now.Add(5 * time.Hour).Format(time.RFC3339), "state": "active"}}, + "tls": map[string]any{"certificateFile": "tls.crt", "privateKeyFile": "tls.key", "clientCAFile": "clients.pem"}, + "clients": []any{ + map[string]any{"uri": "spiffe://mecatl.test/client/mecak8s", "mayAttestOwner": true, "administrator": true}, + map[string]any{"uri": "spiffe://mecatl.test/client/intruder", "mayAttestOwner": true, "administrator": false}, + map[string]any{"uri": "spiffe://mecatl.test/client/operations", "mayAttestOwner": true, "administrator": true, "administratorFor": []string{"spiffe://mecatl.test/client/mecak8s"}}, + map[string]any{"uri": "spiffe://mecatl.test/client/wrong-scope", "mayAttestOwner": true, "administrator": true, "administratorFor": []string{"spiffe://mecatl.test/client/intruder"}}, + }, + }) + must(err) + write(filepath.Join(dir, "manifest.json"), append(manifest, '\n'), 0o600) + must(issue(dir, "oidc", ca, caKey, []string{"oidc-issuer", "oidc-issuer.execution-qualification.svc", "oidc-issuer.execution-qualification.svc.cluster.local"}, "", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth})) + jwtKey, err := rsa.GenerateKey(rand.Reader, 2048) + must(err) + jwtDER, err := x509.MarshalPKCS8PrivateKey(jwtKey) + must(err) + write(filepath.Join(dir, "oidc-jwt-key.pem"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: jwtDER}), 0o600) +} + +func issue(dir, name string, ca *x509.Certificate, caKey *rsa.PrivateKey, dns []string, uri string, usages []x509.ExtKeyUsage) error { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return err + } + cert := &x509.Certificate{SerialNumber: serial(), Subject: pkix.Name{CommonName: name}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(6 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, ExtKeyUsage: usages, DNSNames: dns} + if uri != "" { + u, err := url.Parse(uri) + if err != nil { + return err + } + cert.URIs = []*url.URL{u} + } + der, err := x509.CreateCertificate(rand.Reader, cert, ca, &key.PublicKey, caKey) + if err != nil { + return err + } + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + return err + } + write(filepath.Join(dir, name+".crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600) + write(filepath.Join(dir, name+".key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}), 0o600) + return nil +} + +func serial() *big.Int { + n, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + must(err) + return n +} +func write(path string, data []byte, mode os.FileMode) { must(os.WriteFile(path, data, mode)) } +func must(err error) { + if err != nil { + panic(err) + } +} diff --git a/e2e/k8s_execution/fixture/rotation/main.go b/e2e/k8s_execution/fixture/rotation/main.go new file mode 100644 index 0000000000..3eb0c748fe --- /dev/null +++ b/e2e/k8s_execution/fixture/rotation/main.go @@ -0,0 +1,140 @@ +//go:build kind_execution_e2e + +// Command rotation creates synthetic execution-provider rotation candidates. +// It consumes only fixture material generated for the current qualification run. +package main + +import ( + "crypto/ed25519" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/json" + "encoding/pem" + "math/big" + "net/url" + "os" + "path/filepath" + "time" +) + +func main() { + if len(os.Args) != 3 { + panic("usage: rotation INITIAL_PKI OUTPUT_DIRECTORY") + } + generate(os.Args[1], os.Args[2]) +} + +func generate(initial, out string) { + must(os.MkdirAll(out, 0o700)) + now := time.Now().UTC() + oldCA := read(filepath.Join(initial, "ca.crt")) + oldGrant := readGrant(filepath.Join(initial, "grant-key.pem")) + + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + must(err) + ca := &x509.Certificate{SerialNumber: serial(), Subject: pkix.Name{CommonName: "mecatl execution qualification rotation CA"}, NotBefore: now.Add(-time.Minute), NotAfter: now.Add(6 * time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) + must(err) + newCA := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}) + write(filepath.Join(out, "new-ca.crt"), newCA) + write(filepath.Join(out, "client-roots.pem"), append(append([]byte{}, oldCA...), newCA...)) + write(filepath.Join(out, "bridge-clients.pem"), append(append([]byte{}, oldCA...), newCA...)) + write(filepath.Join(out, "final-clients.pem"), newCA) + issue(out, "provider-new", ca, caKey, []string{"mecatl-execution", "mecatl-execution.execution-qualification.svc", "mecatl-execution.execution-qualification.svc.cluster.local"}, "", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}) + issue(out, "mecak8s-new", ca, caKey, nil, "spiffe://mecatl.test/client/mecak8s", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}) + + pub2, key2, err := ed25519.GenerateKey(rand.Reader) + must(err) + der2, err := x509.MarshalPKCS8PrivateKey(key2) + must(err) + write(filepath.Join(out, "grant-k2.pem"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der2})) + + writeManifest(filepath.Join(out, "invalid-same-generation.json"), manifest(1, "changed-policy.invalid", "k1", []keyEntry{key("k1", 1, "grant-k1.pem", oldGrant.Public().(ed25519.PublicKey), "active", now.Add(5*time.Hour))}, "tls.crt", "tls.key", "clients.pem")) + keys := []keyEntry{ + key("k1", 1, "grant-k1.pem", oldGrant.Public().(ed25519.PublicKey), "active", now.Add(5*time.Hour)), + key("k2", 2, "grant-k2.pem", pub2, "active", now.Add(5*time.Hour)), + } + writeManifest(filepath.Join(out, "bridge.json"), manifest(2, "mecatl-execution", "k2", keys, "tls.crt", "tls.key", "bridge-clients.pem")) + keys[0].State = "revoked" + writeManifest(filepath.Join(out, "final.json"), manifest(3, "mecatl-execution", "k2", keys, "provider-new.crt", "provider-new.key", "final-clients.pem")) + writeManifest(filepath.Join(out, "restore-fixture-clients.json"), manifest(4, "mecatl-execution", "k2", keys, "provider-new.crt", "provider-new.key", "bridge-clients.pem")) +} + +type keyEntry struct { + ID, File, Fingerprint, State string + Version uint64 + ActivateAt, VerifyUntil time.Time +} + +func key(id string, version uint64, file string, pub ed25519.PublicKey, state string, until time.Time) keyEntry { + sum := sha256.Sum256(pub) + return keyEntry{id, file, hex.EncodeToString(sum[:]), state, version, time.Now().UTC().Add(-time.Minute), until} +} + +func manifest(generation uint64, audience, active string, keys []keyEntry, cert, privateKey, clients string) map[string]any { + entries := make([]any, 0, len(keys)) + for _, k := range keys { + entries = append(entries, map[string]any{"id": k.ID, "version": k.Version, "file": k.File, "publicKeySHA256": k.Fingerprint, "activateAt": k.ActivateAt.Format(time.RFC3339), "verifyUntil": k.VerifyUntil.Format(time.RFC3339), "state": k.State}) + } + return map[string]any{ + "version": 1, "generation": generation, "issuer": "https://mecatl.execution.test", "audience": audience, "activeKeyID": active, "grantTTL": "1m", "clockSkew": "5s", "keys": entries, + "tls": map[string]any{"certificateFile": cert, "privateKeyFile": privateKey, "clientCAFile": clients}, + "clients": []any{ + map[string]any{"uri": "spiffe://mecatl.test/client/mecak8s", "mayAttestOwner": true, "administrator": true}, + map[string]any{"uri": "spiffe://mecatl.test/client/intruder", "mayAttestOwner": true, "administrator": false}, + map[string]any{"uri": "spiffe://mecatl.test/client/operations", "mayAttestOwner": true, "administrator": true, "administratorFor": []string{"spiffe://mecatl.test/client/mecak8s"}}, + map[string]any{"uri": "spiffe://mecatl.test/client/wrong-scope", "mayAttestOwner": true, "administrator": true, "administratorFor": []string{"spiffe://mecatl.test/client/intruder"}}, + }, + } +} + +func writeManifest(path string, value any) { + data, err := json.Marshal(value) + must(err) + write(path, append(data, '\n')) +} +func readGrant(path string) ed25519.PrivateKey { + block, _ := pem.Decode(read(path)) + if block == nil { + panic("invalid synthetic grant key") + } + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + must(err) + out, ok := key.(ed25519.PrivateKey) + if !ok { + panic("unexpected synthetic grant key type") + } + return out +} +func issue(dir, name string, ca *x509.Certificate, caKey *rsa.PrivateKey, dns []string, uri string, usages []x509.ExtKeyUsage) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + must(err) + cert := &x509.Certificate{SerialNumber: serial(), Subject: pkix.Name{CommonName: name}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(6 * time.Hour), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, ExtKeyUsage: usages, DNSNames: dns} + if uri != "" { + u, err := url.Parse(uri) + must(err) + cert.URIs = []*url.URL{u} + } + der, err := x509.CreateCertificate(rand.Reader, cert, ca, &key.PublicKey, caKey) + must(err) + keyDER, err := x509.MarshalPKCS8PrivateKey(key) + must(err) + write(filepath.Join(dir, name+".crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + write(filepath.Join(dir, name+".key"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER})) +} +func serial() *big.Int { + n, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + must(err) + return n +} +func read(path string) []byte { data, err := os.ReadFile(path); must(err); return data } +func write(path string, data []byte) { must(os.WriteFile(path, data, 0o600)) } +func must(err error) { + if err != nil { + panic(err) + } +} diff --git a/e2e/k8s_execution/fixture/rotation/main_test.go b/e2e/k8s_execution/fixture/rotation/main_test.go new file mode 100644 index 0000000000..bbbd6a5f9b --- /dev/null +++ b/e2e/k8s_execution/fixture/rotation/main_test.go @@ -0,0 +1,127 @@ +//go:build kind_execution_e2e + +package main + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "math/big" + "net" + "os" + "path/filepath" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +func TestForwardRestoreTrustBundleSupportsFixtureClient(t *testing.T) { + now := time.Now().UTC() + initial := filepath.Join(t.TempDir(), "initial") + out := filepath.Join(t.TempDir(), "rotation") + if err := os.MkdirAll(initial, 0o700); err != nil { + t.Fatal(err) + } + + oldCAKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + oldCA := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "fixture CA"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign} + oldCADER, err := x509.CreateCertificate(rand.Reader, oldCA, oldCA, &oldCAKey.PublicKey, oldCAKey) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(initial, "ca.crt"), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: oldCADER}), 0o600); err != nil { + t.Fatal(err) + } + _, grant, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + grantDER, err := x509.MarshalPKCS8PrivateKey(grant) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(initial, "grant-key.pem"), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: grantDER}), 0o600); err != nil { + t.Fatal(err) + } + generate(initial, out) + for _, tc := range []struct { + file, cert, key, ca string + }{ + {"bridge.json", "tls.crt", "tls.key", "bridge-clients.pem"}, + {"final.json", "provider-new.crt", "provider-new.key", "final-clients.pem"}, + {"restore-fixture-clients.json", "provider-new.crt", "provider-new.key", "bridge-clients.pem"}, + } { + var manifest struct { + TLS struct { + CertificateFile, PrivateKeyFile, ClientCAFile string + } + } + if err := json.Unmarshal(read(filepath.Join(out, tc.file)), &manifest); err != nil { + t.Fatal(err) + } + if manifest.TLS.CertificateFile != tc.cert || manifest.TLS.PrivateKeyFile != tc.key || manifest.TLS.ClientCAFile != tc.ca { + t.Errorf("%s must reference immutable material names", tc.file) + } + } + + clientKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + clientTemplate := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "fixture client"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + clientDER, err := x509.CreateCertificate(rand.Reader, clientTemplate, oldCA, &clientKey.PublicKey, oldCAKey) + if err != nil { + t.Fatal(err) + } + clientCert := tls.Certificate{Certificate: [][]byte{clientDER}, PrivateKey: clientKey} + serverCert, err := tls.LoadX509KeyPair(filepath.Join(out, "provider-new.crt"), filepath.Join(out, "provider-new.key")) + if err != nil { + t.Fatal(err) + } + bundle, err := os.ReadFile(filepath.Join(out, "client-roots.pem")) + if err != nil { + t.Fatal(err) + } + roots := x509.NewCertPool() + if !roots.AppendCertsFromPEM(bundle) { + t.Fatal("generated forward trust bundle contained no certificates") + } + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{serverCert}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: roots}))) + healthpb.RegisterHealthServer(server, health.NewServer()) + serveDone := make(chan error, 1) + go func() { serveDone <- server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + _ = listener.Close() + <-serveDone + }) + + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, ServerName: "mecatl-execution.execution-qualification.svc.cluster.local", RootCAs: roots, Certificates: []tls.Certificate{clientCert}}))) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + if _, err := healthpb.NewHealthClient(conn).Check(ctx, &healthpb.HealthCheckRequest{}); err != nil { + t.Fatal("forward-restored TLS gRPC handshake failed") + } +} diff --git a/e2e/k8s_execution/holder_loss_helpers_test.go b/e2e/k8s_execution/holder_loss_helpers_test.go new file mode 100644 index 0000000000..c1ff8aa4ed --- /dev/null +++ b/e2e/k8s_execution/holder_loss_helpers_test.go @@ -0,0 +1,61 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "errors" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" +) + +func terminateOwnedExecutor(ctx context.Context, kube kubernetes.Interface, environmentID, environmentUID, podUID string) error { + pods, err := kube.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: "execution.mecatl.dev/environment=" + environmentID}) + if err != nil { + return err + } + if len(pods.Items) != 1 || podUID == "" || environmentUID == "" { + return errors.New("exact owned executor unavailable") + } + pod := pods.Items[0] + if string(pod.UID) != podUID || len(pod.OwnerReferences) != 1 { + return errors.New("executor identity changed") + } + owner := pod.OwnerReferences[0] + if owner.APIVersion != "execution.mecatl.dev/v1alpha1" || owner.Kind != "ExecutionEnvironment" || owner.Name != environmentID || string(owner.UID) != environmentUID || owner.Controller == nil || !*owner.Controller { + return errors.New("executor owner changed") + } + uid, grace := types.UID(podUID), int64(30) + return kube.CoreV1().Pods(namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &grace, Preconditions: &metav1.Preconditions{UID: &uid}}) +} + +func executorTerminalProof(pod *corev1.Pod, uid string) bool { + if pod == nil || string(pod.UID) != uid || uid == "" || (pod.Status.Phase != corev1.PodFailed && pod.Status.Phase != corev1.PodSucceeded) { + return false + } + names := make(map[string]bool) + for _, c := range pod.Spec.Containers { + names[c.Name] = false + } + for _, c := range pod.Spec.InitContainers { + names[c.Name] = false + } + for _, c := range pod.Spec.EphemeralContainers { + names[c.Name] = false + } + statuses := append(append(append([]corev1.ContainerStatus{}, pod.Status.ContainerStatuses...), pod.Status.InitContainerStatuses...), pod.Status.EphemeralContainerStatuses...) + if len(pod.Spec.Containers) == 0 || len(statuses) != len(names) { + return false + } + for _, s := range statuses { + done, found := names[s.Name] + if !found || done || s.State.Terminated == nil { + return false + } + names[s.Name] = true + } + return true +} diff --git a/e2e/k8s_execution/holder_loss_test.go b/e2e/k8s_execution/holder_loss_test.go new file mode 100644 index 0000000000..f4525d930f --- /dev/null +++ b/e2e/k8s_execution/holder_loss_test.go @@ -0,0 +1,68 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + kubefake "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" +) + +func TestHolderLossTerminalStimulusPinsOwnedPod(t *testing.T) { + for _, changed := range []string{"", "uid", "owner", "absent"} { + t.Run(changed, func(t *testing.T) { + controlled := true + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "executor", Namespace: namespace, UID: "pod-uid", Labels: map[string]string{"execution.mecatl.dev/environment": "env"}, OwnerReferences: []metav1.OwnerReference{{APIVersion: "execution.mecatl.dev/v1alpha1", Kind: "ExecutionEnvironment", Name: "env", UID: "env-uid", Controller: &controlled}}}} + if changed == "uid" { + pod.UID = "foreign" + } + if changed == "owner" { + pod.OwnerReferences[0].UID = "foreign" + } + k := kubefake.NewClientset(pod) + if changed == "absent" { + k = kubefake.NewClientset() + } + deletes := 0 + k.PrependReactor("delete", "pods", func(a ktesting.Action) (bool, runtime.Object, error) { + deletes++ + opts := a.(ktesting.DeleteAction).GetDeleteOptions() + if opts.GracePeriodSeconds == nil || *opts.GracePeriodSeconds != 30 || opts.Preconditions == nil || opts.Preconditions.UID == nil || *opts.Preconditions.UID != "pod-uid" { + t.Fatal("deletion must pin UID with normal grace") + } + return true, nil, nil + }) + err := terminateOwnedExecutor(context.Background(), k, "env", "env-uid", "pod-uid") + if (err == nil) != (changed == "") || deletes != map[bool]int{true: 1, false: 0}[changed == ""] { + t.Fatalf("deletes=%d err=%v", deletes, err) + } + for _, a := range k.Actions() { + if a.GetVerb() != "list" && a.GetVerb() != "delete" { + t.Fatal("stimulus must not patch finalizers") + } + } + }) + } +} + +func TestHolderLossTerminalProofRequiresExactCompleteEvidence(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{UID: "pod-uid"}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "executor"}}, InitContainers: []corev1.Container{{Name: "init"}}}, Status: corev1.PodStatus{Phase: corev1.PodFailed, ContainerStatuses: []corev1.ContainerStatus{{Name: "executor", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{}}}}, InitContainerStatuses: []corev1.ContainerStatus{{Name: "init", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{}}}}}} + if !executorTerminalProof(pod, "pod-uid") { + t.Fatal("terminal proof rejected") + } + for _, mutate := range []func(*corev1.Pod){func(p *corev1.Pod) { p.UID = "foreign" }, func(p *corev1.Pod) { p.Status.Phase = corev1.PodRunning }, func(p *corev1.Pod) { p.Status.InitContainerStatuses = nil }, func(p *corev1.Pod) { p.Status.ContainerStatuses[0].Name = "other" }, func(p *corev1.Pod) { p.Status.ContainerStatuses[0].State.Terminated = nil }} { + candidate := pod.DeepCopy() + mutate(candidate) + if executorTerminalProof(candidate, "pod-uid") { + t.Fatal("incomplete or foreign evidence accepted") + } + } + if executorTerminalProof(nil, "pod-uid") { + t.Fatal("absence is not terminal proof") + } +} diff --git a/e2e/k8s_execution/live_qualification_test.go b/e2e/k8s_execution/live_qualification_test.go new file mode 100644 index 0000000000..19faca92ff --- /dev/null +++ b/e2e/k8s_execution/live_qualification_test.go @@ -0,0 +1,564 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptrace" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + liveModel = "anthropic/claude-haiku-4.5" + liveCredentialKey = "OPENROUTER_API_KEY" +) + +func TestKindExecutionLiveQualification(t *testing.T) { + if os.Getenv("MECATL_EXECUTION_LIVE") != "1" { + t.Skip("explicit live qualification only") + } + state := os.Getenv("MECATL_EXECUTION_QUAL_STATE") + kubeconfig := filepath.Join(state, "kubeconfig") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancel() + + tokenForward := portForward(t, ctx, kubeconfig, "service/oidc-issuer", 8443) + alice := fixtureToken(t, ctx, tokenForward.addr, filepath.Join(state, "pki"), "alice") + tokenForward.stop() + agentForward := portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + stopAgentForward := cleanupOnce(agentForward.stop) + defer stopAgentForward() + + preflightCtx, preflightCancel := context.WithTimeout(ctx, 15*time.Second) + t.Log("live stage=authenticated_http_preflight reason=begin") + preflightStatus, _ := request(t, preflightCtx, http.MethodGet, "http://"+agentForward.addr+"/v1/info", alice, nil) + preflightCancel() + if preflightStatus != http.StatusOK { + t.Fatalf("live stage=authenticated_http_preflight reason=rejected status=%d", preflightStatus) + } + t.Log("live stage=authenticated_http_preflight reason=ok") + created := createLiveSession(t, ctx, agentForward.addr, alice) + secretName := os.Getenv("MECATL_EXECUTION_LIVE_SECRET") + if !strings.HasPrefix(secretName, "mecak8s-live-") { + t.Fatal("run-scoped provider Secret name is unavailable") + } + assertCredentialScope(t, ctx, kubeconfig, secretName) + if created.ResolvedModel == nil || created.ResolvedModel.ProviderId != "openrouter" || created.ResolvedModel.ModelId != liveModel { + t.Fatal("live session resolved an unexpected provider or model") + } + nonce := fmt.Sprintf("qual_%d", time.Now().UnixNano()) + promptText := "Create a small Go 1.27 module with arithmetic/sum.go defining Sum(a, b int) int and a table-driven arithmetic/sum_test.go covering positive, negative, and zero inputs. Include the comment // qualification: " + nonce + " in sum.go. Use the Write tool to create the files, then use Shell with the exact command `go test ./...`. Fix any failures and finish with a concise result only after that exact command passes. Do not add external dependencies." + events := livePrompt(t, ctx, agentForward.addr, created.SessionId, alice, promptText) + + calls := map[string]bool{} + modelShellPassed := modelRanExactShell(events, "go test ./...") + var usage *mecatlv1.Usage + stop := "" + for _, ev := range events { + if ev.ToolCall != nil { + calls[ev.ToolCall.Name] = true + } + if ev.ToolResult != nil && ev.ToolResult.IsError { + t.Fatal("live coding smoke had a tool error") + } + if ev.Result != nil { + stop = ev.Result.Stop + usage = ev.Result.Usage + } + } + if stop != string(session.StopEndTurn) { + t.Fatalf("real provider run did not finish successfully (stop=%s)", stop) + } + if !calls["Write"] { + t.Fatal("real provider did not call a filesystem write tool") + } + if !calls["Shell"] { + t.Fatal("real provider did not call Shell") + } + if !modelShellPassed { + t.Fatal("real provider Shell result did not report exit code 0") + } + if usage == nil || usage.InputTokens <= 0 || usage.OutputTokens <= 0 { + t.Fatal("real provider returned no positive token usage") + } + t.Logf("live stage=model_coding reason=ok input_tokens=%d output_tokens=%d shell_exit_code=0", usage.InputTokens, usage.OutputTokens) + + t.Log("live stage=independent_typed_verification reason=begin") + providerForward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + stopProviderForward := cleanupOnce(providerForward.stop) + defer stopProviderForward() + client, err := executionclient.New(providerForward.addr, loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local")) + if err != nil { + t.Fatal("typed execution client setup failed") + } + closeClient := cleanupOnce(client.Close) + defer closeClient() + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "alice"} + lookup := environmentForBinding(t, ctx, kubeconfig, created.SessionId) + attached := waitReady(t, ctx, client, owner, created.SessionId, lookup) + if attached.Environment != lookup { + t.Fatal("typed execution reattachment returned a different environment") + } + rc, release := acquireRun(t, ctx, client, owner, created.SessionId, attached, fmt.Sprintf("live-verify-%d", time.Now().UnixNano())) + release = cleanupOnce(release) + defer release() + before := readQualificationArtifacts(ctx, t, client, rc, nonce) + installQualificationHelper(ctx, t, client, rc) + beforeCommand := runQualificationTests(ctx, t, client, rc) + t.Logf("live stage=independent_typed_verification reason=ok artifact_count=%d command_exit_code=%d", len(before), beforeCommand.Result.ExitCode) + + // No verifier claim or connection may survive either deployment restart. + release() + assertRunReleased(ctx, t, client, rc) + closeClient() + stopProviderForward() + stopAgentForward() + + t.Log("live stage=restart_verification reason=begin") + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=240s") + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecak8s", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecak8s", "-n", namespace, "--timeout=240s") + + freshTokenForward := portForward(t, ctx, kubeconfig, "service/oidc-issuer", 8443) + freshAlice := fixtureToken(t, ctx, freshTokenForward.addr, filepath.Join(state, "pki"), "alice") + freshTokenForward.stop() + freshAgentForward := portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + defer freshAgentForward.stop() + if status := getSession(t, ctx, freshAgentForward.addr, created.SessionId, freshAlice); status != http.StatusOK { + t.Fatalf("authenticated post-restart session load status=%d, want 200", status) + } + + postRestartLookup := environmentForBinding(t, ctx, kubeconfig, created.SessionId) + if postRestartLookup != lookup { + t.Fatal("session binding resolved a different environment after restart") + } + freshProviderForward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + defer freshProviderForward.stop() + freshClient, err := executionclient.New(freshProviderForward.addr, loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local")) + if err != nil { + t.Fatal("post-restart typed execution client setup failed") + } + defer freshClient.Close() + waitProviderRPCReady(t, ctx, freshClient, freshProviderForward.addr, loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local")) + postRestartAttached := waitReady(t, ctx, freshClient, owner, created.SessionId, postRestartLookup) + if postRestartAttached.Environment != lookup { + t.Fatal("typed execution reattachment changed the persisted environment after restart") + } + postRestartRC, releasePostRestart := acquireRun(t, ctx, freshClient, owner, created.SessionId, postRestartAttached, fmt.Sprintf("live-restart-verify-%d", time.Now().UnixNano())) + defer releasePostRestart() + after := readQualificationArtifacts(ctx, t, freshClient, postRestartRC, nonce) + if !qualificationArtifactsEqual(before, after) { + t.Fatal("qualification artifact bytes changed across restart") + } + assertQualificationHelper(ctx, t, freshClient, postRestartRC) + afterCommand := runQualificationTests(ctx, t, freshClient, postRestartRC) + t.Logf("live stage=restart_verification reason=ok artifact_count=%d before_exit_code=%d after_exit_code=%d", len(after), beforeCommand.Result.ExitCode, afterCommand.Result.ExitCode) + + observedTools := make([]string, 0, 4) + for _, name := range []string{"Write", "Read", "Edit", "Shell"} { + if calls[name] { + observedTools = append(observedTools, name) + } + } + cluster := os.Getenv("MECATL_EXECUTION_LIVE_CLUSTER") + if cluster == "" { + t.Fatal("verified qualification cluster identity is unavailable") + } + summary := liveSummary{ + Cluster: cluster, + Provider: created.ResolvedModel.ProviderId, + Model: created.ResolvedModel.ModelId, + InputTokens: usage.InputTokens, + OutputTokens: usage.OutputTokens, + ToolNames: observedTools, + ArtifactCount: len(after), + RestartVerified: true, + BeforeRestartExitCode: beforeCommand.Result.ExitCode, + AfterRestartExitCode: afterCommand.Result.ExitCode, + Timestamp: time.Now().UTC().Format(time.RFC3339), + } + blob, err := json.MarshalIndent(summary, "", " ") + if err != nil { + t.Fatal("marshal sanitized summary failed") + } + if err := os.WriteFile(filepath.Join(state, "live-summary.json"), append(blob, '\n'), 0o600); err != nil { + t.Fatal("write sanitized summary failed") + } +} + +const ( + qualificationHelperPath = "arithmetic/qualification_external_test.go" + qualificationHelperFile = `package arithmetic + +import "testing" + +func TestQualificationSemantics(t *testing.T) { + for _, tc := range []struct { + name string + a, b int + result int + }{ + {name: "positive", a: 2, b: 3, result: 5}, + {name: "negative", a: -2, b: 5, result: 3}, + {name: "zero", a: 0, b: 0, result: 0}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := Sum(tc.a, tc.b); got != tc.result { + t.Fatalf("Sum(%d, %d) = %d, want %d", tc.a, tc.b, got, tc.result) + } + }) + } +} +` +) + +var qualificationArtifactPaths = [...]string{ + "arithmetic/sum.go", + "arithmetic/sum_test.go", +} + +func cleanupOnce(cleanup func()) func() { + var once sync.Once + return func() { once.Do(cleanup) } +} + +func readQualificationArtifacts(ctx context.Context, t *testing.T, client *executionclient.Client, rc executionenv.RequestContext, nonce string) map[string][]byte { + t.Helper() + artifacts := make(map[string][]byte, len(qualificationArtifactPaths)) + for _, path := range qualificationArtifactPaths { + file, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: path}) + if err != nil || len(file.Data) == 0 { + t.Fatalf("typed gRPC artifact verification failed for %s", path) + } + artifacts[path] = bytes.Clone(file.Data) + } + if !containsExactLine(artifacts["arithmetic/sum.go"], "// qualification: "+nonce) { + t.Fatal("typed gRPC implementation artifact omitted the exact qualification nonce marker") + } + return artifacts +} + +func containsExactLine(content []byte, want string) bool { + for _, line := range bytes.Split(content, []byte("\n")) { + if string(bytes.TrimSpace(line)) == want { + return true + } + } + return false +} + +func installQualificationHelper(ctx context.Context, t *testing.T, client *executionclient.Client, rc executionenv.RequestContext) { + t.Helper() + if _, err := client.File(ctx, executionenv.FileRequest{ + Context: rc, + Operation: executionenv.OpFileCreate, + Path: qualificationHelperPath, + Data: []byte(qualificationHelperFile), + }); err != nil { + t.Fatal("create fixed qualification semantics test failed") + } + assertQualificationHelper(ctx, t, client, rc) +} + +func assertQualificationHelper(ctx context.Context, t *testing.T, client *executionclient.Client, rc executionenv.RequestContext) { + t.Helper() + file, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: qualificationHelperPath}) + if err != nil || !bytes.Equal(file.Data, []byte(qualificationHelperFile)) { + t.Fatal("fixed qualification semantics test changed or was unavailable") + } +} + +func runQualificationTests(ctx context.Context, t *testing.T, client *executionclient.Client, rc executionenv.RequestContext) executionenv.CommandStartResponse { + t.Helper() + command, err := client.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go test ./...", TimeoutMillis: 120000}) + if err != nil || command.State != executionenv.CommandSucceeded || command.Result.ExitCode != 0 { + t.Fatal("typed gRPC independent go test failed") + } + return command +} + +func assertRunReleased(ctx context.Context, t *testing.T, client *executionclient.Client, rc executionenv.RequestContext) { + t.Helper() + _, err := client.RenewRun(ctx, executionenv.RunClaimRequest{ + Environment: rc.Environment, + Owner: rc.Owner, + BindingID: rc.BindingID, + RunID: rc.RunID, + ClaimID: rc.ClaimID, + Epoch: rc.Epoch, + GrantGeneration: rc.GrantGeneration, + OperationID: "verify-released-" + rc.RunID, + TTL: time.Minute, + }) + if !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("released verifier claim remained renewable: code=%s", remoteErrorCode(err)) + } +} + +func qualificationArtifactsEqual(before, after map[string][]byte) bool { + if len(before) != len(qualificationArtifactPaths) || len(after) != len(qualificationArtifactPaths) { + return false + } + for _, path := range qualificationArtifactPaths { + if !bytes.Equal(before[path], after[path]) { + return false + } + } + return true +} + +func modelRanExactShell(events []*mecatlv1.Event, command string) bool { + matching := map[string]struct{}{} + for _, ev := range events { + if ev.ToolCall != nil && ev.ToolCall.Name == "Shell" { + var args map[string]json.RawMessage + if json.Unmarshal([]byte(ev.ToolCall.Args), &args) != nil { + continue + } + var requested string + if json.Unmarshal(args["command"], &requested) == nil && strings.TrimSpace(requested) == command { + matching[ev.ToolCall.Id] = struct{}{} + } + } + if ev.ToolResult != nil { + if _, ok := matching[ev.ToolResult.CallId]; ok && !ev.ToolResult.IsError && strings.Contains(ev.ToolResult.Content, "[exit code: 0]") { + return true + } + } + } + return false +} + +type liveSummary struct { + Cluster string `json:"cluster"` + Provider string `json:"provider"` + Model string `json:"model"` + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + ToolNames []string `json:"tool_names"` + ArtifactCount int `json:"artifact_count"` + RestartVerified bool `json:"restart_verified"` + BeforeRestartExitCode int `json:"before_restart_exit_code"` + AfterRestartExitCode int `json:"after_restart_exit_code"` + Timestamp string `json:"timestamp"` +} + +func assertCredentialScope(t *testing.T, ctx context.Context, kubeconfig, secretName string) { + t.Helper() + var agent appsv1.Deployment + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "deployment/mecak8s", "-n", namespace, "-o", "json"), &agent); err != nil { + t.Fatal("decode mecak8s credential-scope metadata failed") + } + assertRealProviderArgs(t, agent.Spec.Template.Spec) + want := secretReference{Container: "agent", Source: "env", Name: secretName, Key: liveCredentialKey, EnvName: liveCredentialKey} + refs := referencesNamed(secretReferences(agent.Spec.Template.Spec), secretName) + if len(refs) != 1 || refs[0] != want { + t.Fatal("run-scoped provider credential is not confined to the mecak8s agent environment") + } + + var provider appsv1.Deployment + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "deployment/mecatl-execution", "-n", namespace, "-o", "json"), &provider); err != nil { + t.Fatal("decode provider credential-scope metadata failed") + } + var executors corev1.PodList + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "execution.mecatl.dev/environment", "-o", "json"), &executors); err != nil { + t.Fatal("decode executor credential-scope metadata failed") + } + if containsSecretReference(secretReferences(provider.Spec.Template.Spec), secretName) { + t.Fatal("provider credential escaped into the execution provider") + } + for _, pod := range executors.Items { + if containsSecretReference(secretReferences(pod.Spec), secretName) { + t.Fatal("provider credential escaped into an executor pod") + } + } +} + +func assertRealProviderArgs(t *testing.T, spec corev1.PodSpec) { + t.Helper() + if err := realProviderConfigError(spec); err != nil { + t.Fatal(err) + } +} + +func realProviderConfigError(spec corev1.PodSpec) error { + for _, volume := range spec.Volumes { + if volume.Name == "execution-mock" || (volume.ConfigMap != nil && volume.ConfigMap.Name == "execution-mock") { + return errors.New("live deployment retained the execution-mock volume") + } + if volume.Projected != nil { + for _, source := range volume.Projected.Sources { + if source.ConfigMap != nil && source.ConfigMap.Name == "execution-mock" { + return errors.New("live deployment retained a projected execution-mock source") + } + } + } + } + for _, container := range spec.Containers { + for _, mount := range container.VolumeMounts { + if mount.Name == "execution-mock" { + return errors.New("live deployment retained the execution-mock mount") + } + } + if container.Name != "agent" { + continue + } + seen := map[string]bool{} + for _, arg := range container.Args { + seen[arg] = true + if arg == "--mock" || strings.HasPrefix(arg, "--mock-script=") { + return errors.New("live deployment retained mock-provider arguments") + } + if strings.Contains(arg, "base-url") { + return errors.New("live deployment overrides the provider endpoint") + } + } + for _, required := range []string{"--default-provider=openrouter", "--model=" + liveModel, "--max-run-tokens=32000"} { + if !seen[required] { + return fmt.Errorf("live deployment is missing required argument %s", required) + } + } + return nil + } + return errors.New("live deployment has no agent container") +} + +type secretReference struct { + Container string + Source string + Name string + Key string + EnvName string +} + +func secretReferences(spec corev1.PodSpec) []secretReference { + var refs []secretReference + collectContainer := func(container corev1.Container, kind string) { + for _, env := range container.Env { + if env.ValueFrom != nil && env.ValueFrom.SecretKeyRef != nil { + refs = append(refs, secretReference{Container: kind + container.Name, Source: "env", Name: env.ValueFrom.SecretKeyRef.Name, Key: env.ValueFrom.SecretKeyRef.Key, EnvName: env.Name}) + } + } + for _, envFrom := range container.EnvFrom { + if envFrom.SecretRef != nil { + refs = append(refs, secretReference{Container: kind + container.Name, Source: "envFrom", Name: envFrom.SecretRef.Name}) + } + } + } + for _, container := range spec.Containers { + collectContainer(container, "") + } + for _, container := range spec.InitContainers { + collectContainer(container, "init:") + } + for _, volume := range spec.Volumes { + if volume.Secret != nil { + refs = append(refs, secretReference{Source: "volume", Name: volume.Secret.SecretName}) + } + if volume.Projected != nil { + for _, source := range volume.Projected.Sources { + if source.Secret != nil { + refs = append(refs, secretReference{Source: "projected", Name: source.Secret.Name}) + } + } + } + } + for _, pullSecret := range spec.ImagePullSecrets { + refs = append(refs, secretReference{Source: "imagePullSecret", Name: pullSecret.Name}) + } + return refs +} + +func referencesNamed(refs []secretReference, name string) []secretReference { + matched := make([]secretReference, 0, 1) + for _, ref := range refs { + if ref.Name == name { + matched = append(matched, ref) + } + } + return matched +} + +func containsSecretReference(refs []secretReference, name string) bool { + return len(referencesNamed(refs, name)) != 0 +} + +type liveCreateResponse struct { + SessionId string `json:"session_id"` + ResolvedModel *mecatlv1.ResolvedModel `json:"resolved_model"` +} + +func createLiveSession(t *testing.T, ctx context.Context, addr, token string) liveCreateResponse { + t.Helper() + t.Log("live stage=http_create reason=begin") + ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ + WroteHeaders: func() { t.Log("live stage=http_create reason=headers_sent") }, + GotFirstResponseByte: func() { t.Log("live stage=http_create reason=response_started") }, + }) + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions", token, []byte(`{"mode":"default","limits":{"max_turns":8,"max_tool_calls":20,"max_consecutive_failures":3}}`)) + if status != http.StatusCreated { + t.Fatalf("live session creation failed (HTTP %d)", status) + } + var out liveCreateResponse + if json.Unmarshal(body, &out) != nil || out.SessionId == "" { + t.Fatal("live session response invalid") + } + return out +} + +func livePrompt(t *testing.T, ctx context.Context, addr, id, token, text string) []*mecatlv1.Event { + t.Helper() + raw, _ := json.Marshal(map[string]string{"text": text}) + req, _ := http.NewRequestWithContext(ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+id+"/prompt", strings.NewReader(string(raw))) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "text/event-stream") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal("live prompt transport failed") + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 2<<20)) + t.Fatalf("live prompt rejected (HTTP %d; body redacted)", resp.StatusCode) + } + var events []*mecatlv1.Event + s := bufio.NewScanner(io.LimitReader(resp.Body, 4<<20)) + s.Buffer(make([]byte, 64<<10), 1<<20) + for s.Scan() { + line := s.Text() + if !strings.HasPrefix(line, "data: ") { + continue + } + var ev mecatlv1.Event + if json.Unmarshal([]byte(strings.TrimPrefix(line, "data: ")), &ev) != nil { + t.Fatal("invalid live SSE event") + } + events = append(events, &ev) + } + if s.Err() != nil { + t.Fatal("live SSE stream failed") + } + return events +} diff --git a/e2e/k8s_execution/live_qualification_unit_test.go b/e2e/k8s_execution/live_qualification_unit_test.go new file mode 100644 index 0000000000..362057a1ff --- /dev/null +++ b/e2e/k8s_execution/live_qualification_unit_test.go @@ -0,0 +1,190 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "encoding/json" + "go/format" + "testing" + + corev1 "k8s.io/api/core/v1" + + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestReplacementProofRequestsUsePostClaimEpochForPositiveRequest(t *testing.T) { + ref := executionenv.EnvironmentRef{ID: "fixture", Revision: "revision"} + owner := executionenv.Owner{Issuer: "issuer", Subject: "subject"} + staleStatus := executionStatus{Epoch: 4, PodUID: "pod", PVCUID: "pvc"} + current := executionStatus{Epoch: 6, PodUID: "pod", PVCUID: "pvc"} + stale, wrong, exact := replacementProofRequests(ref, owner, staleStatus, current) + if stale.ExpectedEpoch != staleStatus.Epoch || stale.ExpectedPodUID != staleStatus.PodUID { + t.Fatal("stale negative request did not preserve the pre-claim authority") + } + if wrong.ExpectedEpoch != current.Epoch || wrong.ExpectedPodUID == current.PodUID { + t.Fatal("wrong-UID negative request did not use current authority") + } + if exact.ExpectedEpoch != current.Epoch || exact.ExpectedPodUID != current.PodUID || exact.ExpectedPVCUID != current.PVCUID { + t.Fatal("positive exact request did not use post-release authoritative state") + } +} + +func TestModelRanExactShellRequiresDecodedCommandAndMatchingCallID(t *testing.T) { + call := func(id, args string) *mecatlv1.Event { + return &mecatlv1.Event{ToolCall: &mecatlv1.ToolCall{Id: id, Name: "Shell", Args: args}} + } + result := func(id string) *mecatlv1.Event { + return &mecatlv1.Event{ToolResult: &mecatlv1.ToolResult{CallId: id, Content: "[exit code: 0]"}} + } + for name, events := range map[string][]*mecatlv1.Event{ + "echo": {call("shell", `{"command":"echo ok"}`), result("shell")}, + "invalid JSON": {call("shell", `{"command":`), result("shell")}, + "wrong result ID": {call("shell", `{"command":"go test ./..."}`), result("other")}, + "unrelated success": {call("echo", `{"command":"echo ok"}`), result("echo"), call("test", `{"command":"go test ./..."}`)}, + } { + t.Run(name, func(t *testing.T) { + if modelRanExactShell(events, "go test ./...") { + t.Fatal("unqualified Shell call counted") + } + }) + } + if !modelRanExactShell([]*mecatlv1.Event{call("test", `{"command":" go test ./... "}`), result("test")}, "go test ./...") { + t.Fatal("matching successful Shell call was not counted") + } +} + +func TestLiveRestartProofHelpersAcceptExactArtifactsAndCleanupOnce(t *testing.T) { + before := map[string][]byte{ + "arithmetic/sum.go": []byte("package arithmetic\n// qualification: qual_fixture\nfunc Sum(a, b int) int { return a + b }\n"), + "arithmetic/sum_test.go": []byte("package arithmetic\nfunc ExampleSum() { _ = Sum(1, 2) }\n"), + } + after := map[string][]byte{ + "arithmetic/sum.go": append([]byte(nil), before["arithmetic/sum.go"]...), + "arithmetic/sum_test.go": append([]byte(nil), before["arithmetic/sum_test.go"]...), + } + if !qualificationArtifactsEqual(before, after) { + t.Fatal("byte-identical fixed qualification artifacts were rejected") + } + if !containsExactLine(before["arithmetic/sum.go"], "// qualification: qual_fixture") { + t.Fatal("exact nonce marker line was rejected") + } + + calls := 0 + cleanup := cleanupOnce(func() { calls++ }) + cleanup() + cleanup() + if calls != 1 { + t.Fatalf("cleanup calls=%d, want 1", calls) + } +} + +func TestQualificationHelperIsValidCanonicalGo(t *testing.T) { + formatted, err := format.Source([]byte(qualificationHelperFile)) + if err != nil { + t.Fatalf("fixed qualification helper is invalid Go: %v", err) + } + if string(formatted) != qualificationHelperFile { + t.Fatal("fixed qualification helper is not gofmt-canonical") + } +} + +func TestLiveSummaryIncludesFixedRestartEvidence(t *testing.T) { + blob, err := json.Marshal(liveSummary{ + Cluster: "fixture", + Provider: "openrouter", + Model: liveModel, + InputTokens: 1, + OutputTokens: 2, + ToolNames: []string{"Write", "Shell"}, + ArtifactCount: 2, + RestartVerified: true, + BeforeRestartExitCode: 0, + AfterRestartExitCode: 0, + Timestamp: "2026-09-23T00:00:00Z", + }) + if err != nil { + t.Fatal(err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(blob, &fields); err != nil { + t.Fatal(err) + } + for _, name := range []string{"restart_verified", "before_restart_exit_code", "after_restart_exit_code"} { + if _, ok := fields[name]; !ok { + t.Fatalf("summary omitted %s", name) + } + } + if len(fields) != 11 { + t.Fatalf("summary field count=%d, want fixed allowlist of 11", len(fields)) + } + var verified bool + if err := json.Unmarshal(fields["restart_verified"], &verified); err != nil || !verified { + t.Fatal("summary did not preserve positive restart verdict") + } +} + +func TestRealProviderArgsRejectExecutionMockSources(t *testing.T) { + tests := []corev1.PodSpec{ + {Volumes: []corev1.Volume{{Name: "execution-mock"}}}, + {Volumes: []corev1.Volume{{Name: "other", VolumeSource: corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "execution-mock"}}}}}}, + { + Volumes: []corev1.Volume{{ + Name: "other", + VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{Sources: []corev1.VolumeProjection{{ + ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "execution-mock"}}, + }}}}, + }}, + }, + {Containers: []corev1.Container{{Name: "sidecar", VolumeMounts: []corev1.VolumeMount{{Name: "execution-mock"}}}}}, + } + for i, spec := range tests { + if err := realProviderConfigError(spec); err == nil { + t.Fatalf("case %d did not reject execution-mock source", i) + } + } +} + +func TestSecretReferencesFindEveryCredentialEscape(t *testing.T) { + const sentinel = "synthetic-live-secret-sentinel" + tests := map[string]corev1.PodSpec{ + "sidecar env": { + Containers: []corev1.Container{{Name: "sidecar", Env: []corev1.EnvVar{{Name: "TOKEN", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}, Key: "token"}}}}}}, + }, + "sidecar envFrom": { + Containers: []corev1.Container{{Name: "sidecar", EnvFrom: []corev1.EnvFromSource{{SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}}}}}}, + }, + "init container env": { + InitContainers: []corev1.Container{{Name: "setup", Env: []corev1.EnvVar{{Name: "TOKEN", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}, Key: "token"}}}}}}, + }, + "secret volume": { + Volumes: []corev1.Volume{{Name: "credentials", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: sentinel}}}}, + }, + "projected secret volume": { + Volumes: []corev1.Volume{{Name: "credentials", VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{Sources: []corev1.VolumeProjection{{Secret: &corev1.SecretProjection{LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}}}}}}}}, + }, + } + for name, spec := range tests { + t.Run(name, func(t *testing.T) { + if !containsSecretReference(secretReferences(spec), sentinel) { + t.Fatal("credential escape was not detected") + } + }) + } +} + +func TestExpectedHarnessCredentialReferenceIsExact(t *testing.T) { + const sentinel = "synthetic-live-secret-sentinel" + spec := corev1.PodSpec{Containers: []corev1.Container{{ + Name: "agent", + Env: []corev1.EnvVar{{Name: liveCredentialKey, ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: sentinel}, + Key: liveCredentialKey, + }}}}, + }}} + want := secretReference{Container: "agent", Source: "env", Name: sentinel, Key: liveCredentialKey, EnvName: liveCredentialKey} + refs := secretReferences(spec) + if len(refs) != 1 || refs[0] != want { + t.Fatal("expected harness credential reference was not projected exactly") + } +} diff --git a/e2e/k8s_execution/pr1614_runtime_blockers_test.go b/e2e/k8s_execution/pr1614_runtime_blockers_test.go new file mode 100644 index 0000000000..07ebb1ae46 --- /dev/null +++ b/e2e/k8s_execution/pr1614_runtime_blockers_test.go @@ -0,0 +1,259 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/adapter/permpolicy" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/adapter/mcp" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestKindExecutionRevocationStopsActiveWriterAndRetainsFenceUncertainty(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + client, _ := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, client, "revoke-active-writer") + rc, _ := acquireRun(t, ctx, client, owner, binding, attached, "revoke-active-writer") + initial := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + + done := make(chan error, 1) + go func() { + _, err := client.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: `i=0; while [ $i -lt 120 ]; do i=$((i+1)); printf '%s\n' "$i" > revoke-counter; sleep 1; done`, TimeoutMillis: 150000}) + done <- err + }() + waitActiveOperationOrCommandError(t, ctx, kubeconfig, attached.Environment.ID, done) + if _, err := client.RevokeEnvironment(ctx, attached.Environment, owner, rc.GrantGeneration, "revoke-active-writer"); err != nil { + t.Fatal(err) + } + select { + case err := <-done: + if err == nil { + t.Fatal("revoked active writer returned clean completion") + } + case <-time.After(25 * time.Second): + t.Fatal("revoked active writer did not stop promptly") + } + fenced := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { + return s.FenceState == "FenceUnknown" && s.ActiveOperationID != "" + }) + if _, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "post-revoke-writer", Data: []byte("forbidden")}); err == nil { + t.Fatal("revoked grant admitted a new writer") + } + pod := kubeValue(t, ctx, kubeconfig, "get", "pod", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[0].metadata.name}") + first := strings.TrimSpace(string(runKubectl(t, ctx, kubeconfig, "exec", "-n", namespace, pod, "--", "cat", "/workspace/revoke-counter"))) + time.Sleep(2 * time.Second) + second := strings.TrimSpace(string(runKubectl(t, ctx, kubeconfig, "exec", "-n", namespace, pod, "--", "cat", "/workspace/revoke-counter"))) + if first == "" || first != second { + t.Fatalf("workspace kept changing after RPC stopped: first=%q second=%q", first, second) + } + + contextName := os.Getenv("MECATL_KUBE_CONTEXT") + if contextName == "" { + t.Fatal("explicit fixture Kubernetes context required") + } + config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, &clientcmd.ConfigOverrides{CurrentContext: contextName}).ClientConfig() + if err != nil { + t.Fatal(err) + } + config.Timeout = 15 * time.Second + kube, err := kubernetes.NewForConfig(config) + if err != nil { + t.Fatal(err) + } + envUID := kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") + if err := terminateOwnedExecutor(ctx, kube, attached.Environment.ID, envUID, initial.PodUID); err != nil { + t.Fatal(err) + } + waitExecutorTerminal(ctx, t, kubeconfig, attached.Environment.ID, initial.PodUID) + if err := client.RecoverEnvironment(ctx, executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: initial.Epoch, ExpectedPodUID: initial.PodUID, ExpectedPVCUID: fenced.PVCUID, OperationID: "recover-revoke-active-writer"}); err != nil { + t.Fatal(err) + } + retireSyntheticEnvironment(t, ctx, client, kubeconfig, owner, binding, attached.Environment) +} + +func TestKindExecutionExpiredRunAllowsFinalReferenceRetireAndDelete(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + client, _ := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, client, "expired-final-reference") + claim, err := client.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "expired-final-reference", OperationID: "acquire-expired-final-reference", TTL: executionenv.MinRunTTL}) + if err != nil { + t.Fatal(err) + } + rc := executionenv.RequestContext{Environment: claim.Environment, Owner: owner, BindingID: binding, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant} + status := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + commandDone := make(chan error, 1) + go func() { + _, commandErr := client.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "sleep 3", TimeoutMillis: 10000}) + commandDone <- commandErr + }() + waitActiveOperationOrCommandError(t, ctx, kubeconfig, attached.Environment.ID, commandDone) + deleteRef := executionenv.ReferenceRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, OperationID: "delete-expired-final-reference"} + if err := client.PrepareReferenceDelete(ctx, deleteRef); err != nil { + t.Fatal(err) + } + if err := client.ConfirmReferenceDelete(ctx, deleteRef); err != nil { + t.Fatal(err) + } + retire := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: status.Epoch, ExpectedPodUID: status.PodUID, ExpectedPVCUID: status.PVCUID, OperationID: "retire-expired-final-reference"} + if err := client.RetireEnvironment(ctx, retire); err == nil { + t.Fatal("retirement admitted active work") + } + if err := <-commandDone; err != nil { + t.Fatal(err) + } + if err := client.RetireEnvironment(ctx, retire); err == nil { + t.Fatal("retirement admitted an unexpired run") + } + wait := time.Until(claim.ExpiresAt) + time.Second + if wait > 0 { + timer := time.NewTimer(wait) + select { + case <-timer.C: + case <-ctx.Done(): + timer.Stop() + t.Fatal(ctx.Err()) + } + } + if err := client.RetireEnvironment(ctx, retire); err != nil { + t.Fatalf("expired exact run blocked retirement: %v", err) + } + retired := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Retired && s.ExecutorTerminated }) + if err := client.DeleteRetiredEnvironment(ctx, attached.Environment, owner, retired.PVCUID, "delete-retained-expired-final-reference"); err != nil { + t.Fatal(err) + } + waitResourceAbsent(t, ctx, kubeconfig, "executionenvironment", attached.Environment.ID) +} + +type kindInitialCreateBarrierStore struct { + *memstore.Store + target session.SessionID + entered int + mu sync.Mutex + release chan struct{} +} + +func (s *kindInitialCreateBarrierStore) Load(ctx context.Context, id session.SessionID) (*session.Session, error) { + if id == s.target { + s.mu.Lock() + if s.entered < 2 { + s.entered++ + if s.entered == 2 { + close(s.release) + } + gate := s.release + s.mu.Unlock() + select { + case <-gate: + case <-ctx.Done(): + return nil, ctx.Err() + } + } else { + s.mu.Unlock() + } + } + return s.Store.Load(ctx, id) +} + +func TestKindExecutionServiceExplicitIDRetriesReuseOneAllocation(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + client, _ := productionClient(t, ctx, state, kubeconfig) + provider, err := executionclient.NewProvider(client, "go") + if err != nil { + t.Fatal(err) + } + principal := &session.Principal{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: fmt.Sprintf("service-explicit-%d", time.Now().UnixNano()), GrantType: session.GrantTypeUser} + principalCtx := session.WithPrincipal(ctx, principal) + id := session.SessionID(fmt.Sprintf("sched--kind-explicit-%d", time.Now().UnixNano())) + store := &kindInitialCreateBarrierStore{Store: memstore.New(), target: id, release: make(chan struct{})} + newService := func() *server.Service { + engine := agent.NewEngine(agent.Deps{LLM: mockllm.New(mockllm.TextTurn("ok")), Catalog: tool.NewCatalog(), Policy: permpolicy.NewPolicy(nil, nil), Model: "test"}) + svc, newErr := server.NewService(server.Config{Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "kind-explicit-id", OwnershipEnforced: true, SessionEngine: func(context.Context, server.ProviderSelector, []mcp.ServerConfig, server.SessionProfile, string, session.PermissionMode) (server.SessionEngineResult, error) { + return server.SessionEngineResult{Engine: engine}, nil + }}) + if newErr != nil { + t.Fatal(newErr) + } + return svc + } + type createResult struct { + created *session.Session + err error + } + createConcurrently := func(services []*server.Service) []*session.Session { + t.Helper() + out := make(chan createResult, len(services)) + for _, svc := range services { + go func() { + created, createErr := svc.CreateSessionWithProfile(principalCtx, session.ModeDefault, session.Limits{}, server.ProviderSelector{}, server.ProfileDefault, server.WithSessionID(id)) + out <- createResult{created: created, err: createErr} + }() + } + results := make([]*session.Session, 0, len(services)) + for range services { + select { + case result := <-out: + if result.err != nil || result.created == nil { + t.Fatalf("concurrent create=(%+v,%v), want success", result.created, result.err) + } + results = append(results, result.created) + case <-ctx.Done(): + t.Fatalf("concurrent create did not finish: %v", ctx.Err()) + } + } + return results + } + initialServices := []*server.Service{newService(), newService()} + initial := createConcurrently(initialServices) + for _, svc := range initialServices { + svc.Close() + } + first := initial[0] + if initial[1].EnvironmentRef != first.EnvironmentRef { + t.Fatalf("initial creates diverged: first=%+v second=%+v", first.EnvironmentRef, initial[1].EnvironmentRef) + } + services := []*server.Service{newService(), newService()} + defer services[0].Close() + defer services[1].Close() + results := createConcurrently(services) + for i := range results { + if results[i].EnvironmentRef != first.EnvironmentRef { + t.Fatalf("retry %d=%+v, want ref %+v", i, results[i].EnvironmentRef, first.EnvironmentRef) + } + } + other := session.WithPrincipal(ctx, &session.Principal{Issuer: principal.Issuer, Subject: principal.Subject + "-other", GrantType: session.GrantTypeUser}) + if _, err := services[0].CreateSessionWithProfile(other, session.ModeDefault, session.Limits{}, server.ProviderSelector{}, server.ProfileDefault, server.WithSessionID(id)); err == nil { + t.Fatal("changed owner adopted explicit-ID allocation") + } + if _, err := services[0].CreateSessionWithProfile(principalCtx, session.ModePlan, session.Limits{}, server.ProviderSelector{}, server.ProfileDefault, server.WithSessionID(id)); !errors.Is(err, server.ErrInvalidArgument) { + t.Fatalf("changed mode error=%v", err) + } + if _, err := services[0].CreateSessionWithProfile(principalCtx, session.ModeDefault, session.Limits{MaxTurns: 1}, server.ProviderSelector{}, server.ProfileDefault, server.WithSessionID(id)); !errors.Is(err, server.ErrInvalidArgument) { + t.Fatalf("changed limits error=%v", err) + } + if _, err := services[0].CreateSessionWithProfile(principalCtx, session.ModeDefault, session.Limits{}, server.ProviderSelector{}, server.ProfileNoFS, server.WithSessionID(id)); !errors.Is(err, server.ErrInvalidArgument) { + t.Fatalf("changed profile error=%v", err) + } + owner := executionenv.Owner{Issuer: principal.Issuer, Subject: principal.Subject} + retireSyntheticEnvironment(t, ctx, client, kubeconfig, owner, string(id), executionenv.EnvironmentRef{ID: first.EnvironmentRef.ID, Revision: first.EnvironmentRef.Revision}) +} diff --git a/e2e/k8s_execution/production_qualification_test.go b/e2e/k8s_execution/production_qualification_test.go new file mode 100644 index 0000000000..86f0e51ad8 --- /dev/null +++ b/e2e/k8s_execution/production_qualification_test.go @@ -0,0 +1,1504 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func requireProduction(t *testing.T) (string, string, context.Context, context.CancelFunc) { + t.Helper() + if os.Getenv("MECATL_EXECUTION_QUAL_PROFILE") != "production" { + t.Skip("production qualification profile only") + } + state := os.Getenv("MECATL_EXECUTION_QUAL_STATE") + if state == "" { + t.Fatal("MECATL_EXECUTION_QUAL_STATE is required") + } + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + return state, filepath.Join(state, "kubeconfig"), ctx, cancel +} + +func productionClient(t *testing.T, ctx context.Context, state, kubeconfig string) (*executionclient.Client, *forward) { + t.Helper() + f := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + tlsConfig := loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") + client, err := executionclient.New(f.addr, tlsConfig) + if err != nil { + t.Fatal(err) + } + waitProviderRPCReady(t, ctx, client, f.addr, tlsConfig) + t.Cleanup(func() { client.Close() }) + return client, f +} + +func productionReplicaClients(t *testing.T, ctx context.Context, state, kubeconfig string) ([2]*executionclient.Client, [2]string) { + t.Helper() + var pods corev1.PodList + raw := runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "app.kubernetes.io/name=mecatl-execution", "-o", "json") + if err := json.Unmarshal(raw, &pods); err != nil || len(pods.Items) != 2 { + t.Fatalf("decode two provider replicas: pods=%d err=%v", len(pods.Items), err) + } + if pods.Items[0].UID == "" || pods.Items[1].UID == "" || pods.Items[0].UID == pods.Items[1].UID { + t.Fatal("provider replica Pod UIDs are missing or identical") + } + var clients [2]*executionclient.Client + var podUIDs [2]string + for i := range clients { + forward := portForward(t, ctx, kubeconfig, "pod/"+pods.Items[i].Name, 8443) + client, err := executionclient.New(forward.addr, loadTLS(t, filepath.Join(state, "pki"), "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local")) + if err != nil { + t.Fatal(err) + } + clients[i] = client + podUIDs[i] = string(pods.Items[i].UID) + t.Cleanup(func() { client.Close() }) + } + return clients, podUIDs +} + +func createProductionEnvironment(t *testing.T, ctx context.Context, c *executionclient.Client, suffix string) (executionenv.Owner, string, executionenv.AttachEnvironmentResponse) { + t.Helper() + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "production-" + suffix} + binding := fmt.Sprintf("production-%s-%d", suffix, time.Now().UnixNano()) + op := "ensure-" + binding + ensured, err := c.Ensure(ctx, binding, "go", owner, op) + if err != nil { + t.Fatalf("ensure production environment: %v", err) + } + if err := c.CommitReference(ctx, executionenv.ReferenceRequest{Environment: ensured.Environment, Owner: owner, BindingID: binding, OperationID: op}); err != nil { + t.Fatalf("commit production reference: %v", err) + } + return owner, binding, waitReady(t, ctx, c, owner, binding, ensured.Environment) +} + +func TestKindExecutionProductionNetworkPolicyEnforced(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + client, _ := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, client, "network") + rc, release := acquireRun(t, ctx, client, owner, binding, attached, fmt.Sprintf("network-%d", time.Now().UnixNano())) + defer release() + + assertProductionExecutorPod(t, ctx, kubeconfig, attached.Environment.ID) + probeSource := []byte("package main\nimport (\"net\";\"os\";\"time\")\nfunc main(){c,e:=net.DialTimeout(\"tcp\",os.Args[1],2*time.Second);if e!=nil{os.Exit(42)};c.Close()}\n") + if _, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "network_probe.go", Data: probeSource}); err != nil { + t.Fatalf("install network probe in owned workspace: %v", err) + } + built, err := client.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go build -o network-probe network_probe.go", TimeoutMillis: 30000}) + if err != nil || built.State != executionenv.CommandSucceeded || built.Result.ExitCode != 0 { + t.Fatalf("build network probe: state=%q exit=%d err=%v", built.State, built.Result.ExitCode, err) + } + fixtureIP := kubeValue(t, ctx, kubeconfig, "get", "pod/network-fixture", "-n", namespace, "-o", "jsonpath={.status.podIP}") + if out := runOwnedProbe(t, ctx, client, rc, fixtureIP+":8080"); out.State != executionenv.CommandSucceeded || out.Result.ExitCode != 0 { + t.Fatal("profile-specific package endpoint was not reachable") + } + providerIP := kubeValue(t, ctx, kubeconfig, "get", "service/mecatl-execution", "-n", namespace, "-o", "jsonpath={.spec.clusterIP}") + peerIP := kubeValue(t, ctx, kubeconfig, "get", "pod/network-intruder", "-n", namespace, "-o", "jsonpath={.status.podIP}") + for name, endpoint := range map[string]string{ + "provider": providerIP + ":8443", "api": "10.96.0.1:443", "peer": peerIP + ":8080", + "dns": "10.96.0.10:53", "metadata": "169.254.169.254:80", "internet": "1.1.1.1:443", + } { + out := runOwnedProbe(t, ctx, client, rc, endpoint) + if out.State != executionenv.CommandFailed || out.Result.State != executionenv.CommandFailed || out.Result.ExitCode != 42 { + t.Fatalf("default-denied executor probe %s state=%q result_state=%q exit=%d, want failed/failed/42", name, out.State, out.Result.State, out.Result.ExitCode) + } + } + + out, err := command(ctx, kubeconfig, "exec", "-n", namespace, "pod/network-intruder", "--", "/ko-app/netprobe", "-target", providerIP+":8443", "-timeout", "2s").CombinedOutput() + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 42 { + t.Fatalf("unrelated pod denial exit proof=%q err=%v, want 42", strings.TrimSpace(string(out)), err) + } + if got := resourceCount(t, ctx, kubeconfig, "pods -l app.kubernetes.io/name=mecatl-execution"); got != 2 { + t.Fatalf("ready provider replicas=%d, want 2", got) + } +} + +func runOwnedProbe(t *testing.T, ctx context.Context, c *executionclient.Client, rc executionenv.RequestContext, endpoint string) executionenv.CommandStartResponse { + t.Helper() + out, err := c.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "./network-probe " + endpoint, TimeoutMillis: 15000}) + if err != nil { + t.Fatalf("network probe RPC failed: %v", err) + } + return out +} + +func assertProductionExecutorPod(t *testing.T, ctx context.Context, kubeconfig, environmentID string) { + t.Helper() + var pods corev1.PodList + raw := runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "execution.mecatl.dev/environment="+environmentID, "-o", "json") + if err := json.Unmarshal(raw, &pods); err != nil || len(pods.Items) != 1 { + t.Fatal("decode exact executor pod") + } + pod := pods.Items[0] + if pod.Spec.RuntimeClassName == nil || *pod.Spec.RuntimeClassName != "qualification-runc" { + t.Fatal("executor does not use the qualified RuntimeClass") + } + if pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { + t.Fatal("executor received a service-account token") + } + if len(pod.Spec.Containers) != 1 { + t.Fatal("executor container shape drifted") + } + c := pod.Spec.Containers[0] + if c.SecurityContext == nil || c.SecurityContext.AllowPrivilegeEscalation == nil || *c.SecurityContext.AllowPrivilegeEscalation || c.SecurityContext.Capabilities == nil || len(c.SecurityContext.Capabilities.Drop) != 1 || c.SecurityContext.Capabilities.Drop[0] != "ALL" { + t.Fatal("executor capability confinement drifted") + } + for _, name := range []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory, corev1.ResourceEphemeralStorage} { + request, requestOK := c.Resources.Requests[name] + limit, limitOK := c.Resources.Limits[name] + if !requestOK || !limitOK || request.IsZero() || limit.IsZero() { + t.Fatalf("executor resource %s is unbounded", name) + } + } + for _, volume := range pod.Spec.Volumes { + if volume.Name == "tmp" && volume.EmptyDir != nil && volume.EmptyDir.SizeLimit != nil && !volume.EmptyDir.SizeLimit.IsZero() { + return + } + } + t.Fatal("executor /tmp is unbounded") +} + +func TestKindExecutionProductionSecurityRotation(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + oldClient, forward := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, oldClient, "rotation") + oldClaim, err := oldClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-old-grant", OperationID: "acquire-rotation-old-grant", TTL: time.Minute}) + if err != nil { + t.Fatal(err) + } + oldRun := executionenv.RequestContext{Environment: oldClaim.Environment, Owner: owner, BindingID: binding, RunID: oldClaim.RunID, ClaimID: oldClaim.ClaimID, Epoch: oldClaim.Epoch, GrantGeneration: oldClaim.GrantGeneration, Grant: oldClaim.Grant} + if _, err := oldClient.File(ctx, executionenv.FileRequest{Context: oldRun, Operation: executionenv.OpFileCreate, Path: "rotation-sentinel.txt", Data: []byte("old-authority\n")}); err != nil { + t.Fatal(err) + } + + rotationDir := filepath.Join(state, "rotation") + generator := exec.CommandContext(ctx, "go", "run", "-tags", "kind_execution_e2e", "./e2e/k8s_execution/fixture/rotation", filepath.Join(state, "pki"), rotationDir) + generator.Dir = repoRoot(t) + generator.Env = cleanEnv() + if out, err := generator.CombinedOutput(); err != nil { + t.Fatalf("generate synthetic rotation material: %v: %s", err, out) + } + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cleanupCancel() + restoreFixtureSecurity(t, cleanupCtx, kubeconfig, rotationDir) + }) + + // A changed policy at the already-authoritative generation is rejected by + // both replicas. The old snapshot cannot continue authorizing RPCs while the + // mounted candidate disagrees with the durable authority ledger. + applySecurityCandidate(t, ctx, kubeconfig, filepath.Join(state, "pki"), filepath.Join(rotationDir, "invalid-same-generation.json"), "initial") + waitProviderReadyReplicas(t, ctx, kubeconfig, 0) + if _, err := oldClient.File(ctx, executionenv.FileRequest{Context: oldRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); err == nil { + t.Fatal("same-generation changed policy continued authorizing an existing gRPC connection") + } + applySecurityCandidate(t, ctx, kubeconfig, filepath.Join(state, "pki"), filepath.Join(state, "pki", "manifest.json"), "initial") + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + recovered, err := oldClient.RenewRun(ctx, executionenv.RunClaimRequest{Environment: oldRun.Environment, Owner: owner, BindingID: binding, RunID: oldRun.RunID, ClaimID: oldRun.ClaimID, Epoch: oldRun.Epoch, GrantGeneration: oldRun.GrantGeneration, OperationID: "renew-rotation-recovered-grant", TTL: time.Minute}) + if isRemoteCode(err, executionenv.CodeConflict) { + recovered, err = oldClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-recovered-grant", OperationID: "acquire-rotation-recovered-grant", TTL: time.Minute}) + } + if err != nil { + t.Fatalf("establish fresh claim after authority recovery: code=%s", remoteErrorCode(err)) + } + recoveredRun := executionenv.RequestContext{Environment: recovered.Environment, Owner: owner, BindingID: binding, RunID: recovered.RunID, ClaimID: recovered.ClaimID, Epoch: recovered.Epoch, GrantGeneration: recovered.GrantGeneration, Grant: recovered.Grant} + waitFileContent(t, ctx, oldClient, recoveredRun, "rotation-sentinel.txt", "old-authority\n", "valid authority recovery") + if err := oldClient.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: recovered.Environment, Owner: owner, BindingID: binding, RunID: recovered.RunID, ClaimID: recovered.ClaimID, Epoch: recovered.Epoch, GrantGeneration: recovered.GrantGeneration, OperationID: "rotation-release-recovered"}); err != nil { + t.Fatalf("release recovery-phase claim: code=%s", remoteErrorCode(err)) + } + + // Generation 2 bridges client trust before changing the server certificate. + // The old connection remains usable, while a new-CA client can establish its + // own connection and receives grants from k2. + applySecurityCandidate(t, ctx, kubeconfig, rotationDir, filepath.Join(rotationDir, "bridge.json"), "bridge") + waitSecurityGeneration(ctx, t, kubeconfig, 2) + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + newTLS, err := executionclient.LoadTLSConfig(executionclient.TLSFiles{CA: filepath.Join(rotationDir, "client-roots.pem"), Cert: filepath.Join(rotationDir, "mecak8s-new.crt"), Key: filepath.Join(rotationDir, "mecak8s-new.key")}) + if err != nil { + t.Fatal(err) + } + newTLS.ServerName = "mecatl-execution.execution-qualification.svc.cluster.local" + newClient, err := executionclient.New(forward.addr, newTLS) + if err != nil { + t.Fatal(err) + } + defer newClient.Close() + newAttached := waitReady(t, ctx, newClient, owner, binding, attached.Environment) + if _, err := newClient.RenewRun(ctx, executionenv.RunClaimRequest{Environment: recoveredRun.Environment, Owner: owner, BindingID: binding, RunID: recoveredRun.RunID, ClaimID: recoveredRun.ClaimID, Epoch: recoveredRun.Epoch, GrantGeneration: recoveredRun.GrantGeneration, OperationID: "rotation-renew-released-phase-claim", TTL: time.Minute}); !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("released recovery-phase claim renewal code=%s, want conflict", remoteErrorCode(err)) + } + renewed, err := newClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: newAttached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-k2-grant", OperationID: "rotation-acquire-k2", TTL: time.Minute}) + if err != nil { + t.Fatalf("acquire bridge-authority claim: code=%s", remoteErrorCode(err)) + } + newRun := executionenv.RequestContext{Environment: renewed.Environment, Owner: owner, BindingID: binding, RunID: renewed.RunID, ClaimID: renewed.ClaimID, Epoch: renewed.Epoch, GrantGeneration: renewed.GrantGeneration, Grant: renewed.Grant} + waitFileContent(t, ctx, newClient, newRun, "rotation-sentinel.txt", "old-authority\n", "bridge authority") + if err := newClient.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: renewed.Environment, Owner: owner, BindingID: binding, RunID: renewed.RunID, ClaimID: renewed.ClaimID, Epoch: renewed.Epoch, GrantGeneration: renewed.GrantGeneration, OperationID: "rotation-release-k2-bridge"}); err != nil { + t.Fatalf("release bridge-phase claim: code=%s", remoteErrorCode(err)) + } + + // Generation 3 switches server TLS, removes the old client CA, and revokes + // k1. Authorization is checked on every RPC, so the established old-client + // HTTP/2 connection is denied rather than passing until reconnect. + applySecurityCandidate(t, ctx, kubeconfig, rotationDir, filepath.Join(rotationDir, "final.json"), "final") + waitSecurityGeneration(ctx, t, kubeconfig, 3) + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + finalAttached := waitReady(t, ctx, newClient, owner, binding, attached.Environment) + finalRequest := executionenv.RunClaimRequest{Environment: finalAttached.Environment, Owner: owner, BindingID: binding, RunID: "rotation-final-grant", OperationID: "rotation-acquire-final", TTL: time.Minute} + finalClaim, err := acquireCurrentAuthorityRun(ctx, finalRequest, newClient.AcquireRun, 500*time.Millisecond) + if err != nil { + var remote *executionenv.Error + t.Fatalf("acquire final-authority claim: errorCode=%s retryable=%t", remoteErrorCode(err), errors.As(err, &remote) && remote.Retryable) + } + finalRun := executionenv.RequestContext{Environment: finalClaim.Environment, Owner: owner, BindingID: binding, RunID: finalClaim.RunID, ClaimID: finalClaim.ClaimID, Epoch: finalClaim.Epoch, GrantGeneration: finalClaim.GrantGeneration, Grant: finalClaim.Grant} + waitFileContent(t, ctx, newClient, finalRun, "rotation-sentinel.txt", "old-authority\n", "final authority") + // Both fixture certificates attest the same URI. Only the CA differs; the + // current claim below has just succeeded and has not been released. + if _, err := oldClient.File(ctx, executionenv.FileRequest{Context: finalRun, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); !isRemoteCode(err, executionenv.CodeUnauthenticated) { + var remote *executionenv.Error + t.Fatalf("removed client CA rejection: errorCode=%s retryable=%t", remoteErrorCode(err), errors.As(err, &remote) && remote.Retryable) + } + assertRetiredFixtureKeyDenied(ctx, t, newClient, state, finalRun, "rotation-sentinel.txt", "old-authority\n") + freshClient, err := executionclient.New(forward.addr, newTLS) + if err != nil { + t.Fatal(err) + } + defer freshClient.Close() + if _, err := freshClient.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); err != nil { + t.Fatalf("fresh new-CA client failed after TLS switch: %v", err) + } + + if err := newClient.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: finalClaim.Environment, Owner: owner, BindingID: binding, RunID: finalClaim.RunID, ClaimID: finalClaim.ClaimID, Epoch: finalClaim.Epoch, GrantGeneration: finalClaim.GrantGeneration, OperationID: "rotation-release-final"}); err != nil { + t.Fatal(err) + } + generation, err := newClient.RevokeEnvironment(ctx, attached.Environment, owner, newAttached.GrantGeneration, "rotation-revoke-replay") + if err != nil { + t.Fatal(err) + } + replayed, err := freshClient.RevokeEnvironment(ctx, attached.Environment, owner, newAttached.GrantGeneration, "rotation-revoke-replay") + if err != nil || replayed != generation { + t.Fatalf("durable revoke receipt replay generation=%d want=%d err=%v", replayed, generation, err) + } + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=240s") + postRestartForward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + postRestartClient, err := executionclient.New(postRestartForward.addr, newTLS) + if err != nil { + t.Fatal(err) + } + defer postRestartClient.Close() + postRestartAttached := waitReady(t, ctx, postRestartClient, owner, binding, attached.Environment) + if postRestartAttached.GrantGeneration != generation || generation != newAttached.GrantGeneration+1 { + t.Fatalf("persisted revocation generation=%d want=%d", postRestartAttached.GrantGeneration, generation) + } + replayed, err = postRestartClient.RevokeEnvironment(ctx, attached.Environment, owner, newAttached.GrantGeneration, "rotation-revoke-replay") + if err != nil || replayed != generation { + t.Fatalf("post-restart revoke receipt generation=%d want=%d code=%s", replayed, generation, remoteErrorCode(err)) + } + currentRun, releaseCurrent := acquireRun(t, ctx, postRestartClient, owner, binding, postRestartAttached, "rotation-post-restart") + defer releaseCurrent() + if currentRun.GrantGeneration != generation { + t.Fatalf("post-restart claim generation=%d want=%d", currentRun.GrantGeneration, generation) + } + // Keep the live claim's identity and lifetime, signing with the trusted k2. + // Only the generation is stale: neither a released claim nor retired k1 can + // explain the denial. Successful reads bracket the probe on this connection. + stale := resignFixtureGrant(t, currentRun, filepath.Join(rotationDir, "grant-k2.pem"), "k2", newAttached.GrantGeneration) + waitFileContent(t, ctx, postRestartClient, currentRun, "rotation-sentinel.txt", "old-authority\n", "post-restart revocation positive control") + if _, err := postRestartClient.File(ctx, executionenv.FileRequest{Context: stale, Operation: executionenv.OpFileRead, Path: "rotation-sentinel.txt"}); !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("revoked claim generation rejection: code=%s, want conflict", remoteErrorCode(err)) + } + waitFileContent(t, ctx, postRestartClient, currentRun, "rotation-sentinel.txt", "old-authority\n", "post-restart revocation positive control after probe") + // Restore fixture client compatibility through a higher generation; this is + // another forward rotation, never a high-water-mark rollback. + restoreFixtureSecurity(t, ctx, kubeconfig, rotationDir) + qualifyDistinctAdministrator(t, ctx, state, kubeconfig, "rotated") +} + +func acquireCurrentAuthorityRun(ctx context.Context, req executionenv.RunClaimRequest, acquire func(context.Context, executionenv.RunClaimRequest) (executionenv.RunClaim, error), retryDelay time.Duration) (executionenv.RunClaim, error) { + const maxAttempts = 8 + + var lastErr error + for attempt := 1; attempt <= maxAttempts; attempt++ { + if err := ctx.Err(); err != nil { + return executionenv.RunClaim{}, err + } + claim, err := acquire(ctx, req) + if err == nil { + return claim, nil + } + if !isRetryableAuthorityConvergence(err) { + return executionenv.RunClaim{}, err + } + lastErr = err + if attempt == maxAttempts { + break + } + if retryDelay <= 0 { + continue + } + timer := time.NewTimer(retryDelay) + select { + case <-ctx.Done(): + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + return executionenv.RunClaim{}, ctx.Err() + case <-timer.C: + } + } + return executionenv.RunClaim{}, fmt.Errorf("final-authority acquire did not converge after %d attempts: %w", maxAttempts, lastErr) +} + +func isRetryableAuthorityConvergence(err error) bool { + var remote *executionenv.Error + return errors.As(err, &remote) && remote.Code == executionenv.CodeNotReady && remote.Retryable +} + +func TestAcquireCurrentAuthorityRunRetriesOnlyRetryableNotReady(t *testing.T) { + req := executionenv.RunClaimRequest{Environment: executionenv.EnvironmentRef{ID: "environment", Revision: "revision"}, Owner: executionenv.Owner{Issuer: "issuer", Subject: "subject"}, BindingID: "binding", RunID: "run", OperationID: "same-operation", TTL: time.Minute} + success := executionenv.RunClaim{Environment: req.Environment, BindingID: req.BindingID, RunID: req.RunID, ClaimID: "claim"} + + for _, tc := range []struct { + name string + errs []error + wantCalls int + wantErr bool + }{ + {name: "retryable not ready", errs: []error{&executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true}}, wantCalls: 2}, + {name: "not ready without retryability", errs: []error{&executionenv.Error{Code: executionenv.CodeNotReady}}, wantCalls: 1, wantErr: true}, + {name: "retryable unauthenticated", errs: []error{&executionenv.Error{Code: executionenv.CodeUnauthenticated, Retryable: true}}, wantCalls: 1, wantErr: true}, + {name: "permission denied", errs: []error{&executionenv.Error{Code: executionenv.CodePermissionDenied}}, wantCalls: 1, wantErr: true}, + {name: "internal", errs: []error{&executionenv.Error{Code: executionenv.CodeInternal}}, wantCalls: 1, wantErr: true}, + {name: "malformed error", errs: []error{errors.New("unexpected failure")}, wantCalls: 1, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + claim, err := acquireCurrentAuthorityRun(t.Context(), req, func(_ context.Context, got executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + calls++ + if got != req { + t.Fatalf("retry request changed: got %+v want %+v", got, req) + } + if calls <= len(tc.errs) { + return executionenv.RunClaim{}, tc.errs[calls-1] + } + return success, nil + }, 0) + if calls != tc.wantCalls { + t.Fatalf("calls=%d want=%d", calls, tc.wantCalls) + } + if tc.wantErr { + if err == nil { + t.Fatal("expected terminal error") + } + return + } + if err != nil || claim != success { + t.Fatalf("claim=%+v err=%v, want %+v", claim, err, success) + } + }) + } +} + +func TestAcquireCurrentAuthorityRunStopsAtBoundAndHonorsContext(t *testing.T) { + req := executionenv.RunClaimRequest{OperationID: "same-operation"} + t.Run("bound", func(t *testing.T) { + calls := 0 + _, err := acquireCurrentAuthorityRun(t.Context(), req, func(context.Context, executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + calls++ + return executionenv.RunClaim{}, &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true} + }, 0) + if calls != 8 || !isRemoteCode(err, executionenv.CodeNotReady) { + t.Fatalf("calls=%d code=%s, want 8/not_ready", calls, remoteErrorCode(err)) + } + }) + t.Run("cancelled", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + calls := 0 + _, err := acquireCurrentAuthorityRun(ctx, req, func(context.Context, executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + calls++ + return executionenv.RunClaim{}, nil + }, 0) + if calls != 0 || !errors.Is(err, context.Canceled) { + t.Fatalf("calls=%d err=%v, want 0/context canceled", calls, err) + } + }) +} + +func waitFileContent(t *testing.T, ctx context.Context, client *executionclient.Client, rc executionenv.RequestContext, path, want, proof string) { + t.Helper() + if err := readCurrentAuthorityContent(ctx, func(ctx context.Context) (executionenv.FileResponse, error) { + return client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: path}) + }, want, 500*time.Millisecond); err != nil { + var remote *executionenv.Error + retryable := errors.As(err, &remote) && remote.Retryable + t.Fatalf("%s did not authorize preserved data: errorCode=%s retryable=%t", proof, remoteErrorCode(err), retryable) + } +} + +func readCurrentAuthorityContent(ctx context.Context, read func(context.Context) (executionenv.FileResponse, error), want string, delay time.Duration) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + const maxAttempts = 60 + for attempt := 0; ; attempt++ { + if err := ctx.Err(); err != nil { + return err + } + got, err := read(ctx) + if err == nil { + if string(got.Data) != want { + return errors.New("authority read returned unexpected content") + } + return nil + } + if !isRetryableAuthorityConvergence(err) || attempt == maxAttempts-1 { + return err + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} + +func TestReadCurrentAuthorityContentRetriesOnlyPreDispatchLag(t *testing.T) { + for _, tc := range []struct { + name string + err error + content string + wantCalls int + wantErr bool + }{ + {"success", nil, "sentinel", 1, false}, + {"authority lag", &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true}, "", 2, false}, + {"wrong content", nil, "wrong", 1, true}, + {"store not ready", &executionenv.Error{Code: executionenv.CodeNotReady}, "", 1, true}, + {"expired client", &executionenv.Error{Code: executionenv.CodeUnauthenticated, Retryable: true}, "", 1, true}, + {"revoked key", &executionenv.Error{Code: executionenv.CodePermissionDenied}, "", 1, true}, + {"unknown error", errors.New("not_ready"), "", 1, true}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + err := readCurrentAuthorityContent(t.Context(), func(context.Context) (executionenv.FileResponse, error) { + calls++ + if calls == 1 { + return executionenv.FileResponse{Data: []byte(tc.content)}, tc.err + } + return executionenv.FileResponse{Data: []byte("sentinel")}, nil + }, "sentinel", 0) + if calls != tc.wantCalls || (err != nil) != tc.wantErr { + t.Fatalf("calls=%d error=%t", calls, err != nil) + } + }) + } +} + +func TestReadCurrentAuthorityContentBoundAndCancellation(t *testing.T) { + t.Run("persistent lag", func(t *testing.T) { + calls := 0 + err := readCurrentAuthorityContent(t.Context(), func(context.Context) (executionenv.FileResponse, error) { + calls++ + return executionenv.FileResponse{}, &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true} + }, "sentinel", 0) + if calls != 60 || !isRetryableAuthorityConvergence(err) { + t.Fatalf("calls=%d errorCode=%s", calls, remoteErrorCode(err)) + } + }) + for _, cancelInCall := range []bool{false, true} { + ctx, cancel := context.WithCancel(t.Context()) + if !cancelInCall { + cancel() + } + calls := 0 + err := readCurrentAuthorityContent(ctx, func(ctx context.Context) (executionenv.FileResponse, error) { + calls++ + if _, ok := ctx.Deadline(); !ok { + t.Fatal("read has no bounded deadline") + } + cancel() + return executionenv.FileResponse{}, &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: true} + }, "sentinel", time.Hour) + cancel() + wantCalls := 0 + if cancelInCall { + wantCalls = 1 + } + if !errors.Is(err, context.Canceled) || calls != wantCalls { + t.Fatalf("calls=%d cancelled=%t", calls, errors.Is(err, context.Canceled)) + } + } +} + +func applySecurityCandidate(t *testing.T, ctx context.Context, kubeconfig, materialDir, manifest, mode string) { + t.Helper() + pki := filepath.Join(os.Getenv("MECATL_EXECUTION_QUAL_STATE"), "pki") + // Retain every previously referenced name with its original bytes. Secret + // and manifest projections may arrive in either order at each replica. + secretArgs := []string{"create", "secret", "generic", "execution-security", "-n", namespace, + "--from-file=grant-k1.pem=" + filepath.Join(pki, "grant-key.pem"), + "--from-file=tls.crt=" + filepath.Join(pki, "provider.crt"), + "--from-file=tls.key=" + filepath.Join(pki, "provider.key"), + "--from-file=clients.pem=" + filepath.Join(pki, "ca.crt")} + switch mode { + case "initial": + case "bridge", "final", "restore": + for _, name := range []string{"grant-k2.pem", "provider-new.crt", "provider-new.key", "bridge-clients.pem", "final-clients.pem"} { + secretArgs = append(secretArgs, "--from-file="+name+"="+filepath.Join(materialDir, name)) + } + default: + t.Fatalf("unknown security candidate mode %q", mode) + } + secretArgs = append(secretArgs, "--dry-run=client", "-o", "yaml") + applyKubectlInput(t, ctx, kubeconfig, runKubectl(t, ctx, kubeconfig, secretArgs...)) + config := runKubectl(t, ctx, kubeconfig, "create", "configmap", "mecatl-execution-security-manifest", "-n", namespace, "--from-file=manifest.json="+manifest, "--dry-run=client", "-o", "yaml") + applyKubectlInput(t, ctx, kubeconfig, config) +} + +func restoreFixtureSecurity(t *testing.T, ctx context.Context, kubeconfig, rotationDir string) { + t.Helper() + applySecurityCandidate(t, ctx, kubeconfig, rotationDir, filepath.Join(rotationDir, "restore-fixture-clients.json"), "restore") + waitSecurityGeneration(ctx, t, kubeconfig, 4) + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + + pki := filepath.Join(os.Getenv("MECATL_EXECUTION_QUAL_STATE"), "pki") + roots, err := os.ReadFile(filepath.Join(rotationDir, "client-roots.pem")) + if err != nil { + t.Fatal("read synthetic provider trust bundle") + } + if err := os.WriteFile(filepath.Join(pki, "provider-roots.pem"), roots, 0o600); err != nil { + t.Fatal("publish synthetic provider trust bundle") + } + clientSecret := runKubectl(t, ctx, kubeconfig, "create", "secret", "generic", "execution-client-tls", "-n", namespace, + "--from-file=ca.crt="+filepath.Join(rotationDir, "client-roots.pem"), "--from-file=tls.crt="+filepath.Join(pki, "mecak8s.crt"), "--from-file=tls.key="+filepath.Join(pki, "mecak8s.key"), "--dry-run=client", "-o", "yaml") + applyKubectlInput(t, ctx, kubeconfig, clientSecret) + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecak8s", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecak8s", "-n", namespace, "--timeout=240s") +} + +func applyKubectlInput(t *testing.T, ctx context.Context, kubeconfig string, input []byte) { + t.Helper() + cmd := command(ctx, kubeconfig, "apply", "-f", "-") + cmd.Stdin = bytes.NewReader(input) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("kubectl apply synthetic fixture: %v: %s", err, out) + } +} + +// PodReady can still describe the prior generation. Observe the nonsecret +// ledger first; the pinned replica's RPC barrier then proves local convergence. +func waitSecurityGeneration(ctx context.Context, t *testing.T, kubeconfig string, want uint64) { + t.Helper() + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + for { + cm := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-security-authority") + var ledger struct{ Generation uint64 } + if err := json.Unmarshal([]byte(cm.Data["state.json"]), &ledger); err != nil || ledger.Generation > want { + t.Fatal("unexpected authority ledger generation") + } + if ledger.Generation == want { + return + } + timer := time.NewTimer(500 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + t.Fatalf("authority ledger did not reach generation %d", want) + case <-timer.C: + } + } +} + +func waitProviderReadyReplicas(t *testing.T, ctx context.Context, kubeconfig string, want int) { + t.Helper() + deadline := time.Now().Add(2 * time.Minute) + for time.Now().Before(deadline) { + var pods corev1.PodList + raw := runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "app.kubernetes.io/name=mecatl-execution", "-o", "json") + if json.Unmarshal(raw, &pods) == nil { + ready := 0 + for _, pod := range pods.Items { + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue { + ready++ + } + } + } + if ready == want { + return + } + } + time.Sleep(time.Second) + } + t.Fatalf("provider Ready replicas did not converge to %d", want) +} + +func TestKindExecutionProductionReplicaLifecycle(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + replicas, replicaUIDs := productionReplicaClients(t, ctx, state, kubeconfig) + client := replicas[0] + owner, binding, attached := createProductionEnvironment(t, ctx, client, "lifecycle") + first := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + if first.SpecSchema != 2 || first.StatusSchema != 2 || first.PodUID == "" || first.PVCUID == "" { + t.Fatal("schema-v2 exact runtime identities were not persisted") + } + + type acquireResult struct { + claim executionenv.RunClaim + err error + } + startAcquire := make(chan struct{}) + acquired := make(chan acquireResult, 2) + for i := range replicas { + i := i + go func() { + <-startAcquire + claim, err := replicas[i].AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: fmt.Sprintf("run-%d", i), OperationID: fmt.Sprintf("acquire-%d", i), TTL: time.Minute}) + acquired <- acquireResult{claim: claim, err: err} + }() + } + close(startAcquire) + var claim1 executionenv.RunClaim + conflicts := 0 + for range replicas { + result := <-acquired + if result.err == nil { + claim1 = result.claim + } else if isRemoteCode(result.err, executionenv.CodeConflict) { + conflicts++ + } else { + t.Fatalf("cross-replica run acquisition returned unexpected error: %v", result.err) + } + } + if claim1.ClaimID == "" || conflicts != 1 { + t.Fatalf("cross-replica run acquisition pod_uids=%v winner=%t conflicts=%d, want one each", replicaUIDs, claim1.ClaimID != "", conflicts) + } + statusWithClaim := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + if statusWithClaim.ActiveGrantGeneration != claim1.GrantGeneration { + t.Fatal("CRD pruned activeRun.grantGeneration") + } + if _, err := client.RenewRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: claim1.RunID, ClaimID: "stale", Epoch: claim1.Epoch, GrantGeneration: claim1.GrantGeneration, OperationID: "renew-stale", TTL: time.Minute}); err == nil { + t.Fatal("stale renewal unexpectedly succeeded") + } + rc := executionenv.RequestContext{Environment: claim1.Environment, Owner: owner, BindingID: binding, RunID: claim1.RunID, ClaimID: claim1.ClaimID, Epoch: claim1.Epoch, GrantGeneration: claim1.GrantGeneration, Grant: claim1.Grant} + if _, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "lifecycle-proof.txt", Data: []byte("preserved\n")}); err != nil { + t.Fatal(err) + } + if err := client.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: claim1.Environment, Owner: owner, BindingID: binding, RunID: claim1.RunID, ClaimID: claim1.ClaimID, Epoch: claim1.Epoch, GrantGeneration: claim1.GrantGeneration, OperationID: "release-a"}); err != nil { + t.Fatal(err) + } + + current := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + stale, wrong, exact := replacementProofRequests(attached.Environment, owner, first, current) + if err := client.ReplaceExecutor(ctx, stale); err == nil { + t.Fatal("stale-epoch replacement unexpectedly succeeded") + } + if err := client.ReplaceExecutor(ctx, wrong); err == nil { + t.Fatal("wrong Pod UID replacement unexpectedly succeeded") + } + if got := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID).PodUID; got != first.PodUID { + t.Fatal("wrong-UID replacement changed executor") + } + if err := client.ReplaceExecutor(ctx, exact); err != nil { + t.Fatalf("replace exact executor: %v", err) + } + second := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Ready && s.PodUID != "" && s.PodUID != first.PodUID }) + if second.PVCUID != first.PVCUID || second.Epoch <= first.Epoch { + t.Fatal("replacement did not preserve exact PVC and advance epoch") + } + reattached := waitReady(t, ctx, client, owner, binding, attached.Environment) + rc2, release2 := acquireRun(t, ctx, client, owner, binding, reattached, "verify-replacement") + proof, err := client.File(ctx, executionenv.FileRequest{Context: rc2, Operation: executionenv.OpFileRead, Path: "lifecycle-proof.txt"}) + if err != nil || string(proof.Data) != "preserved\n" { + t.Fatal("replacement lost PVC data") + } + release2() + + revokeResults := make(chan struct { + generation uint64 + err error + }, 2) + startRevoke := make(chan struct{}) + for i := range replicas { + i := i + go func() { + <-startRevoke + generation, err := replicas[i].RevokeEnvironment(ctx, attached.Environment, owner, reattached.GrantGeneration, "revoke-lifecycle") + revokeResults <- struct { + generation uint64 + err error + }{generation: generation, err: err} + }() + } + close(startRevoke) + var newGeneration uint64 + for range replicas { + result := <-revokeResults + if result.err != nil || result.generation <= reattached.GrantGeneration { + t.Fatalf("cross-replica revoke failed: generation=%d err=%v", result.generation, result.err) + } + if newGeneration != 0 && result.generation != newGeneration { + t.Fatalf("cross-replica revoke replay diverged: got=%d want=%d", result.generation, newGeneration) + } + newGeneration = result.generation + } + if _, err := client.File(ctx, executionenv.FileRequest{Context: rc2, Operation: executionenv.OpFileRead, Path: "lifecycle-proof.txt"}); err == nil { + t.Fatal("revoked grant remained usable") + } + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=240s") + client, _ = productionClient(t, ctx, state, kubeconfig) + if refreshed := waitReady(t, ctx, client, owner, binding, attached.Environment); refreshed.GrantGeneration != newGeneration { + t.Fatal("revocation generation was not durable across provider restart") + } + + refReq := executionenv.ReferenceRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, OperationID: "delete-ref-lifecycle"} + if err := client.PrepareReferenceDelete(ctx, refReq); err != nil { + t.Fatal(err) + } + if err := client.ConfirmReferenceDelete(ctx, refReq); err != nil { + t.Fatal(err) + } + retireStatus := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + retire := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: retireStatus.Epoch, ExpectedPodUID: retireStatus.PodUID, ExpectedPVCUID: retireStatus.PVCUID, OperationID: "retire-lifecycle"} + if err := client.RetireEnvironment(ctx, retire); err != nil { + t.Fatalf("retire exact environment: %v", err) + } + retired := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Retired && s.ExecutorTerminated }) + if resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+attached.Environment.ID) != 1 || resourceCount(t, ctx, kubeconfig, "pods -l execution.mecatl.dev/environment="+attached.Environment.ID) != 0 { + t.Fatal("retirement did not stop executor while retaining PVC") + } + if err := client.DeleteRetiredEnvironment(ctx, attached.Environment, owner, retired.PVCUID, "delete-retired-lifecycle"); err != nil { + t.Fatalf("delete retired environment: %v", err) + } + waitResourceAbsent(t, ctx, kubeconfig, "executionenvironment", attached.Environment.ID) + if resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+attached.Environment.ID) != 0 { + t.Fatal("explicit retired deletion retained its owned synthetic PVC") + } +} + +type executionStatus struct { + SpecSchema, StatusSchema int + Epoch uint64 + PodUID, PVCUID string + ActiveOperationID, FenceState string + ActiveGrantGeneration uint64 + Ready, Retired, ExecutorTerminated bool + ReadyReason string +} + +func replacementProofRequests(ref executionenv.EnvironmentRef, owner executionenv.Owner, staleStatus, current executionStatus) (executionenv.RetireEnvironmentRequest, executionenv.RetireEnvironmentRequest, executionenv.RetireEnvironmentRequest) { + stale := executionenv.RetireEnvironmentRequest{Environment: ref, Owner: owner, ExpectedEpoch: staleStatus.Epoch, ExpectedPodUID: staleStatus.PodUID, ExpectedPVCUID: staleStatus.PVCUID, OperationID: "replace-stale-epoch"} + wrong := executionenv.RetireEnvironmentRequest{Environment: ref, Owner: owner, ExpectedEpoch: current.Epoch, ExpectedPodUID: "wrong-pod-uid", ExpectedPVCUID: current.PVCUID, OperationID: "replace-wrong"} + exact := wrong + exact.ExpectedPodUID = current.PodUID + exact.OperationID = "replace-exact" + return stale, wrong, exact +} + +func readExecutionStatus(t *testing.T, ctx context.Context, kubeconfig, name string) executionStatus { + t.Helper() + var object map[string]any + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "executionenvironment", name, "-n", namespace, "-o", "json"), &object); err != nil { + t.Fatal(err) + } + nested := func(path ...string) any { + var v any = object + for _, p := range path { + m, ok := v.(map[string]any) + if !ok { + return nil + } + v = m[p] + } + return v + } + number := func(path ...string) uint64 { + if v, ok := nested(path...).(float64); ok { + return uint64(v) + } + return 0 + } + text := func(path ...string) string { v, _ := nested(path...).(string); return v } + condition := func(kind string) bool { + items, _ := nested("status", "conditions").([]any) + for _, item := range items { + m, _ := item.(map[string]any) + if m["type"] == kind && m["status"] == "True" { + return true + } + } + return false + } + conditionReason := func(kind string) string { + items, _ := nested("status", "conditions").([]any) + for _, item := range items { + m, _ := item.(map[string]any) + if m["type"] == kind { + reason, _ := m["reason"].(string) + return reason + } + } + return "" + } + return executionStatus{SpecSchema: int(number("spec", "schemaVersion")), StatusSchema: int(number("status", "schemaVersion")), Epoch: number("status", "epoch"), PodUID: text("status", "pod", "uid"), PVCUID: text("status", "pvc", "uid"), ActiveOperationID: text("status", "activeOperation", "id"), FenceState: text("status", "fenceState"), ActiveGrantGeneration: number("status", "activeRun", "grantGeneration"), Ready: condition("Ready"), Retired: condition("Retired"), ExecutorTerminated: condition("ExecutorTerminated"), ReadyReason: conditionReason("Ready")} +} + +func waitExecutionStatus(t *testing.T, ctx context.Context, kubeconfig, name string, accept func(executionStatus) bool) executionStatus { + t.Helper() + deadline := time.Now().Add(4 * time.Minute) + var last executionStatus + for time.Now().Before(deadline) { + last = readExecutionStatus(t, ctx, kubeconfig, name) + if accept(last) { + return last + } + time.Sleep(time.Second) + } + t.Fatalf("execution status did not reach required lifecycle state: environment=%q ready=%t ready_reason=%q fence_state=%q active_operation=%t", name, last.Ready, last.ReadyReason, last.FenceState, last.ActiveOperationID != "") + return executionStatus{} +} + +func waitActiveOperationOrCommandError(t *testing.T, ctx context.Context, kubeconfig, name string, commandDone <-chan error) { + t.Helper() + deadline := time.NewTimer(4 * time.Minute) + defer deadline.Stop() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case err := <-commandDone: + t.Fatalf("command ended before active-operation publication: code=%s", remoteErrorCode(err)) + case <-ticker.C: + if readExecutionStatus(t, ctx, kubeconfig, name).ActiveOperationID != "" { + return + } + case <-deadline.C: + t.Fatal("active-operation publication timed out while command remained in flight") + case <-ctx.Done(): + t.Fatal("active-operation publication context ended") + } + } +} + +func waitResourceAbsent(t *testing.T, ctx context.Context, kubeconfig, resource, name string) { + t.Helper() + deadline := time.Now().Add(3 * time.Minute) + for time.Now().Before(deadline) { + cmd := command(ctx, kubeconfig, "get", resource, name, "-n", namespace, "-o", "name") + if err := cmd.Run(); err != nil { + return + } + time.Sleep(time.Second) + } + t.Fatal("owned synthetic resource was not deleted") +} + +func kubeValue(t *testing.T, ctx context.Context, kubeconfig string, args ...string) string { + t.Helper() + return strings.TrimSpace(string(runKubectl(t, ctx, kubeconfig, args...))) +} + +func TestKindExecutionProductionHolderLossFencesActiveOperation(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + replicas, replicaUIDs := productionReplicaClients(t, ctx, state, kubeconfig) + holderClient, survivingClient := replicas[0], replicas[1] + owner, binding, attached := createProductionEnvironment(t, ctx, holderClient, "holder-loss") + rc, _ := acquireRun(t, ctx, holderClient, owner, binding, attached, "holder-loss-run") + initial := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + + var providers corev1.PodList + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "app.kubernetes.io/name=mecatl-execution", "-o", "json"), &providers); err != nil || len(providers.Items) != 2 { + t.Fatal("holder-loss proof requires two provider replicas") + } + holderPod := "" + for i := range providers.Items { + if string(providers.Items[i].UID) == replicaUIDs[0] { + holderPod = providers.Items[i].Name + } + } + if holderPod == "" { + t.Fatalf("holder replica UID %s no longer identifies a provider Pod", replicaUIDs[0]) + } + + commandDone := make(chan error, 1) + go func() { + _, commandErr := holderClient.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: `i=0; trap '' HUP TERM; while [ $i -lt 90 ]; do i=$((i+1)); printf '%s\n' "$i" > holder-loss-nonce; sleep 1; done`, TimeoutMillis: 110000}) + commandDone <- commandErr + }() + waitActiveOperationOrCommandError(t, ctx, kubeconfig, attached.Environment.ID, commandDone) + runKubectl(t, ctx, kubeconfig, "delete", "pod/"+holderPod, "-n", namespace, "--wait=true", "--timeout=90s") + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=180s") + + if _, err := survivingClient.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: "second-writer", Data: []byte("must-not-run")}); err == nil { + t.Fatal("surviving replica admitted an overlapping writer after holder loss") + } + if _, err := survivingClient.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: "overlap-run", OperationID: "overlap-run-acquire", TTL: time.Minute}); err == nil { + t.Fatal("surviving replica admitted a new run while old operation ownership was unresolved") + } + fenced := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { + return s.FenceState == "FenceUnknown" && s.ActiveOperationID != "" && !s.Ready + }) + if fenced.ReadyReason != "FenceUnknown" { + t.Fatalf("holder loss reason=%q, want bounded FenceUnknown", fenced.ReadyReason) + } + select { + case <-commandDone: + case <-time.After(5 * time.Second): + // A disconnected exec stream is permitted to remain locally blocked; the + // durable operation identity above, not RPC completion, is the proof. + } + + // Exec disconnect cancellation kills the command process group and reaped + // descendants, so a child timer cannot reliably terminate PID 1. Only after + // proving unresolved ownership above, ask the kubelet to stop this exact Pod. + contextName := os.Getenv("MECATL_KUBE_CONTEXT") + if contextName == "" { + t.Fatal("explicit fixture Kubernetes context required") + } + config, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(&clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfig}, &clientcmd.ConfigOverrides{CurrentContext: contextName}).ClientConfig() + if err != nil { + t.Fatal("load explicit fixture Kubernetes configuration") + } + config.Timeout = 15 * time.Second + kube, err := kubernetes.NewForConfig(config) + if err != nil { + t.Fatal("construct fixture Kubernetes client") + } + envUID := kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") + if err := terminateOwnedExecutor(ctx, kube, attached.Environment.ID, envUID, initial.PodUID); err != nil { + t.Fatal("exact owned executor termination request failed") + } + waitExecutorTerminal(ctx, t, kubeconfig, attached.Environment.ID, initial.PodUID) + recovery := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: initial.Epoch, ExpectedPodUID: initial.PodUID, ExpectedPVCUID: initial.PVCUID, OperationID: "recover-holder-loss"} + if err := survivingClient.RecoverEnvironment(ctx, recovery); err != nil { + t.Fatalf("recover exact terminal executor: %v", err) + } + if err := survivingClient.ReplaceExecutor(ctx, executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: initial.Epoch, ExpectedPodUID: initial.PodUID, ExpectedPVCUID: initial.PVCUID, OperationID: "replace-holder-loss"}); err != nil { + t.Fatalf("replace recovered executor: %v", err) + } + replaced := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Ready && s.PodUID != "" && s.PodUID != initial.PodUID }) + if replaced.PVCUID != initial.PVCUID { + t.Fatal("holder-loss recovery replaced the retained workspace") + } + reattached := waitReady(t, ctx, survivingClient, owner, binding, attached.Environment) + verify, release := acquireRun(t, ctx, survivingClient, owner, binding, reattached, "holder-loss-verify") + if _, err := survivingClient.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "holder-loss-nonce"}); err != nil { + t.Fatalf("post-recovery sentinel unavailable: %v", err) + } + if _, err := survivingClient.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "second-writer"}); err == nil { + t.Fatal("denied overlapping writer nevertheless reached the executor") + } + release() +} + +func waitExecutorTerminal(ctx context.Context, t *testing.T, kubeconfig, environmentID, podUID string) { + t.Helper() + deadline := time.Now().Add(2 * time.Minute) + for time.Now().Before(deadline) { + var pods corev1.PodList + raw := runKubectl(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "execution.mecatl.dev/environment="+environmentID, "-o", "json") + if json.Unmarshal(raw, &pods) == nil && len(pods.Items) == 1 && executorTerminalProof(&pods.Items[0], podUID) { + return + } + time.Sleep(time.Second) + } + t.Fatal("test-owned executor did not produce terminal kubelet evidence") +} + +func TestKindExecutionProductionQuotaSaturation(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + replicas, replicaUIDs := productionReplicaClients(t, ctx, state, kubeconfig) + client := replicas[0] + + // First prove the provider's profile-wide CAS authority under concurrent + // requests. Exactly one distinct allocation may consume quota-cas's sole slot. + type ensureResult struct { + owner executionenv.Owner + binding string + out executionenv.EnsureEnvironmentResponse + err error + } + start := make(chan struct{}) + results := make(chan ensureResult, 2) + for i := 0; i < 2; i++ { + i := i + go func() { + <-start + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: fmt.Sprintf("quota-cas-%d", i)} + binding := fmt.Sprintf("quota-cas-%d-%d", i, time.Now().UnixNano()) + out, err := replicas[i].Ensure(ctx, binding, "quota-cas", owner, "ensure-"+binding) + results <- ensureResult{owner: owner, binding: binding, out: out, err: err} + }() + } + close(start) + var accepted *ensureResult + denied := 0 + for range 2 { + result := <-results + if result.err == nil { + copy := result + accepted = © + } else if isRemoteCode(result.err, executionenv.CodeResourceExhausted) { + denied++ + } else { + t.Fatalf("quota CAS returned unexpected error: %v", result.err) + } + } + if accepted == nil || denied != 1 { + t.Fatalf("quota CAS pod_uids=%v accepted=%v resource_exhausted=%d, want one each", replicaUIDs, accepted != nil, denied) + } + if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: accepted.out.Environment, Owner: accepted.owner, BindingID: accepted.binding, OperationID: "ensure-" + accepted.binding}); err != nil { + t.Fatal(err) + } + waitReady(t, ctx, client, accepted.owner, accepted.binding, accepted.out.Environment) + + // Then exercise Kubernetes admission itself. Tighten only the fixture-owned + // namespace's PVC count to one additional claim and restore the known chart + // value before converging and explicitly deleting these synthetic allocations. + pvcBaseline := resourceCount(t, ctx, kubeconfig, "pvc") + runKubectl(t, ctx, kubeconfig, "patch", "resourcequota/mecatl-execution", "-n", namespace, "--type=merge", "-p", fmt.Sprintf(`{"spec":{"hard":{"persistentvolumeclaims":"%d"}}}`, pvcBaseline+1)) + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute) + defer cleanupCancel() + _ = command(cleanupCtx, kubeconfig, "patch", "resourcequota/mecatl-execution", "-n", namespace, "--type=merge", "-p", `{"spec":{"hard":{"persistentvolumeclaims":"100"}}}`).Run() + }) + + allocations := make([]ensureResult, 0, 2) + for i := 0; i < 2; i++ { + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: fmt.Sprintf("quota-kube-%d", i)} + binding := fmt.Sprintf("quota-kube-%d-%d", i, time.Now().UnixNano()) + out, err := client.Ensure(ctx, binding, "quota-kube", owner, "ensure-"+binding) + if err != nil { + t.Fatalf("create quota admission candidate: %v", err) + } + allocations = append(allocations, ensureResult{owner: owner, binding: binding, out: out}) + } + deadline := time.Now().Add(2 * time.Minute) + var blocked ensureResult + for time.Now().Before(deadline) { + for _, allocation := range allocations { + status := readExecutionStatus(t, ctx, kubeconfig, allocation.out.Environment.ID) + if !status.Ready && status.ReadyReason == "PVCUnavailable" && resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+allocation.out.Environment.ID) == 0 { + blocked = allocation + } + } + if blocked.binding != "" { + break + } + time.Sleep(time.Second) + } + if blocked.binding == "" { + t.Fatal("Kubernetes PVC quota did not leave one allocation NotReady with reason PVCUnavailable") + } + if resourceCount(t, ctx, kubeconfig, "pvc") != pvcBaseline+1 { + t.Fatal("PVC quota admitted more than one additional workspace") + } + runKubectl(t, ctx, kubeconfig, "patch", "resourcequota/mecatl-execution", "-n", namespace, "--type=merge", "-p", `{"spec":{"hard":{"persistentvolumeclaims":"100"}}}`) + waitExecutionStatus(t, ctx, kubeconfig, blocked.out.Environment.ID, func(s executionStatus) bool { return s.Ready }) + + all := append(allocations, *accepted) + for _, allocation := range all { + if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: allocation.out.Environment, Owner: allocation.owner, BindingID: allocation.binding, OperationID: "ensure-" + allocation.binding}); err != nil { + t.Fatal(err) + } + waitReady(t, ctx, client, allocation.owner, allocation.binding, allocation.out.Environment) + retireSyntheticEnvironment(t, ctx, client, kubeconfig, allocation.owner, allocation.binding, allocation.out.Environment) + } +} + +func retireSyntheticEnvironment(t *testing.T, ctx context.Context, client *executionclient.Client, kubeconfig string, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) { + t.Helper() + deleteRef := executionenv.ReferenceRequest{Environment: ref, Owner: owner, BindingID: binding, OperationID: "delete-ref-" + binding} + if err := client.PrepareReferenceDelete(ctx, deleteRef); err != nil { + t.Fatal(err) + } + if err := client.ConfirmReferenceDelete(ctx, deleteRef); err != nil { + t.Fatal(err) + } + status := readExecutionStatus(t, ctx, kubeconfig, ref.ID) + request := executionenv.RetireEnvironmentRequest{Environment: ref, Owner: owner, ExpectedEpoch: status.Epoch, ExpectedPodUID: status.PodUID, ExpectedPVCUID: status.PVCUID, OperationID: "retire-" + binding} + if err := client.RetireEnvironment(ctx, request); err != nil { + t.Fatal(err) + } + retired := waitExecutionStatus(t, ctx, kubeconfig, ref.ID, func(s executionStatus) bool { return s.Retired && s.ExecutorTerminated }) + if err := client.DeleteRetiredEnvironment(ctx, ref, owner, retired.PVCUID, "delete-retired-"+binding); err != nil { + t.Fatal(err) + } + waitResourceAbsent(t, ctx, kubeconfig, "executionenvironment", ref.ID) +} + +func TestKindExecutionProductionPendingDeleteOutageRecovery(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + client, _ := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, client, "pending-delete") + request := executionenv.ReferenceRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, OperationID: "pending-delete-outage"} + if err := client.PrepareReferenceDelete(ctx, request); err != nil { + t.Fatal(err) + } + if got := executionReferences(t, ctx, kubeconfig, attached.Environment.ID)[binding]; got != string(executionenv.ReferencePendingDelete) { + t.Fatalf("prepared reference state=%q", got) + } + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=240s") + client, _ = productionClient(t, ctx, state, kubeconfig) + intents, err := client.ListReferenceIntents(ctx, owner) + if err != nil { + t.Fatal(err) + } + found := false + for _, intent := range intents { + if intent.Environment == attached.Environment && intent.BindingID == binding && intent.State == executionenv.ReferencePendingDelete && intent.OperationID == request.OperationID { + found = true + } + } + if !found { + exact, exactErr := client.FindReferenceIntent(ctx, owner, attached.Environment, binding) + t.Fatalf("pending-delete intent was not durable across provider outage: listed=%d exact_code=%s exact_state=%q exact_operation_match=%t", len(intents), remoteErrorCode(exactErr), exact.State, exact.OperationID == request.OperationID) + } + intruderTLS := loadTLS(t, filepath.Join(state, "pki"), "intruder", "mecatl-execution.execution-qualification.svc.cluster.local") + forward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + intruder, err := executionclient.New(forward.addr, intruderTLS) + if err != nil { + t.Fatal(err) + } + defer intruder.Close() + private, err := intruder.ListReferenceIntents(ctx, owner) + if err != nil { + t.Fatal(err) + } + if len(private) != 0 { + t.Fatal("pending reference intent crossed the mTLS client privacy boundary") + } + if err := client.ConfirmReferenceDelete(ctx, request); err != nil { + t.Fatal(err) + } + waitReferenceSet(t, ctx, kubeconfig, attached.Environment.ID) + if resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+attached.Environment.ID) != 1 { + t.Fatal("pending-delete recovery removed retained storage") + } + status := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + retire := executionenv.RetireEnvironmentRequest{Environment: attached.Environment, Owner: owner, ExpectedEpoch: status.Epoch, ExpectedPodUID: status.PodUID, ExpectedPVCUID: status.PVCUID, OperationID: "retire-pending-delete"} + if err := client.RetireEnvironment(ctx, retire); err != nil { + t.Fatal(err) + } + retired := waitExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID, func(s executionStatus) bool { return s.Retired && s.ExecutorTerminated }) + if err := client.DeleteRetiredEnvironment(ctx, attached.Environment, owner, retired.PVCUID, "delete-pending-delete"); err != nil { + t.Fatal(err) + } +} + +type legacyMigrationFixture struct { + Environment executionenv.EnvironmentRef `json:"environment"` + Owner executionenv.Owner `json:"owner"` + Binding string `json:"binding"` + PodUID string `json:"pod_uid"` + PVCUID string `json:"pvc_uid"` + Malformed executionenv.EnvironmentRef `json:"malformed_environment"` + MalformedPodUID string `json:"malformed_pod_uid"` + MalformedPVCUID string `json:"malformed_pvc_uid"` + Insecure executionenv.EnvironmentRef `json:"insecure_environment"` + InsecurePodUID string `json:"insecure_pod_uid"` + InsecurePVCUID string `json:"insecure_pvc_uid"` +} + +func runKindExecutionProductionCompatiblePrototypeMigration(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + client, _ := productionClient(t, ctx, state, kubeconfig) + var fixture legacyMigrationFixture + raw, err := os.ReadFile(filepath.Join(state, "legacy-migration.json")) + if err != nil || json.Unmarshal(raw, &fixture) != nil { + t.Fatalf("load pre-upgrade legacy API fixture: %v", err) + } + if fixture.Environment.ID == "" || fixture.Environment.Revision == "" || fixture.PodUID == "" || fixture.PVCUID == "" || fixture.Binding == "" { + t.Fatal("pre-upgrade legacy API fixture identities are incomplete") + } + + var stored map[string]any + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "executionenvironment", fixture.Environment.ID, "-n", namespace, "-o", "json"), &stored); err != nil { + t.Fatal(err) + } + status, _ := stored["status"].(map[string]any) + references, _ := status["references"].([]any) + if len(references) != 1 || references[0] != fixture.Binding { + t.Fatalf("legacy string references were not persisted through the real API upgrade: %#v", references) + } + before := readExecutionStatus(t, ctx, kubeconfig, fixture.Environment.ID) + if before.SpecSchema != 1 || before.StatusSchema != 1 || before.PodUID != fixture.PodUID || before.PVCUID != fixture.PVCUID { + t.Fatal("pre-upgrade fixture did not preserve schema-1 exact runtime identity") + } + if _, err := client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: fixture.Environment, Owner: fixture.Owner, BindingID: fixture.Binding}, Purpose: executionenv.PurposeSession}); err == nil { + t.Fatal("prototype schema attached before explicit migration") + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, "foreign-pod-uid", before.PVCUID, "migration-wrong-uid"); err == nil { + t.Fatal("migration adopted a foreign Pod UID") + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, "", before.PVCUID, "migration-missing-uid"); err == nil { + t.Fatal("migration accepted a missing runtime UID") + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 2, before.PodUID, before.PVCUID, "migration-unknown-version"); err == nil { + t.Fatal("migration accepted an unrecognized source schema") + } + if err := client.MigrateEnvironment(ctx, fixture.Malformed, fixture.Owner, 1, fixture.MalformedPodUID, fixture.MalformedPVCUID, "migration-malformed-shape"); err == nil || !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("migration accepted a legacy reference shape outside the recognized prototype: %v", err) + } + if err := client.MigrateEnvironment(ctx, fixture.Insecure, fixture.Owner, 1, fixture.InsecurePodUID, fixture.InsecurePVCUID, "migration-insecure-runtime"); err == nil || !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("migration accepted an insecure legacy executor: %v", err) + } + if got := readExecutionStatus(t, ctx, kubeconfig, fixture.Insecure.ID); got.SpecSchema != 1 || got.StatusSchema != 1 { + t.Fatal("rejected insecure legacy executor was rewritten") + } + // Normalize the deliberately rejected test-owned object through the API so it + // cannot poison later client-scoped reference reconciliation in this retained cluster. + normalized := fmt.Sprintf(`[{"bindingID":"legacy-malformed-binding","state":"Published","operationID":"migration-fixture-normalize","createdAt":%q}]`, time.Now().UTC().Format(time.RFC3339Nano)) + runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+fixture.Malformed.ID, "-n", namespace, "--subresource=status", "--type=merge", "-p", `{"status":{"references":`+normalized+`}}`) + insecureNormalized := fmt.Sprintf(`[{"bindingID":"legacy-insecure-binding","state":"Published","operationID":"migration-insecure-fixture-normalize","createdAt":%q}]`, time.Now().UTC().Format(time.RFC3339Nano)) + runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+fixture.Insecure.ID, "-n", namespace, "--subresource=status", "--type=merge", "-p", `{"status":{"references":`+insecureNormalized+`}}`) + if err := client.MigrateEnvironment(ctx, fixture.Malformed, fixture.Owner, 1, fixture.MalformedPodUID, fixture.MalformedPVCUID, "migration-normalized-shape"); err != nil { + t.Fatalf("normalize rejected migration fixture: %v", err) + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, before.PodUID, before.PVCUID, "migration-compatible-v1"); err != nil { + t.Fatalf("migrate compatible prototype: %v", err) + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 1, before.PodUID, before.PVCUID, "migration-compatible-v1"); err != nil { + t.Fatalf("exact migration replay: %v", err) + } + if err := client.MigrateEnvironment(ctx, fixture.Environment, fixture.Owner, 0, before.PodUID, before.PVCUID, "migration-compatible-v1"); !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatal("migration receipt accepted changed source schema:", remoteErrorCode(err)) + } + if got := kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", fixture.Environment.ID, "-n", namespace, "-o", "jsonpath={.status.lastMigrationFromSchema}"); got != "1" { + t.Fatal("API server pruned the exact migration receipt") + } + runKubectl(t, ctx, kubeconfig, "patch", "executionenvironment/"+fixture.Environment.ID, "-n", namespace, "--subresource=status", "--type=merge", "--dry-run=server", "-p", `{"status":{"lastMigrationFromSchema":0}}`) + invalid := command(ctx, kubeconfig, "patch", "executionenvironment/"+fixture.Environment.ID, "-n", namespace, "--subresource=status", "--type=merge", "--dry-run=server", "-p", `{"status":{"lastMigrationFromSchema":2}}`) + if out, err := invalid.CombinedOutput(); err == nil || !bytes.Contains(out, []byte("lastMigrationFromSchema")) || !bytes.Contains(out, []byte("Unsupported value")) { + t.Fatal("API server did not reject an unsupported migration receipt schema") + } + after := waitExecutionStatus(t, ctx, kubeconfig, fixture.Environment.ID, func(s executionStatus) bool { return s.Ready && s.SpecSchema == 2 && s.StatusSchema == 2 }) + if after.PodUID != before.PodUID || after.PVCUID != before.PVCUID || after.Epoch != before.Epoch+1 { + t.Fatal("migration changed runtime identity or failed to advance only the fence epoch") + } + if got := executionReferences(t, ctx, kubeconfig, fixture.Environment.ID); got[fixture.Binding] != string(executionenv.ReferencePublished) { + t.Fatal("migration did not convert the persisted legacy string reference") + } + reattached := waitReady(t, ctx, client, fixture.Owner, fixture.Binding, fixture.Environment) + verify, verifyRelease := acquireRun(t, ctx, client, fixture.Owner, fixture.Binding, reattached, "migration-verify") + got, err := client.File(ctx, executionenv.FileRequest{Context: verify, Operation: executionenv.OpFileRead, Path: "migration-sentinel"}) + if err != nil || string(got.Data) != "prototype-data\n" { + t.Fatal("migration lost prototype workspace data") + } + verifyRelease() +} + +func TestKindExecutionProductionClearForkLifecycle(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + provider, _ := productionClient(t, ctx, state, kubeconfig) + + tokenForward := portForward(t, ctx, kubeconfig, "service/oidc-issuer", 8443) + alice := fixtureToken(t, ctx, tokenForward.addr, filepath.Join(state, "pki"), "alice") + bob := fixtureToken(t, ctx, tokenForward.addr, filepath.Join(state, "pki"), "bob") + tokenForward.stop() + agent := portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + defer agent.stop() + + source := createSession(t, ctx, agent.addr, alice) + assertMockJourney(t, prompt(t, ctx, agent.addr, source, alice, "run the scripted remote qualification")) + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "alice"} + allocation := executionenv.EnsureEnvironmentResponse{Environment: environmentForBinding(t, ctx, kubeconfig, source)} + attached := waitReady(t, ctx, provider, owner, source, allocation.Environment) + before := readExecutionStatus(t, ctx, kubeconfig, allocation.Environment.ID) + + forkID := successorSession(t, ctx, agent.addr, source, alice, "fork") + clearID := successorSession(t, ctx, agent.addr, source, alice, "clear") + if forkID == source || clearID == source || forkID == clearID { + t.Fatal("Clear/Fork did not publish distinct session identities") + } + if got := getSession(t, ctx, agent.addr, forkID, bob); got != http.StatusNotFound { + t.Fatalf("unknown owner observed fork: status=%d", got) + } + if got := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); got < 1 { + t.Fatal("successor publication lost its source environment") + } + waitReferenceSet(t, ctx, kubeconfig, allocation.Environment.ID, source, forkID, clearID) + after := readExecutionStatus(t, ctx, kubeconfig, allocation.Environment.ID) + if after.PVCUID != before.PVCUID || after.PodUID != before.PodUID || after.Epoch != before.Epoch { + t.Fatal("Clear/Fork changed the exact remote execution identity") + } + for _, binding := range []string{forkID, clearID} { + if _, err := provider.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: allocation.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); err != nil { + t.Fatalf("successor %s did not attach to source environment: %v", binding, err) + } + } + + claim, err := provider.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: source, RunID: "source-blocker", OperationID: "source-blocker-acquire", TTL: time.Minute}) + if err != nil { + t.Fatal(err) + } + if _, err := provider.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: clearID, RunID: "clear-overlap", OperationID: "clear-overlap-acquire", TTL: time.Minute}); err == nil || !isRemoteCode(err, executionenv.CodeConflict) { + t.Fatalf("second session admitted an overlapping run: %v", err) + } + if err := provider.ReleaseRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: source, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, OperationID: "source-blocker-release"}); err != nil { + t.Fatal(err) + } + clearAttached := waitReady(t, ctx, provider, owner, clearID, allocation.Environment) + _, release := acquireRun(t, ctx, provider, owner, clearID, clearAttached, "clear-after-release") + release() + + for _, id := range []string{forkID, clearID, source} { + status, body := request(t, ctx, http.MethodPost, "http://"+agent.addr+"/v1/sessions/"+id+"/delete", alice, nil) + if status != http.StatusNoContent { + t.Fatalf("delete session %s status=%d body=%s", id, status, body) + } + } + waitReferenceSet(t, ctx, kubeconfig, allocation.Environment.ID) + final := readExecutionStatus(t, ctx, kubeconfig, allocation.Environment.ID) + if final.PVCUID != before.PVCUID || resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+allocation.Environment.ID) != 1 { + t.Fatal("session deletion removed or replaced the retained workspace") + } +} + +func environmentForBinding(t *testing.T, ctx context.Context, kubeconfig, binding string) executionenv.EnvironmentRef { + t.Helper() + var list struct { + Items []struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + Revision string `json:"revision"` + } `json:"spec"` + Status struct { + References []struct { + BindingID string `json:"bindingID"` + } `json:"references"` + } `json:"status"` + } `json:"items"` + } + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "executionenvironments", "-n", namespace, "-o", "json"), &list); err != nil { + t.Fatal(err) + } + for _, item := range list.Items { + for _, ref := range item.Status.References { + if ref.BindingID == binding && item.Metadata.Name != "" && item.Spec.Revision != "" { + return executionenv.EnvironmentRef{ID: item.Metadata.Name, Revision: item.Spec.Revision} + } + } + } + t.Fatalf("no exact execution environment owns binding %s", binding) + return executionenv.EnvironmentRef{} +} + +func successorSession(t *testing.T, ctx context.Context, addr, source, token, operation string) string { + t.Helper() + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+source+"/"+operation, token, []byte(`{}`)) + if status != http.StatusCreated { + t.Fatalf("%s session status=%d body=%s", operation, status, body) + } + var out struct { + SessionID string `json:"session_id"` + } + if json.Unmarshal(body, &out) != nil || out.SessionID == "" { + t.Fatalf("%s session returned no identity", operation) + } + return out.SessionID +} + +func waitReferenceSet(t *testing.T, ctx context.Context, kubeconfig, environmentID string, want ...string) { + t.Helper() + deadline := time.Now().Add(90 * time.Second) + for time.Now().Before(deadline) { + got := executionReferences(t, ctx, kubeconfig, environmentID) + if len(got) == len(want) { + matched := true + for _, binding := range want { + if got[binding] != string(executionenv.ReferencePublished) { + matched = false + } + } + if matched { + return + } + } + time.Sleep(500 * time.Millisecond) + } + t.Fatalf("environment references did not converge to published set %v", want) +} + +func executionReferences(t *testing.T, ctx context.Context, kubeconfig, environmentID string) map[string]string { + t.Helper() + var object struct { + Status struct { + References []struct { + BindingID string `json:"bindingID"` + State string `json:"state"` + } `json:"references"` + } `json:"status"` + } + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "executionenvironment", environmentID, "-n", namespace, "-o", "json"), &object); err != nil { + t.Fatal(err) + } + out := make(map[string]string, len(object.Status.References)) + for _, ref := range object.Status.References { + out[ref.BindingID] = ref.State + } + return out +} + +func TestKindExecutionProductionFailureArtifactBoundary(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + ctx, stop := context.WithTimeout(ctx, time.Minute) + defer stop() + output := filepath.Join(state, fmt.Sprintf("artifact-boundary-%d.jsonl", time.Now().UnixNano())) + cmd := exec.CommandContext(ctx, "sh", "../../deploy/mecatl-execution-kind/collect-failure.sh", kubeconfig, os.Getenv("MECATL_KUBE_CONTEXT"), output) + cmd.Env = cleanEnv() + if err := cmd.Run(); err != nil { + t.Fatal("bounded failure collector failed") + } + artifact, err := os.ReadFile(output) + if err != nil { + t.Fatal("sanitized failure artifact unavailable") + } + if len(artifact) > 1<<20 { + t.Fatal("sanitized failure artifact exceeds one MiB") + } + for _, forbidden := range []string{"kubeconfig", "BEGIN PRIVATE KEY", "Bearer ", "grant-key", "ownerSubject", "clientHash", "Secret/data"} { + if strings.Contains(string(artifact), forbidden) { + t.Fatalf("sanitized failure artifact contains forbidden class %q", forbidden) + } + } +} diff --git a/e2e/k8s_execution/qualification_test.go b/e2e/k8s_execution/qualification_test.go new file mode 100644 index 0000000000..82ae212915 --- /dev/null +++ b/e2e/k8s_execution/qualification_test.go @@ -0,0 +1,617 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" + + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const namespace = "execution-qualification" + +func TestKindExecutionQualification(t *testing.T) { + state := os.Getenv("MECATL_EXECUTION_QUAL_STATE") + if state == "" || os.Getenv("MECATL_KUBE_CONTEXT") == "" { + t.Fatal("MECATL_EXECUTION_QUAL_STATE and MECATL_KUBE_CONTEXT are required; run task e2e:k8s:execution") + } + kubeconfig := filepath.Join(state, "kubeconfig") + pki := filepath.Join(state, "pki") + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Minute) + defer cancel() + applyMockScript(t, ctx, kubeconfig, "mock-script.json") + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecak8s", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecak8s", "-n", namespace, "--timeout=240s") + + baseline := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev") + providerForward := portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + providerTLS := loadTLS(t, pki, "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") + providerClient, err := executionclient.New(providerForward.addr, providerTLS) + if err != nil { + t.Fatal(err) + } + defer providerClient.Close() + if _, err = providerClient.ValidateProfile(ctx, "go"); err != nil { + t.Fatalf("validate profile: %v", err) + } + if got := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); got != baseline { + t.Fatalf("profile validation changed environment count from %d to %d", baseline, got) + } + + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "alice"} + binding := fmt.Sprintf("direct-binding-%d", time.Now().UnixNano()) + operationID := fmt.Sprintf("direct-ensure-%d", time.Now().UnixNano()) + first, err := providerClient.Ensure(ctx, binding, "go", owner, operationID) + if err != nil { + t.Fatalf("ensure: %v", err) + } + if err := providerClient.CommitReference(ctx, executionenv.ReferenceRequest{Environment: first.Environment, Owner: owner, BindingID: binding, OperationID: operationID}); err != nil { + t.Fatalf("commit reference: %v", err) + } + if first.Environment.ID == "" || first.Environment.Revision == "" { + t.Fatal("ensure returned an empty exact environment reference") + } + ready := waitReady(t, ctx, providerClient, owner, binding, first.Environment) + if os.Getenv("MECATL_EXECUTION_QUAL_PROFILE") == "production" { + assertProductionExecutorPod(t, ctx, kubeconfig, first.Environment.ID) + } + if ready.Environment != first.Environment { + t.Fatalf("ready reference drifted: got %+v want %+v", ready.Environment, first.Environment) + } + second, err := providerClient.Ensure(ctx, binding, "go", owner, operationID) + if err != nil { + t.Fatalf("repeat ensure: %v", err) + } + if second.Environment != first.Environment { + t.Fatalf("repeat ensure changed identity: got %+v want %+v", second.Environment, first.Environment) + } + if got := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); got != baseline+1 { + t.Fatalf("repeat ensure left %d environments, want baseline + 1 (%d)", got, baseline+1) + } + if got := resourceCount(t, ctx, kubeconfig, "pods -l execution.mecatl.dev/environment="+first.Environment.ID); got != 1 { + t.Fatalf("executor pods = %d, want 1", got) + } + if got := resourceCount(t, ctx, kubeconfig, "pvc -l execution.mecatl.dev/environment="+first.Environment.ID); got != 1 { + t.Fatalf("workspace PVCs = %d, want 1", got) + } + + intruderTLS := loadTLS(t, pki, "intruder", "mecatl-execution.execution-qualification.svc.cluster.local") + intruder, err := executionclient.New(providerForward.addr, intruderTLS) + if err != nil { + t.Fatal(err) + } + defer intruder.Close() + _, err = intruder.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: first.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}) + if err == nil || (!isRemoteCode(err, executionenv.CodePermissionDenied) && !isRemoteCode(err, executionenv.CodeNotFound)) { + t.Fatalf("different client attach error = %v, want an existence-hiding denial", err) + } + rc := executionenv.RequestContext{Environment: first.Environment, Owner: owner, BindingID: binding, Epoch: ready.Epoch, Grant: ready.Grant} + for name, call := range map[string]func() error{ + "command status": func() error { + _, err := intruder.CommandStatus(ctx, executionenv.CommandQueryRequest{Context: rc, CommandID: "not-owned"}) + return err + }, + "command cancel": func() error { + _, err := intruder.CancelCommand(ctx, executionenv.CommandQueryRequest{Context: rc, CommandID: "not-owned"}) + return err + }, + "environment retire": func() error { + return intruder.RetireEnvironment(ctx, executionenv.RetireEnvironmentRequest{Environment: rc.Environment, Owner: rc.Owner, ExpectedEpoch: rc.Epoch, ExpectedPodUID: "wrong-pod", ExpectedPVCUID: "wrong-pvc", OperationID: "unauthorized-retire"}) + }, + } { + if err := call(); err == nil || (!isRemoteCode(err, executionenv.CodePermissionDenied) && !isRemoteCode(err, executionenv.CodeUnauthenticated)) { + t.Fatalf("different client %s error = %v, want denial", name, err) + } + } + providerForward.stop() + if _, err := providerClient.ValidateProfile(ctx, "go"); err == nil { + t.Fatal("provider endpoint loss unexpectedly fell back") + } + providerForward = portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + providerTLS = loadTLS(t, pki, "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") + providerClient, err = executionclient.New(providerForward.addr, providerTLS) + if err != nil { + t.Fatal(err) + } + waitProviderRPCReady(t, ctx, providerClient, providerForward.addr, providerTLS) + defer providerClient.Close() + + tokenForward := portForward(t, ctx, kubeconfig, "service/oidc-issuer", 8443) + alice := fixtureToken(t, ctx, tokenForward.addr, pki, "alice") + bob := fixtureToken(t, ctx, tokenForward.addr, pki, "bob") + tokenForward.stop() + agentForward := portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + beforeNoFS := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev") + status, noFSBody := request(t, ctx, http.MethodPost, "http://"+agentForward.addr+"/v1/sessions", alice, []byte(`{"profile":"no-fs","mode":"default"}`)) + if status != http.StatusCreated { + t.Fatalf("create no-fs session status=%d body=%s", status, noFSBody) + } + if afterNoFS := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); afterNoFS != beforeNoFS { + t.Fatalf("no-fs session allocated execution resources: before=%d after=%d", beforeNoFS, afterNoFS) + } + sessionID := createSession(t, ctx, agentForward.addr, alice) + body := prompt(t, ctx, agentForward.addr, sessionID, alice, "run the scripted remote qualification") + if err := os.WriteFile(filepath.Join(state, "mock-journey.sse"), body, 0o600); err != nil { + t.Fatal(err) + } + assertMockJourney(t, body) + lookup := environmentForBinding(t, ctx, kubeconfig, sessionID) + attached := waitReady(t, ctx, providerClient, owner, sessionID, lookup) + rc, releaseVerification := acquireRun(t, ctx, providerClient, owner, sessionID, attached, fmt.Sprintf("independent-verify-%d", time.Now().UnixNano())) + proof, err := providerClient.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileRead, Path: "proof.txt"}) + if err != nil || string(proof.Data) != "beta\n" { + t.Fatalf("independent file proof failed: err=%v", err) + } + verification, err := providerClient.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go test ./...", TimeoutMillis: 120000}) + if err != nil || verification.State != executionenv.CommandSucceeded || verification.Result.ExitCode != 0 { + t.Fatalf("independent go test proof failed: err=%v state=%s", err, verification.State) + } + releaseVerification() + qualifyRemoteFileTools(t, ctx, providerClient, owner, sessionID, lookup) + if status := getSession(t, ctx, agentForward.addr, sessionID, bob); status != http.StatusNotFound { + t.Fatalf("different OIDC owner read status = %d, want 404", status) + } + agentForward.stop() + + applyReattachScript(t, ctx, kubeconfig, state) + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecatl-execution", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecatl-execution", "-n", namespace, "--timeout=180s") + providerForward = portForward(t, ctx, kubeconfig, "service/mecatl-execution", 8443) + providerTLS = loadTLS(t, pki, "mecak8s", "mecatl-execution.execution-qualification.svc.cluster.local") + providerClient, err = executionclient.New(providerForward.addr, providerTLS) + if err != nil { + t.Fatal(err) + } + waitProviderRPCReady(t, ctx, providerClient, providerForward.addr, providerTLS) + defer providerClient.Close() + if refreshed := waitReady(t, ctx, providerClient, owner, sessionID, lookup); refreshed.Environment != lookup { + t.Fatal("provider did not reattach the exact environment after restart") + } + runKubectl(t, ctx, kubeconfig, "rollout", "restart", "deployment/mecak8s", "-n", namespace) + runKubectl(t, ctx, kubeconfig, "rollout", "status", "deployment/mecak8s", "-n", namespace, "--timeout=240s") + agentForward = portForward(t, ctx, kubeconfig, "service/mecak8s", 8081) + defer agentForward.stop() + body = promptEventually(t, ctx, agentForward.addr, sessionID, alice, "verify exact reattachment after both deployments restarted") + for _, marker := range []string{"reattach-read", "reattach-shell", "beta", "REMOTE_EXECUTION_REATTACH_COMPLETE", "ok"} { + if !bytes.Contains(body, []byte(marker)) { + t.Fatalf("reattach SSE omitted %q", marker) + } + } + if bytes.Contains(body, []byte(`"is_error":true`)) { + t.Fatal("reattached remote operation contained an error result") + } + if got := resourceCount(t, ctx, kubeconfig, "executionenvironments.execution.mecatl.dev"); got != baseline+2 { + t.Fatalf("post-restart environments = %d, want baseline + direct + session (%d)", got, baseline+2) + } +} + +func assertMockJourney(t *testing.T, body []byte) { + t.Helper() + expected := map[string]struct { + name string + key string + want string + }{ + "write-proof": {"Write", "path", "proof.txt"}, "read-before-edit": {"Read", "path", "proof.txt"}, + "edit-proof": {"Edit", "path", "proof.txt"}, "copy-proof": {"Copy", "destination", "copy.txt"}, + "move-proof": {"Move", "destination", "moved.txt"}, "glob-proof": {"Glob", "pattern", "*.txt"}, + "grep-proof": {"Grep", "pattern", "beta"}, "write-mod": {"Write", "path", "go.mod"}, + "write-test": {"Write", "path", "remote_test.go"}, "shell-go-test": {"Shell", "command", "go test ./..."}, + "remove-moved": {"Remove", "path", "moved.txt"}, "read-final": {"Read", "path", "proof.txt"}, + } + calls := map[string]string{} + results := map[string]*mecatlv1.ToolResult{} + stop := "" + for _, line := range bytes.Split(body, []byte("\n")) { + if !bytes.HasPrefix(line, []byte("data: ")) { + continue + } + var ev mecatlv1.Event + if err := json.Unmarshal(bytes.TrimPrefix(line, []byte("data: ")), &ev); err != nil { + t.Fatalf("decode harness SSE event: %v", err) + } + if ev.ToolCall != nil { + want, ok := expected[ev.ToolCall.Id] + if !ok { + t.Fatalf("unexpected tool call id %q", ev.ToolCall.Id) + } + if ev.ToolCall.Name != want.name { + t.Fatalf("tool call %s name=%s want=%s", ev.ToolCall.Id, ev.ToolCall.Name, want.name) + } + var args map[string]json.RawMessage + var got string + if json.Unmarshal([]byte(ev.ToolCall.Args), &args) != nil || json.Unmarshal(args[want.key], &got) != nil || got != want.want { + t.Fatalf("tool call %s did not carry expected structured argument", ev.ToolCall.Id) + } + calls[ev.ToolCall.Id] = ev.ToolCall.Name + } + if ev.ToolResult != nil { + results[ev.ToolResult.CallId] = ev.ToolResult + } + if ev.Result != nil { + stop = ev.Result.Stop + } + } + for id := range expected { + if calls[id] == "" { + t.Fatalf("missing structured tool call %s (%s)", id, mockSSEStatus(body)) + } + result := results[id] + if result == nil || result.IsError { + t.Fatalf("tool call %s has no correlated successful result", id) + } + } + if result := results["shell-go-test"]; !strings.Contains(result.Content, "[exit code: 0]") { + t.Fatal("model-issued exact go test did not return exit code 0") + } + if result := results["read-final"]; !strings.Contains(result.Content, "beta") { + t.Fatal("final Read result did not contain persisted proof") + } + if stop != "end_turn" { + t.Fatalf("mock harness stop=%q, want end_turn", stop) + } +} + +func mockSSEStatus(body []byte) string { + var names []string + errorClasses := map[string]int{} + stop := "missing" + for _, line := range bytes.Split(body, []byte("\n")) { + if !bytes.HasPrefix(line, []byte("data: ")) { + continue + } + var ev mecatlv1.Event + if json.Unmarshal(bytes.TrimPrefix(line, []byte("data: ")), &ev) != nil { + continue + } + if ev.ToolCall != nil { + names = append(names, ev.ToolCall.Name) + } + if ev.ToolResult != nil && ev.ToolResult.IsError { + errorClasses[classifyMockToolError(ev.ToolResult.Content)]++ + } + if ev.Result != nil { + stop = ev.Result.Stop + } + } + return fmt.Sprintf("tools=%s error_classes=%v stop=%s", strings.Join(names, ","), errorClasses, stop) +} + +func classifyMockToolError(content string) string { + content = strings.ToLower(content) + for _, class := range []string{"grant", "permission", "owner", "stale", "unavailable", "not found", "version", "placement", "environment", "connection", "timeout", "binding", "argument", "request", "profile", "reference", "invalid", "read before", "already exists"} { + if strings.Contains(content, class) { + return strings.ReplaceAll(class, " ", "_") + } + } + return "other_redacted" +} + +func acquireRun(t *testing.T, ctx context.Context, c *executionclient.Client, owner executionenv.Owner, binding string, attached executionenv.AttachEnvironmentResponse, runID string) (executionenv.RequestContext, func()) { + t.Helper() + operationID := "acquire-" + runID + claim, err := c.AcquireRun(ctx, executionenv.RunClaimRequest{Environment: attached.Environment, Owner: owner, BindingID: binding, RunID: runID, OperationID: operationID, TTL: time.Minute}) + if err != nil { + t.Fatalf("acquire run: %v", err) + } + rc := executionenv.RequestContext{Environment: claim.Environment, Owner: owner, BindingID: binding, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant} + release := func() { + err := c.ReleaseRun(context.WithoutCancel(ctx), executionenv.RunClaimRequest{Environment: claim.Environment, Owner: owner, BindingID: binding, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, OperationID: "release-" + runID}) + if err != nil { + t.Errorf("release run: %v", err) + } + } + return rc, release +} + +func waitReady(t *testing.T, ctx context.Context, c *executionclient.Client, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef) executionenv.AttachEnvironmentResponse { + t.Helper() + deadline := time.Now().Add(3 * time.Minute) + lastCode := "none" + lastReady := false + for time.Now().Before(deadline) { + out, err := c.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}) + lastCode, lastReady = remoteErrorCode(err), out.Ready + if err == nil && out.Ready { + return out + } + time.Sleep(500 * time.Millisecond) + } + t.Fatalf("execution environment %q did not become ready: last_code=%s last_ready=%t", ref.ID, lastCode, lastReady) + return executionenv.AttachEnvironmentResponse{} +} +func isRemoteCode(err error, code executionenv.ErrorCode) bool { + var remote *executionenv.Error + return errors.As(err, &remote) && remote.Code == code +} + +func remoteErrorCode(err error) string { + if err == nil { + return "none" + } + var remote *executionenv.Error + if errors.As(err, &remote) && remote.Code.Valid() { + return string(remote.Code) + } + return "transport" +} + +func waitProviderRPCReady(t *testing.T, ctx context.Context, c *executionclient.Client, addr string, cfg *tls.Config) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + var lastErr error + for time.Now().Before(deadline) { + attemptCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + _, lastErr = c.ValidateProfile(attemptCtx, "go") + cancel() + if lastErr == nil { + return + } + time.Sleep(250 * time.Millisecond) + } + t.Fatalf("provider RPC did not become ready: code=%s tls=%s", remoteErrorCode(lastErr), classifyTLSFailure(addr, cfg)) +} + +func classifyTLSFailure(addr string, cfg *tls.Config) string { + dialer := &net.Dialer{Timeout: 2 * time.Second} + conn, err := tls.DialWithDialer(dialer, "tcp", addr, cfg.Clone()) + if err == nil { + _ = conn.Close() + return "ok" + } + var unknown x509.UnknownAuthorityError + if errors.As(err, &unknown) { + return "unknown-ca" + } + var invalid x509.CertificateInvalidError + if errors.As(err, &invalid) { + if invalid.Reason == x509.Expired { + return "expired" + } + return "bad-certificate" + } + var hostname x509.HostnameError + if errors.As(err, &hostname) { + return "bad-certificate" + } + if errors.Is(err, syscall.ECONNREFUSED) { + return "dial-refused" + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return "dial-timeout" + } + return "handshake-failed" +} + +func loadTLS(t *testing.T, dir, name, serverName string) *tls.Config { + t.Helper() + ca := filepath.Join(dir, "provider-roots.pem") + if _, err := os.Stat(ca); errors.Is(err, os.ErrNotExist) { + ca = filepath.Join(dir, "ca.crt") + } else if err != nil { + t.Fatal("inspect synthetic provider trust bundle") + } + cfg, err := executionclient.LoadTLSConfig(executionclient.TLSFiles{CA: ca, Cert: filepath.Join(dir, name+".crt"), Key: filepath.Join(dir, name+".key")}) + if err != nil { + t.Fatal(err) + } + cfg.ServerName = serverName + return cfg +} + +type forward struct { + addr string + cmd *exec.Cmd + stopOnce sync.Once +} + +func (f *forward) stop() { + if f == nil { + return + } + f.stopOnce.Do(func() { + if f.cmd == nil || f.cmd.Process == nil { + return + } + _ = f.cmd.Process.Signal(os.Interrupt) + _, _ = f.cmd.Process.Wait() + }) +} +func portForward(t *testing.T, ctx context.Context, kubeconfig, target string, remote int) *forward { + t.Helper() + local := freePort(t) + cmd := command(ctx, kubeconfig, "port-forward", "-n", namespace, target, fmt.Sprintf("%d:%d", local, remote)) + var stderr bytes.Buffer + cmd.Stdout = io.Discard + cmd.Stderr = &stderr + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + f := &forward{addr: fmt.Sprintf("127.0.0.1:%d", local), cmd: cmd} + t.Cleanup(f.stop) + deadline := time.Now().Add(30 * time.Second) + for time.Now().Before(deadline) { + conn, err := net.DialTimeout("tcp", f.addr, 200*time.Millisecond) + if err == nil { + _ = conn.Close() + return f + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("port-forward %s did not start: %s", target, stderr.String()) + return nil +} +func freePort(t *testing.T) int { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + return ln.Addr().(*net.TCPAddr).Port +} + +func cleanEnv() []string { + return []string{"HOME=" + os.Getenv("HOME"), "PATH=" + os.Getenv("PATH")} +} +func command(ctx context.Context, kubeconfig string, args ...string) *exec.Cmd { + contextName := os.Getenv("MECATL_KUBE_CONTEXT") + fullArgs := append([]string{"--kubeconfig", kubeconfig, "--context", contextName}, args...) + cmd := exec.CommandContext(ctx, "kubectl", fullArgs...) + cmd.Env = cleanEnv() + return cmd +} +func runKubectl(t *testing.T, ctx context.Context, kubeconfig string, args ...string) []byte { + t.Helper() + out, err := command(ctx, kubeconfig, args...).CombinedOutput() + if err != nil { + t.Fatalf("kubectl %s failed: %v: %s", args[0], err, out) + } + return out +} +func resourceCount(t *testing.T, ctx context.Context, kubeconfig, spec string) int { + t.Helper() + args := append([]string{"get"}, strings.Fields(spec)...) + args = append(args, "-n", namespace, "-o", "name") + out := runKubectl(t, ctx, kubeconfig, args...) + if len(bytes.TrimSpace(out)) == 0 { + return 0 + } + return len(bytes.Split(bytes.TrimSpace(out), []byte{'\n'})) +} + +func fixtureToken(t *testing.T, ctx context.Context, addr, pki, sub string) string { + t.Helper() + ca, err := os.ReadFile(filepath.Join(pki, "ca.crt")) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(ca) { + t.Fatal("fixture CA invalid") + } + tr := &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: pool, ServerName: "oidc-issuer.execution-qualification.svc.cluster.local"}} + req, _ := http.NewRequestWithContext(ctx, http.MethodGet, "https://"+addr+"/token?sub="+sub, nil) + resp, err := (&http.Client{Transport: tr}).Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out struct { + IDToken string `json:"id_token"` + } + if resp.StatusCode != 200 || json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&out) != nil || out.IDToken == "" { + t.Fatal("fixture token issuance failed") + } + return out.IDToken +} +func createSession(t *testing.T, ctx context.Context, addr, token string) string { + t.Helper() + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions", token, []byte(`{"mode":"default","limits":{"max_turns":16,"max_tool_calls":24,"max_consecutive_failures":3}}`)) + if status != http.StatusCreated { + t.Fatalf("create session status=%d body=%s", status, body) + } + var out struct { + SessionID string `json:"session_id"` + } + if json.Unmarshal(body, &out) != nil || out.SessionID == "" { + t.Fatal("create session returned no id") + } + return out.SessionID +} +func prompt(t *testing.T, ctx context.Context, addr, id, token, text string) []byte { + t.Helper() + raw, _ := json.Marshal(map[string]string{"text": text}) + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+id+"/prompt", token, raw) + if status/100 != 2 { + t.Fatalf("prompt status=%d body=%s", status, body) + } + return body +} +func promptEventually(t *testing.T, ctx context.Context, addr, id, token, text string) []byte { + t.Helper() + raw, _ := json.Marshal(map[string]string{"text": text}) + deadline := time.Now().Add(90 * time.Second) + for { + status, body := request(t, ctx, http.MethodPost, "http://"+addr+"/v1/sessions/"+id+"/prompt", token, raw) + if status/100 == 2 { + return body + } + if status != http.StatusConflict || time.Now().After(deadline) { + t.Fatalf("reattach prompt status=%d body=%s", status, body) + } + time.Sleep(time.Second) + } +} +func getSession(t *testing.T, ctx context.Context, addr, id, token string) int { + t.Helper() + status, _ := request(t, ctx, http.MethodGet, "http://"+addr+"/v1/sessions/"+id, token, nil) + return status +} +func request(t *testing.T, ctx context.Context, method, url, token string, body []byte) (int, []byte) { + t.Helper() + req, _ := http.NewRequestWithContext(ctx, method, url, bytes.NewReader(body)) + req.Header.Set("Authorization", "Bearer "+token) + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("Accept", "text/event-stream") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + raw, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) + if err != nil { + t.Fatal(err) + } + return resp.StatusCode, raw +} +func applyReattachScript(t *testing.T, ctx context.Context, kubeconfig, state string) { + t.Helper() + root := filepath.Clean(filepath.Join(state, "..", "..", "..")) + _ = root + applyMockScript(t, ctx, kubeconfig, "mock-script-reattach.json") +} +func applyMockScript(t *testing.T, ctx context.Context, kubeconfig, filename string) { + t.Helper() + source := filepath.Join(repoRoot(t), "deploy", "mecatl-execution-kind", filename) + cmd := command(ctx, kubeconfig, "create", "configmap", "execution-mock", "-n", namespace, "--from-file=mock-script.json="+source, "--dry-run=client", "-o", "yaml") + rendered, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + apply := command(ctx, kubeconfig, "apply", "-f", "-") + apply.Stdin = bytes.NewReader(rendered) + if out, err := apply.CombinedOutput(); err != nil { + t.Fatalf("update mock script: %v: %s", err, out) + } +} +func repoRoot(t *testing.T) string { + t.Helper() + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + return filepath.Clean(filepath.Join(wd, "..", "..")) +} diff --git a/e2e/k8s_execution/zz_production_helm_lifetime_test.go b/e2e/k8s_execution/zz_production_helm_lifetime_test.go new file mode 100644 index 0000000000..b1c68a012d --- /dev/null +++ b/e2e/k8s_execution/zz_production_helm_lifetime_test.go @@ -0,0 +1,565 @@ +//go:build kind_execution_e2e + +package k8s_execution_test + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + + "github.com/stacklok/mecatl/internal/adapter/executionclient" + "github.com/stacklok/mecatl/internal/executionenv" +) + +// Last in the serial production suite: rotation and migration share this +// throwaway release. Always restore its current authority, never the initial one. +func TestKindExecutionProductionHelmLifetime(t *testing.T) { + state, kubeconfig, ctx, cancel := requireProduction(t) + defer cancel() + requireOwnedHelmFixture(t, state, kubeconfig) + client, _ := productionClient(t, ctx, state, kubeconfig) + owner, binding, attached := createProductionEnvironment(t, ctx, client, "helm-lifetime") + rc, release := acquireRun(t, ctx, client, owner, binding, attached, "helm-lifetime-write") + probe := []byte("package main\nimport (\"net\";\"os\";\"time\")\nfunc main(){c,e:=net.DialTimeout(\"tcp\",os.Args[1],2*time.Second);if e!=nil{os.Exit(42)};c.Close()}\n") + for path, data := range map[string][]byte{"helm-sentinel": []byte("retained-helm-data\n"), "helm-probe.go": probe} { + if _, err := client.File(ctx, executionenv.FileRequest{Context: rc, Operation: executionenv.OpFileCreate, Path: path, Data: data}); err != nil { + t.Fatal("create lifetime workspace data:", remoteErrorCode(err)) + } + } + built, err := client.StartCommand(ctx, executionenv.CommandStartRequest{Context: rc, Command: "go build -o helm-probe helm-probe.go", TimeoutMillis: 30000}) + if err != nil || built.State != executionenv.CommandSucceeded || built.Result.ExitCode != 0 { + t.Fatal("build lifetime probe failed") + } + assertRetiredFixtureKeyDenied(ctx, t, client, state, rc, "helm-sentinel", "retained-helm-data\n") + release() + // Occupy the single-slot profile; its reservation must still constrain Ensure + // after adoption, rather than only matching a ConfigMap annotation. + quotaBinding := fmt.Sprintf("helm-quota-%d", time.Now().UnixNano()) + quota, err := client.Ensure(ctx, quotaBinding, "quota-cas", owner, "ensure-"+quotaBinding) + if err != nil { + t.Fatal("reserve lifetime capacity:", remoteErrorCode(err)) + } + if err := client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: quota.Environment, Owner: owner, BindingID: quotaBinding, OperationID: "ensure-" + quotaBinding}); err != nil { + t.Fatal("commit lifetime capacity:", remoteErrorCode(err)) + } + waitReady(t, ctx, client, owner, quotaBinding, quota.Environment) + + before := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + envUID := kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") + pod := kubeValue(t, ctx, kubeconfig, "get", "pods", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[0].metadata.name}") + if before.PVCUID != kubeValue(t, ctx, kubeconfig, "get", "pvc", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[*].metadata.uid}") || before.PodUID != kubeValue(t, ctx, kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "jsonpath={.metadata.uid}") { + t.Fatal("runtime status does not name the actual PVC/Pod UIDs") + } + fixtureIP := kubeValue(t, ctx, kubeconfig, "get", "pod/network-fixture", "-n", namespace, "-o", "jsonpath={.status.podIP}") + peerIP := kubeValue(t, ctx, kubeconfig, "get", "pod/network-intruder", "-n", namespace, "-o", "jsonpath={.status.podIP}") + + // Only nonsecret chart values and manifests are captured. No Helm manifest or + // Secret dump is written to artifacts; command errors report status only. + raw, err := helmLifetime(ctx, kubeconfig, "get", "values", "mecatl-execution", "-o", "json") + if err != nil { + t.Fatal("read owned release values:", err) + } + var values map[string]any + if err := json.Unmarshal(raw, &values); err != nil { + t.Fatal("decode release values") + } + current := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-security-manifest") + oldManifest := current.Data["manifest.json"] + var manifest map[string]any + if err := json.Unmarshal([]byte(oldManifest), &manifest); err != nil { + t.Fatal("decode nonsecret authority manifest") + } + generation, ok := manifest["generation"].(float64) + if !ok || generation < 1 { + t.Fatal("missing authority generation") + } + manifest["generation"] = generation + 1 + newManifest, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + provider, ok := values["provider"].(map[string]any) + if !ok { + t.Fatal("missing provider values") + } + provider["securityManifest"] = string(newManifest) + dir, err := os.MkdirTemp(state, "helm-lifetime-") + if err != nil { + t.Fatal(err) + } + valuesPath := filepath.Join(dir, "values.json") + raw, err = json.Marshal(values) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(valuesPath, raw, 0o600); err != nil { + t.Fatal(err) + } + chart := filepath.Join(repoRoot(t), "deploy/helm/mecatl-execution") + restore := func(ctx context.Context) error { + if err := quiesceLifetimeProvider(ctx, kubeconfig); err != nil { + return err + } + _, err := helmLifetime(ctx, kubeconfig, "upgrade", "--install", "mecatl-execution", chart, "-f", valuesPath, "--wait", "--timeout=4m") + return err + } + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 7*time.Minute) + defer cleanupCancel() + requireOwnedHelmFixture(t, state, kubeconfig) + if err := restore(cleanupCtx); err != nil { + t.Error("restore owned release/current authority failed:", err) + } + }) + capacityBefore := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-profile-allocations") + authorityBefore := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-security-authority") + var priorHistory struct { + Fingerprints map[string]string `json:"fingerprints"` + } + if err := json.Unmarshal([]byte(authorityBefore.Data["state.json"]), &priorHistory); err != nil || len(priorHistory.Fingerprints) == 0 { + t.Fatal("missing pre-upgrade authority history") + } + policiesBefore := lifetimePolicies(ctx, t, kubeconfig) + liveLedgers := map[string]corev1.ConfigMap{capacityBefore.Name: capacityBefore, authorityBefore.Name: authorityBefore, current.Name: current} + liveLedgers["mecatl-execution-profiles"] = lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-profiles") + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + if _, err := helmLifetime(ctx, kubeconfig, "upgrade", "mecatl-execution", chart, "-f", valuesPath, "--wait", "--timeout=4m"); !errors.Is(err, errLifetimeNotQuiesced) { + t.Fatal("live same-release upgrade did not reject provider writers with the quiescence error") + } + assertLifetimeRetained(ctx, t, kubeconfig, liveLedgers, policiesBefore) + if status := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID); status != before || envUID != kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") || before.PodUID != kubeValue(t, ctx, kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "jsonpath={.metadata.uid}") || before.PVCUID != kubeValue(t, ctx, kubeconfig, "get", "pvc", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[*].metadata.uid}") { + t.Fatal("rejected live upgrade changed runtime identity or status") + } + // Quiesce all writers before lookup snapshots are rendered into the upgrade. + if err := restore(ctx); err != nil { + t.Fatal("compatible Helm upgrade failed:", err) + } + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + ledgers := map[string]corev1.ConfigMap{} + for _, name := range []string{"mecatl-execution-security-authority", "mecatl-execution-profile-allocations", "mecatl-execution-profiles", "mecatl-execution-security-manifest"} { + ledgers[name] = lifetimeConfigMap(ctx, t, kubeconfig, name) + } + var highWater struct { + Generation uint64 `json:"generation"` + Fingerprints map[string]string `json:"fingerprints"` + } + if err := json.Unmarshal([]byte(ledgers["mecatl-execution-security-authority"].Data["state.json"]), &highWater); err != nil || highWater.Generation != uint64(generation+1) || !reflect.DeepEqual(highWater.Fingerprints, priorHistory.Fingerprints) { + t.Fatal("upgrade did not preserve key history and publish higher authority") + } + if capacityAfter := ledgers[capacityBefore.Name]; capacityAfter.UID != capacityBefore.UID || !reflect.DeepEqual(capacityAfter.Data, capacityBefore.Data) { + t.Fatal("compatible upgrade changed capacity reservations") + } + policies := lifetimePolicies(ctx, t, kubeconfig) + if len(policies) < 2 { + t.Fatal("workload policies missing before uninstall") + } + if _, err := helmLifetime(ctx, kubeconfig, "uninstall", "mecatl-execution", "--wait", "--timeout=2m"); err != nil { + t.Fatal("uninstall owned release failed:", err) + } + waitResourceAbsent(t, ctx, kubeconfig, "deployment", "mecatl-execution") + assertLifetimeRetained(ctx, t, kubeconfig, ledgers, policies) + after := readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + if after.PVCUID != before.PVCUID || after.PodUID != before.PodUID || before.PVCUID != kubeValue(t, ctx, kubeconfig, "get", "pvc", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[*].metadata.uid}") || before.PodUID != kubeValue(t, ctx, kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "jsonpath={.metadata.uid}") || envUID != kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") { + t.Fatal("uninstall replaced retained runtime identity") + } + // Bypass the absent provider only to observe the *same executor*. Positive + // reachability prevents an all-network-down result masquerading as isolation. + lifetimeProbe(ctx, t, kubeconfig, pod, fixtureIP+":8080", 0) + lifetimeProbe(ctx, t, kubeconfig, pod, peerIP+":8080", 42) + lifetimeProbe(ctx, t, kubeconfig, pod, "10.96.0.1:443", 42) + lifetimeProbe(ctx, t, kubeconfig, pod, "1.1.1.1:443", 42) + data := runKubectl(t, ctx, kubeconfig, "exec", "-n", namespace, pod, "--", "cat", "/workspace/helm-sentinel") + if string(data) != "retained-helm-data\n" { + t.Fatal("retained workspace data changed during provider absence") + } + // An incompatible profile is refused before a new release can mutate anything. + if _, err := helmLifetime(ctx, kubeconfig, "install", "mecatl-execution", chart, "-f", valuesPath, "--set", "profiles.go.maxEnvironments=30", "--dry-run=server"); err == nil { + t.Fatal("incompatible reinstall was accepted") + } + // Refusal must happen without adopting another release or bootstrapping a + // new authority name over this namespace's retained allocations. + for _, args := range [][]string{ + {"install", "foreign-execution", chart, "-f", valuesPath, "--dry-run=server"}, + {"install", "mecatl-execution", chart, "-f", valuesPath, "--set", "fullnameOverride=missing-history", "--dry-run=server"}, + } { + if _, err := helmLifetime(ctx, kubeconfig, args...); err == nil { + t.Fatal("foreign/missing-history reinstall was accepted") + } + } + assertLifetimeRetained(ctx, t, kubeconfig, ledgers, policies) + if _, err := helmLifetime(ctx, kubeconfig, "install", "mecatl-execution", chart, "-f", valuesPath, "--wait", "--timeout=4m"); err != nil { + t.Fatal("same-identity Helm reinstall/adoption failed:", err) + } + assertLifetimeRetained(ctx, t, kubeconfig, ledgers, policies) + reattachedClient, _ := productionClient(t, ctx, state, kubeconfig) + reattached := waitReady(t, ctx, reattachedClient, owner, binding, attached.Environment) + if reattached.Environment != attached.Environment { + t.Fatal("reattach changed exact environment revision") + } + after = readExecutionStatus(t, ctx, kubeconfig, attached.Environment.ID) + if after.PVCUID != before.PVCUID || after.PodUID != before.PodUID || before.PVCUID != kubeValue(t, ctx, kubeconfig, "get", "pvc", "-n", namespace, "-l", "execution.mecatl.dev/environment="+attached.Environment.ID, "-o", "jsonpath={.items[*].metadata.uid}") || before.PodUID != kubeValue(t, ctx, kubeconfig, "get", "pod", pod, "-n", namespace, "-o", "jsonpath={.metadata.uid}") || envUID != kubeValue(t, ctx, kubeconfig, "get", "executionenvironment", attached.Environment.ID, "-n", namespace, "-o", "jsonpath={.metadata.uid}") { + t.Fatal("adoption replaced retained runtime identity") + } + rc, release = acquireRun(t, ctx, reattachedClient, owner, binding, reattached, "helm-lifetime-read") + waitFileContent(t, ctx, reattachedClient, rc, "helm-sentinel", "retained-helm-data\n", "same-release Helm adoption") + assertRetiredFixtureKeyDenied(ctx, t, reattachedClient, state, rc, "helm-sentinel", "retained-helm-data\n") + release() + if _, err := reattachedClient.Ensure(ctx, quotaBinding+"-excess", "quota-cas", owner, "ensure-"+quotaBinding+"-excess"); !isRemoteCode(err, executionenv.CodeResourceExhausted) { + t.Fatal("retained capacity did not reject excess allocation:", remoteErrorCode(err)) + } + + // The old manifest still has valid key material, windows, and clients. Only + // its generation is older. An existing connection must fail after reload. + patch, err := json.Marshal(map[string]any{"data": map[string]string{"manifest.json": oldManifest}}) + if err != nil { + t.Fatal(err) + } + runKubectl(t, ctx, kubeconfig, "patch", "configmap/mecatl-execution-security-manifest", "-n", namespace, "--type=merge", "-p", string(patch)) + waitProviderReadyReplicas(t, ctx, kubeconfig, 0) + if _, err := reattachedClient.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}); !isRemoteCode(err, executionenv.CodeNotReady) { + t.Fatal("older still-valid authority did not fail closed with not_ready after reinstall:", remoteErrorCode(err)) + } + authority := lifetimeConfigMap(ctx, t, kubeconfig, "mecatl-execution-security-authority") + if !reflect.DeepEqual(authority.Data, ledgers[authority.Name].Data) { + t.Fatal("stale candidate reset authority history") + } + if err := restore(ctx); err != nil { + t.Fatal("restore current authority:", err) + } + waitProviderReadyReplicas(t, ctx, kubeconfig, 2) + restoredClient, _ := productionClient(t, ctx, state, kubeconfig) + waitReady(t, ctx, restoredClient, owner, binding, attached.Environment) + // Retain the test's data by default, as the chart does; no forced cleanup or + // finalizer removal. The explicit test-cluster owner controls final disposal. +} + +// Quiesced upgrades can outlive a grant/claim. Re-sign the CURRENT claim with +// the fixture's retired k1, before and after adoption: expiry, released claims, +// and stale epochs cannot masquerade as durable authority rejection. +func assertRetiredFixtureKeyDenied(ctx context.Context, t *testing.T, client *executionclient.Client, state string, rc executionenv.RequestContext, path, want string) { + t.Helper() + old := resignFixtureGrant(t, rc, filepath.Join(state, "pki", "grant-key.pem"), "k1", rc.GrantGeneration) + waitFileContent(t, ctx, client, rc, path, want, "current authority before retired-key probe") + if _, err := client.File(ctx, executionenv.FileRequest{Context: old, Operation: executionenv.OpFileRead, Path: path}); !isRemoteCode(err, executionenv.CodePermissionDenied) { + var remote *executionenv.Error + t.Fatalf("retired signing authority rejection: errorCode=%s retryable=%t", remoteErrorCode(err), errors.As(err, &remote) && remote.Retryable) + } + waitFileContent(t, ctx, client, rc, path, want, "current authority after retired-key probe") +} + +func resignFixtureGrant(t *testing.T, rc executionenv.RequestContext, keyPath, keyID string, generation uint64) executionenv.RequestContext { + t.Helper() + parts := strings.Split(rc.Grant, ".") + if len(parts) != 3 { + t.Fatal("invalid current fixture grant") + } + payload, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + t.Fatal("decode current fixture grant") + } + var claims executionenv.GrantClaims + if err := json.Unmarshal(payload, &claims); err != nil { + t.Fatal("decode current fixture claims") + } + material, err := os.ReadFile(keyPath) + if err != nil { + t.Fatal("read test-owned signing key") + } + block, _ := pem.Decode(material) + if block == nil { + t.Fatal("decode test-owned signing key") + } + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + t.Fatal("parse test-owned signing key") + } + private, ok := key.(ed25519.PrivateKey) + if !ok { + t.Fatal("fixture signing key is not Ed25519") + } + claims.KeyID = keyID + claims.GrantGeneration = generation + probe := rc + probe.GrantGeneration = generation + probe.Grant, err = executionenv.SignGrant(private, claims) + if err != nil { + t.Fatal("sign fixture grant") + } + verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{keyID: private.Public().(ed25519.PublicKey)}, Issuer: claims.Issuer, Audience: claims.Audience, MaxLifetime: time.Minute} + if _, err := verifier.Verify(probe.Grant, executionenv.GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: generation, Operation: executionenv.OpFileRead}); err != nil { + t.Fatal("fixture probe is not cryptographically valid and current") + } + return probe +} + +func TestResignFixtureGrantChangesOnlyRequestedAuthority(t *testing.T) { + key := ed25519.NewKeyFromSeed(bytes.Repeat([]byte{7}, ed25519.SeedSize)) + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal("marshal synthetic signing key") + } + keyPath := filepath.Join(t.TempDir(), "synthetic.pem") + if err := os.WriteFile(keyPath, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), 0o600); err != nil { + t.Fatal("write synthetic signing key") + } + now := time.Now().UTC().Truncate(time.Second) + claims := executionenv.GrantClaims{KeyID: "k2", Issuer: "fixture", Audience: "fixture", Client: "fixture-client", OwnerHash: "fixture-owner", BindingID: "binding", RunID: "run", ClaimID: "claim", Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, Epoch: 3, GrantGeneration: 2, Operations: []executionenv.Operation{executionenv.OpFileRead}, NotBefore: now.Add(-time.Second), ExpiresAt: now.Add(30 * time.Second), Nonce: "fixture-nonce"} + grant, err := executionenv.SignGrant(key, claims) + if err != nil { + t.Fatal("sign synthetic control") + } + rc := executionenv.RequestContext{Environment: claims.Environment, Owner: executionenv.Owner{Issuer: "issuer", Subject: "subject"}, BindingID: claims.BindingID, RunID: claims.RunID, ClaimID: claims.ClaimID, Epoch: claims.Epoch, GrantGeneration: claims.GrantGeneration, Grant: grant} + for _, probe := range []struct { + keyID string + generation uint64 + }{{"k2", 1}, {"k1", 2}} { + resigned := resignFixtureGrant(t, rc, keyPath, probe.keyID, probe.generation) + verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{probe.keyID: key.Public().(ed25519.PublicKey)}, Issuer: claims.Issuer, Audience: claims.Audience, MaxLifetime: time.Minute} + got, err := verifier.Verify(resigned.Grant, executionenv.GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: probe.generation, Operation: executionenv.OpFileRead}) + want := claims + want.KeyID, want.GrantGeneration = probe.keyID, probe.generation + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatal("probe changed claims beyond key ID and generation") + } + resigned.Grant, resigned.GrantGeneration = rc.Grant, rc.GrantGeneration + if resigned != rc { + t.Fatal("probe changed request identity") + } + } +} + +func requireOwnedHelmFixture(t *testing.T, state, kubeconfig string) { + t.Helper() + ownership, err := os.ReadFile(filepath.Join(state, "ownership")) + if err != nil { + t.Fatal("lifetime test requires fixture ownership record") + } + fields := map[string]string{} + for line := range strings.SplitSeq(string(ownership), "\n") { + key, value, ok := strings.Cut(line, "=") + if ok { + fields[key] = value + } + } + cluster := fields["cluster"] + if !strings.HasPrefix(cluster, "mecatl-execution-qual-") || fields["context"] != "kind-"+cluster || fields["context"] != os.Getenv("MECATL_KUBE_CONTEXT") || fields["kubeconfig"] != kubeconfig || fields["namespace"] != namespace || fields["profile"] != "production" { + t.Fatal("lifetime test refuses non-owned cluster/context/namespace") + } +} + +func lifetimeConfigMap(ctx context.Context, t *testing.T, kubeconfig, name string) corev1.ConfigMap { + t.Helper() + var cm corev1.ConfigMap + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "configmap", name, "-n", namespace, "-o", "json"), &cm); err != nil { + t.Fatal("decode nonsecret lifetime ConfigMap") + } + if cm.UID == "" || cm.Labels["app.kubernetes.io/managed-by"] != "Helm" || cm.Annotations["meta.helm.sh/release-name"] != "mecatl-execution" || cm.Annotations["meta.helm.sh/release-namespace"] != namespace { + t.Fatal("refuse foreign lifetime ConfigMap", name) + } + return cm +} + +func lifetimePolicies(ctx context.Context, t *testing.T, kubeconfig string) map[string]networkingv1.NetworkPolicy { + t.Helper() + var list networkingv1.NetworkPolicyList + if err := json.Unmarshal(runKubectl(t, ctx, kubeconfig, "get", "networkpolicy", "-n", namespace, "-o", "json"), &list); err != nil { + t.Fatal("decode lifetime policies") + } + result := map[string]networkingv1.NetworkPolicy{} + for _, p := range list.Items { + if p.Name == "mecatl-execution-workload-default-deny" || strings.HasPrefix(p.Name, "mecatl-execution-profile-") { + result[p.Name] = p + } + } + return result +} + +func assertLifetimeRetained(ctx context.Context, t *testing.T, kubeconfig string, cms map[string]corev1.ConfigMap, policies map[string]networkingv1.NetworkPolicy) { + t.Helper() + for name, before := range cms { + after := lifetimeConfigMap(ctx, t, kubeconfig, name) + if before.UID != after.UID || !reflect.DeepEqual(before.Data, after.Data) { + t.Fatal("retained ConfigMap identity/data changed", name) + } + } + live := lifetimePolicies(ctx, t, kubeconfig) + if len(live) != len(policies) { + t.Fatal("retained policy set changed") + } + for name, before := range policies { + after := live[name] + if before.UID != after.UID || !reflect.DeepEqual(before.Spec, after.Spec) { + t.Fatal("retained policy identity/confinement changed", name) + } + } +} + +func lifetimeProbe(ctx context.Context, t *testing.T, kubeconfig, pod, endpoint string, want int) { + t.Helper() + cmd := command(ctx, kubeconfig, "exec", "-n", namespace, pod, "--", "/workspace/helm-probe", endpoint) + cmd.Stdout = io.Discard + cmd.Stderr = io.Discard + err := cmd.Run() + if want == 0 && err == nil { + return + } + var exit *exec.ExitError + if want != 0 && errors.As(err, &exit) && exit.ExitCode() == want { + return + } + t.Fatalf("retained executor network probe did not produce exit %d", want) +} + +func quiesceLifetimeProvider(ctx context.Context, kubeconfig string) error { + cmd := command(ctx, kubeconfig, "get", "deployment/mecatl-execution", "-n", namespace, "--ignore-not-found", "-o", "json") + var out lifetimeOutput + cmd.Stdout = &out + cmd.Stderr = io.Discard + if err := cmd.Run(); err != nil { + return fmt.Errorf("read owned provider deployment: %w", err) + } + if out.Len() == 0 { + return nil + } + var deployment struct { + Metadata struct{ Labels, Annotations map[string]string } + } + if err := json.Unmarshal(out.Bytes(), &deployment); err != nil { + return errors.New("decode owned provider deployment") + } + if deployment.Metadata.Labels["app.kubernetes.io/managed-by"] != "Helm" || deployment.Metadata.Annotations["meta.helm.sh/release-name"] != "mecatl-execution" || deployment.Metadata.Annotations["meta.helm.sh/release-namespace"] != namespace { + return errors.New("refuse to quiesce a foreign provider deployment") + } + if err := command(ctx, kubeconfig, "scale", "deployment/mecatl-execution", "-n", namespace, "--replicas=0").Run(); err != nil { + return fmt.Errorf("quiesce owned provider: %w", err) + } + if err := command(ctx, kubeconfig, "wait", "--for=delete", "pod", "-n", namespace, "-l", "app.kubernetes.io/name=mecatl-execution", "--timeout=2m").Run(); err != nil { + return fmt.Errorf("wait for provider quiescence: %w", err) + } + return nil +} + +var errLifetimeNotQuiesced = errors.New("execution provider must be quiesced") + +func helmLifetime(ctx context.Context, kubeconfig string, args ...string) ([]byte, error) { + if helmLifecycleMutation(args) { + versionArgs := []string{"version", "--template", "{{.Version}}"} + cmd := exec.CommandContext(ctx, "helm", versionArgs...) + cmd.Env = cleanEnv() + version, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("detect Helm version: %w", err) + } + major, _, _ := strings.Cut(strings.TrimPrefix(strings.TrimSpace(string(version)), "v"), ".") + if major == "4" { + // Helm 4 defaults chart lifecycle writes to server-side apply. Use + // client-side mode so reviewed external ConfigMap rotation and the + // following quiesced chart update do not contend for field ownership. + args = append(args, "--server-side=false") + } + } + full := append([]string{"--kubeconfig", kubeconfig, "--kube-context", os.Getenv("MECATL_KUBE_CONTEXT"), "--namespace", namespace}, args...) + cmd := exec.CommandContext(ctx, "helm", full...) + cmd.Env = cleanEnv() + var out, stderr lifetimeOutput + cmd.Stdout = &out + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + if bytes.Contains(stderr.Bytes(), []byte("quiesce the execution provider (scale to zero and wait for its Pods to disappear) before upgrading")) { + return nil, errLifetimeNotQuiesced + } + return nil, fmt.Errorf("Helm %s failed: %w (output suppressed)", args[0], err) + } + return out.Bytes(), nil +} + +func helmLifecycleMutation(args []string) bool { + if len(args) == 0 || args[0] != "install" && args[0] != "upgrade" { + return false + } + for _, arg := range args { + if strings.HasPrefix(arg, "--dry-run") { + return false + } + } + return true +} + +func TestHelmLifetimeSelectsClientSideWritesOnlyForHelm4(t *testing.T) { + for _, tc := range []struct { + name string + version string + args []string + wantVersion bool + wantFlag bool + }{ + {name: "Helm 4 upgrade", version: "v4.0.0", args: []string{"upgrade", "mecatl-execution", "chart"}, wantVersion: true, wantFlag: true}, + {name: "Helm 3 install", version: "v3.16.4", args: []string{"install", "mecatl-execution", "chart"}, wantVersion: true}, + {name: "Helm 4 server dry run", version: "v4.0.0", args: []string{"install", "mecatl-execution", "chart", "--dry-run=server"}}, + {name: "Helm 4 read", version: "v4.0.0", args: []string{"get", "values", "mecatl-execution"}}, + {name: "Helm 4 uninstall", version: "v4.0.0", args: []string{"uninstall", "mecatl-execution"}}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + marker := filepath.Join(dir, "calls") + script := fmt.Sprintf("#!/bin/sh\nset -eu\nprintf '%%s\\n' \"$*\" >> %q\nif [ \"$1\" = version ]; then printf '%%s\\n' %q; fi\n", marker, tc.version) + if err := os.WriteFile(filepath.Join(dir, "helm"), []byte(script), 0o700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) + if _, err := helmLifetime(t.Context(), "fixture-kubeconfig", tc.args...); err != nil { + t.Fatal(err) + } + calls, err := os.ReadFile(marker) + if err != nil { + t.Fatal(err) + } + got := string(calls) + if strings.Contains(got, "version --template {{.Version}}") != tc.wantVersion { + t.Fatalf("version probe mismatch: %q", got) + } + if strings.Contains(got, "--server-side=false") != tc.wantFlag { + t.Fatalf("client-side mode mismatch: %q", got) + } + }) + } +} + +func TestLifetimeOutputRejectsOversizedArtifacts(t *testing.T) { + var out lifetimeOutput + if _, err := io.Copy(&out, strings.NewReader(strings.Repeat("x", 1<<20))); err != nil || out.Len() != 1<<20 { + t.Fatal("bounded output did not accept its exact limit") + } + if _, err := io.Copy(&out, strings.NewReader("overflow")); err == nil || out.Len() != 1<<20 { + t.Fatal("output exceeded its bound through io.Copy") + } +} + +type lifetimeOutput struct{ buffer bytes.Buffer } + +func (b *lifetimeOutput) Len() int { return b.buffer.Len() } +func (b *lifetimeOutput) Bytes() []byte { return b.buffer.Bytes() } + +func (b *lifetimeOutput) Write(p []byte) (int, error) { + if b.Len()+len(p) > 1<<20 { + return 0, errors.New("lifetime output exceeds 1 MiB") + } + return b.buffer.Write(p) +} diff --git a/engine/agent/authority_evaluator_test.go b/engine/agent/authority_evaluator_test.go index 70bec40d10..6cedc58805 100644 --- a/engine/agent/authority_evaluator_test.go +++ b/engine/agent/authority_evaluator_test.go @@ -3,6 +3,7 @@ package agent_test import ( "context" "encoding/json" + "errors" "strings" "sync" "sync/atomic" @@ -431,6 +432,54 @@ func TestADR_0233_AuthorityEvaluator_Scenario7_ResourceAttributeIsDerivedWithout }) } +type failingAuthorityWorkspace struct { + tool.Workspace + err error +} + +func (w failingAuthorityWorkspace) AuthorityResourcePath(string) (string, string, error) { + return "", "", w.err +} + +func TestAuthorityResourceResolutionFailureDeniesBeforeEvaluator(t *testing.T) { + for _, tc := range []struct { + name string + err error + }{ + {name: "remote resolver unavailable", err: context.DeadlineExceeded}, + {name: "physical symlink escape", err: errors.New("path is outside workspace")}, + } { + t.Run(tc.name, func(t *testing.T) { + read := &authorityTool{name: "Read"} + evaluator := &recordingAuthorityEvaluator{decision: port.AuthorityDecision{Allowed: true}} + eng := newEngine(agent.Deps{ + LLM: mockllm.New(mockllm.ToolCallTurn(toolCall("read", "Read", `{"path":"escape"}`))), + Catalog: catalogWith(t, read), + AuthorityEvaluator: evaluator, + }) + base := agent.MemEnv("/workspace") + env := tool.MustEnvironment(base.Ref(), failingAuthorityWorkspace{Workspace: base.Workspace(), err: tc.err}, base.ReadLedger(), nil) + sess := authoritySession(t, "Read") + if err := sess.Rehome(env.Ref()); err != nil { + t.Fatal(err) + } + events := drain(eng.Run(context.Background(), sess, env, agent.RunRequest{Text: "read"})) + if read.ran.Load() != 0 || evaluator.calls() != 0 { + t.Fatalf("tool executions=%d evaluator calls=%d, want 0/0", read.ran.Load(), evaluator.calls()) + } + for _, event := range events { + if event.ToolResult != nil && event.ToolResult.CallID == "read" { + if !event.ToolResult.IsError || !strings.Contains(event.ToolResult.Content, "authority resource") { + t.Fatalf("tool result=%+v, want fail-closed authority resolution error", event.ToolResult) + } + return + } + } + t.Fatal("missing fail-closed tool result") + }) + } +} + // sequencedAuthorityEvaluator returns one decision per call in order, so a test // can distinguish the source-resource request from the destination-resource // request in a dual-resource call (Copy/Move). diff --git a/go.work.sum b/go.work.sum index ac0bf8d7d9..487fbbffea 100644 --- a/go.work.sum +++ b/go.work.sum @@ -504,6 +504,7 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.26.0 h1:DPGjXackMpJWH680oGY4lZhYjIameYmR+/6RBdDGmaI= github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/certificate-transparency-go v1.3.3 h1:hq/rSxztSkXN2tx/3jQqF6Xc0O565UQPdHrOWvZwybo= github.com/google/certificate-transparency-go v1.3.3/go.mod h1:iR17ZgSaXRzSa5qvjFl8TnVD5h8ky2JMVio+dzoKMgA= diff --git a/internal/adapter/executionclient/client.go b/internal/adapter/executionclient/client.go new file mode 100644 index 0000000000..ac8a664010 --- /dev/null +++ b/internal/adapter/executionclient/client.go @@ -0,0 +1,1199 @@ +// Package executionclient implements the private mTLS client and placement adapter +// for the independently deployed Kubernetes execution provider. +// +//nolint:revive // Private protocol methods are intentionally explicit across adapter boundaries. +package executionclient + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "net" + "os" + "strings" + "sync" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + grpccredentials "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/creatediag" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + remoteRoot = "/workspace" + authorityResolveRequestTimeout = 10 * time.Second +) + +// TLSFiles names the runtime-only mTLS material mounted into mecak8s. +type TLSFiles struct{ CA, Cert, Key string } + +// LoadTLSConfig loads provider trust and client identity at process startup. +func LoadTLSConfig(files TLSFiles) (*tls.Config, error) { + if files.CA == "" || files.Cert == "" || files.Key == "" { + return nil, errors.New("execution client: CA, certificate, and key are required") + } + ca, err := os.ReadFile(files.CA) + if err != nil { + return nil, fmt.Errorf("execution client: load CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(ca) { + return nil, errors.New("execution client: CA contains no certificates") + } + cert, err := tls.LoadX509KeyPair(files.Cert, files.Key) + if err != nil { + return nil, fmt.Errorf("execution client: load client identity: %w", err) + } + return &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: pool, Certificates: []tls.Certificate{cert}}, nil +} + +// Client is a bounded client for the private execution-provider gRPC API. +type Client struct { + conn *grpc.ClientConn + rpc executionv1.ExecutionProviderServiceClient +} + +// New constructs a production mTLS execution-provider client. Endpoint is a +// plain host:port authority; URI schemes and alternate resolvers are rejected. +func New(endpoint string, tlsConfig *tls.Config) (*Client, error) { + if strings.Contains(endpoint, "://") || strings.ContainsAny(endpoint, "/?#@") { + return nil, errors.New("execution client: endpoint must be host:port") + } + host, port, err := net.SplitHostPort(endpoint) + if err != nil || host == "" || port == "" { + return nil, errors.New("execution client: endpoint must be host:port") + } + if tlsConfig == nil || len(tlsConfig.Certificates) == 0 || tlsConfig.RootCAs == nil { + return nil, errors.New("execution client: production mTLS configuration is required") + } + cfg := tlsConfig.Clone() + if cfg.MinVersion < tls.VersionTLS13 { + cfg.MinVersion = tls.VersionTLS13 + } + conn, err := grpc.NewClient(endpoint, + grpc.WithTransportCredentials(grpccredentials.NewTLS(cfg)), + grpc.WithDisableRetry(), + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(executionenv.MaxMessageBytes), grpc.MaxCallSendMsgSize(executionenv.MaxMessageBytes)), + ) + if err != nil { + return nil, fmt.Errorf("execution client: connect: %w", err) + } + return &Client{conn: conn, rpc: executionv1.NewExecutionProviderServiceClient(conn)}, nil +} + +// Close releases the provider connection. +func (c *Client) Close() { + if c != nil && c.conn != nil { + _ = c.conn.Close() + } +} + +// ValidateProfile validates a provider profile without allocating an environment. +func (c *Client) ValidateProfile(ctx context.Context, profile string) (executionenv.ValidateProfileResponse, error) { + v, err := c.rpc.ValidateProfile(ctx, &executionv1.ValidateProfileRequest{Profile: profile}) + if err != nil { + return executionenv.ValidateProfileResponse{}, decodeError(ctx, err) + } + return executionenv.ValidateProfileResponse{Profile: v.Profile, Digest: v.Digest, Capabilities: v.Capabilities, MaxFileBytes: v.MaxFileBytes, MaxCommandBytes: v.MaxCommandBytes, MaxCommandDurationMillis: v.MaxCommandDurationMillis}, nil +} + +func newOperationID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +func bindOperationID(owner executionenv.Owner, binding, profile string) string { + sum := sha256.Sum256([]byte("bind/v1\x00" + owner.Issuer + "\x00" + owner.Subject + "\x00" + binding + "\x00" + profile)) + return "bind-v1-" + hex.EncodeToString(sum[:]) +} + +// Ensure idempotently allocates or resolves the environment for a binding. +func (c *Client) Ensure(ctx context.Context, binding, profile string, owner executionenv.Owner, operationID string) (executionenv.EnsureEnvironmentResponse, error) { + v, err := c.rpc.EnsureEnvironment(ctx, &executionv1.EnsureEnvironmentRequest{BindingId: binding, Profile: profile, Owner: ownerToProto(owner), OperationId: operationID}) + if err != nil { + return executionenv.EnsureEnvironmentResponse{}, decodeError(ctx, err) + } + return ensureFromProto(v) +} + +// Attach obtains a fresh grant for an exact environment reference. +func (c *Client) Attach(ctx context.Context, req executionenv.AttachEnvironmentRequest) (executionenv.AttachEnvironmentResponse, error) { + v, err := c.rpc.AttachEnvironment(ctx, &executionv1.AttachEnvironmentRequest{Context: contextToProto(req.Context), Purpose: req.Purpose}) + if err != nil { + return executionenv.AttachEnvironmentResponse{}, decodeError(ctx, err) + } + return attachFromProto(v) +} + +// AcquireRun acquires one environment-wide run claim. +func (c *Client) AcquireRun(ctx context.Context, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + v, err := c.rpc.AcquireRun(ctx, &executionv1.AcquireRunRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), BindingId: req.BindingID, RunId: req.RunID, OperationId: req.OperationID, TtlMillis: req.TTL.Milliseconds()}) + if err != nil { + return executionenv.RunClaim{}, decodeError(ctx, err) + } + return runClaimFromProto(v) +} +func (c *Client) RenewRun(ctx context.Context, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + v, err := c.rpc.RenewRun(ctx, &executionv1.RenewRunRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), BindingId: req.BindingID, RunId: req.RunID, ClaimId: req.ClaimID, Epoch: req.Epoch, GrantGeneration: req.GrantGeneration, OperationId: req.OperationID, TtlMillis: req.TTL.Milliseconds()}) + if err != nil { + return executionenv.RunClaim{}, decodeError(ctx, err) + } + return runClaimFromProto(v) +} +func (c *Client) ReleaseRun(ctx context.Context, req executionenv.RunClaimRequest) error { + _, err := c.rpc.ReleaseRun(ctx, &executionv1.ReleaseRunRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), BindingId: req.BindingID, RunId: req.RunID, ClaimId: req.ClaimID, Epoch: req.Epoch, GrantGeneration: req.GrantGeneration, OperationId: req.OperationID}) + return decodeError(ctx, err) +} +func (c *Client) referenceMutation(ctx context.Context, req executionenv.ReferenceRequest, call func(context.Context, *executionv1.ReferenceMutationRequest, ...grpc.CallOption) (*emptypb.Empty, error)) error { + _, err := call(ctx, &executionv1.ReferenceMutationRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), BindingId: req.BindingID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} +func (c *Client) CommitReference(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.CommitReference) +} +func (c *Client) AbortReference(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.AbortReference) +} +func (c *Client) PrepareReferenceDelete(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.PrepareReferenceDelete) +} +func (c *Client) ConfirmReferenceDelete(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.ConfirmReferenceDelete) +} +func (c *Client) CancelReferenceDelete(ctx context.Context, req executionenv.ReferenceRequest) error { + return c.referenceMutation(ctx, req, c.rpc.CancelReferenceDelete) +} +func (c *Client) ReserveSuccessor(ctx context.Context, req executionenv.ReferenceRequest) error { + _, err := c.rpc.ReserveSuccessor(ctx, &executionv1.ReserveSuccessorRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), SourceBindingId: req.SourceBindingID, DestinationBindingId: req.BindingID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} + +func (c *Client) ListReferenceIntents(ctx context.Context, owner executionenv.Owner) ([]executionenv.ReferenceIntent, error) { + return c.listReferenceIntents(ctx, &executionv1.ListReferenceIntentsRequest{Owner: ownerToProto(owner), Limit: 64}) +} + +func (c *Client) FindReferenceIntent(ctx context.Context, owner executionenv.Owner, environment executionenv.EnvironmentRef, binding string) (executionenv.ReferenceIntent, error) { + intents, err := c.listReferenceIntents(ctx, &executionv1.ListReferenceIntentsRequest{Owner: ownerToProto(owner), Limit: 1, Environment: refToProto(environment), BindingId: binding}) + if err != nil { + return executionenv.ReferenceIntent{}, err + } + if len(intents) != 1 { + return executionenv.ReferenceIntent{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference intent not found"} + } + return intents[0], nil +} + +func (c *Client) ListAllReferenceIntents(ctx context.Context) ([]executionenv.ReferenceIntent, error) { + return c.listReferenceIntents(ctx, &executionv1.ListReferenceIntentsRequest{Limit: 64}) +} + +func (c *Client) listReferenceIntents(ctx context.Context, req *executionv1.ListReferenceIntentsRequest) ([]executionenv.ReferenceIntent, error) { + v, err := c.rpc.ListReferenceIntents(ctx, req) + if err != nil { + return nil, decodeError(ctx, err) + } + out := make([]executionenv.ReferenceIntent, 0, len(v.GetIntents())) + for _, intent := range v.GetIntents() { + created, timeErr := checkedTime(intent.GetCreatedAt()) + if timeErr != nil { + return nil, sanitizedProviderError() + } + state := executionenv.ReferenceState(intent.GetState()) + if state != executionenv.ReferencePendingCreate && state != executionenv.ReferencePendingDelete { + return nil, sanitizedProviderError() + } + owner := executionenv.Owner{} + ownerOK := intent.GetOwner() == nil + if intent.GetOwner() != nil { + owner = executionenv.Owner{Issuer: intent.GetOwner().GetIssuer(), Subject: intent.GetOwner().GetSubject()} + ownerOK = owner.Issuer != "" && owner.Subject != "" && len(owner.Issuer) <= executionenv.MaxIdentityBytes && len(owner.Subject) <= executionenv.MaxIdentityBytes + } + if !ownerOK { + return nil, sanitizedProviderError() + } + out = append(out, executionenv.ReferenceIntent{Environment: refFromProto(intent.GetEnvironment()), Owner: owner, BindingID: intent.GetBindingId(), State: state, OperationID: intent.GetOperationId(), SourceBindingID: intent.GetSourceBindingId(), CreatedAt: created}) + } + return out, nil +} + +// File executes one authorized filesystem operation. +func (c *Client) File(ctx context.Context, req executionenv.FileRequest) (executionenv.FileResponse, error) { + if req.Limit < 0 || req.Limit > executionenv.MaxListEntries { + return executionenv.FileResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "invalid file request limit"} + } + v, err := c.rpc.Files(ctx, &executionv1.FileRequest{Context: contextToProto(req.Context), Operation: fileOperationToProto(req.Operation), Path: req.Path, Destination: req.Destination, Pattern: req.Pattern, Data: req.Data, Version: []byte(req.Version), Limit: int32(req.Limit)}) //nolint:gosec // bounded above + if err != nil { + return executionenv.FileResponse{}, decodeError(ctx, err) + } + return fileFromProto(v), nil +} + +// StartCommand executes one foreground command. +func (c *Client) StartCommand(ctx context.Context, req executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + v, err := c.rpc.StartCommand(ctx, &executionv1.CommandStartRequest{Context: contextToProto(req.Context), Command: req.Command, TimeoutMillis: req.TimeoutMillis}) + if err != nil { + return executionenv.CommandStartResponse{}, decodeError(ctx, err) + } + return commandStartFromProto(v) +} + +// CommandStatus queries a command after the request has been authorized. +func (c *Client) CommandStatus(ctx context.Context, req executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + v, err := c.rpc.CommandStatus(ctx, &executionv1.CommandQueryRequest{Context: contextToProto(req.Context), CommandId: req.CommandID, Offset: req.Offset}) + if err != nil { + return executionenv.CommandStatusResponse{}, decodeError(ctx, err) + } + return commandStatusFromProto(v) +} + +// CancelCommand cancels a command after the request has been authorized. +func (c *Client) CancelCommand(ctx context.Context, req executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + v, err := c.rpc.CancelCommand(ctx, &executionv1.CommandQueryRequest{Context: contextToProto(req.Context), CommandId: req.CommandID, Offset: req.Offset}) + if err != nil { + return executionenv.CommandStatusResponse{}, decodeError(ctx, err) + } + return commandStatusFromProto(v) +} + +// ReleaseReference releases one durable binding reference. +func (c *Client) ReleaseReference(ctx context.Context, req executionenv.ReferenceReleaseRequest) error { + _, err := c.rpc.ReleaseReference(ctx, &executionv1.ReleaseReferenceRequest{Context: contextToProto(req.Context)}) + return decodeError(ctx, err) +} + +// ReplaceExecutor requests UID-checked administrative executor replacement. +func (c *Client) ReplaceExecutor(ctx context.Context, req executionenv.RetireEnvironmentRequest) error { + _, err := c.rpc.ReplaceExecutor(ctx, &executionv1.ReplaceExecutorRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), ExpectedExecutionEpoch: req.ExpectedEpoch, ExpectedPodUid: req.ExpectedPodUID, ExpectedPvcUid: req.ExpectedPVCUID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} + +// RetireEnvironment requests administrative retirement. +func (c *Client) RetireEnvironment(ctx context.Context, req executionenv.RetireEnvironmentRequest) error { + _, err := c.rpc.RetireEnvironment(ctx, &executionv1.RetireEnvironmentRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), ExpectedExecutionEpoch: req.ExpectedEpoch, ExpectedPodUid: req.ExpectedPodUID, ExpectedPvcUid: req.ExpectedPVCUID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} + +// DeleteRetiredEnvironment deletes retained storage after retirement proof. +func (c *Client) DeleteRetiredEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, owner executionenv.Owner, expectedPVCUID, operationID string) error { + _, err := c.rpc.DeleteRetiredEnvironment(ctx, &executionv1.DeleteRetiredEnvironmentRequest{Environment: refToProto(ref), Owner: ownerToProto(owner), ExpectedPvcUid: expectedPVCUID, OperationId: operationID}) + return decodeError(ctx, err) +} + +// RevokeEnvironment atomically fences grants and replays a matching operation receipt. +func (c *Client) RevokeEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, owner executionenv.Owner, expectedGeneration uint64, operationID string) (uint64, error) { + out, err := c.rpc.RevokeEnvironment(ctx, &executionv1.RevokeEnvironmentRequest{Environment: refToProto(ref), Owner: ownerToProto(owner), ExpectedGrantGeneration: expectedGeneration, OperationId: operationID}) + if err != nil { + return 0, decodeError(ctx, err) + } + return out.GetGrantGeneration(), nil +} + +// MigrateEnvironment upgrades one explicitly identified legacy schema. +func (c *Client) MigrateEnvironment(ctx context.Context, ref executionenv.EnvironmentRef, owner executionenv.Owner, expectedSchema uint32, podUID, pvcUID, operationID string) error { + _, err := c.rpc.MigrateEnvironment(ctx, &executionv1.MigrateEnvironmentRequest{Environment: refToProto(ref), Owner: ownerToProto(owner), ExpectedSchemaVersion: &expectedSchema, ExpectedPodUid: podUID, ExpectedPvcUid: pvcUID, OperationId: operationID}) + return decodeError(ctx, err) +} + +// RecoverEnvironment clears a fence only after the provider verifies exact terminal evidence. +func (c *Client) RecoverEnvironment(ctx context.Context, req executionenv.RetireEnvironmentRequest) error { + _, err := c.rpc.RecoverEnvironment(ctx, &executionv1.RecoverEnvironmentRequest{Environment: refToProto(req.Environment), Owner: ownerToProto(req.Owner), ExpectedExecutionEpoch: req.ExpectedEpoch, ExpectedPodUid: req.ExpectedPodUID, ExpectedPvcUid: req.ExpectedPVCUID, OperationId: req.OperationID}) + return decodeError(ctx, err) +} + +func decodeError(ctx context.Context, err error) error { + if err == nil { + return nil + } + if ctx != nil && ctx.Err() != nil { + return ctx.Err() + } + st, ok := status.FromError(err) + if !ok || len(st.Details()) != 1 { + return sanitizedProviderError() + } + detail, ok := st.Details()[0].(*executionv1.ErrorDetail) + if !ok { + return sanitizedProviderError() + } + code := executionenv.ErrorCode(detail.Code) + if !code.Valid() || st.Code() != grpcCodeForError(code) || (detail.Retryable && code != executionenv.CodeNotReady && code != executionenv.CodeUnauthenticated) { + return sanitizedProviderError() + } + return &executionenv.Error{Code: code, Message: "execution provider request failed", Retryable: detail.Retryable} +} + +func sanitizedProviderError() error { + return &executionenv.Error{Code: executionenv.CodeInternal, Message: "execution provider request failed"} +} + +func grpcCodeForError(code executionenv.ErrorCode) codes.Code { + switch code { + case executionenv.CodeInvalidArgument: + return codes.InvalidArgument + case executionenv.CodeUnauthenticated: + return codes.Unauthenticated + case executionenv.CodePermissionDenied: + return codes.PermissionDenied + case executionenv.CodeNotFound: + return codes.NotFound + case executionenv.CodeAlreadyExists: + return codes.AlreadyExists + case executionenv.CodeConflict, executionenv.CodeVersionMismatch, executionenv.CodeDirectoryNotEmpty: + return codes.Aborted + case executionenv.CodeNotReady: + return codes.Unavailable + case executionenv.CodeFenceUnknown: + return codes.FailedPrecondition + case executionenv.CodeResourceExhausted: + return codes.ResourceExhausted + default: + return codes.Internal + } +} +func ownerToProto(o executionenv.Owner) *executionv1.Owner { + return &executionv1.Owner{Issuer: o.Issuer, Subject: o.Subject} +} +func refToProto(r executionenv.EnvironmentRef) *executionv1.EnvironmentRef { + return &executionv1.EnvironmentRef{Id: r.ID, Revision: r.Revision} +} +func contextToProto(v executionenv.RequestContext) *executionv1.RequestContext { + return &executionv1.RequestContext{Environment: refToProto(v.Environment), Owner: ownerToProto(v.Owner), BindingId: v.BindingID, RunId: v.RunID, ClaimId: v.ClaimID, Epoch: v.Epoch, GrantGeneration: v.GrantGeneration, Grant: v.Grant} +} +func refFromProto(r *executionv1.EnvironmentRef) executionenv.EnvironmentRef { + return executionenv.EnvironmentRef{ID: r.GetId(), Revision: r.GetRevision()} +} +func runClaimFromProto(v *executionv1.RunClaimResponse) (executionenv.RunClaim, error) { + expiry, err := checkedTime(v.GetExpiresAt()) + if err != nil || v.GetClaimId() == "" || v.GetRunId() == "" || v.GetEpoch() == 0 || v.GetGrantGeneration() == 0 || v.GetGrant() == "" { + return executionenv.RunClaim{}, sanitizedProviderError() + } + return executionenv.RunClaim{Environment: refFromProto(v.GetEnvironment()), BindingID: v.GetBindingId(), RunID: v.GetRunId(), ClaimID: v.GetClaimId(), Epoch: v.GetEpoch(), GrantGeneration: v.GetGrantGeneration(), Grant: v.GetGrant(), ExpiresAt: expiry}, nil +} + +func ensureFromProto(v *executionv1.EnsureEnvironmentResponse) (executionenv.EnsureEnvironmentResponse, error) { + if v == nil { + return executionenv.EnsureEnvironmentResponse{}, sanitizedProviderError() + } + return executionenv.EnsureEnvironmentResponse{Environment: refFromProto(v.GetEnvironment()), Epoch: v.GetEpoch(), Ready: v.GetReady(), GrantGeneration: v.GetGrantGeneration()}, nil +} +func attachFromProto(v *executionv1.AttachEnvironmentResponse) (executionenv.AttachEnvironmentResponse, error) { + if v == nil { + return executionenv.AttachEnvironmentResponse{}, sanitizedProviderError() + } + return executionenv.AttachEnvironmentResponse{Environment: refFromProto(v.GetEnvironment()), Epoch: v.GetEpoch(), Ready: v.GetReady(), GrantGeneration: v.GetGrantGeneration()}, nil +} +func checkedTime(v interface { + CheckValid() error + AsTime() time.Time +}) (time.Time, error) { + if v == nil { + return time.Time{}, errors.New("execution provider omitted timestamp") + } + if err := v.CheckValid(); err != nil { + return time.Time{}, errors.New("execution provider returned invalid timestamp") + } + return v.AsTime(), nil +} +func fileOperationToProto(op executionenv.Operation) executionv1.FileOperation { + switch op { + case executionenv.OpFileRead: + return executionv1.FileOperation_FILE_OPERATION_READ + case executionenv.OpFileResolveAuthority: + return executionv1.FileOperation_FILE_OPERATION_RESOLVE_AUTHORITY + case executionenv.OpFileStat: + return executionv1.FileOperation_FILE_OPERATION_STAT + case executionenv.OpFileCreate: + return executionv1.FileOperation_FILE_OPERATION_CREATE + case executionenv.OpFileReplace: + return executionv1.FileOperation_FILE_OPERATION_REPLACE + case executionenv.OpFileList: + return executionv1.FileOperation_FILE_OPERATION_LIST + case executionenv.OpFileRemove: + return executionv1.FileOperation_FILE_OPERATION_REMOVE + case executionenv.OpFileRename: + return executionv1.FileOperation_FILE_OPERATION_RENAME + case executionenv.OpFileCopy: + return executionv1.FileOperation_FILE_OPERATION_COPY + case executionenv.OpFileGlob: + return executionv1.FileOperation_FILE_OPERATION_GLOB + case executionenv.OpFileGrep: + return executionv1.FileOperation_FILE_OPERATION_GREP + } + return executionv1.FileOperation_FILE_OPERATION_UNSPECIFIED +} +func fileFromProto(v *executionv1.FileResponse) executionenv.FileResponse { + r := executionenv.FileResponse{Data: v.GetData(), Version: string(v.GetVersion()), Paths: v.GetPaths(), AuthorityTarget: v.GetAuthorityTarget(), AuthorityWorkspace: v.GetAuthorityWorkspace()} + if v.Info != nil { + x := fileInfoFromProto(v.Info) + r.Info = &x + } + for _, x := range v.Entries { + r.Entries = append(r.Entries, fileInfoFromProto(x)) + } + for _, x := range v.Matches { + r.Matches = append(r.Matches, executionenv.GrepMatch{Path: x.Path, Line: int(x.Line), Text: x.Text}) + } + return r +} +func fileInfoFromProto(v *executionv1.FileInfo) executionenv.FileInfo { + var mt time.Time + if v.ModTime != nil && v.ModTime.CheckValid() == nil { + mt = v.ModTime.AsTime() + } + return executionenv.FileInfo{Name: v.Name, Size: v.Size, Mode: v.Mode, ModTime: mt, IsDir: v.IsDir} +} +func commandStartFromProto(v *executionv1.CommandStartResponse) (executionenv.CommandStartResponse, error) { + if v == nil { + return executionenv.CommandStartResponse{}, sanitizedProviderError() + } + state, ok := commandStateFromProto(v.GetState()) + if !ok { + return executionenv.CommandStartResponse{}, sanitizedProviderError() + } + result, err := commandStatusFromProto(v.GetResult()) + if err != nil { + return executionenv.CommandStartResponse{}, err + } + return executionenv.CommandStartResponse{CommandID: v.GetCommandId(), State: state, Result: result}, nil +} +func commandStatusFromProto(v *executionv1.CommandStatusResponse) (executionenv.CommandStatusResponse, error) { + if v == nil { + return executionenv.CommandStatusResponse{}, sanitizedProviderError() + } + state, ok := commandStateFromProto(v.State) + if !ok { + return executionenv.CommandStatusResponse{}, sanitizedProviderError() + } + return executionenv.CommandStatusResponse{CommandID: v.CommandId, State: state, ExitCode: int(v.ExitCode), Stdout: v.Stdout, Stderr: v.Stderr, NextOffset: v.NextOffset, Truncated: v.Truncated, TerminalReceipt: v.TerminalReceipt}, nil +} +func commandStateFromProto(v executionv1.CommandState) (executionenv.CommandState, bool) { + switch v { + case executionv1.CommandState_COMMAND_STATE_RUNNING: + return executionenv.CommandRunning, true + case executionv1.CommandState_COMMAND_STATE_SUCCEEDED: + return executionenv.CommandSucceeded, true + case executionv1.CommandState_COMMAND_STATE_FAILED: + return executionenv.CommandFailed, true + case executionv1.CommandState_COMMAND_STATE_CANCELLED: + return executionenv.CommandCancelled, true + case executionv1.CommandState_COMMAND_STATE_FENCE_UNKNOWN: + return executionenv.CommandFenceUnknown, true + default: + return "", false + } +} + +// Provider adapts the execution service to server placement binding. +type Provider struct { + client *Client + profile string +} + +// NewProvider binds a client to one operator-selected profile. +func NewProvider(client *Client, profile string) (*Provider, error) { + if client == nil || profile == "" { + return nil, errors.New("execution client: client and profile are required") + } + return &Provider{client: client, profile: profile}, nil +} + +type referenceDeleteHandle struct { + client *Client + req executionenv.ReferenceRequest +} + +func (h *referenceDeleteHandle) Confirm(ctx context.Context) error { + return h.client.ConfirmReferenceDelete(ctx, h.req) +} +func (h *referenceDeleteHandle) Cancel(ctx context.Context) error { + return h.client.CancelReferenceDelete(ctx, h.req) +} +func (p *Provider) PrepareReferenceDelete(ctx context.Context, req server.PlacementSuccessorRequest) (server.ReferenceDeleteHandle, error) { + if !p.Applies(req.Ref) || req.SourceBindingID == "" { + return nil, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return nil, err + } + op := "" + exact := executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision} + intent, err := p.client.FindReferenceIntent(ctx, owner, exact, string(req.SourceBindingID)) + if err == nil { + if intent.State != executionenv.ReferencePendingDelete || intent.Environment != exact || intent.BindingID != string(req.SourceBindingID) { + return nil, server.ErrPlacementChanged + } + op = intent.OperationID + } else { + var remote *executionenv.Error + if !errors.As(err, &remote) || remote.Code != executionenv.CodeNotFound { + return nil, mapPlacementError(err) + } + } + if op == "" { + op, err = newOperationID() + if err != nil { + return nil, server.ErrPlacementUnavailable + } + } + mutation := executionenv.ReferenceRequest{Environment: executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision}, Owner: owner, BindingID: string(req.SourceBindingID), OperationID: op} + if err := p.client.PrepareReferenceDelete(ctx, mutation); err != nil { + return nil, mapPlacementError(err) + } + return &referenceDeleteHandle{client: p.client, req: mutation}, nil +} + +func (p *Provider) ReserveSuccessor(ctx context.Context, req server.PlacementSuccessorRequest) (server.PlacementBinding, error) { + if !p.Applies(req.Ref) || req.SourceBindingID == "" || req.DestinationBindingID == "" { + return server.PlacementBinding{}, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return server.PlacementBinding{}, err + } + op, err := newOperationID() + if err != nil { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + ref := executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision} + if err := p.client.ReserveSuccessor(ctx, executionenv.ReferenceRequest{Environment: ref, Owner: owner, SourceBindingID: string(req.SourceBindingID), BindingID: string(req.DestinationBindingID), OperationID: op}); err != nil { + return server.PlacementBinding{}, mapPlacementError(err) + } + attached, err := p.client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: string(req.SourceBindingID)}, Purpose: executionenv.PurposeSession}) + if err != nil { + return server.PlacementBinding{}, mapPlacementError(err) + } + base, err := p.binding(owner, string(req.DestinationBindingID), attached) + if err != nil { + return server.PlacementBinding{}, err + } + return p.withExclusiveReferenceTransaction(base, owner, string(req.DestinationBindingID), op), nil +} + +func (p *Provider) Applies(ref session.EnvironmentRef) bool { + return ref.Kind == session.EnvironmentKind("kubernetes") +} + +type runHandle struct { + provider *Provider + owner executionenv.Owner + claim executionenv.RunClaimRequest + credentials *grantContext + environment tool.Environment + mu sync.Mutex +} + +func (h *runHandle) Environment() tool.Environment { return h.environment } +func (h *runHandle) RenewalDeadline() time.Time { + h.mu.Lock() + defer h.mu.Unlock() + return h.credentials.expiry() +} +func (h *runHandle) Renew(ctx context.Context) error { + h.mu.Lock() + defer h.mu.Unlock() + op, err := newOperationID() + if err != nil { + return err + } + h.claim.OperationID = op + claim, err := h.provider.client.RenewRun(ctx, h.claim) + if err != nil { + return err + } + h.claim.Epoch, h.claim.ClaimID = claim.Epoch, claim.ClaimID + h.credentials.replace(claim) + return nil +} +func (h *runHandle) Release(ctx context.Context) error { + h.mu.Lock() + defer h.mu.Unlock() + op, err := newOperationID() + if err != nil { + return err + } + h.claim.OperationID = op + return h.provider.client.ReleaseRun(ctx, h.claim) +} + +func (p *Provider) ListReferenceIntents(ctx context.Context, limit int) ([]server.ReferenceIntent, error) { + if limit <= 0 || limit > 64 { + limit = 64 + } + intents, err := p.client.ListAllReferenceIntents(ctx) + if err != nil { + return nil, mapPlacementError(err) + } + if len(intents) > limit { + intents = intents[:limit] + } + out := make([]server.ReferenceIntent, 0, len(intents)) + for _, intent := range intents { + if intent.Owner.Issuer == "" || intent.Owner.Subject == "" || intent.BindingID == "" || intent.OperationID == "" { + return nil, server.ErrPlacementUnavailable + } + out = append(out, server.ReferenceIntent{ + Ref: toSessionRef(intent.Environment), + Principal: &session.Principal{Issuer: intent.Owner.Issuer, Subject: intent.Owner.Subject}, + BindingID: session.SessionID(intent.BindingID), + SourceBindingID: session.SessionID(intent.SourceBindingID), + OperationID: intent.OperationID, + PendingDelete: intent.State == executionenv.ReferencePendingDelete, + }) + } + return out, nil +} + +func (p *Provider) referenceIntentRequest(intent server.ReferenceIntent) (executionenv.ReferenceRequest, error) { + owner, err := ownerOf(intent.Principal) + if err != nil || !p.Applies(intent.Ref) || intent.BindingID == "" || intent.OperationID == "" { + return executionenv.ReferenceRequest{}, server.ErrInvalidPlacementSelection + } + return executionenv.ReferenceRequest{Environment: executionenv.EnvironmentRef{ID: intent.Ref.ID, Revision: intent.Ref.Revision}, Owner: owner, BindingID: string(intent.BindingID), SourceBindingID: string(intent.SourceBindingID), OperationID: intent.OperationID}, nil +} + +func (p *Provider) CommitReferenceIntent(ctx context.Context, intent server.ReferenceIntent) error { + req, err := p.referenceIntentRequest(intent) + if err != nil { + return err + } + return mapPlacementError(p.client.CommitReference(ctx, req)) +} +func (p *Provider) ConfirmReferenceIntentDelete(ctx context.Context, intent server.ReferenceIntent) error { + req, err := p.referenceIntentRequest(intent) + if err != nil { + return err + } + return mapPlacementError(p.client.ConfirmReferenceDelete(ctx, req)) +} +func (p *Provider) CancelReferenceIntentDelete(ctx context.Context, intent server.ReferenceIntent) error { + req, err := p.referenceIntentRequest(intent) + if err != nil { + return err + } + return mapPlacementError(p.client.CancelReferenceDelete(ctx, req)) +} + +func (p *Provider) AcquireRun(ctx context.Context, req server.ExecutionRunRequest) (server.ExecutionRunHandle, error) { + if !p.Applies(req.Ref) || req.BindingID == "" || req.RunID == "" { + return nil, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return nil, err + } + intents, err := p.client.ListReferenceIntents(ctx, owner) + if err != nil { + return nil, mapPlacementError(err) + } + for _, intent := range intents { + if intent.Environment == (executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision}) && intent.BindingID == string(req.BindingID) && intent.State == executionenv.ReferencePendingCreate { + if err := p.client.CommitReference(ctx, executionenv.ReferenceRequest{Environment: intent.Environment, Owner: owner, BindingID: intent.BindingID, OperationID: intent.OperationID}); err != nil { + return nil, mapPlacementError(err) + } + } + } + op, err := newOperationID() + if err != nil { + return nil, server.ErrPlacementUnavailable + } + claimReq := executionenv.RunClaimRequest{Environment: executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision}, Owner: owner, BindingID: string(req.BindingID), RunID: req.RunID, OperationID: op, TTL: executionenv.DefaultRunTTL} + claim, err := p.client.AcquireRun(ctx, claimReq) + if err != nil { + return nil, mapPlacementError(err) + } + if toSessionRef(claim.Environment) != req.Ref || claim.BindingID != string(req.BindingID) || claim.RunID != req.RunID { + return nil, server.ErrPlacementChanged + } + credentials := &grantContext{client: p.client, context: executionenv.RequestContext{Environment: claim.Environment, Owner: owner, BindingID: claim.BindingID, RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant}, expiresAt: claim.ExpiresAt} + ws := &workspace{credentials: credentials} + runner := &runner{credentials: credentials} + env, envErr := tool.NewEnvironment(req.Ref, ws, memledger.New(), runner) + if envErr != nil { + return nil, server.ErrPlacementUnavailable + } + claimReq.ClaimID, claimReq.Epoch, claimReq.GrantGeneration = claim.ClaimID, claim.Epoch, claim.GrantGeneration + handle := &runHandle{provider: p, owner: owner, claim: claimReq, credentials: credentials, environment: env} + credentials.renewClaim = handle.Renew + return handle, nil +} + +// ValidatePlacement performs side-effect-free profile preflight. +func (p *Provider) ValidatePlacement(ctx context.Context) error { + _, err := p.client.ValidateProfile(ctx, p.profile) + return mapPlacementError(err) +} + +func ownerOf(principal *session.Principal) (executionenv.Owner, error) { + if principal == nil || principal.Issuer == "" || principal.Subject == "" { + return executionenv.Owner{}, server.ErrPlacementNotFound + } + return executionenv.Owner{Issuer: principal.Issuer, Subject: principal.Subject}, nil +} + +// Bind allocates the environment for the final session binding. +func (p *Provider) Bind(ctx context.Context, req server.PlacementBindRequest) (server.PlacementBinding, error) { + if !req.Selector.IsDefault() || req.BindingID == "" { + return server.PlacementBinding{}, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return server.PlacementBinding{}, err + } + operationID := bindOperationID(owner, string(req.BindingID), p.profile) + ensureDone := creatediag.Begin(ctx, "bind_ensure") + ensured, err := p.client.Ensure(ctx, string(req.BindingID), p.profile, owner, operationID) + ensureDone(err) + if err != nil { + return server.PlacementBinding{}, mapPlacementError(err) + } + if ensured.Environment.ID == "" || ensured.Environment.Revision == "" { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + return p.waitForBinding(ctx, owner, string(req.BindingID), ensured.Environment, operationID) +} + +// Reattach exactly resolves a persisted remote environment for its binding. +func (p *Provider) Reattach(ctx context.Context, req server.PlacementReattachRequest) (server.PlacementBinding, error) { + if req.BindingID == "" { + return server.PlacementBinding{}, server.ErrInvalidPlacementSelection + } + owner, err := ownerOf(req.Principal) + if err != nil { + return server.PlacementBinding{}, err + } + ref := executionenv.EnvironmentRef{ID: req.Ref.ID, Revision: req.Ref.Revision} + attached, err := p.client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: string(req.BindingID)}, Purpose: executionenv.PurposeSession}) + if err != nil { + return server.PlacementBinding{}, mapPlacementError(err) + } + if toSessionRef(attached.Environment) != req.Ref { + return server.PlacementBinding{}, server.ErrPlacementChanged + } + return p.binding(owner, string(req.BindingID), attached) +} + +func (p *Provider) waitForBinding(ctx context.Context, owner executionenv.Owner, binding string, ref executionenv.EnvironmentRef, operationID string) (server.PlacementBinding, error) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + calls, last := 0, "begin" + creatediag.Note(ctx, "attach_poll", last, calls) + defer func() { creatediag.Note(ctx, "attach_poll_end", last, calls) }() + for { + calls++ + attached, err := p.client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: binding}, Purpose: executionenv.PurposeSession}) + state := bindingPollState(ctx, attached.Ready, err) + if state != last { + last = state + creatediag.Note(ctx, "attach_poll", state, calls) + } + if err == nil && attached.Ready { + if attached.Environment != ref { + return server.PlacementBinding{}, server.ErrPlacementChanged + } + base, bindErr := p.binding(owner, binding, attached) + if bindErr != nil { + return server.PlacementBinding{}, bindErr + } + return p.withSharedInitialReferenceTransaction(base, owner, binding, operationID), nil + } + if err != nil { + var remote *executionenv.Error + if !errors.As(err, &remote) || (remote.Code != executionenv.CodeNotReady && !remote.Retryable) { + return server.PlacementBinding{}, mapPlacementError(err) + } + } + select { + case <-ctx.Done(): + last = bindingPollState(ctx, false, ctx.Err()) + return server.PlacementBinding{}, server.ErrPlacementUnavailable + case <-ticker.C: + } + } +} + +func bindingPollState(ctx context.Context, ready bool, err error) string { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return "deadline" + } + if ctx.Err() != nil { + return "cancelled" + } + var remote *executionenv.Error + if errors.As(err, &remote) { + if remote.Code == executionenv.CodeNotReady { + if remote.Retryable { + return "not_ready_retryable" + } + return "not_ready_nonretryable" + } + if remote.Retryable { + return "remote_retryable" + } + return "remote_terminal" + } + if err != nil { + return "transport_error" + } + if ready { + return "ready" + } + return "pending" +} + +func (p *Provider) binding(owner executionenv.Owner, binding string, attached executionenv.AttachEnvironmentResponse) (server.PlacementBinding, error) { + if !attached.Ready || attached.Environment.ID == "" || attached.Environment.Revision == "" || attached.Epoch == 0 { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + credentials := &grantContext{client: p.client, context: executionenv.RequestContext{Environment: attached.Environment, Owner: owner, BindingID: binding, Epoch: attached.Epoch, GrantGeneration: attached.GrantGeneration}, expiresAt: time.Time{}} + ws := &workspace{credentials: credentials} + runner := &runner{credentials: credentials} + sref := toSessionRef(attached.Environment) + env, err := tool.NewEnvironment(sref, ws, memledger.New(), runner) + if err != nil { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + return server.PlacementBinding{Environment: env, Ref: sref, Metadata: server.PlacementMetadata{Kind: "kubernetes", Label: "Remote Kubernetes workspace", Revision: attached.Environment.Revision}}, nil +} +func (p *Provider) withSharedInitialReferenceTransaction(binding server.PlacementBinding, owner executionenv.Owner, bindingID, operationID string) server.PlacementBinding { + // Initial Bind reservations are deterministic and may be shared by another + // replica, so local pre-publication cleanup must retain them. + return p.withReferenceTransaction(binding, owner, bindingID, operationID, nil) +} + +func (p *Provider) withExclusiveReferenceTransaction(binding server.PlacementBinding, owner executionenv.Owner, bindingID, operationID string) server.PlacementBinding { + return p.withReferenceTransaction(binding, owner, bindingID, operationID, func(ctx context.Context, req executionenv.ReferenceRequest) error { + return p.client.AbortReference(ctx, req) + }) +} + +func (p *Provider) withReferenceTransaction(binding server.PlacementBinding, owner executionenv.Owner, bindingID, operationID string, abort func(context.Context, executionenv.ReferenceRequest) error) server.PlacementBinding { + ref := executionenv.EnvironmentRef{ID: binding.Ref.ID, Revision: binding.Ref.Revision} + req := executionenv.ReferenceRequest{Environment: ref, Owner: owner, BindingID: bindingID, OperationID: operationID} + var mu sync.Mutex + finalized := false + binding.Commit = func(ctx context.Context) error { + mu.Lock() + defer mu.Unlock() + if finalized { + return nil + } + // Once dispatched, a missing response is ambiguous: the provider may have + // committed. Close must never turn that uncertainty into an abort. + finalized = true + commitDone := creatediag.Begin(ctx, "reference_commit") + err := p.client.CommitReference(ctx, req) + commitDone(err) + return err + } + binding.Close = func() error { + mu.Lock() + defer mu.Unlock() + if finalized { + return nil + } + finalized = true + if abort == nil { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return abort(ctx, req) + } + return binding +} + +func toSessionRef(ref executionenv.EnvironmentRef) session.EnvironmentRef { + return session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: ref.ID, Revision: ref.Revision} +} +func mapPlacementError(err error) error { + if err == nil { + return nil + } + var remote *executionenv.Error + if errors.As(err, &remote) { + switch remote.Code { + case executionenv.CodeNotFound, executionenv.CodePermissionDenied, executionenv.CodeUnauthenticated: + return server.ErrPlacementNotFound + case executionenv.CodeConflict, executionenv.CodeVersionMismatch, executionenv.CodeDirectoryNotEmpty: + return server.ErrPlacementChanged + case executionenv.CodeInvalidArgument: + return server.ErrInvalidPlacementSelection + } + } + return server.ErrPlacementUnavailable +} + +type grantContext struct { + mu sync.Mutex + client *Client + context executionenv.RequestContext + expiresAt time.Time + renewClaim func(context.Context) error +} + +func (g *grantContext) current(_ context.Context) (executionenv.RequestContext, error) { + g.mu.Lock() + defer g.mu.Unlock() + if g.context.Grant == "" || !time.Now().Before(g.expiresAt) { + return executionenv.RequestContext{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "run grant unavailable", Retryable: true} + } + return g.context, nil +} + +func (g *grantContext) refresh(ctx context.Context) (executionenv.RequestContext, error) { + g.mu.Lock() + before := g.context.Grant + if before == "" || !time.Now().Before(g.expiresAt) { + g.mu.Unlock() + return executionenv.RequestContext{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "active run grant expired; retry the run", Retryable: true} + } + renew := g.renewClaim + g.mu.Unlock() + g.mu.Lock() + if g.context.Grant != before && time.Now().Before(g.expiresAt) { + out := g.context + g.mu.Unlock() + return out, nil + } + g.mu.Unlock() + if renew == nil { + return executionenv.RequestContext{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "active run grant cannot be refreshed", Retryable: true} + } + if err := renew(ctx); err != nil { + return executionenv.RequestContext{}, err + } + return g.current(ctx) +} + +func (g *grantContext) expiry() time.Time { + g.mu.Lock() + defer g.mu.Unlock() + return g.expiresAt +} + +func (g *grantContext) replace(claim executionenv.RunClaim) { + g.mu.Lock() + defer g.mu.Unlock() + g.context.RunID, g.context.ClaimID, g.context.Epoch, g.context.GrantGeneration, g.context.Grant = claim.RunID, claim.ClaimID, claim.Epoch, claim.GrantGeneration, claim.Grant + g.expiresAt = claim.ExpiresAt +} + +// workspace is bound to one provider-issued environment grant. +type workspace struct { + credentials *grantContext + opMu sync.Mutex +} + +var _ tool.AuthorityResourceResolver = (*workspace)(nil) + +func (*workspace) Root() string { return remoteRoot } + +// AuthorityResourcePath asks the authenticated executor to derive the same +// physical, confined identity that the subsequent filesystem operation uses. +func (w *workspace) AuthorityResourcePath(path string) (target, root string, err error) { + ctx, cancel := context.WithTimeout(context.Background(), authorityResolveRequestTimeout) + defer cancel() + response, err := w.call(ctx, executionenv.OpFileResolveAuthority, path, "", "", nil, "") + if err != nil { + return "", "", err + } + if response.AuthorityWorkspace != remoteRoot || (response.AuthorityTarget != remoteRoot && !strings.HasPrefix(response.AuthorityTarget, remoteRoot+"/")) { + return "", "", errors.New("execution provider returned an invalid authority resource identity") + } + return response.AuthorityTarget, response.AuthorityWorkspace, nil +} + +func (w *workspace) call(ctx context.Context, op executionenv.Operation, path, dest, pattern string, data []byte, version string) (executionenv.FileResponse, error) { + // The provider's CRD deliberately carries one active operation. The engine may + // dispatch read-only sibling tools concurrently, so serialize this bound + // workspace before claiming that provider-side operation slot. + w.opMu.Lock() + defer w.opMu.Unlock() + credentials, err := w.credentials.current(ctx) + if err != nil { + return executionenv.FileResponse{}, mapFileError(path, err) + } + limit := 0 + if op == executionenv.OpFileList || op == executionenv.OpFileGlob || op == executionenv.OpFileGrep { + limit = executionenv.MaxListEntries + } + request := executionenv.FileRequest{Context: credentials, Operation: op, Path: path, Destination: dest, Pattern: pattern, Data: data, Version: version, Limit: limit} + out, err := w.credentials.client.File(ctx, request) + if isDefinitiveAuthDenial(err) && readOnlyFileOperation(op) { + credentials, refreshErr := w.credentials.refresh(ctx) + if refreshErr != nil { + return executionenv.FileResponse{}, mapFileError(path, refreshErr) + } + request.Context = credentials + out, err = w.credentials.client.File(ctx, request) + } + return out, mapFileError(path, err) +} + +func isDefinitiveAuthDenial(err error) bool { + var remote *executionenv.Error + return errors.As(err, &remote) && remote.Code == executionenv.CodeUnauthenticated && remote.Retryable +} + +func readOnlyFileOperation(op executionenv.Operation) bool { + switch op { + case executionenv.OpFileRead, executionenv.OpFileResolveAuthority, executionenv.OpFileStat, executionenv.OpFileList, executionenv.OpFileGlob, executionenv.OpFileGrep: + return true + default: + return false + } +} +func (w *workspace) Read(ctx context.Context, path string) ([]byte, error) { + r, e := w.call(ctx, executionenv.OpFileRead, path, "", "", nil, "") + return r.Data, e +} +func (w *workspace) ReadVersion(ctx context.Context, path string) ([]byte, tool.FileVersion, error) { + r, e := w.call(ctx, executionenv.OpFileRead, path, "", "", nil, "") + if e != nil { + return nil, tool.FileVersion{}, e + } + return r.Data, tool.DecodeFileVersion(r.Version), nil +} +func (w *workspace) Stat(ctx context.Context, path string) (tool.FileInfo, error) { + r, e := w.call(ctx, executionenv.OpFileStat, path, "", "", nil, "") + if e != nil { + return tool.FileInfo{}, e + } + if r.Info == nil { + return tool.FileInfo{}, errors.New("execution provider omitted file metadata") + } + return fileInfo(*r.Info), nil +} +func (w *workspace) CreateFile(ctx context.Context, path string, data []byte) (tool.FileVersion, error) { + r, e := w.call(ctx, executionenv.OpFileCreate, path, "", "", data, "") + if e != nil { + return tool.FileVersion{}, e + } + return tool.DecodeFileVersion(r.Version), nil +} +func (w *workspace) ReplaceFile(ctx context.Context, path string, old tool.FileVersion, data []byte) (tool.FileVersion, error) { + v, e := tool.EncodeFileVersion(old) + if e != nil { + if _, statErr := w.Stat(ctx, path); statErr != nil { + return tool.FileVersion{}, statErr + } + return tool.FileVersion{}, e + } + r, e := w.call(ctx, executionenv.OpFileReplace, path, "", "", data, v) + if e != nil { + return tool.FileVersion{}, e + } + return tool.DecodeFileVersion(r.Version), nil +} +func (w *workspace) ReadDir(ctx context.Context, path string) ([]tool.FileInfo, error) { + r, e := w.call(ctx, executionenv.OpFileList, path, "", "", nil, "") + if e != nil { + return nil, e + } + out := make([]tool.FileInfo, len(r.Entries)) + for i := range r.Entries { + out[i] = fileInfo(r.Entries[i]) + } + return out, nil +} +func (w *workspace) Remove(ctx context.Context, path string) error { + _, e := w.call(ctx, executionenv.OpFileRemove, path, "", "", nil, "") + return e +} +func (w *workspace) Rename(ctx context.Context, a, b string) error { + _, e := w.call(ctx, executionenv.OpFileRename, a, b, "", nil, "") + return e +} +func (w *workspace) CopyFile(ctx context.Context, a, b string) (tool.FileVersion, error) { + r, e := w.call(ctx, executionenv.OpFileCopy, a, b, "", nil, "") + if e != nil { + return tool.FileVersion{}, e + } + return tool.DecodeFileVersion(r.Version), nil +} +func (w *workspace) Glob(ctx context.Context, pattern string) ([]string, error) { + r, e := w.call(ctx, executionenv.OpFileGlob, "", "", pattern, nil, "") + return r.Paths, e +} +func (w *workspace) Grep(ctx context.Context, pattern, pathGlob string) ([]tool.GrepMatch, error) { + r, e := w.call(ctx, executionenv.OpFileGrep, pathGlob, "", pattern, nil, "") + if e != nil { + return nil, e + } + out := make([]tool.GrepMatch, len(r.Matches)) + for i, m := range r.Matches { + out[i] = tool.GrepMatch{Path: m.Path, Line: m.Line, Text: m.Text} + } + return out, nil +} +func fileInfo(i executionenv.FileInfo) tool.FileInfo { + return tool.FileInfo{Name: i.Name, Size: i.Size, Mode: fs.FileMode(i.Mode), ModTime: i.ModTime, IsDir: i.IsDir} +} +func mapFileError(path string, err error) error { + if err == nil { + return nil + } + var remote *executionenv.Error + if errors.As(err, &remote) { + switch remote.Code { + case executionenv.CodeNotFound: + return fmt.Errorf("%s: %w", path, fs.ErrNotExist) + case executionenv.CodeAlreadyExists: + return fmt.Errorf("%s: %w", path, fs.ErrExist) + case executionenv.CodeVersionMismatch: + return &tool.VersionMismatchError{Path: path} + case executionenv.CodeDirectoryNotEmpty: + return fmt.Errorf("%s: %w", path, tool.ErrDirectoryNotEmpty) + } + } + return err +} + +// runner deliberately implements only foreground CommandRunner. +type runner struct { + credentials *grantContext +} + +func (*runner) BoundWorkspaceRoot() string { return remoteRoot } +func (r *runner) Run(ctx context.Context, command string) (tool.CommandResult, error) { + credentials, err := r.credentials.current(ctx) + if err != nil { + return tool.CommandResult{}, err + } + resp, err := r.credentials.client.StartCommand(ctx, executionenv.CommandStartRequest{Context: credentials, Command: command}) + if err != nil { + return tool.CommandResult{}, err + } + result := resp.Result + out := tool.CommandResult{Stdout: session.ToValidUTF8(string(result.Stdout)), Stderr: session.ToValidUTF8(string(result.Stderr)), ExitCode: result.ExitCode} + switch result.State { + case executionenv.CommandSucceeded, executionenv.CommandFailed: + return out, nil + case executionenv.CommandCancelled: + return out, context.Canceled + case executionenv.CommandFenceUnknown: + return out, errors.New("execution provider lost terminal command receipt; environment fencing is unknown") + default: + return out, errors.New("execution provider returned a non-terminal foreground command") + } +} diff --git a/internal/adapter/executionclient/client_test.go b/internal/adapter/executionclient/client_test.go new file mode 100644 index 0000000000..59f86ddcf5 --- /dev/null +++ b/internal/adapter/executionclient/client_test.go @@ -0,0 +1,352 @@ +//nolint:revive // Test doubles mirror the private protocol's complete method set. +package executionclient + +import ( + "context" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "math/big" + "net" + "net/url" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type integrationBackend struct { + allocation executioncontroller.Allocation + ensureCalls, attachCalls, fileCalls int + expireNextRead bool +} + +func (*integrationBackend) ValidateProfile(context.Context, string) (executioncontroller.Profile, error) { + return executioncontroller.Profile{Name: "coding", Digest: "sha256:test"}, nil +} +func (b *integrationBackend) Ensure(_ context.Context, client, owner, binding, _, _ string) (executioncontroller.Allocation, error) { + b.ensureCalls++ + b.allocation = executioncontroller.Allocation{Environment: executionenv.EnvironmentRef{ID: "env-real", Revision: "rev-1"}, Epoch: 1, OwnerHash: owner, BindingID: binding, Client: client, Ready: true} + return b.allocation, nil +} +func (b *integrationBackend) Attach(_ context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) (executioncontroller.Allocation, error) { + b.attachCalls++ + if ref != b.allocation.Environment || client != b.allocation.Client || owner != b.allocation.OwnerHash || binding != b.allocation.BindingID { + return executioncontroller.Allocation{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "not found"} + } + return b.allocation, nil +} +func (*integrationBackend) ReleaseReference(context.Context, executionenv.EnvironmentRef, string, string, string) error { + return nil +} +func (*integrationBackend) Retire(context.Context, executionenv.EnvironmentRef, string) error { + return nil +} +func (b *integrationBackend) File(_ context.Context, _, _ string, q executionenv.FileRequest) (executionenv.FileResponse, error) { + b.fileCalls++ + if b.expireNextRead && q.Operation == executionenv.OpFileRead { + b.expireNextRead = false + return executionenv.FileResponse{}, &executionenv.Error{Code: executionenv.CodeUnauthenticated, Message: "expired grant", Retryable: true} + } + if q.Operation == executionenv.OpFileResolveAuthority { + return executionenv.FileResponse{AuthorityTarget: "/workspace/main.go", AuthorityWorkspace: "/workspace"}, nil + } + return executionenv.FileResponse{Data: []byte("from-grpc"), Version: string([]byte{0xff, 0, 1})}, nil +} +func (*integrationBackend) StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + return executionenv.CommandStartResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Result: executionenv.CommandStatusResponse{CommandID: "c1", State: executionenv.CommandSucceeded, Stdout: []byte("ok\n\xff"), Stderr: []byte("err\xe2")}}, nil +} +func (*integrationBackend) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeInternal, Message: "unimplemented"} +} +func (*integrationBackend) CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeInternal, Message: "unimplemented"} +} + +func (b *integrationBackend) EnsurePending(ctx context.Context, client, owner, binding, profile, fp, _ string) (executioncontroller.Allocation, error) { + return b.Ensure(ctx, client, owner, binding, profile, fp) +} +func (b *integrationBackend) AcquireRun(_ context.Context, ref executionenv.EnvironmentRef, client, owner, binding, run, _ string, ttl time.Duration) (executionenv.RunClaim, error) { + return executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: run, ClaimID: "claim", Epoch: b.allocation.Epoch + 1, GrantGeneration: 1, ExpiresAt: time.Now().Add(ttl)}, nil +} +func (b *integrationBackend) RenewRun(_ context.Context, _ executionenv.EnvironmentRef, _ string, _ string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + return executionenv.RunClaim{Environment: req.Environment, BindingID: req.BindingID, RunID: req.RunID, ClaimID: req.ClaimID, Epoch: req.Epoch, GrantGeneration: 1, ExpiresAt: time.Now().Add(req.TTL)}, nil +} +func (*integrationBackend) ReleaseRun(context.Context, executionenv.EnvironmentRef, string, string, executionenv.RunClaimRequest) error { + return nil +} +func (*integrationBackend) CommitReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) AbortReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) ReserveSuccessor(context.Context, executionenv.EnvironmentRef, string, string, string, string, string) error { + return nil +} +func (*integrationBackend) PrepareReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) ConfirmReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) CancelReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*integrationBackend) ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) { + return nil, nil +} +func (*integrationBackend) FindReferenceIntent(context.Context, executionenv.EnvironmentRef, string, string, string) (executionenv.ReferenceIntent, error) { + return executionenv.ReferenceIntent{}, nil +} + +func certificate(t *testing.T, parent *x509.Certificate, parentKey *ecdsa.PrivateKey, serverName string, client bool) (tls.Certificate, *x509.Certificate, *ecdsa.PrivateKey) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tmpl := &x509.Certificate{SerialNumber: big.NewInt(time.Now().UnixNano()), Subject: pkix.Name{CommonName: serverName}, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, IsCA: parent == nil, BasicConstraintsValid: true} + if client { + tmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + u, _ := url.Parse("spiffe://example.test/mecak8s") + tmpl.URIs = []*url.URL{u} + } else if parent != nil { + tmpl.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth} + tmpl.DNSNames = []string{"example.test"} + } + issuer, signer := tmpl, key + if parent != nil { + issuer, signer = parent, parentKey + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, issuer, &key.PublicKey, signer) + if err != nil { + t.Fatal(err) + } + parsed, err := x509.ParseCertificate(der) + if err != nil { + t.Fatal(err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key, Leaf: parsed}, parsed, key +} + +type grpcFixture struct { + endpoint string + clientTLS *tls.Config + stop func() +} + +func startFixture(t *testing.T, backend executioncontroller.Backend, ready func() bool) grpcFixture { + t.Helper() + _, ca, caKey := certificate(t, nil, nil, "ca", false) + serverCert, _, _ := certificate(t, ca, caKey, "example.test", false) + clientCert, _, _ := certificate(t, ca, caKey, "client", true) + _, private, _ := ed25519.GenerateKey(rand.Reader) + h := executioncontroller.NewHandler(executioncontroller.HandlerConfig{Clients: map[string]executioncontroller.ClientPolicy{"spiffe://example.test/mecak8s": {MayAttestOwner: true}}, Signer: executioncontroller.GrantSigner{KeyID: "k1", PrivateKey: private, Issuer: "provider", Audience: "executor", Lifetime: time.Minute}, Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": private.Public().(ed25519.PublicKey)}, Issuer: "provider", Audience: "executor", MaxLifetime: 2 * time.Minute}, Ready: ready}, backend) + pool := x509.NewCertPool() + pool.AddCert(ca) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + s := grpc.NewServer(grpc.Creds(credentials.NewTLS(executioncontroller.TLSConfig(serverCert, pool))), grpc.MaxRecvMsgSize(executionenv.MaxMessageBytes), grpc.MaxSendMsgSize(executionenv.MaxMessageBytes)) + executionv1.RegisterExecutionProviderServiceServer(s, h) + go func() { _ = s.Serve(ln) }() + roots := x509.NewCertPool() + roots.AddCert(ca) + return grpcFixture{endpoint: ln.Addr().String(), clientTLS: &tls.Config{MinVersion: tls.VersionTLS13, RootCAs: roots, Certificates: []tls.Certificate{clientCert}, ServerName: "example.test"}, stop: func() { s.Stop(); _ = ln.Close() }} +} + +func TestProviderThroughRealGRPCSignedHandlerRefreshesAndReattachesExactly(t *testing.T) { + backend := &integrationBackend{} + fx := startFixture(t, backend, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, _ := NewProvider(client, "coding") + principal := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + binding, err := provider.Bind(context.Background(), server.PlacementBindRequest{Selector: server.DefaultPlacement(), Principal: principal, BindingID: "session-real"}) + if err != nil { + t.Fatal(err) + } + if binding.Commit != nil { + if err := binding.Commit(context.Background()); err != nil { + t.Fatal(err) + } + } + reattached, err := provider.Reattach(context.Background(), server.PlacementReattachRequest{Ref: binding.Ref, Principal: principal, BindingID: "session-real"}) + if err != nil { + t.Fatal(err) + } + handle, err := provider.AcquireRun(context.Background(), server.ExecutionRunRequest{Ref: reattached.Ref, Principal: principal, BindingID: "session-real", RunID: "run-real"}) + if err != nil { + t.Fatal(err) + } + defer handle.Release(context.Background()) + runEnv := handle.Environment() + resolver := runEnv.Workspace().(tool.AuthorityResourceResolver) + target, root, err := resolver.AuthorityResourcePath("main.go") + if err != nil || target != "/workspace/main.go" || root != "/workspace" { + t.Fatalf("authority=(%q,%q) err=%v", target, root, err) + } + data, version, err := runEnv.Workspace().ReadVersion(context.Background(), "main.go") + if err != nil || string(data) != "from-grpc" { + t.Fatalf("read=%q err=%v", data, err) + } + encoded, err := tool.EncodeFileVersion(version) + if err != nil || encoded != string([]byte{0xff, 0, 1}) { + t.Fatalf("opaque version=%q err=%v", encoded, err) + } + result, err := runEnv.CommandRunner().Run(context.Background(), "go test ./...") + if err != nil || result.Stdout != "ok\n�" || result.Stderr != "err�" { + t.Fatalf("result=%+v err=%v", result, err) + } + if backend.ensureCalls != 1 || backend.attachCalls < 2 || backend.fileCalls != 2 { + t.Fatalf("ensure=%d attach=%d file=%d", backend.ensureCalls, backend.attachCalls, backend.fileCalls) + } +} + +func TestCommandStatusIsUnimplementedOnlyAfterAuthorization(t *testing.T) { + backend := &integrationBackend{} + fx := startFixture(t, backend, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + ensured, err := client.Ensure(context.Background(), "binding", "coding", owner, "create") + if err != nil { + t.Fatal(err) + } + claim, err := client.AcquireRun(context.Background(), executionenv.RunClaimRequest{Environment: ensured.Environment, Owner: owner, BindingID: "binding", RunID: "run", OperationID: "acquire", TTL: time.Minute}) + if err != nil { + t.Fatal(err) + } + rc := executionenv.RequestContext{Environment: claim.Environment, Owner: owner, BindingID: "binding", RunID: claim.RunID, ClaimID: claim.ClaimID, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant} + _, err = client.rpc.CommandStatus(context.Background(), &executionv1.CommandQueryRequest{Context: contextToProto(rc), CommandId: "c1"}) + if status.Code(err) != codes.Unimplemented { + t.Fatalf("authorized status code=%v", status.Code(err)) + } + rc.Owner.Subject = "mallory" + _, err = client.rpc.CommandStatus(context.Background(), &executionv1.CommandQueryRequest{Context: contextToProto(rc), CommandId: "c1"}) + if status.Code(err) != codes.PermissionDenied { + t.Fatalf("wrong-owner status code=%v", status.Code(err)) + } +} + +func TestGRPCReadinessAndMTLSAreMandatory(t *testing.T) { + fx := startFixture(t, &integrationBackend{}, func() bool { return false }) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + _, err = client.ValidateProfile(context.Background(), "coding") + var remote *executionenv.Error + if !errors.As(err, &remote) || remote.Code != executionenv.CodeNotReady { + t.Fatalf("error=%v", err) + } + noCert := fx.clientTLS.Clone() + noCert.Certificates = nil + conn, err := grpc.NewClient(fx.endpoint, grpc.WithTransportCredentials(credentials.NewTLS(noCert))) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := executionv1.NewExecutionProviderServiceClient(conn).ValidateProfile(ctx, &executionv1.ValidateProfileRequest{Profile: "coding"}); err == nil { + t.Fatal("server accepted client without certificate") + } +} + +func TestDecodeErrorRejectsUntrustedOrContradictoryMetadata(t *testing.T) { + const marker = "/var/run/secrets/provider-key" + typed := func(code executionenv.ErrorCode, retry bool, grpcCode codes.Code) error { + st, err := status.New(grpcCode, marker).WithDetails(&executionv1.ErrorDetail{Code: string(code), Retryable: retry}) + if err != nil { + t.Fatal(err) + } + return st.Err() + } + multiple, err := status.New(codes.NotFound, marker).WithDetails( + &executionv1.ErrorDetail{Code: string(executionenv.CodeNotFound)}, + &executionv1.ErrorDetail{Code: string(executionenv.CodePermissionDenied)}, + ) + if err != nil { + t.Fatal(err) + } + cases := []error{ + errors.New(marker), + status.Error(codes.Internal, marker), + typed("", false, codes.Internal), + typed(executionenv.ErrorCode("future"), false, codes.Internal), + typed(executionenv.CodeNotFound, false, codes.PermissionDenied), + typed(executionenv.CodeNotFound, true, codes.NotFound), + multiple.Err(), + } + for i, input := range cases { + got := decodeError(context.Background(), input) + var remote *executionenv.Error + if !errors.As(got, &remote) || remote.Code != executionenv.CodeInternal || remote.Retryable || strings.Contains(got.Error(), marker) { + t.Fatalf("case %d escaped or misclassified: %v", i, got) + } + } +} + +func TestDecodeErrorAcceptsClosedTypedErrorAndLocalCancellation(t *testing.T) { + st, err := status.New(codes.Unauthenticated, "peer text").WithDetails(&executionv1.ErrorDetail{Code: string(executionenv.CodeUnauthenticated), Retryable: true}) + if err != nil { + t.Fatal(err) + } + var remote *executionenv.Error + if got := decodeError(context.Background(), st.Err()); !errors.As(got, &remote) || remote.Code != executionenv.CodeUnauthenticated || !remote.Retryable { + t.Fatalf("typed error=%v", got) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if got := decodeError(ctx, status.Error(codes.Canceled, "peer text")); !errors.Is(got, context.Canceled) { + t.Fatalf("local cancellation=%v", got) + } +} + +func TestCommandResponsesRejectUnknownStates(t *testing.T) { + if _, err := commandStartFromProto(&executionv1.CommandStartResponse{State: executionv1.CommandState_COMMAND_STATE_UNSPECIFIED, Result: &executionv1.CommandStatusResponse{State: executionv1.CommandState_COMMAND_STATE_SUCCEEDED}}); err == nil { + t.Fatal("accepted unspecified command-start state") + } + if _, err := commandStatusFromProto(&executionv1.CommandStatusResponse{State: executionv1.CommandState(99), TerminalReceipt: "clean"}); err == nil { + t.Fatal("accepted unknown command-status state") + } +} + +func TestEndpointRejectsHTTPAndAlternateResolvers(t *testing.T) { + cfg := &tls.Config{RootCAs: x509.NewCertPool(), Certificates: []tls.Certificate{{Certificate: [][]byte{{1}}}}} + for _, endpoint := range []string{"https://provider:8443", "dns:///provider:8443", "provider"} { + if c, err := New(endpoint, cfg); err == nil { + c.Close() + t.Fatalf("accepted %q", endpoint) + } + } +} diff --git a/internal/adapter/executionclient/create_diagnostics_test.go b/internal/adapter/executionclient/create_diagnostics_test.go new file mode 100644 index 0000000000..1ba2fb76ca --- /dev/null +++ b/internal/adapter/executionclient/create_diagnostics_test.go @@ -0,0 +1,153 @@ +package executionclient + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/adapter/slogdiag" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type pendingCreateBackend struct { + integrationBackend + calls atomic.Int64 + waiting chan struct{} + release chan struct{} + retryable bool +} + +func (b *pendingCreateBackend) Attach(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) (executioncontroller.Allocation, error) { + if b.calls.Add(1) <= 3 { + return executioncontroller.Allocation{}, &executionenv.Error{Code: executionenv.CodeNotReady, Retryable: b.retryable, Message: "PRIVATE-ERROR-SENTINEL"} + } + close(b.waiting) + select { + case <-b.release: + return b.integrationBackend.Attach(ctx, ref, client, owner, binding) + case <-ctx.Done(): + return executioncontroller.Allocation{}, ctx.Err() + } +} + +func TestRemoteCreateAttachPollingDiagnosticsAndCancellation(t *testing.T) { + for _, tc := range []struct { + name string + retryable, cancel bool + }{ + {"ready", true, false}, {"not-ready-nonretryable", false, false}, {"cancel", true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + backend := &pendingCreateBackend{waiting: make(chan struct{}), release: make(chan struct{}), retryable: tc.retryable} + fx := startFixture(t, backend, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + var logs bytes.Buffer + store := memstore.New() + engine := agent.NewEngine(agent.Deps{LLM: mockllm.New(), Catalog: tool.NewCatalog(), Model: "offline"}) + svc, err := server.NewService(server.Config{ + Engine: engine, Store: store, PlacementProvider: provider, ExecutionAccess: provider, + PlacementScope: "fixture", NewID: func() session.SessionID { return "fixed" }, + Diagnostics: slogdiag.New(&logs, true, port.LevelDebug), + }) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + ctx = session.WithPrincipal(ctx, &session.Principal{Issuer: "PRIVATE-ISSUER", Subject: "PRIVATE-OWNER", GrantType: session.GrantTypeUser}) + req := httptest.NewRequest(http.MethodPost, "/v1/sessions", strings.NewReader(`{}`)).WithContext(ctx) + response := httptest.NewRecorder() + done := make(chan struct{}) + go func() { server.NewHTTPHandler(svc).ServeHTTP(response, req); close(done) }() + select { + case <-backend.waiting: + case <-ctx.Done(): + t.Fatal("Attach polling did not reach gated call") + } + if tc.cancel { + cancel() + } else { + close(backend.release) + } + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("create did not return after release/cancel") + } + _, loadErr := store.Load(t.Context(), "fixed") + if tc.cancel { + if !errors.Is(loadErr, port.ErrSessionNotFound) || response.Code == http.StatusCreated { + t.Fatal("cancelled create persisted") + } + } else if loadErr != nil || response.Code != http.StatusCreated { + t.Fatalf("ready create status=%d error=%v", response.Code, loadErr) + } + if strings.Contains(logs.String(), "PRIVATE") || strings.Contains(logs.String(), "env-real") || strings.Contains(logs.String(), "/workspace") { + t.Fatal("private data leaked") + } + reasons := map[string]int{} + stages := map[string]bool{} + var finalCalls float64 + for _, line := range strings.Split(strings.TrimSpace(logs.String()), "\n") { + var row map[string]any + if err := json.Unmarshal([]byte(line), &row); err != nil { + t.Fatal(err) + } + if row["msg"] != "remote create stage" { + continue + } + stage, _ := row["stage"].(string) + stages[stage] = true + if stage == "attach_poll" { + reasons[row["reason"].(string)]++ + } + if stage == "attach_poll_end" { + finalCalls = row["calls"].(float64) + } + } + reason := "not_ready_retryable" + if !tc.retryable { + reason = "not_ready_nonretryable" + } + if reasons[reason] != 1 || finalCalls != 4 { + t.Fatalf("polling not bounded/discriminating: reasons=%v calls=%v", reasons, finalCalls) + } + for _, stage := range []string{"http_handler", "session_id_probe", "bind_ensure", "attach_poll", "attach_poll_end"} { + if !stages[stage] { + t.Errorf("missing stage %s", stage) + } + } + if !tc.cancel && (!stages["session_persist"] || !stages["reference_commit"] || !stages["http_response"]) { + t.Fatal("missing persistence/commit/response stages") + } + if tc.cancel && (stages["session_persist"] || stages["reference_commit"]) { + t.Fatal("cancel reached publication") + } + }) + } +} diff --git a/internal/adapter/executionclient/explicit_id_service_runtime_test.go b/internal/adapter/executionclient/explicit_id_service_runtime_test.go new file mode 100644 index 0000000000..f6c11476c1 --- /dev/null +++ b/internal/adapter/executionclient/explicit_id_service_runtime_test.go @@ -0,0 +1,427 @@ +package executionclient + +import ( + "context" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/adapter/permpolicy" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/adapter/mcp" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type initialCreateBarrierStore struct { + *memstore.Store + target session.SessionID + entered int + mu sync.Mutex + release chan struct{} + winnerMode session.PermissionMode + persisted chan struct{} + releaseCommit chan struct{} +} + +func (s *initialCreateBarrierStore) Load(ctx context.Context, id session.SessionID) (*session.Session, error) { + if id == s.target { + s.mu.Lock() + if s.entered < 2 { + s.entered++ + if s.entered == 2 { + close(s.release) + } + gate := s.release + s.mu.Unlock() + select { + case <-gate: + case <-ctx.Done(): + return nil, ctx.Err() + } + } else { + s.mu.Unlock() + } + } + return s.Store.Load(ctx, id) +} + +func (s *initialCreateBarrierStore) Create(ctx context.Context, sess *session.Session) error { + if sess.ID != s.target || s.persisted == nil { + return s.Store.Create(ctx, sess) + } + if sess.Mode != s.winnerMode { + select { + case <-s.persisted: + case <-ctx.Done(): + return ctx.Err() + } + return s.Store.Create(ctx, sess) + } + if err := s.Store.Create(ctx, sess); err != nil { + return err + } + close(s.persisted) + select { + case <-s.releaseCommit: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func TestServiceExplicitIDConcurrentMismatchCannotAbortPersistedWinner(t *testing.T) { + for _, sharedEngine := range []bool{true, false} { + t.Run(map[bool]string{true: "shared_engine", false: "per_session_engine"}[sharedEngine], func(t *testing.T) { + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{executioncontroller.ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + controllerStore := executioncontroller.NewStore(dyn, "ns", explicitIDProfiles(t), &recordingExecutor{}) + fixture := startFixture(t, controllerStore, nil) + defer fixture.stop() + client, err := New(fixture.endpoint, fixture.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + stopReady := make(chan struct{}) + readyDone := make(chan struct{}) + go func() { + defer close(readyDone) + publishCreatedEnvironmentsReady(dyn, stopReady) + }() + defer func() { close(stopReady); <-readyDone }() + + const id session.SessionID = "sched--remote-explicit-mismatch" + store := &initialCreateBarrierStore{ + Store: memstore.New(), target: id, release: make(chan struct{}), winnerMode: session.ModeDefault, + persisted: make(chan struct{}), releaseCommit: make(chan struct{}), + } + engine := agent.NewEngine(agent.Deps{LLM: mockllm.New(mockllm.TextTurn("ok")), Catalog: tool.NewCatalog(), Policy: permpolicy.NewPolicy(nil, nil), Model: "test"}) + newService := func() *server.Service { + cfg := server.Config{ + Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "test", OwnershipEnforced: true, + SessionEngine: func(context.Context, server.ProviderSelector, []mcp.ServerConfig, server.SessionProfile, string, session.PermissionMode) (server.SessionEngineResult, error) { + return server.SessionEngineResult{Engine: engine}, nil + }, + } + if sharedEngine { + cfg.SharedEngineRoot = remoteRoot + } + svc, svcErr := server.NewService(cfg) + if svcErr != nil { + t.Fatal(svcErr) + } + return svc + } + winnerSvc, loserSvc := newService(), newService() + defer winnerSvc.Close() + defer loserSvc.Close() + principal := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + ctx, cancel := context.WithTimeout(session.WithPrincipal(context.Background(), principal), 10*time.Second) + defer cancel() + type result struct { + sess *session.Session + err error + } + winnerDone := make(chan result, 1) + loserDone := make(chan result, 1) + go func() { + sess, createErr := winnerSvc.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, server.ProviderSelector{}, server.ProfileDefault, server.WithSessionID(id)) + winnerDone <- result{sess: sess, err: createErr} + }() + go func() { + sess, createErr := loserSvc.CreateSessionWithProfile(ctx, session.ModePlan, session.Limits{}, server.ProviderSelector{}, server.ProfileDefault, server.WithSessionID(id)) + loserDone <- result{sess: sess, err: createErr} + }() + + loser := <-loserDone + if !errors.Is(loser.err, server.ErrInvalidArgument) { + t.Fatalf("mismatched loser error=%v, want original ErrInvalidArgument", loser.err) + } + if loser.sess != nil { + t.Fatalf("mismatched loser returned stale session %+v", loser.sess) + } + intents, err := client.ListReferenceIntents(ctx, executionenv.Owner{Issuer: principal.Issuer, Subject: principal.Subject}) + if err != nil || len(intents) != 1 || intents[0].State != executionenv.ReferencePendingCreate { + t.Fatalf("pending intents after loser close=%+v err=%v, want one retained PendingCreate", intents, err) + } + close(store.releaseCommit) + winner := <-winnerDone + if winner.err != nil || winner.sess == nil { + t.Fatalf("winner=(%+v,%v), want success", winner.sess, winner.err) + } + attached, err := client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{ + Environment: executionenv.EnvironmentRef{ID: winner.sess.EnvironmentRef.ID, Revision: winner.sess.EnvironmentRef.Revision}, + Owner: executionenv.Owner{Issuer: principal.Issuer, Subject: principal.Subject}, BindingID: string(id), + }, Purpose: executionenv.PurposeSession}) + if err != nil || !attached.Ready || toSessionRef(attached.Environment) != winner.sess.EnvironmentRef { + t.Fatalf("published attach=(%+v,%v), want exact winner ref %+v", attached, err, winner.sess.EnvironmentRef) + } + }) + } +} + +func TestServiceExplicitIDPrepublicationFailureCannotAbortPersistedWinner(t *testing.T) { + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{executioncontroller.ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + controllerStore := executioncontroller.NewStore(dyn, "ns", explicitIDProfiles(t), &recordingExecutor{}) + fixture := startFixture(t, controllerStore, nil) + defer fixture.stop() + client, err := New(fixture.endpoint, fixture.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + stopReady := make(chan struct{}) + readyDone := make(chan struct{}) + go func() { defer close(readyDone); publishCreatedEnvironmentsReady(dyn, stopReady) }() + defer func() { close(stopReady); <-readyDone }() + + const id session.SessionID = "sched--remote-explicit-prepublish" + store := &initialCreateBarrierStore{ + Store: memstore.New(), target: id, release: make(chan struct{}), winnerMode: session.ModeDefault, + persisted: make(chan struct{}), releaseCommit: make(chan struct{}), + } + engine := agent.NewEngine(agent.Deps{LLM: mockllm.New(mockllm.TextTurn("ok")), Catalog: tool.NewCatalog(), Policy: permpolicy.NewPolicy(nil, nil), Model: "test"}) + factoryErr := errors.New("factory rejected loser") + newService := func() *server.Service { + svc, svcErr := server.NewService(server.Config{ + Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "test", OwnershipEnforced: true, + SessionEngine: func(ctx context.Context, _ server.ProviderSelector, _ []mcp.ServerConfig, _ server.SessionProfile, _ string, mode session.PermissionMode) (server.SessionEngineResult, error) { + if mode == session.ModePlan { + select { + case <-store.persisted: + case <-ctx.Done(): + return server.SessionEngineResult{}, ctx.Err() + } + return server.SessionEngineResult{}, factoryErr + } + return server.SessionEngineResult{Engine: engine}, nil + }, + }) + if svcErr != nil { + t.Fatal(svcErr) + } + return svc + } + winnerSvc, loserSvc := newService(), newService() + defer winnerSvc.Close() + defer loserSvc.Close() + principal := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + ctx, cancel := context.WithTimeout(session.WithPrincipal(context.Background(), principal), 10*time.Second) + defer cancel() + type result struct { + sess *session.Session + err error + } + winnerDone := make(chan result, 1) + go func() { + created, createErr := winnerSvc.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, server.ProviderSelector{}, server.ProfileDefault, server.WithSessionID(id)) + winnerDone <- result{sess: created, err: createErr} + }() + loser, loserErr := loserSvc.CreateSessionWithProfile(ctx, session.ModePlan, session.Limits{}, server.ProviderSelector{}, server.ProfileDefault, server.WithSessionID(id)) + if !errors.Is(loserErr, server.ErrInvalidArgument) || !strings.Contains(loserErr.Error(), "different request") { + t.Fatalf("prepublication loser error=%v, want persisted-winner request mismatch", loserErr) + } + if loser != nil { + t.Fatalf("prepublication loser returned stale session %+v", loser) + } + intents, err := client.ListReferenceIntents(ctx, executionenv.Owner{Issuer: principal.Issuer, Subject: principal.Subject}) + if err != nil || len(intents) != 1 || intents[0].State != executionenv.ReferencePendingCreate { + t.Fatalf("pending intents after loser close=%+v err=%v, want one retained PendingCreate", intents, err) + } + close(store.releaseCommit) + winner := <-winnerDone + if winner.err != nil || winner.sess == nil { + t.Fatalf("winner=(%+v,%v), want success", winner.sess, winner.err) + } + attached, err := client.Attach(ctx, executionenv.AttachEnvironmentRequest{Context: executionenv.RequestContext{ + Environment: executionenv.EnvironmentRef{ID: winner.sess.EnvironmentRef.ID, Revision: winner.sess.EnvironmentRef.Revision}, + Owner: executionenv.Owner{Issuer: principal.Issuer, Subject: principal.Subject}, BindingID: string(id), + }, Purpose: executionenv.PurposeSession}) + if err != nil || !attached.Ready || toSessionRef(attached.Environment) != winner.sess.EnvironmentRef { + t.Fatalf("published attach=(%+v,%v), want exact winner ref %+v", attached, err, winner.sess.EnvironmentRef) + } +} + +func TestServiceExplicitIDRetryReusesRemoteAllocationAcrossRestartAndConcurrency(t *testing.T) { + profiles := explicitIDProfiles(t) + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{executioncontroller.ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + controllerStore := executioncontroller.NewStore(dyn, "ns", profiles, &recordingExecutor{}) + fixture := startFixture(t, controllerStore, nil) + defer fixture.stop() + client, err := New(fixture.endpoint, fixture.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + stopReady := make(chan struct{}) + readyDone := make(chan struct{}) + go func() { + defer close(readyDone) + publishCreatedEnvironmentsReady(dyn, stopReady) + }() + defer func() { + close(stopReady) + <-readyDone + }() + + const id session.SessionID = "sched--remote-explicit-retry" + store := &initialCreateBarrierStore{Store: memstore.New(), target: id, release: make(chan struct{})} + newService := func() *server.Service { + engine := agent.NewEngine(agent.Deps{LLM: mockllm.New(mockllm.TextTurn("ok")), Catalog: tool.NewCatalog(), Policy: permpolicy.NewPolicy(nil, nil), Model: "test"}) + svc, svcErr := server.NewService(server.Config{ + Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "test", OwnershipEnforced: true, + SessionEngine: func(context.Context, server.ProviderSelector, []mcp.ServerConfig, server.SessionProfile, string, session.PermissionMode) (server.SessionEngineResult, error) { + return server.SessionEngineResult{Engine: engine}, nil + }, + }) + if svcErr != nil { + t.Fatal(svcErr) + } + return svc + } + alice := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + ctx, cancel := context.WithTimeout(session.WithPrincipal(context.Background(), alice), 10*time.Second) + defer cancel() + type createResult struct { + session *session.Session + err error + } + createConcurrently := func(services []*server.Service) []*session.Session { + t.Helper() + out := make(chan createResult, len(services)) + for _, svc := range services { + go func() { + created, createErr := svc.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, server.ProviderSelector{}, server.ProfileDefault, server.WithSessionID(id)) + out <- createResult{session: created, err: createErr} + }() + } + results := make([]*session.Session, 0, len(services)) + for range services { + select { + case result := <-out: + if result.err != nil || result.session == nil { + t.Fatalf("concurrent create=(%+v,%v), want success", result.session, result.err) + } + results = append(results, result.session) + case <-ctx.Done(): + t.Fatalf("concurrent create did not finish: %v", ctx.Err()) + } + } + return results + } + + initialServices := []*server.Service{newService(), newService()} + initial := createConcurrently(initialServices) + for _, svc := range initialServices { + svc.Close() + } + first := initial[0] + if initial[1].EnvironmentRef != first.EnvironmentRef { + t.Fatalf("initial creates diverged: first=%+v second=%+v", first.EnvironmentRef, initial[1].EnvironmentRef) + } + + services := []*server.Service{newService(), newService()} + defer services[0].Close() + defer services[1].Close() + results := createConcurrently(services) + for i := range results { + if results[i].EnvironmentRef != first.EnvironmentRef { + t.Fatalf("identical retry %d=%+v, want same ref %+v", i, results[i].EnvironmentRef, first.EnvironmentRef) + } + } + list, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").List(t.Context(), metav1.ListOptions{}) + if err != nil || len(list.Items) != 1 { + t.Fatalf("allocations=%d err=%v, want one", len(list.Items), err) + } + allocationBefore := list.Items[0].DeepCopy() + + bobCtx := session.WithPrincipal(context.Background(), &session.Principal{Issuer: "issuer", Subject: "bob", GrantType: session.GrantTypeUser}) + if _, err := services[0].CreateSessionWithProfile(bobCtx, session.ModeDefault, session.Limits{}, server.ProviderSelector{}, server.ProfileDefault, server.WithSessionID(id)); err == nil { + t.Fatal("changed owner adopted explicit-ID session") + } + if _, err := services[0].CreateSessionWithProfile(ctx, session.ModePlan, session.Limits{}, server.ProviderSelector{}, server.ProfileDefault, server.WithSessionID(id)); !errors.Is(err, server.ErrInvalidArgument) { + t.Fatalf("changed mode error=%v, want ErrInvalidArgument", err) + } + if _, err := services[0].CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{MaxTurns: 1}, server.ProviderSelector{}, server.ProfileDefault, server.WithSessionID(id)); !errors.Is(err, server.ErrInvalidArgument) { + t.Fatalf("changed limits error=%v, want ErrInvalidArgument", err) + } + if _, err := services[0].CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, server.ProviderSelector{}, server.ProfileNoFS, server.WithSessionID(id)); !errors.Is(err, server.ErrInvalidArgument) { + t.Fatalf("changed profile error=%v, want ErrInvalidArgument", err) + } + afterList, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").List(t.Context(), metav1.ListOptions{}) + if err != nil || len(afterList.Items) != 1 { + t.Fatalf("rejected retries changed allocation count to %d: %v", len(afterList.Items), err) + } + after, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), allocationBefore.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + allocationBefore.SetResourceVersion(after.GetResourceVersion()) + if allocationBefore.GetUID() != after.GetUID() || !reflect.DeepEqual(allocationBefore.Object["spec"], after.Object["spec"]) { + t.Fatal("rejected retry adopted a different allocation identity") + } +} + +func explicitIDProfiles(t *testing.T) *executioncontroller.Profiles { + t.Helper() + path := filepath.Join(t.TempDir(), "profiles.yaml") + body := []byte("profiles:\n coding:\n image: example.test/executor@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n storageClass: standard\n storageSize: 1Gi\n cpuRequest: 100m\n memoryRequest: 128Mi\n cpuLimit: 1\n memoryLimit: 1Gi\n ephemeralStorageRequest: 256Mi\n ephemeralStorageLimit: 1Gi\n tmpSizeLimit: 128Mi\n runtimeClassName: gvisor\n maxFileBytes: 5242880\n maxCommandBytes: 1048576\n maxCommandDuration: 1m\n maxEnvironments: 10\n") + if err := os.WriteFile(path, body, 0o600); err != nil { + t.Fatal(err) + } + profiles, err := executioncontroller.LoadProfiles(path) + if err != nil { + t.Fatal(err) + } + return profiles +} + +func publishCreatedEnvironmentsReady(client *dynamicfake.FakeDynamicClient, stop <-chan struct{}) { + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + list, err := client.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").List(context.Background(), metav1.ListOptions{}) + if err != nil { + continue + } + for i := range list.Items { + o := list.Items[i].DeepCopy() + _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": "executor"}, "status", "pod") + _ = unstructured.SetNestedSlice(o.Object, []any{map[string]any{"type": "Ready", "status": "True"}}, "status", "conditions") + _, _ = client.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(context.Background(), o, metav1.UpdateOptions{}) + } + } + } +} diff --git a/internal/adapter/executionclient/files_conformance_test.go b/internal/adapter/executionclient/files_conformance_test.go new file mode 100644 index 0000000000..ebf9985ef8 --- /dev/null +++ b/internal/adapter/executionclient/files_conformance_test.go @@ -0,0 +1,124 @@ +package executionclient + +import ( + "context" + "encoding/json" + "errors" + "io/fs" + "strings" + "testing" + + "github.com/stacklok/mecatl/engine/adapter/fsconformance" + "github.com/stacklok/mecatl/engine/adapter/fstools" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" + "github.com/stacklok/mecatl/internal/executionexecutor" +) + +type executorBackend struct { + integrationBackend + executor *executionexecutor.Executor +} + +func (b *executorBackend) File(ctx context.Context, _, _ string, q executionenv.FileRequest) (executionenv.FileResponse, error) { + r, err := b.executor.Execute(ctx, executionenv.ExecutorRequest{Operation: q.Operation, Path: q.Path, Destination: q.Destination, Pattern: q.Pattern, Data: q.Data, Version: q.Version, Limit: q.Limit}) + return r.FileResponse, err +} + +func remoteExecutorEnvironment(t *testing.T) tool.Environment { + t.Helper() + x, err := executionexecutor.New(t.TempDir(), executionexecutor.Limits{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = x.Close() }) + fx := startFixture(t, &executorBackend{executor: x}, nil) + t.Cleanup(fx.stop) + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + t.Cleanup(client.Close) + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + principal := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + binding, err := provider.Bind(t.Context(), server.PlacementBindRequest{Selector: server.DefaultPlacement(), Principal: principal, BindingID: "files"}) + if err != nil { + t.Fatal(err) + } + if err := binding.Commit(t.Context()); err != nil { + t.Fatal(err) + } + handle, err := provider.AcquireRun(t.Context(), server.ExecutionRunRequest{Ref: binding.Ref, Principal: principal, BindingID: "files", RunID: "run-files"}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = handle.Release(context.Background()) }) + return handle.Environment() +} + +func TestRemoteWorkspaceConformanceThroughSignedGRPC(t *testing.T) { + factory := func(t *testing.T) tool.Workspace { return remoteExecutorEnvironment(t).Workspace() } + fsconformance.Run(t, factory) + fsconformance.RunNamespace(t, factory) +} + +func TestRemoteFileToolsPreserveLedgerAndNamespaceContracts(t *testing.T) { + env := remoteExecutorEnvironment(t) + ctx := t.Context() + call := func(body tool.Tool, args string, wantError bool, contains string) { + t.Helper() + result, err := body.Execute(ctx, session.ToolCall{ID: "file", Name: body.Spec().Name, Args: json.RawMessage(args)}, env) + if err != nil || result.IsError != wantError || !strings.Contains(result.Content, contains) { + t.Fatalf("%s: result=%+v error=%v", body.Spec().Name, result, err) + } + } + if _, err := env.Workspace().CreateFile(ctx, "main.go", []byte("package main\n// before\n")); err != nil { + t.Fatal(err) + } + call(fstools.EditTool{}, `{"path":"main.go","old_string":"before","new_string":"after"}`, true, "") + call(fstools.WriteTool{}, `{"path":"main.go","content":"clobber"}`, true, "") + call(fstools.ReadTool{}, `{"path":"main.go"}`, false, "before") + call(fstools.EditTool{}, `{"path":"main.go","old_string":"before","new_string":"after"}`, false, "") + call(fstools.ReadTool{}, `{"path":"main.go"}`, false, "after") + // A mutation outside the read ledger invalidates both Edit and existing Write. + _, version, err := env.Workspace().ReadVersion(ctx, "main.go") + if err != nil { + t.Fatal(err) + } + if _, err := env.Workspace().ReplaceFile(ctx, "main.go", version, []byte("package main\n// concurrent\n")); err != nil { + t.Fatal(err) + } + call(fstools.EditTool{}, `{"path":"main.go","old_string":"concurrent","new_string":"lost"}`, true, "") + call(fstools.WriteTool{}, `{"path":"main.go","content":"lost"}`, true, "") + call(fstools.ReadTool{}, `{"path":"main.go"}`, false, "concurrent") + call(fstools.WriteTool{}, `{"path":"main.go","content":"package main\n// final\n"}`, false, "") + call(fstools.ListDirTool{}, `{"path":"."}`, false, "main.go") + call(fstools.GrepTool{}, `{"path":"*.go","pattern":"final"}`, false, "final") + call(fstools.GlobTool{}, `{"pattern":"**/*.go"}`, false, "main.go") + call(fstools.WriteTool{}, `{"path":"fresh.go","content":"package main\n"}`, false, "") + // Seed without Write's automatic ledger update: namespace operations need no ledger. + if _, err := env.Workspace().CreateFile(ctx, "unread.go", []byte("package unread\n")); err != nil { + t.Fatal(err) + } + call(fstools.CopyTool{}, `{"source":"unread.go","destination":"copied.go"}`, false, "") + call(fstools.MoveTool{}, `{"source":"copied.go","destination":"moved.go"}`, false, "") + data, err := env.Workspace().Read(ctx, "moved.go") + if err != nil || string(data) != "package unread\n" { + t.Fatalf("moved=%q error=%v", data, err) + } + call(fstools.CopyTool{}, `{"source":"unread.go","destination":"main.go"}`, true, "") + call(fstools.MoveTool{}, `{"source":"moved.go","destination":"main.go"}`, true, "") + call(fstools.RemoveTool{}, `{"path":"moved.go"}`, false, "") + if _, err := env.Workspace().Stat(ctx, "moved.go"); !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("removed file: %v", err) + } + data, err = env.Workspace().Read(ctx, "main.go") + if err != nil || string(data) != "package main\n// final\n" { + t.Fatalf("final=%q error=%v", data, err) + } +} diff --git a/internal/adapter/executionclient/harness_context_integration_test.go b/internal/adapter/executionclient/harness_context_integration_test.go new file mode 100644 index 0000000000..6d5fe7a1f6 --- /dev/null +++ b/internal/adapter/executionclient/harness_context_integration_test.go @@ -0,0 +1,226 @@ +package executionclient + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/stacklok/mecatl/engine/adapter/memfs" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/engine/prompt" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/memory" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/app" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type nativeContextInstructions string + +func (s nativeContextInstructions) Assemble(context.Context) ([]session.Message, error) { + return []session.Message{session.NewUserMessage(string(s))}, nil +} + +type contextPublicationBackend struct { + integrationBackend + failCommit bool + commits, aborts atomic.Int64 +} + +func (b *contextPublicationBackend) CommitReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + b.commits.Add(1) + if b.failCommit { + return &executionenv.Error{Code: executionenv.CodeInternal, Message: "publication response lost"} + } + return nil +} + +func (b *contextPublicationBackend) AbortReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + b.aborts.Add(1) + return nil +} + +func TestNativeBuildPreservesIndependentContext(t *testing.T) { + for _, tc := range []struct { + name string + compatibility, failCommit bool + }{ + {name: "selected"}, + {name: "published despite commit failure", failCommit: true}, + {name: "compatibility", compatibility: true}, + } { + t.Run(tc.name, func(t *testing.T) { + for _, key := range []string{"HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME"} { + t.Setenv(key, t.TempDir()) + } + backend := &contextPublicationBackend{failCommit: tc.failCommit} + fx := startFixture(t, backend, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + ctx := session.WithPrincipal(t.Context(), &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser}) + var requests []port.LLMRequest + cfg := app.Config{ + UseMock: true, MockProvider: mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { requests = append(requests, req) })}, mockllm.TextTurn("done")), + Workspace: t.TempDir(), StoreDir: t.TempDir(), UserModelDir: t.TempDir(), MemoryDir: t.TempDir(), NoSoul: true, + RemoteExecution: true, TrustProject: true, Headless: true, EnableCommands: true, OwnershipEnforced: true, + PlacementProvider: provider, PlacementScope: "native-test", + } + for name, body := range map[string]string{ + "AGENTS.md": "POISON_HOST_INSTRUCTIONS", + ".claude/rules/poison.md": "POISON_HOST_RULES", + ".claude/commands/poison.md": "POISON_HOST_COMMAND", + } { + path := filepath.Join(cfg.Workspace, name) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + memoryStore, err := memory.New(cfg.MemoryDir) + if err != nil { + t.Fatal(err) + } + callerMemory := memory.NewCallerStore(memoryStore, true) + for root, marker := range map[string]string{cfg.Workspace: "POISON_HOST_MEMORY", "/workspace": "SELECTED_REMOTE_MEMORY"} { + memoryCtx := memory.WithWorkspace(ctx, root) + if _, err := callerMemory.Remember(memoryCtx, tool.MemoryEntry{Key: "project/fixture", Value: marker, Description: marker}, tool.MemoryCurrent{}); err != nil { + t.Fatal(err) + } + index, err := callerMemory.Index(memoryCtx) + if err != nil || len(index) != 1 || index[0].Description != marker { + t.Fatalf("memory fixture not readable: %v, %v", index, err) + } + } + var binds, closes atomic.Int64 + if tc.compatibility { + cfg.CommandsDir = "operator-commands" + if err := os.MkdirAll(filepath.Join(cfg.Workspace, cfg.CommandsDir), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cfg.Workspace, cfg.CommandsDir, "selected.md"), []byte("INDEPENDENT_COMMAND_BODY"), 0o600); err != nil { + t.Fatal(err) + } + } else { + source := memfs.NewWorkspace("/independent-context") + if _, err := source.CreateFile(t.Context(), "commands/selected.md", []byte("INDEPENDENT_COMMAND_BODY")); err != nil { + t.Fatal(err) + } + settings := filepath.Join(t.TempDir(), "settings.yaml") + if err := os.WriteFile(settings, []byte(`harness_context: + enabled_sources: [independent, pvc] + kinds: + instructions: {sources: [independent], mode: combine} + commands: {sources: [independent, pvc], mode: combine} + rules: {sources: [], mode: combine} + skills: {sources: [], mode: combine} + agent_defs: {sources: [], mode: combine} +`), 0o600); err != nil { + t.Fatal(err) + } + cfg.PermissionConfigs = []string{settings} + cfg.HarnessInstructionSources = []app.HarnessSourceRegistration[prompt.InstructionAssembler]{ + {ID: "independent", Provenance: app.HarnessProvenancePolicy{Fixed: "explicit"}, Bind: func(context.Context, app.HarnessSourceScope) (prompt.InstructionAssembler, func() error, error) { + return nativeContextInstructions("INDEPENDENT_INSTRUCTIONS"), nil, nil + }}, + } + cfg.HarnessCommandSources = []app.HarnessSourceRegistration[server.CommandSourceBinding]{ + {ID: "independent", Scope: app.HarnessSourceScopePrincipal, Provenance: app.HarnessProvenancePolicy{Fixed: "explicit"}, Bind: func(_ context.Context, scope app.HarnessSourceScope) (server.CommandSourceBinding, func() error, error) { + if scope.SessionID == "" || scope.Principal == nil { + t.Error("source bound without session identity") + } + if scope.AcquireExecutionWorkspace != nil { + t.Error("independent source received execution workspace capability") + } + binds.Add(1) + return prompt.NewDirCommandExpander(source, "commands"), func() error { closes.Add(1); return nil }, nil + }}, + {ID: "pvc", Scope: app.HarnessSourceScopePrincipal, UsesExecutionWorkspace: true, Provenance: app.HarnessProvenancePolicy{Fixed: "project"}, Bind: func(context.Context, app.HarnessSourceScope) (server.CommandSourceBinding, func() error, error) { + t.Error("native execution source admitted") + return prompt.NoopExpander{}, nil, nil + }}, + } + } + built, err := app.Build(t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + defer built.Close() + sess, err := built.Service.CreateSession(ctx, session.ModeDefault, session.Limits{}) + if tc.failCommit { + if !errors.Is(err, server.ErrInternal) || sess == nil { + t.Fatalf("create=%v, %v; want persisted session and publication error", sess, err) + } + } else if err != nil { + t.Fatal(err) + } + stored, err := built.Service.GetSession(ctx, sess.ID) + if err != nil || stored.EnvironmentRef != sess.EnvironmentRef { + t.Fatalf("session not persisted: %v", err) + } + if backend.commits.Load() != 1 || backend.aborts.Load() != 0 || closes.Load() != 0 { + t.Fatalf("published context retired: commits=%d aborts=%d closes=%d", backend.commits.Load(), backend.aborts.Load(), closes.Load()) + } + commands, err := built.Service.ListCommandsForSession(ctx, sess.ID) + if err != nil || len(commands) != 1 || commands[0].Name != "selected" { + t.Fatalf("independent listing=%v, %v", commands, err) + } + if backend.ensureCalls != 1 || backend.attachCalls != 1 || backend.fileCalls != 0 { + t.Fatalf("context discovery touched native provider: ensure=%d attach=%d files=%d", backend.ensureCalls, backend.attachCalls, backend.fileCalls) + } + run, err := built.Service.StartRun(ctx, sess.ID, "/selected") + if err != nil { + t.Fatal(err) + } + for range run.Events() { + } + built.Service.FinishRun(sess.ID, run) + if backend.ensureCalls != 1 || backend.attachCalls != 1 || backend.fileCalls != 0 { + t.Fatalf("run changed native attachment ownership: ensure=%d attach=%d files=%d", backend.ensureCalls, backend.attachCalls, backend.fileCalls) + } + if len(requests) != 1 { + t.Fatalf("model requests=%d", len(requests)) + } + data, err := json.Marshal(requests[0]) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "INDEPENDENT_COMMAND_BODY") || strings.Contains(string(data), "POISON_HOST_") { + t.Fatal("model request lost selected command or admitted host poison") + } + if !tc.compatibility && !strings.Contains(string(data), "INDEPENDENT_INSTRUCTIONS") { + t.Fatal("native factory dropped independent instructions") + } + if strings.Contains(string(data), "SELECTED_REMOTE_MEMORY") { + t.Fatal("guest execution root became a governance memory root") + } + if !strings.Contains(requests[0].System.StablePrefix, "persistent remote Kubernetes workspace") || !strings.Contains(requests[0].System.StablePrefix, "explicitly selected independent commands") { + t.Fatal("native factory lost system posture or independent command affordance") + } + built.Service.CloseSession(sess.ID) + built.Close() + if !tc.compatibility && (binds.Load() != 1 || closes.Load() != 1) { + t.Fatalf("binding lifecycle binds=%d closes=%d", binds.Load(), closes.Load()) + } + if backend.aborts.Load() != 0 { + t.Fatal("ambiguous published reference was aborted") + } + }) + } +} diff --git a/internal/adapter/executionclient/native_pending_approval_tui_integration_test.go b/internal/adapter/executionclient/native_pending_approval_tui_integration_test.go new file mode 100644 index 0000000000..6350233fa6 --- /dev/null +++ b/internal/adapter/executionclient/native_pending_approval_tui_integration_test.go @@ -0,0 +1,575 @@ +package executionclient + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + tea "charm.land/bubbletea/v2" + "github.com/charmbracelet/x/ansi" + "github.com/charmbracelet/x/exp/teatest/v2" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + + mecatuiclient "github.com/stacklok/mecatl/cmd/mecatui/client" + "github.com/stacklok/mecatl/cmd/mecatui/theme" + "github.com/stacklok/mecatl/cmd/mecatui/ui" + mecatlv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/v1" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/adapter/store/jsonlstore" + "github.com/stacklok/mecatl/internal/app" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + nativeBridgeOwnerToken = "alice-test-token" + nativeBridgeForeignToken = "bob-test-token" +) + +type nativeBridgeRPCStats struct { + ownerConverse atomic.Int32 + ownerResolve atomic.Int32 + foreignCalls atomic.Int32 + ownerResolved chan struct{} + converseDone chan struct{} + resolveOnce sync.Once + converseOnce sync.Once + mu sync.Mutex + unmappedKinds []string + terminalStops []string + ownerControl *mecatlv1.ResolveRunAskRequest +} + +type nativeBridgePrincipalStream struct { + grpc.ServerStream + ctx context.Context + stats *nativeBridgeRPCStats +} + +func (s nativeBridgePrincipalStream) SendMsg(message any) error { + if frame, ok := message.(*mecatlv1.WatchSessionEventsResponse); ok { + if event := frame.GetEvent(); event != nil { + s.stats.mu.Lock() + if mecatuiclient.EventToMsg(event) == nil { + s.stats.unmappedKinds = append(s.stats.unmappedKinds, event.GetType()) + } + if event.GetType() == "result" { + s.stats.terminalStops = append(s.stats.terminalStops, event.GetResult().GetStop()) + } + s.stats.mu.Unlock() + } + } + return s.ServerStream.SendMsg(message) +} + +func (s nativeBridgePrincipalStream) Context() context.Context { return s.ctx } + +func (s *nativeBridgeRPCStats) principal(ctx context.Context) (*session.Principal, bool) { + md, _ := metadata.FromIncomingContext(ctx) + values := md.Get("authorization") + if len(values) != 1 { + return nil, false + } + switch values[0] { + case "Bearer " + nativeBridgeOwnerToken: + return &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser}, true + case "Bearer " + nativeBridgeForeignToken: + s.foreignCalls.Add(1) + return &session.Principal{Issuer: "issuer", Subject: "bob", GrantType: session.GrantTypeUser}, true + default: + return nil, false + } +} + +func (s *nativeBridgeRPCStats) unary(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + principal, ok := s.principal(ctx) + if !ok { + return nil, status.Error(codes.Unauthenticated, "test principal required") + } + if strings.HasSuffix(info.FullMethod, "/ResolveRunAsk") && principal.Subject == "alice" { + s.ownerResolve.Add(1) + s.mu.Lock() + s.ownerControl = req.(*mecatlv1.ResolveRunAskRequest) + s.mu.Unlock() + s.resolveOnce.Do(func() { close(s.ownerResolved) }) + } + return handler(session.WithPrincipal(ctx, principal), req) +} + +func (s *nativeBridgeRPCStats) stream(srv any, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + principal, ok := s.principal(stream.Context()) + if !ok { + return status.Error(codes.Unauthenticated, "test principal required") + } + if strings.HasSuffix(info.FullMethod, "/Converse") && principal.Subject == "alice" { + s.ownerConverse.Add(1) + if s.converseDone != nil { + defer s.converseOnce.Do(func() { close(s.converseDone) }) + } + } + return handler(srv, nativeBridgePrincipalStream{ServerStream: stream, ctx: session.WithPrincipal(stream.Context(), principal), stats: s}) +} + +type nativeBridgePendingClient struct{ *mecatuiclient.Client } + +func (c nativeBridgePendingClient) WatchPendingApprovalRun(ctx context.Context, approval mecatuiclient.PendingApproval) (ui.PendingApprovalWatch, error) { + return c.Client.WatchPendingApprovalRun(ctx, approval) +} + +// nativeBridgeModel observes only the public rendered view on Bubble Tea's +// goroutine, so barriers do not depend on the terminal renderer's flush timer. +type nativeBridgeModel struct { + ui.Model + ready, terminal chan string + lastView *atomic.Value +} + +func (m *nativeBridgeModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + next, cmd := m.Model.Update(msg) + m.Model = next.(ui.Model) + return m, cmd +} + +func (m *nativeBridgeModel) View() tea.View { + view := m.Model.View() + text := ansi.Strip(view.Content) + m.lastView.Store(text) + if m.ready != nil && strings.Contains(text, "Permission required") && strings.Contains(text, "bridge-command") { + m.ready <- text + m.ready = nil + } + if m.terminal != nil && (strings.Contains(text, "connection failed") || + strings.Contains(text, "continuation complete") && strings.Contains(text, "done") && !strings.Contains(text, "enter queue")) { + m.terminal <- text + m.terminal = nil + } + return view +} + +// This is a composition proof, not CLI flag resolution, OIDC login, or a live +// Kubernetes proof. Only authentication and the final executor are test-owned; +// authorization, public RPCs, durable history and native run access are real. +func TestNativePendingApprovalStartupRecovery(t *testing.T) { + for _, tc := range []struct { + name string + key rune + wantRun bool + wantOutput string + }{ + {name: "allow-once", key: 'a', wantRun: true, wantOutput: "continuation complete"}, + {name: "deny", key: 'd', wantOutput: "continuation complete"}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + for _, key := range []string{"HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME"} { + t.Setenv(key, t.TempDir()) + } + + profiles := explicitIDProfiles(t) + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{executioncontroller.ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + release := make(chan struct{}) + releaseCommand := sync.OnceFunc(func() { close(release) }) + defer releaseCommand() + executor := &recordingExecutor{executed: make(chan executionenv.ExecutorRequest, 1), release: release} + controllerStore := executioncontroller.NewStore(dyn, "ns", profiles, executor) + profile, err := controllerStore.ValidateProfile(ctx, "coding") + if err != nil { + t.Fatal(err) + } + owner := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + ownerSum := sha256.Sum256([]byte(owner.Issuer + "\x00" + owner.Subject)) + clientSum := sha256.Sum256([]byte("spiffe://example.test/mecak8s")) + now := time.Now().UTC() + sessionID := session.SessionID("native-ui-" + tc.name) + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env-" + tc.name, Revision: "rev-1"} + envObj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": ref.ID, "namespace": "ns"}, + "spec": map[string]any{"schemaVersion": int64(2), "revision": ref.Revision, "ownerHash": hex.EncodeToString(ownerSum[:]), "ownerIssuer": owner.Issuer, "ownerSubject": owner.Subject, "clientHash": hex.EncodeToString(clientSum[:]), "bindingID": string(sessionID), "profile": "coding", "profileDigest": profile.Digest, "desired": "Active"}, + "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "pod": map[string]any{"name": "executor-pod"}, "references": []any{map[string]any{"bindingID": string(sessionID), "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}, + }} + if _, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Create(ctx, envObj, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + executionFixture := startFixture(t, controllerStore, nil) + defer executionFixture.stop() + executionClient, err := New(executionFixture.endpoint, executionFixture.clientTLS) + if err != nil { + t.Fatal(err) + } + defer executionClient.Close() + provider, err := NewProvider(executionClient, "coding") + if err != nil { + t.Fatal(err) + } + + storeDir, workspace := t.TempDir(), t.TempDir() + store, err := jsonlstore.New(storeDir) + if err != nil { + t.Fatal(err) + } + parked := session.New(sessionID, session.ModeDefault, ref, session.Limits{MaxTurns: 4}, now) + if err := parked.RestoreLabels(owner, session.Authority{}); err != nil { + t.Fatal(err) + } + if err := store.Save(ctx, parked); err != nil { + t.Fatal(err) + } + settings := filepath.Join(t.TempDir(), "settings.yaml") + if err := os.WriteFile(settings, []byte("permissions:\n ask:\n - Shell\n"), 0o600); err != nil { + t.Fatal(err) + } + baseConfig := func(model port.LLMProvider) app.Config { + return app.Config{ + UseMock: true, MockProvider: model, Workspace: workspace, StoreDir: storeDir, + UserModelDir: t.TempDir(), MemoryDir: t.TempDir(), NoSoul: true, Headless: true, + RemoteExecution: true, TrustProject: true, OwnershipEnforced: true, + PlacementProvider: provider, PlacementScope: "native-ui-test", PermissionConfigs: []string{settings}, + } + } + builtA, err := app.Build(ctx, baseConfig(mockllm.New(mockllm.ToolCallTurn( + session.NewToolCall("native-shell", "Shell", json.RawMessage(`{"command":"printf bridge-command"}`)), + )))) + if err != nil { + t.Fatal(err) + } + closeA := sync.OnceFunc(func() { builtA.Close() }) + defer closeA() + firstStats := &nativeBridgeRPCStats{converseDone: make(chan struct{})} + firstListener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + closeA() + t.Fatal(err) + } + firstServer := grpc.NewServer(grpc.UnaryInterceptor(firstStats.unary), grpc.StreamInterceptor(firstStats.stream)) + mecatlv1.RegisterHarnessServiceServer(firstServer, server.NewHarnessServer(builtA.Service)) + go func() { _ = firstServer.Serve(firstListener) }() + defer func() { + firstServer.Stop() + _ = firstListener.Close() + }() + firstClient, err := mecatuiclient.Dial(mecatuiclient.DialConfig{Server: firstListener.Addr().String(), AuthToken: nativeBridgeOwnerToken}) + if err != nil { + firstServer.Stop() + _ = firstListener.Close() + closeA() + t.Fatal(err) + } + defer func() { _ = firstClient.Close() }() + runCtx, cancelRun := context.WithCancel(ctx) + defer cancelRun() + stream, err := firstClient.OpenConverseForSession(runCtx, string(sessionID)) + if err != nil { + cancelRun() + _ = firstClient.Close() + firstServer.Stop() + _ = firstListener.Close() + closeA() + t.Fatal(err) + } + messages := make(chan tea.Msg, 16) + readerDone := make(chan struct{}) + go func() { + defer close(readerDone) + stream.ReadLoop(runCtx, messages) + }() + defer func() { + cancelRun() + select { + case <-readerDone: + case <-time.After(5 * time.Second): + t.Error("initial public stream reader did not exit") + } + }() + if err := stream.SendPrompt(string(sessionID), "run native shell", nil); err != nil { + cancelRun() + t.Fatal(err) + } + var runID, askID string + for askID == "" { + select { + case message, ok := <-messages: + if !ok { + t.Fatal("initial public stream closed before the ask") + } + if ask, ok := message.(mecatuiclient.PermissionAskMsg); ok { + runID, askID = ask.RunID, ask.AskID + } + case <-ctx.Done(): + cancelRun() + t.Fatalf("native Shell run did not reach its persisted ask: %v", ctx.Err()) + } + } + persisted, err := firstClient.DiscoverPendingApproval(ctx, string(sessionID)) + if err != nil || persisted.RunID != runID || persisted.AskID != askID || persisted.Cursor == "" { + cancelRun() + t.Fatalf("persist initial ask before restart: pending=%+v err=%v", persisted, err) + } + closeA() + select { + case <-firstStats.converseDone: + case <-ctx.Done(): + cancelRun() + t.Fatalf("parked public transport did not drain: %v", ctx.Err()) + } + cancelRun() + _ = firstClient.Close() + firstServer.Stop() + _ = firstListener.Close() + + var continuationRequests atomic.Int32 + builtB, err := app.Build(ctx, baseConfig(mockllm.NewWith([]mockllm.Option{ + mockllm.WithRequestObserver(func(port.LLMRequest) { continuationRequests.Add(1) }), + }, mockllm.TextTurn("continuation complete")))) + if err != nil { + t.Fatal(err) + } + defer builtB.Close() + stats := &nativeBridgeRPCStats{ownerResolved: make(chan struct{})} + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + grpcServer := grpc.NewServer(grpc.UnaryInterceptor(stats.unary), grpc.StreamInterceptor(stats.stream)) + mecatlv1.RegisterHarnessServiceServer(grpcServer, server.NewHarnessServer(builtB.Service)) + go func() { _ = grpcServer.Serve(listener) }() + defer func() { + grpcServer.Stop() + _ = listener.Close() + }() + + ownerClient, err := mecatuiclient.Dial(mecatuiclient.DialConfig{Server: listener.Addr().String(), AuthToken: nativeBridgeOwnerToken}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = ownerClient.Close() }() + foreignClient, err := mecatuiclient.Dial(mecatuiclient.DialConfig{Server: listener.Addr().String(), AuthToken: nativeBridgeForeignToken}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = foreignClient.Close() }() + + snapshot, err := ownerClient.GetSession(ctx, string(sessionID)) + if err != nil { + t.Fatal(err) + } + transcript, err := ownerClient.GetSessionTranscript(ctx, string(sessionID)) + if err != nil { + t.Fatal(err) + } + pending, err := ownerClient.DiscoverPendingApproval(ctx, string(sessionID)) + if err != nil { + stats.mu.Lock() + stops := append([]string(nil), stats.terminalStops...) + stats.mu.Unlock() + t.Fatalf("discover after shutdown: snapshot=%q durable result.stop=%q: %v", snapshot.State, stops, err) + } + current, err := ownerClient.GetSession(ctx, string(sessionID)) + if err != nil { + t.Fatal(err) + } + if snapshot.State != "awaiting" || current.State != "awaiting" || pending.RunID != runID || pending.AskID != askID || pending.Cursor == "" { + t.Fatalf("actual recovery payload: snapshot=%q current=%q pending=%+v", snapshot.State, current.State, pending) + } + if !transcript.Complete || transcript.SessionID != string(sessionID) || len(transcript.Messages) == 0 { + t.Fatalf("actual transcript = %+v", transcript) + } + var foundOriginalCall bool + for _, message := range transcript.Messages { + for _, call := range message.ToolCalls { + foundOriginalCall = foundOriginalCall || call.ID == "native-shell" && call.Name == "Shell" && strings.Contains(call.Args, "bridge-command") + } + } + if !foundOriginalCall { + t.Fatalf("original pending tool call absent from transcript: %+v", transcript.Messages) + } + + if _, err := foreignClient.GetSession(ctx, string(sessionID)); !mecatuiclient.IsNotFound(err) { + t.Fatalf("foreign session lookup = %v, want concealed not found", err) + } + if foreignPending, err := foreignClient.DiscoverPendingApproval(ctx, string(sessionID)); !mecatuiclient.IsPendingApprovalFailure(err, mecatuiclient.PendingApprovalUnavailable) || foreignPending != (mecatuiclient.PendingApproval{}) { + t.Fatalf("foreign discovery leaked a handle or wrong refusal: %v", err) + } + if err := foreignClient.ResolvePendingApproval(ctx, pending, mecatuiclient.VerdictAllowOnce); !mecatuiclient.IsPendingApprovalFailure(err, mecatuiclient.PendingApprovalUnavailable) { + t.Fatalf("foreign resolution = %v, want unavailable", err) + } + assertNoNativeActiveRun(t, dyn, ref.ID) + executor.mu.Lock() + operationsBeforeUI := len(executor.operations) + executor.mu.Unlock() + if operationsBeforeUI != 0 || continuationRequests.Load() != 0 { + t.Fatalf("foreign checks executed work: operations=%d model-requests=%d", operationsBeforeUI, continuationRequests.Load()) + } + + observer, err := ownerClient.WatchPendingApprovalRun(ctx, pending) + if err != nil { + t.Fatal(err) + } + defer observer.Close() + if boundary, err := observer.Recv(); err != nil || boundary.Kind != mecatuiclient.PendingApprovalEventBoundary { + t.Fatalf("observer boundary: event=%+v err=%v", boundary, err) + } + selection := &mecatuiclient.ResumeSelection{ + Row: mecatuiclient.SessionListItem{ID: string(sessionID), Title: snapshot.Title, Kind: transcript.Kind}, + Transcript: transcript, Snapshot: current, Pending: &pending, + } + model := ui.New(ui.Deps{ + Conv: ownerClient, PendingApprovals: nativeBridgePendingClient{ownerClient}, Resume: selection, + Theme: theme.New("aztec", theme.AztecPalette()), Ctx: ctx, Workspace: workspace, + InitialPrompt: "draft only", NoAltScreen: true, NoBanner: true, + }) + ready, terminal := make(chan string, 1), make(chan string, 1) + lastView := &atomic.Value{} + tm := teatest.NewTestModel(t, &nativeBridgeModel{Model: model, ready: ready, terminal: terminal, lastView: lastView}, teatest.WithInitialTermSize(100, 40)) + defer func() { + cancel() + if err := tm.Quit(); err != nil { + t.Errorf("quit TUI: %v", err) + } + tm.WaitFinished(t, teatest.WithFinalTimeout(5*time.Second)) + }() + select { + case view := <-ready: + if !strings.Contains(view, "Shell") || !strings.Contains(view, "draft only") { + t.Fatalf("approval view lost original call or draft:\n%s", view) + } + case <-ctx.Done(): + t.Fatalf("TUI did not display approval: %v", ctx.Err()) + } + if stats.ownerConverse.Load() != 0 || stats.ownerResolve.Load() != 0 || continuationRequests.Load() != 0 { + t.Fatalf("startup entered work before keypress: converse=%d resolve=%d model=%d", stats.ownerConverse.Load(), stats.ownerResolve.Load(), continuationRequests.Load()) + } + executor.mu.Lock() + operationsBeforeKey := len(executor.operations) + executor.mu.Unlock() + if operationsBeforeKey != 0 { + t.Fatalf("native operations before keypress = %d", operationsBeforeKey) + } + assertNoNativeActiveRun(t, dyn, ref.ID) + + tm.Send(tea.KeyPressMsg{Code: tc.key, Text: string(tc.key)}) + select { + case <-stats.ownerResolved: + case <-ctx.Done(): + t.Fatalf("owner verdict did not reach public gRPC: %v", ctx.Err()) + } + stats.mu.Lock() + control := stats.ownerControl + stats.mu.Unlock() + wantVerdict := mecatlv1.ApprovalVerdict_APPROVAL_VERDICT_DENY + if tc.wantRun { + wantVerdict = mecatlv1.ApprovalVerdict_APPROVAL_VERDICT_ALLOW_ONCE + } + if control.GetSessionId() != string(sessionID) || control.GetExpectedRunId() != runID || control.GetAskId() != askID || control.GetVerdict() != wantVerdict { + t.Fatal("keypress did not resolve the original session/run/ask with the selected verdict") + } + if tc.wantRun { + select { + case request := <-executor.executed: + if request.Operation != executionenv.OpCommandStart || request.Command != "printf bridge-command" { + t.Fatalf("native request operation=%v command=%q", request.Operation, request.Command) + } + case <-ctx.Done(): + t.Fatalf("allowed command did not reach native RPC: %v", ctx.Err()) + } + stored, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, ref.ID, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + activeRun, found, err := unstructured.NestedMap(stored.Object, "status", "activeRun") + if err != nil || !found || activeRun["runID"] != runID || activeRun["bindingID"] != string(sessionID) || activeRun["ownerHash"] != hex.EncodeToString(ownerSum[:]) { + t.Fatalf("native claim before command return: found=%t run=%v err=%v", found, activeRun["runID"], err) + } + releaseCommand() + } + var ( + observedKinds []mecatuiclient.PendingApprovalEventKind + observedNil bool + toolResults int + ) + for { + event, err := observer.Recv() + if err != nil { + t.Fatalf("observe resumed run after %v: %v", observedKinds, err) + } + observedKinds = append(observedKinds, event.Kind) + observedNil = observedNil || event.Kind == mecatuiclient.PendingApprovalEventOther && event.Message == nil + if result, ok := event.Message.(mecatuiclient.ToolResultMsg); ok { + if result.CallID != "native-shell" || result.IsError == tc.wantRun { + t.Fatal("resumed tool result identity/outcome mismatch") + } + toolResults++ + } + if event.Kind == mecatuiclient.PendingApprovalEventTerminal { + result, ok := event.Message.(mecatuiclient.ResultMsg) + if !ok || result.Stop != "end_turn" { + t.Fatal("resumed run did not end normally") + } + break + } + } + select { + case <-terminal: + case <-time.After(5 * time.Second): + t.Fatalf("TUI did not render terminal continuation:\n%v", lastView.Load()) + } + if err := tm.Quit(); err != nil { + t.Fatalf("quit TUI: %v", err) + } + tm.WaitFinished(t, teatest.WithFinalTimeout(5*time.Second)) + final, ok := tm.FinalModel(t).(*nativeBridgeModel) + if !ok { + t.Fatalf("final model = %T", tm.FinalModel(t)) + } + stats.mu.Lock() + unmappedKinds := append([]string(nil), stats.unmappedKinds...) + stats.mu.Unlock() + t.Logf("unmapped public event.type values: %q", unmappedKinds) + view := ansi.Strip(final.View().Content) + if strings.Contains(strings.ToLower(view), "connection failed") { + t.Fatalf("actual TUI rejected the native continuation (unmapped ordinary event=%t; events=%v):\n%s", observedNil, observedKinds, view) + } + if final.ActiveSessionID() != string(sessionID) { + t.Fatalf("active session = %q, want %q", final.ActiveSessionID(), sessionID) + } + if toolResults != 1 || continuationRequests.Load() != 1 || stats.ownerConverse.Load() != 0 || stats.ownerResolve.Load() != 1 || stats.foreignCalls.Load() != 3 { + t.Fatalf("unexpected work counts: tool-results=%d model=%d converse=%d resolve=%d foreign=%d", toolResults, continuationRequests.Load(), stats.ownerConverse.Load(), stats.ownerResolve.Load(), stats.foreignCalls.Load()) + } + if strings.Count(view, `command: "printf bridge-command"`) != 1 || strings.Count(view, "│ ✓ Shell")+strings.Count(view, "│ ✗ Shell") != 1 || !strings.Contains(view, tc.wantOutput) || !strings.Contains(view, "draft only") || strings.Contains(view, "enter queue") || strings.Contains(view, "… Shell") { + t.Fatalf("terminal view lost draft, continuation, or single resolved tool card:\n%s", view) + } + executor.mu.Lock() + operations := append([]executionenv.Operation(nil), executor.operations...) + executor.mu.Unlock() + if tc.wantRun { + if len(operations) != 1 || operations[0] != executionenv.OpCommandStart || !strings.Contains(view, "out�") || !strings.Contains(view, "[exit code: 0]") { + t.Fatalf("allow result projection: operations=%v\n%s", operations, view) + } + } else if len(operations) != 0 || !strings.Contains(view, "denied") { + t.Fatalf("deny executed original command or lost refusal: operations=%v\n%s", operations, view) + } + }) + } +} + +var _ ui.PendingApprovalController = nativeBridgePendingClient{} diff --git a/internal/adapter/executionclient/persisted_ask_integration_test.go b/internal/adapter/executionclient/persisted_ask_integration_test.go new file mode 100644 index 0000000000..e43c08f87f --- /dev/null +++ b/internal/adapter/executionclient/persisted_ask_integration_test.go @@ -0,0 +1,227 @@ +package executionclient + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "sync" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/adapter/store/jsonlstore" + "github.com/stacklok/mecatl/internal/app" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestNativeBuildResolvePersistedShellAskAfterRestart(t *testing.T) { + for _, key := range []string{"HOME", "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME"} { + t.Setenv(key, t.TempDir()) + } + profilePath := filepath.Join(t.TempDir(), "profiles.yaml") + profileYAML := []byte("profiles:\n coding:\n image: example.test/executor@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n storageClass: standard\n storageSize: 1Gi\n cpuRequest: 100m\n memoryRequest: 128Mi\n cpuLimit: 1\n memoryLimit: 1Gi\n ephemeralStorageRequest: 256Mi\n ephemeralStorageLimit: 1Gi\n tmpSizeLimit: 128Mi\n runtimeClassName: gvisor\n maxFileBytes: 5242880\n maxCommandBytes: 1048576\n maxCommandDuration: 1m\n maxEnvironments: 10\n") + if err := os.WriteFile(profilePath, profileYAML, 0o600); err != nil { + t.Fatal(err) + } + profiles, err := executioncontroller.LoadProfiles(profilePath) + if err != nil { + t.Fatal(err) + } + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{executioncontroller.ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + executor := &recordingExecutor{} + controllerStore := executioncontroller.NewStore(dyn, "ns", profiles, executor) + profile, err := controllerStore.ValidateProfile(t.Context(), "coding") + if err != nil { + t.Fatal(err) + } + owner := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + ownerSum := sha256.Sum256([]byte(owner.Issuer + "\x00" + owner.Subject)) + clientSum := sha256.Sum256([]byte("spiffe://example.test/mecak8s")) + now := time.Now().UTC() + const sessionID = session.SessionID("persisted-native-ask") + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env-persisted", Revision: "rev-1"} + envObj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": ref.ID, "namespace": "ns"}, + "spec": map[string]any{"schemaVersion": int64(2), "revision": ref.Revision, "ownerHash": hex.EncodeToString(ownerSum[:]), "ownerIssuer": owner.Issuer, "ownerSubject": owner.Subject, "clientHash": hex.EncodeToString(clientSum[:]), "bindingID": string(sessionID), "profile": "coding", "profileDigest": profile.Digest, "desired": "Active"}, + "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "pod": map[string]any{"name": "executor-pod"}, "references": []any{map[string]any{"bindingID": string(sessionID), "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}, + }} + if _, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Create(t.Context(), envObj, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + fx := startFixture(t, controllerStore, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + + storeDir := t.TempDir() + store, err := jsonlstore.New(storeDir) + if err != nil { + t.Fatal(err) + } + parked := session.New(sessionID, session.ModeDefault, ref, session.Limits{MaxTurns: 4}, now) + if err := parked.RestoreLabels(owner, session.Authority{}); err != nil { + t.Fatal(err) + } + if err := store.Save(t.Context(), parked); err != nil { + t.Fatal(err) + } + settings := filepath.Join(t.TempDir(), "settings.yaml") + if err := os.WriteFile(settings, []byte("permissions:\n ask:\n - Shell\n"), 0o600); err != nil { + t.Fatal(err) + } + baseConfig := func(model port.LLMProvider) app.Config { + return app.Config{ + UseMock: true, MockProvider: model, Workspace: t.TempDir(), StoreDir: storeDir, + UserModelDir: t.TempDir(), MemoryDir: t.TempDir(), NoSoul: true, Headless: true, + RemoteExecution: true, TrustProject: true, OwnershipEnforced: true, + PlacementProvider: provider, PlacementScope: "native-test", PermissionConfigs: []string{settings}, + } + } + ctx := session.WithPrincipal(t.Context(), owner) + builtA, err := app.Build(t.Context(), baseConfig(mockllm.New(mockllm.ToolCallTurn( + session.NewToolCall("shell", "Shell", json.RawMessage(`{"command":"true"}`)), + )))) + if err != nil { + t.Fatal(err) + } + run, err := builtA.Service.StartRunContent(ctx, sessionID, "run shell", nil) + if err != nil { + builtA.Close() + t.Fatal(err) + } + var runID, askID string + for event := range run.Events() { + if event.Type == session.EvPermissionAsk && event.Ask != nil { + runID, askID = event.RunID, event.Ask.AskID + builtA.Service.Persist(ctx, sessionID) + break + } + } + if runID == "" || askID == "" { + builtA.Close() + t.Fatal("native Shell run did not persist an exact ask") + } + drained := make(chan struct{}) + go func() { + for range run.Events() { + } + builtA.Service.FinishRun(sessionID, run) + close(drained) + }() + builtA.Close() + select { + case <-drained: + case <-time.After(time.Second): + t.Fatal("normal transport relay did not drain the parked run during shutdown") + } + + continuationEntered := make(chan struct{}) + continueModel := make(chan struct{}) + builtB, err := app.Build(t.Context(), baseConfig(mockllm.NewWith([]mockllm.Option{ + mockllm.WithRequestObserver(func(port.LLMRequest) { + close(continuationEntered) + <-continueModel + }), + }, mockllm.TextTurn("done")))) + if err != nil { + t.Fatal(err) + } + defer builtB.Close() + releaseModel := sync.OnceFunc(func() { close(continueModel) }) + defer releaseModel() + foreign := session.WithPrincipal(t.Context(), &session.Principal{Issuer: owner.Issuer, Subject: "mallory", GrantType: session.GrantTypeUser}) + if _, err := builtB.Service.ResolveRunAsk(foreign, sessionID, runID, askID, session.VerdictAllowOnce); !errors.Is(err, server.ErrNotFound) { + t.Fatalf("foreign resolve = %v, want concealed not found", err) + } + assertNoNativeActiveRun(t, dyn, ref.ID) + + if _, err := builtB.Service.ResolveRunAsk(ctx, sessionID, runID, askID, session.VerdictAllowOnce); err != nil { + t.Fatal(err) + } + select { + case <-continuationEntered: + case <-time.After(time.Second): + t.Fatal("resumed run did not reach its model continuation") + } + stored, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), ref.ID, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + activeRun, found, err := unstructured.NestedMap(stored.Object, "status", "activeRun") + if err != nil || !found { + t.Fatalf("native claim not held during continuation: found=%t err=%v", found, err) + } + for field, want := range map[string]string{ + "runID": runID, "bindingID": string(sessionID), "ownerHash": hex.EncodeToString(ownerSum[:]), + } { + if got := activeRun[field]; got != want { + t.Fatalf("native claim %s = %v, want %q", field, got, want) + } + } + competing, err := provider.AcquireRun(ctx, server.ExecutionRunRequest{Ref: ref, Principal: owner, BindingID: sessionID, RunID: runID}) + if err == nil { + _ = competing.Release(t.Context()) + t.Fatal("competing native claim acquired while resumed run was active") + } + releaseModel() + waitForNoNativeActiveRun(t, dyn, ref.ID) + executor.mu.Lock() + operations := append([]executionenv.Operation(nil), executor.operations...) + executor.mu.Unlock() + if len(operations) != 1 || operations[0] != executionenv.OpCommandStart { + t.Fatalf("native operations = %v, want one command dispatch", operations) + } +} + +func assertNoNativeActiveRun(t *testing.T, dyn *dynamicfake.FakeDynamicClient, id string) { + t.Helper() + stored, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), id, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if _, found, err := unstructured.NestedMap(stored.Object, "status", "activeRun"); err != nil || found { + t.Fatalf("unexpected native claim: found=%t err=%v", found, err) + } +} + +func waitForNoNativeActiveRun(t *testing.T, dyn *dynamicfake.FakeDynamicClient, id string) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + ticker := time.NewTicker(time.Millisecond) + defer ticker.Stop() + for { + stored, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, id, metav1.GetOptions{}) + if err == nil { + if _, found, nestedErr := unstructured.NestedMap(stored.Object, "status", "activeRun"); nestedErr == nil && !found { + return + } + } + select { + case <-ctx.Done(): + t.Fatal("native claim was not released after detached relay drain") + case <-ticker.C: + } + } +} diff --git a/internal/adapter/executionclient/reference_ambiguity_test.go b/internal/adapter/executionclient/reference_ambiguity_test.go new file mode 100644 index 0000000000..8fc96fcdf8 --- /dev/null +++ b/internal/adapter/executionclient/reference_ambiguity_test.go @@ -0,0 +1,98 @@ +package executionclient + +import ( + "context" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type droppedCommitRPC struct { + executionv1.ExecutionProviderServiceClient + commits int + aborts int + lastAbort *executionv1.ReferenceMutationRequest + lastSuccessor *executionv1.ReserveSuccessorRequest +} + +func (r *droppedCommitRPC) CommitReference(context.Context, *executionv1.ReferenceMutationRequest, ...grpc.CallOption) (*emptypb.Empty, error) { + r.commits++ + return nil, status.Error(codes.Unavailable, "response lost") +} +func (r *droppedCommitRPC) AbortReference(_ context.Context, req *executionv1.ReferenceMutationRequest, _ ...grpc.CallOption) (*emptypb.Empty, error) { + r.aborts++ + r.lastAbort = req + return &emptypb.Empty{}, nil +} +func (r *droppedCommitRPC) ReserveSuccessor(_ context.Context, req *executionv1.ReserveSuccessorRequest, _ ...grpc.CallOption) (*executionv1.ReferenceReservationResponse, error) { + r.lastSuccessor = req + return &executionv1.ReferenceReservationResponse{}, nil +} +func (*droppedCommitRPC) AttachEnvironment(_ context.Context, req *executionv1.AttachEnvironmentRequest, _ ...grpc.CallOption) (*executionv1.AttachEnvironmentResponse, error) { + return &executionv1.AttachEnvironmentResponse{Environment: req.GetContext().GetEnvironment(), Epoch: 1, Ready: true, GrantGeneration: 1}, nil +} + +func TestExclusiveSuccessorReservationAbortsOnPrepublicationFailure(t *testing.T) { + rpc := &droppedCommitRPC{} + provider := &Provider{client: &Client{rpc: rpc}, profile: "coding"} + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + binding, err := provider.ReserveSuccessor(t.Context(), server.PlacementSuccessorRequest{ + Ref: ref, Principal: &session.Principal{Issuer: "issuer", Subject: "alice"}, + SourceBindingID: "source", DestinationBindingID: "destination", + }) + if err != nil { + t.Fatal(err) + } + if err := binding.Close(); err != nil { + t.Fatal(err) + } + if rpc.commits != 0 || rpc.aborts != 1 { + t.Fatalf("commits=%d aborts=%d, want one conclusive successor abort", rpc.commits, rpc.aborts) + } + if rpc.lastSuccessor == nil || rpc.lastAbort.GetBindingId() != "destination" || rpc.lastAbort.GetOperationId() != rpc.lastSuccessor.GetOperationId() || rpc.lastAbort.GetOperationId() == "" { + t.Fatalf("reserve=%+v abort binding=%q operation=%q, want exact successor destination and operation", rpc.lastSuccessor, rpc.lastAbort.GetBindingId(), rpc.lastAbort.GetOperationId()) + } + if err := binding.Close(); err != nil { + t.Fatal(err) + } + if rpc.aborts != 1 { + t.Fatalf("second close issued another abort: %d", rpc.aborts) + } +} + +func TestSharedInitialReservationIsRetainedOnPrepublicationFailure(t *testing.T) { + rpc := &droppedCommitRPC{} + provider := &Provider{client: &Client{rpc: rpc}, profile: "coding"} + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + binding := provider.withSharedInitialReferenceTransaction(server.PlacementBinding{Ref: ref}, executionenv.Owner{Issuer: "issuer", Subject: "alice"}, "session", "operation") + if err := binding.Close(); err != nil { + t.Fatal(err) + } + if rpc.commits != 0 || rpc.aborts != 0 { + t.Fatalf("commits=%d aborts=%d, shared initial reservation must remain pending", rpc.commits, rpc.aborts) + } +} + +func TestAmbiguousCommitIsRetainedAndNeverAbortedByCleanup(t *testing.T) { + rpc := &droppedCommitRPC{} + provider := &Provider{client: &Client{rpc: rpc}, profile: "coding"} + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + binding := provider.withSharedInitialReferenceTransaction(server.PlacementBinding{Ref: ref}, executionenv.Owner{Issuer: "issuer", Subject: "alice"}, "session", "operation") + if err := binding.Commit(t.Context()); err == nil { + t.Fatal("commit transport ambiguity was hidden") + } + if err := binding.Close(); err != nil { + t.Fatal(err) + } + if rpc.commits != 1 || rpc.aborts != 0 { + t.Fatalf("commits=%d aborts=%d", rpc.commits, rpc.aborts) + } +} diff --git a/internal/adapter/executionclient/service_integration_test.go b/internal/adapter/executionclient/service_integration_test.go new file mode 100644 index 0000000000..0c29878d2d --- /dev/null +++ b/internal/adapter/executionclient/service_integration_test.go @@ -0,0 +1,232 @@ +package executionclient + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + "time" + "unicode/utf8" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + + "github.com/stacklok/mecatl/engine/adapter/fstools" + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/adapter/permpolicy" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/executioncontroller" + "github.com/stacklok/mecatl/internal/adapter/server" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type recordingExecutor struct { + mu sync.Mutex + operations []executionenv.Operation + executed chan executionenv.ExecutorRequest + release <-chan struct{} +} + +func (e *recordingExecutor) Execute(ctx context.Context, _ string, req executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + e.mu.Lock() + e.operations = append(e.operations, req.Operation) + e.mu.Unlock() + if e.executed != nil { + select { + case e.executed <- req: + case <-ctx.Done(): + return executionenv.ExecutorResponse{}, ctx.Err() + } + select { + case <-e.release: + case <-ctx.Done(): + return executionenv.ExecutorResponse{}, ctx.Err() + } + } + switch req.Operation { + case executionenv.OpFileResolveAuthority: + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{AuthorityTarget: "/workspace/main.go", AuthorityWorkspace: "/workspace"}}, nil + case executionenv.OpFileRead: + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Data: []byte("package main\n"), Version: "v1"}}, nil + case executionenv.OpCommandStart: + return executionenv.ExecutorResponse{Command: &executionenv.CommandStatusResponse{CommandID: req.CommandID, State: executionenv.CommandSucceeded, Stdout: []byte("out\xff"), Stderr: []byte("err\xe2"), TerminalReceipt: "complete"}}, nil + default: + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "unsupported test operation"} + } +} + +func TestServiceUsesRealMTLSProviderStoreAndReleasesOnlyAfterDrain(t *testing.T) { + profilePath := filepath.Join(t.TempDir(), "profiles.yaml") + profileYAML := []byte("profiles:\n coding:\n image: example.test/executor@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n storageClass: standard\n storageSize: 1Gi\n cpuRequest: 100m\n memoryRequest: 128Mi\n cpuLimit: 1\n memoryLimit: 1Gi\n ephemeralStorageRequest: 256Mi\n ephemeralStorageLimit: 1Gi\n tmpSizeLimit: 128Mi\n runtimeClassName: gvisor\n maxFileBytes: 5242880\n maxCommandBytes: 1048576\n maxCommandDuration: 1m\n maxEnvironments: 10\n") + if err := os.WriteFile(profilePath, profileYAML, 0o600); err != nil { + t.Fatal(err) + } + profiles, err := executioncontroller.LoadProfiles(profilePath) + if err != nil { + t.Fatal(err) + } + dyn := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{executioncontroller.ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + executor := &recordingExecutor{} + controllerStore := executioncontroller.NewStore(dyn, "ns", profiles, executor) + profile, err := controllerStore.ValidateProfile(t.Context(), "coding") + if err != nil { + t.Fatal(err) + } + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + ownerSum := sha256.Sum256([]byte(owner.Issuer + "\x00" + owner.Subject)) + clientSum := sha256.Sum256([]byte("spiffe://example.test/mecak8s")) + now := time.Now().UTC() + envObj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env-real", "namespace": "ns"}, + "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev-1", "ownerHash": hex.EncodeToString(ownerSum[:]), "ownerIssuer": owner.Issuer, "ownerSubject": owner.Subject, "clientHash": hex.EncodeToString(clientSum[:]), "bindingID": "service-session", "profile": "coding", "profileDigest": profile.Digest, "desired": "Active"}, + "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "pod": map[string]any{"name": "executor-pod"}, "references": []any{map[string]any{"bindingID": "service-session", "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}, + }} + if _, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Create(t.Context(), envObj, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + fx := startFixture(t, controllerStore, nil) + defer fx.stop() + client, err := New(fx.endpoint, fx.clientTLS) + if err != nil { + t.Fatal(err) + } + defer client.Close() + provider, err := NewProvider(client, "coding") + if err != nil { + t.Fatal(err) + } + catalog := tool.NewCatalog() + catalog.MustRegister(fstools.ReadTool{}) + catalog.MustRegister(agent.NewShellTool()) + var modelShellResult string + llm := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { + for _, message := range req.Messages { + if result := message.ToolResult; result != nil && result.CallID == "shell" { + modelShellResult = result.Content + } + } + })}, + mockllm.ToolCallTurn(session.NewToolCall("read", "Read", json.RawMessage(`{"path":"main.go"}`))), + mockllm.ToolCallTurn(session.NewToolCall("background", "Shell", json.RawMessage(`{"command":"touch background-leak","background":true}`))), + mockllm.ToolCallTurn(session.NewToolCall("shell", "Shell", json.RawMessage(`{"command":"true"}`))), + mockllm.TextTurn("done"), + mockllm.TextTurn("continued"), + ) + engine := agent.NewEngine(agent.Deps{LLM: llm, Catalog: catalog, Policy: permpolicy.NewPolicy(permpolicy.AllowAllFloorRules(), nil), Model: "test"}) + sessions := memstore.New() + svc, err := server.NewService(server.Config{Engine: engine, Store: sessions, PlacementProvider: provider, PlacementScope: "test", ExecutionAccess: provider, DefaultLimits: session.Limits{MaxTurns: 5}}) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + principal := &session.Principal{Issuer: owner.Issuer, Subject: owner.Subject, GrantType: session.GrantTypeUser} + sess := session.New("service-session", session.ModeDefault, session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env-real", Revision: "rev-1"}, session.Limits{MaxTurns: 5}, now) + if err := sess.RestoreLabels(principal, session.Authority{}); err != nil { + t.Fatal(err) + } + if err := sessions.Save(t.Context(), sess); err != nil { + t.Fatal(err) + } + ctx := session.WithPrincipal(t.Context(), principal) + run, err := svc.StartRunContent(ctx, sess.ID, "inspect", nil) + if err != nil { + t.Fatal(err) + } + backgroundRejected := false + var streamedShellResult string + for event := range run.Events() { + if result := event.ToolResult; result != nil && result.CallID == "shell" { + streamedShellResult = result.Content + } + if result := event.ToolResult; result != nil && result.CallID == "background" { + backgroundRejected = result.IsError && strings.Contains(result.Content, "not supported by this command runner") + } + } + if !backgroundRejected { + t.Fatal("remote background Shell did not return the named capability error") + } + if !utf8.ValidString(streamedShellResult) || !strings.Contains(streamedShellResult, "out�") || !strings.Contains(streamedShellResult, "err�") || modelShellResult != streamedShellResult { + t.Fatalf("private protobuf bytes were not repaired identically for client/model: stream=%q model=%q", streamedShellResult, modelShellResult) + } + executor.mu.Lock() + operations := append([]executionenv.Operation(nil), executor.operations...) + executor.mu.Unlock() + wantOperations := []executionenv.Operation{executionenv.OpFileRead, executionenv.OpCommandStart} + if !reflect.DeepEqual(operations, wantOperations) { + t.Fatalf("executor operations=%v, want %v", operations, wantOperations) + } + if _, err := svc.StartRunContent(ctx, sess.ID, "early", nil); err == nil { + t.Fatal("replacement run acquired before relay release") + } + svc.FinishRun(sess.ID, run) + stored, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env-real", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if _, found, err := unstructured.NestedMap(stored.Object, "status", "activeRun"); err != nil || found { + t.Fatalf("native claim remained after actual drain release: found=%t err=%v", found, err) + } + continued, err := svc.StartRunContent(ctx, sess.ID, "continue", nil) + if err != nil { + t.Fatal(err) + } + for range continued.Events() { + } + svc.Persist(ctx, sess.ID) + svc.FinishRun(sess.ID, continued) + + stored, err = dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, "env-real", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + before, err := sessions.Load(ctx, sess.ID) + if err != nil || len(before.Conversation.Messages) == 0 { + t.Fatalf("source history missing: %v", err) + } + placement := server.SuccessorPlacement{Selector: "unsupported-worktree", SelectorPresent: true} + // A well-formed selector and binding ID reach the native provider's unsupported + // selection gate, not the Service's malformed-request validation. + if _, err := provider.Bind(ctx, server.PlacementBindRequest{Selector: server.SelectWorktree(sess.ID, sess.EnvironmentRef, placement.Selector), BindingID: "unused-destination", Principal: principal}); !errors.Is(err, server.ErrInvalidPlacementSelection) { + t.Fatalf("native selector gate: %v", err) + } + for _, operation := range []string{"clear", "fork"} { + t.Run(operation+"-unsupported-selector", func(t *testing.T) { + var destination session.SessionID + var err error + if operation == "clear" { + destination, err = svc.ClearSessionSuccessor(ctx, sess.ID, placement) + } else { + destination, err = svc.ForkSessionSuccessor(ctx, server.ForkSuccessorRequest{Source: sess.ID, Placement: placement}) + } + if !errors.Is(err, server.ErrInvalidPlacementSelection) || destination != "" { + t.Fatalf("destination=%q error=%v", destination, err) + } + after, err := sessions.Load(ctx, sess.ID) + if err != nil || after.EnvironmentRef != before.EnvironmentRef || !reflect.DeepEqual(after.Conversation, before.Conversation) { + t.Fatalf("unsupported successor changed source ref/history: %v", err) + } + all, err := svc.ListSessions(ctx) + if err != nil || len(all) != 1 || all[0].SessionID != string(sess.ID) { + t.Fatalf("unsupported successor published a destination: %+v, %v", all, err) + } + current, err := dyn.Resource(executioncontroller.ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, "env-real", metav1.GetOptions{}) + if err != nil || !reflect.DeepEqual(current.Object, stored.Object) { + t.Fatalf("unsupported successor mutated provider references: %v", err) + } + }) + } +} diff --git a/internal/adapter/executioncontroller/admin_lifecycle.go b/internal/adapter/executioncontroller/admin_lifecycle.go new file mode 100644 index 0000000000..ecb0b3f9e8 --- /dev/null +++ b/internal/adapter/executioncontroller/admin_lifecycle.go @@ -0,0 +1,484 @@ +package executioncontroller + +import ( + "context" + "math" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +type adminLifecycleRequest struct { + Environment executionenv.EnvironmentRef + OwnerHash string + Client string + AdministratorFor []string + ExpectedEpoch uint64 + ExpectedPodUID string + ExpectedPVCUID string + OperationID string + ExpectedSchema int64 +} + +// ReplaceExecutor starts an exact, crash-recoverable executor replacement. +func (s *Store) ReplaceExecutor(ctx context.Context, q adminLifecycleRequest) error { + return s.startLifecycle(ctx, q, "ReplaceExecutor", false) +} + +// RetireExact starts exact executor retirement while retaining its workspace. +func (s *Store) RetireExact(ctx context.Context, q adminLifecycleRequest) error { + return s.startLifecycle(ctx, q, "RetireEnvironment", true) +} + +func (s *Store) startLifecycle(ctx context.Context, q adminLifecycleRequest, kind string, requireNoRefs bool) error { //nolint:gocyclo // Exact admission keeps all identity and receipt checks together. + if q.ExpectedEpoch == 0 || q.ExpectedEpoch > math.MaxInt64 || q.ExpectedPodUID == "" || q.ExpectedPVCUID == "" || q.OperationID == "" { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "exact lifecycle identity is required"} + } + if err := s.persistObservedTerminalProof(ctx, q); err != nil { + return err + } + return s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { + if kind == "ReplaceExecutor" && replacementReceiptMatches(o, q) { + return nil + } + if err := exactAdminSubject(o, q); err != nil { + return err + } + if kind == "RetireEnvironment" && conditionTrue(o, "Retired") { + if exactTerminationProofOperationMatches(o, q) { + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is already retired by another operation"} + } + if existing, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation"); found { + if text(existing, "id") == q.OperationID && text(existing, "type") == kind && text(existing, "expectedPodUID") == q.ExpectedPodUID && text(existing, "expectedPVCUID") == q.ExpectedPVCUID && intNested(existing, "expectedEpoch") == int64(q.ExpectedEpoch) { //nolint:gosec // validated above. + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "another lifecycle operation is active"} + } + if conditionTrue(o, "Retired") { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is already retired"} + } + expiredRun := expiredLifecycleClaimMatches(o, q, s.now()) + if lifecycleAdmissionBlocked(o, q, expiredRun) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not healthy and idle"} + } + refs, err := referenceRecords(o) + if err != nil { + return err + } + if requireNoRefs && len(refs) != 0 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment still has references"} + } + if expiredRun { + unstructured.RemoveNestedField(o.Object, "status", "activeRun") + } + operation := map[string]any{"id": q.OperationID, "type": kind, "phase": "Quiescing", "expectedEpoch": int64(q.ExpectedEpoch), "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID, "createdAt": s.now().Format(time.RFC3339Nano)} //nolint:gosec // validated above. + setConditionObject(o, "Ready", false, "Quiescing", "administrator lifecycle operation is quiescing the executor") + return unstructured.SetNestedMap(o.Object, operation, "status", "lifecycleOperation") + }) +} + +func lifecycleAdmissionBlocked(o *unstructured.Unstructured, q adminLifecycleRequest, expiredRun bool) bool { + return textNested(o.Object, "status", "activeRun", "claimID") != "" && !expiredRun || + textNested(o.Object, "status", "activeOperation", "id") != "" || + textNested(o.Object, "status", "fenceState") != fenceHealthy || + !conditionTrue(o, "Ready") && !exactTerminationProofMatches(o, q) +} + +func expiredLifecycleClaimMatches(o *unstructured.Unstructured, q adminLifecycleRequest, now time.Time) bool { + if textNested(o.Object, "status", "activeRun", "claimID") == "" { + return false + } + claim, _, expiry, ok := activeRunFrom(o) + return ok && !now.Before(expiry) && intNested(o.Object, "status", "activeRun", "epoch") == int64(q.ExpectedEpoch) && //nolint:gosec // q.ExpectedEpoch is bounded by startLifecycle. + claim.Epoch == q.ExpectedEpoch && epochMatches(o, q.ExpectedEpoch) && generationMatches(o, claim.GrantGeneration) +} + +func replacementReceiptMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + return textNested(o.Object, "status", "lastReplacement", operationIDField) == q.OperationID && + textNested(o.Object, "status", "lastReplacement", "previousPodUID") == q.ExpectedPodUID && + textNested(o.Object, "status", "lastReplacement", "pvcUID") == q.ExpectedPVCUID && + intNested(o.Object, "status", "lastReplacement", "previousEpoch") == int64(q.ExpectedEpoch) //nolint:gosec // q.ExpectedEpoch is bounded by startLifecycle. +} + +func exactTerminationProofMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + return textNested(o.Object, "status", "terminationProof", "podUID") == q.ExpectedPodUID && + textNested(o.Object, "status", "terminationProof", "pvcUID") == q.ExpectedPVCUID && + intNested(o.Object, "status", "terminationProof", "epoch") == int64(q.ExpectedEpoch) //nolint:gosec // q.ExpectedEpoch is bounded above. +} + +func exactTerminationProofOperationMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + return exactTerminationProofMatches(o, q) && textNested(o.Object, "status", "terminationProof", operationIDField) == q.OperationID +} + +func (s *Store) persistObservedTerminalProof(ctx context.Context, q adminLifecycleRequest) error { //nolint:gocyclo // Exact terminal proof deliberately fails closed at every observable mismatch. + o, err := s.resources.Get(ctx, q.Environment.ID, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if err != nil { + return err + } + if err := adminSubject(o, q); err != nil { + return err + } + if err := requireCurrentSchema(o); err != nil { + return err + } + if replacementReceiptMatches(o, q) { + return nil + } + if err := exactAdminSubject(o, q); err != nil { + return err + } + if conditionTrue(o, "Ready") || conditionTrue(o, "Retired") || exactTerminationProofOperationMatches(o, q) { + return nil + } + expiredRun := expiredLifecycleClaimMatches(o, q, s.now()) + if textNested(o.Object, "status", "activeRun", "claimID") != "" && !expiredRun || textNested(o.Object, "status", "activeOperation", "id") != "" || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not healthy and idle"} + } + if exactTerminationProofMatches(o, q) { + return nil + } + if s.kube == nil { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not healthy and idle"} + } + podName, pvcName := textNested(o.Object, "status", "pod", "name"), textNested(o.Object, "status", "pvc", "name") + pod, err := s.kube.CoreV1().Pods(s.namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil || string(pod.UID) != q.ExpectedPodUID || !podTerminal(pod) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "exact terminal executor proof is unavailable"} + } + pvc, err := s.kube.CoreV1().PersistentVolumeClaims(s.namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil || string(pvc.UID) != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "exact workspace identity is unavailable"} + } + return s.retryAdminStatus(ctx, q, func(current *unstructured.Unstructured) error { + if err := exactAdminSubject(current, q); err != nil { + return err + } + if conditionTrue(current, "Ready") || exactTerminationProofOperationMatches(current, q) { + return nil + } + expiredRun := expiredLifecycleClaimMatches(current, q, s.now()) + if textNested(current.Object, "status", "activeRun", "claimID") != "" && !expiredRun || textNested(current.Object, "status", "activeOperation", "id") != "" || textNested(current.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not healthy and idle"} + } + if exactTerminationProofMatches(current, q) { + return nil + } + return unstructured.SetNestedMap(current.Object, terminationProof(q, pod), "status", "terminationProof") + }) +} + +// adminSubject uses only the policy captured by authentication for this request. +// The actor remains distinct from the immutable creator, including on receipt replay. +func adminSubject(o *unstructured.Unstructured, q adminLifecycleRequest) error { + creator := textNested(o.Object, "spec", "clientHash") + allowed := creator == hashText(q.Client) + for _, uri := range q.AdministratorFor { + allowed = allowed || creator == hashText(uri) + } + if !allowed || textNested(o.Object, "spec", "ownerHash") != q.OwnerHash || textNested(o.Object, "spec", "revision") != q.Environment.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + return nil +} + +func (s *Store) retryAdminStatus(ctx context.Context, q adminLifecycleRequest, mutate func(*unstructured.Unstructured) error) error { + return s.retryUpdateStatusRaw(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if err := adminSubject(o, q); err != nil { + return err + } + if err := requireCurrentSchema(o); err != nil { + return err + } + return mutate(o) + }) +} + +func exactAdminSubject(o *unstructured.Unstructured, q adminLifecycleRequest) error { + if err := adminSubject(o, q); err != nil { + return err + } + if !epochMatches(o, q.ExpectedEpoch) || textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment execution identity changed"} + } + return nil +} + +// RecoverEnvironment clears uncertainty only after built-in exact terminal verification. +func (s *Store) RecoverEnvironment(ctx context.Context, q adminLifecycleRequest) error { + if s.kube == nil || q.ExpectedPodUID == "" || q.ExpectedPVCUID == "" || q.OperationID == "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "built-in recovery verifier is unavailable"} + } + podName, pvcName := "", "" + if err := s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { + if err := exactAdminSubject(o, q); err != nil { + return err + } + if exactTerminationProofOperationMatches(o, q) { + return nil + } + if textNested(o.Object, "status", "fenceState") != "FenceUnknown" { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not fenced"} + } + podName, pvcName = textNested(o.Object, "status", "pod", "name"), textNested(o.Object, "status", "pvc", "name") + return nil + }); err != nil { + return err + } + if podName == "" { + return nil + } + pod, err := s.kube.CoreV1().Pods(s.namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil || string(pod.UID) != q.ExpectedPodUID || !podTerminal(pod) { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "terminal executor proof unavailable; use the external-fencing operator runbook"} + } + pvc, err := s.kube.CoreV1().PersistentVolumeClaims(s.namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil || string(pvc.UID) != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "workspace identity cannot be verified"} + } + return s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { + if err := exactAdminSubject(o, q); err != nil { + return err + } + if exactTerminationProofOperationMatches(o, q) { + return nil + } + proof := terminationProof(q, pod) + if err := unstructured.SetNestedMap(o.Object, proof, "status", "terminationProof"); err != nil { + return err + } + unstructured.RemoveNestedField(o.Object, "status", "activeOperation") + unstructured.RemoveNestedField(o.Object, "status", "activeRun") + return unstructured.SetNestedField(o.Object, fenceHealthy, "status", "fenceState") + }) +} + +func terminationProof(q adminLifecycleRequest, pod *corev1.Pod) map[string]any { + return map[string]any{operationIDField: q.OperationID, "podUID": q.ExpectedPodUID, "pvcUID": q.ExpectedPVCUID, "epoch": int64(q.ExpectedEpoch), "podPhase": string(pod.Status.Phase), "observedAt": time.Now().UTC().Format(time.RFC3339Nano)} //nolint:gosec // epoch is validated by caller. +} + +func podTerminal(pod *corev1.Pod) bool { + if pod == nil || pod.Status.Phase != corev1.PodSucceeded && pod.Status.Phase != corev1.PodFailed || len(pod.Spec.Containers) == 0 { + return false + } + if !allDeclaredContainersTerminated(pod.Spec.InitContainers, pod.Status.InitContainerStatuses) || !allDeclaredContainersTerminated(pod.Spec.Containers, pod.Status.ContainerStatuses) { + return false + } + return allDeclaredContainersTerminated(pod.Spec.EphemeralContainers, pod.Status.EphemeralContainerStatuses) +} + +func allDeclaredContainersTerminated[T interface { + corev1.Container | corev1.EphemeralContainer +}](declared []T, statuses []corev1.ContainerStatus) bool { + if len(statuses) != len(declared) { + return false + } + names := make(map[string]struct{}, len(declared)) + for _, container := range declared { + var name string + switch c := any(container).(type) { + case corev1.Container: + name = c.Name + case corev1.EphemeralContainer: + name = c.Name + } + if name == "" { + return false + } + names[name] = struct{}{} + } + if len(names) != len(declared) { + return false + } + seen := make(map[string]struct{}, len(statuses)) + for _, status := range statuses { + if _, declared := names[status.Name]; !declared || status.State.Terminated == nil { + return false + } + if _, duplicate := seen[status.Name]; duplicate { + return false + } + seen[status.Name] = struct{}{} + } + return len(seen) == len(names) +} + +// DeleteRetiredEnvironment starts exact deletion of an explicitly retained workspace. +func (s *Store) DeleteRetiredEnvironment(ctx context.Context, q adminLifecycleRequest) error { + if q.ExpectedPVCUID == "" || q.OperationID == "" { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "exact retained workspace identity is required"} + } + return s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "retained workspace identity changed"} + } + if existing, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation"); found { + if text(existing, "id") == q.OperationID && text(existing, "type") == deleteRetiredEnvironment { + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "another lifecycle operation is active"} + } + refs, err := referenceRecords(o) + if err != nil || len(refs) != 0 || textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "activeOperation", "id") != "" || !conditionTrue(o, "Retired") || !conditionTrue(o, "ExecutorTerminated") { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not safely retained and unreferenced"} + } + return unstructured.SetNestedMap(o.Object, map[string]any{"id": q.OperationID, "type": deleteRetiredEnvironment, "phase": "DeletingPVC", "expectedPVCUID": q.ExpectedPVCUID, "createdAt": s.now().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + }) +} + +// MigrateEnvironment explicitly upgrades a recognized, verified prototype schema. +func (s *Store) MigrateEnvironment(ctx context.Context, q adminLifecycleRequest) error { //nolint:gocyclo // Two Kubernetes subresources require a durable phased migration. + if s.kube == nil || q.ExpectedSchema < 0 || q.ExpectedSchema > 1 || q.OperationID == "" || q.ExpectedPodUID == "" || q.ExpectedPVCUID == "" { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "recognized prototype schema and exact runtime identities are required"} + } + if err := s.retryUpdateStatusRaw(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if err := adminSubject(o, q); err != nil { + return err + } + if textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype runtime identity mismatch"} + } + if completedMigrationMatches(o, q) { + return nil + } + if migrationID := textNested(o.Object, "status", "migrationOperation", "id"); migrationID != "" { + fromSchema, found, err := unstructured.NestedInt64(o.Object, "status", "migrationOperation", "fromSchema") + if migrationID != q.OperationID || err != nil || !found || fromSchema != q.ExpectedSchema { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "another schema migration is active"} + } + return nil + } + if intNested(o.Object, "spec", "schemaVersion") != q.ExpectedSchema || intNested(o.Object, "status", "schemaVersion") != q.ExpectedSchema || textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "activeOperation", "id") != "" || textNested(o.Object, "status", "lifecycleOperation", "id") != "" || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype environment is not eligible for migration"} + } + if err := s.verifyRuntimeUIDs(ctx, o, q); err != nil { + return err + } + refs, err := migrateReferenceRecords(o, q.OperationID, s.now()) + if err != nil { + return err + } + if err := setReferenceRecords(o, refs); err != nil { + return err + } + return unstructured.SetNestedMap(o.Object, map[string]any{"id": q.OperationID, "fromSchema": q.ExpectedSchema, "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID}, "status", "migrationOperation") + }); err != nil { + return err + } + for attempt := 0; attempt < 5; attempt++ { + o, err := s.resources.Get(ctx, q.Environment.ID, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if err != nil { + return err + } + if err := adminSubject(o, q); err != nil { + return err + } + if intNested(o.Object, "spec", "schemaVersion") == currentSchemaVersion { + break + } + if textNested(o.Object, "status", "migrationOperation", "id") != q.OperationID || intNested(o.Object, "spec", "schemaVersion") != q.ExpectedSchema { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "schema migration changed concurrently"} + } + _ = unstructured.SetNestedField(o.Object, currentSchemaVersion, "spec", "schemaVersion") + if _, err = s.resources.Update(ctx, o, metav1.UpdateOptions{}); err == nil { + break + } else if !apierrors.IsConflict(err) { + return err + } else if attempt == 4 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "schema migration conflict", Retryable: true} + } + } + return s.retryUpdateStatusRaw(ctx, q.Environment.ID, func(o *unstructured.Unstructured) error { + if err := adminSubject(o, q); err != nil { + return err + } + if completedMigrationMatches(o, q) { + return nil + } + fromSchema, found, schemaErr := unstructured.NestedInt64(o.Object, "status", "migrationOperation", "fromSchema") + if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion || textNested(o.Object, "status", "migrationOperation", "id") != q.OperationID || intNested(o.Object, "status", "schemaVersion") != q.ExpectedSchema || schemaErr != nil || !found || fromSchema != q.ExpectedSchema || textNested(o.Object, "status", "pod", "uid") != q.ExpectedPodUID || textNested(o.Object, "status", "pvc", "uid") != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "schema migration phase mismatch"} + } + epoch, ok := epochValue(o) + if !ok || epoch >= math.MaxInt64 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype epoch is invalid"} + } + _ = unstructured.SetNestedField(o.Object, int64(epoch+1), "status", "epoch") //nolint:gosec // checked above. + _ = unstructured.SetNestedField(o.Object, currentSchemaVersion, "status", "schemaVersion") + _ = unstructured.SetNestedField(o.Object, q.OperationID, "status", "lastMigrationOperationID") + _ = unstructured.SetNestedField(o.Object, q.ExpectedSchema, "status", "lastMigrationFromSchema") + unstructured.RemoveNestedField(o.Object, "status", "migrationOperation") + return nil + }) +} + +func completedMigrationMatches(o *unstructured.Unstructured, q adminLifecycleRequest) bool { + fromSchema, found, err := unstructured.NestedInt64(o.Object, "status", "lastMigrationFromSchema") + return err == nil && found && fromSchema == q.ExpectedSchema && + intNested(o.Object, "spec", "schemaVersion") == currentSchemaVersion && + intNested(o.Object, "status", "schemaVersion") == currentSchemaVersion && + textNested(o.Object, "status", "lastMigrationOperationID") == q.OperationID && + textNested(o.Object, "status", "pod", "uid") == q.ExpectedPodUID && + textNested(o.Object, "status", "pvc", "uid") == q.ExpectedPVCUID +} + +func (s *Store) verifyRuntimeUIDs(ctx context.Context, o *unstructured.Unstructured, q adminLifecycleRequest) error { + profile, ok := s.profiles.get(textNested(o.Object, "spec", "profile")) + if !ok || profile.Digest != textNested(o.Object, "spec", "profileDigest") { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype profile is unavailable or changed"} + } + pod, err := s.kube.CoreV1().Pods(s.namespace).Get(ctx, textNested(o.Object, "status", "pod", "name"), metav1.GetOptions{}) + if err != nil || string(pod.UID) != q.ExpectedPodUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype executor identity is not observable"} + } + pvc, err := s.kube.CoreV1().PersistentVolumeClaims(s.namespace).Get(ctx, textNested(o.Object, "status", "pvc", "name"), metav1.GetOptions{}) + if err != nil || string(pvc.UID) != q.ExpectedPVCUID { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype workspace identity is not observable"} + } + if err := validatePVC(o, profile, pvc); err != nil { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype workspace shape is incompatible"} + } + if err := validatePod(o, profile, pvc.Name, pod); err != nil { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype executor shape is insecure or incompatible"} + } + return nil +} + +func migrateReferenceRecords(o *unstructured.Unstructured, operationID string, now time.Time) ([]referenceRecord, error) { + raw, found, err := unstructured.NestedSlice(o.Object, "status", "references") + if err != nil || !found { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} + } + out := make([]referenceRecord, 0, len(raw)) + for _, value := range raw { + switch item := value.(type) { + case string: + if item == "" { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} + } + out = append(out, referenceRecord{BindingID: item, State: executionenv.ReferencePublished, OperationID: "migration-" + operationID, CreatedAt: now}) + case map[string]any: + created, parseErr := time.Parse(time.RFC3339Nano, text(item, "createdAt")) + r := referenceRecord{BindingID: text(item, "bindingID"), State: executionenv.ReferenceState(text(item, "state")), OperationID: text(item, operationIDField), SourceBindingID: text(item, "sourceBindingID"), CreatedAt: created} + if parseErr != nil || r.BindingID == "" || r.OperationID == "" || r.State != executionenv.ReferencePendingCreate && r.State != executionenv.ReferencePublished && r.State != executionenv.ReferencePendingDelete { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} + } + out = append(out, r) + default: + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "prototype references are malformed"} + } + } + return out, nil +} diff --git a/internal/adapter/executioncontroller/admin_lifecycle_expired_run_test.go b/internal/adapter/executioncontroller/admin_lifecycle_expired_run_test.go new file mode 100644 index 0000000000..a5565cdf47 --- /dev/null +++ b/internal/adapter/executioncontroller/admin_lifecycle_expired_run_test.go @@ -0,0 +1,170 @@ +package executioncontroller + +import ( + "errors" + "reflect" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestExpiredExactRunDoesNotPermanentlyBlockFinalReferenceRetirement(t *testing.T) { + now := time.Date(2026, 9, 22, 12, 0, 0, 0, time.UTC) + env := lifecycleAdminEnvironment(2, []any{map[string]any{"bindingID": "binding", "state": string(executionenv.ReferencePublished), operationIDField: "seed", "createdAt": now.Add(-time.Hour).Format(time.RFC3339Nano)}}) + setLifecycleRunClaim(env, now.Add(-time.Second), 4, 1) + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + store.now = func() time.Time { return now } + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + if err := store.ReleaseReference(t.Context(), ref, "client", "owner", "binding"); err != nil { + t.Fatalf("remove final reference after client loss: %v", err) + } + q := adminRequestFixture() + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatalf("expired exact run permanently blocked retirement: %v", err) + } + started, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(started.Object, "status", "lifecycleOperation", "type") != "RetireEnvironment" { + t.Fatalf("retirement did not start: %v", started.Object["status"]) + } + + // Model the controller's already-covered retirement completion so this test + // also proves the retained-delete admission immediately following this path. + if err := store.retryUpdateStatus(t.Context(), "env", func(o *unstructured.Unstructured) error { + unstructured.RemoveNestedField(o.Object, "status", "activeRun") + unstructured.RemoveNestedField(o.Object, "status", "lifecycleOperation") + setConditionObject(o, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(o, "ExecutorTerminated", true, "Verified", "terminated") + return nil + }); err != nil { + t.Fatal(err) + } + q.OperationID = "delete-retained" + if err := store.DeleteRetiredEnvironment(t.Context(), q); err != nil { + t.Fatalf("retained delete after expired-run retirement: %v", err) + } +} + +func TestExpiredExactRunAdmitsReadyFalseLifecycleWithTerminalProof(t *testing.T) { + now := time.Date(2026, 9, 22, 12, 0, 0, 0, time.UTC) + for _, tc := range []struct { + name string + existingProof bool + administrator bool + }{ + {name: "existing exact proof from prior operation", existingProof: true}, + {name: "newly verified terminal pod and pvc"}, + {name: "scoped administrator uses creator run tuple", existingProof: true, administrator: true}, + } { + t.Run(tc.name, func(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Ready", false, "PodTerminated", "executor exited") + setLifecycleRunClaim(env, now.Add(-time.Second), 4, 1) + if tc.existingProof { + setLifecycleTerminationProof(env, "prior-operation") + } + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + if !tc.existingProof { + store = store.WithKubeClient(kubernetesfake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + } + store.now = func() time.Time { return now } + q := adminRequestFixture() + if tc.administrator { + q.Client = "administrator" + q.AdministratorFor = []string{"client"} + } + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatalf("retire with exact terminal proof: %v", err) + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(got.Object, "status", "activeRun", "claimID") != "" || textNested(got.Object, "status", "lifecycleOperation", "id") != q.OperationID || !exactTerminationProofMatches(got, q) { + t.Fatalf("expired claim and proof were not atomically advanced: %v", got.Object["status"]) + } + }) + } +} + +func TestLifecycleRefusesLiveMalformedMismatchedClaimsAndActiveOperationWithoutMutation(t *testing.T) { + now := time.Date(2026, 9, 22, 12, 0, 0, 0, time.UTC) + cases := []struct { + name string + plant func(*unstructured.Unstructured) + }{ + {name: "live claim", plant: func(o *unstructured.Unstructured) { setLifecycleRunClaim(o, now.Add(time.Minute), 4, 1) }}, + {name: "malformed claim", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedMap(o.Object, map[string]any{"claimID": "claim", "expiresAt": "not-a-time"}, "status", "activeRun") + }}, + {name: "mismatched epoch", plant: func(o *unstructured.Unstructured) { setLifecycleRunClaim(o, now.Add(-time.Second), 3, 1) }}, + {name: "mismatched generation", plant: func(o *unstructured.Unstructured) { setLifecycleRunClaim(o, now.Add(-time.Second), 4, 2) }}, + {name: "wrong active-run owner", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, "wrong-owner", "status", "activeRun", "ownerHash") + }}, + {name: "missing active-run owner", plant: func(o *unstructured.Unstructured) { + unstructured.RemoveNestedField(o.Object, "status", "activeRun", "ownerHash") + }}, + {name: "wrong active-run client", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, hashText("wrong-client"), "status", "activeRun", "clientHash") + }}, + {name: "missing active-run client", plant: func(o *unstructured.Unstructured) { + unstructured.RemoveNestedField(o.Object, "status", "activeRun", "clientHash") + }}, + {name: "active operation", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedMap(o.Object, map[string]any{"id": "writer", "operation": string(executionenv.OpFileReplace), "claimID": "claim", "runID": "run", "epoch": int64(4), "holderID": "holder"}, "status", "activeOperation") + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Ready", false, "PodTerminated", "executor exited") + setLifecycleRunClaim(env, now.Add(-time.Second), 4, 1) + setLifecycleTerminationProof(env, "prior-operation") + tc.plant(env) + before := env.DeepCopy() + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + store.now = func() time.Time { return now } + err := store.RetireExact(t.Context(), adminRequestFixture()) + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("retirement error=%v, want conflict", err) + } + after, getErr := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if getErr != nil { + t.Fatal(getErr) + } + before.SetResourceVersion(after.GetResourceVersion()) + if !reflect.DeepEqual(before.Object, after.Object) { + t.Fatalf("refused lifecycle changed state\nbefore=%v\nafter=%v", before.Object["status"], after.Object["status"]) + } + }) + } +} + +func setLifecycleRunClaim(o *unstructured.Unstructured, expiry time.Time, epoch, generation int64) { + _ = unstructured.SetNestedMap(o.Object, map[string]any{ + "bindingID": "binding", "runID": "run", "claimID": "claim", operationIDField: "acquire", + "ownerHash": "owner", "clientHash": hashText("client"), + "epoch": epoch, "grantGeneration": generation, "expiresAt": expiry.UTC().Format(time.RFC3339Nano), + }, "status", "activeRun") +} + +func setLifecycleTerminationProof(o *unstructured.Unstructured, operationID string) { + _ = unstructured.SetNestedMap(o.Object, map[string]any{ + operationIDField: operationID, "podUID": "pod-uid", "pvcUID": "pvc-uid", "epoch": int64(4), + "podPhase": "Failed", "observedAt": time.Now().UTC().Format(time.RFC3339Nano), + }, "status", "terminationProof") +} diff --git a/internal/adapter/executioncontroller/admin_scope_security_test.go b/internal/adapter/executioncontroller/admin_scope_security_test.go new file mode 100644 index 0000000000..8e7992dce2 --- /dev/null +++ b/internal/adapter/executioncontroller/admin_scope_security_test.go @@ -0,0 +1,91 @@ +package executioncontroller + +import ( + "encoding/json" + "fmt" + "reflect" + "testing" + "time" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func scopedManifestClient(t *testing.T, uri string, admin bool, scope []string) securityClientManifest { + t.Helper() + return securityClientManifest{URI: uri, Administrator: admin, AdministratorFor: scope} +} + +func TestAdministratorScopeManifestValidation(t *testing.T) { + for _, tc := range []struct { + name, raw string + valid bool + }{ + {"absent", `{"uri":"spiffe://example/admin","administrator":true}`, true}, + {"empty", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":[]}`, true}, + {"scoped", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/creator"]}`, true}, + {"not admin", `{"uri":"spiffe://example/admin","administratorFor":["spiffe://example/creator"]}`, false}, + {"duplicate", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/creator","spiffe://example/creator"]}`, false}, + {"wildcard", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/*"]}`, false}, + {"escaped wildcard", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/%2A"]}`, false}, + {"case", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://EXAMPLE/creator"]}`, false}, + {"query", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/creator?q=1"]}`, false}, + {"fragment", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://example/creator#x"]}`, false}, + {"userinfo", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["spiffe://user@example/creator"]}`, false}, + {"malformed", `{"uri":"spiffe://example/admin","administrator":true,"administratorFor":["creator"]}`, false}, + {"unknown", `{"uri":"spiffe://example/admin","administrator":true,"adminFor":[]}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var entry securityClientManifest + err := executionenv.DecodeStrict([]byte(tc.raw), &entry) + if err == nil { + _, err = clientPolicies([]securityClientManifest{entry}) + } + if (err == nil) != tc.valid { + t.Fatalf("valid=%v err=%v", tc.valid, err) + } + }) + } + for _, count := range []int{256, 257} { + scope := make([]string, count) + for i := range scope { + scope[i] = fmt.Sprintf("spiffe://example/creator-%d", i) + } + entry := scopedManifestClient(t, "spiffe://example/admin", true, scope) + if _, err := clientPolicies([]securityClientManifest{entry}); (err == nil) != (count == 256) { + t.Fatalf("scope count=%d err=%v", count, err) + } + } +} + +func TestAdministratorScopeDigestIsNormalizedAndAuthorityBound(t *testing.T) { + a, b := "spiffe://example/a", "spiffe://example/b" + mf := securityManifest{Clients: []securityClientManifest{scopedManifestClient(t, "spiffe://example/admin", true, []string{b, a})}} + before, _ := json.Marshal(mf) + digest := func(m securityManifest) string { + t.Helper() + d, err := authorityDigest(m, time.Minute, time.Second, nil, nil, "server") + if err != nil { + t.Fatal(err) + } + return d + } + original := digest(mf) + after, _ := json.Marshal(mf) + if !reflect.DeepEqual(before, after) { + t.Fatal("digest mutated source manifest") + } + mf.Clients[0] = scopedManifestClient(t, "spiffe://example/admin", true, []string{a, b}) + if digest(mf) != original { + t.Fatal("scope ordering changed authority digest") + } + mf.Clients[0] = scopedManifestClient(t, "spiffe://example/admin", true, []string{a}) + if digest(mf) == original { + t.Fatal("scope removal did not change authority digest") + } + mf.Clients[0] = scopedManifestClient(t, "spiffe://example/admin", true, nil) + empty := digest(mf) + mf.Clients[0] = scopedManifestClient(t, "spiffe://example/admin", true, []string{}) + if digest(mf) != empty { + t.Fatal("empty and absent scope differ") + } +} diff --git a/internal/adapter/executioncontroller/admin_scope_test.go b/internal/adapter/executioncontroller/admin_scope_test.go new file mode 100644 index 0000000000..c922393415 --- /dev/null +++ b/internal/adapter/executioncontroller/admin_scope_test.go @@ -0,0 +1,568 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "net" + "os" + "path/filepath" + "reflect" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +const scopeCreator = "spiffe://example/creator" +const scopeAdmin = "spiffe://example/operations" + +var scopeOwner = executionenv.Owner{Issuer: "https://issuer.example", Subject: "owner"} + +type adminScopeFixture struct { + client executionv1.ExecutionProviderServiceClient + dynamic *dynamicfake.FakeDynamicClient + manager *SecurityManager + manifest securityManifest + path string +} + +func newAdminScopeFixture(t *testing.T, route, actor string) *adminScopeFixture { + t.Helper() + env := lifecycleAdminEnvironment(2, []any{}) + if route == "data" { + env = runFixtureEnvironment(4, 4) + } + if route == "migrate" { + _ = unstructured.SetNestedField(env.Object, int64(1), "spec", "schemaVersion") + _ = unstructured.SetNestedField(env.Object, int64(1), "status", "schemaVersion") + } + if route == "recover" { + _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") + } + if route == "delete" { + setConditionObject(env, "Retired", true, "Retired", "retired") + setConditionObject(env, "ExecutorTerminated", true, "Terminated", "terminated") + } + _ = unstructured.SetNestedField(env.Object, hashText(scopeCreator), "spec", "clientHash") + _ = unstructured.SetNestedField(env.Object, ownerHash(scopeOwner), "spec", "ownerHash") + if textNested(env.Object, "status", "activeRun", "claimID") != "" { + _ = unstructured.SetNestedField(env.Object, hashText(scopeCreator), "status", "activeRun", "clientHash") + _ = unstructured.SetNestedField(env.Object, ownerHash(scopeOwner), "status", "activeRun", "ownerHash") + } + _ = unstructured.SetNestedField(env.Object, int64(4), "status", "grantGeneration") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC()) + store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) + dir, now := t.TempDir(), time.Now().UTC() + ca, cert, key, clientCert := rpcSecurityPKI(t, now, actor) + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + writePKCS8(t, filepath.Join(dir, "grant.pem"), priv) + for name, data := range map[string][]byte{"server.crt": cert, "server.key": key, "clients.pem": ca} { + if err := os.WriteFile(filepath.Join(dir, name), data, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(pub) + mf := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: actor}}} + path := filepath.Join(dir, "manifest.json") + writeManifest(t, path, mf) + manager := NewSecurityManager(path, dir, "ns", "authority", k) + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig()))) + executionv1.RegisterExecutionProviderServiceServer(server, NewHandler(HandlerConfig{Security: manager}, store)) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(ca) + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, ServerName: "provider.test", RootCAs: roots, Certificates: []tls.Certificate{clientCert}}))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + return &adminScopeFixture{client: executionv1.NewExecutionProviderServiceClient(conn), dynamic: d, manager: manager, manifest: mf, path: path} +} + +func (f *adminScopeFixture) policy(t *testing.T, admin bool, scope []string) { + t.Helper() + f.manifest.Generation++ + f.manifest.Clients[0] = scopedManifestClient(t, f.manifest.Clients[0].URI, admin, scope) + writeManifest(t, f.path, f.manifest) + if err := f.manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } +} + +func callScopedAdmin(ctx context.Context, c executionv1.ExecutionProviderServiceClient, route string, q adminLifecycleRequest, owner executionenv.Owner) error { + ref := &executionv1.EnvironmentRef{Id: q.Environment.ID, Revision: q.Environment.Revision} + o := &executionv1.Owner{Issuer: owner.Issuer, Subject: owner.Subject} + var err error + switch route { + case "retire": + _, err = c.RetireEnvironment(ctx, &executionv1.RetireEnvironmentRequest{Environment: ref, Owner: o, ExpectedExecutionEpoch: q.ExpectedEpoch, ExpectedPodUid: q.ExpectedPodUID, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "replace": + _, err = c.ReplaceExecutor(ctx, &executionv1.ReplaceExecutorRequest{Environment: ref, Owner: o, ExpectedExecutionEpoch: q.ExpectedEpoch, ExpectedPodUid: q.ExpectedPodUID, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "recover": + _, err = c.RecoverEnvironment(ctx, &executionv1.RecoverEnvironmentRequest{Environment: ref, Owner: o, ExpectedExecutionEpoch: q.ExpectedEpoch, ExpectedPodUid: q.ExpectedPodUID, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "delete": + _, err = c.DeleteRetiredEnvironment(ctx, &executionv1.DeleteRetiredEnvironmentRequest{Environment: ref, Owner: o, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "migrate": + schema := uint32(q.ExpectedSchema) + _, err = c.MigrateEnvironment(ctx, &executionv1.MigrateEnvironmentRequest{Environment: ref, Owner: o, ExpectedSchemaVersion: &schema, ExpectedPodUid: q.ExpectedPodUID, ExpectedPvcUid: q.ExpectedPVCUID, OperationId: q.OperationID}) + case "revoke": + _, err = c.RevokeEnvironment(ctx, &executionv1.RevokeEnvironmentRequest{Environment: ref, Owner: o, ExpectedGrantGeneration: q.ExpectedEpoch, OperationId: q.OperationID}) + default: + panic("unknown test route") + } + return err +} + +func TestScopedAdminAllRoutesOverMTLS(t *testing.T) { + for _, route := range []string{"retire", "replace", "recover", "delete", "migrate", "revoke"} { + t.Run(route, func(t *testing.T) { + f := newAdminScopeFixture(t, route, scopeAdmin) + q := adminRequestFixture() + q.ExpectedSchema = 1 + call := func(q adminLifecycleRequest, owner executionenv.Owner) error { + return callScopedAdmin(t.Context(), f.client, route, q, owner) + } + if err := call(q, scopeOwner); status.Code(err) != codes.PermissionDenied { + t.Fatalf("nonadmin: %v", err) + } + for _, scope := range [][]string{nil, {"spiffe://example/unrelated"}} { + f.policy(t, true, scope) + denied := call(q, scopeOwner) + missing := q + missing.Environment.ID = "absent" + absent := call(missing, scopeOwner) + if status.Code(denied) != codes.NotFound || status.Convert(denied).Message() != status.Convert(absent).Message() || status.Code(absent) != codes.NotFound { + t.Fatalf("scope/missing disclosure: denied=%v absent=%v", denied, absent) + } + } + // Creator login is deliberately absent from the allowlist. + f.policy(t, true, []string{scopeCreator}) + wrongOwner := scopeOwner + wrongOwner.Subject = "another-owner" + if err := call(q, wrongOwner); status.Code(err) != codes.NotFound { + t.Fatalf("owner mismatch: %v", err) + } + stale := q + stale.Environment.Revision = "stale" + if err := call(stale, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("revision mismatch: %v", err) + } + stale = q + if route == "revoke" { + stale.ExpectedEpoch++ + } else { + stale.ExpectedPVCUID = "stale" + } + if err := call(stale, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("stale runtime identity: %v", err) + } + if route == "retire" || route == "replace" || route == "recover" { + stale = q + stale.ExpectedEpoch++ + if err := call(stale, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("stale epoch: %v", err) + } + } + if route != "delete" && route != "revoke" { + stale = q + stale.ExpectedPodUID = "stale" + if err := call(stale, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("stale Pod UID: %v", err) + } + } + if route == "migrate" { + stale = q + stale.ExpectedSchema = 0 + if err := call(stale, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("wrong schema: %v", err) + } + } + before, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := call(q, scopeOwner); err != nil { + t.Fatalf("scoped admin rejected: %v", err) + } + if err := call(q, scopeOwner); err != nil { + t.Fatalf("exact replay: %v", err) + } + after, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if reflect.DeepEqual(before.Object["status"], after.Object["status"]) { + t.Fatal("successful administration did not persist its transition") + } + if route == "revoke" { + receipts, _, err := unstructured.NestedSlice(after.Object, "status", "revocationReceipts") + if err != nil || len(receipts) != 1 || intNested(after.Object, "status", "grantGeneration") != 5 { + t.Fatalf("revocation replay was not idempotent: %v", after.Object["status"]) + } + if text(receipts[0].(map[string]any), "fingerprint") != fingerprint("env", "rev", scopeAdmin, ownerHash(scopeOwner), "4") { + t.Fatal("revocation receipt replaced the authenticated actor with the creator") + } + } + for _, field := range []string{"clientHash", "ownerHash", "revision"} { + if textNested(before.Object, "spec", field) != textNested(after.Object, "spec", field) { + t.Fatalf("admin rewrote immutable %s", field) + } + } + // Even receipt replay is a new admission on the established connection. + f.policy(t, true, nil) + if err := call(q, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("scope removal did not revoke replay: %v", err) + } + }) + t.Run(route+"/creator", func(t *testing.T) { + f := newAdminScopeFixture(t, route, scopeCreator) + q := adminRequestFixture() + q.ExpectedSchema = 1 + if err := callScopedAdmin(t.Context(), f.client, route, q, scopeOwner); status.Code(err) != codes.PermissionDenied { + t.Fatalf("normal creator is admin: %v", err) + } + f.policy(t, true, nil) + if err := callScopedAdmin(t.Context(), f.client, route, q, scopeOwner); err != nil { + t.Fatalf("legacy self-admin: %v", err) + } + }) + } +} + +func TestScopedAdminCASRetryRechecksSubject(t *testing.T) { + for _, route := range []string{"retire", "replace", "recover", "delete", "migrate", "revoke"} { + t.Run(route, func(t *testing.T) { + f := newAdminScopeFixture(t, route, scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + var conflicted atomic.Bool + f.dynamic.PrependReactor("update", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + if conflicted.Swap(true) { + return false, nil, nil + } + current, err := f.dynamic.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + if err != nil { + return true, nil, err + } + o := current.(*unstructured.Unstructured) + _ = unstructured.SetNestedField(o.Object, hashText("spiffe://example/other"), "spec", "clientHash") + if err := f.dynamic.Tracker().Update(ExecutionEnvironmentGVR, o, "ns"); err != nil { + return true, nil, err + } + return true, nil, apierrors.NewConflict(ExecutionEnvironmentGVR.GroupResource(), "env", nil) + }) + q := adminRequestFixture() + q.ExpectedSchema = 1 + if err := callScopedAdmin(t.Context(), f.client, route, q, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("CAS retry escaped scope: %v", err) + } + if !conflicted.Load() { + t.Fatal("did not exercise CAS retry") + } + }) + } +} + +func TestScopedAdminUsesAdmittedPolicySnapshot(t *testing.T) { + f := newAdminScopeFixture(t, "revoke", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + var changed atomic.Bool + f.manifest.Generation++ + f.manifest.Clients[0] = scopedManifestClient(t, scopeAdmin, true, nil) + writeManifest(t, f.path, f.manifest) + f.dynamic.PrependReactor("get", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + if !changed.Swap(true) { + // Authentication has already admitted this request; change the live + // snapshot before the store makes its subject decision. + if err := f.manager.Reload(t.Context()); err != nil { + return true, nil, err + } + } + return false, nil, nil + }) + q := adminRequestFixture() + if err := callScopedAdmin(t.Context(), f.client, "revoke", q, scopeOwner); err != nil { + t.Fatalf("admitted snapshot was replaced mid-request: %v", err) + } + if !changed.Load() { + t.Fatal("policy did not change during admission") + } + if err := callScopedAdmin(t.Context(), f.client, "revoke", q, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("next request reused old scope: %v", err) + } +} + +func TestScopedAdminCompletedReplacementReceiptReauthorizes(t *testing.T) { + f := newAdminScopeFixture(t, "replace", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + q := adminRequestFixture() + env, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + _ = unstructured.SetNestedMap(env.Object, map[string]any{"operationID": q.OperationID, "previousPodUID": q.ExpectedPodUID, "replacementPodUID": "new-pod", "pvcUID": q.ExpectedPVCUID, "previousEpoch": int64(4), "replacementEpoch": int64(5)}, "status", "lastReplacement") + _ = unstructured.SetNestedField(env.Object, int64(5), "status", "epoch") + _ = unstructured.SetNestedField(env.Object, "new-pod", "status", "pod", "uid") + if _, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(t.Context(), env, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := callScopedAdmin(t.Context(), f.client, "replace", q, scopeOwner); err != nil { + t.Fatal(err) + } + wrongOwner := scopeOwner + wrongOwner.Subject = "other" + if err := callScopedAdmin(t.Context(), f.client, "replace", q, wrongOwner); status.Code(err) != codes.NotFound { + t.Fatalf("receipt bypassed owner check: %v", err) + } + f.policy(t, true, nil) + if err := callScopedAdmin(t.Context(), f.client, "replace", q, scopeOwner); status.Code(err) != codes.NotFound { + t.Fatalf("receipt bypassed removed scope: %v", err) + } +} + +func TestScopedAdminMigrationReceiptRetainsUIDPreconditions(t *testing.T) { + f := newAdminScopeFixture(t, "migrate", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + q := adminRequestFixture() + q.ExpectedSchema = 1 + if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); err != nil { + t.Fatal(err) + } + if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); err != nil { + t.Fatalf("exact migration replay failed: %v", err) + } + q.ExpectedSchema = 0 + if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("migration receipt bypassed source schema: %v", err) + } + q.ExpectedSchema = 1 + q.ExpectedPVCUID = "stale" + if err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner); status.Code(err) != codes.Aborted { + t.Fatalf("migration receipt bypassed UID: %v", err) + } +} + +func TestMigrationCompletedCASReplayRequiresExactSourceSchema(t *testing.T) { + for _, receipt := range []string{"exact", "changed", "absent", "expired"} { + t.Run(receipt, func(t *testing.T) { + f := newAdminScopeFixture(t, "migrate", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + var raced bool + f.dynamic.PrependReactor("update", "executionenvironments", func(action k8stesting.Action) (bool, runtime.Object, error) { + u := action.(k8stesting.UpdateAction).GetObject().(*unstructured.Unstructured) + if raced || textNested(u.Object, "status", "lastMigrationOperationID") == "" { + return false, nil, nil + } + raced = true + completed := u.DeepCopy() + switch receipt { + case "changed": + _ = unstructured.SetNestedField(completed.Object, int64(0), "status", "lastMigrationFromSchema") + case "absent": + unstructured.RemoveNestedField(completed.Object, "status", "lastMigrationFromSchema") + case "expired": + unstructured.RemoveNestedField(completed.Object, "status", "lastMigrationOperationID") + unstructured.RemoveNestedField(completed.Object, "status", "lastMigrationFromSchema") + } + if err := f.dynamic.Tracker().Update(ExecutionEnvironmentGVR, completed, "ns"); err != nil { + t.Fatal(err) + } + return true, nil, apierrors.NewConflict(ExecutionEnvironmentGVR.GroupResource(), "env", nil) + }) + q := adminRequestFixture() + q.ExpectedSchema = 1 + err := callScopedAdmin(t.Context(), f.client, "migrate", q, scopeOwner) + want := codes.Aborted + if receipt == "exact" { + want = codes.OK + } + if !raced || status.Code(err) != want { + t.Fatalf("CAS replay raced=%t code=%v, want %v", raced, status.Code(err), want) + } + }) + } +} + +func TestScopedAdminEqualGenerationDriftFailsClosed(t *testing.T) { + f := newAdminScopeFixture(t, "revoke", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + q := adminRequestFixture() + if err := callScopedAdmin(t.Context(), f.client, "revoke", q, scopeOwner); err != nil { + t.Fatal(err) + } + f.manifest.Clients[0] = scopedManifestClient(t, scopeAdmin, true, nil) + writeManifest(t, f.path, f.manifest) + if err := f.manager.Reload(t.Context()); err == nil { + t.Fatal("equal-generation scope drift admitted") + } + if err := callScopedAdmin(t.Context(), f.client, "revoke", q, scopeOwner); status.Code(err) != codes.Unavailable { + t.Fatalf("stale snapshot admitted RPC: %v", err) + } +} + +func TestScopedAdminWithOwnerAttestationCannotUseAnotherCreatorsDataPlane(t *testing.T) { + f := newAdminScopeFixture(t, "data", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + f.manifest.Generation++ + f.manifest.Clients[0].MayAttestOwner = true + writeManifest(t, f.path, f.manifest) + if err := f.manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + ref := &executionv1.EnvironmentRef{Id: "env", Revision: "rev"} + owner := &executionv1.Owner{Issuer: scopeOwner.Issuer, Subject: scopeOwner.Subject} + before, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + refs, err := referenceRecords(before) + if err != nil { + t.Fatal(err) + } + refs = append(refs, referenceRecord{BindingID: "pending", State: executionenv.ReferencePendingCreate, OperationID: "pending-op", CreatedAt: time.Now().UTC()}) + if err := setReferenceRecords(before, refs); err != nil { + t.Fatal(err) + } + if _, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(t.Context(), before, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + for name, call := range map[string]func() error{ + "attach": func() error { + _, err := f.client.AttachEnvironment(t.Context(), &executionv1.AttachEnvironmentRequest{Context: &executionv1.RequestContext{Environment: ref, Owner: owner, BindingId: "binding"}, Purpose: "session"}) + return err + }, + "acquire": func() error { + _, err := f.client.AcquireRun(t.Context(), &executionv1.AcquireRunRequest{Environment: ref, Owner: owner, BindingId: "binding", RunId: "run", OperationId: "acquire", TtlMillis: 60000}) + return err + }, + "reference": func() error { + _, err := f.client.CommitReference(t.Context(), &executionv1.ReferenceMutationRequest{Environment: ref, Owner: owner, BindingId: "pending", OperationId: "pending-op"}) + return err + }, + "exact discovery": func() error { + _, err := f.client.ListReferenceIntents(t.Context(), &executionv1.ListReferenceIntentsRequest{Environment: ref, Owner: owner, BindingId: "pending"}) + return err + }, + } { + if err := call(); status.Code(err) != codes.NotFound { + t.Errorf("%s: expected creator-bound not found, got %v", name, err) + } + } + for _, query := range []*executionv1.ListReferenceIntentsRequest{{Owner: owner}, {}} { + list, err := f.client.ListReferenceIntents(t.Context(), query) + if err != nil || len(list.GetIntents()) != 0 { + t.Fatal("discovery exposed another creator's pending reference") + } + } + after, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil || !reflect.DeepEqual(before.Object, after.Object) { + t.Fatal("creator-bound denials changed foreign environment") + } + own, err := f.client.EnsureEnvironment(t.Context(), &executionv1.EnsureEnvironmentRequest{BindingId: "own", Profile: "go", Owner: owner, OperationId: "own-create"}) + if err != nil || own.GetEnvironment().GetId() == "env" { + t.Fatalf("explicit owner attestation did not permit own allocation: %v", err) + } + list, err := f.client.ListReferenceIntents(t.Context(), &executionv1.ListReferenceIntentsRequest{Owner: owner}) + if err != nil || len(list.GetIntents()) != 1 || list.Intents[0].GetEnvironment().GetId() != own.GetEnvironment().GetId() { + t.Fatal("discovery did not isolate the actor's own pending allocation") + } +} + +func TestScopedAdminDoesNotGrantAttestationOrDataPlane(t *testing.T) { + f := newAdminScopeFixture(t, "data", scopeAdmin) + f.policy(t, true, []string{scopeCreator}) + ref := &executionv1.EnvironmentRef{Id: "env", Revision: "rev"} + owner := &executionv1.Owner{Issuer: scopeOwner.Issuer, Subject: scopeOwner.Subject} + request := &executionv1.RequestContext{Environment: ref, Owner: owner, BindingId: "binding", Epoch: 4, RunId: "run", ClaimId: "claim", GrantGeneration: 4} + // A correctly signed actor-bound grant still cannot cross creator ownership. + h := NewHandler(HandlerConfig{Security: f.manager}, nil) + claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 4, GrantGeneration: 4} + grant, _, err := h.signClaim(t.Context(), claim, scopeAdmin, ownerHash(scopeOwner)) + if err != nil { + t.Fatal(err) + } + request.Grant = grant + calls := map[string]func() error{ + "ensure attestation": func() error { + _, err := f.client.EnsureEnvironment(t.Context(), &executionv1.EnsureEnvironmentRequest{BindingId: "new", Profile: "go", Owner: owner, OperationId: "ensure"}) + return err + }, + "attach": func() error { + _, err := f.client.AttachEnvironment(t.Context(), &executionv1.AttachEnvironmentRequest{Context: request, Purpose: "session"}) + return err + }, + "run": func() error { + _, err := f.client.AcquireRun(t.Context(), &executionv1.AcquireRunRequest{Environment: ref, Owner: owner, BindingId: "binding", RunId: "run", OperationId: "acquire", TtlMillis: 60000}) + return err + }, + "reference": func() error { + _, err := f.client.CommitReference(t.Context(), &executionv1.ReferenceMutationRequest{Environment: ref, Owner: owner, BindingId: "binding", OperationId: "commit"}) + return err + }, + "reference discovery": func() error { + _, err := f.client.ListReferenceIntents(t.Context(), &executionv1.ListReferenceIntentsRequest{Owner: owner}) + return err + }, + "files": func() error { + _, err := f.client.Files(t.Context(), &executionv1.FileRequest{Context: request, Operation: executionv1.FileOperation_FILE_OPERATION_READ, Path: "file"}) + return err + }, + "shell": func() error { + _, err := f.client.StartCommand(t.Context(), &executionv1.CommandStartRequest{Context: request, Command: "true"}) + return err + }, + } + before, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + for name, call := range calls { + want := codes.PermissionDenied + switch name { + case "run", "reference", "reference discovery": + want = codes.InvalidArgument + case "files", "shell": + want = codes.NotFound + } + if err := call(); status.Code(err) != want { + t.Errorf("%s: want %v, got %v", name, want, err) + } + } + after, err := f.dynamic.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before.Object, after.Object) { + t.Fatal("denied data-plane calls changed environment") + } +} diff --git a/internal/adapter/executioncontroller/controller.go b/internal/adapter/executioncontroller/controller.go new file mode 100644 index 0000000000..576ab78e3e --- /dev/null +++ b/internal/adapter/executioncontroller/controller.go @@ -0,0 +1,470 @@ +package executioncontroller + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/dynamic/dynamicinformer" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + environmentFinalizer = "execution.mecatl.dev/retain-workspace" + executorFinalizer = "execution.mecatl.dev/verify-termination" + fenceHealthy = "Healthy" + statusField = "status" +) + +// Reconciler maintains executor Pods and retained workspace PVCs. +type Reconciler struct { + dynamic dynamic.Interface + kube kubernetes.Interface + namespace string + profiles *Profiles + queue workqueue.TypedRateLimitingInterface[string] + initOnce sync.Once + initErr error + ready atomic.Bool + now func() time.Time +} + +// NewReconciler constructs a reconciler for one namespace. +func NewReconciler(d dynamic.Interface, k kubernetes.Interface, namespace string, p *Profiles) *Reconciler { + return &Reconciler{dynamic: d, kube: k, namespace: namespace, profiles: p, queue: workqueue.NewTypedRateLimitingQueue(workqueue.DefaultTypedControllerRateLimiter[string]()), now: func() time.Time { return time.Now().UTC() }} +} + +// Initialize synchronizes informer caches before the provider may expose mutating APIs. +// Startup never mutates operation ownership: another replica may still be its live holder. +func (r *Reconciler) Initialize(ctx context.Context) error { + r.initOnce.Do(func() { + if err := r.preflightProfiles(ctx); err != nil { + r.initErr = err + return + } + factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(r.dynamic, 0, r.namespace, nil) + environments := factory.ForResource(ExecutionEnvironmentGVR).Informer() + _, err := environments.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: r.enqueue, UpdateFunc: r.enqueueUpdate}) + if err != nil { + r.initErr = err + return + } + runtimeFactory := informers.NewSharedInformerFactoryWithOptions(r.kube, 0, informers.WithNamespace(r.namespace), informers.WithTweakListOptions(func(options *metav1.ListOptions) { + options.LabelSelector = "execution.mecatl.dev/environment" + })) + pods := runtimeFactory.Core().V1().Pods().Informer() + pvcs := runtimeFactory.Core().V1().PersistentVolumeClaims().Informer() + for _, informer := range []cache.SharedIndexInformer{pods, pvcs} { + if _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{AddFunc: r.enqueueRuntime, UpdateFunc: func(_, next any) { r.enqueueRuntime(next) }, DeleteFunc: r.enqueueRuntime}); err != nil { + r.initErr = err + return + } + } + factory.Start(ctx.Done()) + runtimeFactory.Start(ctx.Done()) + if !cache.WaitForCacheSync(ctx.Done(), environments.HasSynced, pods.HasSynced, pvcs.HasSynced) { + r.initErr = errors.New("execution environment informer failed to sync") + return + } + go wait.UntilWithContext(ctx, r.worker, time.Second) + r.ready.Store(true) + }) + return r.initErr +} + +func (r *Reconciler) preflightProfiles(ctx context.Context) error { + runtimeClasses, storageClasses := r.profiles.clusterResources() + for _, name := range runtimeClasses { + if _, err := r.kube.NodeV1().RuntimeClasses().Get(ctx, name, metav1.GetOptions{}); err != nil { + return fmt.Errorf("profile preflight: RuntimeClass %q unavailable: %w", name, err) + } + } + for _, name := range storageClasses { + if _, err := r.kube.StorageV1().StorageClasses().Get(ctx, name, metav1.GetOptions{}); err != nil { + return fmt.Errorf("profile preflight: StorageClass %q unavailable: %w", name, err) + } + } + return nil +} + +// Ready reports whether startup fencing, profile preflight, and informer synchronization completed. +func (r *Reconciler) Ready() bool { return r.ready.Load() } + +// ReadinessReason returns a bounded operator-facing reason class. +func (r *Reconciler) ReadinessReason() string { + if r.ready.Load() { + return "ready" + } + if r.initErr != nil && strings.HasPrefix(r.initErr.Error(), "profile preflight:") { + return "profile-resource-unavailable" + } + if r.initErr != nil { + return "controller-cache-unavailable" + } + return "controller-starting" +} + +// Run initializes the reconciler and blocks until shutdown. +func (r *Reconciler) Run(ctx context.Context) error { + if err := r.Initialize(ctx); err != nil { + return err + } + <-ctx.Done() + r.queue.ShutDown() + return nil +} +func (r *Reconciler) enqueue(obj any) { + u, ok := obj.(*unstructured.Unstructured) + if ok { + r.queue.Add(u.GetName()) + } +} + +func (r *Reconciler) enqueueUpdate(oldObj, newObj any) { + oldEnv, oldOK := oldObj.(*unstructured.Unstructured) + newEnv, newOK := newObj.(*unstructured.Unstructured) + if !oldOK || !newOK || !reconcileInputsEqual(oldEnv, newEnv) { + r.enqueue(newObj) + } +} + +func reconcileInputsEqual(oldEnv, newEnv *unstructured.Unstructured) bool { + if oldEnv.GetGeneration() != newEnv.GetGeneration() || + !reflect.DeepEqual(oldEnv.GetDeletionTimestamp(), newEnv.GetDeletionTimestamp()) || + !reflect.DeepEqual(oldEnv.GetFinalizers(), newEnv.GetFinalizers()) { + return false + } + for _, path := range [][]string{{"spec"}, {statusField, "activeOperation"}, {statusField, "lifecycleOperation"}, {statusField, "migrationOperation"}, {statusField, "references"}, {statusField, "fenceState"}} { + oldValue, _, _ := unstructured.NestedFieldNoCopy(oldEnv.Object, path...) + newValue, _, _ := unstructured.NestedFieldNoCopy(newEnv.Object, path...) + if !reflect.DeepEqual(oldValue, newValue) { + return false + } + } + return true +} + +func (r *Reconciler) enqueueRuntime(obj any) { + if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { + obj = tombstone.Obj + } + var labels map[string]string + switch value := obj.(type) { + case *corev1.Pod: + labels = value.Labels + case *corev1.PersistentVolumeClaim: + labels = value.Labels + } + if name := labels["execution.mecatl.dev/environment"]; name != "" { + r.queue.Add(name) + } +} +func (r *Reconciler) worker(ctx context.Context) { + for r.process(ctx) { + } +} +func (r *Reconciler) process(ctx context.Context) bool { + name, shutdown := r.queue.Get() + if shutdown { + return false + } + defer r.queue.Done(name) + if err := r.Reconcile(ctx, name); err != nil { + r.queue.AddRateLimited(name) + } else { + r.queue.Forget(name) + } + return true +} + +// Reconcile converges one ExecutionEnvironment and its runtime resources. +func (r *Reconciler) Reconcile(ctx context.Context, name string) error { + res := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace) + env, err := res.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + if requireCurrentSchema(env) != nil { + return r.setCondition(ctx, env, "Ready", false, "IncompatibleSchema", "explicit administrator migration to schema version 2 is required") + } + if expires := textNested(env.Object, "status", "activeOperation", "expiresAt"); expires != "" { + deadline, parseErr := time.Parse(time.RFC3339Nano, expires) + now := r.now() + if parseErr != nil || !now.Before(deadline) { + return r.setFenceUnknown(ctx, env, "operation holder lease expired; operation identity retained for recovery") + } + r.queue.AddAfter(name, deadline.Sub(now)) + } + // The admitted durable delete owns finalization, including after DELETE has + // set deletionTimestamp. A peer must not re-arm the generic finalizer. + if textNested(env.Object, "status", "lifecycleOperation", "type") == deleteRetiredEnvironment { + return r.reconcileLifecycle(ctx, env) + } + if env.GetDeletionTimestamp() != nil { + return r.reconcileDeletion(ctx, res, env) + } + if !contains(env.GetFinalizers(), environmentFinalizer) { + updated := env.DeepCopy() + updated.SetFinalizers(append(updated.GetFinalizers(), environmentFinalizer)) + if _, err := res.Update(ctx, updated, metav1.UpdateOptions{}); err != nil { + return err + } + return nil + } + if textNested(env.Object, "status", "lifecycleOperation", "id") != "" { + return r.reconcileLifecycle(ctx, env) + } + if conditionTrue(env, "Retired") { + return nil + } + if textNested(env.Object, "spec", "desired") == "Retiring" { + return r.setCondition(ctx, env, "Ready", false, "UnsupportedRetirementRequest", "retirement requires the exact administrator lifecycle operation") + } + profileName := textNested(env.Object, "spec", "profile") + p, ok := r.profiles.get(profileName) + if !ok || p.Digest != textNested(env.Object, "spec", "profileDigest") { + return r.setCondition(ctx, env, "Ready", false, "InvalidProfile", "configured profile is unavailable or changed") + } + pvcName := resourceName("workspace", name) + podName := resourceName("executor", name) + pvc, err := r.ensurePVC(ctx, env, p, pvcName) + if err != nil { + return errors.Join(err, r.setCondition(ctx, env, "Ready", false, "PVCUnavailable", "workspace PVC unavailable")) + } + pod, err := r.ensurePod(ctx, env, p, podName, pvcName) + if err != nil { + return errors.Join(err, r.setCondition(ctx, env, "Ready", false, "ExecutorUnavailable", "executor Pod unavailable")) + } + ready := podReady(pod) + if err := r.updateRuntimeStatus(ctx, env, pvc, pod, ready); err != nil { + return err + } + if !ready { + r.queue.AddAfter(name, time.Second) + } + return nil +} +func (r *Reconciler) ensurePVC(ctx context.Context, env *unstructured.Unstructured, p resolvedProfile, name string) (*corev1.PersistentVolumeClaim, error) { + pvcs := r.kube.CoreV1().PersistentVolumeClaims(r.namespace) + cur, err := pvcs.Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return cur, validatePVC(env, p, cur) + } + if !apierrors.IsNotFound(err) { + return nil, err + } + if textNested(env.Object, "status", "pvc", "uid") != "" { + return nil, errors.New("authoritative PVC disappeared; replacement is forbidden") + } + qty := p.StorageSize + mode := corev1.PersistentVolumeFilesystem + created, err := pvcs.Create(ctx, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/revision": textNested(env.Object, "spec", "revision"), "execution.mecatl.dev/allocation-uid": string(env.GetUID())}}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, StorageClassName: &p.Spec.StorageClass, VolumeMode: &mode, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: qty}}}}, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + created, err = pvcs.Get(ctx, name, metav1.GetOptions{}) + } + if err != nil { + return nil, err + } + return created, validatePVC(env, p, created) +} +func (r *Reconciler) ensurePod(ctx context.Context, env *unstructured.Unstructured, p resolvedProfile, name, pvc string) (*corev1.Pod, error) { + pods := r.kube.CoreV1().Pods(r.namespace) + cur, err := pods.Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return cur, validatePod(env, p, pvc, cur) + } + if !apierrors.IsNotFound(err) { + return nil, err + } + if textNested(env.Object, "status", "pod", "uid") != "" { + return nil, errors.Join( + &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "previous executor termination unconfirmed"}, + r.setFenceUnknown(ctx, env, "previous executor disappearance is not externally fenced"), + ) + } + cpuReq := p.CPURequest + memReq := p.MemoryRequest + cpuLim := p.CPULimit + memLim := p.MemoryLimit + ephemeralReq := p.EphemeralStorageRequest + ephemeralLim := p.EphemeralStorageLimit + tmpLim := p.TmpSizeLimit + nonroot := true + uid := int64(65532) + noPriv := false + ro := true + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/profile": hashText(textNested(env.Object, "spec", "profile"))[:16]}, Finalizers: []string{executorFinalizer}, OwnerReferences: []metav1.OwnerReference{{APIVersion: env.GetAPIVersion(), Kind: env.GetKind(), Name: env.GetName(), UID: env.GetUID(), Controller: &nonroot}}}, Spec: corev1.PodSpec{AutomountServiceAccountToken: &noPriv, RuntimeClassName: &p.Spec.RuntimeClassName, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &nonroot, RunAsUser: &uid, RunAsGroup: &uid, FSGroup: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "executor", Image: p.Spec.Image, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &noPriv, ReadOnlyRootFilesystem: &ro, RunAsNonRoot: &nonroot, RunAsUser: &uid, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: cpuReq, corev1.ResourceMemory: memReq, corev1.ResourceEphemeralStorage: ephemeralReq}, Limits: corev1.ResourceList{corev1.ResourceCPU: cpuLim, corev1.ResourceMemory: memLim, corev1.ResourceEphemeralStorage: ephemeralLim}}, VolumeMounts: []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}, {Name: "tmp", MountPath: "/tmp"}}}}, Volumes: []corev1.Volume{{Name: "workspace", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc}}}, {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &tmpLim}}}}}} + created, err := pods.Create(ctx, pod, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + created, err = pods.Get(ctx, name, metav1.GetOptions{}) + } + if err != nil { + return nil, err + } + return created, validatePod(env, p, pvc, created) +} + +func validatePVC(env *unstructured.Unstructured, p resolvedProfile, pvc *corev1.PersistentVolumeClaim) error { + storedUID := textNested(env.Object, "status", "pvc", "uid") + qty, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage] + if pvc.Labels["execution.mecatl.dev/environment"] != env.GetName() || pvc.Labels["execution.mecatl.dev/revision"] != textNested(env.Object, "spec", "revision") || pvc.Labels["execution.mecatl.dev/allocation-uid"] != string(env.GetUID()) || len(pvc.OwnerReferences) != 0 || (storedUID != "" && storedUID != string(pvc.UID)) || pvc.Spec.StorageClassName == nil || *pvc.Spec.StorageClassName != p.Spec.StorageClass || len(pvc.Spec.AccessModes) != 1 || pvc.Spec.AccessModes[0] != corev1.ReadWriteOnce || pvc.Spec.VolumeMode == nil || *pvc.Spec.VolumeMode != corev1.PersistentVolumeFilesystem || !ok || qty.Cmp(p.StorageSize) != 0 { + return errors.New("PVC ownership or immutable specification mismatch") + } + return nil +} + +func validatePod(env *unstructured.Unstructured, p resolvedProfile, pvcName string, pod *corev1.Pod) error { //nolint:gocyclo // Every security-sensitive immutable field is checked explicitly. + if pod.DeletionTimestamp != nil { + return errors.New("pod is terminating") + } + storedUID := textNested(env.Object, "status", "pod", "uid") + if pod.Labels["execution.mecatl.dev/environment"] != env.GetName() || pod.Labels["execution.mecatl.dev/profile"] != hashText(textNested(env.Object, "spec", "profile"))[:16] || (storedUID != "" && storedUID != string(pod.UID)) || len(pod.OwnerReferences) != 1 { + return errors.New("pod ownership identity mismatch") + } + owner := pod.OwnerReferences[0] + if owner.APIVersion != env.GetAPIVersion() || owner.Kind != env.GetKind() || owner.Name != env.GetName() || owner.UID != env.GetUID() || owner.Controller == nil || !*owner.Controller || !reflect.DeepEqual(pod.Finalizers, []string{executorFinalizer}) { + return errors.New("pod controller ownership mismatch") + } + if pod.Spec.RestartPolicy != corev1.RestartPolicyNever || pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken || pod.Spec.RuntimeClassName == nil || *pod.Spec.RuntimeClassName != p.Spec.RuntimeClassName || len(pod.Spec.Containers) != 1 || len(pod.Spec.InitContainers) != 0 || len(pod.Spec.EphemeralContainers) != 0 || len(pod.Spec.Volumes) != 2 { + return errors.New("pod immutable specification mismatch") + } + psc := pod.Spec.SecurityContext + if psc == nil || psc.RunAsNonRoot == nil || !*psc.RunAsNonRoot || psc.RunAsUser == nil || *psc.RunAsUser != 65532 || psc.RunAsGroup == nil || *psc.RunAsGroup != 65532 || psc.FSGroup == nil || *psc.FSGroup != 65532 || psc.SeccompProfile == nil || psc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + return errors.New("pod security context mismatch") + } + c := pod.Spec.Containers[0] + if c.Name != "executor" || c.Image != p.Spec.Image || !reflect.DeepEqual(c.Command, []string{"/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"}) || !reflect.DeepEqual(c.VolumeMounts, []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}, {Name: "tmp", MountPath: "/tmp"}}) { + return errors.New("executor container specification mismatch") + } + cs := c.SecurityContext + if cs == nil || cs.AllowPrivilegeEscalation == nil || *cs.AllowPrivilegeEscalation || cs.ReadOnlyRootFilesystem == nil || !*cs.ReadOnlyRootFilesystem || cs.RunAsNonRoot == nil || !*cs.RunAsNonRoot || cs.RunAsUser == nil || *cs.RunAsUser != 65532 || cs.Capabilities == nil || !reflect.DeepEqual(cs.Capabilities.Drop, []corev1.Capability{"ALL"}) { + return errors.New("executor container security mismatch") + } + wantRequests := corev1.ResourceList{corev1.ResourceCPU: p.CPURequest, corev1.ResourceMemory: p.MemoryRequest, corev1.ResourceEphemeralStorage: p.EphemeralStorageRequest} + wantLimits := corev1.ResourceList{corev1.ResourceCPU: p.CPULimit, corev1.ResourceMemory: p.MemoryLimit, corev1.ResourceEphemeralStorage: p.EphemeralStorageLimit} + if !reflect.DeepEqual(c.Resources.Requests, wantRequests) || !reflect.DeepEqual(c.Resources.Limits, wantLimits) || pod.Spec.Volumes[0].Name != "workspace" || pod.Spec.Volumes[0].PersistentVolumeClaim == nil || pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName != pvcName || pod.Spec.Volumes[1].Name != "tmp" || pod.Spec.Volumes[1].EmptyDir == nil || pod.Spec.Volumes[1].EmptyDir.SizeLimit == nil || pod.Spec.Volumes[1].EmptyDir.SizeLimit.Cmp(p.TmpSizeLimit) != 0 { + return errors.New("executor resources or volumes mismatch") + } + return nil +} + +func (r *Reconciler) reconcileDeletion(ctx context.Context, _ dynamic.ResourceInterface, env *unstructured.Unstructured) error { + return r.setCondition(ctx, env, "DeletionBlocked", true, "ExactLifecycleRequired", "deletion requires the exact retained-environment lifecycle operation") +} +func (r *Reconciler) updateRuntimeStatus(ctx context.Context, env *unstructured.Unstructured, pvc *corev1.PersistentVolumeClaim, pod *corev1.Pod, ready bool) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + if !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pvc.Name, "uid": string(pvc.UID)}, "status", "pvc") + _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pod.Name, "uid": string(pod.UID)}, "status", "pod") + _ = unstructured.SetNestedField(o.Object, o.GetGeneration(), "status", "observedGeneration") + setConditionObject(o, "Ready", ready, "Reconciled", map[bool]string{true: "PVC and executor Pod are ready", false: "waiting for executor Pod readiness"}[ready]) + return nil + }) +} +func (r *Reconciler) setFenceUnknown(ctx context.Context, env *unstructured.Unstructured, msg string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + if !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + _ = unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") + setConditionObject(o, "Ready", false, "FenceUnknown", msg) + return nil + }) +} +func (r *Reconciler) setCondition(ctx context.Context, env *unstructured.Unstructured, name string, status bool, reason, msg string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { setConditionObject(o, name, status, reason, msg); return nil }) +} +func (r *Reconciler) updateStatus(ctx context.Context, env *unstructured.Unstructured, fn func(*unstructured.Unstructured) error) error { + res := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace) + for i := 0; i < 5; i++ { + cur, err := res.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + return err + } + before, _, _ := unstructured.NestedMap(cur.Object, "status") + if err := fn(cur); err != nil { + return err + } + after, _, _ := unstructured.NestedMap(cur.Object, "status") + if reflect.DeepEqual(before, after) { + return nil + } + if _, err = res.UpdateStatus(ctx, cur, metav1.UpdateOptions{}); err == nil { + return nil + } else if !apierrors.IsConflict(err) { + return err + } + } + return errors.New("status update conflict limit exceeded") +} +func setConditionObject(o *unstructured.Unstructured, name string, status bool, reason, msg string) { + conds, _, _ := unstructured.NestedSlice(o.Object, "status", "conditions") + st := "False" + if status { + st = "True" + } + replacement := map[string]any{"type": name, "status": st, "reason": reason, "message": msg, "observedGeneration": o.GetGeneration()} + next := make([]any, 0, len(conds)+1) + replaced := false + for _, value := range conds { + condition, ok := value.(map[string]any) + if !ok || text(condition, "type") != name { + next = append(next, value) + continue + } + if replaced { + continue + } + transitionTime := text(condition, "lastTransitionTime") + if text(condition, "status") != st || transitionTime == "" { + transitionTime = time.Now().UTC().Format(time.RFC3339) + } + replacement["lastTransitionTime"] = transitionTime + next = append(next, replacement) + replaced = true + } + if !replaced { + replacement["lastTransitionTime"] = time.Now().UTC().Format(time.RFC3339) + next = append(next, replacement) + } + _ = unstructured.SetNestedSlice(o.Object, next, "status", "conditions") +} +func podReady(p *corev1.Pod) bool { + if p == nil || p.DeletionTimestamp != nil { + return false + } + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + return true + } + } + return false +} +func resourceName(prefix, env string) string { + name := prefix + "-" + strings.TrimPrefix(env, "exec-") + if len(name) > 63 { + name = name[:63] + } + return strings.TrimRight(name, "-") +} diff --git a/internal/adapter/executioncontroller/controller_lifecycle.go b/internal/adapter/executioncontroller/controller_lifecycle.go new file mode 100644 index 0000000000..21a330829d --- /dev/null +++ b/internal/adapter/executioncontroller/controller_lifecycle.go @@ -0,0 +1,312 @@ +package executioncontroller + +import ( + "context" + "errors" + "reflect" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +const deleteRetiredEnvironment = "DeleteRetiredEnvironment" + +func (r *Reconciler) reconcileLifecycle(ctx context.Context, env *unstructured.Unstructured) error { //nolint:gocyclo // Durable phases are deliberately explicit. + op, found, err := unstructured.NestedMap(env.Object, "status", "lifecycleOperation") + if err != nil || !found { + return nil + } + kind, phase := text(op, "type"), text(op, "phase") + operationID := text(op, "id") + podUID, pvcUID := text(op, "expectedPodUID"), text(op, "expectedPVCUID") + podName, pvcName := textNested(env.Object, "status", "pod", "name"), textNested(env.Object, "status", "pvc", "name") + if operationID == "" || pvcUID == "" { + return r.setLifecycleFenceUnknown(ctx, env, "lifecycle operation identity is malformed") + } + proofMatches := textNested(env.Object, "status", "terminationProof", operationIDField) == operationID && textNested(env.Object, "status", "terminationProof", "podUID") == podUID && textNested(env.Object, "status", "terminationProof", "pvcUID") == pvcUID && intNested(env.Object, "status", "terminationProof", "epoch") == intNested(op, "expectedEpoch") + podIdentityMatches := textNested(env.Object, "status", "pod", "uid") == podUID + if phase == "CreatingReplacement" { + podIdentityMatches = textNested(env.Object, "status", "pod", "uid") == "" && proofMatches + } + if kind == deleteRetiredEnvironment { + return r.reconcileRetainedDelete(ctx, env, op, pvcName, pvcUID) + } + if kind == "ReplaceExecutor" { + if _, ok := r.replacementProfile(env); !ok { + return r.setLifecycleFenceUnknown(ctx, env, "replacement profile is unavailable") + } + } + epoch := intNested(op, "expectedEpoch") + if podUID == "" || epoch <= 0 || !podIdentityMatches || textNested(env.Object, "status", "pvc", "uid") != pvcUID || intNested(env.Object, "status", "epoch") != epoch { + return r.setLifecycleFenceUnknown(ctx, env, "lifecycle operation no longer matches the exact executor and workspace") + } + pvc, err := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err != nil || string(pvc.UID) != pvcUID { + if err != nil && !apierrors.IsNotFound(err) { + return err + } + return r.setLifecycleFenceUnknown(ctx, env, "authoritative workspace identity is unavailable") + } + switch phase { + case "Quiescing": + pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if getErr != nil || string(pod.UID) != podUID { + if getErr != nil && !apierrors.IsNotFound(getErr) { + return getErr + } + return r.setLifecycleFenceUnknown(ctx, env, "executor disappeared before durable terminal proof") + } + uid := types.UID(podUID) + grace := int64(30) + if deleteErr := r.kube.CoreV1().Pods(r.namespace).Delete(ctx, podName, metav1.DeleteOptions{GracePeriodSeconds: &grace, Preconditions: &metav1.Preconditions{UID: &uid}}); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return deleteErr + } + return r.setLifecyclePhase(ctx, env, operationID, "Quiescing", "WaitingForTermination") + case "WaitingForTermination": + pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if apierrors.IsNotFound(getErr) { + return r.setLifecycleFenceUnknown(ctx, env, "executor disappeared before durable terminal proof") + } + if getErr != nil { + return getErr + } + if string(pod.UID) != podUID { + return r.setLifecycleFenceUnknown(ctx, env, "executor UID changed during quiescing") + } + if !podTerminal(pod) { + return r.setLifecycleCondition(ctx, env, "Ready", false, "AwaitingTerminalExecutor", "waiting for kubelet terminal phase and terminated container states") + } + q := adminLifecycleRequest{ExpectedEpoch: uint64(epoch), ExpectedPodUID: podUID, ExpectedPVCUID: pvcUID, OperationID: operationID} //nolint:gosec // positivity checked above. + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + current, ok, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if !ok || text(current, "id") != operationID || text(current, "phase") != phase || !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + if err := unstructured.SetNestedMap(o.Object, terminationProof(q, pod), "status", "terminationProof"); err != nil { + return err + } + current["phase"] = "RemovingPodFinalizer" + return unstructured.SetNestedMap(o.Object, current, "status", "lifecycleOperation") + }) + case "RemovingPodFinalizer": + if !proofMatches { + return r.setLifecycleFenceUnknown(ctx, env, "durable terminal proof is missing or mismatched") + } + pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if getErr == nil { + if string(pod.UID) != podUID || !podTerminal(pod) { + return r.setLifecycleFenceUnknown(ctx, env, "terminal executor identity changed before finalizer removal") + } + pod = pod.DeepCopy() + pod.Finalizers = withoutString(pod.Finalizers, executorFinalizer) + if _, updateErr := r.kube.CoreV1().Pods(r.namespace).Update(ctx, pod, metav1.UpdateOptions{}); updateErr != nil { + return updateErr + } + } else if !apierrors.IsNotFound(getErr) { + return getErr + } + return r.setLifecyclePhase(ctx, env, operationID, "RemovingPodFinalizer", "WaitingForPodDeletion") + case "WaitingForPodDeletion": + if !proofMatches { + return r.setLifecycleFenceUnknown(ctx, env, "durable terminal proof is missing or mismatched") + } + pod, getErr := r.kube.CoreV1().Pods(r.namespace).Get(ctx, podName, metav1.GetOptions{}) + if getErr == nil { + if string(pod.UID) != podUID { + return r.setLifecycleFenceUnknown(ctx, env, "unexpected executor occupies the retained name") + } + uid := types.UID(podUID) + if deleteErr := r.kube.CoreV1().Pods(r.namespace).Delete(ctx, podName, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return deleteErr + } + return nil + } + if !apierrors.IsNotFound(getErr) { + return getErr + } + if kind == "RetireEnvironment" { + return r.finishRetirement(ctx, env, operationID) + } + return r.prepareReplacement(ctx, env, operationID) + case "CreatingReplacement": + return r.finishReplacement(ctx, env, operationID) + default: + return r.setLifecycleFenceUnknown(ctx, env, "unknown lifecycle phase") + } +} + +func (r *Reconciler) setLifecyclePhase(ctx context.Context, env *unstructured.Unstructured, operationID, expectedPhase, nextPhase string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + op, found, err := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if err != nil || !found || text(op, "id") != operationID || text(op, "phase") != expectedPhase || !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + op["phase"] = nextPhase + return unstructured.SetNestedMap(o.Object, op, "status", "lifecycleOperation") + }) +} + +func (r *Reconciler) finishRetirement(ctx context.Context, env *unstructured.Unstructured, operationID string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + op, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if !found || text(op, "id") != operationID || text(op, "type") != "RetireEnvironment" || text(op, "phase") != "WaitingForPodDeletion" || !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + unstructured.RemoveNestedField(o.Object, "status", "lifecycleOperation") + setConditionObject(o, "ExecutorTerminated", true, "TerminalPodProof", "terminal executor proof is durable") + setConditionObject(o, "Retired", true, "WorkspaceRetained", "executor retired; workspace PVC retained") + setConditionObject(o, "Ready", false, "Retained", "environment is retired with retained storage") + return nil + }) +} + +func (r *Reconciler) prepareReplacement(ctx context.Context, env *unstructured.Unstructured, operationID string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + op, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if !found || text(op, "id") != operationID || text(op, "type") != "ReplaceExecutor" || text(op, "phase") != "WaitingForPodDeletion" || !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + unstructured.RemoveNestedField(o.Object, "status", "pod") + op["phase"] = "CreatingReplacement" + return unstructured.SetNestedMap(o.Object, op, "status", "lifecycleOperation") + }) +} + +func (r *Reconciler) replacementProfile(env *unstructured.Unstructured) (resolvedProfile, bool) { + profile, ok := r.profiles.get(textNested(env.Object, "spec", "profile")) + return profile, ok && profile.Digest == textNested(env.Object, "spec", "profileDigest") +} + +func (r *Reconciler) finishReplacement(ctx context.Context, env *unstructured.Unstructured, operationID string) error { + profile, ok := r.replacementProfile(env) + if !ok { + return r.setLifecycleFenceUnknown(ctx, env, "replacement profile is unavailable") + } + pvcName := textNested(env.Object, "status", "pvc", "name") + pod, err := r.ensurePod(ctx, env, profile, resourceName("executor", env.GetName()), pvcName) + if err != nil { + return err + } + if !podReady(pod) { + return r.setLifecycleCondition(ctx, env, "Ready", false, "ReplacementStarting", "waiting for replacement executor readiness") + } + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + op, found, _ := unstructured.NestedMap(o.Object, "status", "lifecycleOperation") + if !found || text(op, "id") != operationID || text(op, "type") != "ReplaceExecutor" || text(op, "phase") != "CreatingReplacement" || !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + epoch := intNested(o.Object, "status", "epoch") + if epoch <= 0 { + return errors.New("execution epoch cannot advance") + } + _ = unstructured.SetNestedField(o.Object, epoch+1, "status", "epoch") + _ = unstructured.SetNestedMap(o.Object, map[string]any{"name": pod.Name, "uid": string(pod.UID)}, "status", "pod") + // Completed migration receipts are bound to the runtime being replaced. + unstructured.RemoveNestedField(o.Object, "status", "lastMigrationOperationID") + unstructured.RemoveNestedField(o.Object, "status", "lastMigrationFromSchema") + _ = unstructured.SetNestedMap(o.Object, map[string]any{operationIDField: operationID, "previousPodUID": text(op, "expectedPodUID"), "replacementPodUID": string(pod.UID), "pvcUID": text(op, "expectedPVCUID"), "previousEpoch": intNested(op, "expectedEpoch"), "replacementEpoch": epoch + 1}, "status", "lastReplacement") + unstructured.RemoveNestedField(o.Object, "status", "lifecycleOperation") + setConditionObject(o, "Ready", true, "ReplacementReady", "replacement executor is ready on the retained workspace") + return nil + }) +} + +func (r *Reconciler) reconcileRetainedDelete(ctx context.Context, env *unstructured.Unstructured, op map[string]any, pvcName, pvcUID string) error { //nolint:gocyclo // Exact retained-deletion phases deliberately fail closed at each identity check. + phase := text(op, "phase") + if (phase != "DeletingPVC" && phase != "ReleasingSlot") || !conditionTrue(env, "Retired") || !conditionTrue(env, "ExecutorTerminated") { + return r.setLifecycleFenceUnknown(ctx, env, "retained deletion state is invalid") + } + pvc, err := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Get(ctx, pvcName, metav1.GetOptions{}) + if err == nil { + if phase != "DeletingPVC" { + return r.setLifecycleFenceUnknown(ctx, env, "retained PVC reappeared after deallocation began") + } + if string(pvc.UID) != pvcUID { + return r.setLifecycleFenceUnknown(ctx, env, "retained PVC UID changed") + } + uid := types.UID(pvcUID) + if deleteErr := r.kube.CoreV1().PersistentVolumeClaims(r.namespace).Delete(ctx, pvcName, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); deleteErr != nil && !apierrors.IsNotFound(deleteErr) { + return deleteErr + } + return nil + } + if !apierrors.IsNotFound(err) { + return err + } + if phase == "DeletingPVC" { + return r.setLifecyclePhase(ctx, env, text(op, "id"), "DeletingPVC", "ReleasingSlot") + } + res := r.dynamic.Resource(ExecutionEnvironmentGVR).Namespace(r.namespace) + current, err := res.Get(ctx, env.GetName(), metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + currentOp, found, _ := unstructured.NestedMap(current.Object, "status", "lifecycleOperation") + if current.GetUID() != env.GetUID() || !found || text(currentOp, "id") != text(op, "id") || text(currentOp, "type") != text(op, "type") || text(currentOp, "phase") != "ReleasingSlot" || !runtimeObservationMatches(current, env) { + return lifecycleConflict() + } + if err := releaseProfileSlot(ctx, r.kube, r.namespace, textNested(current.Object, "spec", "profile"), current.GetName()); err != nil { + return err + } + current.SetFinalizers(withoutString(current.GetFinalizers(), environmentFinalizer)) + updated, err := res.Update(ctx, current, metav1.UpdateOptions{}) + if err != nil { + return err + } + uid, version := updated.GetUID(), updated.GetResourceVersion() + if err := res.Delete(ctx, updated.GetName(), metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid, ResourceVersion: &version}}); err != nil && !apierrors.IsNotFound(err) { + return err + } + return nil +} + +func lifecycleConflict() error { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "lifecycle operation changed concurrently"} +} + +func runtimeObservationMatches(current, observed *unstructured.Unstructured) bool { + currentOp, currentHasOp, _ := unstructured.NestedMap(current.Object, "status", "lifecycleOperation") + observedOp, observedHasOp, _ := unstructured.NestedMap(observed.Object, "status", "lifecycleOperation") + return currentHasOp == observedHasOp && reflect.DeepEqual(currentOp, observedOp) && + intNested(current.Object, "status", "epoch") == intNested(observed.Object, "status", "epoch") && + textNested(current.Object, "status", "pod", "uid") == textNested(observed.Object, "status", "pod", "uid") && + textNested(current.Object, "status", "pvc", "uid") == textNested(observed.Object, "status", "pvc", "uid") +} + +func (r *Reconciler) setLifecycleCondition(ctx context.Context, env *unstructured.Unstructured, name string, status bool, reason, msg string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + if !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + setConditionObject(o, name, status, reason, msg) + return nil + }) +} + +func (r *Reconciler) setLifecycleFenceUnknown(ctx context.Context, env *unstructured.Unstructured, msg string) error { + return r.updateStatus(ctx, env, func(o *unstructured.Unstructured) error { + if !runtimeObservationMatches(o, env) { + return lifecycleConflict() + } + _ = unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") + setConditionObject(o, "Ready", false, "FenceUnknown", msg) + return nil + }) +} + +func withoutString(values []string, remove string) []string { + out := make([]string, 0, len(values)) + for _, value := range values { + if value != remove { + out = append(out, value) + } + } + return out +} diff --git a/internal/adapter/executioncontroller/controller_test.go b/internal/adapter/executioncontroller/controller_test.go new file mode 100644 index 0000000000..fe80cb70dd --- /dev/null +++ b/internal/adapter/executioncontroller/controller_test.go @@ -0,0 +1,412 @@ +package executioncontroller + +import ( + "context" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + nodev1 "k8s.io/api/node/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/util/workqueue" + clocktesting "k8s.io/utils/clock/testing" +) + +func TestReconcileRefusesForeignExistingPVCWithoutPersistingUID(t *testing.T) { + ctx := context.Background() + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + foreign := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "workspace-test", Namespace: "ns", UID: types.UID("foreign"), Labels: map[string]string{"execution.mecatl.dev/environment": env.GetName(), "execution.mecatl.dev/revision": "rev"}}} + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(foreign) + r := NewReconciler(d, k, "ns", testProfiles()) + if err := r.Reconcile(ctx, env.GetName()); err == nil { + t.Fatal("foreign PVC ownership mismatch was not returned") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if uid := textNested(got.Object, "status", "pvc", "uid"); uid != "" { + t.Fatalf("foreign PVC UID persisted as authoritative: %q", uid) + } + pods, err := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if err != nil || len(pods.Items) != 0 { + t.Fatalf("executor created over foreign PVC: pods=%d err=%v", len(pods.Items), err) + } +} + +func TestReconcileCreatesTokenlessNonRootPodAndRetainedPVC(t *testing.T) { + ctx := context.Background() + env := testEnvironment() + scheme := runtime.NewScheme() + d := dynamicfake.NewSimpleDynamicClient(scheme, env) + k := kubefake.NewSimpleClientset() + r := NewReconciler(d, k, "ns", testProfiles()) + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + pvcs, _ := k.CoreV1().PersistentVolumeClaims("ns").List(ctx, metav1.ListOptions{}) + if len(pvcs.Items) != 1 { + t.Fatalf("pvcs=%d", len(pvcs.Items)) + } + if len(pvcs.Items[0].OwnerReferences) != 0 { + t.Fatal("PVC must not have owner reference") + } + pods, _ := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if len(pods.Items) != 1 { + t.Fatalf("pods=%d", len(pods.Items)) + } + p := pods.Items[0] + if p.Spec.AutomountServiceAccountToken == nil || *p.Spec.AutomountServiceAccountToken { + t.Fatal("service account token mounted") + } + if p.Spec.SecurityContext == nil || p.Spec.SecurityContext.RunAsNonRoot == nil || !*p.Spec.SecurityContext.RunAsNonRoot { + t.Fatal("pod is not non-root") + } + c := p.Spec.Containers[0] + if c.SecurityContext == nil || c.SecurityContext.ReadOnlyRootFilesystem == nil || !*c.SecurityContext.ReadOnlyRootFilesystem || c.SecurityContext.AllowPrivilegeEscalation == nil || *c.SecurityContext.AllowPrivilegeEscalation { + t.Fatal("container security context is not hardened") + } + if p.Spec.RuntimeClassName == nil || *p.Spec.RuntimeClassName != "sandboxed" || c.Resources.Requests.Cpu().IsZero() || c.Resources.Requests.Memory().IsZero() || c.Resources.Requests.StorageEphemeral().IsZero() || c.Resources.Limits.StorageEphemeral().IsZero() { + t.Fatal("runtime class or resource bounds are missing") + } + if p.Spec.Volumes[1].EmptyDir == nil || p.Spec.Volumes[1].EmptyDir.SizeLimit == nil || p.Spec.Volumes[1].EmptyDir.SizeLimit.String() != "256Mi" { + t.Fatal("tmp volume is not profile-bounded") + } +} +func TestTerminatingPodIsUnavailableDuringReconcile(t *testing.T) { + ctx := t.Context() + env := testEnvironment() + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + r := NewReconciler(d, k, "ns", testProfiles()) + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + pod, err := k.CoreV1().Pods("ns").Get(ctx, "executor-test", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + now := metav1.Now() + pod.DeletionTimestamp = &now + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if _, err := k.CoreV1().Pods("ns").Update(ctx, pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err == nil { + t.Fatal("terminating Pod was not rejected") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if conditionTrue(got, "Ready") { + t.Fatalf("terminating pod remained ready: %v", got.Object["status"]) + } +} + +func TestRepeatedReconcileDoesNotRewriteUnchangedStatus(t *testing.T) { + ctx := context.Background() + env := testEnvironment() + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + r := NewReconciler(d, k, "ns", testProfiles()) + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + writes := statusUpdateCount(d.Actions()) + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + transition := conditionTransition(got, "Ready") + if transition == "" { + t.Fatal("Ready condition has no transition timestamp") + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if extra := statusUpdateCount(d.Actions()) - writes; extra != 0 { + t.Fatalf("unchanged reconcile performed %d status writes", extra) + } + got, err = d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if current := conditionTransition(got, "Ready"); current != transition { + t.Fatalf("lastTransitionTime changed from %q to %q", transition, current) + } + + pods, err := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if err != nil || len(pods.Items) != 1 { + t.Fatalf("pods=%d err=%v", len(pods.Items), err) + } + pod := pods.Items[0].DeepCopy() + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if _, err := k.CoreV1().Pods("ns").UpdateStatus(ctx, pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + got, err = d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !conditionTrue(got, "Ready") { + t.Fatalf("Ready did not follow Pod readiness: %v", got.Object["status"]) + } + if statusUpdateCount(d.Actions()) != writes+1 { + t.Fatalf("status writes=%d, want %d", statusUpdateCount(d.Actions()), writes+1) + } +} + +func TestActiveOperationExpiryRequeuesWithoutAnotherEvent(t *testing.T) { + ctx := t.Context() + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + kubeClient := kubefake.NewSimpleClientset() + r := NewReconciler(dynamicClient, kubeClient, "ns", testProfiles()) + initialQueue := r.queue + t.Cleanup(initialQueue.ShutDown) + + // Establish a fully reconciled, ready environment so only operation expiry can + // cause the transition below. + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + pod, err := kubeClient.CoreV1().Pods("ns").Get(ctx, "executor-test", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if _, err := kubeClient.CoreV1().Pods("ns").UpdateStatus(ctx, pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + initialQueue.ShutDown() + + now := time.Date(2026, 9, 21, 12, 0, 0, 0, time.UTC) + deadline := now.Add(100 * time.Millisecond) + current, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedMap(current.Object, map[string]any{ + "id": "operation", "expiresAt": deadline.Format(time.RFC3339Nano), + }, "status", "activeOperation"); err != nil { + t.Fatal(err) + } + if _, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(ctx, current, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + + fakeClock := clocktesting.NewFakeClock(now) + delayingQueue := workqueue.NewTypedDelayingQueueWithConfig[string](workqueue.TypedDelayingQueueConfig[string]{Clock: fakeClock}) + replacementQueue := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[string](), + workqueue.TypedRateLimitingQueueConfig[string]{DelayingQueue: delayingQueue}, + ) + t.Cleanup(replacementQueue.ShutDown) + r.queue = replacementQueue + r.now = fakeClock.Now + if err := r.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + beforeExpiry, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !conditionTrue(beforeExpiry, "Ready") || textNested(beforeExpiry.Object, "status", "fenceState") != fenceHealthy { + t.Fatalf("future operation lease disturbed ready state: %v", beforeExpiry.Object["status"]) + } + + waitUntil := time.Now().Add(time.Second) + for fakeClock.Waiters() < 2 && time.Now().Before(waitUntil) { + time.Sleep(time.Millisecond) + } + if fakeClock.Waiters() < 2 { + t.Fatal("reconcile did not register the operation-expiry deadline with the delayed queue") + } + + done := make(chan struct{}) + go func() { + r.worker(ctx) + close(done) + }() + t.Cleanup(func() { + replacementQueue.ShutDown() + <-done + }) + + fakeClock.Step(99 * time.Millisecond) + preDeadline, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !conditionTrue(preDeadline, "Ready") || textNested(preDeadline.Object, "status", "fenceState") != fenceHealthy { + t.Fatalf("operation fenced before its deadline: %v", preDeadline.Object["status"]) + } + + fakeClock.Step(time.Millisecond) + workerDeadline := time.Now().Add(2 * time.Second) + for time.Now().Before(workerDeadline) { + got, getErr := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if getErr != nil { + t.Fatal(getErr) + } + if textNested(got.Object, "status", "fenceState") == "FenceUnknown" { + if textNested(got.Object, "status", "activeOperation", "id") != "operation" || conditionTrue(got, "Ready") { + t.Fatalf("expiry did not retain unresolved operation identity and clear readiness: %v", got.Object["status"]) + } + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("worker did not reconcile the active-operation deadline without another Kubernetes event") +} + +func TestRuntimeAndRelevantStatusUpdatesEnqueue(t *testing.T) { + r := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kubefake.NewSimpleClientset(), "ns", testProfiles()) + r.enqueueRuntime(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"execution.mecatl.dev/environment": "exec-pod"}}}) + r.enqueueRuntime(&corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"execution.mecatl.dev/environment": "exec-pvc"}}}) + if r.queue.Len() != 2 { + t.Fatalf("runtime queue length=%d", r.queue.Len()) + } + + oldEnv := testEnvironment() + newEnv := oldEnv.DeepCopy() + _ = unstructured.SetNestedField(newEnv.Object, "op", "status", "activeOperation", "id") + r.enqueueUpdate(oldEnv, newEnv) + if r.queue.Len() != 3 { + t.Fatalf("relevant status update was not queued: len=%d", r.queue.Len()) + } + lifecycle := newEnv.DeepCopy() + _ = unstructured.SetNestedMap(lifecycle.Object, map[string]any{"id": "retire", "phase": "Quiescing"}, "status", "lifecycleOperation") + lifecycleQueue := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kubefake.NewSimpleClientset(), "ns", testProfiles()) + lifecycleQueue.enqueueUpdate(newEnv, lifecycle) + if lifecycleQueue.queue.Len() != 1 { + t.Fatalf("lifecycle start was not queued: len=%d", lifecycleQueue.queue.Len()) + } + advanced := lifecycle.DeepCopy() + _ = unstructured.SetNestedField(advanced.Object, "WaitingForTermination", "status", "lifecycleOperation", "phase") + phaseQueue := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kubefake.NewSimpleClientset(), "ns", testProfiles()) + phaseQueue.enqueueUpdate(lifecycle, advanced) + if phaseQueue.queue.Len() != 1 { + t.Fatalf("lifecycle phase change was not queued: len=%d", phaseQueue.queue.Len()) + } + controllerStatus := advanced.DeepCopy() + _ = unstructured.SetNestedField(controllerStatus.Object, "pod", "status", "pod", "name") + r.enqueueUpdate(advanced, controllerStatus) + if r.queue.Len() != 3 { + t.Fatalf("controller-owned status update was queued: len=%d", r.queue.Len()) + } +} + +func statusUpdateCount(actions []k8stesting.Action) int { + count := 0 + for _, action := range actions { + if action.GetVerb() == "update" && action.GetSubresource() == "status" { + count++ + } + } + return count +} + +func conditionTransition(env *unstructured.Unstructured, name string) string { + conditions, _, _ := unstructured.NestedSlice(env.Object, "status", "conditions") + for _, value := range conditions { + condition, ok := value.(map[string]any) + if ok && text(condition, "type") == name { + return text(condition, "lastTransitionTime") + } + } + return "" +} + +func TestStartupDoesNotFenceLivePeerOperation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + env := testEnvironment() + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "old", "operation": "file.replace"}, "status", "activeOperation") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + r := NewReconciler(d, profileResourceClient(), "ns", testProfiles()) + peer := NewReconciler(d, profileResourceClient(), "ns", testProfiles()) + if r.Ready() || peer.Ready() { + t.Fatal("reconciler reported ready before cache synchronization") + } + if err := r.Initialize(ctx); err != nil { + t.Fatal(err) + } + if err := peer.Initialize(ctx); err != nil { + t.Fatal(err) + } + if !r.Ready() || !peer.Ready() { + t.Fatal("reconcilers did not report ready after cache synchronization") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(got.Object, "status", "fenceState") != "Healthy" || textNested(got.Object, "status", "activeOperation", "id") != "old" { + t.Fatalf("startup mutated a potentially live peer operation: status=%v", got.Object["status"]) + } +} +func TestInitializeRefusesMissingRuntimeClassBeforeCreatingPods(t *testing.T) { + kube := kubefake.NewSimpleClientset(&storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "standard"}}) + r := NewReconciler(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), kube, "ns", testProfiles()) + err := r.Initialize(t.Context()) + if err == nil || !strings.Contains(err.Error(), `profile preflight: RuntimeClass "sandboxed" unavailable`) { + t.Fatalf("Initialize error = %v", err) + } + if r.Ready() { + t.Fatal("reconciler reported ready after failed profile preflight") + } + for _, action := range kube.Actions() { + if action.GetVerb() == "create" && action.GetResource().Resource == "pods" { + t.Fatal("profile preflight created a Pod") + } + } +} + +func profileResourceClient() *kubefake.Clientset { + return kubefake.NewSimpleClientset( + &nodev1.RuntimeClass{ObjectMeta: metav1.ObjectMeta{Name: "sandboxed"}}, + &storagev1.StorageClass{ObjectMeta: metav1.ObjectMeta{Name: "standard"}}, + ) +} + +func testProfiles() *Profiles { + spec := ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100} + profile, err := validateProfile("go", spec) + if err != nil { + panic(err) + } + profile.Spec = spec + profile.Digest = "sha256:profile" + return &Profiles{byName: map[string]resolvedProfile{"go": profile}} +} +func testEnvironment() *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "exec-test", "namespace": "ns", "uid": string(types.UID("uid"))}, "spec": map[string]any{"schemaVersion": int64(2), "profile": "go", "profileDigest": "sha256:profile", "revision": "rev", "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "references": []any{}, "fenceState": "Healthy"}}} +} diff --git a/internal/adapter/executioncontroller/crd_schema_test.go b/internal/adapter/executioncontroller/crd_schema_test.go new file mode 100644 index 0000000000..4ccfed4d8f --- /dev/null +++ b/internal/adapter/executioncontroller/crd_schema_test.go @@ -0,0 +1,90 @@ +package executioncontroller + +import ( + "os" + "testing" + + apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apiextensionsvalidation "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/validation" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema" + "k8s.io/apiextensions-apiserver/pkg/apiserver/schema/pruning" + apiservervalidation "k8s.io/apiextensions-apiserver/pkg/apiserver/validation" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +func loadExecutionEnvironmentCRD(t *testing.T) (*apiextensionsv1.CustomResourceDefinition, *apiextensions.CustomResourceDefinition) { + t.Helper() + raw, err := os.ReadFile("../../../deploy/helm/mecatl-execution/crds/executionenvironment.yaml") + if err != nil { + t.Fatal(err) + } + var crd apiextensionsv1.CustomResourceDefinition + if err := yaml.Unmarshal(raw, &crd); err != nil { + t.Fatal(err) + } + apiextensionsv1.SetDefaults_CustomResourceDefinition(&crd) + var internal apiextensions.CustomResourceDefinition + if err := apiextensionsv1.Convert_v1_CustomResourceDefinition_To_apiextensions_CustomResourceDefinition(&crd, &internal, nil); err != nil { + t.Fatal(err) + } + return &crd, &internal +} + +func TestExecutionEnvironmentCRDIsAcceptedByKubernetesValidation(t *testing.T) { + _, internal := loadExecutionEnvironmentCRD(t) + if errs := apiextensionsvalidation.ValidateCustomResourceDefinition(t.Context(), internal); len(errs) != 0 { + t.Fatalf("CRD is not installable: %v", errs.ToAggregate()) + } +} + +func TestMigrationReceiptSchemaAdmission(t *testing.T) { + crd, _ := loadExecutionEnvironmentCRD(t) + field := crd.Spec.Versions[0].Schema.OpenAPIV3Schema.Properties["status"].Properties["lastMigrationFromSchema"] + var internal apiextensions.JSONSchemaProps + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(&field, &internal, nil); err != nil { + t.Fatal(err) + } + validator, _, err := apiservervalidation.NewSchemaValidator(&internal) + if err != nil { + t.Fatal(err) + } + for _, value := range []any{int64(0), int64(1), int64(-1), int64(2), "0"} { + errs := apiservervalidation.ValidateCustomResource(nil, value, validator) + valid := value == int64(0) || value == int64(1) + if (len(errs) == 0) != valid { + t.Fatalf("receipt source schema %v: valid=%t errors=%v", value, valid, errs) + } + } +} + +func TestExecutionEnvironmentCRDPreservesControllerStatus(t *testing.T) { + crd, _ := loadExecutionEnvironmentCRD(t) + var internal apiextensions.JSONSchemaProps + if err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(crd.Spec.Versions[0].Schema.OpenAPIV3Schema, &internal, nil); err != nil { + t.Fatal(err) + } + structural, err := schema.NewStructural(&internal) + if err != nil { + t.Fatal(err) + } + obj := map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env"}, "spec": map[string]any{}, "status": map[string]any{ + "schemaVersion": int64(2), "observedGeneration": int64(9), "epoch": int64(3), "grantGeneration": int64(4), "fenceState": "Healthy", + "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "op", "sourceBindingID": "source", "createdAt": "2026-09-17T00:00:00Z"}}, + "pvc": map[string]any{"name": "pvc", "uid": "pvc-uid"}, "pod": map[string]any{"name": "pod", "uid": "pod-uid"}, + "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "ownerHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "clientHash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "epoch": int64(3), "grantGeneration": int64(4), "expiresAt": "2026-09-17T00:01:00Z"}, + "renewReceipts": []any{map[string]any{"operationID": "renew", "fingerprint": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", "expiresAt": "2026-09-17T00:01:00Z"}}, + "activeOperation": map[string]any{"id": "file", "operation": "file.read", "startedAt": "2026-09-17T00:00:00Z", "claimID": "claim", "runID": "run", "epoch": int64(3), "holderID": "holder", "renewedAt": "2026-09-17T00:00:01Z", "expiresAt": "2026-09-17T00:01:00Z"}, + "lifecycleOperation": map[string]any{"id": "life", "type": "ReplaceExecutor", "phase": "Quiescing", "expectedEpoch": int64(3), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid", "createdAt": "2026-09-17T00:00:00Z"}, + "terminationProof": map[string]any{"operationID": "life", "podUID": "pod-uid", "pvcUID": "pvc-uid", "epoch": int64(3), "podPhase": "Failed", "observedAt": "2026-09-17T00:00:00Z"}, + "migrationOperation": map[string]any{"id": "migration", "fromSchema": int64(1), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid"}, "lastMigrationOperationID": "migration", "lastMigrationFromSchema": int64(0), + "conditions": []any{map[string]any{"type": "Ready", "status": "True", "reason": "Reconciled", "message": "ready", "observedGeneration": int64(9), "lastTransitionTime": "2026-09-17T00:00:00Z"}}, + }} + pruning.Prune(obj, structural, true) + for _, path := range [][]string{{"status", "schemaVersion"}, {"status", "observedGeneration"}, {"status", "epoch"}, {"status", "grantGeneration"}, {"status", "references"}, {"status", "pvc"}, {"status", "pod"}, {"status", "activeRun", "grantGeneration"}, {"status", "renewReceipts"}, {"status", "activeOperation"}, {"status", "lifecycleOperation"}, {"status", "terminationProof"}, {"status", "migrationOperation"}, {"status", "lastMigrationOperationID"}, {"status", "lastMigrationFromSchema"}, {"status", "conditions"}} { + if _, found, err := unstructured.NestedFieldNoCopy(obj, path...); err != nil || !found { + t.Fatalf("CRD pruning removed controller field %v (found=%t err=%v)", path, found, err) + } + } +} diff --git a/internal/adapter/executioncontroller/disabled_composition_test.go b/internal/adapter/executioncontroller/disabled_composition_test.go new file mode 100644 index 0000000000..3de7bfe3da --- /dev/null +++ b/internal/adapter/executioncontroller/disabled_composition_test.go @@ -0,0 +1,95 @@ +package executioncontroller + +import ( + "reflect" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/internal/app" +) + +func TestDisabledCompositionPreservesAllocationsAndIndependentReconciliation(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + t.Setenv("XDG_STATE_HOME", t.TempDir()) + ctx := t.Context() + env := testEnvironment() + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewClientset() + provider := NewReconciler(d, k, "ns", testProfiles()) + // First persist the finalizer, then provision the runtime. + if err := provider.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + if err := provider.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + resources := d.Resource(ExecutionEnvironmentGVR).Namespace("ns") + before, err := resources.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + pods, err := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if err != nil || len(pods.Items) != 1 { + t.Fatalf("pods=%v err=%v", pods, err) + } + pvcs, err := k.CoreV1().PersistentVolumeClaims("ns").List(ctx, metav1.ListOptions{}) + if err != nil || len(pvcs.Items) != 1 { + t.Fatalf("pvcs=%v err=%v", pvcs, err) + } + d.ClearActions() + k.ClearActions() + // No execution integration is passed to the real composition root. Existing + // runtime objects belong to the independent provider, not the harness lifetime. + built, err := app.Build(ctx, app.Config{UseMock: true, NoSoul: true, NoUserModel: true}) + if err != nil { + t.Fatal(err) + } + sess, err := built.Service.CreateSession(ctx, session.ModeDefault, session.Limits{}) + if err != nil { + built.Close() + t.Fatal(err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "offline") + if err != nil { + built.Close() + t.Fatal(err) + } + for range run.Events() { + } + built.Service.FinishRun(sess.ID, run) + built.Close() + if len(d.Actions()) != 0 || len(k.Actions()) != 0 { + t.Fatal("disabled composition touched provider resources") + } + after, err := resources.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil || !reflect.DeepEqual(before, after) { + t.Fatalf("allocation changed: %v", err) + } + afterPods, err := k.CoreV1().Pods("ns").List(ctx, metav1.ListOptions{}) + if err != nil || !reflect.DeepEqual(pods.Items, afterPods.Items) { + t.Fatalf("executor changed: %v", err) + } + afterPVCs, err := k.CoreV1().PersistentVolumeClaims("ns").List(ctx, metav1.ListOptions{}) + if err != nil || !reflect.DeepEqual(pvcs.Items, afterPVCs.Items) { + t.Fatalf("storage changed: %v", err) + } + pod := pods.Items[0].DeepCopy() + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if _, err := k.CoreV1().Pods("ns").UpdateStatus(ctx, pod, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := provider.Reconcile(ctx, env.GetName()); err != nil { + t.Fatal(err) + } + ready, err := resources.Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil || !conditionTrue(ready, "Ready") { + t.Fatalf("independent provider failed to converge: %v", err) + } +} diff --git a/internal/adapter/executioncontroller/file_authorization_test.go b/internal/adapter/executioncontroller/file_authorization_test.go new file mode 100644 index 0000000000..d7558b6048 --- /dev/null +++ b/internal/adapter/executioncontroller/file_authorization_test.go @@ -0,0 +1,101 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type countedFileBackend struct { + *fakeBackend + calls int +} + +func (b *countedFileBackend) File(context.Context, string, string, executionenv.FileRequest) (executionenv.FileResponse, error) { + b.calls++ + return executionenv.FileResponse{}, nil +} + +func TestEveryFileOperationRequiresExactCurrentGrant(t *testing.T) { + public, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + const client = "spiffe://cluster/ns/mecak8s" + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + for value, name := range executionv1.FileOperation_name { + op, ok := operationFromProto(executionv1.FileOperation(value)) + if !ok { + continue + } + t.Run(name, func(t *testing.T) { + for _, failure := range []string{"client", "owner", "environment", "revision", "epoch", "operation", "expired", "revoked"} { + t.Run(failure, func(t *testing.T) { + backend := &countedFileBackend{fakeBackend: newFakeBackend()} + verifier := executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"key": public}, Issuer: "provider", Audience: "executor", MaxLifetime: time.Minute, Now: func() time.Time { return now }} + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{client: {MayAttestOwner: true}}, Verifier: verifier}, backend) + claims := executionenv.GrantClaims{KeyID: "key", Issuer: "provider", Audience: "executor", Client: client, OwnerHash: ownerHash(owner), BindingID: "binding", RunID: "run", ClaimID: "claim", Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, Epoch: 1, GrantGeneration: 1, Operations: []executionenv.Operation{op}, NotBefore: now.Add(-time.Second), ExpiresAt: now.Add(30 * time.Second), Nonce: "nonce"} + sign := func() string { + token, err := executionenv.SignGrant(private, claims) + if err != nil { + t.Fatal(err) + } + return token + } + req := &executionv1.FileRequest{Context: &executionv1.RequestContext{Environment: refToProto(claims.Environment), Owner: &executionv1.Owner{Issuer: owner.Issuer, Subject: owner.Subject}, BindingId: "binding", RunId: "run", ClaimId: "claim", Epoch: 1, GrantGeneration: 1, Grant: sign()}, Operation: executionv1.FileOperation(value), Path: "file"} + switch op { + case executionenv.OpFileReplace: + req.Version = []byte("version") + case executionenv.OpFileCopy, executionenv.OpFileRename: + req.Destination = "destination" + case executionenv.OpFileGlob: + req.Path = "" + req.Pattern = "*" + case executionenv.OpFileGrep: + req.Pattern = "text" + } + ctx := authenticatedContext(client) + if _, err := h.Files(ctx, req); err != nil || backend.calls != 1 { + t.Fatalf("positive control: calls=%d err=%v", backend.calls, err) + } + negative := proto.Clone(req).(*executionv1.FileRequest) + want := codes.PermissionDenied + switch failure { + case "client": + claims.Client = "spiffe://cluster/ns/other" + case "owner": + claims.OwnerHash = "other" + case "environment": + claims.Environment.ID = "other" + case "revision": + claims.Environment.Revision = "other" + case "epoch": + claims.Epoch++ + case "operation": + claims.Operations = []executionenv.Operation{executionenv.OpCommandStart} + case "expired": + claims.NotBefore = now.Add(-time.Minute) + claims.ExpiresAt = now.Add(-time.Second) + want = codes.Unauthenticated + case "revoked": + h.cfg.Verifier.RevokedNonces = map[string]struct{}{"nonce": {}} + } + negative.Context.Grant = sign() + if _, err := h.Files(ctx, negative); status.Code(err) != want || backend.calls != 1 { + t.Fatalf("negative control: calls=%d code=%v err=%v", backend.calls, status.Code(err), err) + } + }) + } + }) + } +} diff --git a/internal/adapter/executioncontroller/handler.go b/internal/adapter/executioncontroller/handler.go new file mode 100644 index 0000000000..b334865d7f --- /dev/null +++ b/internal/adapter/executioncontroller/handler.go @@ -0,0 +1,947 @@ +// Package executioncontroller implements the authenticated provider gRPC boundary. +// +//nolint:revive // Generated gRPC method names define this private handler surface. +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "errors" + "io" + "math" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" + apierrors "k8s.io/apimachinery/pkg/api/errors" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/internal/executionenv" +) + +// ClientPolicy defines capabilities assigned to an authenticated provider client. +type ClientPolicy struct { + MayAttestOwner, Administrator bool + AdministratorFor []string +} + +// GrantSigner configures short-lived environment grant issuance. +type GrantSigner struct { + KeyID string + PrivateKey ed25519.PrivateKey + Issuer, Audience string + Lifetime time.Duration +} + +// HandlerConfig configures authentication and capability grants. +type HandlerConfig struct { + Clients map[string]ClientPolicy + Signer GrantSigner + Verifier executionenv.GrantVerifier + Ready func() bool + Security *SecurityManager + Diagnostics port.Diagnostics +} + +// Profile is the externally visible immutable execution profile. +type Profile struct { + Name, Digest string + MaxFileBytes, MaxCommandBytes int64 + MaxCommandDuration time.Duration + Capabilities []string +} + +// Allocation is an exact environment allocation and authorization binding. +type Allocation struct { + Environment executionenv.EnvironmentRef + Epoch, GrantGeneration uint64 + OwnerHash, BindingID, Client string + Ready bool +} + +// Backend implements provider-side authorization state and executor dispatch. +type Backend interface { + ValidateProfile(context.Context, string) (Profile, error) + Ensure(context.Context, string, string, string, string, string) (Allocation, error) + Attach(context.Context, executionenv.EnvironmentRef, string, string, string) (Allocation, error) + ReleaseReference(context.Context, executionenv.EnvironmentRef, string, string, string) error + File(context.Context, string, string, executionenv.FileRequest) (executionenv.FileResponse, error) + StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) + CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) + CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) +} + +type runClaimValidator interface { + ValidateRunClaim(context.Context, string, string, executionenv.RequestContext) error +} + +type lifecycleBackend interface { + EnsurePending(context.Context, string, string, string, string, string, string) (Allocation, error) + AcquireRun(context.Context, executionenv.EnvironmentRef, string, string, string, string, string, time.Duration) (executionenv.RunClaim, error) + RenewRun(context.Context, executionenv.EnvironmentRef, string, string, executionenv.RunClaimRequest) (executionenv.RunClaim, error) + ReleaseRun(context.Context, executionenv.EnvironmentRef, string, string, executionenv.RunClaimRequest) error + CommitReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + AbortReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + ReserveSuccessor(context.Context, executionenv.EnvironmentRef, string, string, string, string, string) error + PrepareReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + ConfirmReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + CancelReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error + ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) + FindReferenceIntent(context.Context, executionenv.EnvironmentRef, string, string, string) (executionenv.ReferenceIntent, error) +} + +type ownerIntentBackend interface { + EnsurePendingOwned(context.Context, string, string, executionenv.Owner, string, string, string, string) (Allocation, error) + ListReferenceIntentsForClient(context.Context, string, int) ([]executionenv.ReferenceIntent, error) +} + +type adminLifecycleBackend interface { + ReplaceExecutor(context.Context, adminLifecycleRequest) error + RetireExact(context.Context, adminLifecycleRequest) error + RecoverEnvironment(context.Context, adminLifecycleRequest) error + DeleteRetiredEnvironment(context.Context, adminLifecycleRequest) error + MigrateEnvironment(context.Context, adminLifecycleRequest) error + RevokeEnvironment(context.Context, adminLifecycleRequest, uint64) (uint64, error) +} + +// Handler is the authenticated private execution-provider gRPC service. +type Handler struct { + executionv1.UnimplementedExecutionProviderServiceServer + cfg HandlerConfig + backend Backend + lifecycleBackend lifecycleBackend + adminLifecycleBackend adminLifecycleBackend +} + +// NewHandler constructs the authenticated private gRPC service. +func NewHandler(cfg HandlerConfig, b Backend) *Handler { + lifecycle, _ := b.(lifecycleBackend) + adminLifecycle, _ := b.(adminLifecycleBackend) + return &Handler{cfg: cfg, backend: b, lifecycleBackend: lifecycle, adminLifecycleBackend: adminLifecycle} +} + +type authenticatedClient struct { + id string + policy ClientPolicy +} + +func (h *Handler) client(ctx context.Context) (authenticatedClient, error) { + if h.backend == nil || (h.cfg.Ready != nil && !h.cfg.Ready()) { + return authenticatedClient{}, wireError(executionenv.CodeNotReady, true) + } + chain, handshakeVerified, err := clientCertificates(ctx) + if err != nil || h.cfg.Security == nil && !handshakeVerified { + return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) + } + leaf := chain[0] + if h.cfg.Security != nil { + if !h.cfg.Security.Ready() { + return authenticatedClient{}, wireError(executionenv.CodeNotReady, true) + } + id, policy, err := h.cfg.Security.authorize(ctx, chain) + if err != nil { + return authenticatedClient{}, securityAuthorizationError(err) + } + return authenticatedClient{id: id, policy: policy}, nil + } + id, err := canonicalClientIdentity(leaf) + if err != nil { + return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) + } + policy, ok := h.cfg.Clients[id] + if !ok { + return authenticatedClient{}, wireError(executionenv.CodeUnauthenticated, false) + } + return authenticatedClient{id: id, policy: policy}, nil +} + +func canonicalClientIdentity(c *x509.Certificate) (string, error) { + if len(c.URIs) != 1 { + return "", errors.New("certificate must have exactly one URI SAN") + } + ids := make([]string, 0, 1) + for _, u := range c.URIs { + if u == nil || u.Scheme == "" || u.Host == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" { + continue + } + v := *u + v.Scheme, v.Host = strings.ToLower(v.Scheme), strings.ToLower(v.Host) + ids = append(ids, v.String()) + } + if len(ids) != 1 { + return "", errors.New("certificate must have exactly one canonical URI SAN") + } + return ids[0], nil +} + +func (h *Handler) lifecycle() (lifecycleBackend, error) { + if h.lifecycleBackend == nil { + return nil, wireError(executionenv.CodeNotReady, false) + } + return h.lifecycleBackend, nil +} + +// ValidateProfile validates one operator-defined execution profile. +func (h *Handler) ValidateProfile(ctx context.Context, q *executionv1.ValidateProfileRequest) (*executionv1.ValidateProfileResponse, error) { + if _, err := h.client(ctx); err != nil { + return nil, err + } + if q == nil || q.Profile == "" || len(q.Profile) > 63 { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + p, err := h.backend.ValidateProfile(ctx, q.Profile) + if err != nil { + return nil, backendError(err) + } + return &executionv1.ValidateProfileResponse{Profile: valid(p.Name), Digest: valid(p.Digest), Capabilities: validAll(p.Capabilities), MaxFileBytes: p.MaxFileBytes, MaxCommandBytes: p.MaxCommandBytes, MaxCommandDurationMillis: p.MaxCommandDuration.Milliseconds()}, nil +} + +// EnsureEnvironment idempotently resolves or allocates an environment. +func (h *Handler) EnsureEnvironment(ctx context.Context, q *executionv1.EnsureEnvironmentRequest) (*executionv1.EnsureEnvironmentResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + owner, ok := ownerFromProto(q.GetOwner()) + if q == nil || !c.policy.MayAttestOwner || !ok || q.BindingId == "" || len(q.BindingId) > executionenv.MaxBindingBytes || q.Profile == "" || len(q.Profile) > 63 || !validOperationID(q.OperationId) { + return nil, wireError(executionenv.CodePermissionDenied, false) + } + oh := ownerHash(owner) + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + fp := fingerprint(c.id, oh, q.BindingId, q.Profile) + var a Allocation + if owned, ok := h.backend.(ownerIntentBackend); ok { + a, err = owned.EnsurePendingOwned(ctx, c.id, oh, owner, q.BindingId, q.Profile, fp, q.OperationId) + } else { + a, err = lifecycle.EnsurePending(ctx, c.id, oh, q.BindingId, q.Profile, fp, q.OperationId) + } + if err != nil { + return nil, backendError(err) + } + return ensureResponse(a, "", time.Time{}), nil +} + +// AttachEnvironment exactly reattaches and refreshes a short-lived grant. +func (h *Handler) AttachEnvironment(ctx context.Context, q *executionv1.AttachEnvironmentRequest) (*executionv1.AttachEnvironmentResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + rc, ok := attachContext(q.GetContext()) + if q == nil || q.Purpose != executionenv.PurposeSession || !c.policy.MayAttestOwner || !ok { + return nil, wireError(executionenv.CodePermissionDenied, false) + } + oh := ownerHash(rc.Owner) + a, err := h.backend.Attach(ctx, rc.Environment, c.id, oh, rc.BindingID) + if err != nil { + return nil, backendError(err) + } + return attachResponse(a, "", time.Time{}), nil +} + +// AcquireRun takes environment-wide execution ownership and issues a run-bound grant. +func (h *Handler) AcquireRun(ctx context.Context, q *executionv1.AcquireRunRequest) (*executionv1.RunClaimResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + owner, ok := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if q == nil || !c.policy.MayAttestOwner || !ok || !validRef(ref) || !validBinding(q.BindingId) || !validIdentity(q.RunId) || !validOperationID(q.OperationId) || q.TtlMillis < executionenv.MinRunTTL.Milliseconds() || q.TtlMillis > executionenv.MaxRunTTL.Milliseconds() { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + claim, err := lifecycle.AcquireRun(ctx, ref, c.id, ownerHash(owner), q.BindingId, q.RunId, q.OperationId, time.Duration(q.TtlMillis)*time.Millisecond) + if err != nil { + return nil, backendError(err) + } + return h.runClaimResponse(ctx, claim, c.id, ownerHash(owner)) +} + +func (h *Handler) RenewRun(ctx context.Context, q *executionv1.RenewRunRequest) (*executionv1.RunClaimResponse, error) { + c, owner, req, err := h.runClaimRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetBindingId(), q.GetRunId(), q.GetClaimId(), q.GetEpoch(), q.GetGrantGeneration(), q.GetOperationId(), q.GetTtlMillis()) + if err != nil { + return nil, err + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + claim, backendErr := lifecycle.RenewRun(ctx, req.Environment, c.id, owner, req) + if backendErr != nil { + return nil, backendError(backendErr) + } + return h.runClaimResponse(ctx, claim, c.id, owner) +} + +func (h *Handler) ReleaseRun(ctx context.Context, q *executionv1.ReleaseRunRequest) (*emptypb.Empty, error) { + c, owner, req, err := h.runClaimRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetBindingId(), q.GetRunId(), q.GetClaimId(), q.GetEpoch(), q.GetGrantGeneration(), q.GetOperationId(), executionenv.DefaultRunTTL.Milliseconds()) + if err != nil { + return nil, err + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + if backendErr := lifecycle.ReleaseRun(ctx, req.Environment, c.id, owner, req); backendErr != nil { + return nil, backendError(backendErr) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) runClaimRequest(ctx context.Context, pRef *executionv1.EnvironmentRef, pOwner *executionv1.Owner, binding, runID, claimID string, epoch, generation uint64, operationID string, ttlMillis int64) (authenticatedClient, string, executionenv.RunClaimRequest, error) { + c, err := h.client(ctx) + if err != nil { + return c, "", executionenv.RunClaimRequest{}, err + } + owner, ok := ownerFromProto(pOwner) + ref := refFromProto(pRef) + if !c.policy.MayAttestOwner || !ok || !validRef(ref) || !validBinding(binding) || !validIdentity(runID) || !validIdentity(claimID) || epoch == 0 || epoch > math.MaxInt64 || generation == 0 || generation > math.MaxInt64 || !validOperationID(operationID) || ttlMillis < executionenv.MinRunTTL.Milliseconds() || ttlMillis > executionenv.MaxRunTTL.Milliseconds() { + return c, "", executionenv.RunClaimRequest{}, wireError(executionenv.CodeInvalidArgument, false) + } + return c, ownerHash(owner), executionenv.RunClaimRequest{Environment: ref, Owner: owner, BindingID: binding, RunID: runID, ClaimID: claimID, Epoch: epoch, GrantGeneration: generation, OperationID: operationID, TTL: time.Duration(ttlMillis) * time.Millisecond}, nil +} + +func (h *Handler) runClaimResponse(ctx context.Context, claim executionenv.RunClaim, client, owner string) (*executionv1.RunClaimResponse, error) { + grant, expiry, err := h.signClaim(ctx, claim, client, owner) + if err != nil { + if errors.Is(err, errAuthorityUnavailable) { + return nil, securityAuthorizationError(err) + } + return nil, wireError(executionenv.CodeInternal, false) + } + return &executionv1.RunClaimResponse{Environment: refToProto(claim.Environment), BindingId: valid(claim.BindingID), RunId: valid(claim.RunID), ClaimId: valid(claim.ClaimID), Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: valid(grant), ExpiresAt: timestamppb.New(expiry)}, nil +} + +func (h *Handler) referenceRequest(ctx context.Context, q *executionv1.ReferenceMutationRequest) (authenticatedClient, string, executionenv.EnvironmentRef, string, string, error) { + c, err := h.client(ctx) + if err != nil { + return c, "", executionenv.EnvironmentRef{}, "", "", err + } + owner, ok := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if q == nil || !c.policy.MayAttestOwner || !ok || !validRef(ref) || !validBinding(q.BindingId) || !validOperationID(q.OperationId) { + return c, "", ref, "", "", wireError(executionenv.CodeInvalidArgument, false) + } + return c, ownerHash(owner), ref, q.BindingId, q.OperationId, nil +} +func (h *Handler) referenceMutation(ctx context.Context, q *executionv1.ReferenceMutationRequest, operation executionenv.Operation) (*emptypb.Empty, error) { + c, owner, ref, binding, operationID, err := h.referenceRequest(ctx, q) + if err != nil { + return nil, err + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + switch operation { + case executionenv.OpReferenceCommit: + err = lifecycle.CommitReference(ctx, ref, c.id, owner, binding, operationID) + case executionenv.OpReferenceAbort: + err = lifecycle.AbortReference(ctx, ref, c.id, owner, binding, operationID) + case executionenv.OpReferenceDeletePrepare: + err = lifecycle.PrepareReferenceDelete(ctx, ref, c.id, owner, binding, operationID) + case executionenv.OpReferenceDeleteConfirm: + err = lifecycle.ConfirmReferenceDelete(ctx, ref, c.id, owner, binding, operationID) + case executionenv.OpReferenceDeleteCancel: + err = lifecycle.CancelReferenceDelete(ctx, ref, c.id, owner, binding, operationID) + default: + err = &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "invalid reference operation"} + } + if err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} +func (h *Handler) CommitReference(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceCommit) +} +func (h *Handler) AbortReference(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceAbort) +} +func (h *Handler) PrepareReferenceDelete(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceDeletePrepare) +} +func (h *Handler) ConfirmReferenceDelete(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceDeleteConfirm) +} +func (h *Handler) CancelReferenceDelete(ctx context.Context, q *executionv1.ReferenceMutationRequest) (*emptypb.Empty, error) { + return h.referenceMutation(ctx, q, executionenv.OpReferenceDeleteCancel) +} + +func (h *Handler) ReserveSuccessor(ctx context.Context, q *executionv1.ReserveSuccessorRequest) (*executionv1.ReferenceReservationResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + owner, ok := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if q == nil || !c.policy.MayAttestOwner || !ok || !validRef(ref) || !validBinding(q.SourceBindingId) || !validBinding(q.DestinationBindingId) || q.SourceBindingId == q.DestinationBindingId || !validOperationID(q.OperationId) { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + if err = lifecycle.ReserveSuccessor(ctx, ref, c.id, ownerHash(owner), q.SourceBindingId, q.DestinationBindingId, q.OperationId); err != nil { + return nil, backendError(err) + } + return &executionv1.ReferenceReservationResponse{Environment: refToProto(ref)}, nil +} + +func (h *Handler) ListReferenceIntents(ctx context.Context, q *executionv1.ListReferenceIntentsRequest) (*executionv1.ListReferenceIntentsResponse, error) { + c, err := h.client(ctx) + if err != nil { + return nil, err + } + if q == nil || !c.policy.MayAttestOwner || q.Limit < 0 || q.Limit > maxReferences { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + lifecycle, lifecycleErr := h.lifecycle() + if lifecycleErr != nil { + return nil, lifecycleErr + } + var intents []executionenv.ReferenceIntent + exactRef := refFromProto(q.GetEnvironment()) + exactLookup := q.GetEnvironment() != nil || q.GetBindingId() != "" + if exactLookup { + owner, ok := ownerFromProto(q.GetOwner()) + if !ok || !validRef(exactRef) || !validBinding(q.GetBindingId()) { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + intent, findErr := lifecycle.FindReferenceIntent(ctx, exactRef, c.id, ownerHash(owner), q.GetBindingId()) + if findErr != nil { + return nil, backendError(findErr) + } + intent.Owner = owner + intents = []executionenv.ReferenceIntent{intent} + } else if q.Owner != nil { + owner, ok := ownerFromProto(q.Owner) + if !ok { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + intents, err = lifecycle.ListReferenceIntents(ctx, c.id, ownerHash(owner), int(q.Limit)) + for i := range intents { + intents[i].Owner = owner + } + } else if owned, ok := h.backend.(ownerIntentBackend); ok { + intents, err = owned.ListReferenceIntentsForClient(ctx, c.id, int(q.Limit)) + } else { + return nil, wireError(executionenv.CodeNotReady, true) + } + if err != nil { + return nil, backendError(err) + } + out := &executionv1.ListReferenceIntentsResponse{} + for _, intent := range intents { + out.Intents = append(out.Intents, &executionv1.ReferenceIntent{Environment: refToProto(intent.Environment), BindingId: valid(intent.BindingID), State: valid(string(intent.State)), OperationId: valid(intent.OperationID), SourceBindingId: valid(intent.SourceBindingID), CreatedAt: timestamppb.New(intent.CreatedAt), Owner: &executionv1.Owner{Issuer: valid(intent.Owner.Issuer), Subject: valid(intent.Owner.Subject)}}) + } + return out, nil +} + +// ReleaseReference releases one authorized durable binding reference. +func (h *Handler) ReleaseReference(ctx context.Context, q *executionv1.ReleaseReferenceRequest) (*emptypb.Empty, error) { + c, owner, rc, err := h.authorize(ctx, q.GetContext(), executionenv.OpReferenceRelease) + if err != nil { + return nil, err + } + if err := h.backend.ReleaseReference(ctx, rc.Environment, c.id, owner, rc.BindingID); err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +// RetireEnvironment starts exact, proof-producing executor retirement while retaining the PVC. +func (h *Handler) RetireEnvironment(ctx context.Context, q *executionv1.RetireEnvironmentRequest) (*emptypb.Empty, error) { + req, err := h.adminRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetExpectedExecutionEpoch(), q.GetExpectedPodUid(), q.GetExpectedPvcUid(), q.GetOperationId()) + if err != nil { + return nil, err + } + if err := h.adminLifecycleBackend.RetireExact(ctx, req); err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) ReplaceExecutor(ctx context.Context, q *executionv1.ReplaceExecutorRequest) (*emptypb.Empty, error) { + req, err := h.adminRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetExpectedExecutionEpoch(), q.GetExpectedPodUid(), q.GetExpectedPvcUid(), q.GetOperationId()) + if err != nil { + return nil, err + } + if err := h.adminLifecycleBackend.ReplaceExecutor(ctx, req); err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) RecoverEnvironment(ctx context.Context, q *executionv1.RecoverEnvironmentRequest) (*emptypb.Empty, error) { + req, err := h.adminRequest(ctx, q.GetEnvironment(), q.GetOwner(), q.GetExpectedExecutionEpoch(), q.GetExpectedPodUid(), q.GetExpectedPvcUid(), q.GetOperationId()) + if err != nil { + return nil, err + } + if err := h.adminLifecycleBackend.RecoverEnvironment(ctx, req); err != nil { + var controlled *executionenv.Error + if errors.As(err, &controlled) && controlled.Code == executionenv.CodeFenceUnknown { + return nil, wireError(executionenv.CodeFenceUnknown, false) + } + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) DeleteRetiredEnvironment(ctx context.Context, q *executionv1.DeleteRetiredEnvironmentRequest) (*emptypb.Empty, error) { + c, err := h.client(ctx) + owner, ownerOK := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if err != nil { + return nil, err + } + if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) || !validIdentity(q.GetExpectedPvcUid()) || !validOperationID(q.GetOperationId()) { + return nil, wireError(executionenv.CodePermissionDenied, false) + } + req := adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, AdministratorFor: c.policy.AdministratorFor, ExpectedPVCUID: q.GetExpectedPvcUid(), OperationID: q.GetOperationId()} + if err := h.adminLifecycleBackend.DeleteRetiredEnvironment(ctx, req); err != nil { + if h.cfg.Diagnostics != nil { + h.cfg.Diagnostics.Log(ctx, port.LevelWarn, "execution backend failure", "operation", "delete_retired", "reason", backendReasonClass(err)) + } + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) MigrateEnvironment(ctx context.Context, q *executionv1.MigrateEnvironmentRequest) (*emptypb.Empty, error) { + req, err := h.adminRequest(ctx, q.GetEnvironment(), q.GetOwner(), 1, q.GetExpectedPodUid(), q.GetExpectedPvcUid(), q.GetOperationId()) + if err != nil { + return nil, err + } + if q == nil || q.ExpectedSchemaVersion == nil || q.GetExpectedSchemaVersion() > 1 { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + req.ExpectedEpoch = 0 + req.ExpectedSchema = int64(q.GetExpectedSchemaVersion()) + if err := h.adminLifecycleBackend.MigrateEnvironment(ctx, req); err != nil { + return nil, backendError(err) + } + return &emptypb.Empty{}, nil +} + +func (h *Handler) RevokeEnvironment(ctx context.Context, q *executionv1.RevokeEnvironmentRequest) (*executionv1.RevokeEnvironmentResponse, error) { + c, err := h.client(ctx) + owner, ownerOK := ownerFromProto(q.GetOwner()) + ref := refFromProto(q.GetEnvironment()) + if err != nil { + return nil, err + } + if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) { + return nil, wireError(executionenv.CodePermissionDenied, false) + } + if q.GetExpectedGrantGeneration() == 0 || q.GetExpectedGrantGeneration() > math.MaxInt64 || !validOperationID(q.GetOperationId()) { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + req := adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, AdministratorFor: c.policy.AdministratorFor, OperationID: q.GetOperationId()} + generation, err := h.adminLifecycleBackend.RevokeEnvironment(ctx, req, q.GetExpectedGrantGeneration()) + if err != nil { + return nil, backendError(err) + } + return &executionv1.RevokeEnvironmentResponse{GrantGeneration: generation}, nil +} + +func (h *Handler) adminRequest(ctx context.Context, pRef *executionv1.EnvironmentRef, pOwner *executionv1.Owner, epoch uint64, podUID, pvcUID, operationID string) (adminLifecycleRequest, error) { + c, err := h.client(ctx) + if err != nil { + return adminLifecycleRequest{}, err + } + owner, ownerOK := ownerFromProto(pOwner) + ref := refFromProto(pRef) + if h.adminLifecycleBackend == nil || !c.policy.Administrator || !ownerOK || !validRef(ref) || epoch == 0 || epoch > math.MaxInt64 || !validIdentity(podUID) || !validIdentity(pvcUID) || !validOperationID(operationID) { + return adminLifecycleRequest{}, wireError(executionenv.CodePermissionDenied, false) + } + return adminLifecycleRequest{Environment: ref, OwnerHash: ownerHash(owner), Client: c.id, AdministratorFor: c.policy.AdministratorFor, ExpectedEpoch: epoch, ExpectedPodUID: podUID, ExpectedPVCUID: pvcUID, OperationID: operationID}, nil +} + +// Files executes one bounded and authorized filesystem operation. +func (h *Handler) Files(ctx context.Context, q *executionv1.FileRequest) (*executionv1.FileResponse, error) { + if _, err := h.client(ctx); err != nil { + return nil, err + } + op, ok := operationFromProto(q.GetOperation()) + if q == nil || !ok || !validFileRequest(q) { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + c, owner, rc, err := h.authorize(ctx, q.Context, op) + if err != nil { + return nil, err + } + req := executionenv.FileRequest{Context: rc, Operation: op, Path: q.Path, Destination: q.Destination, Pattern: q.Pattern, Data: q.Data, Version: string(q.Version), Limit: int(q.Limit)} + out, err := h.backend.File(ctx, c.id, owner, req) + if err != nil { + return nil, backendError(err) + } + return fileResponse(out), nil +} + +// StartCommand executes one authorized foreground command. +func (h *Handler) StartCommand(ctx context.Context, q *executionv1.CommandStartRequest) (*executionv1.CommandStartResponse, error) { + if _, err := h.client(ctx); err != nil { + return nil, err + } + if q == nil || len(q.Command) == 0 || len(q.Command) > executionenv.MaxCommandBytes || q.TimeoutMillis < 0 { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + c, owner, rc, err := h.authorize(ctx, q.Context, executionenv.OpCommandStart) + if err != nil { + return nil, err + } + out, err := h.backend.StartCommand(ctx, c.id, owner, executionenv.CommandStartRequest{Context: rc, Command: q.Command, TimeoutMillis: q.TimeoutMillis}) + if err != nil { + return nil, backendError(err) + } + response, ok := commandStartResponse(out) + if !ok { + return nil, wireError(executionenv.CodeInternal, false) + } + return response, nil +} + +// CommandStatus authorizes then reports the detached API as unsupported. +func (h *Handler) CommandStatus(ctx context.Context, q *executionv1.CommandQueryRequest) (*executionv1.CommandStatusResponse, error) { + return h.commandQuery(ctx, q, executionenv.OpCommandStatus) +} + +// CancelCommand authorizes then reports the detached API as unsupported. +func (h *Handler) CancelCommand(ctx context.Context, q *executionv1.CommandQueryRequest) (*executionv1.CommandStatusResponse, error) { + return h.commandQuery(ctx, q, executionenv.OpCommandCancel) +} +func (h *Handler) commandQuery(ctx context.Context, q *executionv1.CommandQueryRequest, op executionenv.Operation) (*executionv1.CommandStatusResponse, error) { + _, _, _, err := h.authorize(ctx, q.GetContext(), op) + if err != nil { + return nil, err + } + if q.CommandId == "" || q.Offset < 0 { + return nil, wireError(executionenv.CodeInvalidArgument, false) + } + return nil, status.Error(codes.Unimplemented, "foreground commands have no detached control API") +} + +func (h *Handler) authorize(ctx context.Context, p *executionv1.RequestContext, op executionenv.Operation) (authenticatedClient, string, executionenv.RequestContext, error) { + c, err := h.client(ctx) + if err != nil { + return c, "", executionenv.RequestContext{}, err + } + rc, ok := requestContextFromProto(p) + if !ok { + return c, "", rc, wireError(executionenv.CodePermissionDenied, false) + } + oh := ownerHash(rc.Owner) + verifier := h.cfg.Verifier + if h.cfg.Security != nil { + now := h.cfg.Security.now() + material, materialErr := h.cfg.Security.authoritativeAt(ctx, now) + if materialErr != nil { + return c, "", rc, wireError(executionenv.CodeNotReady, true) + } + verifier = material.verifierAt(now) + } + if _, err := verifier.Verify(rc.Grant, executionenv.GrantExpectation{Client: c.id, OwnerHash: oh, BindingID: rc.BindingID, RunID: rc.RunID, ClaimID: rc.ClaimID, Environment: rc.Environment, Epoch: rc.Epoch, GrantGeneration: rc.GrantGeneration, Operation: op}); err != nil { + if errors.Is(err, executionenv.ErrGrantExpired) { + return c, "", rc, wireError(executionenv.CodeUnauthenticated, true) + } + return c, "", rc, wireError(executionenv.CodePermissionDenied, false) + } + if validator, ok := h.backend.(runClaimValidator); ok { + if err := validator.ValidateRunClaim(ctx, c.id, oh, rc); err != nil { + return c, "", rc, backendError(err) + } + } + return c, oh, rc, nil +} + +func requestContextFromProto(p *executionv1.RequestContext) (executionenv.RequestContext, bool) { + owner, ownerOK := ownerFromProto(p.GetOwner()) + ref := refFromProto(p.GetEnvironment()) + rc := executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: p.GetBindingId(), RunID: p.GetRunId(), ClaimID: p.GetClaimId(), Epoch: p.GetEpoch(), GrantGeneration: p.GetGrantGeneration(), Grant: p.GetGrant()} + ok := p != nil && ownerOK && ref.ID != "" && ref.Revision != "" && len(ref.ID) <= executionenv.MaxIdentityBytes && len(ref.Revision) <= executionenv.MaxIdentityBytes && rc.BindingID != "" && len(rc.BindingID) <= executionenv.MaxBindingBytes && validIdentity(rc.RunID) && validIdentity(rc.ClaimID) && rc.Epoch != 0 && rc.GrantGeneration != 0 && rc.Grant != "" && len(rc.Grant) <= executionenv.MaxGrantBytes + return rc, ok +} +func attachContext(p *executionv1.RequestContext) (executionenv.RequestContext, bool) { + owner, ownerOK := ownerFromProto(p.GetOwner()) + ref := refFromProto(p.GetEnvironment()) + rc := executionenv.RequestContext{Environment: ref, Owner: owner, BindingID: p.GetBindingId()} + return rc, p != nil && ownerOK && ref.ID != "" && ref.Revision != "" && len(ref.ID) <= executionenv.MaxIdentityBytes && len(ref.Revision) <= executionenv.MaxIdentityBytes && rc.BindingID != "" && len(rc.BindingID) <= executionenv.MaxBindingBytes +} +func ownerFromProto(p *executionv1.Owner) (executionenv.Owner, bool) { + o := executionenv.Owner{Issuer: p.GetIssuer(), Subject: p.GetSubject()} + return o, p != nil && o.Issuer != "" && o.Subject != "" && len(o.Issuer) <= executionenv.MaxIdentityBytes && len(o.Subject) <= executionenv.MaxIdentityBytes +} +func refFromProto(p *executionv1.EnvironmentRef) executionenv.EnvironmentRef { + return executionenv.EnvironmentRef{ID: p.GetId(), Revision: p.GetRevision()} +} +func refToProto(r executionenv.EnvironmentRef) *executionv1.EnvironmentRef { + return &executionv1.EnvironmentRef{Id: valid(r.ID), Revision: valid(r.Revision)} +} + +func validFileRequest(q *executionv1.FileRequest) bool { //nolint:gocyclo // Closed operation/field matrix is intentionally explicit. + if len(q.Data) > executionenv.MaxFileBytes || len(q.Path) > executionenv.MaxPathBytes || len(q.Destination) > executionenv.MaxPathBytes || len(q.Pattern) > executionenv.MaxPathBytes || q.Limit < 0 || q.Limit > executionenv.MaxListEntries { + return false + } + path, dest, pattern, data, version, limit := q.Path != "", q.Destination != "", q.Pattern != "", len(q.Data) != 0, len(q.Version) != 0, q.Limit != 0 + switch q.Operation { + case executionv1.FileOperation_FILE_OPERATION_READ, executionv1.FileOperation_FILE_OPERATION_RESOLVE_AUTHORITY, executionv1.FileOperation_FILE_OPERATION_STAT, executionv1.FileOperation_FILE_OPERATION_REMOVE: + return path && !dest && !pattern && !data && !version && !limit + case executionv1.FileOperation_FILE_OPERATION_CREATE: + return path && !dest && !pattern && !version && !limit + case executionv1.FileOperation_FILE_OPERATION_REPLACE: + return path && !dest && !pattern && version && !limit + case executionv1.FileOperation_FILE_OPERATION_LIST: + return path && !dest && !pattern && !data && !version + case executionv1.FileOperation_FILE_OPERATION_RENAME, executionv1.FileOperation_FILE_OPERATION_COPY: + return path && dest && !pattern && !data && !version && !limit + case executionv1.FileOperation_FILE_OPERATION_GLOB: + return !path && !dest && pattern && !data && !version + case executionv1.FileOperation_FILE_OPERATION_GREP: + return path && !dest && pattern && !data && !version + default: + return false + } +} + +func operationFromProto(op executionv1.FileOperation) (executionenv.Operation, bool) { + m := map[executionv1.FileOperation]executionenv.Operation{executionv1.FileOperation_FILE_OPERATION_READ: executionenv.OpFileRead, executionv1.FileOperation_FILE_OPERATION_RESOLVE_AUTHORITY: executionenv.OpFileResolveAuthority, executionv1.FileOperation_FILE_OPERATION_STAT: executionenv.OpFileStat, executionv1.FileOperation_FILE_OPERATION_CREATE: executionenv.OpFileCreate, executionv1.FileOperation_FILE_OPERATION_REPLACE: executionenv.OpFileReplace, executionv1.FileOperation_FILE_OPERATION_LIST: executionenv.OpFileList, executionv1.FileOperation_FILE_OPERATION_REMOVE: executionenv.OpFileRemove, executionv1.FileOperation_FILE_OPERATION_RENAME: executionenv.OpFileRename, executionv1.FileOperation_FILE_OPERATION_COPY: executionenv.OpFileCopy, executionv1.FileOperation_FILE_OPERATION_GLOB: executionenv.OpFileGlob, executionv1.FileOperation_FILE_OPERATION_GREP: executionenv.OpFileGrep} + v, ok := m[op] + return v, ok +} + +func ensureResponse(a Allocation, grant string, expiry time.Time) *executionv1.EnsureEnvironmentResponse { + return &executionv1.EnsureEnvironmentResponse{Environment: refToProto(a.Environment), Epoch: a.Epoch, Ready: a.Ready, GrantGeneration: a.GrantGeneration, Grant: valid(grant), GrantExpiresAt: timestamppb.New(expiry)} +} +func attachResponse(a Allocation, grant string, expiry time.Time) *executionv1.AttachEnvironmentResponse { + return &executionv1.AttachEnvironmentResponse{Environment: refToProto(a.Environment), Epoch: a.Epoch, Ready: a.Ready, GrantGeneration: a.GrantGeneration, Grant: valid(grant), GrantExpiresAt: timestamppb.New(expiry)} +} +func fileResponse(v executionenv.FileResponse) *executionv1.FileResponse { + r := &executionv1.FileResponse{Data: v.Data, Version: []byte(v.Version), Paths: validAll(v.Paths), AuthorityTarget: valid(v.AuthorityTarget), AuthorityWorkspace: valid(v.AuthorityWorkspace)} + if v.Info != nil { + r.Info = fileInfoToProto(*v.Info) + } + for _, x := range v.Entries { + r.Entries = append(r.Entries, fileInfoToProto(x)) + } + for _, x := range v.Matches { + r.Matches = append(r.Matches, &executionv1.GrepMatch{Path: valid(x.Path), Line: boundedInt32(x.Line), Text: valid(x.Text)}) + } + return r +} +func fileInfoToProto(v executionenv.FileInfo) *executionv1.FileInfo { + return &executionv1.FileInfo{Name: valid(v.Name), Size: v.Size, Mode: v.Mode, ModTime: timestamppb.New(v.ModTime), IsDir: v.IsDir} +} +func commandStartResponse(v executionenv.CommandStartResponse) (*executionv1.CommandStartResponse, bool) { + state, ok := commandStateToProto(v.State) + if !ok { + return nil, false + } + result, ok := commandStatusResponse(v.Result) + if !ok { + return nil, false + } + return &executionv1.CommandStartResponse{CommandId: valid(v.CommandID), State: state, Result: result}, true +} +func commandStatusResponse(v executionenv.CommandStatusResponse) (*executionv1.CommandStatusResponse, bool) { + state, ok := commandStateToProto(v.State) + if !ok { + return nil, false + } + return &executionv1.CommandStatusResponse{CommandId: valid(v.CommandID), State: state, ExitCode: boundedInt32(v.ExitCode), Stdout: v.Stdout, Stderr: v.Stderr, NextOffset: v.NextOffset, Truncated: v.Truncated, TerminalReceipt: valid(v.TerminalReceipt)}, true +} +func boundedInt32(v int) int32 { + if v > math.MaxInt32 { + return math.MaxInt32 + } + if v < math.MinInt32 { + return math.MinInt32 + } + return int32(v) //nolint:gosec // bounds checked above +} + +func commandStateToProto(v executionenv.CommandState) (executionv1.CommandState, bool) { + switch v { + case executionenv.CommandRunning: + return executionv1.CommandState_COMMAND_STATE_RUNNING, true + case executionenv.CommandSucceeded: + return executionv1.CommandState_COMMAND_STATE_SUCCEEDED, true + case executionenv.CommandFailed: + return executionv1.CommandState_COMMAND_STATE_FAILED, true + case executionenv.CommandCancelled: + return executionv1.CommandState_COMMAND_STATE_CANCELLED, true + case executionenv.CommandFenceUnknown: + return executionv1.CommandState_COMMAND_STATE_FENCE_UNKNOWN, true + default: + return executionv1.CommandState_COMMAND_STATE_UNSPECIFIED, false + } +} + +func validIdentity(v string) bool { return v != "" && len(v) <= executionenv.MaxIdentityBytes } +func validBinding(v string) bool { return v != "" && len(v) <= executionenv.MaxBindingBytes } +func validOperationID(v string) bool { return validIdentity(v) } +func validRef(v executionenv.EnvironmentRef) bool { + return validIdentity(v.ID) && validIdentity(v.Revision) +} + +func (h *Handler) signClaim(ctx context.Context, claim executionenv.RunClaim, client, owner string) (string, time.Time, error) { + signer := h.cfg.Signer + now := time.Now().UTC() + var signingDeadline time.Time + if h.cfg.Security != nil { + now = h.cfg.Security.now() + material, err := h.cfg.Security.authoritativeAt(ctx, now) + if err != nil { + return "", time.Time{}, err + } + signer = material.signer + signingDeadline = material.activeWindow.verifyUntil + } + life := signer.Lifetime + if life <= 0 { + life = time.Minute + } + expiry := now.Add(life) + if !signingDeadline.IsZero() && signingDeadline.Before(expiry) { + expiry = signingDeadline + } + if !now.Before(expiry) { + return "", time.Time{}, errors.New("active signing key is expired") + } + nonce := make([]byte, 16) + if _, err := rand.Read(nonce); err != nil { + return "", time.Time{}, err + } + ops := []executionenv.Operation{executionenv.OpFileRead, executionenv.OpFileResolveAuthority, executionenv.OpFileStat, executionenv.OpFileCreate, executionenv.OpFileReplace, executionenv.OpFileList, executionenv.OpFileRemove, executionenv.OpFileRename, executionenv.OpFileCopy, executionenv.OpFileGlob, executionenv.OpFileGrep, executionenv.OpCommandStart, executionenv.OpCommandStatus, executionenv.OpCommandCancel} + grant, err := executionenv.SignGrant(signer.PrivateKey, executionenv.GrantClaims{KeyID: signer.KeyID, Issuer: signer.Issuer, Audience: signer.Audience, Client: client, OwnerHash: owner, BindingID: claim.BindingID, RunID: claim.RunID, ClaimID: claim.ClaimID, Environment: claim.Environment, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Operations: ops, NotBefore: now, ExpiresAt: expiry, Nonce: hex.EncodeToString(nonce)}) + return grant, expiry, err +} +func ownerHash(o executionenv.Owner) string { + s := sha256.Sum256([]byte(o.Issuer + "\x00" + o.Subject)) + return hex.EncodeToString(s[:]) +} +func fingerprint(fields ...string) string { + h := sha256.New() + for _, f := range fields { + _, _ = h.Write([]byte{0}) + _, _ = io.WriteString(h, f) + } + return hex.EncodeToString(h.Sum(nil)) +} + +func securityAuthorizationError(err error) error { + if errors.Is(err, errAuthorityUnavailable) { + return wireError(executionenv.CodeNotReady, true) + } + return wireError(executionenv.CodeUnauthenticated, false) +} + +func backendReasonClass(err error) string { + var controlled *executionenv.Error + if errors.As(err, &controlled) && controlled.Code.Valid() { + return string(controlled.Code) + } + switch { + case apierrors.IsInvalid(err): + return "invalid" + case apierrors.IsConflict(err): + return "conflict" + case apierrors.IsTimeout(err), apierrors.IsServerTimeout(err), errors.Is(err, context.DeadlineExceeded): + return "timeout" + case apierrors.IsTooManyRequests(err): + return "rate_limited" + case apierrors.IsForbidden(err): + return "forbidden" + case apierrors.IsUnauthorized(err): + return "unauthorized" + case apierrors.IsNotFound(err): + return "not_found" + case apierrors.IsAlreadyExists(err): + return "already_exists" + case apierrors.IsServiceUnavailable(err): + return "unavailable" + case errors.Is(err, context.Canceled): + return "cancelled" + default: + return "other" + } +} + +func backendError(err error) error { + var e *executionenv.Error + if errors.As(err, &e) && e.Code.Valid() { + return wireError(e.Code, e.Retryable) + } + return wireError(executionenv.CodeInternal, false) +} +func wireError(code executionenv.ErrorCode, retry bool) error { + if !code.Valid() || (retry && code != executionenv.CodeNotReady && code != executionenv.CodeUnauthenticated) { + code, retry = executionenv.CodeInternal, false + } + grpcCode := codes.Internal + switch code { + case executionenv.CodeInvalidArgument: + grpcCode = codes.InvalidArgument + case executionenv.CodeUnauthenticated: + grpcCode = codes.Unauthenticated + case executionenv.CodePermissionDenied: + grpcCode = codes.PermissionDenied + case executionenv.CodeNotFound: + grpcCode = codes.NotFound + case executionenv.CodeAlreadyExists: + grpcCode = codes.AlreadyExists + case executionenv.CodeConflict, executionenv.CodeVersionMismatch, executionenv.CodeDirectoryNotEmpty: + grpcCode = codes.Aborted + case executionenv.CodeNotReady: + grpcCode = codes.Unavailable + case executionenv.CodeFenceUnknown: + grpcCode = codes.FailedPrecondition + case executionenv.CodeResourceExhausted: + grpcCode = codes.ResourceExhausted + } + message := "execution provider request failed" + if code == executionenv.CodeFenceUnknown { + message = "execution provider request failed; verify termination independently or use external fencing" + } + st := status.New(grpcCode, message) + with, err := st.WithDetails(&executionv1.ErrorDetail{Code: string(code), Retryable: retry}) + if err == nil { + st = with + } + return st.Err() +} +func valid(s string) string { + if utf8.ValidString(s) { + return s + } + return strings.ToValidUTF8(s, "�") +} +func validAll(in []string) []string { + out := make([]string, len(in)) + for i := range in { + out[i] = valid(in[i]) + } + return out +} + +// TLSConfig returns the provider's TLS 1.3 mutual-authentication policy. +func TLSConfig(server tls.Certificate, clientCAs *x509.CertPool) *tls.Config { + return &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{server}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: clientCAs} +} diff --git a/internal/adapter/executioncontroller/handler_test.go b/internal/adapter/executioncontroller/handler_test.go new file mode 100644 index 0000000000..4010bbd2e6 --- /dev/null +++ b/internal/adapter/executioncontroller/handler_test.go @@ -0,0 +1,451 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "errors" + "fmt" + "net/url" + "strings" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type fakeBackend struct { + ensureCalls, created int + allocations map[string]Allocation + commandState executionenv.CommandState + intents []executionenv.ReferenceIntent +} + +func newFakeBackend() *fakeBackend { return &fakeBackend{allocations: map[string]Allocation{}} } +func (*fakeBackend) ValidateProfile(context.Context, string) (Profile, error) { + return Profile{Name: "go", Digest: "sha256:test", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute}, nil +} +func (f *fakeBackend) Ensure(_ context.Context, client, owner, binding, _, fp string) (Allocation, error) { + f.ensureCalls++ + if a, ok := f.allocations[fp]; ok { + return a, nil + } + f.created++ + a := Allocation{Environment: executionenv.EnvironmentRef{ID: "env-1", Revision: "rev-1"}, Epoch: 1, OwnerHash: owner, BindingID: binding, Client: client, Ready: true} + f.allocations[fp] = a + return a, nil +} +func (*fakeBackend) Attach(context.Context, executionenv.EnvironmentRef, string, string, string) (Allocation, error) { + panic("unused") +} +func (*fakeBackend) ReleaseReference(context.Context, executionenv.EnvironmentRef, string, string, string) error { + return nil +} +func (*fakeBackend) Retire(context.Context, executionenv.EnvironmentRef, string) error { return nil } +func (*fakeBackend) File(context.Context, string, string, executionenv.FileRequest) (executionenv.FileResponse, error) { + return executionenv.FileResponse{}, nil +} +func (f *fakeBackend) StartCommand(context.Context, string, string, executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + state := f.commandState + if state == "" { + state = executionenv.CommandSucceeded + } + return executionenv.CommandStartResponse{CommandID: "command", State: state, Result: executionenv.CommandStatusResponse{CommandID: "command", State: state}}, nil +} +func (*fakeBackend) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, nil +} +func (*fakeBackend) CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, nil +} + +func (f *fakeBackend) EnsurePending(ctx context.Context, client, owner, binding, profile, fp, _ string) (Allocation, error) { + return f.Ensure(ctx, client, owner, binding, profile, fp) +} +func (*fakeBackend) AcquireRun(_ context.Context, ref executionenv.EnvironmentRef, _ string, _ string, binding, run, _ string, ttl time.Duration) (executionenv.RunClaim, error) { + return executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: run, ClaimID: "claim", Epoch: 2, GrantGeneration: 1, ExpiresAt: time.Now().Add(ttl)}, nil +} +func (*fakeBackend) RenewRun(_ context.Context, _ executionenv.EnvironmentRef, _ string, _ string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + return executionenv.RunClaim{Environment: req.Environment, BindingID: req.BindingID, RunID: req.RunID, ClaimID: req.ClaimID, Epoch: req.Epoch, GrantGeneration: 1, ExpiresAt: time.Now().Add(req.TTL)}, nil +} +func (*fakeBackend) ReleaseRun(context.Context, executionenv.EnvironmentRef, string, string, executionenv.RunClaimRequest) error { + return nil +} +func (*fakeBackend) CommitReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) AbortReference(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) ReserveSuccessor(context.Context, executionenv.EnvironmentRef, string, string, string, string, string) error { + return nil +} +func (*fakeBackend) PrepareReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) ConfirmReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (*fakeBackend) CancelReferenceDelete(context.Context, executionenv.EnvironmentRef, string, string, string, string) error { + return nil +} +func (f *fakeBackend) ListReferenceIntents(context.Context, string, string, int) ([]executionenv.ReferenceIntent, error) { + return f.intents, nil +} +func (*fakeBackend) FindReferenceIntent(context.Context, executionenv.EnvironmentRef, string, string, string) (executionenv.ReferenceIntent, error) { + return executionenv.ReferenceIntent{}, nil +} + +type adminFakeBackend struct { + *fakeBackend + migratedSchema int64 + deleteErr error + deletedRequest adminLifecycleRequest +} + +func (*adminFakeBackend) ReplaceExecutor(context.Context, adminLifecycleRequest) error { return nil } +func (*adminFakeBackend) RetireExact(context.Context, adminLifecycleRequest) error { return nil } +func (*adminFakeBackend) RecoverEnvironment(context.Context, adminLifecycleRequest) error { + return nil +} +func (b *adminFakeBackend) DeleteRetiredEnvironment(_ context.Context, req adminLifecycleRequest) error { + b.deletedRequest = req + return b.deleteErr +} +func (b *adminFakeBackend) MigrateEnvironment(_ context.Context, req adminLifecycleRequest) error { + b.migratedSchema = req.ExpectedSchema + return nil +} +func (*adminFakeBackend) RevokeEnvironment(context.Context, adminLifecycleRequest, uint64) (uint64, error) { + return 2, nil +} + +func authenticatedContext(id string) context.Context { + u, _ := url.Parse(id) + cert := &x509.Certificate{URIs: []*url.URL{u}} + return peer.NewContext(context.Background(), &peer.Peer{AuthInfo: credentials.TLSInfo{State: structTLSState(cert)}}) +} +func structTLSState(cert *x509.Certificate) (s tls.ConnectionState) { + s.PeerCertificates = []*x509.Certificate{cert} + s.VerifiedChains = [][]*x509.Certificate{{cert}} + return s +} + +type handlerDiagnosticRecord struct { + level port.Level + msg string + attrs []any +} + +type handlerDiagnosticRecorder struct { + records []handlerDiagnosticRecord +} + +func (d *handlerDiagnosticRecorder) Log(_ context.Context, level port.Level, msg string, attrs ...any) { + d.records = append(d.records, handlerDiagnosticRecord{level: level, msg: msg, attrs: append([]any(nil), attrs...)}) +} + +func (d *handlerDiagnosticRecorder) With(...any) port.Diagnostics { return d } + +func TestDeleteRetiredBackendFailureDiagnosticIsBounded(t *testing.T) { + const ( + id = "spiffe://cluster/ns/admin" + sentinel = "sentinel-private-operation-delete-retired" + ) + backend := &adminFakeBackend{fakeBackend: newFakeBackend(), deleteErr: errors.New(sentinel + " env=env operation=delete-op")} + diagnostics := &handlerDiagnosticRecorder{} + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{id: {Administrator: true}}, Diagnostics: diagnostics}, backend) + _, err := h.DeleteRetiredEnvironment(authenticatedContext(id), &executionv1.DeleteRetiredEnvironmentRequest{ + Environment: &executionv1.EnvironmentRef{Id: "env", Revision: "rev"}, + Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, + ExpectedPvcUid: "pvc-uid", + OperationId: "delete-op", + }) + if status.Code(err) != codes.Internal { + t.Fatalf("code=%v, want %v", status.Code(err), codes.Internal) + } + if backend.deletedRequest.OperationID != "delete-op" || backend.deletedRequest.Environment.ID != "env" { + t.Fatalf("backend request=%+v", backend.deletedRequest) + } + if len(diagnostics.records) != 1 { + t.Fatalf("diagnostic records=%d, want 1: %+v", len(diagnostics.records), diagnostics.records) + } + record := diagnostics.records[0] + if record.level != port.LevelWarn || record.msg != "execution backend failure" { + t.Fatalf("diagnostic=%+v", record) + } + if len(record.attrs) != 4 || record.attrs[0] != "operation" || record.attrs[1] != "delete_retired" || record.attrs[2] != "reason" || record.attrs[3] != "other" { + t.Fatalf("diagnostic attrs=%#v", record.attrs) + } + if strings.Contains(fmt.Sprint(record), sentinel) { + t.Fatalf("diagnostic leaked backend error: %+v", record) + } +} + +func TestListReferenceIntentsProjectsAttestedOwner(t *testing.T) { + backend := newFakeBackend() + backend.intents = []executionenv.ReferenceIntent{{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", State: executionenv.ReferencePendingDelete, OperationID: "delete", CreatedAt: time.Now().UTC()}} + id := "spiffe://cluster/ns/mecak8s" + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{id: {MayAttestOwner: true}}}, backend) + owner := &executionv1.Owner{Issuer: "issuer", Subject: "alice"} + out, err := h.ListReferenceIntents(authenticatedContext(id), &executionv1.ListReferenceIntentsRequest{Owner: owner, Limit: 64}) + if err != nil { + t.Fatal(err) + } + if len(out.GetIntents()) != 1 || out.GetIntents()[0].GetOwner().GetIssuer() != owner.GetIssuer() || out.GetIntents()[0].GetOwner().GetSubject() != owner.GetSubject() { + t.Fatal("owner-scoped reference intent omitted its attested owner") + } +} + +func TestMigrateEnvironmentRequiresExpectedSchemaPresence(t *testing.T) { + backend := &adminFakeBackend{fakeBackend: newFakeBackend()} + id := "spiffe://cluster/ns/admin" + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{id: {Administrator: true}}}, backend) + ctx := authenticatedContext(id) + base := &executionv1.MigrateEnvironmentRequest{Environment: &executionv1.EnvironmentRef{Id: "env", Revision: "rev"}, Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, ExpectedPodUid: "pod", ExpectedPvcUid: "pvc", OperationId: "migrate"} + if _, err := h.MigrateEnvironment(ctx, base); status.Code(err) != codes.InvalidArgument { + t.Fatalf("omitted schema code=%v", status.Code(err)) + } + zero := uint32(0) + base.ExpectedSchemaVersion = &zero + if _, err := h.MigrateEnvironment(ctx, base); err != nil || backend.migratedSchema != 0 { + t.Fatalf("explicit zero rejected: schema=%d err=%v", backend.migratedSchema, err) + } + unknown := uint32(2) + base.ExpectedSchemaVersion = &unknown + if _, err := h.MigrateEnvironment(ctx, base); status.Code(err) != codes.InvalidArgument { + t.Fatalf("unknown schema code=%v", status.Code(err)) + } +} + +func TestHandlerBlocksAllRequestsUntilStartupReady(t *testing.T) { + backend := newFakeBackend() + h := NewHandler(HandlerConfig{Ready: func() bool { return false }}, backend) + _, err := h.EnsureEnvironment(authenticatedContext("spiffe://cluster/ns/mecak8s"), &executionv1.EnsureEnvironmentRequest{}) + if status.Code(err) != codes.Unavailable { + t.Fatalf("code=%v", status.Code(err)) + } + if backend.ensureCalls != 0 { + t.Fatalf("backend calls before readiness=%d", backend.ensureCalls) + } +} + +func TestHandlerRequiresAllowlistedCanonicalURISAN(t *testing.T) { + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {}}}, newFakeBackend()) + _, err := h.ValidateProfile(authenticatedContext("spiffe://cluster/ns/other"), &executionv1.ValidateProfileRequest{Profile: "go"}) + if status.Code(err) != codes.Unauthenticated { + t.Fatalf("code=%v", status.Code(err)) + } +} + +func TestHandlerValidateIsReadOnlyAndEnsureIdempotent(t *testing.T) { + pub, priv, _ := ed25519.GenerateKey(rand.Reader) + backend := newFakeBackend() + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {MayAttestOwner: true}}, Signer: GrantSigner{KeyID: "k1", PrivateKey: priv, Issuer: "provider", Audience: "execution", Lifetime: time.Minute}, Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: "provider", Audience: "execution"}}, backend) + ctx := authenticatedContext("spiffe://cluster/ns/mecak8s") + if _, err := h.ValidateProfile(ctx, &executionv1.ValidateProfileRequest{Profile: "go"}); err != nil { + t.Fatal(err) + } + if backend.ensureCalls != 0 { + t.Fatal("validation allocated") + } + q := &executionv1.EnsureEnvironmentRequest{BindingId: "session-1", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, OperationId: "create"} + first, err := h.EnsureEnvironment(ctx, q) + if err != nil { + t.Fatal(err) + } + second, err := h.EnsureEnvironment(ctx, q) + if err != nil { + t.Fatal(err) + } + if backend.created != 1 || first.Environment.Id != second.Environment.Id || first.Epoch != second.Epoch { + t.Fatalf("created=%d", backend.created) + } +} + +func TestHandlerWithNilBackendRejectsEveryRPCWithoutPanicking(t *testing.T) { + h := NewHandler(HandlerConfig{}, nil) + ctx := context.Background() + calls := []func(bool) error{ + func(valid bool) error { + var q *executionv1.ValidateProfileRequest + if valid { + q = &executionv1.ValidateProfileRequest{Profile: "go"} + } + _, err := h.ValidateProfile(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.EnsureEnvironmentRequest + if valid { + q = &executionv1.EnsureEnvironmentRequest{BindingId: "binding", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, OperationId: "create"} + } + _, err := h.EnsureEnvironment(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.AttachEnvironmentRequest + if valid { + q = &executionv1.AttachEnvironmentRequest{Purpose: executionenv.PurposeSession} + } + _, err := h.AttachEnvironment(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.ReleaseReferenceRequest + if valid { + q = &executionv1.ReleaseReferenceRequest{Context: &executionv1.RequestContext{}} + } + _, err := h.ReleaseReference(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.RetireEnvironmentRequest + if valid { + q = &executionv1.RetireEnvironmentRequest{Environment: &executionv1.EnvironmentRef{}, Owner: &executionv1.Owner{}} + } + _, err := h.RetireEnvironment(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.FileRequest + if valid { + q = &executionv1.FileRequest{Operation: executionv1.FileOperation_FILE_OPERATION_READ, Path: "x"} + } + _, err := h.Files(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.CommandStartRequest + if valid { + q = &executionv1.CommandStartRequest{Command: "true"} + } + _, err := h.StartCommand(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.CommandQueryRequest + if valid { + q = &executionv1.CommandQueryRequest{CommandId: "command"} + } + _, err := h.CommandStatus(ctx, q) + return err + }, + func(valid bool) error { + var q *executionv1.CommandQueryRequest + if valid { + q = &executionv1.CommandQueryRequest{CommandId: "command"} + } + _, err := h.CancelCommand(ctx, q) + return err + }, + } + for i, call := range calls { + for _, valid := range []bool{false, true} { + if err := call(valid); status.Code(err) != codes.Unavailable { + t.Fatalf("RPC %d valid=%t code=%v", i, valid, status.Code(err)) + } + } + } +} + +func TestHandlerRejectsUnknownBackendCommandState(t *testing.T) { + pub, priv, _ := ed25519.GenerateKey(rand.Reader) + backend := newFakeBackend() + backend.commandState = executionenv.CommandState("future") + h := NewHandler(HandlerConfig{Clients: map[string]ClientPolicy{"spiffe://cluster/ns/mecak8s": {MayAttestOwner: true}}, Signer: GrantSigner{KeyID: "k1", PrivateKey: priv, Issuer: "provider", Audience: "execution", Lifetime: time.Minute}, Verifier: executionenv.GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: "provider", Audience: "execution"}}, backend) + ctx := authenticatedContext("spiffe://cluster/ns/mecak8s") + ensured, err := h.EnsureEnvironment(ctx, &executionv1.EnsureEnvironmentRequest{BindingId: "session-1", Profile: "go", Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, OperationId: "create"}) + if err != nil { + t.Fatal(err) + } + claim, err := h.AcquireRun(ctx, &executionv1.AcquireRunRequest{Environment: ensured.Environment, Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, BindingId: "session-1", RunId: "run", OperationId: "acquire", TtlMillis: time.Minute.Milliseconds()}) + if err != nil { + t.Fatal(err) + } + q := &executionv1.CommandStartRequest{Context: &executionv1.RequestContext{Environment: claim.Environment, Owner: &executionv1.Owner{Issuer: "issuer", Subject: "alice"}, BindingId: "session-1", RunId: claim.RunId, ClaimId: claim.ClaimId, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant}, Command: "true"} + if _, err := h.StartCommand(ctx, q); status.Code(err) != codes.Internal || status.Convert(err).Message() != "execution provider request failed" { + t.Fatalf("unexpected invalid-state result: code=%v message=%q", status.Code(err), status.Convert(err).Message()) + } +} + +func TestCanonicalClientIdentityRejectsCNAndAmbiguousURIs(t *testing.T) { + if _, err := canonicalClientIdentity(&x509.Certificate{Subject: pkix.Name{CommonName: "trusted"}}); err == nil { + t.Fatal("CN accepted") + } + u1, _ := url.Parse("spiffe://b/x") + u2, _ := url.Parse("spiffe://a/x") + if _, err := canonicalClientIdentity(&x509.Certificate{URIs: []*url.URL{u1, u2}}); err == nil { + t.Fatal("ambiguous URI SANs accepted") + } +} + +func TestRunClaimResponseKeepsNonAuthoritySigningFailureInternal(t *testing.T) { + h := NewHandler(HandlerConfig{Signer: GrantSigner{KeyID: "k1", Issuer: "issuer", Audience: "audience", Lifetime: time.Minute}}, newFakeBackend()) + claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1} + _, err := h.runClaimResponse(t.Context(), claim, "spiffe://example/client", "owner") + st := status.Convert(err) + if st.Code() != codes.Internal { + t.Fatalf("code=%v", st.Code()) + } + details := st.Details() + if len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeInternal) || details[0].(*executionv1.ErrorDetail).Retryable { + t.Fatalf("details=%v", details) + } +} + +func TestBackendReasonClassIsBounded(t *testing.T) { + invalid := apierrors.NewInvalid(schema.GroupKind{Group: "example.test", Kind: "Fixture"}, "private-resource-name", field.ErrorList{field.Invalid(field.NewPath("status"), "private-value", "private-detail")}) + cases := []struct { + name string + err error + want string + }{ + {name: "Kubernetes invalid", err: invalid, want: "invalid"}, + {name: "Kubernetes conflict", err: apierrors.NewConflict(schema.GroupResource{Group: "example.test", Resource: "fixtures"}, "private-resource-name", context.Canceled), want: "conflict"}, + {name: "controlled", err: &executionenv.Error{Code: executionenv.CodeNotReady, Message: "private-detail"}, want: "not_ready"}, + {name: "unknown", err: errors.New("private-detail"), want: "other"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := backendReasonClass(tc.err); got != tc.want { + t.Fatalf("reason=%q, want %q", got, tc.want) + } + }) + } +} + +func TestWireErrorDoesNotExposeBackendMessage(t *testing.T) { + err := backendError(&executionenv.Error{Code: executionenv.CodeInternal, Message: "kubectl exec --token=secret"}) + if got := status.Convert(err).Message(); got != "execution provider request failed" { + t.Fatalf("message=%q", got) + } + unknown := backendError(&executionenv.Error{Code: executionenv.ErrorCode("future"), Message: "sensitive path"}) + st := status.Convert(unknown) + if st.Code() != codes.Internal || st.Message() != "execution provider request failed" { + t.Fatalf("unknown error escaped: code=%v message=%q", st.Code(), st.Message()) + } + if details := st.Details(); len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeInternal) { + t.Fatalf("unknown error detail=%v", details) + } + fenced := status.Convert(backendError(&executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "/var/run/provider/private-key"})) + if fenced.Code() != codes.FailedPrecondition || strings.Contains(fenced.Message(), "/var/") { + t.Fatalf("fence error leaked or mapped incorrectly: code=%v message=%q", fenced.Code(), fenced.Message()) + } + if details := fenced.Details(); len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeFenceUnknown) { + t.Fatalf("fence error detail=%v", details) + } +} diff --git a/internal/adapter/executioncontroller/legacy_fixture_kind.go b/internal/adapter/executioncontroller/legacy_fixture_kind.go new file mode 100644 index 0000000000..5266a19c2c --- /dev/null +++ b/internal/adapter/executioncontroller/legacy_fixture_kind.go @@ -0,0 +1,202 @@ +//go:build kind_execution_e2e + +package executioncontroller + +import ( + "context" + "errors" + "fmt" + "slices" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// LegacyMigrationSeed identifies one pre-upgrade API-server fixture. +type LegacyMigrationSeed struct { + Environment executionenv.EnvironmentRef `json:"environment"` + Owner executionenv.Owner `json:"owner"` + Binding string `json:"binding"` + PodUID string `json:"pod_uid"` + PVCUID string `json:"pvc_uid"` + Malformed executionenv.EnvironmentRef `json:"malformed_environment"` + MalformedPodUID string `json:"malformed_pod_uid"` + MalformedPVCUID string `json:"malformed_pvc_uid"` + Insecure executionenv.EnvironmentRef `json:"insecure_environment"` + InsecurePodUID string `json:"insecure_pod_uid"` + InsecurePVCUID string `json:"insecure_pvc_uid"` +} + +// SeedLegacyMigrationFixture creates the narrow security-compatible prototype +// accepted by MigrateEnvironment plus malformed and insecure negative controls. +// It is excluded from regular builds. +func SeedLegacyMigrationFixture(ctx context.Context, d dynamic.Interface, kube kubernetes.Interface, namespace, profilesPath string) (LegacyMigrationSeed, error) { + profiles, err := LoadProfiles(profilesPath) + if err != nil { + return LegacyMigrationSeed{}, err + } + profile, ok := profiles.get("go") + if !ok { + return LegacyMigrationSeed{}, fmt.Errorf("go profile unavailable") + } + owner := executionenv.Owner{Issuer: "https://oidc-issuer.execution-qualification.svc.cluster.local:8443", Subject: "production-migration"} + client := "spiffe://mecatl.test/client/mecak8s" + recognized, err := seedOneLegacyEnvironment(ctx, d, kube, namespace, profile, owner, client, "legacy-migration", "legacy-migration-binding", []any{"legacy-migration-binding"}, false) + if err != nil { + return LegacyMigrationSeed{}, err + } + malformed, err := seedOneLegacyEnvironment(ctx, d, kube, namespace, profile, owner, client, "legacy-migration-malformed", "legacy-malformed-binding", []any{""}, false) + if err != nil { + return LegacyMigrationSeed{}, err + } + insecure, err := seedOneLegacyEnvironment(ctx, d, kube, namespace, profile, owner, client, "legacy-migration-insecure", "legacy-insecure-binding", []any{"legacy-insecure-binding"}, true) + if err != nil { + return LegacyMigrationSeed{}, err + } + return LegacyMigrationSeed{Environment: recognized.ref, Owner: owner, Binding: "legacy-migration-binding", PodUID: recognized.podUID, PVCUID: recognized.pvcUID, Malformed: malformed.ref, MalformedPodUID: malformed.podUID, MalformedPVCUID: malformed.pvcUID, Insecure: insecure.ref, InsecurePodUID: insecure.podUID, InsecurePVCUID: insecure.pvcUID}, nil +} + +// Kubernetes defaults resource-quota usage resync to five minutes. Discovery of +// a new CRD need not enqueue a quota that has not accounted for that resource. +const legacyQuotaWait = 6 * time.Minute + +type seededLegacyEnvironment struct { + ref executionenv.EnvironmentRef + podUID, pvcUID string +} + +func seedOneLegacyEnvironment(ctx context.Context, d dynamic.Interface, kube kubernetes.Interface, namespace string, profile resolvedProfile, owner executionenv.Owner, client, name, binding string, references []any, insecure bool) (seededLegacyEnvironment, error) { + // Deployment readiness does not imply that quota admission has initialized + // accounting, especially for the freshly installed custom resource. + started := time.Now() + var missing []string + if err := wait.PollUntilContextTimeout(ctx, 100*time.Millisecond, legacyQuotaWait, true, func(ctx context.Context) (bool, error) { + quota, err := kube.CoreV1().ResourceQuotas(namespace).Get(ctx, "mecatl-execution", metav1.GetOptions{}) + if err != nil { + return false, err + } + missing = nil + ready := true + for name, configured := range quota.Spec.Hard { + hard, hasHard := quota.Status.Hard[name] + _, hasUsed := quota.Status.Used[name] + if !hasHard || hard.Cmp(configured) != 0 || !hasUsed { + ready = false + switch name { + case "pods", "persistentvolumeclaims", "count/executionenvironments.execution.mecatl.dev", "requests.cpu", "requests.memory", "requests.storage", "requests.ephemeral-storage", "limits.cpu", "limits.memory", "limits.ephemeral-storage": + missing = append(missing, string(name)) + } + } + } + return ready, nil + }); err != nil { + if errors.Is(err, context.DeadlineExceeded) { + err = context.DeadlineExceeded + } else if errors.Is(err, context.Canceled) { + err = context.Canceled + } else if apierrors.IsForbidden(err) { + err = &apierrors.StatusError{ErrStatus: metav1.Status{Reason: metav1.StatusReasonForbidden, Code: 403, Message: "forbidden"}} + } else { + err = errors.New("api_error") + } + // The fixture CLI prints this diagnostic; never wrap raw API response data. + slices.Sort(missing) + return seededLegacyEnvironment{}, fmt.Errorf("wait for legacy fixture quota accounting: missing_or_mismatched_keys=%v elapsed=%s: %w", missing, time.Since(started).Round(time.Millisecond), err) + } + revision, err := randomID() + if err != nil { + return seededLegacyEnvironment{}, err + } + spec := map[string]any{ + "schemaVersion": int64(1), "allocationID": name, "revision": revision, + "ownerHash": ownerHash(owner), "ownerIssuer": owner.Issuer, "ownerSubject": owner.Subject, + "clientHash": hashText(client), "bindingID": binding, "requestFingerprint": hashText("legacy-fixture-" + name), + "profile": "go", "profileDigest": profile.Digest, "image": profile.Spec.Image, + "storageClass": profile.Spec.StorageClass, "storageSize": profile.Spec.StorageSize, + "resources": map[string]any{"cpuRequest": profile.Spec.CPURequest, "memoryRequest": profile.Spec.MemoryRequest, "cpuLimit": profile.Spec.CPULimit, "memoryLimit": profile.Spec.MemoryLimit}, + "desired": "Active", + } + object := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", + "metadata": map[string]any{"name": name, "namespace": namespace, "finalizers": []any{environmentFinalizer}}, + "spec": spec, + }} + resources := d.Resource(ExecutionEnvironmentGVR).Namespace(namespace) + created, err := resources.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + created, err = resources.Create(ctx, object, metav1.CreateOptions{}) + } else if err == nil { + revision = textNested(created.Object, "spec", "revision") + podUID := textNested(created.Object, "status", "pod", "uid") + pvcUID := textNested(created.Object, "status", "pvc", "uid") + if revision != "" && podUID != "" && pvcUID != "" { + return seededLegacyEnvironment{ref: executionenv.EnvironmentRef{ID: name, Revision: revision}, podUID: podUID, pvcUID: pvcUID}, nil + } + } + if err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("create legacy environment %s: %w", name, err) + } + r := NewReconciler(d, kube, namespace, &Profiles{byName: map[string]resolvedProfile{"go": profile}}) + pvcName, podName := resourceName("workspace", name), resourceName("executor", name) + pvc, err := r.ensurePVC(ctx, created, profile, pvcName) + if err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("create legacy PVC %s: %w", name, err) + } + pod, err := r.ensurePod(ctx, created, profile, podName, pvcName) + if err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("create legacy Pod %s: %w", name, err) + } + if insecure { + pod, err = replaceWithInsecureLegacyPod(ctx, kube, namespace, pod) + if err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("create insecure legacy Pod %s: %w", name, err) + } + } + now := time.Now().UTC().Format(time.RFC3339Nano) + created.Object["status"] = map[string]any{ + "schemaVersion": int64(1), "observedGeneration": created.GetGeneration(), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": fenceHealthy, + "references": references, "pvc": map[string]any{"name": pvc.Name, "uid": string(pvc.UID)}, "pod": map[string]any{"name": pod.Name, "uid": string(pod.UID)}, + "conditions": []any{map[string]any{"type": "Ready", "status": "True", "reason": "LegacyFixtureReady", "message": "security-compatible prototype runtime created before CRD upgrade", "observedGeneration": created.GetGeneration(), "lastTransitionTime": now}}, + } + if _, err := resources.UpdateStatus(ctx, created, metav1.UpdateOptions{}); err != nil { + return seededLegacyEnvironment{}, fmt.Errorf("persist legacy status %s: %w", name, err) + } + return seededLegacyEnvironment{ref: executionenv.EnvironmentRef{ID: name, Revision: revision}, podUID: string(pod.UID), pvcUID: string(pvc.UID)}, nil +} + +func replaceWithInsecureLegacyPod(ctx context.Context, kube kubernetes.Interface, namespace string, pod *corev1.Pod) (*corev1.Pod, error) { + pods := kube.CoreV1().Pods(namespace) + if _, err := pods.Patch(ctx, pod.Name, types.MergePatchType, []byte(`{"metadata":{"finalizers":[]}}`), metav1.PatchOptions{}); err != nil { + return nil, err + } + zero := int64(0) + if err := pods.Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &zero}); err != nil { + return nil, err + } + for deadline := time.Now().Add(30 * time.Second); time.Now().Before(deadline); { + if _, err := pods.Get(ctx, pod.Name, metav1.GetOptions{}); apierrors.IsNotFound(err) { + break + } else if err != nil { + return nil, err + } + time.Sleep(100 * time.Millisecond) + } + if _, err := pods.Get(ctx, pod.Name, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + return nil, fmt.Errorf("secure fixture Pod deletion did not complete: %w", err) + } + candidate := pod.DeepCopy() + candidate.ObjectMeta = metav1.ObjectMeta{Name: pod.Name, Namespace: namespace, Labels: pod.Labels, Finalizers: pod.Finalizers, OwnerReferences: pod.OwnerReferences} + candidate.Status = corev1.PodStatus{} + automount := true + candidate.Spec.AutomountServiceAccountToken = &automount + return pods.Create(ctx, candidate, metav1.CreateOptions{}) +} diff --git a/internal/adapter/executioncontroller/legacy_fixture_kind_test.go b/internal/adapter/executioncontroller/legacy_fixture_kind_test.go new file mode 100644 index 0000000000..fc710b19b4 --- /dev/null +++ b/internal/adapter/executioncontroller/legacy_fixture_kind_test.go @@ -0,0 +1,128 @@ +//go:build kind_execution_e2e + +package executioncontroller + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestLegacyQuotaWaitCoversDefaultControllerResync(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + kube := kubefake.NewClientset() + kube.PrependReactor("get", "resourcequotas", func(ktesting.Action) (bool, runtime.Object, error) { + cancel() + return true, nil, context.Canceled + }) + if legacyQuotaWait < 6*time.Minute { + t.Fatal("fixture must cover the five-minute quota resync plus margin") + } + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + _, err := seedOneLegacyEnvironment(ctx, d, kube, "test", resolvedProfile{}, executionenv.Owner{}, "client", "legacy", "binding", nil, false) + if !errors.Is(err, context.Canceled) || len(d.Actions()) != 0 { + t.Fatal("cancellation must stop before creating fixtures") + } +} + +func TestLegacyFixtureWaitsForQuotaAccountingBeforeCreate(t *testing.T) { + for _, mode := range []string{"delayed", "timeout", "cancel", "forbidden", "api-error"} { + t.Run(mode, func(t *testing.T) { + quota := &corev1.ResourceQuota{ObjectMeta: metav1.ObjectMeta{Name: "mecatl-execution", Namespace: "test"}, Spec: corev1.ResourceQuotaSpec{Hard: corev1.ResourceList{corev1.ResourceName("count/executionenvironments.execution.mecatl.dev"): resource.MustParse("10"), corev1.ResourcePods: resource.MustParse("10"), corev1.ResourceName("private-quota-key"): resource.MustParse("12345")}}} + kube := kubefake.NewClientset() + reads, creates := 0, 0 + ctx, cancel := context.WithTimeout(context.Background(), 350*time.Millisecond) + defer cancel() + const hostile = "https://sentinel.invalid/private?token=sk-fake-quota-secret" + kube.PrependReactor("get", "resourcequotas", func(ktesting.Action) (bool, runtime.Object, error) { + reads++ + if mode == "forbidden" { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Resource: "resourcequotas"}, quota.Name, errors.New(hostile)) + } + if mode == "api-error" { + return true, nil, apierrors.NewInternalError(errors.New(hostile)) + } + // Partially initialized status must not pass: all configured resources matter. + quota.Status.Hard = quota.Spec.Hard.DeepCopy() + quota.Status.Used = corev1.ResourceList{corev1.ResourcePods: resource.MustParse("0")} + if mode == "timeout" { + quota.Status.Hard[corev1.ResourcePods] = resource.MustParse("20") + } + if mode == "delayed" && reads >= 2 { + quota.Status.Used = quota.Spec.Hard.DeepCopy() + } + if mode == "cancel" { + cancel() + } + return true, quota.DeepCopy(), nil + }) + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + stop := errors.New("create reached") + d.PrependReactor("create", "executionenvironments", func(ktesting.Action) (bool, runtime.Object, error) { + creates++ + if reads < 2 || mode != "delayed" { + t.Error("create before quota accounting initialized") + } + return true, nil, stop + }) + _, err := seedOneLegacyEnvironment(ctx, d, kube, "test", resolvedProfile{}, executionenv.Owner{}, "client", "legacy", "binding", nil, false) + if mode == "delayed" { + if !errors.Is(err, stop) || creates != 1 { + t.Fatalf("creates=%d reads=%d error=%v", creates, reads, err) + } + return + } + if err == nil || creates != 0 { + t.Fatalf("creates=%d error=%v", creates, err) + } + if mode == "forbidden" || mode == "api-error" { + var reported strings.Builder + fmt.Fprintln(&reported, err) // Same reporting boundary as legacyfixture.fail. + for _, secret := range []string{"https://sentinel.invalid", "sk-fake-quota-secret"} { + if strings.Contains(err.Error(), secret) || strings.Contains(reported.String(), secret) { + t.Fatal("quota error disclosed API response data") + } + } + if reads != 1 { + t.Fatalf("API error retried: reads=%d", reads) + } + } + switch mode { + case "timeout": + if strings.Contains(err.Error(), "private-quota-key") || strings.Contains(err.Error(), "12345") || !strings.Contains(err.Error(), "pods") { + t.Fatalf("quota diagnostics lost mismatch or leaked data: %v", err) + } + if !strings.Contains(err.Error(), "count/executionenvironments.execution.mecatl.dev") || !strings.Contains(err.Error(), "elapsed=") { + t.Fatalf("missing bounded quota diagnostic: %v", err) + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatal(err) + } + case "cancel": + if !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + case "forbidden": + if !apierrors.IsForbidden(err) || reads != 1 { + t.Fatalf("reads=%d error=%v", reads, err) + } + } + }) + } +} diff --git a/internal/adapter/executioncontroller/operation_lease.go b/internal/adapter/executioncontroller/operation_lease.go new file mode 100644 index 0000000000..62ce85a460 --- /dev/null +++ b/internal/adapter/executioncontroller/operation_lease.go @@ -0,0 +1,76 @@ +package executioncontroller + +import ( + "context" + "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +type operationLeaseAuthority struct { + environment, revision, operationID, bindingID, runID, claimID string + client, owner string + epoch, grantGeneration uint64 +} + +func operationMatches(o *unstructured.Unstructured, authority operationLeaseAuthority, holderID string) bool { + m, found, err := unstructured.NestedMap(o.Object, "status", "activeOperation") + return err == nil && found && text(m, "id") == authority.operationID && text(m, "holderID") == holderID && + text(m, "claimID") == authority.claimID && text(m, "runID") == authority.runID && + intNested(m, "epoch") == int64(authority.epoch) //nolint:gosec // epochs are bounded before persistence. +} + +func operationLeaseCurrent(o *unstructured.Unstructured, authority operationLeaseAuthority, holderID string, now time.Time) bool { + if !operationMatches(o, authority, holderID) || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return false + } + expires, err := time.Parse(time.RFC3339Nano, textNested(o.Object, "status", "activeOperation", "expiresAt")) + return err == nil && now.Before(expires) +} + +func (s *Store) operationAuthorityCurrent(o *unstructured.Unstructured, authority operationLeaseAuthority) bool { + claim, _, expiry, ok := activeRunFrom(o) + return ok && s.now().Before(expiry) && intNested(o.Object, "status", "activeRun", "epoch") == int64(authority.epoch) && //nolint:gosec // epochs are bounded before persistence. + claim.Environment.ID == authority.environment && claim.Environment.Revision == authority.revision && + claim.BindingID == authority.bindingID && claim.RunID == authority.runID && claim.ClaimID == authority.claimID && + claim.Epoch == authority.epoch && claim.GrantGeneration == authority.grantGeneration && + epochMatches(o, authority.epoch) && generationMatches(o, authority.grantGeneration) && + textNested(o.Object, "spec", "clientHash") == hashText(authority.client) && textNested(o.Object, "spec", "ownerHash") == authority.owner +} + +func (s *Store) renewOperationLease(ctx context.Context, authority operationLeaseAuthority) error { + interval := s.opTTL / 3 + if interval <= 0 { + interval = operationLeaseTTL / 3 + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + err := s.retryUpdateStatus(ctx, authority.environment, func(o *unstructured.Unstructured) error { + now := s.now() + if !operationLeaseCurrent(o, authority, s.holderID, now) || !s.operationAuthorityCurrent(o, authority) { + return incompatibleOperationError() + } + m, _, _ := unstructured.NestedMap(o.Object, "status", "activeOperation") + m["renewedAt"] = now.Format(time.RFC3339Nano) + m["expiresAt"] = now.Add(s.opTTL).Format(time.RFC3339Nano) + return unstructured.SetNestedMap(o.Object, m, "status", "activeOperation") + }) + if err != nil { + return err + } + } + } +} + +func incompatibleOperationError() error { + return &operationLeaseError{} +} + +type operationLeaseError struct{} + +func (*operationLeaseError) Error() string { return "operation lease ownership changed" } diff --git a/internal/adapter/executioncontroller/operation_lease_runtime_test.go b/internal/adapter/executioncontroller/operation_lease_runtime_test.go new file mode 100644 index 0000000000..0cc614b263 --- /dev/null +++ b/internal/adapter/executioncontroller/operation_lease_runtime_test.go @@ -0,0 +1,225 @@ +package executioncontroller + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +type leaseBlockingExecutor struct { + started chan struct{} + cancelled chan struct{} + release chan struct{} + once sync.Once +} + +func newLeaseBlockingExecutor() *leaseBlockingExecutor { + return &leaseBlockingExecutor{started: make(chan struct{}), cancelled: make(chan struct{}), release: make(chan struct{})} +} + +func (e *leaseBlockingExecutor) Execute(ctx context.Context, _ string, _ executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + close(e.started) + select { + case <-ctx.Done(): + close(e.cancelled) + case <-e.release: + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Data: []byte("ok"), Version: "v1"}}, nil + } + <-e.release + return executionenv.ExecutorResponse{}, ctx.Err() +} + +func (e *leaseBlockingExecutor) unblock() { e.once.Do(func() { close(e.release) }) } + +func TestActiveOperationCancelsWhenExecutionAuthorityIsLost(t *testing.T) { + for _, tc := range []struct { + name string + plant func(*unstructured.Unstructured) + }{ + {name: "run expiry", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano), "status", "activeRun", "expiresAt") + }}, + {name: "operation expiry", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano), "status", "activeOperation", "expiresAt") + }}, + {name: "grant revocation", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, int64(2), "status", "grantGeneration") + }}, + } { + t.Run(tc.name, func(t *testing.T) { + testActiveOperationCancellation(t, nil, tc.plant, nil) + }) + } +} + +func TestActiveOperationCancelsWhenLeaseRenewalPersistenceFails(t *testing.T) { + var fail atomic.Bool + testActiveOperationCancellation(t, func(client *dynamicfake.FakeDynamicClient) { + client.PrependReactor("update", "executionenvironments", func(_ k8stesting.Action) (bool, runtime.Object, error) { + if fail.CompareAndSwap(true, false) { + return true, nil, apierrors.NewInternalError(errors.New("planted renewal write failure")) + } + return false, nil, nil + }) + }, func(*unstructured.Unstructured) {}, func() { fail.Store(true) }) +} + +func testActiveOperationCancellation(t *testing.T, install func(*dynamicfake.FakeDynamicClient), plant func(*unstructured.Unstructured), afterPlant func()) { + t.Helper() + env := runFixtureEnvironment(1, 1) + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + if install != nil { + install(client) + } + exec := newLeaseBlockingExecutor() + defer exec.unblock() + store := NewStore(client, "ns", testProfiles(), exec) + store.opTTL = 60 * time.Millisecond + done := make(chan error, 1) + go func() { + _, err := store.File(context.Background(), "client", "owner", executionenv.FileRequest{Context: operationRequestContext(), Operation: executionenv.OpFileRead, Path: "sentinel"}) + done <- err + }() + select { + case <-exec.started: + case <-time.After(time.Second): + t.Fatal("executor did not start") + } + if err := store.retryUpdateStatus(t.Context(), "env", func(o *unstructured.Unstructured) error { plant(o); return nil }); err != nil { + t.Fatal(err) + } + if afterPlant != nil { + afterPlant() + } + select { + case <-exec.cancelled: + case <-time.After(500 * time.Millisecond): + t.Fatal("active executor was not cancelled after execution authority was lost") + } + exec.unblock() + var controlled *executionenv.Error + select { + case err := <-done: + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeFenceUnknown { + t.Fatalf("operation error=%v, want FenceUnknown", err) + } + case <-time.After(time.Second): + t.Fatal("operation did not finish after backend release") + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(got.Object, "status", "activeOperation", "id") == "" || textNested(got.Object, "status", "fenceState") != "FenceUnknown" { + t.Fatalf("lost authority did not retain operation/fence uncertainty: %v", got.Object["status"]) + } + _, err = store.File(t.Context(), "client", "owner", executionenv.FileRequest{Context: operationRequestContext(), Operation: executionenv.OpFileRead, Path: "takeover"}) + if err == nil { + t.Fatal("new writer was admitted while prior operation remained uncertain") + } +} + +func TestActiveOperationCleanCompletionRechecksRunAuthority(t *testing.T) { + for _, tc := range []struct { + name string + plant func(*unstructured.Unstructured) + }{ + {name: "run expiry", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano), "status", "activeRun", "expiresAt") + }}, + {name: "operation expiry", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano), "status", "activeOperation", "expiresAt") + }}, + {name: "missing operation expiry", plant: func(o *unstructured.Unstructured) { + unstructured.RemoveNestedField(o.Object, "status", "activeOperation", "expiresAt") + }}, + {name: "malformed operation expiry", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, "not-a-time", "status", "activeOperation", "expiresAt") + }}, + {name: "grant revocation", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, int64(2), "status", "grantGeneration") + }}, + } { + t.Run(tc.name, func(t *testing.T) { + env := runFixtureEnvironment(1, 1) + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + exec := newLeaseBlockingExecutor() + defer exec.unblock() + store := NewStore(client, "ns", testProfiles(), exec) + store.opTTL = time.Hour // No renewal tick can observe the planted loss. + done := make(chan error, 1) + go func() { + _, err := store.File(context.Background(), "client", "owner", executionenv.FileRequest{Context: operationRequestContext(), Operation: executionenv.OpFileRead, Path: "sentinel"}) + done <- err + }() + select { + case <-exec.started: + case <-time.After(time.Second): + t.Fatal("executor did not start") + } + if err := store.retryUpdateStatus(t.Context(), "env", func(o *unstructured.Unstructured) error { tc.plant(o); return nil }); err != nil { + t.Fatal(err) + } + exec.unblock() + select { + case err := <-done: + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeFenceUnknown { + t.Fatalf("operation error=%v, want FenceUnknown", err) + } + case <-time.After(time.Second): + t.Fatal("operation did not finish") + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(got.Object, "status", "activeOperation", "id") == "" || textNested(got.Object, "status", "fenceState") != "FenceUnknown" { + t.Fatalf("lost authority was treated as a clean terminal: %v", got.Object["status"]) + } + if _, err := store.File(t.Context(), "client", "owner", executionenv.FileRequest{Context: operationRequestContext(), Operation: executionenv.OpFileRead, Path: "takeover"}); err == nil { + t.Fatal("takeover was admitted after completion authority was lost") + } + }) + } +} + +func TestActiveOperationCleanCompletionClearsLeaseWithoutFencing(t *testing.T) { + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), runFixtureEnvironment(1, 1)) + exec := newLeaseBlockingExecutor() + store := NewStore(client, "ns", testProfiles(), exec) + store.opTTL = 60 * time.Millisecond + done := make(chan error, 1) + go func() { + _, err := store.File(context.Background(), "client", "owner", executionenv.FileRequest{Context: operationRequestContext(), Operation: executionenv.OpFileRead, Path: "sentinel"}) + done <- err + }() + <-exec.started + exec.unblock() + if err := <-done; err != nil { + t.Fatal(err) + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if id := textNested(got.Object, "status", "activeOperation", "id"); id != "" || textNested(got.Object, "status", "fenceState") != fenceHealthy { + t.Fatalf("clean completion left operation uncertainty: %s %v", id, got.Object["status"]) + } +} + +func operationRequestContext() executionenv.RequestContext { + return executionenv.RequestContext{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1} +} diff --git a/internal/adapter/executioncontroller/podexec.go b/internal/adapter/executioncontroller/podexec.go new file mode 100644 index 0000000000..2ed4ce2995 --- /dev/null +++ b/internal/adapter/executioncontroller/podexec.go @@ -0,0 +1,120 @@ +package executioncontroller + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "sync" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// PodExecutor invokes the fixed helper in an already-selected executor Pod. +type PodExecutor struct { + config *rest.Config + kube kubernetes.Interface + namespace string +} + +// NewPodExecutor constructs a namespace-scoped Pod executor transport. +func NewPodExecutor(config *rest.Config, kube kubernetes.Interface, namespace string) *PodExecutor { + return &PodExecutor{config: rest.CopyConfig(config), kube: kube, namespace: namespace} +} + +// Execute runs one bounded helper request in pod. +func (p *PodExecutor) Execute(ctx context.Context, pod string, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + body, err := json.Marshal(q) + if err != nil { + return executionenv.ExecutorResponse{}, err + } + if len(body) > executionenv.MaxJSONBody { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "executor request exceeds limit"} + } + req := p.kube.CoreV1().RESTClient().Post().Resource("pods").Name(pod).Namespace(p.namespace).SubResource("exec").VersionedParams(&corev1.PodExecOptions{Container: "executor", Command: []string{"/mecatl-executor"}, Stdin: true, Stdout: true, Stderr: true, TTY: false}, scheme.ParameterCodec) + exec, err := remotecommand.NewSPDYExecutor(p.config, "POST", req.URL()) + if err != nil { + return executionenv.ExecutorResponse{}, fmt.Errorf("create executor stream: %w", err) + } + var stdout, stderr limitedBuffer + stdout.limit = executionenv.MaxJSONBody + stderr.limit = 64 << 10 + var stdin io.Reader = bytes.NewReader(body) + var closeInput func() + if q.Operation == executionenv.OpCommandStart { + stdin, closeInput = commandInput(ctx, body) + defer closeInput() + } + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{Stdin: stdin, Stdout: &stdout, Stderr: &stderr}) + if err != nil { + return executionenv.ExecutorResponse{}, fmt.Errorf("executor stream failed: %w", err) + } + var envelope executionenv.ExecutorEnvelope + if err := executionenv.DecodeStrict(stdout.Bytes(), &envelope); err != nil { + return executionenv.ExecutorResponse{}, fmt.Errorf("invalid executor response: %w", err) + } + if envelope.Error != nil && envelope.Response != nil { + return executionenv.ExecutorResponse{}, errors.New("executor returned both result and error") + } + if envelope.Error != nil { + return executionenv.ExecutorResponse{}, envelope.Error + } + if envelope.Response == nil { + return executionenv.ExecutorResponse{}, errors.New("executor returned no result") + } + return *envelope.Response, nil +} + +func commandInput(ctx context.Context, body []byte) (io.Reader, func()) { + reader, writer := io.Pipe() + done := make(chan struct{}) + go func() { + payload := append(append(make([]byte, 0, len(body)+1), body...), '\n') + if _, err := writer.Write(payload); err != nil { + _ = writer.CloseWithError(err) + } + }() + go func() { + select { + case <-ctx.Done(): + _ = writer.CloseWithError(ctx.Err()) + case <-done: + } + }() + var once sync.Once + return reader, func() { + once.Do(func() { + close(done) + _ = writer.Close() + _ = reader.Close() + }) + } +} + +type limitedBuffer struct { + bytes.Buffer + limit int + overflow bool +} + +func (b *limitedBuffer) Write(p []byte) (int, error) { + if b.Len()+len(p) > b.limit { + remaining := b.limit - b.Len() + if remaining > 0 { + _, _ = b.Buffer.Write(p[:remaining]) + } + b.overflow = true + return len(p), errors.New("executor output exceeds limit") + } + return b.Buffer.Write(p) +} + +var _ ExecutorTransport = (*PodExecutor)(nil) diff --git a/internal/adapter/executioncontroller/podexec_test.go b/internal/adapter/executioncontroller/podexec_test.go new file mode 100644 index 0000000000..4108f48c93 --- /dev/null +++ b/internal/adapter/executioncontroller/podexec_test.go @@ -0,0 +1,36 @@ +package executioncontroller + +import ( + "context" + "io" + "testing" + "time" +) + +func TestCommandInputClosesIndependentlyOnCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + reader, closeInput := commandInput(ctx, []byte(`{"operation":"command.start"}`)) + defer closeInput() + + line := make([]byte, len(`{"operation":"command.start"}`)+1) + if _, err := io.ReadFull(reader, line); err != nil { + t.Fatal(err) + } + cancel() + readDone := make(chan error, 1) + go func() { + var one [1]byte + _, err := reader.Read(one[:]) + readDone <- err + }() + deadline := time.NewTimer(2 * time.Second) + defer deadline.Stop() + select { + case err := <-readDone: + if err == nil { + t.Fatal("command control pipe remained open after cancellation") + } + case <-deadline.C: + t.Fatal("command control pipe did not close after cancellation") + } +} diff --git a/internal/adapter/executioncontroller/production_lifecycle_test.go b/internal/adapter/executioncontroller/production_lifecycle_test.go new file mode 100644 index 0000000000..721b4150e4 --- /dev/null +++ b/internal/adapter/executioncontroller/production_lifecycle_test.go @@ -0,0 +1,826 @@ +package executioncontroller + +import ( + "context" + "errors" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestPodTerminalRequiresEveryDeclaredContainerByName(t *testing.T) { + terminated := corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 0}} + base := &corev1.Pod{Spec: corev1.PodSpec{InitContainers: []corev1.Container{{Name: "init"}}, Containers: []corev1.Container{{Name: "executor"}}}, Status: corev1.PodStatus{Phase: corev1.PodSucceeded, InitContainerStatuses: []corev1.ContainerStatus{{Name: "init", State: terminated}}, ContainerStatuses: []corev1.ContainerStatus{{Name: "executor", State: terminated}}}} + if !podTerminal(base) { + t.Fatal("exact terminated init and regular containers were rejected") + } + cases := map[string]func(*corev1.Pod){ + "init waiting": func(p *corev1.Pod) { + p.Status.InitContainerStatuses[0].State = corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{}} + }, + "init running": func(p *corev1.Pod) { + p.Status.InitContainerStatuses[0].State = corev1.ContainerState{Running: &corev1.ContainerStateRunning{}} + }, + "missing init": func(p *corev1.Pod) { p.Status.InitContainerStatuses = nil }, + "duplicate regular": func(p *corev1.Pod) { + p.Status.ContainerStatuses = append(p.Status.ContainerStatuses, p.Status.ContainerStatuses[0]) + }, + "wrong regular name": func(p *corev1.Pod) { p.Status.ContainerStatuses[0].Name = "other" }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + pod := base.DeepCopy() + mutate(pod) + if podTerminal(pod) { + t.Fatal("invalid container evidence proved terminal") + } + }) + } +} + +func lifecycleAdminEnvironment(schema int64, refs []any) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", + "metadata": map[string]any{"name": "env", "namespace": "ns", "uid": "env-uid", "finalizers": []any{environmentFinalizer}}, + "spec": map[string]any{"schemaVersion": schema, "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, + "status": map[string]any{"schemaVersion": schema, "epoch": int64(4), "grantGeneration": int64(1), "fenceState": fenceHealthy, "references": refs, + "pvc": map[string]any{"name": "workspace", "uid": "pvc-uid"}, "pod": map[string]any{"name": "executor", "uid": "pod-uid"}, + "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}, + }} +} + +func terminalExecutor() *corev1.Pod { + profile, _ := testProfiles().get("go") + noPriv, nonroot, ro := false, true, true + uid := int64(65532) + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "executor", Namespace: "ns", UID: types.UID("pod-uid"), Finalizers: []string{executorFinalizer}, Labels: map[string]string{"execution.mecatl.dev/environment": "env", "execution.mecatl.dev/profile": hashText("go")[:16]}, OwnerReferences: []metav1.OwnerReference{{APIVersion: "execution.mecatl.dev/v1alpha1", Kind: "ExecutionEnvironment", Name: "env", UID: types.UID("env-uid"), Controller: &nonroot}}}, Spec: corev1.PodSpec{AutomountServiceAccountToken: &noPriv, RuntimeClassName: &profile.Spec.RuntimeClassName, RestartPolicy: corev1.RestartPolicyNever, SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &nonroot, RunAsUser: &uid, RunAsGroup: &uid, FSGroup: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, Containers: []corev1.Container{{Name: "executor", Image: profile.Spec.Image, Command: []string{"/bin/sh", "-c", "trap : TERM INT; sleep infinity & wait"}, SecurityContext: &corev1.SecurityContext{AllowPrivilegeEscalation: &noPriv, ReadOnlyRootFilesystem: &ro, RunAsNonRoot: &nonroot, RunAsUser: &uid, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}}, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: profile.CPURequest, corev1.ResourceMemory: profile.MemoryRequest, corev1.ResourceEphemeralStorage: profile.EphemeralStorageRequest}, Limits: corev1.ResourceList{corev1.ResourceCPU: profile.CPULimit, corev1.ResourceMemory: profile.MemoryLimit, corev1.ResourceEphemeralStorage: profile.EphemeralStorageLimit}}, VolumeMounts: []corev1.VolumeMount{{Name: "workspace", MountPath: "/workspace"}, {Name: "tmp", MountPath: "/tmp"}}}}, Volumes: []corev1.Volume{{Name: "workspace", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "workspace"}}}, {Name: "tmp", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{SizeLimit: &profile.TmpSizeLimit}}}}}, Status: corev1.PodStatus{Phase: corev1.PodFailed, ContainerStatuses: []corev1.ContainerStatus{{Name: "executor", State: corev1.ContainerState{Terminated: &corev1.ContainerStateTerminated{ExitCode: 137}}}}}} +} + +func retainedPVC() *corev1.PersistentVolumeClaim { + profile, _ := testProfiles().get("go") + mode := corev1.PersistentVolumeFilesystem + return &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "workspace", Namespace: "ns", UID: types.UID("pvc-uid"), Labels: map[string]string{"execution.mecatl.dev/environment": "env", "execution.mecatl.dev/revision": "rev", "execution.mecatl.dev/allocation-uid": "env-uid"}}, Spec: corev1.PersistentVolumeClaimSpec{StorageClassName: &profile.Spec.StorageClass, AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, VolumeMode: &mode, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: profile.StorageSize}}}} +} + +func adminRequestFixture() adminLifecycleRequest { + return adminLifecycleRequest{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, OwnerHash: "owner", Client: "client", ExpectedEpoch: 4, ExpectedPodUID: "pod-uid", ExpectedPVCUID: "pvc-uid", OperationID: "admin-operation"} +} + +func TestHealthyTerminalPodCanStartLifecycleWithoutReadyCondition(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Ready", false, "PodTerminated", "executor exited naturally") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + q := adminRequestFixture() + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatal(err) + } + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatalf("retry was not idempotent: %v", err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "terminationProof", operationIDField) != q.OperationID || textNested(got.Object, "status", "lifecycleOperation", "id") != q.OperationID { + t.Fatalf("terminal proof and lifecycle operation were not persisted: %v", got.Object["status"]) + } +} + +func TestHealthyTerminalLifecycleRejectsIncompleteEvidence(t *testing.T) { + cases := map[string]func(*corev1.Pod){ + "missing pod": func(*corev1.Pod) {}, + "init running": func(p *corev1.Pod) { + p.Spec.InitContainers = []corev1.Container{{Name: "init"}} + p.Status.InitContainerStatuses = []corev1.ContainerStatus{{Name: "init", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}} + }, + } + for name, mutate := range cases { + t.Run(name, func(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Ready", false, "PodTerminated", "executor exited naturally") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + objects := []runtime.Object{retainedPVC()} + if name != "missing pod" { + pod := terminalExecutor() + mutate(pod) + objects = append(objects, pod) + } + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(objects...)) + if err := store.RetireExact(t.Context(), adminRequestFixture()); err == nil { + t.Fatal("incomplete terminal evidence admitted lifecycle") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "terminationProof", "podUID") != "" || textNested(got.Object, "status", "lifecycleOperation", "id") != "" { + t.Fatalf("rejected evidence changed lifecycle state: %v", got.Object["status"]) + } + }) + } +} + +func TestRetiredEnvironmentIsTerminalWithoutDesiredSpecTransition(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + setConditionObject(env, "Ready", false, "Retained", "environment is retired with retained storage") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + kube := kubefake.NewSimpleClientset(retainedPVC()) + r := NewReconciler(dynamicClient, kube, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + if actions := kube.Actions(); len(actions) != 0 { + t.Fatalf("retired environment re-entered provisioning with actions %v", actions) + } + got, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(got.Object, "status", "fenceState") != fenceHealthy || !conditionTrue(got, "Retired") { + t.Fatalf("retired environment changed terminal state: %v", got.Object["status"]) + } +} + +func TestRetirementCompletionReceiptReplaysOnlyExactOperation(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, terminationProof(q, terminalExecutor()), "status", "terminationProof") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": q.OperationID, "type": "RetireEnvironment", "phase": "WaitingForPodDeletion", "expectedEpoch": int64(4), "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID, "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + r := NewReconciler(dynamicClient, kubefake.NewSimpleClientset(retainedPVC()), "ns", testProfiles()) + if err := r.finishRetirement(t.Context(), env, q.OperationID); err != nil { + t.Fatal(err) + } + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatalf("lost-reply retry failed: %v", err) + } + q.OperationID = "different-operation" + if err := store.RetireExact(t.Context(), q); err == nil { + t.Fatal("different operation silently succeeded against retired environment") + } + q = adminRequestFixture() + q.OwnerHash = "other-owner" + if err := store.RetireExact(t.Context(), q); err == nil { + t.Fatal("wrong subject replayed retirement receipt") + } +} + +func TestRetiredEnvironmentRejectsNewReplacementButKeepsExactReplaysAndDelete(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, terminationProof(q, terminalExecutor()), "status", "terminationProof") + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + setConditionObject(env, "Ready", false, "Retained", "environment is retired with retained storage") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatalf("exact completed retirement replay failed: %v", err) + } + replacement := q + replacement.OperationID = "replacement-after-retirement" + if err := store.ReplaceExecutor(t.Context(), replacement); err == nil { + t.Fatal("retired environment admitted a new replacement") + } else { + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("new replacement error=%v, want controlled conflict", err) + } + } + got, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(got.Object, "status", "lifecycleOperation", "id") != "" || !conditionTrue(got, "Retired") { + t.Fatalf("rejected replacement mutated retired status: %v", got.Object["status"]) + } + + deleteRequest := q + deleteRequest.OperationID = "delete-after-rejected-replacement" + if err := store.DeleteRetiredEnvironment(t.Context(), deleteRequest); err != nil { + t.Fatalf("retained delete was stranded after rejected replacement: %v", err) + } +} + +func TestCompletedReplacementReplayRemainsIdempotentAfterRetirement(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, map[string]any{ + operationIDField: q.OperationID, "previousPodUID": q.ExpectedPodUID, + "replacementPodUID": "replacement-pod", "pvcUID": q.ExpectedPVCUID, + "previousEpoch": int64(q.ExpectedEpoch), "replacementEpoch": int64(q.ExpectedEpoch + 1), + }, "status", "lastReplacement") + _ = unstructured.SetNestedField(env.Object, int64(q.ExpectedEpoch+1), "status", "epoch") + _ = unstructured.SetNestedField(env.Object, "replacement-pod", "status", "pod", "uid") + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "Ready", false, "Retained", "environment is retired with retained storage") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + + if err := store.ReplaceExecutor(t.Context(), q); err != nil { + t.Fatalf("exact completed replacement replay failed after retirement: %v", err) + } + got, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(got.Object, "status", "lifecycleOperation", "id") != "" || textNested(got.Object, "status", "pod", "uid") != "replacement-pod" || !conditionTrue(got, "Retired") { + t.Fatalf("completed replacement replay mutated retired status: %v", got.Object["status"]) + } +} + +func TestAdmittedRetireAndDeleteIgnoreLaterProfileDrift(t *testing.T) { + for _, mode := range []string{"removed", "changed"} { + t.Run(mode, func(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + pod, pvc := terminalExecutor(), retainedPVC() + pod.Name, pvc.Name = resourceName("executor", "env"), resourceName("workspace", "env") + pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name + _ = unstructured.SetNestedField(env.Object, pod.Name, "status", "pod", "name") + _ = unstructured.SetNestedField(env.Object, pvc.Name, "status", "pvc", "name") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + allocations := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{profileAllocationKey("go"): `["env"]`}} + kube := kubefake.NewSimpleClientset(pod, pvc, allocations) + kube.PrependReactor("delete", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + obj, err := kube.Tracker().Get(corev1.SchemeGroupVersion.WithResource("pods"), "ns", action.(k8stesting.DeleteAction).GetName()) + if err != nil { + return true, nil, err + } + return len(obj.(*corev1.Pod).Finalizers) != 0, nil, nil + }) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kube) + q := adminRequestFixture() + if err := store.RetireExact(t.Context(), q); err != nil { + t.Fatal(err) + } + drifted := &Profiles{byName: map[string]resolvedProfile{}} + if mode == "changed" { + profile, _ := testProfiles().get("go") + profile.Digest = "sha256:changed" + drifted.byName["go"] = profile + } + r := NewReconciler(dynamicClient, kube, "ns", drifted) + t.Cleanup(r.queue.ShutDown) + for range 8 { + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + current, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err == nil && conditionTrue(current, "Retired") { + break + } + } + retired, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatalf("read retired environment after profile %s: %v", mode, err) + } + if !conditionTrue(retired, "Retired") || !conditionTrue(retired, "ExecutorTerminated") || textNested(retired.Object, "status", "lifecycleOperation", "id") != "" { + t.Fatalf("admitted retirement stranded after profile %s: status=%v", mode, retired.Object["status"]) + } + q.OperationID = "delete-after-profile-" + mode + if err := store.DeleteRetiredEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + for range 6 { + err := r.Reconcile(t.Context(), "env") + if apierrors.IsNotFound(err) { + break + } + if err != nil { + t.Fatal(err) + } + if _, getErr := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}); apierrors.IsNotFound(getErr) { + break + } + } + if _, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("retained CR survived delete after profile %s: %v", mode, err) + } + if _, err := kube.CoreV1().PersistentVolumeClaims("ns").Get(t.Context(), pvc.Name, metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("retained PVC survived delete after profile %s: %v", mode, err) + } + if slots := profileSlots(t, kube); len(slots) != 0 { + t.Fatalf("profile slot survived delete after profile %s: %v", mode, slots) + } + }) + } +} + +func TestProfileDriftBlocksEveryReplacementMutation(t *testing.T) { + for _, mode := range []string{"removed", "changed"} { + for _, phase := range []string{"Quiescing", "WaitingForTermination", "RemovingPodFinalizer", "WaitingForPodDeletion", "CreatingReplacement"} { + t.Run(mode+"/"+phase, func(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": q.OperationID, "type": "ReplaceExecutor", "phase": phase, "expectedEpoch": int64(q.ExpectedEpoch), "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID}, "status", "lifecycleOperation") + if phase == "RemovingPodFinalizer" || phase == "WaitingForPodDeletion" || phase == "CreatingReplacement" { + _ = unstructured.SetNestedMap(env.Object, terminationProof(q, terminalExecutor()), "status", "terminationProof") + } + if phase == "CreatingReplacement" { + unstructured.RemoveNestedField(env.Object, "status", "pod") + } + objects := []runtime.Object{retainedPVC()} + if phase != "WaitingForPodDeletion" && phase != "CreatingReplacement" { + objects = append(objects, terminalExecutor()) + } + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + kube := kubefake.NewSimpleClientset(objects...) + profiles := &Profiles{byName: map[string]resolvedProfile{}} + if mode == "changed" { + profile, _ := testProfiles().get("go") + profile.Digest = "sha256:changed" + profiles.byName["go"] = profile + } + r := NewReconciler(dynamicClient, kube, "ns", profiles) + t.Cleanup(r.queue.ShutDown) + + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + got, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if gotPhase := textNested(got.Object, "status", "lifecycleOperation", "phase"); gotPhase != phase { + t.Fatalf("replacement phase advanced after profile %s: got %q, want %q", mode, gotPhase, phase) + } + for _, action := range kube.Actions() { + if action.GetResource().Resource == "pods" && action.GetVerb() != "get" { + t.Fatalf("profile %s caused Pod mutation during %s: %v", mode, phase, action) + } + } + }) + } + } +} + +func TestProfileDriftStillBlocksProvisioningAndReplacementCreation(t *testing.T) { + for _, phase := range []string{"provisioning", "CreatingReplacement"} { + t.Run(phase, func(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + if phase == "CreatingReplacement" { + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, terminationProof(q, terminalExecutor()), "status", "terminationProof") + unstructured.RemoveNestedField(env.Object, "status", "pod") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": q.OperationID, "type": "ReplaceExecutor", "phase": phase, "expectedEpoch": int64(q.ExpectedEpoch), "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID}, "status", "lifecycleOperation") + } + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + kube := kubefake.NewSimpleClientset(retainedPVC()) + r := NewReconciler(dynamicClient, kube, "ns", &Profiles{byName: map[string]resolvedProfile{}}) + t.Cleanup(r.queue.ShutDown) + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + got, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if phase == "provisioning" { + if conditionTrue(got, "Ready") || textNested(got.Object, "status", "fenceState") != fenceHealthy { + t.Fatalf("ordinary provisioning did not reject missing profile: %v", got.Object["status"]) + } + } else if textNested(got.Object, "status", "fenceState") != "FenceUnknown" || textNested(got.Object, "status", "lifecycleOperation", "phase") != phase { + t.Fatalf("replacement creation did not fail closed on missing profile: %v", got.Object["status"]) + } + for _, action := range kube.Actions() { + if action.GetVerb() == "create" { + t.Fatalf("profile drift created runtime resource during %s", phase) + } + } + }) + } +} + +func TestOldSchemaFailsClosedAndExplicitMigrationConvertsReferences(t *testing.T) { + env := lifecycleAdminEnvironment(0, []any{"session-a"}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + kube := kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC()) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kube) + _, err := store.Attach(t.Context(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "session-a") + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeNotReady { + t.Fatalf("old schema attach error=%v", err) + } + q := adminRequestFixture() + q.ExpectedSchema = 0 + if err := store.MigrateEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + if err := store.MigrateEnvironment(t.Context(), q); err != nil { + t.Fatalf("migration retry was not idempotent: %v", err) + } + got, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if intNested(got.Object, "spec", "schemaVersion") != 2 || intNested(got.Object, "status", "schemaVersion") != 2 || intNested(got.Object, "status", "epoch") != 5 { + t.Fatalf("migration did not advance schema and epoch: %v", got.Object) + } + refs, err := referenceRecords(got) + if err != nil || len(refs) != 1 || refs[0].BindingID != "session-a" || refs[0].State != executionenv.ReferencePublished { + t.Fatalf("migrated refs=%+v err=%v", refs, err) + } + fromSchema, found, err := unstructured.NestedInt64(got.Object, "status", "lastMigrationFromSchema") + if err != nil || !found || fromSchema != 0 { + t.Fatal("schema-zero migration lost exact receipt presence") + } + unstructured.RemoveNestedField(got.Object, "status", "lastMigrationFromSchema") + if _, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(t.Context(), got, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := store.MigrateEnvironment(t.Context(), q); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("old receipt without source schema replayed: %v", err) + } +} + +func TestMigrationReceiptExpiresAfterReconciledReplacement(t *testing.T) { + for name, schema := range map[string]int64{"legacy": 0, "prototype": 1} { + t.Run(name, func(t *testing.T) { + env := lifecycleAdminEnvironment(schema, []any{"session-a"}) + pod, pvc := terminalExecutor(), retainedPVC() + pod.Name, pvc.Name = resourceName("executor", "env"), resourceName("workspace", "env") + pod.Spec.Volumes[0].PersistentVolumeClaim.ClaimName = pvc.Name + _ = unstructured.SetNestedField(env.Object, pod.Name, "status", "pod", "name") + _ = unstructured.SetNestedField(env.Object, pvc.Name, "status", "pvc", "name") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(pod, pvc) + // The fake API must retain a finalizer-protected Pod until the real + // reconciler has recorded terminal proof and removed its finalizer. + k.PrependReactor("delete", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + obj, err := k.Tracker().Get(corev1.SchemeGroupVersion.WithResource("pods"), "ns", action.(k8stesting.DeleteAction).GetName()) + if err != nil { + return true, nil, err + } + return len(obj.(*corev1.Pod).Finalizers) != 0, nil, nil + }) + k.PrependReactor("create", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + created := action.(k8stesting.CreateAction).GetObject().(*corev1.Pod) + created.UID = "replacement-pod" + created.Status = replacementExecutor(true).Status + return false, nil, nil + }) + store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) + q := adminRequestFixture() + q.ExpectedSchema = schema + if err := store.MigrateEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + if err := store.MigrateEnvironment(t.Context(), q); err != nil { + t.Fatalf("exact receipt replay before replacement: %v", err) + } + replace := q + replace.OperationID = "replace-after-migration" + replace.ExpectedEpoch++ + if err := store.ReplaceExecutor(t.Context(), replace); err != nil { + t.Fatal(err) + } + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + // Drive every replacement phase, then an ordinary ready reconcile. + for range 8 { + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !conditionTrue(got, "Ready") || textNested(got.Object, "status", "pod", "uid") != "replacement-pod" || textNested(got.Object, "status", "pvc", "uid") != q.ExpectedPVCUID || intNested(got.Object, "status", "epoch") != 6 { + t.Fatalf("replacement did not complete normally: %v", got.Object["status"]) + } + changed := q + changed.ExpectedPodUID = "replacement-pod" + for _, replay := range []adminLifecycleRequest{changed, q} { + var controlled *executionenv.Error + if err := store.MigrateEnvironment(t.Context(), replay); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("expired migration replay with Pod %q: %v", replay.ExpectedPodUID, err) + } + } + if textNested(got.Object, "status", "lastMigrationOperationID") != "" { + t.Fatal("replacement retained migration operation receipt") + } + if _, found, err := unstructured.NestedInt64(got.Object, "status", "lastMigrationFromSchema"); err != nil || found { + t.Fatal("replacement retained migration source schema receipt") + } + }) + } +} + +func TestInsecurePrototypeMigrationRejectedWithoutRewrite(t *testing.T) { + env := lifecycleAdminEnvironment(1, []any{"session-a"}) + pod := terminalExecutor() + automount := true + pod.Spec.AutomountServiceAccountToken = &automount + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(pod, retainedPVC())) + q := adminRequestFixture() + q.ExpectedSchema = 1 + if err := store.MigrateEnvironment(t.Context(), q); err == nil { + t.Fatal("insecure prototype executor migrated") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if intNested(got.Object, "spec", "schemaVersion") != 1 || intNested(got.Object, "status", "schemaVersion") != 1 || intNested(got.Object, "status", "epoch") != 4 { + t.Fatalf("insecure prototype was rewritten: %v", got.Object) + } +} + +func TestUnknownSchemaMigrationRejectedWithoutRewrite(t *testing.T) { + env := lifecycleAdminEnvironment(7, []any{}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + q := adminRequestFixture() + q.ExpectedSchema = 7 + if err := store.MigrateEnvironment(t.Context(), q); err == nil { + t.Fatal("unknown schema migrated") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if intNested(got.Object, "spec", "schemaVersion") != 7 || intNested(got.Object, "status", "epoch") != 4 { + t.Fatalf("unknown schema was rewritten: %v", got.Object) + } +} + +func TestExpiredOperationLeaseFencesWithoutClearingIdentity(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "peer-op", "operation": "file.replace", "claimID": "claim", "runID": "run", "epoch": int64(4), "holderID": "peer", "renewedAt": time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano), "expiresAt": time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano)}, "status", "activeOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + r := NewReconciler(dynamicClient, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "fenceState") != "FenceUnknown" || textNested(got.Object, "status", "activeOperation", "id") != "peer-op" { + t.Fatalf("expired operation was not retained and fenced: %v", got.Object["status"]) + } +} + +func TestRecoverRequiresExactTerminalPodAndPersistsProof(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "uncertain", "claimID": "claim", "epoch": int64(4)}, "status", "activeOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + q := adminRequestFixture() + q.OperationID = "recover-op" + if err := store.RecoverEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "fenceState") != fenceHealthy || textNested(got.Object, "status", "activeOperation", "id") != "" || textNested(got.Object, "status", "terminationProof", "podUID") != "pod-uid" { + t.Fatalf("recovery proof/status=%v", got.Object["status"]) + } +} + +func TestRecoveredTerminalProofCanStartExactReplacement(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") + setConditionObject(env, "Ready", false, "FenceUnknown", "holder lost") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "uncertain", "claimID": "claim", "epoch": int64(4)}, "status", "activeOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC())) + q := adminRequestFixture() + q.OperationID = "recover-op" + if err := store.RecoverEnvironment(t.Context(), q); err != nil { + t.Fatal(err) + } + q.OperationID = "replace-op" + if err := store.ReplaceExecutor(t.Context(), q); err != nil { + t.Fatalf("exact replacement after terminal recovery: %v", err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "lifecycleOperation", "id") != "replace-op" { + t.Fatalf("replacement operation missing after recovery: %v", got.Object["status"]) + } +} + +func TestRecoverMissingPodRemainsFenceUnknown(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedField(env.Object, "FenceUnknown", "status", "fenceState") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kubefake.NewSimpleClientset(retainedPVC())) + if err := store.RecoverEnvironment(t.Context(), adminRequestFixture()); err == nil { + t.Fatal("missing executor accepted as termination proof") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "fenceState") != "FenceUnknown" || textNested(got.Object, "status", "terminationProof", "podUID") != "" { + t.Fatalf("missing executor changed fenced state: %v", got.Object["status"]) + } +} + +func TestCompletedReplacementReceiptReplaysExactOperation(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}}) + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, map[string]any{operationIDField: q.OperationID, "previousPodUID": q.ExpectedPodUID, "replacementPodUID": "new-pod", "pvcUID": q.ExpectedPVCUID, "previousEpoch": int64(q.ExpectedEpoch), "replacementEpoch": int64(q.ExpectedEpoch + 1)}, "status", "lastReplacement") + _ = unstructured.SetNestedField(env.Object, int64(q.ExpectedEpoch+1), "status", "epoch") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"name": "executor", "uid": "new-pod"}, "status", "pod") + store := NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env), "ns", testProfiles(), nil) + if err := store.ReplaceExecutor(t.Context(), q); err != nil { + t.Fatalf("exact completed replacement did not replay: %v", err) + } + q.OperationID = "different" + if err := store.ReplaceExecutor(t.Context(), q); err == nil { + t.Fatal("different operation replayed completed replacement") + } +} + +func TestReplacementQuiescesAndPendingReferenceBlocksRetirement(t *testing.T) { + published := map[string]any{"bindingID": "session", "state": "Published", "operationID": "publish", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)} + env := lifecycleAdminEnvironment(2, []any{published}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + q := adminRequestFixture() + if err := store.ReplaceExecutor(t.Context(), q); err != nil { + t.Fatal(err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "lifecycleOperation", "phase") != "Quiescing" || conditionTrue(got, "Ready") { + t.Fatalf("replacement did not consume admission: %v", got.Object["status"]) + } + if _, err := store.AcquireRun(t.Context(), q.Environment, "client", "owner", "session", "run", "acquire", time.Minute); err == nil { + t.Fatal("run admitted during replacement") + } + + pending := map[string]any{"bindingID": "pending", "state": "PendingDelete", "operationID": "delete", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)} + env = lifecycleAdminEnvironment(2, []any{pending}) + store = NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env), "ns", testProfiles(), nil) + if err := store.RetireExact(t.Context(), q); err == nil { + t.Fatal("pending reference did not block retirement") + } +} + +func TestReplacementPersistsTerminalProofBeforeRemovingPodFinalizer(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "replace", "type": "ReplaceExecutor", "phase": "WaitingForTermination", "expectedEpoch": int64(4), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + kube := kubefake.NewSimpleClientset(terminalExecutor(), retainedPVC()) + r := NewReconciler(dynamicClient, kube, "ns", testProfiles()) + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "terminationProof", "podUID") != "pod-uid" || textNested(got.Object, "status", "lifecycleOperation", "phase") != "RemovingPodFinalizer" { + t.Fatalf("terminal proof was not persisted first: %v", got.Object["status"]) + } + pod, _ := kube.CoreV1().Pods("ns").Get(t.Context(), "executor", metav1.GetOptions{}) + if !contains(pod.Finalizers, executorFinalizer) { + t.Fatal("executor finalizer was removed in the proof-persistence transition") + } + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + pod, err := kube.CoreV1().Pods("ns").Get(t.Context(), "executor", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if contains(pod.Finalizers, executorFinalizer) { + t.Fatal("executor finalizer remained after durable exact proof") + } + got, _ = dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "lifecycleOperation", "phase") != "WaitingForPodDeletion" { + t.Fatalf("phase=%v", got.Object["status"]) + } +} + +func replacementExecutor(ready bool) *corev1.Pod { + pod := terminalExecutor() + pod.UID = types.UID("new-pod") + pod.Status = corev1.PodStatus{Phase: corev1.PodRunning} + if ready { + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + } + return pod +} + +func replacementLifecycleEnvironment() *unstructured.Unstructured { + env := lifecycleAdminEnvironment(2, []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}}) + q := adminRequestFixture() + _ = unstructured.SetNestedMap(env.Object, terminationProof(q, terminalExecutor()), "status", "terminationProof") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": q.OperationID, "type": "ReplaceExecutor", "phase": "CreatingReplacement", "expectedEpoch": int64(4), "expectedPodUID": q.ExpectedPodUID, "expectedPVCUID": q.ExpectedPVCUID, "createdAt": "2026-09-21T00:00:00Z"}, "status", "lifecycleOperation") + unstructured.RemoveNestedField(env.Object, "status", "pod") + setConditionObject(env, "Ready", false, "ReplacementStarting", "waiting") + return env +} + +func TestStaleReplacementObservationCannotOverwriteCompletion(t *testing.T) { + stale := replacementLifecycleEnvironment() + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), stale.DeepCopy()) + kube := kubefake.NewSimpleClientset(replacementExecutor(false), retainedPVC()) + observed, release := make(chan struct{}), make(chan struct{}) + kube.PrependReactor("get", "pods", func(k8stesting.Action) (bool, runtime.Object, error) { + close(observed) + <-release + return false, nil, nil + }) + r := NewReconciler(dynamicClient, kube, "ns", testProfiles()) + done := make(chan error, 1) + go func() { done <- r.finishReplacement(t.Context(), stale, adminRequestFixture().OperationID) }() + <-observed + + current, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + _ = unstructured.SetNestedField(current.Object, int64(5), "status", "epoch") + _ = unstructured.SetNestedMap(current.Object, map[string]any{"name": "executor", "uid": "new-pod"}, "status", "pod") + _ = unstructured.SetNestedMap(current.Object, map[string]any{operationIDField: adminRequestFixture().OperationID, "previousPodUID": "pod-uid", "replacementPodUID": "new-pod", "pvcUID": "pvc-uid", "previousEpoch": int64(4), "replacementEpoch": int64(5)}, "status", "lastReplacement") + unstructured.RemoveNestedField(current.Object, "status", "lifecycleOperation") + setConditionObject(current, "Ready", true, "ReplacementReady", "replacement executor is ready on the retained workspace") + if _, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").UpdateStatus(t.Context(), current, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + close(release) + var controlled *executionenv.Error + if err := <-done; !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("stale replacement result=%v", err) + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if !conditionTrue(got, "Ready") || intNested(got.Object, "status", "epoch") != 5 || textNested(got.Object, "status", "pod", "uid") != "new-pod" { + t.Fatalf("stale unready write damaged completion: %v", got.Object["status"]) + } + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + if allocation, err := store.Attach(t.Context(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "binding"); err != nil || !allocation.Ready || allocation.Epoch != 5 { + t.Fatalf("attach after completed replacement: allocation=%+v err=%v", allocation, err) + } +} + +func TestLifecycleStatusWritersRejectStaleObservations(t *testing.T) { + t.Run("phase regression", func(t *testing.T) { + stale := replacementLifecycleEnvironment() + _ = unstructured.SetNestedField(stale.Object, "Quiescing", "status", "lifecycleOperation", "phase") + current := stale.DeepCopy() + _ = unstructured.SetNestedField(current.Object, "RemovingPodFinalizer", "status", "lifecycleOperation", "phase") + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), current) + r := NewReconciler(client, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if err := r.setLifecyclePhase(t.Context(), stale, adminRequestFixture().OperationID, "Quiescing", "WaitingForTermination"); err == nil { + t.Fatal("stale phase transition succeeded") + } + got, _ := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if phase := textNested(got.Object, "status", "lifecycleOperation", "phase"); phase != "RemovingPodFinalizer" { + t.Fatalf("phase regressed to %q", phase) + } + }) + + t.Run("waiting and fence writes", func(t *testing.T) { + stale := replacementLifecycleEnvironment() + _ = unstructured.SetNestedField(stale.Object, "WaitingForTermination", "status", "lifecycleOperation", "phase") + current := stale.DeepCopy() + _ = unstructured.SetNestedField(current.Object, "RemovingPodFinalizer", "status", "lifecycleOperation", "phase") + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), current) + r := NewReconciler(client, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if err := r.setLifecycleCondition(t.Context(), stale, "Ready", false, "AwaitingTerminalExecutor", "waiting"); err == nil { + t.Fatal("stale waiting condition succeeded") + } + if err := r.setLifecycleFenceUnknown(t.Context(), stale, "stale failure"); err == nil { + t.Fatal("stale fence write succeeded") + } + got, _ := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "fenceState") != fenceHealthy || textNested(got.Object, "status", "lifecycleOperation", "phase") != "RemovingPodFinalizer" { + t.Fatalf("stale writer changed lifecycle status: %v", got.Object["status"]) + } + }) + + t.Run("ordinary reconcile after admission", func(t *testing.T) { + stale := lifecycleAdminEnvironment(2, []any{}) + current := stale.DeepCopy() + _ = unstructured.SetNestedMap(current.Object, map[string]any{"id": "replace", "type": "ReplaceExecutor", "phase": "Quiescing", "expectedEpoch": int64(4), "expectedPodUID": "pod-uid", "expectedPVCUID": "pvc-uid", "createdAt": "2026-09-21T00:00:00Z"}, "status", "lifecycleOperation") + setConditionObject(current, "Ready", false, "Quiescing", "replacement admitted") + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), current) + r := NewReconciler(client, kubefake.NewSimpleClientset(), "ns", testProfiles()) + if err := r.updateRuntimeStatus(t.Context(), stale, retainedPVC(), terminalExecutor(), true); err == nil { + t.Fatal("ordinary reconcile overwrote lifecycle admission") + } + got, _ := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if conditionTrue(got, "Ready") || textNested(got.Object, "status", "lifecycleOperation", "id") != "replace" { + t.Fatalf("ordinary reconcile damaged lifecycle admission: %v", got.Object["status"]) + } + }) +} + +func TestWrongPVCUIDCannotStartRetainedDeletion(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(dynamicClient, "ns", testProfiles(), nil) + q := adminRequestFixture() + q.ExpectedPVCUID = "wrong" + if err := store.DeleteRetiredEnvironment(t.Context(), q); err == nil { + t.Fatal("wrong PVC UID admitted for deletion") + } + got, _ := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if textNested(got.Object, "status", "lifecycleOperation", "id") != "" { + t.Fatalf("wrong UID created delete operation: %v", got.Object["status"]) + } +} diff --git a/internal/adapter/executioncontroller/profiles.go b/internal/adapter/executioncontroller/profiles.go new file mode 100644 index 0000000000..ea9a46d2cd --- /dev/null +++ b/internal/adapter/executioncontroller/profiles.go @@ -0,0 +1,198 @@ +package executioncontroller + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "sort" + "strings" + "time" + + "github.com/goccy/go-yaml" + "k8s.io/apimachinery/pkg/api/resource" +) + +// ProfilesFile is the strict operator profile-file schema. +type ProfilesFile struct { + Profiles map[string]ProfileSpec `yaml:"profiles"` +} + +// ProfileSpec defines immutable image, storage, resource, and operation bounds. +type ProfileSpec struct { + Image string `yaml:"image"` + StorageClass string `yaml:"storageClass"` + StorageSize string `yaml:"storageSize"` + CPURequest string `yaml:"cpuRequest"` + MemoryRequest string `yaml:"memoryRequest"` + CPULimit string `yaml:"cpuLimit"` + MemoryLimit string `yaml:"memoryLimit"` + EphemeralStorageRequest string `yaml:"ephemeralStorageRequest"` + EphemeralStorageLimit string `yaml:"ephemeralStorageLimit"` + TmpSizeLimit string `yaml:"tmpSizeLimit"` + RuntimeClassName string `yaml:"runtimeClassName"` + MaxFileBytes int64 `yaml:"maxFileBytes"` + MaxCommandBytes int64 `yaml:"maxCommandBytes"` + MaxCommandDuration time.Duration `yaml:"maxCommandDuration"` + MaxEnvironments int `yaml:"maxEnvironments"` +} + +// Profiles is a validated immutable profile registry. +type Profiles struct{ byName map[string]resolvedProfile } +type resolvedProfile struct { + Spec ProfileSpec + Digest string + StorageSize resource.Quantity + CPURequest resource.Quantity + MemoryRequest resource.Quantity + CPULimit resource.Quantity + MemoryLimit resource.Quantity + EphemeralStorageRequest resource.Quantity + EphemeralStorageLimit resource.Quantity + TmpSizeLimit resource.Quantity +} + +// LoadProfiles reads and strictly validates an operator profile file. +func LoadProfiles(path string) (*Profiles, error) { + b, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read profiles: %w", err) + } + var f ProfilesFile + if err := yaml.UnmarshalWithOptions(b, &f, yaml.DisallowUnknownField()); err != nil { + return nil, fmt.Errorf("decode profiles: %w", err) + } + if len(f.Profiles) == 0 { + return nil, errors.New("profiles file has no profiles") + } + p := &Profiles{byName: map[string]resolvedProfile{}} + for name, s := range f.Profiles { + quantities, err := validateProfile(name, s) + if err != nil { + return nil, err + } + canonical, err := yaml.Marshal(s) + if err != nil { + return nil, err + } + sum := sha256.Sum256(canonical) + quantities.Spec = s + quantities.Digest = "sha256:" + hex.EncodeToString(sum[:]) + p.byName[name] = quantities + } + return p, nil +} +func validateProfile(name string, s ProfileSpec) (resolvedProfile, error) { + if name == "" || strings.ContainsAny(name, "/\\") { + return resolvedProfile{}, fmt.Errorf("invalid profile name %q", name) + } + if !strings.Contains(s.Image, "@sha256:") { + return resolvedProfile{}, fmt.Errorf("profile %q image must be digest-pinned", name) + } + if s.StorageClass == "" || s.StorageSize == "" { + return resolvedProfile{}, fmt.Errorf("profile %q requires explicit storage class and size", name) + } + if s.CPURequest == "" || s.MemoryRequest == "" || s.CPULimit == "" || s.MemoryLimit == "" || s.EphemeralStorageRequest == "" || s.EphemeralStorageLimit == "" || s.TmpSizeLimit == "" || s.RuntimeClassName == "" { + return resolvedProfile{}, fmt.Errorf("profile %q requires explicit resource requests, limits, tmp size, and runtime class", name) + } + quantities, err := resolveProfileQuantities(name, s) + if err != nil { + return resolvedProfile{}, err + } + if !validExecutionBounds(s) { + return resolvedProfile{}, fmt.Errorf("profile %q has invalid execution bounds", name) + } + return quantities, nil +} + +func resolveProfileQuantities(name string, s ProfileSpec) (resolvedProfile, error) { + storage, err := positiveQuantity(name, "storageSize", s.StorageSize) + if err != nil { + return resolvedProfile{}, err + } + cpuRequest, err := positiveQuantity(name, "cpuRequest", s.CPURequest) + if err != nil { + return resolvedProfile{}, err + } + memoryRequest, err := positiveQuantity(name, "memoryRequest", s.MemoryRequest) + if err != nil { + return resolvedProfile{}, err + } + cpuLimit, err := positiveQuantity(name, "cpuLimit", s.CPULimit) + if err != nil { + return resolvedProfile{}, err + } + memoryLimit, err := positiveQuantity(name, "memoryLimit", s.MemoryLimit) + if err != nil { + return resolvedProfile{}, err + } + ephemeralRequest, err := positiveQuantity(name, "ephemeralStorageRequest", s.EphemeralStorageRequest) + if err != nil { + return resolvedProfile{}, err + } + ephemeralLimit, err := positiveQuantity(name, "ephemeralStorageLimit", s.EphemeralStorageLimit) + if err != nil { + return resolvedProfile{}, err + } + tmpLimit, err := positiveQuantity(name, "tmpSizeLimit", s.TmpSizeLimit) + if err != nil { + return resolvedProfile{}, err + } + if cpuRequest.Cmp(cpuLimit) > 0 { + return resolvedProfile{}, fmt.Errorf("profile %q cpuRequest exceeds cpuLimit", name) + } + if memoryRequest.Cmp(memoryLimit) > 0 { + return resolvedProfile{}, fmt.Errorf("profile %q memoryRequest exceeds memoryLimit", name) + } + if ephemeralRequest.Cmp(ephemeralLimit) > 0 || tmpLimit.Cmp(ephemeralLimit) > 0 { + return resolvedProfile{}, fmt.Errorf("profile %q ephemeral storage request or tmp limit exceeds ephemeral storage limit", name) + } + return resolvedProfile{StorageSize: storage, CPURequest: cpuRequest, MemoryRequest: memoryRequest, CPULimit: cpuLimit, MemoryLimit: memoryLimit, EphemeralStorageRequest: ephemeralRequest, EphemeralStorageLimit: ephemeralLimit, TmpSizeLimit: tmpLimit}, nil +} + +func validExecutionBounds(s ProfileSpec) bool { + return s.MaxFileBytes > 0 && s.MaxFileBytes <= 5<<20 && + s.MaxCommandBytes > 0 && s.MaxCommandBytes <= 1<<20 && + s.MaxCommandDuration > 0 && s.MaxCommandDuration <= 30*time.Minute && + s.MaxEnvironments > 0 && s.MaxEnvironments <= 10_000 +} + +func positiveQuantity(profile, field, value string) (resource.Quantity, error) { + quantity, err := resource.ParseQuantity(value) + if err != nil { + return resource.Quantity{}, fmt.Errorf("profile %q %s is invalid: %w", profile, field, err) + } + if quantity.Sign() <= 0 { + return resource.Quantity{}, fmt.Errorf("profile %q %s must be positive", profile, field) + } + return quantity, nil +} +func (p *Profiles) get(name string) (resolvedProfile, bool) { + if p == nil { + return resolvedProfile{}, false + } + v, ok := p.byName[name] + return v, ok +} + +func (p *Profiles) clusterResources() (runtimeClasses, storageClasses []string) { + if p == nil { + return nil, nil + } + runtimes := make(map[string]struct{}, len(p.byName)) + storage := make(map[string]struct{}, len(p.byName)) + for _, profile := range p.byName { + runtimes[profile.Spec.RuntimeClassName] = struct{}{} + storage[profile.Spec.StorageClass] = struct{}{} + } + for name := range runtimes { + runtimeClasses = append(runtimeClasses, name) + } + for name := range storage { + storageClasses = append(storageClasses, name) + } + sort.Strings(runtimeClasses) + sort.Strings(storageClasses) + return runtimeClasses, storageClasses +} diff --git a/internal/adapter/executioncontroller/profiles_test.go b/internal/adapter/executioncontroller/profiles_test.go new file mode 100644 index 0000000000..563f95ea20 --- /dev/null +++ b/internal/adapter/executioncontroller/profiles_test.go @@ -0,0 +1,101 @@ +package executioncontroller + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadProfilesStrictAndDigestPinned(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "profiles.yaml") + good := validProfileYAML() + if err := os.WriteFile(path, []byte(good), 0o600); err != nil { + t.Fatal(err) + } + p, err := LoadProfiles(path) + if err != nil { + t.Fatal(err) + } + v, ok := p.get("go") + if !ok || !strings.HasPrefix(v.Digest, "sha256:") { + t.Fatalf("profile=%+v", v) + } + if err := os.WriteFile(path, []byte(good+" unknown: true\n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadProfiles(path); err == nil { + t.Fatal("unknown field accepted") + } + unpinned := strings.Replace(good, "@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "", 1) + _ = os.WriteFile(path, []byte(unpinned), 0o600) + if _, err := LoadProfiles(path); err == nil { + t.Fatal("unpinned image accepted") + } +} + +func TestLoadProfilesRejectsInvalidQuantities(t *testing.T) { + cases := map[string]string{ + "malformed": strings.Replace(validProfileYAML(), "cpuRequest: 100m", "cpuRequest: invalid", 1), + "zero": strings.Replace(validProfileYAML(), "memoryRequest: 128Mi", `memoryRequest: "0"`, 1), + "negative": strings.Replace(validProfileYAML(), `cpuLimit: "1"`, `cpuLimit: "-1"`, 1), + "cpu request over limit": strings.Replace(validProfileYAML(), "cpuRequest: 100m", "cpuRequest: 2", 1), + "memory request over limit": strings.Replace(validProfileYAML(), "memoryLimit: 1Gi", "memoryLimit: 64Mi", 1), + "ephemeral request over limit": strings.Replace(validProfileYAML(), "ephemeralStorageLimit: 1Gi", "ephemeralStorageLimit: 32Mi", 1), + "tmp over ephemeral limit": strings.Replace(validProfileYAML(), "tmpSizeLimit: 256Mi", "tmpSizeLimit: 2Gi", 1), + } + for name, content := range cases { + t.Run(name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "profiles.yaml") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + if _, err := LoadProfiles(path); err == nil { + t.Fatal("invalid quantity accepted") + } + }) + } +} + +func TestLoadProfilesAcceptsQuantityUnitVariants(t *testing.T) { + content := validProfileYAML() + content = strings.Replace(content, "storageSize: 2Gi", "storageSize: 500M", 1) + content = strings.Replace(content, "cpuRequest: 100m", "cpuRequest: 0.1", 1) + content = strings.Replace(content, `cpuLimit: "1"`, "cpuLimit: 250m", 1) + content = strings.Replace(content, "memoryRequest: 128Mi", "memoryRequest: 1M", 1) + content = strings.Replace(content, "memoryLimit: 1Gi", "memoryLimit: 2M", 1) + path := filepath.Join(t.TempDir(), "profiles.yaml") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + profiles, err := LoadProfiles(path) + if err != nil { + t.Fatal(err) + } + profile, ok := profiles.get("go") + if !ok || profile.CPURequest.String() != "100m" || profile.MemoryRequest.String() != "1M" || profile.StorageSize.String() != "500M" { + t.Fatalf("parsed quantities=%+v", profile) + } +} + +func validProfileYAML() string { + return `profiles: + go: + image: ghcr.io/example/workload@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + storageClass: standard + storageSize: 2Gi + cpuRequest: 100m + memoryRequest: 128Mi + cpuLimit: "1" + memoryLimit: 1Gi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: sandboxed + maxFileBytes: 1048576 + maxCommandBytes: 65536 + maxCommandDuration: 1m + maxEnvironments: 100 +` +} diff --git a/internal/adapter/executioncontroller/provisioning_retry_test.go b/internal/adapter/executioncontroller/provisioning_retry_test.go new file mode 100644 index 0000000000..d69a7f1476 --- /dev/null +++ b/internal/adapter/executioncontroller/provisioning_retry_test.go @@ -0,0 +1,135 @@ +package executioncontroller + +import ( + "errors" + "sync/atomic" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/client-go/util/workqueue" +) + +func TestProvisioningWorkerConvergesWithoutAnotherEvent(t *testing.T) { + for _, resource := range []string{"persistentvolumeclaims", "pods"} { + t.Run(resource, func(t *testing.T) { + ctx := t.Context() + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + var attempts, pvcCreates, podCreates atomic.Int32 + var restored atomic.Bool + k.PrependReactor("create", "*", func(action k8stesting.Action) (bool, runtime.Object, error) { + if action.GetResource().Resource == resource { + attempts.Add(1) + if !restored.Load() { + return true, nil, errors.New("transient quota saturation") + } + } + switch obj := action.(k8stesting.CreateAction).GetObject().(type) { + case *corev1.PersistentVolumeClaim: + pvcCreates.Add(1) + obj.UID = "pvc-stable" + case *corev1.Pod: + podCreates.Add(1) + obj.UID = "pod-stable" + obj.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + } + return false, nil, nil + }) + r := NewReconciler(d, k, "ns", testProfiles()) + r.queue.ShutDown() + r.queue = workqueue.NewTypedRateLimitingQueue(workqueue.NewTypedItemExponentialFailureRateLimiter[string](10*time.Millisecond, 10*time.Millisecond)) + done := make(chan struct{}) + go func() { r.worker(ctx); close(done) }() + t.Cleanup(func() { r.queue.ShutDown(); <-done }) + start := time.Now() + r.queue.Add(env.GetName()) + // Exceed a customary finite retry budget before restoring admission. + waitProvisioning(t, func() bool { return attempts.Load() >= 24 }) + if time.Since(start) < 200*time.Millisecond { + t.Fatal("provisioning retries hot-looped instead of rate limiting") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + if err != nil || conditionTrue(got, "Ready") || conditionTransition(got, "Ready") == "" { + t.Fatalf("failed provisioning did not retain Ready=false: env=%v err=%v", got, err) + } + if writes := statusUpdateCount(d.Actions()); writes != 1 { + t.Fatalf("unchanged failure rewrote status: writes=%d", writes) + } + // No CR, runtime, or queue event: only the failed external prerequisite changes. + restored.Store(true) + waitProvisioning(t, func() bool { + got, err = d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(ctx, env.GetName(), metav1.GetOptions{}) + return err == nil && conditionTrue(got, "Ready") && r.queue.NumRequeues(env.GetName()) == 0 + }) + if textNested(got.Object, "status", "pvc", "uid") != "pvc-stable" || textNested(got.Object, "status", "pod", "uid") != "pod-stable" || pvcCreates.Load() != 1 || podCreates.Load() != 1 { + t.Fatalf("duplicate or changed runtime identity: pvc=%d pod=%d status=%v", pvcCreates.Load(), podCreates.Load(), got.Object["status"]) + } + }) + } +} + +func waitProvisioning(t *testing.T, ready func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if ready() { + return + } + time.Sleep(time.Millisecond) + } + t.Fatal("worker did not converge without another event") +} + +func TestProvisioningPreservesCreateAndConditionErrors(t *testing.T) { + for _, resource := range []string{"persistentvolumeclaims", "pods"} { + t.Run(resource, func(t *testing.T) { + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + createErr, statusErr := errors.New("create unavailable"), errors.New("status unavailable") + k.PrependReactor("create", resource, func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, createErr }) + d.PrependReactor("update", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, statusErr }) + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + if err := r.Reconcile(t.Context(), env.GetName()); !errors.Is(err, createErr) || !errors.Is(err, statusErr) { + t.Fatalf("lost provisioning/status error: %v", err) + } + }) + } +} + +func TestProvisioningRetriesNeverReplaceAuthoritativeResources(t *testing.T) { + for _, resource := range []string{"pvc", "pod"} { + t.Run(resource, func(t *testing.T) { + env := testEnvironment() + env.SetFinalizers([]string{environmentFinalizer}) + if err := unstructured.SetNestedField(env.Object, "missing-authority", "status", resource, "uid"); err != nil { + t.Fatal(err) + } + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + for range 3 { + if err := r.Reconcile(t.Context(), env.GetName()); err == nil { + t.Fatal("missing authoritative resource was not rejected") + } + } + for _, action := range k.Actions() { + if action.GetVerb() == "create" && (resource == "pvc" || action.GetResource().Resource == "pods") { + t.Fatalf("replaced missing authority: %v", action) + } + } + }) + } +} diff --git a/internal/adapter/executioncontroller/retained_delete_race_test.go b/internal/adapter/executioncontroller/retained_delete_race_test.go new file mode 100644 index 0000000000..3be3ce5011 --- /dev/null +++ b/internal/adapter/executioncontroller/retained_delete_race_test.go @@ -0,0 +1,215 @@ +package executioncontroller + +import ( + "errors" + "strconv" + "testing" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" +) + +func TestRetainedDeletePeerBetweenFinalizerUpdateAndDelete(t *testing.T) { + for _, mode := range []string{"peer-before-delete", "already-terminating", "stale-resource-version"} { + t.Run(mode, func(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + env.SetResourceVersion("1") + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + // The tracker does not implement API-server resource versions or CAS. + versionedUpdate := func(a ktesting.Action) (bool, runtime.Object, error) { + obj, err := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + if err != nil { + return true, nil, err + } + cur := obj.(*unstructured.Unstructured) + next := a.(ktesting.UpdateAction).GetObject().(*unstructured.Unstructured).DeepCopy() + version, err := strconv.Atoi(cur.GetResourceVersion()) + if err != nil || version < 1 || next.GetResourceVersion() == "" { + t.Fatal("fixture lost API-server resource version") + } + if next.GetResourceVersion() != cur.GetResourceVersion() { + return true, nil, apierrors.NewConflict(ExecutionEnvironmentGVR.GroupResource(), "env", errors.New("stale update")) + } + if !contains(cur.GetFinalizers(), environmentFinalizer) && contains(next.GetFinalizers(), environmentFinalizer) { + t.Fatal("peer re-added finalizer during admitted retained deletion") + } + next.SetResourceVersion(strconv.Itoa(version + 1)) + return true, next, d.Tracker().Update(ExecutionEnvironmentGVR, next, "ns") + } + d.PrependReactor("update", "executionenvironments", versionedUpdate) + k := kubefake.NewSimpleClientset(retainedPVC(), &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{profileAllocationKey("go"): `["env"]`}}) + store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) + if err := store.DeleteRetiredEnvironment(t.Context(), adminRequestFixture()); err != nil { + t.Fatal(err) + } + // Separate fake clients avoid the fake's per-client reactor lock, while both + // reconcilers observe the same API-server tracker. + peerClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + peerClient.PrependReactor("*", "*", ktesting.ObjectReaction(d.Tracker())) + peerClient.PrependReactor("update", "executionenvironments", versionedUpdate) + peer := NewReconciler(peerClient, k, "ns", testProfiles()) + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(peer.queue.ShutDown) + t.Cleanup(r.queue.ShutDown) + pvcsDeleted, envsDeleted := 0, 0 + k.PrependReactor("delete", "persistentvolumeclaims", func(a ktesting.Action) (bool, runtime.Object, error) { + opts := a.(ktesting.DeleteAction).GetDeleteOptions() + if opts.Preconditions == nil || opts.Preconditions.UID == nil || *opts.Preconditions.UID != "pvc-uid" { + t.Fatal("PVC deletion lost exact UID") + } + pvcsDeleted++ + return false, nil, nil + }) + staleDeletes := 0 + deletion := func(a ktesting.Action) (bool, runtime.Object, error) { + opts := a.(ktesting.DeleteAction).GetDeleteOptions() + if opts.Preconditions == nil || opts.Preconditions.ResourceVersion == nil || *opts.Preconditions.ResourceVersion == "" { + t.Fatal("CR deletion lost nonempty resource version") + } + obj, err := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + if err != nil { + return true, nil, err + } + cur := obj.(*unstructured.Unstructured) + if opts.Preconditions.UID == nil || *opts.Preconditions.UID != cur.GetUID() { + t.Fatal("CR deletion lost exact UID") + } + if *opts.Preconditions.ResourceVersion != cur.GetResourceVersion() { + staleDeletes++ + return true, nil, apierrors.NewConflict(ExecutionEnvironmentGVR.GroupResource(), "env", errors.New("stale delete")) + } + if contains(cur.GetFinalizers(), environmentFinalizer) { + t.Fatal("admitted retained deletion re-added finalizer") + } + envsDeleted++ + return true, nil, d.Tracker().Delete(ExecutionEnvironmentGVR, "ns", "env") + } + peerClient.PrependReactor("delete", "executionenvironments", deletion) + interleaved := false + d.PrependReactor("delete", "executionenvironments", func(a ktesting.Action) (bool, runtime.Object, error) { + if !interleaved { + interleaved = true + if mode == "stale-resource-version" { + res := peerClient.Resource(ExecutionEnvironmentGVR).Namespace("ns") + cur, err := res.Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + cur.SetAnnotations(map[string]string{"peer-observation": "updated"}) + if _, err := res.Update(t.Context(), cur, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + } else if err := peer.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + } + return deletion(a) + }) + if mode == "already-terminating" { + obj, _ := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + cur := obj.(*unstructured.Unstructured) + now := metav1.Now() + cur.SetDeletionTimestamp(&now) + if _, err := peerClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Update(t.Context(), cur, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + } + conflicts := 0 + for range 5 { + if err := r.Reconcile(t.Context(), "env"); err != nil { + if mode != "stale-resource-version" || !apierrors.IsConflict(err) { + t.Fatal(err) + } + conflicts++ + cur, getErr := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env") + if getErr != nil || contains(cur.(*unstructured.Unstructured).GetFinalizers(), environmentFinalizer) || envsDeleted != 0 { + t.Fatal("stale deletion must preserve CR without re-adding finalizer") + } + } + } + wantConflicts := 0 + if mode == "stale-resource-version" { + wantConflicts = 1 + } + if !interleaved || staleDeletes != wantConflicts || conflicts != wantConflicts { + t.Fatalf("interleaved=%v stale deletes=%d returned conflicts=%d", interleaved, staleDeletes, conflicts) + } + if _, err := d.Tracker().Get(ExecutionEnvironmentGVR, "ns", "env"); !apierrors.IsNotFound(err) { + t.Fatalf("authorized deletion stranded: %v", err) + } + if pvcsDeleted != 1 || envsDeleted != 1 { + t.Fatalf("successful deletes PVC=%d CR=%d", pvcsDeleted, envsDeleted) + } + if len(profileSlots(t, k)) != 0 { + t.Fatal("capacity was not released") + } + }) + } +} + +func TestRetainedDeleteDoesNotRemoveForeignCR(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset() + store := NewStore(d, "ns", testProfiles(), nil).WithKubeClient(k) + if err := store.DeleteRetiredEnvironment(t.Context(), adminRequestFixture()); err != nil { + t.Fatal(err) + } + env, _ = d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + _ = unstructured.SetNestedField(env.Object, "ReleasingSlot", "status", "lifecycleOperation", "phase") + foreign := env.DeepCopy() + foreign.SetUID("foreign") + if err := d.Tracker().Update(ExecutionEnvironmentGVR, foreign, "ns"); err != nil { + t.Fatal(err) + } + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + op, _, _ := unstructured.NestedMap(env.Object, "status", "lifecycleOperation") + if err := r.reconcileRetainedDelete(t.Context(), env, op, "workspace", "pvc-uid"); err == nil { + t.Fatal("foreign CR did not conflict") + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil || got.GetUID() != "foreign" || !contains(got.GetFinalizers(), environmentFinalizer) { + t.Fatal("foreign CR touched") + } + for _, a := range k.Actions() { + if a.GetVerb() != "get" { + t.Fatal("foreign CR released capacity") + } + } +} + +func TestDirectCRDeleteStillBlocked(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + now := metav1.Now() + env.SetDeletionTimestamp(&now) + d := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + k := kubefake.NewSimpleClientset(retainedPVC(), terminalExecutor()) + k.PrependReactor("delete", "*", func(ktesting.Action) (bool, runtime.Object, error) { + t.Error("direct delete destroyed runtime") + return true, nil, apierrors.NewForbidden(schema.GroupResource{}, "", errors.New("unexpected")) + }) + r := NewReconciler(d, k, "ns", testProfiles()) + t.Cleanup(r.queue.ShutDown) + if err := r.Reconcile(t.Context(), "env"); err != nil { + t.Fatal(err) + } + got, err := d.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil || !contains(got.GetFinalizers(), environmentFinalizer) { + t.Fatal("direct deletion lost finalizer") + } + if !conditionTrue(got, "DeletionBlocked") { + t.Fatal("direct deletion not blocked") + } +} diff --git a/internal/adapter/executioncontroller/security.go b/internal/adapter/executioncontroller/security.go new file mode 100644 index 0000000000..11021843de --- /dev/null +++ b/internal/adapter/executioncontroller/security.go @@ -0,0 +1,715 @@ +package executioncontroller + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "maps" + "math" + "net/url" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync/atomic" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +const ( + securityManifestVersion = 1 + maxSecurityKeys = 64 + maxSecurityClients = 256 + securityStateDataKey = "state.json" + keyStateRevoked = "revoked" +) + +type securityKeyManifest struct { + ID string `json:"id"` + Version uint64 `json:"version"` + File string `json:"file"` + PublicSHA256 string `json:"publicKeySHA256"` + ActivateAt time.Time `json:"activateAt"` + VerifyUntil time.Time `json:"verifyUntil"` + State string `json:"state"` +} + +type securityClientManifest struct { + URI string `json:"uri"` + MayAttestOwner bool `json:"mayAttestOwner"` + Administrator bool `json:"administrator"` + AdministratorFor []string `json:"administratorFor,omitempty"` +} + +type securityTLSManifest struct { + CertificateFile string `json:"certificateFile"` + PrivateKeyFile string `json:"privateKeyFile"` + ClientCAFile string `json:"clientCAFile"` +} + +type securityManifest struct { + Version int `json:"version"` + Generation uint64 `json:"generation"` + Issuer string `json:"issuer"` + Audience string `json:"audience"` + ActiveKeyID string `json:"activeKeyID"` + GrantTTL time.Duration `json:"-"` + GrantTTLText string `json:"grantTTL"` + ClockSkewText string `json:"clockSkew"` + Keys []securityKeyManifest `json:"keys"` + TLS securityTLSManifest `json:"tls"` + Clients []securityClientManifest `json:"clients"` +} + +type keyValidity struct { + activateAt, verifyUntil time.Time + state string +} + +type securitySnapshot struct { + generation uint64 + digest string + signer GrantSigner + verifier executionenv.GrantVerifier + keyValidity map[string]keyValidity + activeWindow keyValidity + tlsConfig *tls.Config + clientCAs *x509.CertPool + clients map[string]ClientPolicy + validUntil time.Time + fingerprints map[string]string +} + +type securityLedger struct { + Generation uint64 `json:"generation"` + Digest string `json:"digest"` + Fingerprints map[string]string `json:"fingerprints"` +} + +type securityState struct { + snapshot *securitySnapshot +} + +// SecurityManager reloads one immutable, versioned security snapshot from +// projected files and binds its generation to a durable ConfigMap high-water mark. +type SecurityManager struct { + manifestPath, keyDirectory, namespace, configMap string + kube kubernetes.Interface + now func() time.Time + state atomic.Pointer[securityState] +} + +// NewSecurityManager constructs a fail-closed security material reloader. +func NewSecurityManager(manifestPath, keyDirectory, namespace, configMap string, kube kubernetes.Interface) *SecurityManager { + return &SecurityManager{manifestPath: manifestPath, keyDirectory: keyDirectory, namespace: namespace, configMap: configMap, kube: kube, now: func() time.Time { return time.Now().UTC() }} +} + +// Ready reports whether the current snapshot is authoritative and unexpired. +func (m *SecurityManager) Ready() bool { + state := m.state.Load() + return state != nil && snapshotValidAt(state.snapshot, m.now()) +} + +// CheckReady verifies that the loaded snapshot still matches durable authority. +func (m *SecurityManager) CheckReady(ctx context.Context) bool { + _, err := m.authoritative(ctx) + return err == nil +} + +// Reload atomically validates and publishes a complete security snapshot. +func (m *SecurityManager) Reload(ctx context.Context) error { + observed := m.state.Load() + candidate, err := m.load() + if err != nil { + m.invalidateObservedUnlessReplaced(ctx, observed) + return err + } + if err := m.publishGeneration(ctx, candidate); err != nil { + m.invalidateObservedUnlessReplaced(ctx, observed) + return err + } + if err := m.verifyAuthority(ctx, candidate); err != nil { + m.invalidateObservedUnlessReplaced(ctx, observed) + return err + } + for { + current := m.state.Load() + if current != nil && current.snapshot.generation > candidate.generation { + return errors.New("security manifest generation rollback rejected") + } + if m.state.CompareAndSwap(current, &securityState{snapshot: candidate}) { + return nil + } + } +} + +// Run periodically reloads security material until ctx is cancelled. +func (m *SecurityManager) Run(ctx context.Context, interval time.Duration) { + if interval <= 0 { + interval = 2 * time.Second + } + if interval > time.Minute { + interval = time.Minute + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + _ = m.Reload(ctx) + } + } +} + +// TLSConfig returns a dynamic TLS configuration backed by the current snapshot. +func (m *SecurityManager) TLSConfig() *tls.Config { + return &tls.Config{MinVersion: tls.VersionTLS13, ClientAuth: tls.RequireAnyClientCert, GetConfigForClient: func(*tls.ClientHelloInfo) (*tls.Config, error) { + state := m.state.Load() + if state == nil || !snapshotValidAt(state.snapshot, m.now()) { + return nil, errors.New("security material is not ready") + } + return state.snapshot.tlsConfig.Clone(), nil + }} +} + +func (m *SecurityManager) authorize(ctx context.Context, chain []*x509.Certificate) (string, ClientPolicy, error) { + if len(chain) == 0 || chain[0] == nil { + return "", ClientPolicy{}, errors.New("client certificate is unavailable") + } + now := m.now() + s, err := m.authoritativeAt(ctx, now) + if err != nil { + return "", ClientPolicy{}, err + } + intermediates := x509.NewCertPool() + for _, cert := range chain[1:] { + if cert != nil { + intermediates.AddCert(cert) + } + } + opts := x509.VerifyOptions{Roots: s.clientCAs, Intermediates: intermediates, CurrentTime: now, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}} + leaf := chain[0] + if _, err := leaf.Verify(opts); err != nil { + return "", ClientPolicy{}, err + } + id, err := canonicalClientIdentity(leaf) + if err != nil { + return "", ClientPolicy{}, err + } + policy, ok := s.clients[id] + if !ok { + return "", ClientPolicy{}, errors.New("client is not authorized") + } + return id, policy, nil +} + +func (m *SecurityManager) material(ctx context.Context) (*securitySnapshot, error) { + return m.authoritative(ctx) +} + +func (m *SecurityManager) authoritative(ctx context.Context) (*securitySnapshot, error) { + return m.authoritativeAt(ctx, m.now()) +} + +var errAuthorityUnavailable = errors.New("security authority is unavailable") + +func authorityUnavailable(err error) error { + if err == nil { + return errAuthorityUnavailable + } + return fmt.Errorf("%w: %v", errAuthorityUnavailable, err) +} + +func (m *SecurityManager) authoritativeAt(ctx context.Context, now time.Time) (*securitySnapshot, error) { + state := m.state.Load() + if state == nil || !snapshotValidAt(state.snapshot, now) || m.kube == nil { + return nil, authorityUnavailable(nil) + } + if err := m.verifyAuthority(ctx, state.snapshot); err != nil { + m.state.CompareAndSwap(state, nil) + return nil, authorityUnavailable(err) + } + return state.snapshot, nil +} + +func (m *SecurityManager) verifyAuthority(ctx context.Context, snapshot *securitySnapshot) error { + if snapshot == nil || m.kube == nil { + return errors.New("security material is not ready") + } + cm, err := m.kube.CoreV1().ConfigMaps(m.namespace).Get(ctx, m.configMap, metav1.GetOptions{}) + if err != nil { + return err + } + var ledger securityLedger + if err := executionenv.DecodeStrict([]byte(cm.Data[securityStateDataKey]), &ledger); err != nil { + return err + } + if ledger.Generation != snapshot.generation || ledger.Digest != snapshot.digest || !maps.Equal(ledger.Fingerprints, snapshot.fingerprints) { + return errors.New("loaded security generation is not authoritative") + } + return nil +} + +func (m *SecurityManager) invalidateObservedUnlessReplaced(ctx context.Context, observed *securityState) { + if m.state.CompareAndSwap(observed, nil) { + return + } + current := m.state.Load() + if current != nil && m.verifyAuthority(ctx, current.snapshot) != nil { + m.state.CompareAndSwap(current, nil) + } +} + +func snapshotValidAt(s *securitySnapshot, now time.Time) bool { + return s != nil && !now.Before(s.activeWindow.activateAt) && now.Before(s.activeWindow.verifyUntil) && now.Before(s.validUntil) +} + +func (s *securitySnapshot) verifierAt(now time.Time) executionenv.GrantVerifier { + v := s.verifier + v.Keys = make(map[string]ed25519.PublicKey, len(s.verifier.Keys)) + for id, key := range s.verifier.Keys { + window := s.keyValidity[id] + if window.state != keyStateRevoked && !now.Before(window.activateAt) && now.Before(window.verifyUntil) { + v.Keys[id] = key + } + } + return v +} + +func (m *SecurityManager) load() (*securitySnapshot, error) { + mf, ttl, skew, err := m.loadManifest() + if err != nil { + return nil, err + } + root, err := os.OpenRoot(m.keyDirectory) + if err != nil { + return nil, fmt.Errorf("open security directory: %w", err) + } + defer func() { _ = root.Close() }() + + keys, active, fingerprints, windows, activeWindow, err := m.loadGrantKeys(root, mf) + if err != nil { + return nil, err + } + tlsConfig, clientCAs, tlsValidUntil, caFingerprints, serverFingerprint, err := m.loadTLS(root, mf.TLS) + if err != nil { + return nil, err + } + clients, err := clientPolicies(mf.Clients) + if err != nil { + return nil, err + } + digest, err := authorityDigest(mf, ttl, skew, fingerprints, caFingerprints, serverFingerprint) + if err != nil { + return nil, err + } + validUntil := tlsValidUntil + now := m.now() + for _, window := range windows { + if window.state != keyStateRevoked && window.verifyUntil.Before(validUntil) { + validUntil = window.verifyUntil + } + if window.state != keyStateRevoked && now.Before(window.activateAt) && window.activateAt.Before(validUntil) { + validUntil = window.activateAt + } + } + return &securitySnapshot{ + generation: mf.Generation, digest: digest, + signer: GrantSigner{ + KeyID: mf.ActiveKeyID, PrivateKey: active, Issuer: mf.Issuer, + Audience: mf.Audience, Lifetime: ttl, + }, + verifier: executionenv.GrantVerifier{ + Keys: keys, Issuer: mf.Issuer, Audience: mf.Audience, MaxLifetime: ttl + skew, + }, + keyValidity: windows, activeWindow: activeWindow, + tlsConfig: tlsConfig, clientCAs: clientCAs, clients: clients, + validUntil: validUntil, fingerprints: fingerprints, + }, nil +} + +func (m *SecurityManager) loadManifest() (securityManifest, time.Duration, time.Duration, error) { + manifestBytes, err := os.ReadFile(m.manifestPath) + if err != nil { + return securityManifest{}, 0, 0, fmt.Errorf("read security manifest: %w", err) + } + var mf securityManifest + if err := executionenv.DecodeStrict(manifestBytes, &mf); err != nil { + return securityManifest{}, 0, 0, fmt.Errorf("decode security manifest: %w", err) + } + if mf.Version != securityManifestVersion || mf.Generation == 0 || mf.Generation > math.MaxInt64 || mf.Issuer == "" || mf.Audience == "" || mf.ActiveKeyID == "" || len(mf.Keys) == 0 || len(mf.Keys) > maxSecurityKeys || len(mf.Clients) == 0 || len(mf.Clients) > maxSecurityClients { + return securityManifest{}, 0, 0, errors.New("security manifest identity or bounds are invalid") + } + ttl, err := time.ParseDuration(mf.GrantTTLText) + if err != nil || ttl <= 0 || ttl > 5*time.Minute { + return securityManifest{}, 0, 0, errors.New("grantTTL must be positive and at most 5m") + } + skew, err := time.ParseDuration(mf.ClockSkewText) + if err != nil || skew < 0 || skew >= ttl { + return securityManifest{}, 0, 0, errors.New("clockSkew must be non-negative and less than grantTTL") + } + return mf, ttl, skew, nil +} + +func (m *SecurityManager) loadGrantKeys(root *os.Root, mf securityManifest) (map[string]ed25519.PublicKey, ed25519.PrivateKey, map[string]string, map[string]keyValidity, keyValidity, error) { + keys := make(map[string]ed25519.PublicKey, len(mf.Keys)) + fingerprints := make(map[string]string, len(mf.Keys)) + windows := make(map[string]keyValidity, len(mf.Keys)) + seenVersion := map[uint64]bool{} + seenID := map[string]bool{} + var active ed25519.PrivateKey + var activeWindow keyValidity + now := m.now() + for _, km := range mf.Keys { + if !validKeyManifest(km, seenID, seenVersion) { + return nil, nil, nil, nil, keyValidity{}, errors.New("security key entry is invalid or duplicated") + } + seenID[km.ID] = true + seenVersion[km.Version] = true + key, err := readEd25519(root, km.File) + if err != nil { + return nil, nil, nil, nil, keyValidity{}, fmt.Errorf("load security key %q: %w", km.ID, err) + } + pub := key.Public().(ed25519.PublicKey) + sum := sha256.Sum256(pub) + fp := hex.EncodeToString(sum[:]) + if !strings.EqualFold(fp, km.PublicSHA256) { + return nil, nil, nil, nil, keyValidity{}, fmt.Errorf("security key %q fingerprint mismatch", km.ID) + } + fingerprints[km.ID+":"+strconv.FormatUint(km.Version, 10)] = fp + window := keyValidity{activateAt: km.ActivateAt, verifyUntil: km.VerifyUntil, state: km.State} + windows[km.ID] = window + if km.State != keyStateRevoked { + keys[km.ID] = pub + } + if km.ID == mf.ActiveKeyID { + if km.State != "active" || now.Before(km.ActivateAt) || !now.Before(km.VerifyUntil) { + return nil, nil, nil, nil, keyValidity{}, errors.New("active grant key is outside its activation window") + } + active = key + activeWindow = window + } + } + if active == nil { + return nil, nil, nil, nil, keyValidity{}, errors.New("active grant key is missing") + } + return keys, active, fingerprints, windows, activeWindow, nil +} + +func validKeyManifest(km securityKeyManifest, seenID map[string]bool, seenVersion map[uint64]bool) bool { + return validKeyID(km.ID) && !seenID[km.ID] && km.Version != 0 && !seenVersion[km.Version] && validBaseName(km.File) && + (km.State == "active" || km.State == "verify-only" || km.State == keyStateRevoked) && + !km.VerifyUntil.IsZero() && km.VerifyUntil.After(km.ActivateAt) +} + +func (m *SecurityManager) loadTLS(root *os.Root, manifest securityTLSManifest) (*tls.Config, *x509.CertPool, time.Time, []string, string, error) { + certPEM, err := readRootFile(root, manifest.CertificateFile) + if err != nil { + return nil, nil, time.Time{}, nil, "", err + } + keyPEM, err := readRootFile(root, manifest.PrivateKeyFile) + if err != nil { + return nil, nil, time.Time{}, nil, "", err + } + serverCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return nil, nil, time.Time{}, nil, "", fmt.Errorf("load TLS identity: %w", err) + } + leaf, err := x509.ParseCertificate(serverCert.Certificate[0]) + if err != nil { + return nil, nil, time.Time{}, nil, "", err + } + now := m.now() + if now.Before(leaf.NotBefore) || !now.Before(leaf.NotAfter) || !hasUsage(leaf.ExtKeyUsage, x509.ExtKeyUsageServerAuth) { + return nil, nil, time.Time{}, nil, "", errors.New("server certificate is not currently valid for server authentication") + } + serverCert.Leaf = leaf + caPEM, err := readRootFile(root, manifest.ClientCAFile) + if err != nil { + return nil, nil, time.Time{}, nil, "", err + } + pool := x509.NewCertPool() + caFingerprints, err := appendCAs(pool, caPEM) + if err != nil { + return nil, nil, time.Time{}, nil, "", err + } + serverSum := sha256.Sum256(leaf.Raw) + cfg := &tls.Config{MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{serverCert}, ClientAuth: tls.RequireAnyClientCert} + return cfg, pool, leaf.NotAfter, caFingerprints, hex.EncodeToString(serverSum[:]), nil +} + +func appendCAs(pool *x509.CertPool, pemBytes []byte) ([]string, error) { + var fingerprints []string + for len(bytes.TrimSpace(pemBytes)) != 0 { + block, rest := pem.Decode(pemBytes) + if block == nil || block.Type != "CERTIFICATE" { + return nil, errors.New("client CA bundle is invalid") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil || !cert.IsCA { + return nil, errors.New("client CA bundle contains an invalid CA certificate") + } + pool.AddCert(cert) + sum := sha256.Sum256(cert.Raw) + fingerprints = append(fingerprints, hex.EncodeToString(sum[:])) + pemBytes = rest + } + if len(fingerprints) == 0 { + return nil, errors.New("client CA bundle has no certificates") + } + slices.Sort(fingerprints) + return fingerprints, nil +} + +func clientPolicies(entries []securityClientManifest) (map[string]ClientPolicy, error) { + clients := make(map[string]ClientPolicy, len(entries)) + for _, entry := range entries { + uri, err := url.Parse(entry.URI) + if err != nil { + return nil, errors.New("client URI is invalid") + } + canonical, err := canonicalClientIdentity(&x509.Certificate{URIs: []*url.URL{uri}}) + if err != nil || canonical != entry.URI { + return nil, errors.New("client URI is not canonical") + } + if _, exists := clients[entry.URI]; exists { + return nil, errors.New("client URI is duplicated") + } + if len(entry.AdministratorFor) > maxSecurityClients || len(entry.AdministratorFor) > 0 && !entry.Administrator { + return nil, errors.New("administrator scope requires administrator authority and at most 256 creators") + } + scope := slices.Clone(entry.AdministratorFor) + slices.Sort(scope) + for i, creator := range scope { + uri, err := url.Parse(creator) + if err != nil || strings.ContainsAny(creator, "*?") || strings.ContainsAny(uri.Path, "*?") { + return nil, errors.New("administrator creator URI is invalid") + } + canonical, err := canonicalClientIdentity(&x509.Certificate{URIs: []*url.URL{uri}}) + if err != nil || canonical != creator || i > 0 && scope[i-1] == creator { + return nil, errors.New("administrator creator URI is not canonical or is duplicated") + } + } + clients[entry.URI] = ClientPolicy{MayAttestOwner: entry.MayAttestOwner, Administrator: entry.Administrator, AdministratorFor: scope} + } + return clients, nil +} + +func authorityDigest(mf securityManifest, ttl, skew time.Duration, keyFingerprints map[string]string, caFingerprints []string, serverFingerprint string) (string, error) { + keys := slices.Clone(mf.Keys) + slices.SortFunc(keys, func(a, b securityKeyManifest) int { return strings.Compare(a.ID, b.ID) }) + clients := slices.Clone(mf.Clients) + for i := range clients { + clients[i].AdministratorFor = slices.Clone(clients[i].AdministratorFor) + slices.Sort(clients[i].AdministratorFor) + } + slices.SortFunc(clients, func(a, b securityClientManifest) int { return strings.Compare(a.URI, b.URI) }) + canonical := struct { + Version int + Generation uint64 + Issuer, Audience, ActiveKeyID string + GrantTTL, ClockSkew int64 + Keys []securityKeyManifest + TLS securityTLSManifest + Clients []securityClientManifest + KeyFingerprints map[string]string + CAFingerprints []string + ServerFingerprint string + }{ + Version: mf.Version, Generation: mf.Generation, Issuer: mf.Issuer, Audience: mf.Audience, ActiveKeyID: mf.ActiveKeyID, + GrantTTL: int64(ttl), ClockSkew: int64(skew), Keys: keys, TLS: mf.TLS, Clients: clients, + KeyFingerprints: keyFingerprints, CAFingerprints: slices.Clone(caFingerprints), ServerFingerprint: serverFingerprint, + } + raw, err := json.Marshal(canonical) + if err != nil { + return "", err + } + sum := sha256.Sum256(raw) + return hex.EncodeToString(sum[:]), nil +} + +func (m *SecurityManager) publishGeneration(ctx context.Context, s *securitySnapshot) error { + if m.kube == nil || m.configMap == "" { + return errors.New("security authority ConfigMap is required") + } + cms := m.kube.CoreV1().ConfigMaps(m.namespace) + for range 5 { + cm, exists, ledger, err := m.readSecurityLedger(ctx, cms) + if err != nil { + return err + } + changed, err := advanceSecurityLedger(&ledger, s) + if err != nil || !changed { + return err + } + raw, err := json.Marshal(ledger) + if err != nil { + return err + } + if cm.Data == nil { + cm.Data = map[string]string{} + } + cm.Data[securityStateDataKey] = string(raw) + if !exists { + _, err = cms.Create(ctx, cm, metav1.CreateOptions{}) + } else { + _, err = cms.Update(ctx, cm, metav1.UpdateOptions{}) + } + if err == nil { + return nil + } + if !apierrors.IsAlreadyExists(err) && !apierrors.IsConflict(err) { + return err + } + } + return errors.New("security authority state changed concurrently") +} + +func (m *SecurityManager) readSecurityLedger(ctx context.Context, cms corev1client.ConfigMapInterface) (*corev1.ConfigMap, bool, securityLedger, error) { + cm, err := cms.Get(ctx, m.configMap, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + cm = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: m.configMap, Namespace: m.namespace}, Data: map[string]string{}} + return cm, false, securityLedger{Fingerprints: map[string]string{}}, nil + } + if err != nil { + return nil, false, securityLedger{}, err + } + ledger := securityLedger{Fingerprints: map[string]string{}} + if raw := cm.Data[securityStateDataKey]; raw != "" { + if err := executionenv.DecodeStrict([]byte(raw), &ledger); err != nil || ledger.Generation == 0 || ledger.Digest == "" || ledger.Fingerprints == nil || len(ledger.Fingerprints) > maxSecurityKeys { + return nil, false, securityLedger{}, errors.New("security authority state is invalid") + } + } + return cm, true, ledger, nil +} + +func advanceSecurityLedger(ledger *securityLedger, s *securitySnapshot) (bool, error) { + if s.generation < ledger.Generation { + return false, errors.New("security manifest generation rollback rejected") + } + if s.generation == ledger.Generation && ledger.Generation != 0 { + if ledger.Digest != s.digest || !containsFingerprints(ledger.Fingerprints, s.fingerprints) { + return false, errors.New("security generation authority drift rejected") + } + s.fingerprints = maps.Clone(ledger.Fingerprints) + return false, nil + } + for identity, fp := range s.fingerprints { + old, exists := ledger.Fingerprints[identity] + if exists && old != fp { + return false, errors.New("security key identity reuse rejected") + } + if !exists && !keyVersionAdvances(ledger.Fingerprints, identity) { + return false, errors.New("security key version rollback rejected") + } + ledger.Fingerprints[identity] = fp + } + if len(ledger.Fingerprints) > maxSecurityKeys { + return false, errors.New("security key tombstone limit reached; rotate issuer") + } + ledger.Generation = s.generation + ledger.Digest = s.digest + s.fingerprints = maps.Clone(ledger.Fingerprints) + return true, nil +} + +func readRootFile(root *os.Root, name string) ([]byte, error) { + if !validBaseName(name) { + return nil, errors.New("security filename must be a basename") + } + f, err := root.Open(name) + if err != nil { + return nil, err + } + b, readErr := io.ReadAll(io.LimitReader(f, 1<<20)) + return b, errors.Join(readErr, f.Close()) +} +func readEd25519(root *os.Root, name string) (ed25519.PrivateKey, error) { + b, err := readRootFile(root, name) + if err != nil { + return nil, err + } + block, rest := pem.Decode(b) + if block == nil || block.Type != "PRIVATE KEY" || len(bytes.TrimSpace(rest)) != 0 { + return nil, errors.New("key must be one PKCS8 PEM block") + } + raw, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, errors.New("invalid PKCS8 key") + } + key, ok := raw.(ed25519.PrivateKey) + if !ok { + return nil, errors.New("key is not Ed25519") + } + return key, nil +} +func validBaseName(v string) bool { return v != "" && filepath.Base(v) == v && v != "." && v != ".." } +func validKeyID(v string) bool { + if len(v) == 0 || len(v) > 64 { + return false + } + for _, r := range v { + if r != '-' && r != '_' && (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r < '0' || r > '9') { + return false + } + } + return true +} +func hasUsage(usages []x509.ExtKeyUsage, wanted x509.ExtKeyUsage) bool { + for _, u := range usages { + if u == wanted || u == x509.ExtKeyUsageAny { + return true + } + } + return false +} +func containsFingerprints(authority, candidate map[string]string) bool { + for identity, fingerprint := range candidate { + if authority[identity] != fingerprint { + return false + } + } + return true +} + +func keyVersionAdvances(authority map[string]string, identity string) bool { + separator := strings.LastIndexByte(identity, ':') + if separator <= 0 { + return false + } + id := identity[:separator] + version, err := strconv.ParseUint(identity[separator+1:], 10, 64) + if err != nil { + return false + } + for previous := range authority { + previousSeparator := strings.LastIndexByte(previous, ':') + if previousSeparator <= 0 || previous[:previousSeparator] != id { + continue + } + previousVersion, err := strconv.ParseUint(previous[previousSeparator+1:], 10, 64) + if err != nil || version <= previousVersion { + return false + } + } + return true +} diff --git a/internal/adapter/executioncontroller/security_limit.go b/internal/adapter/executioncontroller/security_limit.go new file mode 100644 index 0000000000..d172248c50 --- /dev/null +++ b/internal/adapter/executioncontroller/security_limit.go @@ -0,0 +1,83 @@ +package executioncontroller + +import ( + "context" + "crypto/x509" + "errors" + "sync" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +// RPCLimiter bounds active unary RPCs globally and per authorized client. +type RPCLimiter struct { + security *SecurityManager + globalMax int + clientMax int + mu sync.Mutex + global int + clients map[string]int +} + +// NewRPCLimiter constructs a limiter whose client map is bounded by the security allowlist. +func NewRPCLimiter(security *SecurityManager, globalMax, clientMax int) (*RPCLimiter, error) { + if security == nil || globalMax < 1 || clientMax < 1 || clientMax > globalMax { + return nil, status.Error(codes.InvalidArgument, "invalid RPC concurrency limits") + } + return &RPCLimiter{security: security, globalMax: globalMax, clientMax: clientMax, clients: map[string]int{}}, nil +} + +// UnaryInterceptor authenticates against current policy before reserving capacity. +func (l *RPCLimiter) UnaryInterceptor(ctx context.Context, req any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + chain, _, err := clientCertificates(ctx) + if err != nil { + return nil, status.Error(codes.Unauthenticated, "execution provider request failed") + } + id, _, err := l.security.authorize(ctx, chain) + if err != nil { + return nil, securityAuthorizationError(err) + } + if !l.acquire(id) { + return nil, status.Error(codes.ResourceExhausted, "execution provider is at its concurrency limit") + } + defer l.release(id) + return handler(ctx, req) +} + +func clientCertificates(ctx context.Context) ([]*x509.Certificate, bool, error) { + p, ok := peer.FromContext(ctx) + if !ok { + return nil, false, errors.New("peer is unavailable") + } + tlsInfo, ok := p.AuthInfo.(credentials.TLSInfo) + if !ok || len(tlsInfo.State.PeerCertificates) == 0 { + return nil, false, errors.New("client certificate is unavailable") + } + return tlsInfo.State.PeerCertificates, len(tlsInfo.State.VerifiedChains) != 0, nil +} + +func (l *RPCLimiter) acquire(id string) bool { + l.mu.Lock() + defer l.mu.Unlock() + if l.global >= l.globalMax || l.clients[id] >= l.clientMax { + return false + } + l.global++ + l.clients[id]++ + return true +} + +func (l *RPCLimiter) release(id string) { + l.mu.Lock() + defer l.mu.Unlock() + l.global-- + if l.clients[id] == 1 { + delete(l.clients, id) + } else { + l.clients[id]-- + } +} diff --git a/internal/adapter/executioncontroller/security_limit_test.go b/internal/adapter/executioncontroller/security_limit_test.go new file mode 100644 index 0000000000..4600fce13c --- /dev/null +++ b/internal/adapter/executioncontroller/security_limit_test.go @@ -0,0 +1,29 @@ +package executioncontroller + +import "testing" + +func TestRPCLimiterBoundsGlobalPerClientAndIdentityMap(t *testing.T) { + limiter, err := NewRPCLimiter(&SecurityManager{}, 2, 1) + if err != nil { + t.Fatal(err) + } + if !limiter.acquire("client-a") || limiter.acquire("client-a") { + t.Fatal("per-client bound was not enforced") + } + if !limiter.acquire("client-b") || limiter.acquire("client-c") { + t.Fatal("global bound was not enforced") + } + limiter.release("client-a") + limiter.release("client-b") + if len(limiter.clients) != 0 || limiter.global != 0 { + t.Fatalf("released limiter retained cardinality: global=%d clients=%v", limiter.global, limiter.clients) + } +} + +func TestRPCLimiterRejectsInvalidBounds(t *testing.T) { + for _, limits := range [][2]int{{0, 1}, {1, 0}, {1, 2}} { + if _, err := NewRPCLimiter(&SecurityManager{}, limits[0], limits[1]); err == nil { + t.Fatalf("accepted limits %v", limits) + } + } +} diff --git a/internal/adapter/executioncontroller/security_rotation_test.go b/internal/adapter/executioncontroller/security_rotation_test.go new file mode 100644 index 0000000000..8595844b4c --- /dev/null +++ b/internal/adapter/executioncontroller/security_rotation_test.go @@ -0,0 +1,176 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/hex" + "os" + "path/filepath" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +// These are synthetic fixture names, including the former mutable Secret keys. +func rotationSecurityFixture(t *testing.T) (*SecurityManager, securityManifest, context.Context, map[string][]byte) { + t.Helper() + dir := t.TempDir() + now := time.Now().UTC() + oldCA, oldCert, oldKey := syntheticPKI(t, now) + newCA, newCert, newKey, client := rpcSecurityPKI(t, now, "spiffe://example/client") + writeRotationMaterial(t, dir, map[string][]byte{"tls.crt": oldCert, "tls.key": oldKey, "clients.pem": append(oldCA, newCA...)}) + manifest := securityManifest{Version: 1, Generation: 2, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k2", GrantTTLText: "1m", ClockSkewText: "5s", TLS: securityTLSManifest{CertificateFile: "tls.crt", PrivateKeyFile: "tls.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + for i, id := range []string{"k1", "k2"} { + pub, private, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + name := "grant-" + id + ".pem" + writePKCS8(t, filepath.Join(dir, name), private) + fp := sha256.Sum256(pub) + manifest.Keys = append(manifest.Keys, securityKeyManifest{ID: id, Version: uint64(i + 1), File: name, PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}) + } + path := filepath.Join(dir, "manifest.json") + writeManifest(t, path, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(path, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + leaf, err := x509.ParseCertificate(client.Certificate[0]) + if err != nil { + t.Fatal(err) + } + ctx := peer.NewContext(t.Context(), &peer.Peer{AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}, VerifiedChains: [][]*x509.Certificate{{leaf}}}}}) + return manager, manifest, ctx, map[string][]byte{"tls.crt": newCert, "tls.key": newKey, "clients.pem": newCA} +} + +func writeRotationMaterial(t *testing.T, dir string, files map[string][]byte) { + t.Helper() + for name, contents := range files { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } +} + +func TestSecurityRotationPinnedReplicaReadConvergesBeforeDispatch(t *testing.T) { + manager, manifest, ctx, final := rotationSecurityFixture(t) + backend := &countedFileBackend{fakeBackend: newFakeBackend()} + h := NewHandler(HandlerConfig{Security: manager}, backend) + owner := &executionv1.Owner{Issuer: "issuer", Subject: "alice"} + claim, err := h.AcquireRun(ctx, &executionv1.AcquireRunRequest{Environment: &executionv1.EnvironmentRef{Id: "env", Revision: "rev"}, Owner: owner, BindingId: "binding", RunId: "run", OperationId: "acquire", TtlMillis: time.Minute.Milliseconds()}) + if err != nil { + t.Fatal("bridge k2 acquire failed") + } + request := &executionv1.FileRequest{Context: &executionv1.RequestContext{Environment: claim.Environment, Owner: owner, BindingId: claim.BindingId, RunId: claim.RunId, ClaimId: claim.ClaimId, Epoch: claim.Epoch, GrantGeneration: claim.GrantGeneration, Grant: claim.Grant}, Operation: executionv1.FileOperation_FILE_OPERATION_READ, Path: "sentinel"} + if _, err := h.Files(ctx, request); err != nil || backend.calls != 1 { + t.Fatal("bridge claim positive control failed") + } + manifest.Generation = 3 + manifest.Keys[0].State = "revoked" + manifest.TLS = securityTLSManifest{CertificateFile: "provider-new.crt", PrivateKeyFile: "provider-new.key", ClientCAFile: "final-clients.pem"} + writeRotationMaterial(t, manager.keyDirectory, map[string][]byte{"provider-new.crt": final["tls.crt"], "provider-new.key": final["tls.key"], "final-clients.pem": final["clients.pem"]}) + writeManifest(t, manager.manifestPath, manifest) + other := NewSecurityManager(manager.manifestPath, manager.keyDirectory, "ns", "authority", manager.kube) + other.now = manager.now + if err := other.Reload(t.Context()); err != nil { + t.Fatal(err) + } + backend.calls = 0 + _, err = h.Files(ctx, request) + st := status.Convert(err) + details := st.Details() + if st.Code() != codes.Unavailable || len(details) != 1 || backend.calls != 0 { + t.Fatalf("lagging replica: code=%s dispatches=%d", st.Code(), backend.calls) + } + detail, ok := details[0].(*executionv1.ErrorDetail) + if !ok || detail.Code != string(executionenv.CodeNotReady) || !detail.Retryable { + t.Fatal("lag was not structured retryable not_ready") + } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if _, err := h.Files(ctx, request); err != nil || backend.calls != 1 { + t.Fatalf("same current claim after reload: code=%s dispatches=%d", status.Code(err), backend.calls) + } +} + +func TestSecurityRotationMutableNamesPoisonSameGeneration(t *testing.T) { + manager, manifest, _, final := rotationSecurityFixture(t) + manifest.Generation = 3 + manifest.Keys[0].State = "revoked" + // ConfigMap projection arrives before the Secret: all old names still exist. + writeManifest(t, manager.manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatalf("mixed bundle did not reproduce publication: %v", err) + } + mixed := manager.state.Load().snapshot.digest + writeRotationMaterial(t, manager.keyDirectory, final) + complete, err := manager.load() + if err != nil || complete.digest == mixed { + t.Fatal("complete bundle did not differ from published mixed bundle") + } + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("same-generation material drift was not rejected") + } + if err := manager.Reload(t.Context()); err == nil { + t.Fatal("retry unexpectedly repaired the poisoned ledger") + } + // Recovery requires a forward generation, not a retry or ledger reset. + manifest.Generation++ + writeManifest(t, manager.manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } +} + +func TestSecurityRotationImmutableNamesHandleProjectionSkew(t *testing.T) { + for _, manifestFirst := range []bool{false, true} { + name := "material-first" + if manifestFirst { + name = "manifest-first" + } + t.Run(name, func(t *testing.T) { + manager, manifest, _, final := rotationSecurityFixture(t) + before := manager.state.Load().snapshot.digest + manifest.Generation = 3 + manifest.Keys[0].State = "revoked" + manifest.TLS = securityTLSManifest{CertificateFile: "provider-new.crt", PrivateKeyFile: "provider-new.key", ClientCAFile: "final-clients.pem"} + if manifestFirst { + writeManifest(t, manager.manifestPath, manifest) + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("missing immutable material did not fail closed") + } + } + writeRotationMaterial(t, manager.keyDirectory, map[string][]byte{"provider-new.crt": final["tls.crt"], "provider-new.key": final["tls.key"], "final-clients.pem": final["clients.pem"]}) + if !manifestFirst { + if err := manager.Reload(t.Context()); err != nil || manager.state.Load().snapshot.digest != before { + t.Fatal("staging new names changed the old manifest's authority") + } + writeManifest(t, manager.manifestPath, manifest) + } + if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { + t.Fatalf("complete immutable bundle did not converge: %v", err) + } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal("same-generation immutable reload drifted") + } + }) + } +} diff --git a/internal/adapter/executioncontroller/security_test.go b/internal/adapter/executioncontroller/security_test.go new file mode 100644 index 0000000000..9de09f5009 --- /dev/null +++ b/internal/adapter/executioncontroller/security_test.go @@ -0,0 +1,743 @@ +package executioncontroller + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "math/big" + "net" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + grpcpeer "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + executionv1 "github.com/stacklok/mecatl/contracts/gen/go/mecatl/execution/v1" + "github.com/stacklok/mecatl/internal/executionenv" +) + +type durableClaimBackend struct { + *fakeBackend + mu sync.Mutex + acquired, renewed map[string]executionenv.RunClaim + acquireCalls, renewalCalls int +} + +func newDurableClaimBackend() *durableClaimBackend { + return &durableClaimBackend{fakeBackend: newFakeBackend(), acquired: make(map[string]executionenv.RunClaim), renewed: make(map[string]executionenv.RunClaim)} +} + +func (b *durableClaimBackend) AcquireRun(_ context.Context, ref executionenv.EnvironmentRef, _ string, _ string, binding, run, operation string, ttl time.Duration) (executionenv.RunClaim, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.acquireCalls++ + if claim, ok := b.acquired[operation]; ok { + return claim, nil + } + claim := executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: run, ClaimID: "claim-acquire", Epoch: 2, GrantGeneration: 1, ExpiresAt: time.Now().Add(ttl)} + b.acquired[operation] = claim + return claim, nil +} + +func (b *durableClaimBackend) RenewRun(_ context.Context, _ executionenv.EnvironmentRef, _ string, _ string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + b.mu.Lock() + defer b.mu.Unlock() + b.renewalCalls++ + if claim, ok := b.renewed[req.OperationID]; ok { + return claim, nil + } + claim := executionenv.RunClaim{Environment: req.Environment, BindingID: req.BindingID, RunID: req.RunID, ClaimID: req.ClaimID, Epoch: req.Epoch, GrantGeneration: req.GrantGeneration + 1, ExpiresAt: time.Now().Add(req.TTL)} + b.renewed[req.OperationID] = claim + return claim, nil +} + +func (b *durableClaimBackend) claim(operation string, renew bool) (executionenv.RunClaim, int, bool) { + b.mu.Lock() + defer b.mu.Unlock() + if renew { + claim, ok := b.renewed[operation] + return claim, b.renewalCalls, ok + } + claim, ok := b.acquired[operation] + return claim, b.acquireCalls, ok +} + +func TestSecurityManagerReloadRollbackAndRecovery(t *testing.T) { + dir := t.TempDir() + data := filepath.Join(dir, "..data") + if err := os.Mkdir(data, 0o700); err != nil { + t.Fatal(err) + } + now := time.Now().UTC() + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(data, "grant.pem"), grantPriv) + caPEM, serverCert, serverKey := syntheticPKI(t, now) + if err := os.WriteFile(filepath.Join(data, "server.crt"), serverCert, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(data, "server.key"), serverKey, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(data, "clients.pem"), caPEM, 0o600); err != nil { + t.Fatal(err) + } + for _, name := range []string{"grant.pem", "server.crt", "server.key", "clients.pem"} { + if err := os.Symlink(filepath.Join("..data", name), filepath.Join(dir, name)); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true, Administrator: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if !manager.Ready() { + t.Fatal("valid snapshot is not ready") + } + manifest.Clients[0].Administrator = false + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("same-generation client policy drift was accepted") + } + manifest.Clients[0].Administrator = true + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatalf("restore same generation: %v", err) + } + manifest.Generation = 0 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("invalid candidate retained authorization readiness") + } + manifest.Generation = 2 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { + t.Fatalf("recovery reload: ready=%v err=%v", manager.Ready(), err) + } + firstCertificate := append([]byte(nil), manager.state.Load().snapshot.tlsConfig.Certificates[0].Certificate[0]...) + cm, err := kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + originalState := cm.Data[securityStateDataKey] + cm.Data[securityStateDataKey] = `{"generation":2,"fingerprints":{"k1:1":"forged"}}` + if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if _, err := manager.material(t.Context()); err == nil || manager.Ready() { + t.Fatal("authoritative fingerprint drift retained authorization") + } + cm, err = kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + cm.Data[securityStateDataKey] = originalState + if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := manager.Reload(t.Context()); err != nil { + t.Fatalf("restore authority: %v", err) + } + newCAPEM, newServerCert, newServerKey := syntheticPKI(t, now) + for name, contents := range map[string][]byte{"server.crt": newServerCert, "server.key": newServerKey, "clients.pem": newCAPEM} { + if err := os.WriteFile(filepath.Join(data, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + manifest.Generation = 3 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { + t.Fatalf("TLS/CA correction did not recover: ready=%v err=%v", manager.Ready(), err) + } + if string(firstCertificate) == string(manager.state.Load().snapshot.tlsConfig.Certificates[0].Certificate[0]) { + t.Fatal("TLS identity did not rotate atomically") + } + manifest.Generation = 4 + writeManifest(t, manifestPath, manifest) + peer := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + peer.now = manager.now + if err := peer.Reload(t.Context()); err != nil { + t.Fatalf("peer authority advance: %v", err) + } + if manager.CheckReady(t.Context()) || manager.Ready() { + t.Fatal("readiness retained a snapshot superseded by a peer") + } + manifest.Generation = 1 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err == nil || manager.Ready() { + t.Fatal("generation rollback accepted") + } +} + +func TestSecurityManagerStaleAuthorityFailureCannotPoisonNewerState(t *testing.T) { + now := time.Now().UTC() + window := keyValidity{activateAt: now.Add(-time.Minute), verifyUntil: now.Add(time.Hour)} + oldSnapshot := &securitySnapshot{generation: 1, digest: "old", activeWindow: window, validUntil: now.Add(time.Hour), fingerprints: map[string]string{"k1:1": "old"}} + newSnapshot := &securitySnapshot{generation: 2, digest: "new", activeWindow: window, validUntil: now.Add(time.Hour), fingerprints: map[string]string{"k1:1": "old", "k1:2": "new"}} + raw, err := json.Marshal(securityLedger{Generation: newSnapshot.generation, Digest: newSnapshot.digest, Fingerprints: newSnapshot.fingerprints}) + if err != nil { + t.Fatal(err) + } + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{securityStateDataKey: string(raw)}}) + started, release := make(chan struct{}), make(chan struct{}) + var startOnce sync.Once + kube.PrependReactor("get", "configmaps", func(_ k8stesting.Action) (bool, runtime.Object, error) { + startOnce.Do(func() { close(started) }) + <-release + return false, nil, nil + }) + manager := NewSecurityManager("", "", "ns", "authority", kube) + manager.now = func() time.Time { return now } + oldState := &securityState{snapshot: oldSnapshot} + manager.state.Store(oldState) + done := make(chan error, 1) + go func() { + _, authErr := manager.authoritative(t.Context()) + done <- authErr + }() + <-started + newState := &securityState{snapshot: newSnapshot} + manager.state.Store(newState) + close(release) + if err := <-done; !errors.Is(err, errAuthorityUnavailable) { + t.Fatalf("stale authority check error=%v", err) + } + if manager.state.Load() != newState || !manager.Ready() { + t.Fatal("stale authority check invalidated the newer published state") + } + + manager.invalidateObservedUnlessReplaced(t.Context(), oldState) + if manager.state.Load() != newState || !manager.Ready() { + t.Fatal("stale reload failure invalidated the newer authoritative state") + } +} + +func TestSecurityLedgerRejectsKeyVersionRollbackAcrossRestart(t *testing.T) { + ledger := securityLedger{Generation: 2, Fingerprints: map[string]string{"key:2": "old"}} + candidate := &securitySnapshot{generation: 3, fingerprints: map[string]string{"key:1": "new"}} + if _, err := advanceSecurityLedger(&ledger, candidate); err == nil { + t.Fatal("key version rollback was accepted") + } + candidate.fingerprints = map[string]string{"key:3": "new"} + changed, err := advanceSecurityLedger(&ledger, candidate) + if err != nil || !changed || candidate.fingerprints["key:2"] != "old" { + t.Fatalf("monotonic rotation failed: changed=%v err=%v fingerprints=%v", changed, err, candidate.fingerprints) + } +} + +func TestSecurityManagerGuardsEveryRPCOnExistingConnection(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC() + caPEM, serverPEM, serverKeyPEM, clientCert := rpcSecurityPKI(t, now, "spiffe://example/client") + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(dir, "grant.pem"), grantPriv) + for name, contents := range map[string][]byte{"server.crt": serverPEM, "server.key": serverKeyPEM, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + store := NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), "ns", testProfiles(), nil) + limiter, err := NewRPCLimiter(manager, 4, 2) + if err != nil { + t.Fatal(err) + } + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig())), grpc.UnaryInterceptor(limiter.UnaryInterceptor)) + executionv1.RegisterExecutionProviderServiceServer(server, NewHandler(HandlerConfig{Security: manager}, store)) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(caPEM) + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, ServerName: "provider.test", RootCAs: roots, Certificates: []tls.Certificate{clientCert}}))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + client := executionv1.NewExecutionProviderServiceClient(conn) + call := func() error { + _, err := client.ValidateProfile(t.Context(), &executionv1.ValidateProfileRequest{Profile: "go"}) + return err + } + if err := call(); err != nil { + t.Fatalf("initial RPC: %v", err) + } + cm, err := kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + authoritativeState := cm.Data[securityStateDataKey] + var drifted securityLedger + if err := executionenv.DecodeStrict([]byte(authoritativeState), &drifted); err != nil { + t.Fatal(err) + } + drifted.Digest = strings.Repeat("0", 64) + raw, _ := json.Marshal(drifted) + cm.Data[securityStateDataKey] = string(raw) + if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := call(); status.Code(err) != codes.Unavailable || manager.Ready() { + t.Fatalf("old snapshot survived durable authority drift: ready=%v err=%v", manager.Ready(), err) + } else if details := status.Convert(err).Details(); len(details) != 1 || details[0].(*executionv1.ErrorDetail).Code != string(executionenv.CodeNotReady) || !details[0].(*executionv1.ErrorDetail).Retryable { + t.Fatalf("authority lag was not a structured retryable not_ready: %v", details) + } + cm, _ = kube.CoreV1().ConfigMaps("ns").Get(t.Context(), "authority", metav1.GetOptions{}) + cm.Data[securityStateDataKey] = authoritativeState + if _, err := kube.CoreV1().ConfigMaps("ns").Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + newCAPEM, newServerPEM, newServerKeyPEM, _ := rpcSecurityPKI(t, now, "spiffe://example/other") + for name, contents := range map[string][]byte{"server.crt": newServerPEM, "server.key": newServerKeyPEM, "clients.pem": newCAPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + manifest.Generation = 2 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if err := call(); status.Code(err) != codes.Unauthenticated { + t.Fatalf("old connection survived CA rotation: %v", err) + } + if err := os.WriteFile(manifestPath, []byte(`{"version":2}`), 0o600); err != nil { + t.Fatal(err) + } + if err := manager.Reload(t.Context()); err == nil { + t.Fatal("invalid manifest reloaded") + } + if err := call(); status.Code(err) != codes.Unavailable { + t.Fatalf("RPC remained authorized during invalid manifest: %v", err) + } + for name, contents := range map[string][]byte{"server.crt": serverPEM, "server.key": serverKeyPEM, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + manifest.Generation = 3 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil || call() != nil { + t.Fatalf("valid generation did not recover existing connection: %v", err) + } +} + +func TestRunClaimSigningAuthorityOutageIsRetryableAfterDurableMutation(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC() + caPEM, serverPEM, serverKeyPEM, clientCert := rpcSecurityPKI(t, now, "spiffe://example/client") + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(dir, "grant.pem"), grantPriv) + for name, contents := range map[string][]byte{"server.crt": serverPEM, "server.key": serverKeyPEM, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + var reads, failAt atomic.Int64 + kube.PrependReactor("get", "configmaps", func(_ k8stesting.Action) (bool, runtime.Object, error) { + read := reads.Add(1) + if target := failAt.Load(); target != 0 && read == target { + return true, nil, errors.New("injected authority read outage") + } + return false, nil, nil + }) + armSigningOutage := func() { + reads.Store(0) + failAt.Store(2) // Authorization reads first; response signing reads second. + } + recoverAuthority := func() { + failAt.Store(0) + if err := manager.Reload(t.Context()); err != nil { + t.Fatalf("reload authority: %v", err) + } + } + assertNotReady := func(err error) { + t.Helper() + st := status.Convert(err) + details := st.Details() + if st.Code() != codes.Unavailable || len(details) != 1 { + t.Fatalf("code=%v details=%v", st.Code(), details) + } + detail, ok := details[0].(*executionv1.ErrorDetail) + if !ok || detail.Code != string(executionenv.CodeNotReady) || !detail.Retryable { + t.Fatalf("detail=%v", details) + } + } + leaf, err := x509.ParseCertificate(clientCert.Certificate[0]) + if err != nil { + t.Fatal(err) + } + ctx := grpcpeer.NewContext(t.Context(), &grpcpeer.Peer{AuthInfo: credentials.TLSInfo{State: tls.ConnectionState{PeerCertificates: []*x509.Certificate{leaf}, VerifiedChains: [][]*x509.Certificate{{leaf}}}}}) + backend := newDurableClaimBackend() + h := NewHandler(HandlerConfig{Security: manager}, backend) + owner := &executionv1.Owner{Issuer: "issuer", Subject: "alice"} + ref := &executionv1.EnvironmentRef{Id: "env", Revision: "rev"} + acquire := &executionv1.AcquireRunRequest{Environment: ref, Owner: owner, BindingId: "binding", RunId: "run", OperationId: "acquire-op", TtlMillis: time.Minute.Milliseconds()} + + armSigningOutage() + _, err = h.AcquireRun(ctx, acquire) + assertNotReady(err) + storedAcquire, calls, ok := backend.claim(acquire.OperationId, false) + if !ok || calls != 1 { + t.Fatalf("acquire mutation missing or retried automatically: stored=%t calls=%d", ok, calls) + } + recoverAuthority() + acquired, err := h.AcquireRun(ctx, acquire) + if err != nil { + t.Fatalf("explicit acquire retry: %v", err) + } + if callsClaim, retryCalls, ok := backend.claim(acquire.OperationId, false); !ok || retryCalls != 2 || callsClaim.ClaimID != storedAcquire.ClaimID || acquired.ClaimId != storedAcquire.ClaimID { + t.Fatalf("acquire retry did not converge: response=%q stored=%+v calls=%d", acquired.GetClaimId(), callsClaim, retryCalls) + } + + renew := &executionv1.RenewRunRequest{Environment: acquired.Environment, Owner: owner, BindingId: acquired.BindingId, RunId: acquired.RunId, ClaimId: acquired.ClaimId, Epoch: acquired.Epoch, GrantGeneration: acquired.GrantGeneration, OperationId: "renew-op", TtlMillis: time.Minute.Milliseconds()} + armSigningOutage() + _, err = h.RenewRun(ctx, renew) + assertNotReady(err) + storedRenew, calls, ok := backend.claim(renew.OperationId, true) + if !ok || calls != 1 { + t.Fatalf("renew mutation missing or retried automatically: stored=%t calls=%d", ok, calls) + } + recoverAuthority() + renewed, err := h.RenewRun(ctx, renew) + if err != nil { + t.Fatalf("explicit renew retry: %v", err) + } + if callsClaim, retryCalls, ok := backend.claim(renew.OperationId, true); !ok || retryCalls != 2 || callsClaim.ClaimID != storedRenew.ClaimID || callsClaim.GrantGeneration != storedRenew.GrantGeneration || renewed.ClaimId != storedRenew.ClaimID || renewed.GrantGeneration != storedRenew.GrantGeneration { + t.Fatalf("renew retry did not converge: response=%q/%d stored=%+v calls=%d", renewed.GetClaimId(), renewed.GetGrantGeneration(), callsClaim, retryCalls) + } +} + +func TestSecurityManagerAuthorizesPresentedIntermediateAndRevokesRootOnExistingConnection(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC() + caPEM, serverPEM, serverKeyPEM, clientCert := rpcSecurityIntermediatePKI(t, now, "spiffe://example/client") + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(dir, "grant.pem"), grantPriv) + for name, contents := range map[string][]byte{"server.crt": serverPEM, "server.key": serverKeyPEM, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(time.Hour), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + manager.now = func() time.Time { return now } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + store := NewStore(dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()), "ns", testProfiles(), nil) + limiter, err := NewRPCLimiter(manager, 4, 2) + if err != nil { + t.Fatal(err) + } + server := grpc.NewServer(grpc.Creds(credentials.NewTLS(manager.TLSConfig())), grpc.UnaryInterceptor(limiter.UnaryInterceptor)) + executionv1.RegisterExecutionProviderServiceServer(server, NewHandler(HandlerConfig{Security: manager}, store)) + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = server.Serve(listener) }() + t.Cleanup(server.Stop) + roots := x509.NewCertPool() + roots.AppendCertsFromPEM(caPEM) + conn, err := grpc.NewClient(listener.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS13, ServerName: "provider.test", RootCAs: roots, Certificates: []tls.Certificate{clientCert}}))) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + client := executionv1.NewExecutionProviderServiceClient(conn) + call := func() error { + _, err := client.ValidateProfile(t.Context(), &executionv1.ValidateProfileRequest{Profile: "go"}) + return err + } + if err := call(); err != nil { + t.Fatalf("intermediate-signed client RPC: %v", err) + } + removedRoot, _, _, _ := rpcSecurityPKI(t, now, "spiffe://example/other") + if err := os.WriteFile(filepath.Join(dir, "clients.pem"), removedRoot, 0o600); err != nil { + t.Fatal(err) + } + manifest.Generation = 2 + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + if err := call(); status.Code(err) != codes.Unauthenticated { + t.Fatalf("existing connection survived removal of intermediate issuer root: %v", err) + } +} + +func rpcSecurityIntermediatePKI(t *testing.T, now time.Time, clientURI string) ([]byte, []byte, []byte, tls.Certificate) { + t.Helper() + rootPub, rootKey, _ := ed25519.GenerateKey(rand.Reader) + root := &x509.Certificate{SerialNumber: big.NewInt(201), Subject: pkix.Name{CommonName: "root"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + rootDER, err := x509.CreateCertificate(rand.Reader, root, root, rootPub, rootKey) + if err != nil { + t.Fatal(err) + } + intermediatePub, intermediateKey, _ := ed25519.GenerateKey(rand.Reader) + intermediate := &x509.Certificate{SerialNumber: big.NewInt(202), Subject: pkix.Name{CommonName: "intermediate"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + intermediateDER, err := x509.CreateCertificate(rand.Reader, intermediate, root, intermediatePub, rootKey) + if err != nil { + t.Fatal(err) + } + issue := func(serial int64, parent *x509.Certificate, parentKey ed25519.PrivateKey, uriText string, usages []x509.ExtKeyUsage, dns []string) ([]byte, ed25519.PrivateKey) { + pub, key, _ := ed25519.GenerateKey(rand.Reader) + tmpl := &x509.Certificate{SerialNumber: big.NewInt(serial), NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: usages, DNSNames: dns} + if uriText != "" { + u, _ := url.Parse(uriText) + tmpl.URIs = []*url.URL{u} + } + der, issueErr := x509.CreateCertificate(rand.Reader, tmpl, parent, pub, parentKey) + if issueErr != nil { + t.Fatal(issueErr) + } + return der, key + } + serverDER, serverKey := issue(203, root, rootKey, "spiffe://example/provider", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, []string{"provider.test"}) + clientDER, clientKey := issue(204, intermediate, intermediateKey, clientURI, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, nil) + encodeKey := func(key ed25519.PrivateKey) []byte { + raw, marshalErr := x509.MarshalPKCS8PrivateKey(key) + if marshalErr != nil { + t.Fatal(marshalErr) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: raw}) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: rootDER}), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}), encodeKey(serverKey), tls.Certificate{Certificate: [][]byte{clientDER, intermediateDER}, PrivateKey: clientKey} +} + +func rpcSecurityPKI(t *testing.T, now time.Time, clientURI string) ([]byte, []byte, []byte, tls.Certificate) { + t.Helper() + caPub, caKey, _ := ed25519.GenerateKey(rand.Reader) + ca := &x509.Certificate{SerialNumber: big.NewInt(101), NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, caPub, caKey) + if err != nil { + t.Fatal(err) + } + issue := func(serial int64, uriText string, usages []x509.ExtKeyUsage, dns []string) ([]byte, ed25519.PrivateKey) { + pub, key, _ := ed25519.GenerateKey(rand.Reader) + tmpl := &x509.Certificate{SerialNumber: big.NewInt(serial), NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: usages, DNSNames: dns} + if uriText != "" { + u, _ := url.Parse(uriText) + tmpl.URIs = []*url.URL{u} + } + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca, pub, caKey) + if err != nil { + t.Fatal(err) + } + return der, key + } + serverDER, serverKey := issue(102, "spiffe://example/provider", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, []string{"provider.test"}) + clientDER, clientKey := issue(103, clientURI, []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, nil) + encodeKey := func(key ed25519.PrivateKey) []byte { + raw, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: raw}) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}), encodeKey(serverKey), tls.Certificate{Certificate: [][]byte{clientDER}, PrivateKey: clientKey} +} + +func TestAuthorityDigestCoversSecurityMeaningfulState(t *testing.T) { + base := securityManifest{Version: 1, Generation: 7, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "key.pem", PublicSHA256: "fingerprint", ActivateAt: time.Unix(1, 0).UTC(), VerifyUntil: time.Unix(100, 0).UTC(), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "ca.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client"}}} + digest := func(m securityManifest, ca, server string) string { + t.Helper() + ttl, err := time.ParseDuration(m.GrantTTLText) + if err != nil { + t.Fatal(err) + } + skew, err := time.ParseDuration(m.ClockSkewText) + if err != nil { + t.Fatal(err) + } + got, err := authorityDigest(m, ttl, skew, map[string]string{"k1:1": "fingerprint"}, []string{ca}, server) + if err != nil { + t.Fatal(err) + } + return got + } + original := digest(base, "ca-1", "server-1") + mutations := map[string]func(*securityManifest){ + "capability": func(m *securityManifest) { m.Clients[0].Administrator = true }, + "key revoke": func(m *securityManifest) { m.Keys[0].State = "revoked" }, + "active key": func(m *securityManifest) { m.ActiveKeyID = "k2" }, + "activation time": func(m *securityManifest) { m.Keys[0].ActivateAt = m.Keys[0].ActivateAt.Add(time.Second) }, + "grant ttl": func(m *securityManifest) { m.GrantTTLText = "2m" }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + candidate := base + candidate.Keys = slices.Clone(base.Keys) + candidate.Clients = slices.Clone(base.Clients) + mutate(&candidate) + if digest(candidate, "ca-1", "server-1") == original { + t.Fatal("mutation did not change authority digest") + } + }) + } + if digest(base, "ca-2", "server-1") == original || digest(base, "ca-1", "server-2") == original { + t.Fatal("CA or server identity swap did not change authority digest") + } +} + +func TestSecurityManagerSigningWindowIsCheckedOnEveryRequest(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC() + grantPub, grantPriv, _ := ed25519.GenerateKey(rand.Reader) + writePKCS8(t, filepath.Join(dir, "grant.pem"), grantPriv) + caPEM, serverCert, serverKey := syntheticPKI(t, now) + for name, contents := range map[string][]byte{"server.crt": serverCert, "server.key": serverKey, "clients.pem": caPEM} { + if err := os.WriteFile(filepath.Join(dir, name), contents, 0o600); err != nil { + t.Fatal(err) + } + } + fp := sha256.Sum256(grantPub) + manifest := securityManifest{Version: 1, Generation: 1, Issuer: "issuer", Audience: "audience", ActiveKeyID: "k1", GrantTTLText: "1m", ClockSkewText: "5s", Keys: []securityKeyManifest{{ID: "k1", Version: 1, File: "grant.pem", PublicSHA256: hex.EncodeToString(fp[:]), ActivateAt: now.Add(-time.Minute), VerifyUntil: now.Add(10 * time.Second), State: "active"}}, TLS: securityTLSManifest{CertificateFile: "server.crt", PrivateKeyFile: "server.key", ClientCAFile: "clients.pem"}, Clients: []securityClientManifest{{URI: "spiffe://example/client", MayAttestOwner: true}}} + manifestPath := filepath.Join(dir, "manifest.json") + writeManifest(t, manifestPath, manifest) + kube := fake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "authority", Namespace: "ns"}, Data: map[string]string{}}) + manager := NewSecurityManager(manifestPath, dir, "ns", "authority", kube) + clock := now + manager.now = func() time.Time { return clock } + if err := manager.Reload(t.Context()); err != nil { + t.Fatal(err) + } + h := NewHandler(HandlerConfig{Security: manager}, &fakeBackend{}) + claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1} + if _, expiry, err := h.signClaim(t.Context(), claim, "spiffe://example/client", "owner"); err != nil || expiry.After(manifest.Keys[0].VerifyUntil) { + t.Fatalf("initial sign expiry=%v err=%v", expiry, err) + } + clock = manifest.Keys[0].VerifyUntil + if manager.Ready() { + t.Fatal("expired active key remained ready between reload ticks") + } + if _, _, err := h.signClaim(t.Context(), claim, "spiffe://example/client", "owner"); err == nil { + t.Fatal("expired active key signed between reload ticks") + } + manifest.Generation = 2 + manifest.Keys[0].VerifyUntil = clock.Add(time.Hour) + writeManifest(t, manifestPath, manifest) + if err := manager.Reload(t.Context()); err != nil || !manager.Ready() { + t.Fatalf("corrected generation did not recover: ready=%v err=%v", manager.Ready(), err) + } +} + +func TestSecurityManagerRejectsEscapingProjectedSymlink(t *testing.T) { + dir := t.TempDir() + if err := os.Symlink("../../outside", filepath.Join(dir, "grant.pem")); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(dir) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := root.Close(); err != nil { + t.Errorf("close root: %v", err) + } + }) + if _, err := readRootFile(root, "grant.pem"); err == nil { + t.Fatal("escaping symlink was read") + } +} + +func writeManifest(t *testing.T, path string, manifest securityManifest) { + t.Helper() + b, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, b, 0o600); err != nil { + t.Fatal(err) + } +} +func writePKCS8(t *testing.T, path string, key ed25519.PrivateKey) { + t.Helper() + b, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: b}), 0o600); err != nil { + t.Fatal(err) + } +} +func syntheticPKI(t *testing.T, now time.Time) ([]byte, []byte, []byte) { + t.Helper() + caPub, caKey, _ := ed25519.GenerateKey(rand.Reader) + ca := &x509.Certificate{SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "test-ca"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(2 * time.Hour), IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature} + caDER, err := x509.CreateCertificate(rand.Reader, ca, ca, caPub, caKey) + if err != nil { + t.Fatal(err) + } + serverPub, serverKey, _ := ed25519.GenerateKey(rand.Reader) + uri, _ := url.Parse("spiffe://example/provider") + server := &x509.Certificate{SerialNumber: big.NewInt(2), Subject: pkix.Name{CommonName: "provider"}, NotBefore: now.Add(-time.Hour), NotAfter: now.Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, URIs: []*url.URL{uri}} + serverDER, err := x509.CreateCertificate(rand.Reader, server, ca, serverPub, caKey) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalPKCS8PrivateKey(serverKey) + if err != nil { + t.Fatal(err) + } + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}), pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: keyDER}) +} diff --git a/internal/adapter/executioncontroller/store.go b/internal/adapter/executioncontroller/store.go new file mode 100644 index 0000000000..af52133581 --- /dev/null +++ b/internal/adapter/executioncontroller/store.go @@ -0,0 +1,631 @@ +//nolint:revive // Private store methods implement adapter-only lifecycle interfaces. +package executioncontroller + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "math" + "slices" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/util/retry" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// ExecutionEnvironmentGVR identifies the private execution-environment CRD. +var ExecutionEnvironmentGVR = schema.GroupVersionResource{Group: "execution.mecatl.dev", Version: "v1alpha1", Resource: "executionenvironments"} + +const ( + maxReferences = 64 + environmentNotFoundMessage = "environment not found" + currentSchemaVersion = int64(2) + operationLeaseTTL = 30 * time.Second + transitioningMessage = "environment is transitioning" + operationIDField = "operationID" +) + +// ExecutorTransport dispatches one request to a fixed workload helper. +type ExecutorTransport interface { + Execute(context.Context, string, executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) +} + +// Store persists authorization and fencing state in ExecutionEnvironment status. +type Store struct { + resources dynamic.ResourceInterface + profiles *Profiles + executor ExecutorTransport + kube kubernetes.Interface + namespace string + holderID string + now func() time.Time + opTTL time.Duration +} + +// NewStore constructs a namespace-scoped execution state store. +func NewStore(client dynamic.Interface, namespace string, profiles *Profiles, executor ExecutorTransport) *Store { + holder, err := randomID() + if err != nil { + holder = "unavailable" + } + return &Store{resources: client.Resource(ExecutionEnvironmentGVR).Namespace(namespace), profiles: profiles, executor: executor, namespace: namespace, holderID: holder, now: func() time.Time { return time.Now().UTC() }, opTTL: 30 * time.Second} +} + +// WithKubeClient enables provider-owned executor and retained-volume lifecycle operations. +func (s *Store) WithKubeClient(kube kubernetes.Interface) *Store { + s.kube = kube + return s +} + +// ValidateProfile returns one validated immutable profile. +func (s *Store) ValidateProfile(_ context.Context, name string) (Profile, error) { + p, ok := s.profiles.get(name) + if !ok { + return Profile{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "profile not found"} + } + return Profile{Name: name, Digest: p.Digest, MaxFileBytes: p.Spec.MaxFileBytes, MaxCommandBytes: p.Spec.MaxCommandBytes, MaxCommandDuration: p.Spec.MaxCommandDuration, Capabilities: []string{"filesystem", "foreground-command"}}, nil +} + +// Ensure idempotently allocates an environment for an immutable request fingerprint. +func (s *Store) Ensure(ctx context.Context, client, owner, binding, profile, fp string) (Allocation, error) { + return s.EnsurePending(ctx, client, owner, binding, profile, fp, "legacy-"+fp) +} + +func (s *Store) EnsurePending(ctx context.Context, client, owner, binding, profile, fp, operationID string) (Allocation, error) { + return s.ensurePending(ctx, client, owner, executionenv.Owner{}, binding, profile, fp, operationID) +} + +// EnsurePendingOwned persists the attested principal needed for owner-safe intent reconciliation. +func (s *Store) EnsurePendingOwned(ctx context.Context, client, expectedOwnerHash string, owner executionenv.Owner, binding, profile, fp, operationID string) (Allocation, error) { + if owner.Issuer == "" || owner.Subject == "" || ownerHash(owner) != expectedOwnerHash { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "owner attestation is invalid"} + } + return s.ensurePending(ctx, client, expectedOwnerHash, owner, binding, profile, fp, operationID) +} + +func (s *Store) ensurePending(ctx context.Context, client, owner string, attested executionenv.Owner, binding, profile, fp, operationID string) (Allocation, error) { + p, ok := s.profiles.get(profile) + if !ok { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "profile not found"} + } + name := allocationName(client, owner, binding) + cur, err := s.resources.Get(ctx, name, metav1.GetOptions{}) + if err == nil { + if err := s.initializeStatus(ctx, cur, binding, operationID); err != nil { + return Allocation{}, err + } + cur, err = s.resources.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return Allocation{}, err + } + if !referenceOperationMatches(cur, binding, operationID) { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference operation conflicts with existing environment"} + } + return allocationFrom(cur, client, owner, binding, profile, fp) + } + if !apierrors.IsNotFound(err) { + return Allocation{}, fmt.Errorf("get execution environment: %w", err) + } + revision, err := randomID() + if err != nil { + return Allocation{}, err + } + if s.kube != nil { + if err := s.reserveProfileSlot(ctx, profile, name, p.Spec.MaxEnvironments); err != nil { + return Allocation{}, err + } + } + spec := map[string]any{"schemaVersion": currentSchemaVersion, "allocationID": name, "revision": revision, "ownerHash": owner, "clientHash": hashText(client), "bindingID": binding, "requestFingerprint": fp, "profile": profile, "profileDigest": p.Digest, "image": p.Spec.Image, "storageClass": p.Spec.StorageClass, "storageSize": p.Spec.StorageSize, "resources": map[string]any{"cpuRequest": p.Spec.CPURequest, "memoryRequest": p.Spec.MemoryRequest, "cpuLimit": p.Spec.CPULimit, "memoryLimit": p.Spec.MemoryLimit}, "desired": "Active"} + if attested.Issuer != "" && attested.Subject != "" { + spec["ownerIssuer"] = attested.Issuer + spec["ownerSubject"] = attested.Subject + } + obj := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": name, "finalizers": []any{environmentFinalizer}}, "spec": spec}} + created, err := s.resources.Create(ctx, obj, metav1.CreateOptions{}) + if apierrors.IsAlreadyExists(err) { + created, err = s.resources.Get(ctx, name, metav1.GetOptions{}) + } + if err != nil { + if s.kube != nil && definitiveCreateRejection(err) { + if releaseErr := releaseProfileSlot(ctx, s.kube, s.namespace, profile, name); releaseErr != nil { + return Allocation{}, errors.Join(fmt.Errorf("create execution environment: %w", err), fmt.Errorf("release profile allocation: %w", releaseErr)) + } + } + return Allocation{}, fmt.Errorf("create execution environment: %w", err) + } + if err := s.initializeStatus(ctx, created, binding, operationID); err != nil { + return Allocation{}, err + } + created, err = s.resources.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return Allocation{}, err + } + return allocationFrom(created, client, owner, binding, profile, fp) +} + +func definitiveCreateRejection(err error) bool { + return apierrors.IsInvalid(err) || apierrors.IsForbidden(err) || apierrors.IsUnauthorized(err) || apierrors.IsBadRequest(err) || apierrors.IsMethodNotSupported(err) || apierrors.IsRequestEntityTooLargeError(err) +} + +const profileAllocationConfigMap = "mecatl-execution-profile-allocations" + +func profileAllocationKey(profile string) string { + keySum := sha256.Sum256([]byte(profile)) + return "profile-" + hex.EncodeToString(keySum[:16]) + ".json" +} + +func (s *Store) reserveProfileSlot(ctx context.Context, profile, allocationID string, limit int) error { //nolint:gocyclo // Durable capacity CAS and CR reconciliation are intentionally one transaction loop. + cms := s.kube.CoreV1().ConfigMaps(s.namespace) + key := profileAllocationKey(profile) + for range 8 { + cm, err := cms.Get(ctx, profileAllocationConfigMap, metav1.GetOptions{}) + exists := true + if apierrors.IsNotFound(err) { + exists = false + cm = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: s.namespace}, Data: map[string]string{}} + } else if err != nil { + return fmt.Errorf("read profile allocation authority: %w", err) + } + var slots []string + if raw := cm.Data[key]; raw != "" { + if err := executionenv.DecodeStrict([]byte(raw), &slots); err != nil || len(slots) > limit { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "profile allocation authority is invalid"} + } + } + if slices.Contains(slots, allocationID) { + return nil + } + all, err := s.resources.List(ctx, metav1.ListOptions{}) + if err != nil { + return fmt.Errorf("list profile allocations: %w", err) + } + for i := range all.Items { + item := &all.Items[i] + deallocating := conditionTrue(item, "Retired") && conditionTrue(item, "ExecutorTerminated") && textNested(item.Object, "status", "lifecycleOperation", "type") == "DeleteRetiredEnvironment" && textNested(item.Object, "status", "lifecycleOperation", "phase") == "ReleasingSlot" + if textNested(item.Object, "spec", "profile") == profile && !deallocating && !slices.Contains(slots, item.GetName()) { + slots = append(slots, item.GetName()) + } + } + if len(slots) >= limit { + return &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "profile environment limit reached"} + } + slots = append(slots, allocationID) + slices.Sort(slots) + raw, err := json.Marshal(slots) + if err != nil { + return err + } + if cm.Data == nil { + cm.Data = map[string]string{} + } + cm.Data[key] = string(raw) + if exists { + _, err = cms.Update(ctx, cm, metav1.UpdateOptions{}) + } else { + _, err = cms.Create(ctx, cm, metav1.CreateOptions{}) + } + if err == nil { + return nil + } + if !apierrors.IsConflict(err) && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("reserve profile allocation: %w", err) + } + } + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "profile allocation authority changed concurrently", Retryable: true} +} + +func releaseProfileSlot(ctx context.Context, kube kubernetes.Interface, namespace, profile, allocationID string) error { + cms := kube.CoreV1().ConfigMaps(namespace) + key := profileAllocationKey(profile) + for range 8 { + cm, err := cms.Get(ctx, profileAllocationConfigMap, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + var slots []string + if err := executionenv.DecodeStrict([]byte(cm.Data[key]), &slots); err != nil { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "profile allocation authority is invalid"} + } + index := slices.Index(slots, allocationID) + if index < 0 { + return nil + } + slots = append(slots[:index], slots[index+1:]...) + raw, err := json.Marshal(slots) + if err != nil { + return err + } + cm.Data[key] = string(raw) + if _, err = cms.Update(ctx, cm, metav1.UpdateOptions{}); err == nil { + return nil + } + if !apierrors.IsConflict(err) { + return err + } + } + return errors.New("profile allocation authority changed concurrently") +} + +func (s *Store) initializeStatus(ctx context.Context, env *unstructured.Unstructured, binding, operationID string) error { + if operationID == "" { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "operation identity is required"} + } + if intNested(env.Object, "status", "epoch") > 0 { + return requireCurrentSchema(env) + } + return s.retryUpdateStatusRaw(ctx, env.GetName(), func(o *unstructured.Unstructured) error { + if intNested(o.Object, "status", "epoch") > 0 { + return requireCurrentSchema(o) + } + if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion { + return incompatibleSchemaError() + } + if err := unstructured.SetNestedField(o.Object, currentSchemaVersion, "status", "schemaVersion"); err != nil { + return err + } + if err := unstructured.SetNestedField(o.Object, int64(1), "status", "epoch"); err != nil { + return err + } + if err := unstructured.SetNestedField(o.Object, int64(1), "status", "grantGeneration"); err != nil { + return err + } + ref := map[string]any{"bindingID": binding, "state": string(executionenv.ReferencePendingCreate), operationIDField: operationID, "createdAt": time.Now().UTC().Format(time.RFC3339Nano)} + if err := unstructured.SetNestedSlice(o.Object, []any{ref}, "status", "references"); err != nil { + return err + } + return unstructured.SetNestedField(o.Object, fenceHealthy, "status", "fenceState") + }) +} +func allocationName(client, owner, binding string) string { + sum := sha256.Sum256([]byte(client + "\x00" + owner + "\x00" + binding)) + return "exec-" + hex.EncodeToString(sum[:20]) +} +func allocationFrom(o *unstructured.Unstructured, client, owner, binding, profile, fp string) (Allocation, error) { + spec, _, _ := unstructured.NestedMap(o.Object, "spec") + if text(spec, "ownerHash") != owner || text(spec, "clientHash") != hashText(client) || text(spec, "bindingID") != binding || text(spec, "profile") != profile || text(spec, "requestFingerprint") != fp { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "allocation identity conflicts with existing environment"} + } + return exactAllocation(o, client, owner, binding) +} +func exactAllocation(o *unstructured.Unstructured, client, owner, binding string) (Allocation, error) { + spec, _, _ := unstructured.NestedMap(o.Object, "spec") + if text(spec, "ownerHash") != owner || text(spec, "clientHash") != hashText(client) { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + revision := text(spec, "revision") + if revision == "" { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment identity is incomplete"} + } + epoch, ok := epochValue(o) + if !ok { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment epoch is invalid"} + } + generation := intNested(o.Object, "status", "grantGeneration") + if generation <= 0 { + return Allocation{}, &executionenv.Error{Code: executionenv.CodeConflict, Message: "grant generation is invalid"} + } + ready := conditionTrue(o, "Ready") && textNested(o.Object, "status", "fenceState") == fenceHealthy + return Allocation{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: revision}, Epoch: epoch, GrantGeneration: uint64(generation), OwnerHash: owner, BindingID: binding, Client: client, Ready: ready}, nil +} + +// Attach adds a binding reference only to the exact healthy environment. +func (s *Store) Attach(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) (Allocation, error) { + var out Allocation + err := s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + a, err := exactAllocation(o, client, owner, binding) + if err != nil { + return err + } + if a.Environment != ref || textNested(o.Object, "spec", "desired") != "Active" { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment revision unavailable"} + } + if !a.Ready { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment is not ready at requested epoch"} + } + refs, refsErr := referenceRecords(o) + if refsErr != nil { + return refsErr + } + if !attachedReference(refs, binding) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + out = a + return nil + }) + return out, err +} + +// ReleaseReference removes one binding reference without deleting the environment. +func (s *Store) ReleaseReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + a, err := exactAllocation(o, client, owner, binding) + if err != nil { + return err + } + if a.Environment != ref { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, refsErr := referenceRecords(o) + if refsErr != nil { + return refsErr + } + for i := range refs { + if refs[i].BindingID == binding { + if refs[i].State != executionenv.ReferencePublished { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference has a pending transaction"} + } + return setReferenceRecords(o, append(refs[:i], refs[i+1:]...)) + } + } + return nil + }) +} + +// File performs one claimed and fenced filesystem operation. +func (s *Store) File(ctx context.Context, client, owner string, q executionenv.FileRequest) (executionenv.FileResponse, error) { + req := executionenv.ExecutorRequest{Operation: q.Operation, Path: q.Path, Destination: q.Destination, Pattern: q.Pattern, Data: q.Data, Version: q.Version, Limit: q.Limit} + resp, err := s.execute(ctx, client, owner, q.Context, req) + return resp.FileResponse, err +} + +// StartCommand performs one claimed foreground command through terminal receipt. +func (s *Store) StartCommand(ctx context.Context, client, owner string, q executionenv.CommandStartRequest) (executionenv.CommandStartResponse, error) { + id, err := randomID() + if err != nil { + return executionenv.CommandStartResponse{}, err + } + resp, err := s.execute(ctx, client, owner, q.Context, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, Command: q.Command, CommandID: id, TimeoutMillis: q.TimeoutMillis}) + if err != nil { + return executionenv.CommandStartResponse{}, err + } + if resp.Command == nil { + return executionenv.CommandStartResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "executor returned no terminal command receipt"} + } + return executionenv.CommandStartResponse{CommandID: id, State: resp.Command.State, Result: *resp.Command}, nil +} + +// CommandStatus reports that foreground-only commands have no detached status API. +func (*Store) CommandStatus(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "foreground commands have no resumable status stream"} +} + +// CancelCommand refuses cancellation without an attached foreground control stream. +func (*Store) CancelCommand(context.Context, string, string, executionenv.CommandQueryRequest) (executionenv.CommandStatusResponse, error) { + return executionenv.CommandStatusResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "no active foreground stream is attached; external fencing is required"} +} +func (s *Store) execute(ctx context.Context, client, owner string, rc executionenv.RequestContext, req executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { //nolint:gocyclo // Atomic claim, dispatch, and fencing remain one auditable transaction. + if rc.Epoch == 0 || rc.Epoch > math.MaxInt64 || rc.GrantGeneration == 0 || rc.GrantGeneration > math.MaxInt64 { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "invalid execution epoch or grant generation"} + } + epochStatus := int64(rc.Epoch) //nolint:gosec // bounded above by MaxInt64. + if s.executor == nil { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeNotReady, Message: "executor transport unavailable", Retryable: true} + } + opID, err := randomID() + if err != nil { + return executionenv.ExecutorResponse{}, err + } + var pod string + var maxFileBytes, maxCommandBytes int64 + err = s.retryUpdateStatus(ctx, rc.Environment.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + profile, ok := s.profiles.get(textNested(o.Object, "spec", "profile")) + if !ok || profile.Digest != textNested(o.Object, "spec", "profileDigest") { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment profile is unavailable"} + } + maxFileBytes, maxCommandBytes = profile.Spec.MaxFileBytes, profile.Spec.MaxCommandBytes + if int64(len(req.Data)) > maxFileBytes || int64(len(req.Command)) > maxCommandBytes { + return &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "operation exceeds profile bounds"} + } + if req.Operation == executionenv.OpCommandStart { + maxMillis := profile.Spec.MaxCommandDuration.Milliseconds() + if req.TimeoutMillis <= 0 { + req.TimeoutMillis = maxMillis + } else if req.TimeoutMillis > maxMillis { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "command timeout exceeds profile bound"} + } + } + refs, refsErr := referenceRecords(o) + if refsErr != nil { + return refsErr + } + claim, _, expiry, claimOK := activeRunFrom(o) + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || !publishedReference(refs, rc.BindingID) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if !claimOK || !s.now().Before(expiry) || claim.BindingID != rc.BindingID || claim.RunID != rc.RunID || claim.ClaimID != rc.ClaimID || claim.Epoch != rc.Epoch || claim.GrantGeneration != rc.GrantGeneration || !generationMatches(o, rc.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim is not current"} + } + if textNested(o.Object, "spec", "revision") != rc.Environment.Revision || !epochMatches(o, rc.Epoch) || textNested(o.Object, "spec", "desired") != "Active" || !conditionTrue(o, "Ready") || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment is not ready at requested epoch"} + } + if textNested(o.Object, "status", "activeOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment already has an active operation", Retryable: true} + } + pod = textNested(o.Object, "status", "pod", "name") + if pod == "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "executor pod is unavailable", Retryable: true} + } + now := s.now() + return unstructured.SetNestedMap(o.Object, map[string]any{"id": opID, "operation": string(req.Operation), "startedAt": now.Format(time.RFC3339Nano), "claimID": rc.ClaimID, "runID": rc.RunID, "epoch": epochStatus, "holderID": s.holderID, "renewedAt": now.Format(time.RFC3339Nano), "expiresAt": now.Add(s.opTTL).Format(time.RFC3339Nano)}, "status", "activeOperation") + }) + if err != nil { + return executionenv.ExecutorResponse{}, err + } + authority := operationLeaseAuthority{ + environment: rc.Environment.ID, revision: rc.Environment.Revision, operationID: opID, + bindingID: rc.BindingID, runID: rc.RunID, claimID: rc.ClaimID, + client: client, owner: owner, epoch: rc.Epoch, grantGeneration: rc.GrantGeneration, + } + current, err := s.resources.Get(ctx, rc.Environment.ID, metav1.GetOptions{}) + if err != nil || !operationLeaseCurrent(current, authority, s.holderID, s.now()) || !s.operationAuthorityCurrent(current, authority) { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "operation identity changed before executor dispatch"} + } + opCtx, stopExecution := context.WithCancel(ctx) + leaseDone := make(chan error, 1) + go func() { + leaseErr := s.renewOperationLease(opCtx, authority) + if leaseErr != nil { + stopExecution() + } + leaseDone <- leaseErr + }() + resp, dispatchErr := s.executor.Execute(opCtx, pod, req) + stopExecution() + leaseErr := <-leaseDone + authorityLost := leaseErr != nil + if authorityLost { + fenceErr := &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "operation lease renewal failed"} + if dispatchErr != nil { + dispatchErr = errors.Join(fenceErr, dispatchErr) + } else { + dispatchErr = fenceErr + } + } + if dispatchErr == nil && (int64(len(resp.Data)) > maxFileBytes || resp.Command != nil && int64(len(resp.Command.Stdout)+len(resp.Command.Stderr)) > maxCommandBytes) { + dispatchErr = &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "executor response exceeds profile bound"} + } + var controlledErr *executionenv.Error + controlledTerminal := !authorityLost && errors.As(dispatchErr, &controlledErr) && controlledErr.Code != executionenv.CodeFenceUnknown + terminal := !authorityLost && (controlledTerminal || dispatchErr == nil && (req.Operation != executionenv.OpCommandStart || resp.Command != nil && resp.Command.TerminalReceipt != "")) + completionAuthorityLost := false + finishErr := s.retryUpdateStatus(context.WithoutCancel(ctx), rc.Environment.ID, func(o *unstructured.Unstructured) error { + if !operationMatches(o, authority, s.holderID) { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "operation ownership changed before completion"} + } + if terminal && operationLeaseCurrent(o, authority, s.holderID, s.now()) && s.operationAuthorityCurrent(o, authority) { + unstructured.RemoveNestedField(o.Object, "status", "activeOperation") + return nil + } + if terminal { + completionAuthorityLost = true + } + return unstructured.SetNestedField(o.Object, "FenceUnknown", "status", "fenceState") + }) + if finishErr != nil { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "could not persist terminal operation state"} + } + if completionAuthorityLost { + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "operation authority changed before completion"} + } + if controlledTerminal { + return executionenv.ExecutorResponse{}, controlledErr + } + if !terminal { + if authorityLost { + return executionenv.ExecutorResponse{}, dispatchErr + } + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "executor termination is unconfirmed; environment fenced", Retryable: false} + } + return resp, nil +} +func (s *Store) retryUpdateStatus(ctx context.Context, name string, mutate func(*unstructured.Unstructured) error) error { + return s.retryUpdateStatusRaw(ctx, name, func(o *unstructured.Unstructured) error { + if err := requireCurrentSchema(o); err != nil { + return err + } + return mutate(o) + }) +} + +func (s *Store) retryUpdateStatusRaw(ctx context.Context, name string, mutate func(*unstructured.Unstructured) error) error { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + cur, err := s.resources.Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if err != nil { + return err + } + if err := mutate(cur); err != nil { + return err + } + _, err = s.resources.UpdateStatus(ctx, cur, metav1.UpdateOptions{}) + return err + }) + if apierrors.IsConflict(err) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment changed concurrently", Retryable: true} + } + return err +} +func requireCurrentSchema(o *unstructured.Unstructured) error { + if intNested(o.Object, "spec", "schemaVersion") != currentSchemaVersion || intNested(o.Object, "status", "schemaVersion") != currentSchemaVersion { + return incompatibleSchemaError() + } + return nil +} + +func incompatibleSchemaError() error { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment schema is incompatible; explicit migration is required"} +} + +func epochValue(o *unstructured.Unstructured) (uint64, bool) { + epoch, found, err := unstructured.NestedInt64(o.Object, "status", "epoch") + if err != nil || !found || epoch <= 0 { + return 0, false + } + return uint64(epoch), true +} + +func epochMatches(o *unstructured.Unstructured, expected uint64) bool { + epoch, ok := epochValue(o) + return ok && epoch == expected +} + +func conditionTrue(o *unstructured.Unstructured, name string) bool { + conds, _, _ := unstructured.NestedSlice(o.Object, "status", "conditions") + for _, raw := range conds { + m, ok := raw.(map[string]any) + if ok && text(m, "type") == name && text(m, "status") == "True" { + return true + } + } + return false +} +func text(m map[string]any, k string) string { v, _ := m[k].(string); return v } +func textNested(m map[string]any, p ...string) string { + v, _, _ := unstructured.NestedString(m, p...) + return v +} +func intNested(m map[string]any, p ...string) int64 { + v, _, _ := unstructured.NestedInt64(m, p...) + return v +} +func hashText(v string) string { sum := sha256.Sum256([]byte(v)); return hex.EncodeToString(sum[:]) } +func randomID() (string, error) { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} +func contains(v []string, s string) bool { + for _, x := range v { + if x == s { + return true + } + } + return false +} + +var _ Backend = (*Store)(nil) diff --git a/internal/adapter/executioncontroller/store_concurrency_test.go b/internal/adapter/executioncontroller/store_concurrency_test.go new file mode 100644 index 0000000000..65c577f1bb --- /dev/null +++ b/internal/adapter/executioncontroller/store_concurrency_test.go @@ -0,0 +1,237 @@ +package executioncontroller + +import ( + "context" + "errors" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func profileSlots(t *testing.T, kube *kubefake.Clientset) []string { + t.Helper() + cm, err := kube.CoreV1().ConfigMaps("ns").Get(t.Context(), profileAllocationConfigMap, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + var slots []string + if err := executionenv.DecodeStrict([]byte(cm.Data[profileAllocationKey("go")]), &slots); err != nil { + t.Fatal(err) + } + return slots +} + +func TestDefinitiveEnvironmentCreateFailureReleasesProfileSlot(t *testing.T) { + dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + dynamicClient.PrependReactor("create", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: ExecutionEnvironmentGVR.Group, Resource: ExecutionEnvironmentGVR.Resource}, "env", errors.New("policy")) + }) + kube := kubefake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{}}) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kube) + if _, err := store.EnsurePending(t.Context(), "client", "owner", "binding", "go", "fp", "op"); err == nil { + t.Fatal("definitive create rejection succeeded") + } + if slots := profileSlots(t, kube); len(slots) != 0 { + t.Fatalf("definitive create rejection leaked slots: %v", slots) + } +} + +func TestAmbiguousEnvironmentCreateFailureRetainsProfileSlot(t *testing.T) { + dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + dynamicClient.PrependReactor("create", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("transport outcome unknown") + }) + kube := kubefake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{}}) + store := NewStore(dynamicClient, "ns", testProfiles(), nil).WithKubeClient(kube) + if _, err := store.EnsurePending(t.Context(), "client", "owner", "binding", "go", "fp", "op"); err == nil { + t.Fatal("ambiguous create failure succeeded") + } + if slots := profileSlots(t, kube); len(slots) != 1 { + t.Fatalf("ambiguous create outcome did not retain conservative capacity: %v", slots) + } +} + +func TestRetainedDeleteWaitsForSlotReleaseBeforeRemovingCRFinalizer(t *testing.T) { + env := lifecycleAdminEnvironment(2, []any{}) + setConditionObject(env, "Retired", true, "WorkspaceRetained", "retained") + setConditionObject(env, "ExecutorTerminated", true, "TerminalPodProof", "proved") + _ = unstructured.SetNestedMap(env.Object, map[string]any{"id": "delete", "type": "DeleteRetiredEnvironment", "phase": "DeletingPVC", "expectedPVCUID": "pvc-uid", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}, "status", "lifecycleOperation") + dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + allocationID := env.GetName() + raw := `["` + allocationID + `"]` + kube := kubefake.NewSimpleClientset(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns"}, Data: map[string]string{profileAllocationKey("go"): raw}}) + var fail atomic.Bool + fail.Store(true) + kube.PrependReactor("get", "configmaps", func(k8stesting.Action) (bool, runtime.Object, error) { + if fail.Load() { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Resource: "configmaps"}, profileAllocationConfigMap, errors.New("outage")) + } + return false, nil, nil + }) + r := NewReconciler(dynamicClient, kube, "ns", testProfiles()) + if err := r.Reconcile(t.Context(), env.GetName()); err != nil { + t.Fatalf("failed to persist deallocation phase: %v", err) + } + if err := r.Reconcile(t.Context(), env.GetName()); err == nil { + t.Fatal("slot authority outage did not stop deletion") + } + got, err := dynamicClient.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), env.GetName(), metav1.GetOptions{}) + if err != nil || !contains(got.GetFinalizers(), environmentFinalizer) { + t.Fatalf("CR was not retained for retry: finalizers=%v err=%v", got.GetFinalizers(), err) + } + fail.Store(false) + if err := r.Reconcile(t.Context(), env.GetName()); err != nil { + t.Fatal(err) + } + if slots := profileSlots(t, kube); len(slots) != 0 { + t.Fatalf("retry did not release slot: %v", slots) + } +} + +func TestConcurrentProfileReservationsEnforceHardLimit(t *testing.T) { + dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + ledger := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: profileAllocationConfigMap, Namespace: "ns", ResourceVersion: "1"}, Data: map[string]string{}} + kubes := []*kubefake.Clientset{kubefake.NewSimpleClientset(ledger), kubefake.NewSimpleClientset(ledger)} + var mu sync.Mutex + current := ledger.DeepCopy() + barrier := make(chan struct{}) + var reads atomic.Int32 + for _, kube := range kubes { + kube.PrependReactor("get", "configmaps", func(k8stesting.Action) (bool, runtime.Object, error) { + mu.Lock() + out := current.DeepCopy() + mu.Unlock() + if reads.Add(1) == 2 { + close(barrier) + } + if reads.Load() <= 2 { + <-barrier + } + return true, out, nil + }) + kube.PrependReactor("update", "configmaps", func(action k8stesting.Action) (bool, runtime.Object, error) { + candidate := action.(k8stesting.UpdateAction).GetObject().(*corev1.ConfigMap) + mu.Lock() + defer mu.Unlock() + if candidate.ResourceVersion != current.ResourceVersion { + return true, nil, apierrors.NewConflict(schema.GroupResource{Resource: "configmaps"}, candidate.Name, errors.New("resource version changed")) + } + rv, _ := strconv.Atoi(current.ResourceVersion) + current = candidate.DeepCopy() + current.ResourceVersion = strconv.Itoa(rv + 1) + return true, current.DeepCopy(), nil + }) + } + profiles := testProfiles() + profile := profiles.byName["go"] + profile.Spec.MaxEnvironments = 1 + profiles.byName["go"] = profile + stores := []*Store{NewStore(dynamicClient, "ns", profiles, nil).WithKubeClient(kubes[0]), NewStore(dynamicClient, "ns", profiles, nil).WithKubeClient(kubes[1])} + results := make(chan error, 2) + for i := range stores { + go func(i int) { + _, err := stores[i].EnsurePending(context.Background(), "client", "owner", "binding-"+strconv.Itoa(i), "go", "fp-"+strconv.Itoa(i), "op-"+strconv.Itoa(i)) + results <- err + }(i) + } + admitted, exhausted := 0, 0 + for range 2 { + err := <-results + var controlled *executionenv.Error + if err == nil { + admitted++ + } else if errors.As(err, &controlled) && controlled.Code == executionenv.CodeResourceExhausted { + exhausted++ + } else { + t.Fatalf("unexpected result: %v", err) + } + } + if admitted != 1 || exhausted != 1 { + t.Fatalf("admitted=%d exhausted=%d", admitted, exhausted) + } +} + +func TestAcquireRunAndRevokeUseResourceVersionCAS(t *testing.T) { + now := time.Date(2026, 9, 17, 5, 0, 0, 0, time.UTC) + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns", "resourceVersion": "1"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": fenceHealthy, "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} + clients := []*dynamicfake.FakeDynamicClient{ + dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env), + dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env), + } + var mu sync.Mutex + current := env.DeepCopy() + firstReads := make(chan struct{}) + var reads, conflicts atomic.Int32 + for _, client := range clients { + client.PrependReactor("get", "executionenvironments", func(k8stesting.Action) (bool, runtime.Object, error) { + mu.Lock() + out := current.DeepCopy() + mu.Unlock() + if reads.Add(1) == 2 { + close(firstReads) + } + if reads.Load() <= 2 { + <-firstReads + } + return true, out, nil + }) + client.PrependReactor("update", "executionenvironments", func(action k8stesting.Action) (bool, runtime.Object, error) { + candidate := action.(k8stesting.UpdateAction).GetObject().(*unstructured.Unstructured) + mu.Lock() + defer mu.Unlock() + if candidate.GetResourceVersion() != current.GetResourceVersion() { + conflicts.Add(1) + return true, nil, apierrors.NewConflict(schema.GroupResource{Group: ExecutionEnvironmentGVR.Group, Resource: ExecutionEnvironmentGVR.Resource}, candidate.GetName(), errors.New("resource version changed")) + } + rv, _ := strconv.Atoi(current.GetResourceVersion()) + candidate = candidate.DeepCopy() + candidate.SetResourceVersion(strconv.Itoa(rv + 1)) + current = candidate + return true, candidate.DeepCopy(), nil + }) + } + first := NewStore(clients[0], "ns", testProfiles(), nil) + second := NewStore(clients[1], "ns", testProfiles(), nil) + first.now, second.now = func() time.Time { return now }, func() time.Time { return now } + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + results := make(chan error, 2) + go func() { + _, err := first.AcquireRun(context.Background(), ref, "client", "owner", "binding", "run", "acquire", executionenv.MinRunTTL) + results <- err + }() + go func() { + _, err := second.RevokeEnvironment(context.Background(), adminLifecycleRequest{Environment: ref, Client: "client", OwnerHash: "owner", OperationID: "revoke"}, 1) + results <- err + }() + for range 2 { + if err := <-results; err != nil { + t.Fatalf("CAS retry did not converge: %v", err) + } + } + if conflicts.Load() == 0 { + t.Fatal("test did not force a resource-version conflict") + } + mu.Lock() + final := current.DeepCopy() + mu.Unlock() + if generation := intNested(final.Object, "status", "grantGeneration"); generation != 2 { + t.Fatalf("grant generation=%d, want 2", generation) + } + if claim, _, _, ok := activeRunFrom(final); ok && generationMatches(final, claim.GrantGeneration) && claim.GrantGeneration != 2 { + t.Fatalf("stale claim remained current after revoke: %+v", claim) + } +} diff --git a/internal/adapter/executioncontroller/store_lifecycle.go b/internal/adapter/executioncontroller/store_lifecycle.go new file mode 100644 index 0000000000..a985c4f2d7 --- /dev/null +++ b/internal/adapter/executioncontroller/store_lifecycle.go @@ -0,0 +1,283 @@ +//nolint:revive // Private store methods implement adapter-only lifecycle interfaces. +package executioncontroller + +import ( + "context" + "sort" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +type referenceRecord struct { + BindingID string + State executionenv.ReferenceState + OperationID string + SourceBindingID string + CreatedAt time.Time +} + +func referenceRecords(o *unstructured.Unstructured) ([]referenceRecord, error) { + raw, found, err := unstructured.NestedSlice(o.Object, "status", "references") + if err != nil || !found { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment references are invalid"} + } + out := make([]referenceRecord, 0, len(raw)) + for _, item := range raw { + m, ok := item.(map[string]any) + if !ok { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "legacy environment references require explicit migration"} + } + created, parseErr := time.Parse(time.RFC3339Nano, text(m, "createdAt")) + r := referenceRecord{BindingID: text(m, "bindingID"), State: executionenv.ReferenceState(text(m, "state")), OperationID: text(m, operationIDField), SourceBindingID: text(m, "sourceBindingID"), CreatedAt: created} + if parseErr != nil || r.BindingID == "" || r.OperationID == "" || (r.State != executionenv.ReferencePendingCreate && r.State != executionenv.ReferencePublished && r.State != executionenv.ReferencePendingDelete) { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment references are invalid"} + } + out = append(out, r) + } + return out, nil +} + +func setReferenceRecords(o *unstructured.Unstructured, refs []referenceRecord) error { + sort.Slice(refs, func(i, j int) bool { return refs[i].BindingID < refs[j].BindingID }) + raw := make([]any, len(refs)) + for i, r := range refs { + raw[i] = map[string]any{"bindingID": r.BindingID, "state": string(r.State), operationIDField: r.OperationID, "sourceBindingID": r.SourceBindingID, "createdAt": r.CreatedAt.UTC().Format(time.RFC3339Nano)} + } + return unstructured.SetNestedSlice(o.Object, raw, "status", "references") +} + +func referenceOperationMatches(o *unstructured.Unstructured, binding, operation string) bool { + refs, err := referenceRecords(o) + if err != nil { + return false + } + for _, r := range refs { + if r.BindingID == binding { + return r.OperationID == operation && (r.State == executionenv.ReferencePendingCreate || r.State == executionenv.ReferencePublished) + } + } + return false +} + +func attachedReference(refs []referenceRecord, binding string) bool { + for _, r := range refs { + if r.BindingID == binding && (r.State == executionenv.ReferencePendingCreate || r.State == executionenv.ReferencePublished) { + return true + } + } + return false +} + +func publishedReference(refs []referenceRecord, binding string) bool { + for _, r := range refs { + if r.BindingID == binding && r.State == executionenv.ReferencePublished { + return true + } + } + return false +} + +func (s *Store) mutateReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string, from, to executionenv.ReferenceState, remove bool) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, err := referenceRecords(o) + if err != nil { + return err + } + for i := range refs { + if refs[i].BindingID != binding { + continue + } + if refs[i].OperationID == operation && refs[i].State == to { + return nil + } + if refs[i].OperationID == operation && refs[i].State == from { + if remove { + refs = append(refs[:i], refs[i+1:]...) + } else { + refs[i].State = to + } + return setReferenceRecords(o, refs) + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference transaction mismatch"} + } + if remove { + return nil + } + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference not found"} + }) +} + +func (s *Store) CommitReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.mutateReference(ctx, ref, client, owner, binding, operation, executionenv.ReferencePendingCreate, executionenv.ReferencePublished, false) +} +func (s *Store) AbortReference(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.mutateReference(ctx, ref, client, owner, binding, operation, executionenv.ReferencePendingCreate, "", true) +} +func (s *Store) PrepareReferenceDelete(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, err := referenceRecords(o) + if err != nil { + return err + } + for i := range refs { + if refs[i].BindingID != binding { + continue + } + if refs[i].State == executionenv.ReferencePendingDelete && refs[i].OperationID == operation { + return nil + } + if refs[i].State != executionenv.ReferencePublished { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference transaction mismatch"} + } + refs[i].State, refs[i].OperationID, refs[i].CreatedAt = executionenv.ReferencePendingDelete, operation, time.Now().UTC() + return setReferenceRecords(o, refs) + } + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference not found"} + }) +} +func (s *Store) ConfirmReferenceDelete(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.mutateReference(ctx, ref, client, owner, binding, operation, executionenv.ReferencePendingDelete, "", true) +} +func (s *Store) CancelReferenceDelete(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, operation string) error { + return s.mutateReference(ctx, ref, client, owner, binding, operation, executionenv.ReferencePendingDelete, executionenv.ReferencePublished, false) +} + +func (s *Store) ReserveSuccessor(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, source, destination, operation string) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, err := referenceRecords(o) + if err != nil { + return err + } + if !publishedReference(refs, source) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: "source reference not found"} + } + if textNested(o.Object, "status", "activeOperation", "id") != "" || textNested(o.Object, "status", "activeRun", "claimID") != "" || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment is not quiescent"} + } + for _, r := range refs { + if r.BindingID == destination { + if r.State == executionenv.ReferencePendingCreate && r.OperationID == operation && r.SourceBindingID == source { + return nil + } + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "destination reference conflicts"} + } + } + if len(refs) >= maxReferences { + return &executionenv.Error{Code: executionenv.CodeResourceExhausted, Message: "environment reference limit reached"} + } + refs = append(refs, referenceRecord{BindingID: destination, State: executionenv.ReferencePendingCreate, OperationID: operation, SourceBindingID: source, CreatedAt: time.Now().UTC()}) + return setReferenceRecords(o, refs) + }) +} + +func (s *Store) FindReferenceIntent(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding string) (executionenv.ReferenceIntent, error) { + o, err := s.resources.Get(ctx, ref.ID, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return executionenv.ReferenceIntent{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference intent not found"} + } + if err != nil { + return executionenv.ReferenceIntent{}, err + } + if textNested(o.Object, "spec", "revision") != ref.Revision || textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) { + return executionenv.ReferenceIntent{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference intent not found"} + } + refs, err := referenceRecords(o) + if err != nil { + return executionenv.ReferenceIntent{}, err + } + for _, r := range refs { + if r.BindingID == binding && r.State != executionenv.ReferencePublished { + return executionenv.ReferenceIntent{Environment: ref, BindingID: r.BindingID, State: r.State, OperationID: r.OperationID, SourceBindingID: r.SourceBindingID, CreatedAt: r.CreatedAt}, nil + } + } + return executionenv.ReferenceIntent{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "reference intent not found"} +} + +func (s *Store) ListReferenceIntentsForClient(ctx context.Context, client string, limit int) ([]executionenv.ReferenceIntent, error) { + if limit <= 0 || limit > maxReferences { + limit = maxReferences + } + list, err := s.resources.List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + out := make([]executionenv.ReferenceIntent, 0) + for i := range list.Items { + o := &list.Items[i] + if textNested(o.Object, "spec", "clientHash") != hashText(client) { + continue + } + owner := executionenv.Owner{Issuer: textNested(o.Object, "spec", "ownerIssuer"), Subject: textNested(o.Object, "spec", "ownerSubject")} + if owner.Issuer == "" || owner.Subject == "" || ownerHash(owner) != textNested(o.Object, "spec", "ownerHash") { + return nil, &executionenv.Error{Code: executionenv.CodeConflict, Message: "reference intent owner attestation is unavailable"} + } + refs, parseErr := referenceRecords(o) + if parseErr != nil { + return nil, parseErr + } + for _, r := range refs { + if r.State == executionenv.ReferencePublished { + continue + } + out = append(out, executionenv.ReferenceIntent{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: textNested(o.Object, "spec", "revision")}, Owner: owner, BindingID: r.BindingID, State: r.State, OperationID: r.OperationID, SourceBindingID: r.SourceBindingID, CreatedAt: r.CreatedAt}) + if len(out) == limit { + return out, nil + } + } + } + return out, nil +} + +func (s *Store) ListReferenceIntents(ctx context.Context, client, owner string, limit int) ([]executionenv.ReferenceIntent, error) { + if limit <= 0 || limit > maxReferences { + limit = maxReferences + } + list, err := s.resources.List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + out := make([]executionenv.ReferenceIntent, 0) + for i := range list.Items { + o := &list.Items[i] + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) { + continue + } + refs, parseErr := referenceRecords(o) + if parseErr != nil { + return nil, parseErr + } + for _, r := range refs { + if r.State == executionenv.ReferencePublished { + continue + } + out = append(out, executionenv.ReferenceIntent{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: textNested(o.Object, "spec", "revision")}, BindingID: r.BindingID, State: r.State, OperationID: r.OperationID, SourceBindingID: r.SourceBindingID, CreatedAt: r.CreatedAt}) + if len(out) == limit { + return out, nil + } + } + } + return out, nil +} diff --git a/internal/adapter/executioncontroller/store_lifecycle_test.go b/internal/adapter/executioncontroller/store_lifecycle_test.go new file mode 100644 index 0000000000..929f66aab9 --- /dev/null +++ b/internal/adapter/executioncontroller/store_lifecycle_test.go @@ -0,0 +1,300 @@ +package executioncontroller + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func lifecycleEnvironment(now time.Time) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", + "metadata": map[string]any{"name": "env", "namespace": "ns"}, + "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, + "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "references": []any{ + map[string]any{"bindingID": "source", "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}, + }, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}, + }} +} + +func TestEnsurePendingOwnedPersistsOwnerAttestation(t *testing.T) { + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + store := NewStore(client, "ns", testProfiles(), nil) + owner := executionenv.Owner{Issuer: "https://issuer.example", Subject: "alice"} + allocation, err := store.EnsurePendingOwned(t.Context(), "client", ownerHash(owner), owner, "binding", "go", "fingerprint", "ensure-operation") + if err != nil { + t.Fatal(err) + } + env, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), allocation.Environment.ID, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(env.Object, "spec", "ownerIssuer") != owner.Issuer || textNested(env.Object, "spec", "ownerSubject") != owner.Subject { + t.Fatalf("owner attestation was not persisted: %v", env.Object["spec"]) + } +} + +func TestPublishedBindingReattachesWithoutNewEnsureOperation(t *testing.T) { + ctx := t.Context() + client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}) + store := NewStore(client, "ns", testProfiles(), nil) + owner := executionenv.Owner{Issuer: "https://issuer.example", Subject: "alice"} + allocation, err := store.EnsurePendingOwned(ctx, "client", ownerHash(owner), owner, "binding", "go", "fingerprint", "create-operation") + if err != nil { + t.Fatal(err) + } + ref := allocation.Environment + if err := store.CommitReference(ctx, ref, "client", ownerHash(owner), "binding", "create-operation"); err != nil { + t.Fatal(err) + } + if err := store.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + setConditionObject(o, "Ready", true, "Ready", "fixture ready") + return nil + }); err != nil { + t.Fatal(err) + } + resources := client.Resource(ExecutionEnvironmentGVR).Namespace("ns") + before, err := resources.List(ctx, metav1.ListOptions{}) + if err != nil || len(before.Items) != 1 { + t.Fatalf("allocation list: %v, err=%v", before, err) + } + _, err = store.EnsurePendingOwned(ctx, "client", ownerHash(owner), owner, "binding", "go", "fingerprint", "qualification-reattach") + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("changed Ensure operation used as lookup: err=%v", err) + } + lookup := executionenv.EnvironmentRef{ID: before.Items[0].GetName(), Revision: textNested(before.Items[0].Object, "spec", "revision")} + attached, err := store.Attach(ctx, lookup, "client", ownerHash(owner), "binding") + if err != nil || !attached.Ready || attached.Environment != ref || attached.BindingID != "binding" || attached.OwnerHash != ownerHash(owner) { + t.Fatalf("exact published binding did not reattach: %+v, err=%v", attached, err) + } + for _, tc := range []struct { + name string + ref executionenv.EnvironmentRef + owner string + binding string + }{ + {"wrong owner", lookup, "other-owner", "binding"}, + {"wrong session", lookup, ownerHash(owner), "other-binding"}, + {"wrong revision", executionenv.EnvironmentRef{ID: lookup.ID, Revision: "other-revision"}, ownerHash(owner), "binding"}, + {"missing reference", executionenv.EnvironmentRef{}, ownerHash(owner), "binding"}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := store.Attach(ctx, tc.ref, "client", tc.owner, tc.binding); err == nil { + t.Fatal("inexact lookup was accepted") + } + }) + } + after, err := resources.List(ctx, metav1.ListOptions{}) + if err != nil || !reflect.DeepEqual(before.Items, after.Items) { + t.Fatalf("lookup changed allocations or published identity: err=%v", err) + } +} + +func TestRunClaimIsEnvironmentWideAndExact(t *testing.T) { + now := time.Now().UTC() + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), lifecycleEnvironment(now)) + store := NewStore(client, "ns", testProfiles(), nil) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + first, err := store.AcquireRun(t.Context(), ref, "client", "owner", "source", "run-a", "op-a", time.Minute) + if err != nil { + t.Fatal(err) + } + if first.Epoch != 2 || first.ClaimID == "" || first.GrantGeneration != 1 { + t.Fatalf("claim=%+v", first) + } + repeat, err := store.AcquireRun(t.Context(), ref, "client", "owner", "source", "run-a", "op-a", time.Minute) + if err != nil || repeat.ClaimID != first.ClaimID || repeat.Epoch != first.Epoch { + t.Fatalf("repeat=%+v err=%v", repeat, err) + } + if _, err := store.AcquireRun(t.Context(), ref, "client", "owner", "source", "run-b", "op-b", time.Minute); err == nil { + t.Fatal("overlapping claim admitted") + } + stale := executionenv.RunClaimRequest{Environment: ref, BindingID: "source", RunID: "run-a", ClaimID: "wrong", Epoch: first.Epoch, GrantGeneration: first.GrantGeneration, OperationID: "release", TTL: time.Minute} + if err := store.ReleaseRun(t.Context(), ref, "client", "owner", stale); err == nil { + t.Fatal("stale release admitted") + } + stale.ClaimID = first.ClaimID + if err := store.ReleaseRun(t.Context(), ref, "client", "owner", stale); err != nil { + t.Fatal(err) + } + second, err := store.AcquireRun(t.Context(), ref, "client", "owner", "source", "run-b", "op-b", time.Minute) + if err != nil || second.Epoch != first.Epoch+1 { + t.Fatalf("second=%+v err=%v", second, err) + } +} + +func TestExpiredClaimWithActiveOperationCannotBeReplaced(t *testing.T) { + now := time.Now().UTC() + env := lifecycleEnvironment(now) + statusMap, _, _ := unstructured.NestedMap(env.Object, "status") + statusMap["epoch"] = int64(4) + statusMap["activeRun"] = map[string]any{"bindingID": "source", "runID": "old", "claimID": "old-claim", "operationID": "old-acquire", "ownerHash": "owner", "clientHash": hashText("client"), "epoch": int64(4), "grantGeneration": int64(1), "expiresAt": now.Add(-time.Minute).Format(time.RFC3339Nano)} + statusMap["activeOperation"] = map[string]any{"id": "still-running", "claimID": "old-claim", "runID": "old", "epoch": int64(4)} + env.Object["status"] = statusMap + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + _, err := store.AcquireRun(t.Context(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "source", "new", "new-op", time.Minute) + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeFenceUnknown { + t.Fatalf("error=%v", err) + } +} + +func TestExactReferenceIntentLookupIsNotTruncatedByGlobalLimit(t *testing.T) { + now := time.Now().UTC() + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + objects := make([]runtime.Object, 0, 67) + for i := 0; i < 66; i++ { + env := lifecycleEnvironment(now) + env.SetName(fmt.Sprintf("env-%02d", i)) + _ = unstructured.SetNestedField(env.Object, fmt.Sprintf("rev-%02d", i), "spec", "revision") + _ = unstructured.SetNestedField(env.Object, ownerHash(owner), "spec", "ownerHash") + _ = unstructured.SetNestedField(env.Object, owner.Issuer, "spec", "ownerIssuer") + _ = unstructured.SetNestedField(env.Object, owner.Subject, "spec", "ownerSubject") + if err := setReferenceRecords(env, []referenceRecord{{BindingID: fmt.Sprintf("binding-%02d", i), State: executionenv.ReferencePendingDelete, OperationID: fmt.Sprintf("delete-%02d", i), CreatedAt: now}}); err != nil { + t.Fatal(err) + } + objects = append(objects, env) + } + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), objects...) + store := NewStore(client, "ns", testProfiles(), nil) + target := executionenv.EnvironmentRef{ID: "env-65", Revision: "rev-65"} + intent, err := store.FindReferenceIntent(t.Context(), target, "client", ownerHash(owner), "binding-65") + if err != nil || intent.OperationID != "delete-65" { + t.Fatalf("exact target beyond global list limit: intent=%+v err=%v", intent, err) + } + if _, err := store.FindReferenceIntent(t.Context(), target, "other-client", ownerHash(owner), "binding-65"); err == nil { + t.Fatal("other client enumerated exact intent") + } + if _, err := store.FindReferenceIntent(t.Context(), target, "client", ownerHash(executionenv.Owner{Issuer: "issuer", Subject: "mallory"}), "binding-65"); err == nil { + t.Fatal("other owner enumerated exact intent") + } +} + +func TestClientScopedIntentReconciliationRejectsEachOwnerMutationWithoutSideEffects(t *testing.T) { + now := time.Now().UTC() + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + mutations := map[string]func(*unstructured.Unstructured){ + "owner hash": func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, hashText("other-owner"), "spec", "ownerHash") + }, + "owner issuer": func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, "other-issuer", "spec", "ownerIssuer") + }, + "owner subject": func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, "mallory", "spec", "ownerSubject") + }, + "client binding": func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, hashText("other-client"), "spec", "clientHash") + }, + } + for name, mutate := range mutations { + t.Run(name, func(t *testing.T) { + env := lifecycleEnvironment(now) + _ = unstructured.SetNestedField(env.Object, ownerHash(owner), "spec", "ownerHash") + _ = unstructured.SetNestedField(env.Object, owner.Issuer, "spec", "ownerIssuer") + _ = unstructured.SetNestedField(env.Object, owner.Subject, "spec", "ownerSubject") + refs, err := referenceRecords(env) + if err != nil { + t.Fatal(err) + } + refs = append(refs, referenceRecord{BindingID: "pending", State: executionenv.ReferencePendingDelete, OperationID: "delete", CreatedAt: now}) + if err := setReferenceRecords(env, refs); err != nil { + t.Fatal(err) + } + mutate(env) + before := env.DeepCopy() + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + intents, listErr := store.ListReferenceIntentsForClient(t.Context(), "client", 64) + if listErr == nil && len(intents) != 0 { + t.Fatalf("mutated identity exposed intents: %+v", intents) + } + after, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before.Object, after.Object) { + t.Fatal("negative reconciliation mutated durable state") + } + }) + } +} + +func TestClientScopedIntentListReturnsAttestedOwnerOnlyToOwningClient(t *testing.T) { + now := time.Now().UTC() + env := lifecycleEnvironment(now) + owner := executionenv.Owner{Issuer: "issuer", Subject: "alice"} + if err := unstructured.SetNestedField(env.Object, ownerHash(owner), "spec", "ownerHash"); err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(env.Object, owner.Issuer, "spec", "ownerIssuer"); err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedField(env.Object, owner.Subject, "spec", "ownerSubject"); err != nil { + t.Fatal(err) + } + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + if err := store.ReserveSuccessor(t.Context(), ref, "client", ownerHash(owner), "source", "destination", "fork-op"); err != nil { + t.Fatal(err) + } + intents, err := store.ListReferenceIntentsForClient(t.Context(), "client", 64) + if err != nil || len(intents) != 1 || intents[0].Owner != owner || intents[0].BindingID != "destination" { + t.Fatalf("intents=%+v err=%v", intents, err) + } + other, err := store.ListReferenceIntentsForClient(t.Context(), "other-client", 64) + if err != nil || len(other) != 0 { + t.Fatalf("other-client intents=%+v err=%v", other, err) + } +} + +func TestReferenceTransactionsRetainUnknownAndNeverChangeSource(t *testing.T) { + now := time.Now().UTC() + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), lifecycleEnvironment(now)) + store := NewStore(client, "ns", testProfiles(), nil) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + if err := store.ReserveSuccessor(t.Context(), ref, "client", "owner", "source", "destination", "fork-op"); err != nil { + t.Fatal(err) + } + if err := store.ReserveSuccessor(t.Context(), ref, "client", "owner", "source", "destination", "fork-op"); err != nil { + t.Fatal(err) + } + intents, err := store.ListReferenceIntents(t.Context(), "client", "owner", 64) + if err != nil || len(intents) != 1 || intents[0].BindingID != "destination" || intents[0].State != executionenv.ReferencePendingCreate { + t.Fatalf("intents=%+v err=%v", intents, err) + } + if err := store.CommitReference(t.Context(), ref, "client", "owner", "destination", "fork-op"); err != nil { + t.Fatal(err) + } + if err := store.PrepareReferenceDelete(t.Context(), ref, "client", "owner", "destination", "delete-op"); err != nil { + t.Fatal(err) + } + if err := store.CancelReferenceDelete(t.Context(), ref, "client", "owner", "destination", "delete-op"); err != nil { + t.Fatal(err) + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + refs, err := referenceRecords(got) + if err != nil { + t.Fatal(err) + } + if !publishedReference(refs, "source") || !publishedReference(refs, "destination") { + t.Fatalf("references=%+v", refs) + } +} diff --git a/internal/adapter/executioncontroller/store_revoke.go b/internal/adapter/executioncontroller/store_revoke.go new file mode 100644 index 0000000000..88e73db467 --- /dev/null +++ b/internal/adapter/executioncontroller/store_revoke.go @@ -0,0 +1,63 @@ +//nolint:revive // Private store methods implement adapter-only lifecycle interfaces. +package executioncontroller + +import ( + "context" + "fmt" + "math" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +// RevokeEnvironment atomically invalidates every issued grant without claiming +// that an already-dispatched executor operation has stopped. +func (s *Store) RevokeEnvironment(ctx context.Context, q adminLifecycleRequest, expected uint64) (uint64, error) { + ref, client, owner, operationID := q.Environment, q.Client, q.OwnerHash, q.OperationID + if expected == 0 || expected >= math.MaxInt64 || operationID == "" || len(operationID) > executionenv.MaxIdentityBytes { + return 0, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "grant generation or operation identity is invalid"} + } + requestFingerprint := fingerprint(ref.ID, ref.Revision, client, owner, fmt.Sprint(expected)) + var next uint64 + err := s.retryAdminStatus(ctx, q, func(o *unstructured.Unstructured) error { + receipts, found, err := unstructured.NestedSlice(o.Object, "status", "revocationReceipts") + if err != nil { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "revocation receipt ledger is invalid"} + } + if found { + for _, raw := range receipts { + receipt, ok := raw.(map[string]any) + if !ok { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "revocation receipt ledger is invalid"} + } + if text(receipt, operationIDField) != operationID { + continue + } + if text(receipt, "fingerprint") != requestFingerprint { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "operation identity was reused for a different revocation"} + } + generation, ok := receipt["grantGeneration"].(int64) + if !ok || generation <= 0 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "revocation receipt ledger is invalid"} + } + next = uint64(generation) + return nil + } + } + current := intNested(o.Object, "status", "grantGeneration") + if current <= 0 || uint64(current) != expected { //nolint:gosec // positivity is checked before conversion. + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "grant generation changed concurrently"} + } + next = expected + 1 + receipts = append(receipts, map[string]any{operationIDField: operationID, "fingerprint": requestFingerprint, "expectedGrantGeneration": int64(expected), "grantGeneration": int64(next)}) //nolint:gosec // values are bounded above. + if len(receipts) > 32 { + receipts = receipts[len(receipts)-32:] + } + if err := unstructured.SetNestedSlice(o.Object, receipts, "status", "revocationReceipts"); err != nil { + return err + } + return unstructured.SetNestedField(o.Object, int64(next), "status", "grantGeneration") //nolint:gosec // expected is bounded below MaxInt64. + }) + return next, err +} diff --git a/internal/adapter/executioncontroller/store_revoke_test.go b/internal/adapter/executioncontroller/store_revoke_test.go new file mode 100644 index 0000000000..04cda47050 --- /dev/null +++ b/internal/adapter/executioncontroller/store_revoke_test.go @@ -0,0 +1,81 @@ +package executioncontroller + +import ( + "context" + "errors" + "math" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestRevokeEnvironmentFencesOldClaimWithoutChangingExecutionEpoch(t *testing.T) { + env := runFixtureEnvironment(7, 3) + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), &terminalErrorExecutor{}) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + next, err := store.RevokeEnvironment(t.Context(), adminLifecycleRequest{Environment: ref, Client: "client", OwnerHash: "owner", OperationID: "revoke-1"}, 3) + if err != nil || next != 4 { + t.Fatalf("revoke=(%d,%v)", next, err) + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if intNested(got.Object, "status", "epoch") != 7 || intNested(got.Object, "status", "grantGeneration") != 4 { + t.Fatalf("status=%v", got.Object["status"]) + } + otherReplica := NewStore(client, "ns", testProfiles(), &terminalErrorExecutor{}) + _, err = otherReplica.File(t.Context(), "client", "owner", executionenv.FileRequest{Context: executionenv.RequestContext{Environment: ref, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3}, Operation: executionenv.OpFileRead, Path: "x"}) + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("old grant error=%v", err) + } + if _, err := store.RenewRun(t.Context(), ref, "client", "owner", executionenv.RunClaimRequest{Environment: ref, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3, OperationID: "renew", TTL: time.Minute}); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("old renewal error=%v", err) + } + replayed, err := store.RevokeEnvironment(t.Context(), adminLifecycleRequest{Environment: ref, Client: "client", OwnerHash: "owner", OperationID: "revoke-1"}, 3) + if err != nil || replayed != 4 { + t.Fatalf("replayed revoke=(%d,%v)", replayed, err) + } + if _, err := store.RevokeEnvironment(t.Context(), adminLifecycleRequest{Environment: ref, Client: "client", OwnerHash: "owner", OperationID: "revoke-2"}, 3); !errors.As(err, &controlled) || controlled.Code != executionenv.CodeConflict { + t.Fatalf("stale CAS error=%v", err) + } +} + +func TestRevokeEnvironmentOverflowFailsClosed(t *testing.T) { + env := runFixtureEnvironment(1, math.MaxInt64) + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + if _, err := store.RevokeEnvironment(context.Background(), adminLifecycleRequest{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, Client: "client", OwnerHash: "owner", OperationID: "overflow"}, math.MaxInt64); err == nil { + t.Fatal("overflow accepted") + } +} + +func TestProfileEnvironmentAdmissionLimit(t *testing.T) { + env := lifecycleEnvironment(time.Now().UTC()) + client := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(runtime.NewScheme(), map[schema.GroupVersionResource]string{ExecutionEnvironmentGVR: "ExecutionEnvironmentList"}, env) + profiles := testProfiles() + profile := profiles.byName["go"] + profile.Spec.MaxEnvironments = 1 + profiles.byName["go"] = profile + store := NewStore(client, "ns", profiles, nil).WithKubeClient(kubernetesfake.NewSimpleClientset()) + _, err := store.Ensure(t.Context(), "client", "owner", "new-binding", "go", "new-fingerprint") + var controlled *executionenv.Error + if !errors.As(err, &controlled) || controlled.Code != executionenv.CodeResourceExhausted { + t.Fatalf("limit error=%v", err) + } +} + +func runFixtureEnvironment(epoch, generation uint64) *unstructured.Unstructured { + now := time.Now().UTC() + return &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(epoch), "grantGeneration": int64(generation), "fenceState": "Healthy", "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": now.Format(time.RFC3339Nano)}}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "ownerHash": "owner", "clientHash": hashText("client"), "epoch": int64(epoch), "grantGeneration": int64(generation), "expiresAt": now.Add(time.Minute).Format(time.RFC3339Nano)}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} +} diff --git a/internal/adapter/executioncontroller/store_run.go b/internal/adapter/executioncontroller/store_run.go new file mode 100644 index 0000000000..85004f1dc1 --- /dev/null +++ b/internal/adapter/executioncontroller/store_run.go @@ -0,0 +1,228 @@ +//nolint:revive // Private store methods implement adapter-only lifecycle interfaces. +package executioncontroller + +import ( + "context" + "fmt" + "math" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func boundedRunTTL(ttl time.Duration) (time.Duration, error) { + if ttl < executionenv.MinRunTTL || ttl > executionenv.MaxRunTTL { + return 0, &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "run claim ttl is outside the operator bound"} + } + return ttl, nil +} + +func generationMatches(o *unstructured.Unstructured, generation uint64) bool { + return generation > 0 && generation <= math.MaxInt64 && intNested(o.Object, "status", "grantGeneration") == int64(generation) +} + +// ValidateRunClaim rechecks durable generation and ownership before any grant-authorized operation. +func (s *Store) ValidateRunClaim(ctx context.Context, client, owner string, rc executionenv.RequestContext) error { + if rc.Epoch == 0 || rc.Epoch > math.MaxInt64 || rc.GrantGeneration == 0 || rc.GrantGeneration > math.MaxInt64 { + return &executionenv.Error{Code: executionenv.CodeInvalidArgument, Message: "run claim identity is invalid"} + } + o, err := s.resources.Get(ctx, rc.Environment.ID, metav1.GetOptions{}) + if err != nil { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + if err := requireCurrentSchema(o); err != nil { + return err + } + claim, _, expiry, ok := activeRunFrom(o) + if !ok || !s.now().Before(expiry) || claim.Environment != rc.Environment || claim.BindingID != rc.BindingID || claim.RunID != rc.RunID || claim.ClaimID != rc.ClaimID || claim.Epoch != rc.Epoch || claim.GrantGeneration != rc.GrantGeneration || !generationMatches(o, rc.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim is not current"} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + return nil +} + +func activeRunFrom(o *unstructured.Unstructured) (executionenv.RunClaim, string, time.Time, bool) { + m, found, _ := unstructured.NestedMap(o.Object, "status", "activeRun") + if !found { + return executionenv.RunClaim{}, "", time.Time{}, false + } + expires, err := time.Parse(time.RFC3339Nano, text(m, "expiresAt")) + epoch, epochOK := epochValue(o) + generationValue := intNested(m, "grantGeneration") + if generationValue <= 0 { + return executionenv.RunClaim{}, "", time.Time{}, false + } + generation := uint64(generationValue) //nolint:gosec // positivity is checked immediately above. + claim := executionenv.RunClaim{Environment: executionenv.EnvironmentRef{ID: o.GetName(), Revision: textNested(o.Object, "spec", "revision")}, BindingID: text(m, "bindingID"), RunID: text(m, "runID"), ClaimID: text(m, "claimID"), Epoch: epoch, GrantGeneration: generation, ExpiresAt: expires} + ownerHash, clientHash := textNested(o.Object, "spec", "ownerHash"), textNested(o.Object, "spec", "clientHash") + return claim, text(m, operationIDField), expires, err == nil && epochOK && generation > 0 && claim.BindingID != "" && claim.RunID != "" && claim.ClaimID != "" && + ownerHash != "" && clientHash != "" && text(m, "ownerHash") == ownerHash && text(m, "clientHash") == clientHash +} + +//nolint:gocyclo // Run ownership admission keeps every CAS precondition in one auditable transition. +func (s *Store) AcquireRun(ctx context.Context, ref executionenv.EnvironmentRef, client, owner, binding, runID, operationID string, ttl time.Duration) (executionenv.RunClaim, error) { + ttl, err := boundedRunTTL(ttl) + if err != nil { + return executionenv.RunClaim{}, err + } + var out executionenv.RunClaim + err = s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || textNested(o.Object, "spec", "revision") != ref.Revision { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: environmentNotFoundMessage} + } + refs, parseErr := referenceRecords(o) + if parseErr != nil { + return parseErr + } + if !publishedReference(refs, binding) { + return &executionenv.Error{Code: executionenv.CodeNotFound, Message: "published reference not found"} + } + if textNested(o.Object, "spec", "desired") != "Active" || !conditionTrue(o, "Ready") || textNested(o.Object, "status", "fenceState") != fenceHealthy { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: "environment is not ready"} + } + if current, currentOp, expires, found := activeRunFrom(o); found { + if current.BindingID == binding && current.RunID == runID && currentOp == operationID { + if !generationMatches(o, current.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim was revoked"} + } + out = current + return nil + } + if s.now().Before(expires) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment has an active run"} + } + if textNested(o.Object, "status", "activeOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "expired run still has an active operation"} + } + } + if textNested(o.Object, "status", "activeOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment has an active operation"} + } + epoch, ok := epochValue(o) + if !ok || epoch >= math.MaxInt64 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "environment epoch is invalid"} + } + epoch++ + epochStatus := int64(epoch) //nolint:gosec // the persisted epoch was checked below MaxInt64 before increment. + claimID, idErr := randomID() + if idErr != nil { + return idErr + } + expires := s.now().Add(ttl) + generation := intNested(o.Object, "status", "grantGeneration") + if generation <= 0 { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "grant generation is invalid"} + } + if setErr := unstructured.SetNestedField(o.Object, epochStatus, "status", "epoch"); setErr != nil { + return setErr + } + if setErr := unstructured.SetNestedMap(o.Object, map[string]any{"bindingID": binding, "runID": runID, "claimID": claimID, operationIDField: operationID, "ownerHash": owner, "clientHash": hashText(client), "epoch": epochStatus, "grantGeneration": generation, "expiresAt": expires.Format(time.RFC3339Nano)}, "status", "activeRun"); setErr != nil { + return setErr + } + out = executionenv.RunClaim{Environment: ref, BindingID: binding, RunID: runID, ClaimID: claimID, Epoch: epoch, GrantGeneration: uint64(generation), ExpiresAt: expires} + return nil + }) + return out, err +} + +func renewFingerprint(ref executionenv.EnvironmentRef, client, owner string, req executionenv.RunClaimRequest) string { + return fingerprint(ref.ID, ref.Revision, client, owner, req.BindingID, req.RunID, req.ClaimID, fmt.Sprint(req.Epoch), fmt.Sprint(req.GrantGeneration), req.TTL.String()) +} + +func replayRenewReceipt(o *unstructured.Unstructured, operationID, requestFingerprint string, now time.Time, claim executionenv.RunClaim) (executionenv.RunClaim, bool, error) { + receipts, _, err := unstructured.NestedSlice(o.Object, "status", "renewReceipts") + if err != nil { + return executionenv.RunClaim{}, false, &executionenv.Error{Code: executionenv.CodeConflict, Message: "run renewal receipts are invalid"} + } + for _, raw := range receipts { + receipt, ok := raw.(map[string]any) + if !ok || text(receipt, operationIDField) != operationID { + continue + } + if text(receipt, "fingerprint") != requestFingerprint { + return executionenv.RunClaim{}, true, &executionenv.Error{Code: executionenv.CodeConflict, Message: "run renewal operation conflicts"} + } + expires, parseErr := time.Parse(time.RFC3339Nano, text(receipt, "expiresAt")) + if parseErr != nil || !now.Before(expires) { + return executionenv.RunClaim{}, true, &executionenv.Error{Code: executionenv.CodeConflict, Message: "run renewal receipt is expired"} + } + claim.ExpiresAt = expires + return claim, true, nil + } + return executionenv.RunClaim{}, false, nil +} + +func appendRenewReceipt(o *unstructured.Unstructured, operationID, requestFingerprint string, expires time.Time) error { + receipts, _, err := unstructured.NestedSlice(o.Object, "status", "renewReceipts") + if err != nil { + return err + } + receipts = append(receipts, map[string]any{operationIDField: operationID, "fingerprint": requestFingerprint, "expiresAt": expires.Format(time.RFC3339Nano)}) + if len(receipts) > 32 { + receipts = receipts[len(receipts)-32:] + } + return unstructured.SetNestedSlice(o.Object, receipts, "status", "renewReceipts") +} + +func (s *Store) RenewRun(ctx context.Context, ref executionenv.EnvironmentRef, client, owner string, req executionenv.RunClaimRequest) (executionenv.RunClaim, error) { + ttl, err := boundedRunTTL(req.TTL) + if err != nil { + return executionenv.RunClaim{}, err + } + requestFingerprint := renewFingerprint(ref, client, owner, req) + var out executionenv.RunClaim + err = s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + cur, _, expiry, ok := activeRunFrom(o) + if !ok || textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || cur.Environment != ref || cur.BindingID != req.BindingID || cur.RunID != req.RunID || cur.ClaimID != req.ClaimID || cur.Epoch != req.Epoch || cur.GrantGeneration != req.GrantGeneration || !generationMatches(o, req.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim mismatch"} + } + if replay, found, replayErr := replayRenewReceipt(o, req.OperationID, requestFingerprint, s.now(), cur); found { + out = replay + return replayErr + } + if !s.now().Before(expiry) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim mismatch"} + } + expires := s.now().Add(ttl) + m, _, _ := unstructured.NestedMap(o.Object, "status", "activeRun") + m["expiresAt"] = expires.Format(time.RFC3339Nano) + if setErr := unstructured.SetNestedMap(o.Object, m, "status", "activeRun"); setErr != nil { + return setErr + } + if setErr := appendRenewReceipt(o, req.OperationID, requestFingerprint, expires); setErr != nil { + return setErr + } + cur.ExpiresAt = expires + out = cur + return nil + }) + return out, err +} + +func (s *Store) ReleaseRun(ctx context.Context, ref executionenv.EnvironmentRef, client, owner string, req executionenv.RunClaimRequest) error { + return s.retryUpdateStatus(ctx, ref.ID, func(o *unstructured.Unstructured) error { + cur, _, _, ok := activeRunFrom(o) + if !ok { + return nil + } + if textNested(o.Object, "status", "lifecycleOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeNotReady, Message: transitioningMessage} + } + if textNested(o.Object, "spec", "ownerHash") != owner || textNested(o.Object, "spec", "clientHash") != hashText(client) || cur.Environment != ref || cur.BindingID != req.BindingID || cur.RunID != req.RunID || cur.ClaimID != req.ClaimID || cur.Epoch != req.Epoch || cur.GrantGeneration != req.GrantGeneration || !generationMatches(o, req.GrantGeneration) { + return &executionenv.Error{Code: executionenv.CodeConflict, Message: "run claim mismatch"} + } + if textNested(o.Object, "status", "activeOperation", "id") != "" { + return &executionenv.Error{Code: executionenv.CodeFenceUnknown, Message: "run has an active operation"} + } + unstructured.RemoveNestedField(o.Object, "status", "activeRun") + return nil + }) +} diff --git a/internal/adapter/executioncontroller/store_test.go b/internal/adapter/executioncontroller/store_test.go new file mode 100644 index 0000000000..9f56f52154 --- /dev/null +++ b/internal/adapter/executioncontroller/store_test.go @@ -0,0 +1,197 @@ +package executioncontroller + +import ( + "context" + "errors" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + dynamicfake "k8s.io/client-go/dynamic/fake" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestRenewRunRejectsActiveRunOwnerClientTupleDrift(t *testing.T) { + for _, tc := range []struct { + name string + plant func(*unstructured.Unstructured) + }{ + {name: "wrong owner", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, "wrong-owner", "status", "activeRun", "ownerHash") + }}, + {name: "missing owner", plant: func(o *unstructured.Unstructured) { + unstructured.RemoveNestedField(o.Object, "status", "activeRun", "ownerHash") + }}, + {name: "wrong client", plant: func(o *unstructured.Unstructured) { + _ = unstructured.SetNestedField(o.Object, hashText("wrong-client"), "status", "activeRun", "clientHash") + }}, + {name: "missing client", plant: func(o *unstructured.Unstructured) { + unstructured.RemoveNestedField(o.Object, "status", "activeRun", "clientHash") + }}, + } { + t.Run(tc.name, func(t *testing.T) { + env := runFixtureEnvironment(7, 3) + tc.plant(env) + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + req := executionenv.RunClaimRequest{BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3, OperationID: "renew", TTL: executionenv.MinRunTTL} + if _, err := store.RenewRun(t.Context(), ref, "client", "owner", req); err == nil { + t.Fatal("renewal accepted active-run identity drift") + } + }) + } +} + +func TestRenewRunRejectsExpiredClaimAtBoundary(t *testing.T) { + now := time.Date(2026, 9, 17, 4, 0, 0, 0, time.UTC) + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(7), "grantGeneration": int64(3), "fenceState": fenceHealthy, "references": []any{}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "ownerHash": "owner", "clientHash": hashText("client"), "epoch": int64(7), "grantGeneration": int64(3), "expiresAt": now.Format(time.RFC3339Nano)}}}} + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + store.now = func() time.Time { return now } + req := executionenv.RunClaimRequest{BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3, OperationID: "renew", TTL: executionenv.MinRunTTL} + if _, err := store.RenewRun(t.Context(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", req); err == nil { + t.Fatal("renewal at the expiry boundary resurrected the claim") + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(t.Context(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if expiry := textNested(got.Object, "status", "activeRun", "expiresAt"); expiry != now.Format(time.RFC3339Nano) { + t.Fatalf("expired claim changed to %q", expiry) + } +} + +func TestRenewRunReplaysExactOperationWithoutExtendingLease(t *testing.T) { + now := time.Date(2026, 9, 17, 4, 0, 0, 0, time.UTC) + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(7), "grantGeneration": int64(3), "fenceState": fenceHealthy, "references": []any{}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "ownerHash": "owner", "clientHash": hashText("client"), "epoch": int64(7), "grantGeneration": int64(3), "expiresAt": now.Add(time.Minute).Format(time.RFC3339Nano)}}}} + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + store := NewStore(client, "ns", testProfiles(), nil) + clock := now + store.now = func() time.Time { return clock } + ref := executionenv.EnvironmentRef{ID: "env", Revision: "rev"} + req := executionenv.RunClaimRequest{BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 7, GrantGeneration: 3, OperationID: "renew", TTL: executionenv.MinRunTTL} + first, err := store.RenewRun(t.Context(), ref, "client", "owner", req) + if err != nil { + t.Fatal(err) + } + clock = clock.Add(10 * time.Second) + replay, err := store.RenewRun(t.Context(), ref, "client", "owner", req) + if err != nil || !replay.ExpiresAt.Equal(first.ExpiresAt) { + t.Fatalf("replay extended or rejected lease: first=%v replay=%v err=%v", first.ExpiresAt, replay.ExpiresAt, err) + } + req.TTL = time.Minute + if _, err := store.RenewRun(t.Context(), ref, "client", "owner", req); err == nil { + t.Fatal("same operation accepted conflicting inputs") + } + clock = first.ExpiresAt + req.TTL = executionenv.MinRunTTL + if _, err := store.RenewRun(t.Context(), ref, "client", "owner", req); err == nil { + t.Fatal("expired renewal receipt resurrected claim") + } +} + +func TestStoreEnsureUsesStableLookupAndRejectsFingerprintDrift(t *testing.T) { + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme()) + profiles := &Profiles{byName: map[string]resolvedProfile{"go": {Digest: "sha256:profile", Spec: ProfileSpec{Image: "example@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100}}, "other": {Digest: "sha256:other", Spec: ProfileSpec{Image: "example@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", StorageClass: "standard", StorageSize: "1Gi", CPURequest: "100m", MemoryRequest: "64Mi", CPULimit: "1", MemoryLimit: "1Gi", EphemeralStorageRequest: "64Mi", EphemeralStorageLimit: "1Gi", TmpSizeLimit: "256Mi", RuntimeClassName: "sandboxed", MaxFileBytes: 1024, MaxCommandBytes: 1024, MaxCommandDuration: time.Minute, MaxEnvironments: 100}}}} + s := NewStore(client, "ns", profiles, nil) + ctx := context.Background() + a, err := s.Ensure(ctx, "spiffe://client", "owner", "binding", "go", "fp1") + if err != nil { + t.Fatal(err) + } + b, err := s.Ensure(ctx, "spiffe://client", "owner", "binding", "go", "fp1") + if err != nil { + t.Fatal(err) + } + if a.Environment != b.Environment || a.Epoch != b.Epoch { + t.Fatalf("identity changed: %+v %+v", a, b) + } + if _, err := s.Ensure(ctx, "spiffe://client", "owner", "binding", "other", "fp2"); err == nil { + t.Fatal("profile drift adopted existing allocation") + } +} +func TestAllocationNameExcludesMutableProfile(t *testing.T) { + a := allocationName("client", "owner", "binding") + b := allocationName("client", "owner", "binding") + if a != b { + t.Fatal("unstable allocation name") + } +} + +type terminalErrorExecutor struct{ called int } + +type blockingExecutor struct { + started chan struct{} + release chan struct{} +} + +func (e *blockingExecutor) Execute(context.Context, string, executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + close(e.started) + <-e.release + return executionenv.ExecutorResponse{}, nil +} + +func (e *terminalErrorExecutor) Execute(context.Context, string, executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + e.called++ + return executionenv.ExecutorResponse{}, &executionenv.Error{Code: executionenv.CodeNotFound, Message: "path not found"} +} +func TestCancelledStoreOperationStaysActiveUntilBackendStopsThenFences(t *testing.T) { + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "ownerHash": "owner", "clientHash": hashText("client"), "epoch": int64(1), "grantGeneration": int64(1), "expiresAt": time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + exec := &blockingExecutor{started: make(chan struct{}), release: make(chan struct{})} + s := NewStore(client, "ns", testProfiles(), exec) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { + _, err := s.StartCommand(ctx, "client", "owner", executionenv.CommandStartRequest{Context: executionenv.RequestContext{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1}, Command: "sleep 10"}) + done <- err + }() + <-exec.started + cancel() + active, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(active.Object, "status", "activeOperation", "id") == "" { + t.Fatal("active operation was released while executor was still live") + } + close(exec.release) + var pe *executionenv.Error + if err := <-done; !errors.As(err, &pe) || pe.Code != executionenv.CodeFenceUnknown { + t.Fatalf("completion error=%v", err) + } + fenced, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if textNested(fenced.Object, "status", "activeOperation", "id") == "" || textNested(fenced.Object, "status", "fenceState") != "FenceUnknown" { + t.Fatalf("uncertain operation identity was not retained: status=%v", fenced.Object["status"]) + } + _, err = s.Attach(context.Background(), executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, "client", "owner", "binding") + if !errors.As(err, &pe) || pe.Code != executionenv.CodeNotReady { + t.Fatalf("attach after fencing error=%v", err) + } +} + +func TestControlledExecutorErrorClearsOperationWithoutFencing(t *testing.T) { + env := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "execution.mecatl.dev/v1alpha1", "kind": "ExecutionEnvironment", "metadata": map[string]any{"name": "env", "namespace": "ns"}, "spec": map[string]any{"schemaVersion": int64(2), "revision": "rev", "ownerHash": "owner", "clientHash": hashText("client"), "profile": "go", "profileDigest": "sha256:profile", "desired": "Active"}, "status": map[string]any{"schemaVersion": int64(2), "epoch": int64(1), "grantGeneration": int64(1), "fenceState": "Healthy", "references": []any{map[string]any{"bindingID": "binding", "state": "Published", "operationID": "seed", "createdAt": time.Now().UTC().Format(time.RFC3339Nano)}}, "activeRun": map[string]any{"bindingID": "binding", "runID": "run", "claimID": "claim", "operationID": "acquire", "ownerHash": "owner", "clientHash": hashText("client"), "epoch": int64(1), "grantGeneration": int64(1), "expiresAt": time.Now().Add(time.Minute).UTC().Format(time.RFC3339Nano)}, "pod": map[string]any{"name": "pod"}, "conditions": []any{map[string]any{"type": "Ready", "status": "True"}}}}} + client := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme(), env) + exec := &terminalErrorExecutor{} + s := NewStore(client, "ns", testProfiles(), exec) + _, err := s.File(context.Background(), "client", "owner", executionenv.FileRequest{Context: executionenv.RequestContext{Environment: executionenv.EnvironmentRef{ID: "env", Revision: "rev"}, BindingID: "binding", RunID: "run", ClaimID: "claim", Epoch: 1, GrantGeneration: 1}, Operation: executionenv.OpFileRead, Path: "missing"}) + var pe *executionenv.Error + if !errors.As(err, &pe) || pe.Code != executionenv.CodeNotFound { + t.Fatalf("error=%v", err) + } + got, err := client.Resource(ExecutionEnvironmentGVR).Namespace("ns").Get(context.Background(), "env", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if exec.called != 1 || textNested(got.Object, "status", "activeOperation", "id") != "" || textNested(got.Object, "status", "fenceState") != "Healthy" { + t.Fatalf("status=%v calls=%d", got.Object["status"], exec.called) + } +} diff --git a/internal/adapter/server/canonical_seams_repair_test.go b/internal/adapter/server/canonical_seams_repair_test.go index 1aedbcd3a4..a6e2158b71 100644 --- a/internal/adapter/server/canonical_seams_repair_test.go +++ b/internal/adapter/server/canonical_seams_repair_test.go @@ -2,9 +2,11 @@ package server import ( "context" + "errors" "go/ast" "go/parser" "go/token" + "sync" "testing" "time" @@ -57,7 +59,13 @@ func TestInvariant_canonical_placement_seams_remove_legacy_authority(t *testing. } } -type revisionPlacementProvider struct{ binds int } +type revisionPlacementProvider struct { + mu sync.Mutex + binds int + reattachCalls []PlacementReattachRequest + reattachResult *PlacementBinding + reattachErr error +} func (*revisionPlacementProvider) binding(revision string) PlacementBinding { ref := session.EnvironmentRef{Kind: session.EnvKindMem, ID: "/same-root", Revision: revision} @@ -65,11 +73,22 @@ func (*revisionPlacementProvider) binding(revision string) PlacementBinding { } func (p *revisionPlacementProvider) Bind(context.Context, PlacementBindRequest) (PlacementBinding, error) { + p.mu.Lock() + defer p.mu.Unlock() p.binds++ return p.binding(string(rune('0' + p.binds))), nil } -func (*revisionPlacementProvider) Reattach(_ context.Context, req PlacementReattachRequest) (PlacementBinding, error) { +func (p *revisionPlacementProvider) Reattach(_ context.Context, req PlacementReattachRequest) (PlacementBinding, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.reattachCalls = append(p.reattachCalls, req) + if p.reattachErr != nil { + return PlacementBinding{}, p.reattachErr + } + if p.reattachResult != nil { + return *p.reattachResult, nil + } return PlacementBinding{Ref: req.Ref, Environment: tool.MustEnvironment(req.Ref, memfs.NewWorkspace("/same-root"), memledger.New(), nil), GovernanceRoot: "/same-root"}, nil } @@ -77,7 +96,7 @@ func (*revisionPlacementProvider) ListWorktrees(context.Context, PlacementDiscov return nil, nil } -func TestInvariant_create_retry_matches_complete_environment_ref(t *testing.T) { +func TestInvariant_create_retry_reattaches_complete_environment_ref(t *testing.T) { provider := &revisionPlacementProvider{} svc, err := NewService(Config{ Engine: repairEngine(), Store: memstore.New(), PlacementProvider: provider, PlacementScope: "test", SharedEngineRoot: "/same-root", @@ -94,8 +113,197 @@ func TestInvariant_create_retry_matches_complete_environment_ref(t *testing.T) { if first.EnvironmentRef.Revision != "1" { t.Fatalf("first ref = %+v, want provider's complete create binding", first.EnvironmentRef) } - if _, err := svc.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, ProviderSelector{}, ProfileDefault, WithSessionID("fixed")); err == nil { - t.Fatal("retry with a different placement revision matched the existing session") + retry, err := svc.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, ProviderSelector{}, ProfileDefault, WithSessionID("fixed")) + if err != nil { + t.Fatal(err) + } + provider.mu.Lock() + binds := provider.binds + reattachCalls := append([]PlacementReattachRequest(nil), provider.reattachCalls...) + provider.mu.Unlock() + if retry.EnvironmentRef != first.EnvironmentRef || binds != 1 { + t.Fatalf("retry ref=%+v binds=%d, want exact ref %+v without another bind", retry.EnvironmentRef, binds, first.EnvironmentRef) + } + if len(reattachCalls) != 1 { + t.Fatalf("reattach calls=%d, want 1", len(reattachCalls)) + } + gotReq := reattachCalls[0] + if gotReq.Ref != first.EnvironmentRef || gotReq.BindingID != first.ID || gotReq.Scope != "test" || gotReq.Principal == nil || gotReq.Principal.Issuer != "issuer" || gotReq.Principal.Subject != "subject" { + t.Fatalf("reattach request=%+v, want exact persisted ref/binding/principal/scope", gotReq) + } +} + +func TestInvariant_existing_create_validates_and_closes_supplied_placement_before_exact_reattach(t *testing.T) { + provider := &revisionPlacementProvider{} + svc, err := NewService(Config{ + Engine: repairEngine(), Store: memstore.New(), PlacementProvider: provider, PlacementScope: "test", SharedEngineRoot: "/same-root", + OwnershipEnforced: true, NewID: func() session.SessionID { return "fixed" }, Now: func() time.Time { return time.Unix(1, 0) }, + }) + if err != nil { + t.Fatal(err) + } + ctx := session.WithPrincipal(context.Background(), &session.Principal{Issuer: "issuer", Subject: "subject"}) + created, err := svc.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, ProviderSelector{}, ProfileDefault, WithSessionID("fixed")) + if err != nil { + t.Fatal(err) + } + binding := func(ref session.EnvironmentRef, closeCount *int) PlacementBinding { + return PlacementBinding{ + Ref: ref, + Environment: tool.MustEnvironment(ref, memfs.NewWorkspace("/same-root"), memledger.New(), nil), + Close: func() error { *closeCount++; return nil }, + } + } + + t.Run("matching", func(t *testing.T) { + closed := 0 + got, err := svc.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, ProviderSelector{}, ProfileDefault, + WithSessionID("fixed"), WithPlacementBinding(binding(created.EnvironmentRef, &closed))) + if err != nil { + t.Fatal(err) + } + if got.EnvironmentRef != created.EnvironmentRef || closed != 1 { + t.Fatalf("retry ref=%+v close calls=%d, want exact persisted ref and one close", got.EnvironmentRef, closed) + } + }) + + t.Run("mismatched", func(t *testing.T) { + closed := 0 + wrong := created.EnvironmentRef + wrong.Revision = "wrong" + _, err := svc.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, ProviderSelector{}, ProfileDefault, + WithSessionID("fixed"), WithPlacementBinding(binding(wrong, &closed))) + if !errors.Is(err, ErrInvalidPlacementBinding) || closed != 1 { + t.Fatalf("error=%v close calls=%d, want invalid placement and one close", err, closed) + } + }) + + t.Run("invalid", func(t *testing.T) { + closed := 0 + invalid := PlacementBinding{Close: func() error { closed++; return nil }} + _, err := svc.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, ProviderSelector{}, ProfileDefault, + WithSessionID("fixed"), WithPlacementBinding(invalid)) + if !errors.Is(err, ErrInvalidPlacementBinding) || closed != 1 { + t.Fatalf("error=%v close calls=%d, want invalid placement and one close", err, closed) + } + }) +} + +type bindOnlyPlacementProvider struct{ binding PlacementBinding } + +func (p *bindOnlyPlacementProvider) Bind(context.Context, PlacementBindRequest) (PlacementBinding, error) { + return p.binding, nil +} + +func TestInvariant_exact_reattach_fails_closed_when_missing_or_mismatched(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvKindMem, ID: "/same-root", Revision: "exact"} + binding := PlacementBinding{Ref: ref, Environment: tool.MustEnvironment(ref, memfs.NewWorkspace("/same-root"), memledger.New(), nil)} + missing := &PlacementBinder{provider: &bindOnlyPlacementProvider{binding: binding}} + if _, err := missing.Reattach(t.Context(), PlacementReattachRequest{Ref: ref, Scope: "test", BindingID: "fixed"}); !errors.Is(err, ErrPlacementUnavailable) { + t.Fatalf("missing reattach error=%v, want placement unavailable", err) + } + wrongRef := session.EnvironmentRef{Kind: session.EnvKindMem, ID: "/same-root", Revision: "wrong"} + provider := &revisionPlacementProvider{reattachResult: &PlacementBinding{Ref: wrongRef, Environment: tool.MustEnvironment(wrongRef, memfs.NewWorkspace("/same-root"), memledger.New(), nil)}} + mismatched := &PlacementBinder{provider: provider} + if _, err := mismatched.Reattach(t.Context(), PlacementReattachRequest{Ref: ref, Scope: "test", BindingID: "fixed"}); !errors.Is(err, ErrInvalidPlacementBinding) { + t.Fatalf("mismatched reattach error=%v, want invalid binding", err) + } +} + +type concurrentCreatePlacementProvider struct { + mu sync.Mutex + waiting int + gate chan struct{} +} + +func (*concurrentCreatePlacementProvider) binding(id session.SessionID) PlacementBinding { + revision := "probe" + if id != "" { + revision = "rev-" + string(id) + } + ref := session.EnvironmentRef{Kind: session.EnvKindMem, ID: "/same-root", Revision: revision} + return PlacementBinding{Ref: ref, Environment: tool.MustEnvironment(ref, memfs.NewWorkspace("/same-root"), memledger.New(), nil), GovernanceRoot: "/same-root"} +} + +func (p *concurrentCreatePlacementProvider) Bind(ctx context.Context, req PlacementBindRequest) (PlacementBinding, error) { + p.mu.Lock() + gate := p.gate + if gate != nil && req.BindingID != "" { + p.waiting++ + if p.waiting == 2 { + close(gate) + } + } + p.mu.Unlock() + if gate != nil && req.BindingID != "" { + select { + case <-gate: + case <-ctx.Done(): + return PlacementBinding{}, ctx.Err() + } + } + return p.binding(req.BindingID), nil +} + +func (p *concurrentCreatePlacementProvider) Reattach(_ context.Context, req PlacementReattachRequest) (PlacementBinding, error) { + return p.binding(req.BindingID), nil +} + +func TestInvariant_concurrent_initial_create_converges_on_deterministic_binding(t *testing.T) { + provider := &concurrentCreatePlacementProvider{} + store := memstore.New() + newService := func() *Service { + svc, err := NewService(Config{ + Engine: repairEngine(), Store: store, PlacementProvider: provider, PlacementScope: "test", SharedEngineRoot: "/same-root", + OwnershipEnforced: true, NewID: func() session.SessionID { return "fixed" }, Now: func() time.Time { return time.Unix(1, 0) }, + }) + if err != nil { + t.Fatal(err) + } + return svc + } + a, b := newService(), newService() + provider.mu.Lock() + provider.gate = make(chan struct{}) + provider.mu.Unlock() + base := session.WithPrincipal(context.Background(), &session.Principal{Issuer: "issuer", Subject: "subject"}) + ctx, cancel := context.WithTimeout(base, 5*time.Second) + defer cancel() + type createResult struct { + created *session.Session + err error + } + results := make(chan createResult, 2) + for _, svc := range []*Service{a, b} { + go func() { + created, err := svc.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, ProviderSelector{}, ProfileDefault, WithSessionID("fixed")) + results <- createResult{created: created, err: err} + }() + } + var first *session.Session + for range 2 { + select { + case result := <-results: + if result.err != nil { + t.Fatalf("concurrent create: %v", result.err) + } + if first == nil { + first = result.created + } else if result.created.EnvironmentRef != first.EnvironmentRef || result.created.ID != first.ID { + t.Fatalf("concurrent creates diverged: first=%+v second=%+v", first.EnvironmentRef, result.created.EnvironmentRef) + } + case <-ctx.Done(): + t.Fatalf("concurrent create did not finish: %v", ctx.Err()) + } + } + if first.EnvironmentRef.Revision != "rev-fixed" { + t.Fatalf("create ref=%+v, want deterministic binding", first.EnvironmentRef) + } + if _, err := a.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{MaxTurns: 1}, ProviderSelector{}, ProfileDefault, WithSessionID("fixed")); !errors.Is(err, ErrInvalidArgument) { + t.Fatalf("changed limits retry error=%v, want invalid argument", err) + } + if _, err := b.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, ProviderSelector{}, ProfileNoFS, WithSessionID("fixed")); !errors.Is(err, ErrInvalidArgument) { + t.Fatalf("changed profile retry error=%v, want invalid argument", err) } } @@ -124,6 +332,73 @@ func TestInvariant_create_session_rejects_only_legacy_unknown_placement_fields(t } } +func TestInvariant_shared_create_collision_publishes_matching_pending_reference(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvKindMem, ID: "/same-root", Revision: "pending"} + owner := &session.Principal{Issuer: "issuer", Subject: "subject"} + store := memstore.New() + service, err := NewService(Config{ + Engine: repairEngine(), Store: store, OwnershipEnforced: true, PlacementScope: "test", SharedEngineRoot: "/same-root", + PlacementProvider: &repairPlacementProvider{binding: PlacementBinding{Ref: ref, Environment: tool.MustEnvironment(ref, memfs.NewWorkspace("/same-root"), memledger.New(), nil)}}, + }) + if err != nil { + t.Fatal(err) + } + winner := session.New("fixed", session.ModeDefault, ref, session.Limits{}, time.Unix(1, 0)) + winner.Owner = owner + winner.Title = "durable winner" + if err := store.Create(t.Context(), winner); err != nil { + t.Fatal(err) + } + request := newCreateRequest(ref, session.ModeDefault, session.Limits{}, ProviderSelector{}, ProfileDefault, "", createSessionOpts{}) + + t.Run("matching winner commits", func(t *testing.T) { + candidate := session.New("fixed", session.ModeDefault, ref, session.Limits{}, time.Unix(1, 0)) + candidate.Owner = owner + commits, closes := 0, 0 + persisted, err := service.persistPlacedCreatedSession(t.Context(), candidate, owner, &request, &PlacementBinding{ + Ref: ref, Commit: func(context.Context) error { commits++; return nil }, Close: func() error { closes++; return nil }, + }) + if err != nil || persisted == nil || persisted.Title != winner.Title || commits != 1 || closes != 0 { + t.Fatalf("persisted=%+v err=%v commits=%d closes=%d, want durable winner, successful publication, and no abort", persisted, err, commits, closes) + } + }) + + t.Run("mismatched owner or request does not publish or adopt", func(t *testing.T) { + for _, tc := range []struct { + name string + owner *session.Principal + request createRequest + want error + }{ + {name: "owner", owner: &session.Principal{Issuer: "other", Subject: "subject"}, request: request, want: ErrNotFound}, + {name: "request", owner: owner, request: newCreateRequest(ref, session.ModePlan, session.Limits{}, ProviderSelector{}, ProfileDefault, "", createSessionOpts{}), want: ErrInvalidArgument}, + } { + t.Run(tc.name, func(t *testing.T) { + candidate := session.New("fixed", session.ModeDefault, ref, session.Limits{}, time.Unix(1, 0)) + commits := 0 + persisted, err := service.persistPlacedCreatedSession(t.Context(), candidate, tc.owner, &tc.request, &PlacementBinding{Ref: ref, Commit: func(context.Context) error { commits++; return nil }}) + if !errors.Is(err, tc.want) || persisted != nil || commits != 0 { + t.Fatalf("persisted=%p err=%v commits=%d, want %v with no adoption or publication", persisted, err, commits, tc.want) + } + }) + } + }) + + t.Run("commit failure retains pending reference and returns winner", func(t *testing.T) { + candidate := session.New("fixed", session.ModeDefault, ref, session.Limits{}, time.Unix(1, 0)) + candidate.Owner = owner + commits, closes := 0, 0 + persisted, err := service.persistPlacedCreatedSession(t.Context(), candidate, owner, &request, &PlacementBinding{ + Ref: ref, + Commit: func(context.Context) error { commits++; return errors.New("unavailable") }, + Close: func() error { closes++; return nil }, + }) + if !errors.Is(err, ErrInternal) || persisted == nil || persisted.Title != winner.Title || commits != 1 || closes != 0 { + t.Fatalf("persisted=%+v err=%v commits=%d closes=%d, want durable winner, internal error, and retained pending reference", persisted, err, commits, closes) + } + }) +} + func TestInvariant_filesystem_placement_root_selects_policy_engine_for_every_kind(t *testing.T) { ref := session.EnvironmentRef{Kind: "custom-fs", ID: "opaque", Revision: "r1"} provider := &repairPlacementProvider{binding: PlacementBinding{Ref: ref, Environment: tool.MustEnvironment(ref, memfs.NewWorkspace("/root-b"), memledger.New(), nil), GovernanceRoot: "/root-b"}} diff --git a/internal/adapter/server/classification.go b/internal/adapter/server/classification.go index 610a0eeddf..e2cbceb459 100644 --- a/internal/adapter/server/classification.go +++ b/internal/adapter/server/classification.go @@ -248,39 +248,40 @@ func requireCallerOwnedContext(surface string, t reflect.Type, table map[string] // TestInvariant_owned_access_is_classified fails, naming the method. var serviceAccessTable = map[string]ClassificationEntry{ // --- caller-owned: sessions --- - "CreateSession": {KindCallerOwned, "binds the verified context principal as owner atomically with visibility (reserveSessionID)"}, - "CreateSessionWithProvider": {KindCallerOwned, "delegates to CreateSessionWithProfile's atomic owner bind"}, - "CreateSessionWithProfile": {KindCallerOwned, "atomic owner bind at creation (reserveSessionID); ForkSession/carryover sources are authorized via authorizeSession before copying history"}, - "CreateSessionWithMCP": {KindCallerOwned, "delegates to CreateSessionWithProfile's atomic owner bind"}, - "CreateACPSession": {KindCallerOwned, "binds the trusted composition default before treating ACP cwd solely as an equality assertion"}, - "LoadACPSession": {KindCallerOwned, "owner-authorizes and reattaches the exact persisted placement before checking ACP cwd or mounting client MCP"}, - "BindPlacement": {KindCallerOwned, "passes the verified context principal and trusted composition scope to the provider's single atomic authorization-and-resolution operation"}, - "ReattachPlacement": {KindCallerOwned, "passes the exact persisted ref, verified context principal, and trusted scope to the provider without falling back to Bind"}, - "ReattachPlacementInScope": {KindCallerOwned, "requires the durable schedule scope to equal trusted composition scope, then reauthorizes the exact persisted ref as the verified owner without default fallback"}, - "ListCommandsForSession": {KindCallerOwned, "loads and owner-authorizes the source session before exact placement reattachment and command discovery"}, - "ListWorktreesForSession": {KindCallerOwned, "loads and owner-authorizes the source session before exact placement reattachment and scoped selector issuance"}, - "ClearSessionSuccessor": {KindCallerOwned, "serializes and leases an owner-authorized source before atomically publishing an empty-history placed successor"}, - "ForkSessionSuccessor": {KindCallerOwned, "serializes and leases an owner-authorized source before atomically publishing a history-carrying placed successor"}, - "GetSession": {KindCallerOwned, "authorizeSession: owner mismatch or absence both return ErrNotFound"}, - "WithAuthorizedSession": {KindCallerOwned, "ownership preflight excludes foreign lock contention; authoritative reload under runEntryMu precedes the caller-owned effect"}, - "GetTranscript": {KindCallerOwned, "one SessionStore.Load followed by authorizeSession; no run-entry side effects"}, - "LoadSession": {KindCallerOwned, "authorizeSession before rehydration"}, - "LoadSessionWithMCP": {KindCallerOwned, "delegates to LoadSession's authorizeSession before mounting client MCP"}, - "SetMode": {KindCallerOwned, "authorizes via GetSession before changing the session's permission mode"}, - "RenameSession": {KindCallerOwned, "authorizes via GetSession, then revalidates ownership, kind, state, liveness, and lease under runEntryMu before persisting"}, - "CompactSession": {KindCallerOwned, "binds the verified caller, then revalidates ownership, chat purpose, state, liveness, and lease under runEntryMu before saving and appending events"}, - "DeleteSession": {KindCallerOwned, "authorizes via GetSession, then revalidates ownership, kind, state, liveness, and lease under runEntryMu before physical deletion"}, - "EndSession": {KindCallerOwned, "authorizes via GetSession before CloseSession"}, - "ListSessions": {KindCallerOwned, "filters to the caller's own rows before any pagination/count is computed"}, - "ListSessionPage": {KindCallerOwned, "passes caller ownership into the store query before keyset page formation and counting"}, - "StorageHealth": {KindCallerOwned, "gates on the trusted-context management authorizer, NOT caller ownership — the aggregate it reads is store-wide (every session), never scoped to the caller's own rows; see the AccessKind doc comment note on management-authority boundaries"}, - "PlanSessionCleanup": {KindCallerOwned, "gates on management authority, NOT caller ownership — the metadata pager plans over the ENTIRE store (owner scope is nil); the decision resolved here is 'is this caller a management principal', never per-session ownership"}, - "ApplySessionCleanup": {KindCallerOwned, "gates on management authority over the entire store; only the confirmation token/plan is bound to the verified caller, so a stolen or foreign token is rejected before any deletion"}, - "CancelSessionCleanup": {KindCallerOwned, "gates on management authority; matches the verified principal against the bounded job registry — the underlying cleanup scope is store-wide, never caller-owned"}, - "SessionCleanupJob": {KindCallerOwned, "gates on management authority and returns only a caller-bound sanitized job projection; the underlying cleanup data is store-wide, not the caller's own sessions"}, - "StreamSessionEvents": {KindCallerOwned, "event log/live stream resolves through the owning session's authorizeSession check"}, - "WatchSessionEvents": {KindCallerOwned, "durable replay-then-follow watch (ADR 0250); watchLog resolves ownership through the same GetSession check StreamSessionEvents uses, EAGERLY — before any envelope is yielded — because the durable log holds the whole transcript"}, - "Subscribe": {KindCallerOwned, "authorizes via GetSession before registering a live subscriber (issue #368)"}, + "CreateSession": {KindCallerOwned, "binds the verified context principal as owner atomically with visibility (reserveSessionID)"}, + "CreateSessionWithProvider": {KindCallerOwned, "delegates to CreateSessionWithProfile's atomic owner bind"}, + "CreateSessionWithProfile": {KindCallerOwned, "atomic owner bind at creation (reserveSessionID); ForkSession/carryover sources are authorized via authorizeSession before copying history"}, + "CreateSessionWithMCP": {KindCallerOwned, "delegates to CreateSessionWithProfile's atomic owner bind"}, + "CreateACPSession": {KindCallerOwned, "binds the trusted composition default before treating ACP cwd solely as an equality assertion"}, + "LoadACPSession": {KindCallerOwned, "owner-authorizes and reattaches the exact persisted placement before checking ACP cwd or mounting client MCP"}, + "BindPlacement": {KindCallerOwned, "passes the verified context principal and trusted composition scope to the provider's single atomic authorization-and-resolution operation"}, + "ReattachPlacement": {KindCallerOwned, "passes the exact persisted ref, verified context principal, and trusted scope to the provider without falling back to Bind"}, + "ReattachPlacementForBinding": {KindCallerOwned, "passes the exact persisted ref and session binding with the verified context principal to the provider"}, + "ReattachPlacementInScope": {KindCallerOwned, "requires the durable schedule scope to equal trusted composition scope, then reauthorizes the exact persisted ref as the verified owner without default fallback"}, + "ListCommandsForSession": {KindCallerOwned, "loads and owner-authorizes the source session before exact placement reattachment and command discovery"}, + "ListWorktreesForSession": {KindCallerOwned, "loads and owner-authorizes the source session before exact placement reattachment and scoped selector issuance"}, + "ClearSessionSuccessor": {KindCallerOwned, "serializes and leases an owner-authorized source before atomically publishing an empty-history placed successor"}, + "ForkSessionSuccessor": {KindCallerOwned, "serializes and leases an owner-authorized source before atomically publishing a history-carrying placed successor"}, + "GetSession": {KindCallerOwned, "authorizeSession: owner mismatch or absence both return ErrNotFound"}, + "WithAuthorizedSession": {KindCallerOwned, "ownership preflight excludes foreign lock contention; authoritative reload under runEntryMu precedes the caller-owned effect"}, + "GetTranscript": {KindCallerOwned, "one SessionStore.Load followed by authorizeSession; no run-entry side effects"}, + "LoadSession": {KindCallerOwned, "authorizeSession before rehydration"}, + "LoadSessionWithMCP": {KindCallerOwned, "delegates to LoadSession's authorizeSession before mounting client MCP"}, + "SetMode": {KindCallerOwned, "authorizes via GetSession before changing the session's permission mode"}, + "RenameSession": {KindCallerOwned, "authorizes via GetSession, then revalidates ownership, kind, state, liveness, and lease under runEntryMu before persisting"}, + "CompactSession": {KindCallerOwned, "binds the verified caller, then revalidates ownership, chat purpose, state, liveness, and lease under runEntryMu before saving and appending events"}, + "DeleteSession": {KindCallerOwned, "authorizes via GetSession, then revalidates ownership, kind, state, liveness, and lease under runEntryMu before physical deletion"}, + "EndSession": {KindCallerOwned, "authorizes via GetSession before CloseSession"}, + "ListSessions": {KindCallerOwned, "filters to the caller's own rows before any pagination/count is computed"}, + "ListSessionPage": {KindCallerOwned, "passes caller ownership into the store query before keyset page formation and counting"}, + "StorageHealth": {KindCallerOwned, "gates on the trusted-context management authorizer, NOT caller ownership — the aggregate it reads is store-wide (every session), never scoped to the caller's own rows; see the AccessKind doc comment note on management-authority boundaries"}, + "PlanSessionCleanup": {KindCallerOwned, "gates on management authority, NOT caller ownership — the metadata pager plans over the ENTIRE store (owner scope is nil); the decision resolved here is 'is this caller a management principal', never per-session ownership"}, + "ApplySessionCleanup": {KindCallerOwned, "gates on management authority over the entire store; only the confirmation token/plan is bound to the verified caller, so a stolen or foreign token is rejected before any deletion"}, + "CancelSessionCleanup": {KindCallerOwned, "gates on management authority; matches the verified principal against the bounded job registry — the underlying cleanup scope is store-wide, never caller-owned"}, + "SessionCleanupJob": {KindCallerOwned, "gates on management authority and returns only a caller-bound sanitized job projection; the underlying cleanup data is store-wide, not the caller's own sessions"}, + "StreamSessionEvents": {KindCallerOwned, "event log/live stream resolves through the owning session's authorizeSession check"}, + "WatchSessionEvents": {KindCallerOwned, "durable replay-then-follow watch (ADR 0250); watchLog resolves ownership through the same GetSession check StreamSessionEvents uses, EAGERLY — before any envelope is yielded — because the durable log holds the whole transcript"}, + "Subscribe": {KindCallerOwned, "authorizes via GetSession before registering a live subscriber (issue #368)"}, // --- caller-owned: live run verbs --- "RetryFailedRun": {KindCallerOwned, "authorizes and reloads under runEntryMu before failed-step retry eligibility and launch"}, @@ -345,6 +346,7 @@ var serviceAccessTable = map[string]ClassificationEntry{ "LookupRun": {KindDerived, "in-memory run registry read; every caller-facing entry point (Cancel, Persist, Approve*, MaybeAutoApprovePlan) authorizes the session FIRST and only then consults this"}, "IsLive": {KindDerived, "combined Service-run and engine-child process-local registry; consumed by destructive maintenance, not a caller-facing verb"}, "MaintenanceMutationAvailable": {KindDerived, "read-only capability truth consumed by composition before scheduling automatic retention"}, + "ReconcileReferenceIntents": {KindDerived, "composition-owned maintenance joins provider-owned exact intents to durable sessions and invokes only exact retained callbacks"}, "FinishRun": {KindDerived, "deregisters an id the wire adapter already finished draining from its own authorized run"}, "PublishSessionEvent": {KindDerived, "publishes to subscribers already registered via the (caller-owned) Subscribe for this id; PublishSessionEvent itself takes no ctx and makes no independent decision"}, "RecoverNotice": {KindDerived, "pops a notice keyed by id that only the relay's own immediately-preceding, already-authorized StartRunContent call could have set"}, diff --git a/internal/adapter/server/drain_test.go b/internal/adapter/server/drain_test.go index 657fdf7d77..0009daa2ab 100644 --- a/internal/adapter/server/drain_test.go +++ b/internal/adapter/server/drain_test.go @@ -572,71 +572,110 @@ func TestADR_0294_DrainStopsAdmissionBeforeOwnershipChange(t *testing.T) { } } -func TestADR_0294_DrainPreservesAwaitingResumePointThroughGRPCRelay(t *testing.T) { - lease := &fakeLease{} - store := memstore.New() - capability := server.NewSessionMutationCapability(true) - cat := tool.NewCatalog() - cat.MustRegister(&writeAskTool{}) - eng := agent.NewEngine(agent.Deps{ - LLM: mockllm.New(mockllm.ToolCallTurn(session.NewToolCall("relay-call", "Write", json.RawMessage(`{}`)))), - Catalog: cat, Policy: permpolicy.NewPolicy(nil, permstore.New()), Model: "test-model", - Store: capability.GuardStore(store), - }) - svc, err := server.NewService(server.Config{ - Engine: eng, Store: store, SessionLease: lease, LeaseOwner: "relay-drain", - MutationCapability: capability, - LeaseTTL: time.Hour, LeaseRenewInterval: time.Hour, - PlacementProvider: testPlacementProvider{root: "/ws"}, - PlacementScope: "test", - SharedEngineRoot: "/ws", - }) - if err != nil { - t.Fatal(err) - } - defer svc.Close() - client, cleanup := dialGRPC(t, svc) - defer cleanup() - created, err := client.CreateSession(context.Background(), &mecatlv1.CreateSessionRequest{}) - if err != nil { - t.Fatal(err) - } - id := session.SessionID(created.GetSessionId()) - stream, err := client.Converse(affinityContext(string(id))) - if err != nil { - t.Fatal(err) - } - if err := stream.Send(&mecatlv1.ConverseRequest{Kind: &mecatlv1.ConverseRequest_Prompt{Prompt: &mecatlv1.Prompt{SessionId: string(id), Text: "park"}}}); err != nil { - t.Fatal(err) - } - var askID string - for askID == "" { - resp, recvErr := stream.Recv() - if recvErr != nil { - t.Fatalf("receive ask: %v", recvErr) - } - if ask := resp.GetEvent().GetAsk(); ask != nil { - askID = ask.GetAskId() - } - } +func TestADR_0294_ShutdownPreservesAwaitingDurableEventProjection(t *testing.T) { + for _, tc := range []struct { + name string + shutdown func(*server.Service) error + }{ + {name: "graceful drain", shutdown: func(svc *server.Service) error { return svc.GracefulDrain(context.Background()) }}, + {name: "close", shutdown: func(svc *server.Service) error { svc.Close(); return nil }}, + } { + t.Run(tc.name, func(t *testing.T) { + lease := &fakeLease{} + store := memstore.New() + eventLog := memstore.NewEventLog() + capability := server.NewSessionMutationCapability(true) + cat := tool.NewCatalog() + cat.MustRegister(&writeAskTool{}) + eng := agent.NewEngine(agent.Deps{ + LLM: mockllm.New(mockllm.ToolCallTurn(session.NewToolCall("relay-call", "Write", json.RawMessage(`{}`)))), + Catalog: cat, Policy: permpolicy.NewPolicy(nil, permstore.New()), Model: "test-model", + Store: capability.GuardStore(store), + }) + svc, err := server.NewService(server.Config{ + Engine: eng, Store: store, EventLog: eventLog, SessionLease: lease, LeaseOwner: "relay-drain", + MutationCapability: capability, + LeaseTTL: time.Hour, LeaseRenewInterval: time.Hour, + PlacementProvider: testPlacementProvider{root: "/ws"}, + PlacementScope: "test", + SharedEngineRoot: "/ws", + }) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + client, cleanup := dialGRPC(t, svc) + defer cleanup() + created, err := client.CreateSession(context.Background(), &mecatlv1.CreateSessionRequest{}) + if err != nil { + t.Fatal(err) + } + id := session.SessionID(created.GetSessionId()) + stream, err := client.Converse(affinityContext(string(id))) + if err != nil { + t.Fatal(err) + } + if err := stream.Send(&mecatlv1.ConverseRequest{Kind: &mecatlv1.ConverseRequest_Prompt{Prompt: &mecatlv1.Prompt{SessionId: string(id), Text: "park"}}}); err != nil { + t.Fatal(err) + } + var askID string + for askID == "" { + resp, recvErr := stream.Recv() + if recvErr != nil { + t.Fatalf("receive ask: %v", recvErr) + } + if ask := resp.GetEvent().GetAsk(); ask != nil { + askID = ask.GetAskId() + } + } - drained := make(chan error, 1) - go func() { drained <- svc.GracefulDrain(context.Background()) }() - for { - if _, recvErr := stream.Recv(); recvErr != nil { - break - } - } - if err := <-drained; err != nil { - t.Fatalf("GracefulDrain: %v", err) - } - got, err := store.Load(context.Background(), id) - if err != nil { - t.Fatal(err) - } - pending, ok := got.PendingAsk() - if got.State != session.StateAwaiting || !ok || pending.AskID != askID { - t.Fatalf("relay-backed durable resume point changed: state=%q ask=%+v ok=%t", got.State, pending, ok) + shutdownDone := make(chan error, 1) + go func() { shutdownDone <- tc.shutdown(svc) }() + liveCancelled := false + for { + resp, recvErr := stream.Recv() + if recvErr != nil { + break + } + if result := resp.GetEvent().GetResult(); result != nil && result.GetStop() == string(session.StopCancelled) { + liveCancelled = true + } + } + if err := <-shutdownDone; err != nil { + t.Fatalf("shutdown: %v", err) + } + if !liveCancelled { + t.Fatal("live relay did not deliver shutdown cancellation") + } + + got, err := store.Load(context.Background(), id) + if err != nil { + t.Fatal(err) + } + pending, ok := got.PendingAsk() + if got.State != session.StateAwaiting || !ok || pending.AskID != askID { + t.Fatalf("relay-backed durable resume point changed: state=%q ask=%+v ok=%t", got.State, pending, ok) + } + + var durableAsk bool + for ev, readErr := range eventLog.Read(context.Background(), id) { + if readErr != nil { + t.Fatalf("read event log: %v", readErr) + } + if ev.Type == session.EvPermissionAsk && ev.Ask != nil && ev.Ask.AskID == askID { + durableAsk = true + } + if ev.Type == session.EvPermissionRetract && ev.Ask != nil && ev.Ask.AskID == askID { + t.Fatalf("durable replay retracted preserved ask %q", askID) + } + if ev.Type == session.EvResult && ev.Result != nil && ev.Result.Stop == session.StopCancelled { + t.Fatal("durable replay terminated the preserved awaiting run") + } + } + if !durableAsk { + t.Fatalf("durable replay omitted unresolved ask %q", askID) + } + }) } } diff --git a/internal/adapter/server/event_recorder_test.go b/internal/adapter/server/event_recorder_test.go index 8b41be594a..a76165e2d9 100644 --- a/internal/adapter/server/event_recorder_test.go +++ b/internal/adapter/server/event_recorder_test.go @@ -6,10 +6,12 @@ import ( "iter" "strings" "testing" + "testing/synctest" "time" "unicode/utf8" "github.com/stacklok/mecatl/engine/adapter/eventsource" + "github.com/stacklok/mecatl/engine/adapter/memstore" "github.com/stacklok/mecatl/engine/port" "github.com/stacklok/mecatl/engine/session" "github.com/stacklok/mecatl/internal/adapter/store/jsonlstore" @@ -259,6 +261,224 @@ func TestRunEventRecorderDrainToDiscardStillObserves(t *testing.T) { } } +type blockedAdmissionEventLog struct { + countingEventLog + entered chan struct{} + release chan struct{} +} + +func (l *blockedAdmissionEventLog) Append(ctx context.Context, id session.SessionID, ev session.Event) error { + close(l.entered) + <-l.release + return l.countingEventLog.Append(ctx, id, ev) +} + +func TestRunEventRecorderBlockedAppendDoesNotBlockPersistOrDrain(t *testing.T) { + for _, tc := range []struct { + name string + event session.Event + wantPreserve bool + }{ + {name: "cancelled terminal", event: session.Event{Type: session.EvResult, Result: &session.ResultPayload{Stop: session.StopCancelled}}}, + {name: "plan terminal", event: session.Event{Type: session.EvResult, Result: &session.ResultPayload{Stop: session.StopPlanApproved}}}, + {name: "matching retraction", event: session.Event{Type: session.EvPermissionRetract, Ask: &session.PendingAsk{AskID: "matching"}}}, + {name: "unrelated retraction", event: session.Event{Type: session.EvPermissionRetract, Ask: &session.PendingAsk{AskID: "other"}}, wantPreserve: true}, + {name: "other run terminal", event: session.Event{Type: session.EvResult, RunID: "other", Result: &session.ResultPayload{Stop: session.StopCancelled}}, wantPreserve: true}, + {name: "unrelated event", event: session.Event{Type: session.EvHook}, wantPreserve: true}, + } { + for _, postWriteError := range []bool{false, true} { + name := tc.name + "/success" + if postWriteError { + name = tc.name + "/ambiguous append failure" + } + t.Run(name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + id := session.SessionID("blocked-append") + sess, run, askID := liveAwaitingControlRun(t, id) + defer func() { + run.Cancel() + for range run.Events() { + } + }() + store := memstore.New() + if err := store.Save(t.Context(), sess); err != nil { + t.Fatal(err) + } + log := &blockedAdmissionEventLog{ + countingEventLog: countingEventLog{postWriteFailCalls: map[int]bool{1: postWriteError}}, + entered: make(chan struct{}), release: make(chan struct{}), + } + diag := &countingDiagnostics{} + st := &runState{run: run, sess: sess, settled: make(chan struct{}), titleRevision: sess.TitleRevision} + svc := &Service{ + cfg: Config{Store: store, EventLog: log, Diagnostics: diag, MutationCapability: NewSessionMutationCapability(false)}, + runs: map[session.SessionID]*runState{id: st}, + } + svc.Persist(t.Context(), id) + if !st.awaiting.Load() || st.awaitingAskID != askID { + t.Fatal("fixture did not persist the exact awaiting handoff") + } + ev := tc.event + if ev.RunID == "" { + ev.RunID = run.RunID() + } + if ev.Ask != nil && ev.Ask.AskID == "matching" { + ev.Ask = &session.PendingAsk{AskID: askID} + } + ctx := session.WithPrincipal(t.Context(), &session.Principal{Subject: "caller", Issuer: "test"}) + recorder := NewRunEventRecorder(ctx, svc, id) + appended := make(chan struct{}) + go func() { + recorder.Observe(ev) + recorder.Close() + close(appended) + }() + defer func() { + close(log.release) + <-appended + if len(log.attempts) != 1 || len(log.recorded) != 1 { + t.Errorf("attempts/records = %d/%d, want 1/1 even on ambiguous failure", len(log.attempts), len(log.recorded)) + } else if actor := log.recorded[0].Actor; actor == nil || actor.Subject != "caller" || actor.Issuer != "test" { + t.Errorf("actor = %+v, want verified caller", actor) + } + wantWarnings := 0 + if postWriteError { + wantWarnings = 1 + } + if diag.warnings != wantWarnings { + t.Errorf("warnings = %d, want %d; append failure must not be hidden by freeze", diag.warnings, wantWarnings) + } + }() + <-log.entered + synctest.Wait() + if st.cancelSignaled { + t.Fatal("event admission fabricated an owner cancellation") + } + persistedAndFrozen := make(chan struct{}) + go func() { + svc.Persist(t.Context(), id) + svc.snapshotDrainState() + close(persistedAndFrozen) + }() + synctest.Wait() + select { + case <-persistedAndFrozen: + default: + t.Fatal("blocked EventLog.Append holds the Persist/drain lifecycle barrier") + } + if got := st.preserveDurable.Load(); got != tc.wantPreserve { + t.Fatalf("preserved handoff = %t, want %t after event admission", got, tc.wantPreserve) + } + }) + }) + } + } +} + +func TestRunEventRecorderShutdownProjectionIsLifecycleScoped(t *testing.T) { + t.Run("owner cancellation remains durable", func(t *testing.T) { + id := session.SessionID("owner-cancel") + sess, run, askID := liveAwaitingControlRun(t, id) + log := memstore.NewEventLog() + st := &runState{run: run, sess: sess, settled: make(chan struct{}), awaitingAskID: askID} + st.awaiting.Store(true) + svc := &Service{ + cfg: Config{EventLog: log, Diagnostics: port.NopDiagnostics{}}, + runs: map[session.SessionID]*runState{id: st}, + } + recorder := NewRunEventRecorder(context.Background(), svc, id) + recorder.Observe(session.Event{Type: session.EvPermissionAsk, RunID: run.RunID(), Ask: &session.PendingAsk{AskID: askID}}) + if err := svc.cancelLiveRun(id, run, run.RunID()); err != nil { + t.Fatal(err) + } + svc.snapshotDrainState() + for ev := range run.Events() { + recorder.Observe(ev) + } + recorder.Close() + + var cancelled bool + for ev, err := range log.Read(context.Background(), id) { + if err != nil { + t.Fatal(err) + } + if ev.Type == session.EvResult && ev.Result != nil && ev.Result.Stop == session.StopCancelled { + cancelled = true + } + } + if !cancelled { + t.Fatal("owner cancellation was suppressed by later shutdown freeze") + } + }) + + t.Run("preserved shutdown omits only matching cancellation records", func(t *testing.T) { + id := session.SessionID("shutdown-preserved") + sess, run, askID := liveAwaitingControlRun(t, id) + defer func() { + run.Cancel() + for range run.Events() { + } + }() + log := memstore.NewEventLog() + st := &runState{run: run, sess: sess, settled: make(chan struct{}), awaitingAskID: askID} + st.awaiting.Store(true) + svc := &Service{ + cfg: Config{EventLog: log, Diagnostics: port.NopDiagnostics{}}, + runs: map[session.SessionID]*runState{id: st}, + } + svc.snapshotDrainState() + recorder := NewRunEventRecorder(context.Background(), svc, id) + recorder.Observe(session.Event{Type: session.EvHook, RunID: run.RunID(), Text: "unrelated"}) + recorder.Observe(session.Event{Type: session.EvPermissionRetract, RunID: run.RunID(), Ask: &session.PendingAsk{AskID: "other-ask"}}) + recorder.Observe(session.Event{Type: session.EvPermissionRetract, RunID: run.RunID(), Ask: &session.PendingAsk{AskID: askID}}) + recorder.Observe(session.Event{Type: session.EvResult, RunID: run.RunID(), Result: &session.ResultPayload{Stop: session.StopCancelled}}) + recorder.Close() + + var kinds []session.EventType + for ev, err := range log.Read(context.Background(), id) { + if err != nil { + t.Fatal(err) + } + kinds = append(kinds, ev.Type) + if ev.Type == session.EvPermissionRetract && (ev.Ask == nil || ev.Ask.AskID != "other-ask") { + t.Fatalf("recorded retraction = %+v, want only unrelated ask", ev.Ask) + } + } + if len(kinds) != 2 || kinds[0] != session.EvHook || kinds[1] != session.EvPermissionRetract { + t.Fatalf("durable kinds = %v, want unrelated hook and retraction", kinds) + } + }) + + t.Run("failed awaiting save does not suppress cancellation", func(t *testing.T) { + id := session.SessionID("awaiting-save-failed") + sess, run, _ := liveAwaitingControlRun(t, id) + defer func() { + run.Cancel() + for range run.Events() { + } + }() + log := memstore.NewEventLog() + svc := &Service{ + cfg: Config{EventLog: log, Diagnostics: port.NopDiagnostics{}}, + runs: map[session.SessionID]*runState{id: {run: run, sess: sess, settled: make(chan struct{})}}, + } + svc.snapshotDrainState() + recorder := NewRunEventRecorder(context.Background(), svc, id) + recorder.Observe(session.Event{Type: session.EvResult, RunID: run.RunID(), Result: &session.ResultPayload{Stop: session.StopCancelled}}) + recorder.Close() + + for ev, err := range log.Read(context.Background(), id) { + if err != nil { + t.Fatal(err) + } + if ev.Type == session.EvResult && ev.Result != nil && ev.Result.Stop == session.StopCancelled { + return + } + } + t.Fatal("cancellation after failed awaiting save was suppressed") + }) +} + func TestRunEventRecorderOversizedEscapedUTF8RoundTripsThroughJSONL(t *testing.T) { log, err := jsonlstore.New(t.TempDir()) if err != nil { diff --git a/internal/adapter/server/exact_plan_ask.go b/internal/adapter/server/exact_plan_ask.go index 3870ffec17..c2974db7fe 100644 --- a/internal/adapter/server/exact_plan_ask.go +++ b/internal/adapter/server/exact_plan_ask.go @@ -196,6 +196,10 @@ func (s *Service) resolvePersistedPlanAsk(ctx context.Context, id session.Sessio if !leaseHeld() { return RunAskAcknowledgement{}, fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) } + env, err = s.acquireExecution(requestLeaseCtx, st, sess, sess.RunID(), env) + if err != nil { + return RunAskAcknowledgement{}, err + } ownedCtx, stopOwned := s.detachedControlContext(ctx) ownedLeaseCtx, stopOwnedLease, _ := s.mutationLeaseContext(ownedCtx, id) stopRun := func() { stopOwnedLease(); stopOwned(); stopRequestLease() } @@ -236,6 +240,7 @@ func (s *Service) resolvePersistedPlanAsk(ctx context.Context, id session.Sessio st.planContinuation = continuation s.mu.Unlock() continuationReserved = false + s.startExecutionRenewal(id, st, run) go func() { defer s.detachedControlWG.Done() s.relayDetachedControlRun(ownedLeaseCtx, id, run) @@ -268,6 +273,10 @@ func (s *Service) finishExactPlanRun(ctx context.Context, id session.SessionID, s.mu.Unlock() s.completeRelay(ctx, id, run) stop, stopped := st.sess.RecordedStopReason() + // The old relay has drained. Join native renewal and release its claim before + // retiring the operation pin or admitting the reserved continuation. Keep the + // entry lock across provider I/O, but never the global registry mutex. + s.teardownExecution(st) s.removeRunState(id, st) if !stopped || stop != session.StopPlanApproved { continuation.release(s) diff --git a/internal/adapter/server/execution_renewal_test.go b/internal/adapter/server/execution_renewal_test.go new file mode 100644 index 0000000000..06cc3d5ce1 --- /dev/null +++ b/internal/adapter/server/execution_renewal_test.go @@ -0,0 +1,33 @@ +package server + +import ( + "context" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/tool" +) + +type deadlineExecutionHandle struct { + deadline time.Time +} + +func (*deadlineExecutionHandle) Environment() tool.Environment { return tool.Environment{} } +func (*deadlineExecutionHandle) Renew(context.Context) error { return nil } +func (*deadlineExecutionHandle) Release(context.Context) error { return nil } +func (h *deadlineExecutionHandle) RenewalDeadline() time.Time { return h.deadline } + +var _ ExecutionRunHandle = (*deadlineExecutionHandle)(nil) + +func TestExecutionRenewDelayTracksShortGrantExpiry(t *testing.T) { + now := time.Date(2026, 9, 17, 12, 0, 0, 0, time.UTC) + handle := &deadlineExecutionHandle{deadline: now.Add(5 * time.Second)} + delay, usable := executionRenewDelay(handle, now) + if !usable || delay <= 0 || delay >= 5*time.Second || delay < 100*time.Millisecond { + t.Fatalf("five-second grant schedule delay=%v usable=%t", delay, usable) + } + handle.deadline = now.Add(250 * time.Millisecond) + if _, usable := executionRenewDelay(handle, now); usable { + t.Fatal("unusable grant remainder would start execution renewal") + } +} diff --git a/internal/adapter/server/harness_context_test.go b/internal/adapter/server/harness_context_test.go index ac837fcec1..149076b949 100644 --- a/internal/adapter/server/harness_context_test.go +++ b/internal/adapter/server/harness_context_test.go @@ -86,6 +86,95 @@ func TestHarnessContextGeneratedIDCollisionAfterClose(t *testing.T) { } } +func TestHarnessContextGeneratedIDPublicationCollision(t *testing.T) { + owner := &session.Principal{Issuer: "issuer", Subject: "owner", GrantType: session.GrantTypeUser} + for _, tc := range []struct { + name string + winnerOwner *session.Principal + mode session.PermissionMode + want error + }{ + {"identical", owner, session.ModeDefault, nil}, + {"different owner", &session.Principal{Issuer: "issuer", Subject: "other", GrantType: session.GrantTypeUser}, session.ModeDefault, ErrNotFound}, + {"malformed owner", &session.Principal{Issuer: "issuer"}, session.ModeDefault, ErrNotFound}, + {"unowned", nil, session.ModeDefault, ErrNotFound}, + {"different request", owner, session.ModeAccept, ErrInvalidArgument}, + } { + t.Run(tc.name, func(t *testing.T) { + store := memstore.New() + ref := session.EnvironmentRef{Kind: "kubernetes", ID: "placement", Revision: "v1"} + commits := 0 + placement := &repairPlacementProvider{binding: PlacementBinding{ + Ref: ref, Environment: tool.MustEnvironment(ref, memfs.NewWorkspace("/bound"), memledger.New(), nil), + Commit: func(context.Context) error { commits++; return nil }, + }} + eng := repairEngine() + winnerService, err := NewService(Config{Engine: eng, Store: store, PlacementProvider: placement, PlacementScope: "test", SharedEngineRoot: "/bound"}) + if err != nil { + t.Fatal(err) + } + defer winnerService.Close() + resolver := &failedCreateSourceResolver{} + closes := 0 + winnerCommits := 0 + var winner *session.Session + loser, err := NewService(Config{ + Engine: eng, Store: store, OwnershipEnforced: true, + NewID: func() session.SessionID { return "generated-publication-race" }, + PlacementProvider: placement, PlacementScope: "test", SharedEngineRoot: "/bound", + Commands: resolver, + SessionEngine: func(context.Context, ProviderSelector, []mcp.ServerConfig, SessionProfile, string, session.PermissionMode) (SessionEngineResult, error) { + t.Fatal("legacy factory called") + return SessionEngineResult{}, nil + }, + SessionContextEngine: func(ctx context.Context, id session.SessionID, _ *session.Principal, _ ExecutionWorkspaceAcquirer, _ ProviderSelector, _ []mcp.ServerConfig, _ SessionProfile, _ string, _ session.PermissionMode, _ []tool.Tool) (SessionEngineResult, error) { + // The losing Service has already observed absence and reserved its ID. + var err error + winner, err = winnerService.CreateSessionWithProfile(ctx, tc.mode, session.Limits{}, ProviderSelector{}, ProfileDefault, WithSessionID(id), WithOwner(tc.winnerOwner)) + if err != nil { + t.Fatal(err) + } + if commits != 1 { + t.Fatalf("winner commits=%d, want 1", commits) + } + winnerCommits = commits + return SessionEngineResult{Engine: eng, Close: func() error { closes++; return nil }}, nil + }, + }) + if err != nil { + t.Fatal(err) + } + defer loser.Close() + ctx := session.WithPrincipal(t.Context(), owner) + got, err := loser.CreateSession(ctx, session.ModeDefault, session.Limits{}) + if !errors.Is(err, tc.want) { + t.Fatalf("collision error=%v, want %v", err, tc.want) + } + if tc.want == nil { + if got == nil || got.Incarnation() != winner.Incarnation() { + t.Fatal("did not return the identical authorized winner") + } + } else if got != nil { + t.Fatal("adopted an unauthorized or different winner") + } + wantCommits := winnerCommits + if tc.want == nil { + wantCommits++ + } + if commits != wantCommits { + t.Fatalf("commits=%d, want %d (winner=%d)", commits, wantCommits, winnerCommits) + } + if closes != 1 || len(resolver.retired) != 1 || len(loser.sessionEngines) != 0 || len(loser.reservedIDs) != 0 || placement.reattaches != 0 { + t.Fatalf("loser lifecycle: closes=%d retirements=%v engines=%d reservations=%d attaches=%d", closes, resolver.retired, len(loser.sessionEngines), len(loser.reservedIDs), placement.reattaches) + } + persisted, err := store.Load(ctx, winner.ID) + if err != nil || !reflect.DeepEqual(persisted, winner) { + t.Fatalf("collision changed winner: %v", err) + } + }) + } +} + func TestHarnessContextUnpublishedCreateRetiresBinding(t *testing.T) { resolver := &failedCreateSourceResolver{} ref := session.EnvironmentRef{Kind: session.EnvKindMem, ID: "placement", Revision: "v1"} diff --git a/internal/adapter/server/http.go b/internal/adapter/server/http.go index 3eaabe54dd..336897697e 100644 --- a/internal/adapter/server/http.go +++ b/internal/adapter/server/http.go @@ -20,6 +20,7 @@ import ( "github.com/stacklok/mecatl/engine/port" "github.com/stacklok/mecatl/engine/session" "github.com/stacklok/mecatl/internal/adapter/mcp" + "github.com/stacklok/mecatl/internal/creatediag" ) // HTTPHandler is the HTTP/SSE adapter over the shared Service. It serves the @@ -479,6 +480,11 @@ func toContentParts(parts []promptContentBody) ([]session.Content, error) { // createSession handles POST /v1/sessions. func (h *HTTPHandler) createSession(w http.ResponseWriter, r *http.Request) { + if h.svc.cfg.ExecutionAccess != nil { + r = r.WithContext(creatediag.Start(r.Context(), h.svc.Diagnostics())) + } + creatediag.Note(r.Context(), "http_handler", "begin", 0) + defer func() { creatediag.Note(r.Context(), "http_handler", "returned", 1) }() var body createSessionBody // STRICT decode. An unknown field is a 400, not a silent drop. // @@ -549,6 +555,7 @@ func (h *HTTPHandler) createSession(w http.ResponseWriter, r *http.Request) { writeServiceError(w, err) return } + responseDone := creatediag.Begin(r.Context(), "http_response") scaps := h.svc.sessionCapabilitiesFor(sess) writeJSON(w, http.StatusCreated, createSessionResp{ SessionID: string(sess.ID), @@ -556,6 +563,7 @@ func (h *HTTPHandler) createSession(w http.ResponseWriter, r *http.Request) { ResolvedModel: resolvedModelToJSON(h.svc.resolvedModelFor(sess)), Placement: placementMetadataToJSON(sess.Placement), }) + responseDone(r.Context().Err()) } // getSession handles GET /v1/sessions/{id}. diff --git a/internal/adapter/server/native_execution_lifecycle_test.go b/internal/adapter/server/native_execution_lifecycle_test.go new file mode 100644 index 0000000000..817e67d513 --- /dev/null +++ b/internal/adapter/server/native_execution_lifecycle_test.go @@ -0,0 +1,483 @@ +package server_test + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/stacklok/mecatl/engine/adapter/memfs" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/adapter/permpolicy" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" +) + +type nativeRunProvider struct { + ref session.EnvironmentRef + acquireErr error + acquireHook func() + exclusive bool + held atomic.Bool + released chan struct{} + renewed chan struct{} + renewErr error + renewHook func(context.Context) error + releaseHook func(context.Context) error + renewalDeadline time.Time + acquires atomic.Int64 + releases atomic.Int64 +} + +func (p *nativeRunProvider) Bind(context.Context, server.PlacementBindRequest) (server.PlacementBinding, error) { + return p.binding(), nil +} +func (p *nativeRunProvider) Reattach(context.Context, server.PlacementReattachRequest) (server.PlacementBinding, error) { + return p.binding(), nil +} +func (p *nativeRunProvider) Applies(ref session.EnvironmentRef) bool { return ref == p.ref } +func (p *nativeRunProvider) AcquireRun(context.Context, server.ExecutionRunRequest) (server.ExecutionRunHandle, error) { + p.acquires.Add(1) + if p.acquireErr != nil { + return nil, p.acquireErr + } + if p.exclusive && !p.held.CompareAndSwap(false, true) { + return nil, errors.New("native run already held") + } + if p.acquireHook != nil { + p.acquireHook() + } + handle := &nativeRunHandle{provider: p, env: p.binding().Environment} + if !p.renewalDeadline.IsZero() { + return &renewingNativeRunHandle{nativeRunHandle: handle, deadline: p.renewalDeadline}, nil + } + return handle, nil +} +func (p *nativeRunProvider) binding() server.PlacementBinding { + return server.PlacementBinding{Ref: p.ref, Environment: tool.MustEnvironment(p.ref, memfs.NewWorkspace("/native"), memledger.New(), nil), GovernanceRoot: "/native"} +} + +type nativeRunHandle struct { + provider *nativeRunProvider + env tool.Environment +} + +func (h *nativeRunHandle) Environment() tool.Environment { return h.env } +func (*nativeRunHandle) Renew(context.Context) error { return nil } +func (h *nativeRunHandle) Release(ctx context.Context) error { + if h.provider.releaseHook != nil { + if err := h.provider.releaseHook(ctx); err != nil { + return err + } + } + if h.provider.exclusive { + h.provider.held.Store(false) + } + h.provider.releases.Add(1) + if h.provider.released != nil { + h.provider.released <- struct{}{} + } + return nil +} + +type renewingNativeRunHandle struct { + *nativeRunHandle + deadline time.Time +} + +func (h *renewingNativeRunHandle) RenewalDeadline() time.Time { return h.deadline } +func (h *renewingNativeRunHandle) Renew(ctx context.Context) error { + if h.provider.renewed != nil { + h.provider.renewed <- struct{}{} + } + if h.provider.renewHook != nil { + return h.provider.renewHook(ctx) + } + return h.provider.renewErr +} + +func TestNativePersistedResolveAcquisitionFailurePreservesDurableAsk(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + acquireErr := errors.New("native run unavailable") + provider := &nativeRunProvider{ref: ref, acquireErr: acquireErr} + store := memstore.New() + parked, ask := makePersistedControlSession(t, "native-persisted-resolve") + parked.EnvironmentRef = ref + if err := store.Save(t.Context(), parked); err != nil { + t.Fatal(err) + } + var ran atomic.Int64 + cat := tool.NewCatalog() + cat.MustRegister(&writeAskTool{ran: &ran}) + engine := agent.NewEngine(agent.Deps{ + LLM: mockllm.New(mockllm.TextTurn("must not continue")), Catalog: cat, + Policy: permpolicy.NewPolicy(nil, nil), Model: "test", + }) + svc, err := newPlacementTestService(server.Config{ + Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "test", + ExecutionAccess: provider, SharedEngineRoot: "/native", Now: func() time.Time { return time.Unix(1, 0) }, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(svc.Close) + + _, err = svc.ResolveRunAsk(t.Context(), parked.ID, parked.RunID(), ask.AskID, session.VerdictAllowOnce) + if !errors.Is(err, acquireErr) { + t.Fatalf("ResolveRunAsk error = %v, want acquisition error", err) + } + if provider.acquires.Load() != 1 || provider.releases.Load() != 0 || ran.Load() != 0 { + t.Fatalf("acquires=%d releases=%d executions=%d", provider.acquires.Load(), provider.releases.Load(), ran.Load()) + } + got, err := store.Load(t.Context(), parked.ID) + if err != nil { + t.Fatal(err) + } + pending, ok := got.PendingAsk() + if got.State != session.StateAwaiting || !ok || pending.AskID != ask.AskID { + t.Fatalf("durable ask changed: state=%q pending=%+v ok=%t", got.State, pending, ok) + } +} + +func TestNativePersistedResolveEmptyContinuationReleasesClaim(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + provider := &nativeRunProvider{ref: ref, released: make(chan struct{}, 1)} + store := memstore.New() + parked, ask := makePersistedControlSession(t, "native-empty-continuation") + parked.EnvironmentRef = ref + if err := store.Save(t.Context(), parked); err != nil { + t.Fatal(err) + } + var ran atomic.Int64 + cat := tool.NewCatalog() + cat.MustRegister(&writeAskTool{ran: &ran}) + engine := agent.NewEngine(agent.Deps{LLM: mockllm.New(), Catalog: cat, Policy: permpolicy.NewPolicy(nil, nil), Model: "test"}) + svc, err := newPlacementTestService(server.Config{ + Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "test", + ExecutionAccess: provider, SharedEngineRoot: "/native", Now: func() time.Time { return time.Unix(1, 0) }, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(svc.Close) + + if _, err := svc.ResolveRunAsk(t.Context(), parked.ID, parked.RunID(), ask.AskID, session.VerdictAllowOnce); err != nil { + t.Fatal(err) + } + select { + case <-provider.released: + case <-time.After(time.Second): + t.Fatal("empty continuation did not settle its native claim") + } + if provider.acquires.Load() != 1 || provider.releases.Load() != 1 || ran.Load() != 1 { + t.Fatalf("acquires=%d releases=%d executions=%d", provider.acquires.Load(), provider.releases.Load(), ran.Load()) + } +} + +func TestNativePersistedResolveRenewalFailureCancelsAndReleasesAfterDrain(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + provider := &nativeRunProvider{ + ref: ref, released: make(chan struct{}, 1), renewed: make(chan struct{}, 1), + renewErr: errors.New("renewal lost"), renewalDeadline: time.Now().Add(600 * time.Millisecond), + } + store := memstore.New() + parked, ask := makePersistedControlSession(t, "native-renewal-failure") + parked.EnvironmentRef = ref + if err := store.Save(t.Context(), parked); err != nil { + t.Fatal(err) + } + model := &closeJoinProvider{entered: make(chan struct{}), cancelled: make(chan struct{}), release: make(chan struct{})} + var ran atomic.Int64 + cat := tool.NewCatalog() + cat.MustRegister(&writeAskTool{ran: &ran}) + engine := agent.NewEngine(agent.Deps{LLM: model, Catalog: cat, Policy: permpolicy.NewPolicy(nil, nil), Model: "test"}) + svc, err := newPlacementTestService(server.Config{ + Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "test", + ExecutionAccess: provider, SharedEngineRoot: "/native", Now: func() time.Time { return time.Unix(1, 0) }, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(svc.Close) + releaseModel := sync.OnceFunc(func() { close(model.release) }) + defer releaseModel() + + if _, err := svc.ResolveRunAsk(t.Context(), parked.ID, parked.RunID(), ask.AskID, session.VerdictAllowOnce); err != nil { + t.Fatal(err) + } + select { + case <-model.entered: + case <-time.After(time.Second): + t.Fatal("resumed run did not reach its model continuation") + } + select { + case <-provider.renewed: + case <-time.After(time.Second): + t.Fatal("execution renewal did not start") + } + select { + case <-model.cancelled: + case <-time.After(time.Second): + t.Fatal("renewal failure did not cancel the resumed run") + } + if provider.releases.Load() != 0 { + t.Fatal("claim released before the resumed run drained") + } + releaseModel() + select { + case <-provider.released: + case <-time.After(time.Second): + t.Fatal("claim was not released after detached relay drain") + } + if provider.acquires.Load() != 1 || provider.releases.Load() != 1 || ran.Load() != 1 { + t.Fatalf("acquires=%d releases=%d executions=%d", provider.acquires.Load(), provider.releases.Load(), ran.Load()) + } +} + +func TestNativePersistedResolveCloseHoldsClaimUntilDetachedDrain(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + provider := &nativeRunProvider{ref: ref, exclusive: true, released: make(chan struct{}, 2)} + store := memstore.New() + parked, ask := makePersistedControlSession(t, "native-close-claim-drain") + parked.EnvironmentRef = ref + if err := store.Save(t.Context(), parked); err != nil { + t.Fatal(err) + } + model := &closeJoinProvider{entered: make(chan struct{}), cancelled: make(chan struct{}), release: make(chan struct{})} + var ran atomic.Int64 + cat := tool.NewCatalog() + cat.MustRegister(&writeAskTool{ran: &ran}) + engine := agent.NewEngine(agent.Deps{LLM: model, Catalog: cat, Policy: permpolicy.NewPolicy(nil, nil), Model: "test"}) + svc, err := newPlacementTestService(server.Config{ + Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "test", + ExecutionAccess: provider, SharedEngineRoot: "/native", Now: func() time.Time { return time.Unix(1, 0) }, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(svc.Close) + releaseModel := sync.OnceFunc(func() { close(model.release) }) + t.Cleanup(releaseModel) + + if _, err := svc.ResolveRunAsk(t.Context(), parked.ID, parked.RunID(), ask.AskID, session.VerdictAllowOnce); err != nil { + t.Fatal(err) + } + select { + case <-model.entered: + case <-time.After(time.Second): + t.Fatal("resumed run did not reach its model continuation") + } + closed := make(chan struct{}) + go func() { + svc.Close() + close(closed) + }() + select { + case <-model.cancelled: + case <-time.After(time.Second): + t.Fatal("Service.Close did not cancel the resumed continuation") + } + synctest.Wait() + if provider.releases.Load() != 0 { + t.Fatal("Service.Close released the native claim before the detached continuation drained") + } + competing, err := provider.AcquireRun(t.Context(), server.ExecutionRunRequest{Ref: ref, BindingID: "competing", RunID: "competing"}) + if err == nil { + _ = competing.Release(t.Context()) + t.Fatal("competing writer acquired the native claim while the cancelled continuation was still draining") + } + select { + case <-closed: + t.Fatal("Service.Close returned before the detached continuation drained") + default: + } + + releaseModel() + select { + case <-closed: + case <-time.After(time.Second): + t.Fatal("Service.Close did not finish after the detached continuation drained") + } + select { + case <-provider.released: + case <-time.After(time.Second): + t.Fatal("native claim was not released after detached drain") + } + if provider.acquires.Load() != 2 || provider.releases.Load() != 1 || provider.held.Load() || ran.Load() != 1 { + t.Fatalf("acquires=%d releases=%d held=%t executions=%d", provider.acquires.Load(), provider.releases.Load(), provider.held.Load(), ran.Load()) + } + }) +} + +func TestNativePersistedResolveCloseRejectsLateProvisionalClaim(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + provider := &nativeRunProvider{ref: ref, exclusive: true, released: make(chan struct{}, 1)} + acquireEntered := make(chan struct{}) + releaseAcquire := make(chan struct{}) + provider.acquireHook = func() { + close(acquireEntered) + <-releaseAcquire + } + store := memstore.New() + parked, ask := makePersistedControlSession(t, "native-close-provisional-claim") + parked.EnvironmentRef = ref + if err := store.Save(t.Context(), parked); err != nil { + t.Fatal(err) + } + var ran atomic.Int64 + cat := tool.NewCatalog() + cat.MustRegister(&writeAskTool{ran: &ran}) + engine := agent.NewEngine(agent.Deps{LLM: mockllm.New(mockllm.TextTurn("must not continue")), Catalog: cat, Policy: permpolicy.NewPolicy(nil, nil), Model: "test"}) + svc, err := newPlacementTestService(server.Config{ + Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "test", + ExecutionAccess: provider, SharedEngineRoot: "/native", Now: func() time.Time { return time.Unix(1, 0) }, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(svc.Close) + ctx, cancel := context.WithCancel(t.Context()) + unblockAcquire := sync.OnceFunc(func() { close(releaseAcquire) }) + resolveDone := make(chan error, 1) + t.Cleanup(func() { + cancel() + unblockAcquire() + select { + case <-resolveDone: + case <-time.After(time.Second): + t.Error("provisional approval admission did not finish during cleanup") + } + }) + go func() { + defer close(resolveDone) + _, err := svc.ResolveRunAsk(ctx, parked.ID, parked.RunID(), ask.AskID, session.VerdictAllowOnce) + resolveDone <- err + }() + select { + case <-acquireEntered: + case <-time.After(time.Second): + t.Fatal("native claim acquisition did not start") + } + svc.Close() + if provider.releases.Load() != 0 { + t.Fatal("shutdown released a native handle before its blocked acquisition returned") + } + unblockAcquire() + select { + case err := <-resolveDone: + if err == nil { + t.Fatal("provisional approval admission succeeded after shutdown") + } + case <-time.After(time.Second): + t.Fatal("provisional approval admission did not reject its late native handle") + } + select { + case <-provider.released: + case <-time.After(time.Second): + t.Fatal("late native handle was not released") + } + if _, live := svc.LookupRun(parked.ID); live { + t.Fatal("late provisional admission remained published after shutdown") + } + if provider.acquires.Load() != 1 || provider.releases.Load() != 1 || provider.held.Load() || ran.Load() != 0 { + t.Fatalf("acquires=%d releases=%d held=%t executions=%d", provider.acquires.Load(), provider.releases.Load(), provider.held.Load(), ran.Load()) + } +} + +func TestNativePersistedResolveDrainBeforePublicationReleasesClaimOnce(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + provider := &nativeRunProvider{ref: ref, released: make(chan struct{}, 1)} + store := memstore.New() + parked, ask := makePersistedControlSession(t, "native-drain-before-publication") + parked.EnvironmentRef = ref + if err := store.Save(t.Context(), parked); err != nil { + t.Fatal(err) + } + var ran atomic.Int64 + cat := tool.NewCatalog() + cat.MustRegister(&writeAskTool{ran: &ran}) + engine := agent.NewEngine(agent.Deps{ + LLM: mockllm.New(mockllm.TextTurn("must not continue")), Catalog: cat, + Policy: permpolicy.NewPolicy(nil, nil), Model: "test", + }) + var svc *server.Service + provider.acquireHook = func() { svc.Drain() } + var err error + svc, err = newPlacementTestService(server.Config{ + Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "test", + ExecutionAccess: provider, SharedEngineRoot: "/native", Now: func() time.Time { return time.Unix(1, 0) }, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(svc.Close) + + _, err = svc.ResolveRunAsk(t.Context(), parked.ID, parked.RunID(), ask.AskID, session.VerdictAllowOnce) + if err == nil { + t.Fatal("ResolveRunAsk succeeded after drain began") + } + select { + case <-provider.released: + case <-time.After(time.Second): + t.Fatal("acquired claim was not released") + } + if provider.acquires.Load() != 1 || provider.releases.Load() != 1 || ran.Load() != 0 { + t.Fatalf("acquires=%d releases=%d executions=%d", provider.acquires.Load(), provider.releases.Load(), ran.Load()) + } +} + +func TestNativeRunClaimHeldUntilActualDrainRelease(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + provider := &nativeRunProvider{ref: ref} + llm := mockllm.New(mockllm.TextTurn("first"), mockllm.TextTurn("second")) + engine := agent.NewEngine(agent.Deps{LLM: llm, Catalog: tool.NewCatalog(), Policy: permpolicy.NewPolicy(nil, nil), Model: "test"}) + store := memstore.New() + svc, err := newPlacementTestService(server.Config{Engine: engine, Store: store, PlacementProvider: provider, PlacementScope: "test", ExecutionAccess: provider, SharedEngineRoot: "/native", DefaultLimits: session.Limits{MaxTurns: 2}, Now: func() time.Time { return time.Unix(1, 0) }}) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + sess := session.New("native-session", session.ModeDefault, ref, session.Limits{MaxTurns: 2}, time.Unix(1, 0)) + if err := store.Save(t.Context(), sess); err != nil { + t.Fatal(err) + } + + first, err := svc.StartRunContent(t.Context(), sess.ID, "one", nil) + if err != nil { + t.Fatal(err) + } + for range first.Events() { + } + if _, err := svc.StartRunContent(t.Context(), sess.ID, "too early", nil); err == nil { + t.Fatal("native continuation started before the prior relay released its claim") + } + if provider.acquires.Load() != 1 || provider.releases.Load() != 0 { + t.Fatalf("before FinishRun acquires=%d releases=%d", provider.acquires.Load(), provider.releases.Load()) + } + + svc.FinishRun(sess.ID, first) + if provider.releases.Load() != 1 { + t.Fatalf("release count=%d, want 1", provider.releases.Load()) + } + second, err := svc.StartRunContent(t.Context(), sess.ID, "two", nil) + if err != nil { + t.Fatal(err) + } + for range second.Events() { + } + svc.FinishRun(sess.ID, second) + svc.FinishRun(sess.ID, second) + svc.Close() + svc.Close() + if provider.acquires.Load() != 2 || provider.releases.Load() != 2 { + t.Fatalf("after continuation acquires=%d releases=%d", provider.acquires.Load(), provider.releases.Load()) + } +} diff --git a/internal/adapter/server/native_plan_lifecycle_test.go b/internal/adapter/server/native_plan_lifecycle_test.go new file mode 100644 index 0000000000..b2987f3af4 --- /dev/null +++ b/internal/adapter/server/native_plan_lifecycle_test.go @@ -0,0 +1,283 @@ +package server_test + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/adapter/permpolicy" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" +) + +type nativePlanTerminalLog struct { + port.EventLog + entered chan struct{} + release chan struct{} +} + +func (l *nativePlanTerminalLog) Append(ctx context.Context, id session.SessionID, ev session.Event) error { + if ev.Type == session.EvResult && ev.Result != nil && ev.Result.Stop == session.StopEndTurn { + close(l.entered) + <-l.release + } + return l.EventLog.Append(ctx, id, ev) +} + +func TestNativePersistedExactPlanClaimLifecycle(t *testing.T) { + for _, outcome := range []string{"allow", "acquisition failure", "cancel before acceptance"} { + t.Run(outcome, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + provider := &nativeRunProvider{ref: ref, exclusive: true, renewed: make(chan struct{}, 2), renewalDeadline: time.Now().Add(time.Minute)} + var renewExited atomic.Int64 + provider.renewHook = func(ctx context.Context) error { + <-ctx.Done() + renewExited.Add(1) + return ctx.Err() + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + acquireErr := errors.New("native claim unavailable") + switch outcome { + case "acquisition failure": + provider.acquireErr = acquireErr + case "cancel before acceptance": + provider.acquireHook = cancel + } + store := memstore.New() + parked := makePlanControlSession(t, "native-plan-restored") + parked.EnvironmentRef = ref + if err := store.Save(ctx, parked); err != nil { + t.Fatal(err) + } + log := &blockingPlanResultLog{serverLog: memstore.NewEventLog(), seen: make(chan struct{}), release: make(chan struct{})} + allowPlanDrain := sync.OnceFunc(func() { close(log.release) }) + llm := mockllm.New(mockllm.TextTurn("executed")) + cat := tool.NewCatalog() + cat.MustRegister(agent.NewPresentPlanTool()) + engine := agent.NewEngine(agent.Deps{LLM: llm, Catalog: cat, Policy: permpolicy.NewPolicy(allowRules(), nil), Model: "test-model", Interactive: true, Store: store}) + svc, err := newPlacementTestService(server.Config{ + Engine: engine, Store: store, EventLog: log, PlacementProvider: provider, PlacementScope: "test", + ExecutionAccess: provider, SharedEngineRoot: "/native", + }) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + defer allowPlanDrain() + ack, err := svc.ResolvePlanAsk(ctx, parked.ID, parked.RunID(), "plan-ask", session.VerdictAllowOnce) + if outcome != "allow" { + wantErr, wantReleases := acquireErr, int64(0) + if outcome == "cancel before acceptance" { + wantErr, wantReleases = context.Canceled, 1 + } + if !errors.Is(err, wantErr) || provider.acquires.Load() != 1 || provider.releases.Load() != wantReleases || provider.held.Load() || llm.Calls() != 0 { + t.Fatalf("failed admission: err=%v acquires=%d releases=%d held=%t modelCalls=%d", err, provider.acquires.Load(), provider.releases.Load(), provider.held.Load(), llm.Calls()) + } + stored, err := store.Load(t.Context(), parked.ID) + if err != nil { + t.Fatal(err) + } + ask, ok := stored.PendingAsk() + if stored.State != session.StateAwaiting || stored.RunID() != parked.RunID() || !ok || ask.AskID != "plan-ask" || ask.Origin != session.ApprovalOriginPlan { + t.Fatalf("failed admission consumed durable plan ask: state=%s run=%s ask=%+v", stored.State, stored.RunID(), ask) + } + return + } + if err != nil || ack.RunID != parked.RunID() || ack.AskID != "plan-ask" { + t.Fatalf("restored exact acceptance = %+v, %v", ack, err) + } + cancel() // Accepted work belongs to the Service, not the unary caller. + <-log.seen + if provider.acquires.Load() != 1 || provider.releases.Load() != 0 || !provider.held.Load() { + t.Fatalf("restored plan before relay drain: acquires=%d releases=%d held=%t", provider.acquires.Load(), provider.releases.Load(), provider.held.Load()) + } + <-provider.renewed + allowPlanDrain() + synctest.Wait() + if provider.acquires.Load() != 2 || provider.releases.Load() != 2 || provider.held.Load() || renewExited.Load() != 1 || llm.Calls() != 1 { + t.Fatalf("restored continuation: acquires=%d releases=%d held=%t renewExited=%d modelCalls=%d", provider.acquires.Load(), provider.releases.Load(), provider.held.Load(), renewExited.Load(), llm.Calls()) + } + stored, err := store.Load(t.Context(), parked.ID) + if err != nil { + t.Fatal(err) + } + if stored.State != session.StateCompleted || stored.RunID() == parked.RunID() || stored.Mode != session.ModeDefault { + t.Fatalf("restored continuation state=%s run=%s mode=%s", stored.State, stored.RunID(), stored.Mode) + } + }) + }) + } +} + +func TestNativeExactPlanClaimLifecycle(t *testing.T) { + for _, outcome := range []string{"allow", "deny", "owner cancel", "drain after acceptance", "release failure"} { + t.Run(outcome, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + provider := &nativeRunProvider{ref: ref, exclusive: true, renewed: make(chan struct{}, 2), renewalDeadline: time.Now().Add(time.Minute)} + var renewExited, pins, unpins atomic.Int64 + provider.renewHook = func(ctx context.Context) error { + <-ctx.Done() + renewExited.Add(1) + return ctx.Err() + } + releaseEntered, releaseAllowed := make(chan struct{}), make(chan struct{}) + allowRelease := sync.OnceFunc(func() { close(releaseAllowed) }) + provider.releaseHook = func(context.Context) error { + if provider.releases.Load() == 0 { + if renewExited.Load() != 1 || pins.Load()-unpins.Load() != 1 { + t.Errorf("old release before renewal joined or operation pin lost: renewExited=%d pins=%d unpins=%d", renewExited.Load(), pins.Load(), unpins.Load()) + } + close(releaseEntered) + <-releaseAllowed + if outcome == "release failure" { + return errors.New("release uncertain; native claim remains held") + } + } + return nil + } + store := memstore.New() + log := &nativePlanTerminalLog{EventLog: memstore.NewEventLog(), entered: make(chan struct{}), release: make(chan struct{})} + allowTerminal := sync.OnceFunc(func() { close(log.release) }) + llm := mockllm.New(mockllm.ToolCallTurn(call("c1", "PresentPlan", `{"plan":"one"}`)), mockllm.TextTurn("executed")) + cat := tool.NewCatalog() + cat.MustRegister(agent.NewPresentPlanTool()) + engine := agent.NewEngine(agent.Deps{LLM: llm, Catalog: cat, Policy: permpolicy.NewPolicy(allowRules(), nil), Model: "test-model", Interactive: true, Store: store}) + svc, err := newPlacementTestService(server.Config{ + Engine: engine, Store: store, EventLog: log, PlacementProvider: provider, PlacementScope: "test", + ExecutionAccess: provider, SharedEngineRoot: "/native", OwnershipEnforced: true, + OperationPin: func(ctx context.Context) (context.Context, func(), error) { + pins.Add(1) + return ctx, func() { unpins.Add(1) }, nil + }, + }) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + defer allowRelease() + defer allowTerminal() + owner := &session.Principal{Issuer: "test", Subject: "owner"} + ctx := session.WithPrincipal(t.Context(), owner) + sess, err := svc.CreateSession(ctx, session.ModePlan, session.Limits{MaxTurns: 5}) + if err != nil { + t.Fatal(err) + } + run, err := svc.StartInteractiveRunContentWithPlanContinuation(ctx, sess.ID, "plan this", nil) + if err != nil { + t.Fatal(err) + } + askID, _ := awaitLivePlanAsk(t, run, "c1", "one") + svc.Persist(ctx, sess.ID) + <-provider.renewed + foreign := session.WithPrincipal(t.Context(), &session.Principal{Issuer: "test", Subject: "foreign"}) + if _, err := svc.ResolvePlanAsk(foreign, sess.ID, run.RunID(), askID, session.VerdictAllowOnce); !errors.Is(err, server.ErrNotFound) { + t.Fatalf("foreign verdict = %v", err) + } + if _, err := svc.ResolvePlanAsk(ctx, sess.ID, "stale-run", askID, session.VerdictAllowOnce); !errors.Is(err, server.ErrStaleRunControl) { + t.Fatalf("stale run = %v", err) + } + if _, err := svc.ResolvePlanAsk(ctx, sess.ID, run.RunID(), "wrong-ask", session.VerdictAllowOnce); !errors.Is(err, server.ErrAskNotPending) { + t.Fatalf("wrong ask = %v", err) + } + if outcome == "owner cancel" { + if _, err := svc.CancelRun(ctx, sess.ID, run.RunID()); err != nil { + t.Fatal(err) + } + } else { + verdict := session.VerdictAllowOnce + if outcome == "deny" { + verdict = session.VerdictDeny + } + ack, err := svc.ResolvePlanAsk(ctx, sess.ID, run.RunID(), askID, verdict) + if err != nil || ack.RunID != run.RunID() || ack.AskID != askID { + t.Fatalf("exact owner acceptance = %+v, %v", ack, err) + } + } + if provider.releases.Load() != 0 || !provider.held.Load() { + t.Fatal("planning claim released before relay drain") + } + for range run.Events() { + } + svc.Persist(ctx, sess.ID) + if outcome == "allow" || outcome == "release failure" { + if _, err := svc.StartRunContent(ctx, sess.ID, "competing prompt", nil); !errors.Is(err, server.ErrFailedPrecondition) { + t.Fatalf("competing prompt = %v", err) + } + if _, err := svc.RetryFailedRun(ctx, sess.ID); !errors.Is(err, server.ErrFailedPrecondition) || errors.Is(err, server.ErrFailedStepRetryIneligible) { + t.Fatalf("competing retry bypassed reservation: %v", err) + } + } + if outcome == "drain after acceptance" { + svc.Drain() + } + finished := make(chan struct{}) + go func() { + svc.FinishRun(sess.ID, run) + close(finished) + }() + synctest.Wait() + select { + case <-releaseEntered: + default: + t.Fatal("drained plan did not join renewal and release its native claim") + } + // Provider I/O cannot hold s.mu; the old registry entry and operation + // pin must remain until release completes, behind the entry barrier. + if !svc.IsLive(sess.ID) || provider.acquires.Load() != 1 || provider.releases.Load() != 0 { + t.Fatal("old lifecycle retired or continuation acquired during blocked release") + } + allowRelease() + <-finished + synctest.Wait() + if outcome == "allow" { + select { + case <-log.entered: + default: + t.Fatal("accepted native continuation did not reach its terminal relay") + } + if provider.acquires.Load() != 2 || provider.releases.Load() != 1 || !provider.held.Load() || unpins.Load() != 1 { + t.Fatalf("continuation before drain: acquires=%d releases=%d held=%t unpins=%d", provider.acquires.Load(), provider.releases.Load(), provider.held.Load(), unpins.Load()) + } + svc.FinishRun(sess.ID, run) // stale old relay must not retire its successor. + if !svc.IsLive(sess.ID) || provider.releases.Load() != 1 { + t.Fatal("old FinishRun removed the continuation") + } + allowTerminal() + synctest.Wait() + } + wantAcquires, wantReleases, wantCalls := int64(1), int64(1), 1 + switch outcome { + case "allow": + wantAcquires, wantReleases, wantCalls = 2, 2, 2 + case "release failure": + wantAcquires, wantReleases = 2, 0 + } + if provider.acquires.Load() != wantAcquires || provider.releases.Load() != wantReleases || provider.held.Load() != (outcome == "release failure") || llm.Calls() != wantCalls || pins.Load() != unpins.Load() { + t.Fatalf("settled: acquires=%d releases=%d held=%t modelCalls=%d pins=%d unpins=%d", provider.acquires.Load(), provider.releases.Load(), provider.held.Load(), llm.Calls(), pins.Load(), unpins.Load()) + } + failed := false + for ev, err := range log.Read(ctx, sess.ID) { + if err != nil { + t.Fatal(err) + } + failed = failed || ev.Type == session.EvPlanContinuationFailed + } + if failed != (outcome == "release failure" || outcome == "drain after acceptance") { + t.Fatalf("continuation failure recorded = %t for %s", failed, outcome) + } + }) + }) + } +} diff --git a/internal/adapter/server/placement.go b/internal/adapter/server/placement.go index acfbf3faf9..222ce79745 100644 --- a/internal/adapter/server/placement.go +++ b/internal/adapter/server/placement.go @@ -1,3 +1,4 @@ +//nolint:revive // Host-only lifecycle seams are exported solely for internal adapter composition. package server import ( @@ -6,6 +7,7 @@ import ( "fmt" "strings" "sync" + "time" "unicode" "unicode/utf8" @@ -137,6 +139,9 @@ type PlacementBindRequest struct { Principal *session.Principal Scope PlacementScope Operation PlacementOperation + // BindingID is the final server-minted session identity. Providers that + // allocate durable environments use it as their idempotency/reference key. + BindingID session.SessionID } // PlacementMetadata is the bounded, display-safe provider projection returned @@ -160,6 +165,10 @@ type PlacementBinding struct { // when there is no host governance context; guest roots are never used as a // fallback. No-FS bindings always leave it empty. GovernanceRoot string + // Commit publishes a provisional durable reference after the session store + // definitively publishes the matching binding. Close aborts it only while Commit + // has not succeeded; an ambiguous commit must remain recoverable. + Commit func(context.Context) error // Close detaches a published placement binding. Service ownership is // transferred only after the session snapshot is durably created. Close func() error @@ -169,6 +178,24 @@ type PlacementBinding struct { Rollback func() error } +type ExecutionRunRequest struct { + Ref session.EnvironmentRef + Principal *session.Principal + BindingID session.SessionID + RunID string +} + +type ExecutionRunHandle interface { + Environment() tool.Environment + Renew(context.Context) error + Release(context.Context) error +} + +type ExecutionAccess interface { + Applies(session.EnvironmentRef) bool + AcquireRun(context.Context, ExecutionRunRequest) (ExecutionRunHandle, error) +} + // sourceWorkspace exposes the content-reading Workspace surface while refusing // mutations and withholding optional namespace mutation capabilities. type sourceWorkspace struct{ tool.Workspace } @@ -223,6 +250,46 @@ type PlacementProvider interface { Bind(context.Context, PlacementBindRequest) (PlacementBinding, error) } +type PlacementSuccessorRequest struct { + Ref session.EnvironmentRef + Principal *session.Principal + SourceBindingID session.SessionID + DestinationBindingID session.SessionID +} + +type ReferenceDeleteHandle interface { + Confirm(context.Context) error + Cancel(context.Context) error +} +type ReferenceLifecycle interface { + Applies(session.EnvironmentRef) bool + PrepareReferenceDelete(context.Context, PlacementSuccessorRequest) (ReferenceDeleteHandle, error) +} + +// ReferenceIntent is the owner-attested, exact provider operation retained after +// an ambiguous publication or deletion outcome. +type ReferenceIntent struct { + Ref session.EnvironmentRef + Principal *session.Principal + BindingID session.SessionID + SourceBindingID session.SessionID + OperationID string + PendingDelete bool +} + +// ReferenceIntentLifecycle is the optional host reconciliation seam. Its list is +// scoped by the provider to the authenticated mTLS client and bounded by limit. +type ReferenceIntentLifecycle interface { + ListReferenceIntents(context.Context, int) ([]ReferenceIntent, error) + CommitReferenceIntent(context.Context, ReferenceIntent) error + ConfirmReferenceIntentDelete(context.Context, ReferenceIntent) error + CancelReferenceIntentDelete(context.Context, ReferenceIntent) error +} + +type PlacementSuccessorReservoir interface { + ReserveSuccessor(context.Context, PlacementSuccessorRequest) (PlacementBinding, error) +} + // PlacementDiscoveryRequest scopes alternate-worktree discovery to an owned // source and its exact current placement. type PlacementDiscoveryRequest struct { @@ -271,6 +338,14 @@ type PlacementReattachRequest struct { Ref session.EnvironmentRef Principal *session.Principal Scope PlacementScope + // BindingID identifies the durable reference being reattached. + BindingID session.SessionID +} + +// PlacementValidator performs side-effect-free startup validation. Providers +// whose Bind allocates resources implement this seam so preflight never binds. +type PlacementValidator interface { + ValidatePlacement(context.Context) error } // PlacementBinder is the server-owned choke point around one deployment @@ -357,14 +432,14 @@ func configuredPlacementBinder(cfg Config) (*PlacementBinder, error) { return NewPlacementBinder(cfg.PlacementProvider) } -func (s *Service) bindPlacementForCreate(ctx context.Context, profile SessionProfile, owner *session.Principal) (string, *PlacementBinding, error) { +func (s *Service) bindPlacementForCreate(ctx context.Context, profile SessionProfile, owner *session.Principal, bindingID session.SessionID) (string, *PlacementBinding, error) { selector := DefaultPlacement() if profile == ProfileNoFS { selector = NoFSPlacement() } binding, err := s.placementBinder.Bind(ctx, PlacementBindRequest{ Selector: selector, Principal: owner, Scope: s.cfg.PlacementScope, - Operation: PlacementOperationCreate, + Operation: PlacementOperationCreate, BindingID: bindingID, }) if err != nil { s.logPlacementProviderError(ctx, "bind", err) @@ -390,7 +465,17 @@ func (s *Service) persistPlacedCreatedSession(ctx context.Context, sess *session if !sess.EnvironmentRef.Valid() { return nil, fmt.Errorf("%w: placement did not provide an exact environment ref", ErrInvalidPlacementBinding) } - return s.persistCreatedSession(ctx, sess, owner, request) + persisted, err := s.persistCreatedSession(ctx, sess, owner, request) + if err != nil || placement == nil || placement.Commit == nil { + return persisted, err + } + commitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + if err := placement.Commit(commitCtx); err != nil { + s.logPlacementProviderError(ctx, "commit reference", err) + return persisted, fmt.Errorf("%w: session %q persisted but reference publication must be retried", ErrInternal, sess.ID) + } + return persisted, nil } type schedulePlacement struct { @@ -543,7 +628,7 @@ func (s *Service) sessionPlacement(ctx context.Context, sess *session.Session) ( binding.Close = nil return binding, nil } - binding, err := s.ReattachPlacement(ctx, sess.EnvironmentRef) + binding, err := s.ReattachPlacementForBinding(ctx, sess.EnvironmentRef, sess.ID) if err != nil { s.placementAttachMu.Unlock() return PlacementBinding{}, err diff --git a/internal/adapter/server/placement_discovery.go b/internal/adapter/server/placement_discovery.go index 10cec5ec14..bacc71ec42 100644 --- a/internal/adapter/server/placement_discovery.go +++ b/internal/adapter/server/placement_discovery.go @@ -45,7 +45,7 @@ func (s *Service) ownedSessionBinding(ctx context.Context, id session.SessionID) if sess.EnvironmentRef.Kind == session.EnvKindNoFS { return sess, PlacementBinding{}, nil } - binding, err := s.ReattachPlacement(ctx, sess.EnvironmentRef) + binding, err := s.ReattachPlacementForBinding(ctx, sess.EnvironmentRef, sess.ID) if err != nil { return nil, PlacementBinding{}, err } diff --git a/internal/adapter/server/placement_ownership_test.go b/internal/adapter/server/placement_ownership_test.go index bddb077cc2..e05d3104cb 100644 --- a/internal/adapter/server/placement_ownership_test.go +++ b/internal/adapter/server/placement_ownership_test.go @@ -11,6 +11,7 @@ import ( "github.com/stacklok/mecatl/engine/adapter/memfs" "github.com/stacklok/mecatl/engine/adapter/memledger" "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/port" "github.com/stacklok/mecatl/engine/session" "github.com/stacklok/mecatl/engine/tool" "github.com/stacklok/mecatl/internal/adapter/mcp" @@ -23,6 +24,7 @@ type ownershipPlacementProvider struct { detaches atomic.Int32 rollbacks atomic.Int32 rollbackErr error + afterBind func(context.Context, PlacementBindRequest) } func (p *ownershipPlacementProvider) binding(reattach bool) PlacementBinding { @@ -40,8 +42,12 @@ func (p *ownershipPlacementProvider) binding(reattach bool) PlacementBinding { } } -func (p *ownershipPlacementProvider) Bind(context.Context, PlacementBindRequest) (PlacementBinding, error) { - return p.binding(false), nil +func (p *ownershipPlacementProvider) Bind(ctx context.Context, req PlacementBindRequest) (PlacementBinding, error) { + binding := p.binding(false) + if p.afterBind != nil { + p.afterBind(ctx, req) + } + return binding, nil } func (p *ownershipPlacementProvider) Reattach(_ context.Context, req PlacementReattachRequest) (PlacementBinding, error) { @@ -210,23 +216,106 @@ func TestCapacityFailurePrecedesPlacementProvisioning(t *testing.T) { } func TestCollisionAfterProvisioningRollsBackUnpublishedPlacement(t *testing.T) { - ref := session.EnvironmentRef{Kind: "microvm", ID: "logical.environment", Revision: "1"} - provider := &ownershipPlacementProvider{ref: ref} + for _, perSession := range []bool{false, true} { + name := "shared engine" + if perSession { + name = "per-session engine" + } + t.Run(name, func(t *testing.T) { + ref := session.EnvironmentRef{Kind: "microvm", ID: "candidate", Revision: "1"} + provider := &ownershipPlacementProvider{ref: ref} + winnerRef := ref + winnerRef.ID = "winner" + winnerProvider := &ownershipPlacementProvider{ref: winnerRef} + store := memstore.New() + cfg := Config{Engine: repairEngine(), Store: store, PlacementProvider: winnerProvider, PlacementScope: "test", SharedEngineRoot: "/owned", OwnershipEnforced: true} + sel := ProviderSelector{} + if perSession { + sel.ProviderID = "test" + cfg.SessionEngine = func(context.Context, ProviderSelector, []mcp.ServerConfig, SessionProfile, string, session.PermissionMode) (SessionEngineResult, error) { + return SessionEngineResult{Engine: repairEngine()}, nil + } + } + winnerSvc, err := NewService(cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(winnerSvc.Close) + cfg.PlacementProvider = provider + svc, err := NewService(cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(svc.Close) + owner := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + ctx := session.WithPrincipal(t.Context(), owner) + var winner *session.Session + provider.afterBind = func(ctx context.Context, req PlacementBindRequest) { + if provider.binds.Load() != 1 || req.BindingID != "collision" || !req.Principal.SameIdentity(owner) { + t.Fatal("candidate was not provisioned for the exact binding and owner") + } + if _, err := store.Load(ctx, req.BindingID); !errors.Is(err, port.ErrSessionNotFound) { + t.Fatalf("winner published before candidate provisioning: %v", err) + } + // Publish the competing placement only after the candidate's absence + // probe and provisioning, but before its atomic Store.Create. + var err error + winner, err = winnerSvc.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, sel, ProfileDefault, WithSessionID(req.BindingID)) + if err != nil { + t.Fatal(err) + } + } + created, err := svc.CreateSessionWithProfile(ctx, session.ModeDefault, session.Limits{}, sel, ProfileDefault, WithSessionID("collision")) + if !errors.Is(err, ErrInvalidArgument) || created != nil { + t.Fatalf("colliding create = %v, %v; want exact-ref mismatch", created, err) + } + svc.Close() + if provider.binds.Load() != 1 || provider.rollbacks.Load() != 1 || provider.detaches.Load() != 0 { + t.Fatalf("candidate binds=%d rollbacks=%d closes=%d, want 1/1/0", provider.binds.Load(), provider.rollbacks.Load(), provider.detaches.Load()) + } + stored, err := store.Load(ctx, "collision") + if err != nil || winner == nil || stored.Incarnation() != winner.Incarnation() || stored.EnvironmentRef != winnerRef || !stored.Owner.SameIdentity(owner) || stored.Mode != winner.Mode || stored.Limits != winner.Limits || stored.ProviderID != winner.ProviderID { + t.Fatalf("published winner changed: %v, %v", stored, err) + } + if winnerProvider.rollbacks.Load() != 0 || winnerProvider.detaches.Load() != 0 { + t.Fatal("loser cleanup touched the published winner's placement") + } + winnerSvc.CloseSession(winner.ID) + if winnerProvider.detaches.Load() != 1 || winnerProvider.rollbacks.Load() != 0 { + t.Fatal("winner did not retain its own close-only placement lifetime") + } + }) + } +} + +func TestKnownExplicitIDRetryDoesNotProvisionOrRollbackPlacement(t *testing.T) { + ref := session.EnvironmentRef{Kind: "microvm", ID: "winner", Revision: "1"} + winnerProvider := &ownershipPlacementProvider{ref: ref} store := memstore.New() - existing := session.New("collision", session.ModeDefault, ref, session.Limits{}, time.Unix(1, 0)) - if err := store.Save(t.Context(), existing); err != nil { + winnerSvc := newOwnershipService(t, store, winnerProvider) + t.Cleanup(winnerSvc.Close) + owner := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + ctx := session.WithPrincipal(t.Context(), owner) + winner, err := winnerSvc.CreateSession(ctx, session.ModeDefault, session.Limits{}) + if err != nil { t.Fatal(err) } - svc, err := NewService(Config{Engine: repairEngine(), Store: store, PlacementProvider: provider, PlacementScope: "test", SharedEngineRoot: "/owned"}) + retryProvider := &ownershipPlacementProvider{ref: ref} + retrySvc, err := NewService(Config{Engine: repairEngine(), Store: store, PlacementProvider: retryProvider, PlacementScope: "test", SharedEngineRoot: "/owned", OwnershipEnforced: true}) if err != nil { t.Fatal(err) } - _, err = svc.CreateSessionWithProfile(t.Context(), session.ModeDefault, session.Limits{}, ProviderSelector{}, ProfileDefault, WithSessionID("collision")) - if err == nil { - t.Fatal("colliding create succeeded") + t.Cleanup(retrySvc.Close) + retried, err := retrySvc.CreateSessionWithProfile(ctx, winner.Mode, winner.Limits, ProviderSelector{}, ProfileDefault, WithSessionID(winner.ID)) + if err != nil || retried == nil || retried.Incarnation() != winner.Incarnation() || retried.EnvironmentRef != ref { + t.Fatalf("known retry = %v, %v; want persisted winner", retried, err) } - if got := provider.rollbacks.Load(); got != 1 { - t.Fatalf("rollbacks = %d, want 1", got) + retrySvc.Close() + if retryProvider.binds.Load() != 0 || retryProvider.rollbacks.Load() != 0 || retryProvider.reattaches.Load() != 1 || retryProvider.detaches.Load() != 1 { + t.Fatalf("retry binds=%d rollbacks=%d reattaches=%d closes=%d, want 0/0/1/1", retryProvider.binds.Load(), retryProvider.rollbacks.Load(), retryProvider.reattaches.Load(), retryProvider.detaches.Load()) + } + if winnerProvider.detaches.Load() != 0 || winnerProvider.rollbacks.Load() != 0 { + t.Fatal("retry released the original winner's placement ownership") } } diff --git a/internal/adapter/server/placement_successor.go b/internal/adapter/server/placement_successor.go index 21d05c4fec..e553cfcac7 100644 --- a/internal/adapter/server/placement_successor.go +++ b/internal/adapter/server/placement_successor.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "time" "github.com/stacklok/mecatl/engine/port" "github.com/stacklok/mecatl/engine/session" @@ -57,7 +58,7 @@ func (s *Service) ClearSessionSuccessor(ctx context.Context, source session.Sess // non-destructive gate. Exact inherited placement stays on the lease-protected // path below because reattachment may itself wait for lease loss/cancellation. if placement.Selector != "" { - preflight, placementErr := s.successorPlacement(ctx, lockedSource, placement) + preflight, placementErr := s.successorPlacement(ctx, lockedSource, placement, "") if placementErr != nil { return "", placementErr } @@ -158,11 +159,17 @@ func (s *Service) createPlacedSuccessorLocked(ctx context.Context, req ForkSucce if err != nil || absent { return "", err } + destinationID := s.cfg.NewID() + releaseDestination, reserveErr := s.reserveGeneratedSessionID(mutationCtx, destinationID) + if reserveErr != nil { + return "", reserveErr + } + defer releaseDestination() selector, err := successorProviderSelector(source, req) if err != nil { return "", err } - binding, err := s.successorPlacement(mutationCtx, source, req.Placement) + binding, err := s.successorPlacement(mutationCtx, source, req.Placement, destinationID) if err != nil { return "", err } @@ -184,7 +191,7 @@ func (s *Service) createPlacedSuccessorLocked(ctx context.Context, req ForkSucce } } - created := session.New(s.cfg.NewID(), source.Mode, binding.Ref, source.Limits, s.cfg.Now()) + created := session.New(destinationID, source.Mode, binding.Ref, source.Limits, s.cfg.Now()) created.Placement = canonicalPlacementMetadata(binding) authority, bound := source.BoundAuthority() if !bound { @@ -273,6 +280,16 @@ func (s *Service) createPlacedSuccessorLocked(ctx context.Context, req ForkSucce s.logDiscoveryError(ctx, "persist successor placement", err) return "", fmt.Errorf("%w: placement storage failed", ErrInternal) } + if binding.Commit != nil { + commitCtx, cancelCommit := context.WithTimeout(context.WithoutCancel(mutationCtx), 10*time.Second) + commitErr := binding.Commit(commitCtx) + cancelCommit() + if commitErr != nil { + publishedPlacement = true + s.logPlacementProviderError(ctx, "commit successor reference", commitErr) + return created.ID, fmt.Errorf("%w: session %q persisted but reference publication must be retried", ErrInternal, created.ID) + } + } s.installSessionPlacement(created.ID, binding, false) publishedPlacement = true if broker != nil { @@ -305,8 +322,15 @@ func successorProviderSelector(source *session.Session, req ForkSuccessorRequest return selector, nil } -func (s *Service) successorPlacement(ctx context.Context, source *session.Session, requested SuccessorPlacement) (PlacementBinding, error) { +func (s *Service) successorPlacement(ctx context.Context, source *session.Session, requested SuccessorPlacement, destination session.SessionID) (PlacementBinding, error) { if requested.Selector == "" { + if destination != "" && s.cfg.ExecutionAccess != nil && s.cfg.ExecutionAccess.Applies(source.EnvironmentRef) { + reservoir, ok := s.cfg.PlacementProvider.(PlacementSuccessorReservoir) + if !ok { + return PlacementBinding{}, ErrPlacementUnavailable + } + return reservoir.ReserveSuccessor(ctx, PlacementSuccessorRequest{Ref: source.EnvironmentRef, Principal: source.Owner, SourceBindingID: source.ID, DestinationBindingID: destination}) + } binding, release, err := s.borrowSessionPlacement(ctx, source) if err != nil { return PlacementBinding{}, err diff --git a/internal/adapter/server/reference_intent_reconcile_test.go b/internal/adapter/server/reference_intent_reconcile_test.go new file mode 100644 index 0000000000..fd0d694113 --- /dev/null +++ b/internal/adapter/server/reference_intent_reconcile_test.go @@ -0,0 +1,86 @@ +package server_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/adapter/memstore" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/adapter/permpolicy" + "github.com/stacklok/mecatl/engine/agent" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" +) + +type intentLifecycle struct { + intents []server.ReferenceIntent + mu sync.Mutex + committed, confirmed, reset []session.SessionID +} + +func (l *intentLifecycle) ListReferenceIntents(context.Context, int) ([]server.ReferenceIntent, error) { + return append([]server.ReferenceIntent(nil), l.intents...), nil +} +func (l *intentLifecycle) CommitReferenceIntent(_ context.Context, i server.ReferenceIntent) error { + l.mu.Lock() + defer l.mu.Unlock() + l.committed = append(l.committed, i.BindingID) + return nil +} +func (l *intentLifecycle) ConfirmReferenceIntentDelete(_ context.Context, i server.ReferenceIntent) error { + l.mu.Lock() + defer l.mu.Unlock() + l.confirmed = append(l.confirmed, i.BindingID) + return nil +} +func (l *intentLifecycle) CancelReferenceIntentDelete(_ context.Context, i server.ReferenceIntent) error { + l.mu.Lock() + defer l.mu.Unlock() + l.reset = append(l.reset, i.BindingID) + return nil +} + +func TestReferenceIntentReconciliationIsOwnerAndRefExact(t *testing.T) { + owner := &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser} + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env", Revision: "rev"} + otherRef := session.EnvironmentRef{Kind: ref.Kind, ID: ref.ID, Revision: "new-revision"} + store := memstore.New() + persist := func(id session.SessionID, placement session.EnvironmentRef) { + sess := session.New(id, session.ModeDefault, placement, session.Limits{MaxTurns: 1}, time.Unix(1, 0)) + if err := sess.RestoreLabels(owner, session.Authority{}); err != nil { + t.Fatal(err) + } + if err := store.Save(t.Context(), sess); err != nil { + t.Fatal(err) + } + } + persist("create-match", ref) + persist("delete-restored", ref) + persist("reused-id", otherRef) + lifecycle := &intentLifecycle{intents: []server.ReferenceIntent{ + {Ref: ref, Principal: owner, BindingID: "create-match", OperationID: "create-op"}, + {Ref: ref, Principal: owner, BindingID: "delete-gone", OperationID: "delete-op", PendingDelete: true}, + {Ref: ref, Principal: owner, BindingID: "delete-restored", OperationID: "delete-restore-op", PendingDelete: true}, + {Ref: ref, Principal: owner, BindingID: "reused-id", OperationID: "old-delete-op", PendingDelete: true}, + }} + engine := agent.NewEngine(agent.Deps{LLM: mockllm.New(), Catalog: tool.NewCatalog(), Policy: permpolicy.NewPolicy(nil, nil)}) + svc, err := newPlacementTestService(server.Config{Engine: engine, Store: store, PlacementProvider: resolverPlacementProvider{}, PlacementScope: "test", ReferenceIntents: lifecycle}) + if err != nil { + t.Fatal(err) + } + defer svc.Close() + + svc.ReconcileReferenceIntents(t.Context()) + if len(lifecycle.committed) != 1 || lifecycle.committed[0] != "create-match" { + t.Fatalf("committed=%v", lifecycle.committed) + } + if len(lifecycle.confirmed) != 1 || lifecycle.confirmed[0] != "delete-gone" { + t.Fatalf("confirmed=%v", lifecycle.confirmed) + } + if len(lifecycle.reset) != 0 { + t.Fatalf("ambiguous present delete was cancelled=%v", lifecycle.reset) + } +} diff --git a/internal/adapter/server/sdk_typescript_release_test.go b/internal/adapter/server/sdk_typescript_release_test.go index 01f2e17b4c..86ab8cdeb4 100644 --- a/internal/adapter/server/sdk_typescript_release_test.go +++ b/internal/adapter/server/sdk_typescript_release_test.go @@ -235,6 +235,7 @@ func TestSDKTypescriptRelease_Scenario1_PublicServiceProjectionParity(t *testing "LostOwnershipCandidates", "MaintenanceMutationAvailable", "ManualDreamCapabilities", + "ReconcileReferenceIntents", "MaybeAutoApprovePlan", "OwnershipEnforced", "Persist", @@ -242,6 +243,7 @@ func TestSDKTypescriptRelease_Scenario1_PublicServiceProjectionParity(t *testing "ProviderStatuses", "PublishSessionEvent", "ReattachPlacement", + "ReattachPlacementForBinding", "ReattachPlacementInScope", "ReconcileLeaseLossTombstone", "RecoverNotice", diff --git a/internal/adapter/server/service.go b/internal/adapter/server/service.go index 6954d0d92d..103bfdf820 100644 --- a/internal/adapter/server/service.go +++ b/internal/adapter/server/service.go @@ -33,6 +33,7 @@ import ( "github.com/stacklok/mecatl/internal/adapter/memory" "github.com/stacklok/mecatl/internal/adapter/scheduler" "github.com/stacklok/mecatl/internal/adapter/skills" + "github.com/stacklok/mecatl/internal/creatediag" brokercontract "github.com/stacklok/mecatl/internal/mcpbroker" ) @@ -353,6 +354,12 @@ type Config struct { // PlacementScope is the trusted deployment scope supplied to every provider // Bind. It must be non-empty when PlacementProvider is configured. PlacementScope PlacementScope + // ExecutionAccess is an optional host-only run-ownership seam. Local and no-FS + // placements leave it nil and perform no lifecycle RPCs. + ExecutionAccess ExecutionAccess + // ReferenceIntents enables owner-safe reconciliation of provider-retained + // publication/deletion outcomes. Nil is fully inert. + ReferenceIntents ReferenceIntentLifecycle // SessionReadLedger optionally selects a durable read-before-write ledger for // each session independently of its placement's content backend. The returned // handle must be non-nil; nil fails the run closed. @@ -935,6 +942,8 @@ type Service struct { // refresh — background or on-demand — re-projects it). Never nil after // NewService. providerStatus atomic.Pointer[[]*mecatlv1.ProviderStatus] + // referenceIntentWarned bounds diagnostics for retained ambiguous intents. + referenceIntentWarned atomic.Bool // modelsRefresher is the OPTIONAL composition-supplied closure ListModels // calls before returning its snapshot (issue #262, R1.4: a proxy started @@ -1341,8 +1350,14 @@ type runState struct { runContextStop context.CancelFunc // operationRelease drains the immutable direct-MCP runtime pin acquired at // run admission. It is independent from engine cache lifetime. - operationRelease func() - awaiting atomic.Bool + operationRelease func() + execution ExecutionRunHandle + executionMu sync.Mutex + executionClosed bool + executionRenewCancel context.CancelFunc + executionRenewDone chan struct{} + executionTeardown sync.Once + awaiting atomic.Bool // titleRevision is the last title metadata revision successfully persisted and // published for this run. It starts from the admitted durable snapshot so // prompt-ingress changes publish only after their save succeeds. @@ -1354,14 +1369,19 @@ type runState struct { // engine so a delayed relay never snapshots a concurrently-resuming session. // Backend calls admitted before invalidation may still complete. persistMu sync.Mutex - // These fields are guarded by persistMu. The resolution fields close the - // delivery race when a control reaches a run before its relay persists the ask. - // exactApprovalContexts carries only accepted exact verdicts to their - // matching appends; in-stream resolutions never add an entry. + // These fields are guarded by persistMu. They close the delivery race when a + // control reaches a detached/background run after the engine emitted an ask but + // before its relay starts Persist. exactApprovalContexts carries only accepted + // exact verdicts to their matching appends; in-stream resolutions never add one. resolvedAskID string + awaitingAskID string exactApprovalContexts map[string]context.Context acceptedApproval *agent.ApprovalResolution cancelSignaled bool + // terminalEventAdmitted prevents drain from preserving an old awaiting + // snapshot while a terminal append is in flight, even if that append fails. + // Guarded by persistMu; it is not an owner-cancellation signal. + terminalEventAdmitted bool // planContinuation reserves one detached relay before an exact plan allow is // consumed. The terminal relay transfers this ownership to the proceed run. // Guarded by Service.mu. @@ -1370,8 +1390,11 @@ type runState struct { // exact plan verdicts and excludes in-stream plan verdicts on that run. serverOwnedPlanContinuation bool // preserveDurable prevents a shutdown-cancelled local awaiting run from - // overwriting the already-durable PendingAsk handoff point. + // overwriting the already-durable PendingAsk handoff point. preservedAskID is + // guarded by persistMu and identifies the matching retraction that belongs to + // that process-local shutdown. preserveDurable atomic.Bool + preservedAskID string settled chan struct{} settledOnce sync.Once // removeCapabilityOnSettle retains denial when normal teardown releases a @@ -1622,11 +1645,21 @@ func (s *Service) BindPlacement(ctx context.Context, selector PlacementSelector, // ReattachPlacement authorizes and resolves the exact persisted environment // identity. It never invokes Bind and therefore cannot follow a changed default. func (s *Service) ReattachPlacement(ctx context.Context, ref session.EnvironmentRef) (PlacementBinding, error) { + return s.reattachPlacement(ctx, ref, "") +} + +// ReattachPlacementForBinding supplies the durable session reference identity +// required by allocating remote providers. +func (s *Service) ReattachPlacementForBinding(ctx context.Context, ref session.EnvironmentRef, bindingID session.SessionID) (PlacementBinding, error) { + return s.reattachPlacement(ctx, ref, bindingID) +} + +func (s *Service) reattachPlacement(ctx context.Context, ref session.EnvironmentRef, bindingID session.SessionID) (PlacementBinding, error) { if s == nil || s.placementBinder == nil { return PlacementBinding{}, fmt.Errorf("%w: no PlacementProvider is configured", ErrFailedPrecondition) } binding, err := s.placementBinder.Reattach(ctx, PlacementReattachRequest{ - Ref: ref, Principal: session.PrincipalFromContext(ctx), Scope: s.cfg.PlacementScope, + Ref: ref, Principal: session.PrincipalFromContext(ctx), Scope: s.cfg.PlacementScope, BindingID: bindingID, }) if err != nil { s.logPlacementProviderError(ctx, "reattach", err) @@ -2160,53 +2193,84 @@ func (s *Service) classifyCreateWinner(existing *session.Session, owner *session return existing, nil } -// reserveSessionID validates a caller-chosen session id (WithSessionID, ADR 0059 -// decision #7 Phase-2) against THREE collision sources and reserves it for the -// duration of the create, returning a release func the caller MUST defer: -// -// 1. a LIVE per-session engine (sessionEngines — a collision would shadow an -// in-flight session); -// 2. an in-flight create holding the id (reservedIDs — closes the old TOCTOU: -// the prior check released s.mu before the slow factory call and the separate -// registration, so two concurrent creates on the same id both passed); -// 3. a PERSISTED session already in the store (a completed prior create is NOT -// in sessionEngines — e.g. the shared-engine fast path never registers there). -// -// The in-memory reservation (1)+(2) is taken under a single s.mu hold; the store -// probe (3) runs after (no I/O under the mutex). On a collision or an infra probe -// fault the reservation is released before returning the error. Once the session -// is registered (per-session) or persisted (shared) the durable collision sources -// take over, so the reservation only needs to live for the create. -func (s *Service) reserveSessionID(ctx context.Context, id session.SessionID, owner *session.Principal, request createRequest) (existing *session.Session, release func(), err error) { +func (s *Service) classifyExistingCreateRetry(ctx context.Context, existing *session.Session, owner *session.Principal, mode session.PermissionMode, limits session.Limits, sel ProviderSelector, profile SessionProfile, opts createSessionOpts) (*session.Session, error) { + request := newCreateRequest(existing.EnvironmentRef, mode, limits, sel, profile, opts.sourceSessionID, opts) + winner, err := s.classifyCreateWinner(existing, owner, request) + if err != nil { + return nil, err + } + binding, err := s.placementBinder.Reattach(ctx, PlacementReattachRequest{ + Ref: existing.EnvironmentRef, Principal: owner, Scope: s.cfg.PlacementScope, BindingID: existing.ID, + }) + if err != nil { + s.logPlacementProviderError(ctx, "reattach explicit session retry", err) + return nil, err + } + if binding.Close != nil { + defer func() { _ = binding.Close() }() + } + workspace := binding.Environment.Workspace().Root() + if profile == ProfileNoFS && workspace != "" || profile != ProfileNoFS && workspace == "" { + return nil, ErrInvalidPlacementBinding + } + return winner, nil +} + +func (s *Service) reserveCreateID(ctx context.Context, id session.SessionID, owner *session.Principal) (*session.Session, func(), error) { s.mu.Lock() _, liveEngine := s.sessionEngines[id] _, reserved := s.reservedIDs[id] if liveEngine || reserved { s.mu.Unlock() - // Accurate for BOTH cases: a live per-session engine (liveEngine) OR a - // concurrent in-flight create holding the id (reserved). return nil, nil, fmt.Errorf("%w: session id %q is already in use", ErrInvalidArgument, id) } s.reservedIDs[id] = struct{}{} s.mu.Unlock() - release = func() { + release := func() { s.mu.Lock() delete(s.reservedIDs, id) s.mu.Unlock() } - // Probe the store for a persisted session under this id. A not-found error - // means the id is clear; any other infrastructure fault fails closed and is - // exposed only through a content-free public category. - if existing, lerr := s.cfg.Store.Load(ctx, id); lerr == nil && existing != nil { - release() - winner, classifyErr := s.classifyCreateWinner(existing, owner, request) - return winner, nil, classifyErr - } else if lerr != nil && !errors.Is(lerr, port.ErrSessionNotFound) { + existing, err := s.cfg.Store.Load(ctx, id) + if errors.Is(err, port.ErrSessionNotFound) { + return nil, release, nil + } + if err != nil { release() - s.logDiscoveryError(ctx, "probe session placement", lerr) return nil, nil, fmt.Errorf("%w: placement storage failed", ErrInternal) } - return nil, release, nil + sameOwner := existing != nil && (existing.Owner == nil && owner == nil || existing.Owner.SameIdentity(owner)) + if !sameOwner { + release() + return nil, nil, fmt.Errorf("%w: %q", ErrNotFound, id) + } + return existing, release, nil +} + +func (s *Service) reserveGeneratedSessionID(ctx context.Context, id session.SessionID) (func(), error) { + s.mu.Lock() + _, liveEngine := s.sessionEngines[id] + _, reserved := s.reservedIDs[id] + if liveEngine || reserved { + s.mu.Unlock() + return nil, fmt.Errorf("%w: session id %q is already in use", ErrInvalidArgument, id) + } + s.reservedIDs[id] = struct{}{} + s.mu.Unlock() + release := func() { + s.mu.Lock() + delete(s.reservedIDs, id) + s.mu.Unlock() + } + if existing, err := s.cfg.Store.Load(ctx, id); err == nil && existing != nil { + release() + return nil, fmt.Errorf("%w: generated session id %q already exists: %w", ErrInvalidArgument, id, port.ErrSessionAlreadyExists) + } else if err != nil && !errors.Is(err, port.ErrSessionNotFound) { + release() + s.logDiscoveryError(ctx, "probe generated session placement", err) + return nil, fmt.Errorf("%w: placement storage failed", ErrInternal) + } + return release, nil } func (s *Service) persistNewSession(ctx context.Context, sess *session.Session) error { @@ -2221,7 +2285,10 @@ func (s *Service) persistNewSession(ctx context.Context, sess *session.Session) } func (s *Service) persistCreatedSession(ctx context.Context, sess *session.Session, owner *session.Principal, request *createRequest) (*session.Session, error) { - if err := s.persistNewSession(ctx, sess); err != nil { + persistDone := creatediag.Begin(ctx, "session_persist") + persistErr := s.persistNewSession(ctx, sess) + persistDone(persistErr) + if err := persistErr; err != nil { if existing, ok, collisionErr := s.resolveCreateCollision(ctx, sess.ID, owner, request, err); ok || collisionErr != nil { return existing, collisionErr } @@ -2342,6 +2409,51 @@ func (s *Service) createSession(ctx context.Context, mode session.PermissionMode // The owner stamped on the new session: the explicit WithOwner injection, else // the verified principal on the context, else nil (the ownerless no-auth path). owner := resolveOwner(ctx, opts) + // Durable placement allocation is keyed by the final session id. Mint it once + // before Bind; the id is never replaced after a remote allocation succeeds. + finalID := opts.id + generatedID := !opts.idSet + if opts.idSet && finalID == "" { + return nil, fmt.Errorf("%w: session id must not be empty", ErrInvalidArgument) + } + if !opts.idSet { + finalID = s.cfg.NewID() + } + creatediag.Session(ctx, string(finalID)) + probeDone := creatediag.Begin(ctx, "session_id_probe") + var existingCreate *session.Session + var releaseCreate func() + if generatedID { + releaseCreate, err = s.reserveGeneratedSessionID(ctx, finalID) + } else { + existingCreate, releaseCreate, err = s.reserveCreateID(ctx, finalID, owner) + } + probeDone(err) + if err != nil { + return nil, err + } + defer releaseCreate() + if existingCreate != nil { + if opts.placement != nil { + if opts.placement.Close != nil { + defer func() { _ = opts.placement.Close() }() + } + if err := validatePlacementBinding(*opts.placement); err != nil { + return nil, err + } + if opts.placement.Ref != existingCreate.EnvironmentRef { + return nil, ErrInvalidPlacementBinding + } + workspace := opts.placement.Environment.Workspace().Root() + if profile == ProfileNoFS && workspace != "" || profile != ProfileNoFS && workspace == "" { + return nil, ErrInvalidPlacementBinding + } + } + if err := s.bindRelatedIncarnations(ctx, &opts); err != nil { + return nil, err + } + return s.classifyExistingCreateRetry(ctx, existingCreate, owner, mode, limits, sel, profile, opts) + } var placement *PlacementBinding if opts.placement != nil { if err := validatePlacementBinding(*opts.placement); err != nil { @@ -2357,7 +2469,7 @@ func (s *Service) createSession(ctx context.Context, mode session.PermissionMode return nil, ErrInvalidPlacementBinding } } else { - workspace, placement, err = s.bindPlacementForCreate(ctx, profile, owner) + workspace, placement, err = s.bindPlacementForCreate(ctx, profile, owner, finalID) if err != nil { return nil, err } @@ -2375,56 +2487,14 @@ func (s *Service) createSession(ctx context.Context, mode session.PermissionMode return nil, err } - // Resolve the session id: the caller's override (WithSessionID, ADR 0059 - // decision #7 Phase-2) wins; otherwise the Service's NewID generator mints a - // fresh one (the byte-identical pre-Phase-2 path). WithSessionID with an EMPTY - // id is rejected (the doc promises it), distinguished from "never called" by - // idSet. A caller-chosen id is validated + reserved by reserveSessionID (see - // its doc for the three collision sources); the reservation is released on - // EVERY exit path. + // Reserve the final id for the rest of creation. Placement allocation above is + // idempotent on this same key, so a retry cannot allocate a second environment. + request := newCreateRequest(placement.Ref, mode, limits, sel, profile, opts.sourceSessionID, opts) var retryRequest *createRequest - mintID := s.cfg.NewID - if opts.idSet { - if opts.id == "" { - return nil, fmt.Errorf("%w: session id must not be empty", ErrInvalidArgument) - } - request := newCreateRequest(placement.Ref, mode, limits, sel, profile, opts.sourceSessionID, opts) - retryRequest = &request - existing, release, err := s.reserveSessionID(ctx, opts.id, owner, request) - if err != nil { - return nil, err - } - if existing != nil { - return existing, nil - } - defer release() - mintID = func() session.SessionID { return opts.id } - } - // A broker attachment is keyed by the canonical persisted identity. Mint and - // reserve generated IDs before any attachment or catalogue construction. - if (s.cfg.MCPBroker != nil || s.cfg.SessionContextEngine != nil) && !opts.idSet { - id := mintID() - request := newCreateRequest(placement.Ref, mode, limits, sel, profile, opts.sourceSessionID, opts) - // Populate the outer retryRequest too (not just the local var used for - // reserveSessionID above): persistCreatedSession's collision-retry path - // (resolveCreateCollision) needs a non-nil *createRequest to classify an - // idempotent-retry winner on this generated-id branch, exactly as the - // opts.idSet branch above already does. Before this fix, retryRequest - // stayed nil here (the "request" identifier above is a fresh local, not - // the outer var), so resolveCreateCollision's request==nil guard always - // short-circuited and a genuine ErrSessionAlreadyExists from persistNewSession - // always hard-failed instead of resolving to the existing winner. + if !generatedID || s.cfg.MCPBroker != nil || s.cfg.SessionContextEngine != nil { retryRequest = &request - existing, release, reserveErr := s.reserveSessionID(ctx, id, owner, request) - if reserveErr != nil { - return nil, reserveErr - } - if existing != nil { - return nil, fmt.Errorf("%w: generated session id %q already exists", ErrInvalidArgument, id) - } - defer release() - mintID = func() session.SessionID { return id } } + mintID := func() session.SessionID { return finalID } // Issue #20 (model-switch context carryover): when a source session is // named, validate it (turn-boundary) and snapshot its conversation ONCE @@ -2552,6 +2622,7 @@ func (s *Service) createPerSessionEngine(ctx context.Context, mintID func() sess } defer s.finalizeBrokerAttachment(broker, &committed) } + factoryDone := creatediag.Begin(ctx, "engine_factory") acquire := s.executionWorkspaceAcquirer(owner, placement.Ref) res, err = s.callSessionEngine(ctx, id, owner, acquire, sel, specs, profile, workspace, mode, brokerTools(broker)) // Only this reserved, unpublished create may retry a retired attempt. @@ -2566,6 +2637,7 @@ func (s *Service) createPerSessionEngine(ctx context.Context, mintID func() sess res, err = s.callSessionEngine(ctx, id, owner, acquire, sel, specs, profile, workspace, mode, brokerTools(broker)) } } + factoryDone(err) } if err != nil { // Factory maps an unknown/unavailable provider to ErrInvalidArgument; any @@ -2663,11 +2735,29 @@ func (s *Service) createPerSessionEngine(ctx context.Context, mintID func() sess if closeFn != nil { _ = closeFn() } + if perr == nil && persisted != nil && placement != nil && placement.Commit != nil { + commitCtx, cancelCommit := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + commitErr := placement.Commit(commitCtx) + cancelCommit() + if commitErr != nil { + s.logPlacementProviderError(ctx, "commit collided reference", commitErr) + return persisted, fmt.Errorf("%w: session %q persisted but reference publication must be retried", ErrInternal, sess.ID) + } + } return persisted, perr } s.installSessionPlacement(sess.ID, *placement, opts.placementEnvironmentOverride) *publishedPlacement = true contextPublished = true + if placement != nil && placement.Commit != nil { + commitCtx, cancelCommit := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + commitErr := placement.Commit(commitCtx) + cancelCommit() + if commitErr != nil { + s.logPlacementProviderError(ctx, "commit reference", commitErr) + return sess, fmt.Errorf("%w: session %q persisted but reference publication must be retried", ErrInternal, sess.ID) + } + } if broker != nil { commitCtx, cancelCommit := context.WithTimeout(context.WithoutCancel(ctx), engineCloseTimeout) commitErr := s.commitBrokerAttachment(commitCtx, id, broker) @@ -3166,6 +3256,9 @@ func (s *Service) Close() { run.Cancel() } else if admissionCancel != nil { admissionCancel() + // No relay owns a provisional admission. Close its execution slot here so + // an AcquireRun that returns after cancellation releases its late handle. + s.teardownExecution(rs) } } s.waitDetachedControlRelays() @@ -3346,7 +3439,11 @@ func (s *Service) snapshotDrainState() (map[session.SessionID]*runState, []sessi s.mu.Lock() current := s.runs[id] == st if current { - if st.awaiting.Load() { + // A caller control that crossed persistMu first owns the disposition. + // Preserve only an unresolved, successfully persisted handoff that + // shutdown itself is about to cancel in memory. + if st.awaiting.Load() && st.awaitingAskID != "" && !st.cancelSignaled && st.resolvedAskID == "" && !st.terminalEventAdmitted { + st.preservedAskID = st.awaitingAskID st.preserveDurable.Store(true) } st.cancelSignaled = true @@ -3525,10 +3622,101 @@ func (s *Service) StorageReady(ctx context.Context) bool { return p.Ping(ctx) == nil } +// ReconcileReferenceIntents resolves the provider's bounded, client-scoped +// ambiguous publication and deletion outcomes against exact durable sessions. +func (s *Service) ReconcileReferenceIntents(ctx context.Context) { + lifecycle := s.cfg.ReferenceIntents + if lifecycle == nil { + return + } + intents, err := lifecycle.ListReferenceIntents(ctx, 64) + if err != nil { + s.warnReferenceIntentRetention(ctx) + return + } + retained := false + for _, intent := range intents { + if intent.BindingID == "" || intent.OperationID == "" || !intent.Ref.Valid() || intent.Principal == nil { + retained = true + continue + } + sess, loadErr := s.cfg.Store.Load(ctx, intent.BindingID) + if loadErr != nil { + if errors.Is(loadErr, port.ErrSessionNotFound) && intent.PendingDelete { + if err := lifecycle.ConfirmReferenceIntentDelete(ctx, intent); err != nil { + retained = true + } + } else { + // A missing pending create may have been published by another replica + // after this read. It is never safe to abort or TTL-collect it here. + retained = true + } + continue + } + matching := sess != nil && sess.ID == intent.BindingID && sess.EnvironmentRef == intent.Ref && sess.Owner != nil && sess.Owner.SameIdentity(intent.Principal) + if !matching { + retained = true + continue + } + if intent.PendingDelete { + // Presence after an earlier ambiguous delete attempt is not proof that the + // attempt had no effect: a delayed commit may still remove this incarnation. + // Keep the exact pending intent until absence proves deletion or an explicit + // retry obtains a deterministic outcome. + retained = true + continue + } + err = lifecycle.CommitReferenceIntent(ctx, intent) + if err != nil { + retained = true + } + } + if retained { + s.warnReferenceIntentRetention(ctx) + } else { + s.referenceIntentWarned.Store(false) + } +} + +func (s *Service) warnReferenceIntentRetention(ctx context.Context) { + if s.referenceIntentWarned.CompareAndSwap(false, true) { + s.cfg.Diagnostics.Log(ctx, port.LevelWarn, "execution reference reconciliation retained ambiguous intents") + } +} + func (s *Service) saveSession(ctx context.Context, sess *session.Session) error { return s.cfg.MutationCapability.GuardStore(s.cfg.Store).Save(ctx, sess) } +func (s *Service) prepareReferenceDelete(ctx context.Context, sess *session.Session) (ReferenceDeleteHandle, error) { + if sess == nil || s.cfg.ExecutionAccess == nil || !s.cfg.ExecutionAccess.Applies(sess.EnvironmentRef) { + return nil, nil + } + lifecycle, ok := s.cfg.PlacementProvider.(ReferenceLifecycle) + if !ok { + return nil, ErrPlacementUnavailable + } + return lifecycle.PrepareReferenceDelete(ctx, PlacementSuccessorRequest{Ref: sess.EnvironmentRef, Principal: sess.Owner, SourceBindingID: sess.ID}) +} +func (s *Service) cancelReferenceDelete(ctx context.Context, handle ReferenceDeleteHandle) { + if handle == nil { + return + } + cancelCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + if err := handle.Cancel(cancelCtx); err != nil { + s.logPlacementProviderError(ctx, "cancel reference delete", err) + } +} +func (*Service) confirmReferenceDelete(ctx context.Context, handle ReferenceDeleteHandle) error { + if handle == nil { + return nil + } + confirmCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + return handle.Confirm(confirmCtx) +} + func (s *Service) deleteSessionFamily(ctx context.Context, id session.SessionID, store port.PrunableStore) error { if !s.mutationLeaseHeld(id) { return fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) @@ -3762,10 +3950,17 @@ func (s *Service) DeleteSession(ctx context.Context, id session.SessionID) error if err != nil || absent { return err } + referenceDelete, err := s.prepareReferenceDelete(ctx, sess) + if err != nil { + return err + } unlockBroker := s.brokerMu.lock(id) defer unlockBroker() if err := s.deleteSessionFamily(ctx, sess.ID, prunable); err != nil { if errors.Is(err, port.ErrSessionNotFound) { + if confirmErr := s.confirmReferenceDelete(ctx, referenceDelete); confirmErr != nil { + return fmt.Errorf("%w: confirm deleted session reference: %v", ErrInternal, confirmErr) + } // The durable record is already gone: still attempt broker cleanup // (best-effort) before reporting success, so a locally-retained // broker handle is never orphaned by an already-completed delete. @@ -3781,6 +3976,9 @@ func (s *Service) DeleteSession(ctx context.Context, id session.SessionID) error } return fmt.Errorf("%w: delete session: %v", ErrInternal, err) } + if err := s.confirmReferenceDelete(ctx, referenceDelete); err != nil { + return fmt.Errorf("%w: session deleted but reference confirmation must be retried", ErrInternal) + } // The durable record is gone; broker cleanup is now best-effort. Reordered // deliberately (I-8): deleting broker state FIRST left an unrecoverable // partial-deletion window if the durable delete then failed — the snapshot @@ -3832,6 +4030,10 @@ func (s *Service) DeleteSessionForRetentionCandidate(ctx context.Context, candid if !retentionCandidateMatches(sess, candidate) { return errRetentionCandidateChanged } + referenceDelete, err := s.prepareReferenceDelete(ctx, sess) + if err != nil { + return err + } unlockBroker := s.brokerMu.lock(candidate.ID) defer unlockBroker() if !s.mutationLeaseHeld(candidate.ID) { @@ -3845,8 +4047,12 @@ func (s *Service) DeleteSessionForRetentionCandidate(ctx context.Context, candid return fmt.Errorf("%w: delete retention candidate: %v", ErrInternal, err) } if !deleted { + s.cancelReferenceDelete(ctx, referenceDelete) return errRetentionCandidateChanged } + if err := s.confirmReferenceDelete(ctx, referenceDelete); err != nil { + return fmt.Errorf("%w: retention deleted session but reference confirmation must be retried", ErrInternal) + } // The durable record is gone; broker cleanup is now best-effort (I-8: see // deleteBrokerSessionLocked's doc comment for the ordering rationale). if err := s.deleteBrokerSessionLocked(ctx, candidate.ID); err != nil { @@ -3872,6 +4078,8 @@ func retentionCandidateMatches(sess *session.Session, candidate port.SessionDisc // chats, but it still serializes against run entry, acquires the cross-process // mutation lease, and revalidates durable taxonomy and lifecycle after acquiring // that lease. It is an internal composition callback, never a wire operation. +// +//nolint:gocyclo // Retention keeps lease, taxonomy, reference intent, and deletion in one transaction. func (s *Service) DeleteSessionForRetention(ctx context.Context, id session.SessionID) error { unlock := s.runEntryMu.lock(id) defer unlock() @@ -3902,10 +4110,17 @@ func (s *Service) DeleteSessionForRetention(ctx context.Context, id session.Sess if sess.State == session.StateRunning || sess.State == session.StateAwaiting || sess.State == session.StateAuthorizing || s.IsLive(id) { return fmt.Errorf("%w: retention candidate is active or awaiting approval", ErrFailedPrecondition) } + referenceDelete, err := s.prepareReferenceDelete(ctx, sess) + if err != nil { + return err + } unlockBroker := s.brokerMu.lock(id) defer unlockBroker() if err := s.deleteSessionFamily(ctx, id, prunable); err != nil { if errors.Is(err, port.ErrSessionNotFound) { + if confirmErr := s.confirmReferenceDelete(ctx, referenceDelete); confirmErr != nil { + return fmt.Errorf("%w: confirm deleted retention reference: %v", ErrInternal, confirmErr) + } if brokerErr := s.deleteBrokerSessionLocked(ctx, id); brokerErr != nil { s.cfg.Diagnostics.Log(context.WithoutCancel(ctx), port.LevelWarn, "broker cleanup after already-deleted retention candidate failed", "session", string(id), "err", brokerErr.Error()) @@ -3918,6 +4133,9 @@ func (s *Service) DeleteSessionForRetention(ctx context.Context, id session.Sess } return fmt.Errorf("%w: delete retention candidate: %v", ErrInternal, err) } + if err := s.confirmReferenceDelete(ctx, referenceDelete); err != nil { + return fmt.Errorf("%w: retention deleted session but reference confirmation must be retried", ErrInternal) + } // The durable record is gone; broker cleanup is now best-effort (I-8: see // deleteBrokerSessionLocked's doc comment for the ordering rationale). if err := s.deleteBrokerSessionLocked(ctx, id); err != nil { @@ -4678,7 +4896,11 @@ func (s *Service) RetryFailedRun(ctx context.Context, id session.SessionID) (*ag if !leaseHeld() { return nil, fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) } - ctx = memory.WithWorkspace(port.WithRootSessionID(ctx, sess.ID), governanceRoot) + env, err = s.acquireExecution(ctx, st, sess, sess.RunID(), env) + if err != nil { + return nil, err + } + ctx = rootedRunContext(ctx, sess, governanceRoot) run, err := s.promoteRunAdmission(id, st, stopAdmission, func() *agent.Run { return engine.RetryFailedStep(ctx, sess, env) }) @@ -4686,6 +4908,7 @@ func (s *Service) RetryFailedRun(ctx context.Context, id session.SessionID) (*ag return nil, err } promoted = true + s.startExecutionRenewal(id, st, run) return run, nil } @@ -4844,7 +5067,8 @@ func (s *Service) startRunContentLocked(ctx context.Context, id session.SessionI } // A terminal snapshot means the registered run has finished driving but its // relay has not yet removed it. Retire only that run, without displacing a - // continuation reserved by an accepted plan verdict. + // continuation reserved by an accepted plan verdict. Native execution claims + // remain registered until relay drain. if err := s.deregisterTerminalRunForEntry(id, sess, ErrFailedPrecondition); err != nil { return nil, err } @@ -4960,6 +5184,10 @@ func (s *Service) startRunContentLocked(ctx context.Context, id session.SessionI // one mint site for a new run. runID := newRunID() sess.BeginRun(runID) + env, err = s.acquireExecution(ctx, st, sess, runID, env) + if err != nil { + return nil, err + } if !leaseHeld() { return nil, fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) } @@ -4976,6 +5204,7 @@ func (s *Service) startRunContentLocked(ctx context.Context, id session.SessionI *interruptedContinuationOwned = true } promoted = true + s.startExecutionRenewal(id, st, run) return run, nil } @@ -5241,6 +5470,10 @@ func (s *Service) resolvePersistedRunAsk(ctx context.Context, id session.Session if !leaseHeld() { return RunAskAcknowledgement{}, fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) } + env, err = s.acquireExecution(requestLeaseCtx, st, sess, sess.RunID(), env) + if err != nil { + return RunAskAcknowledgement{}, err + } ownedCtx, stopOwned := s.detachedControlContext(ctx) ownedLeaseCtx, stopOwnedLease, _ := s.mutationLeaseContext(ownedCtx, id) @@ -5280,6 +5513,7 @@ func (s *Service) resolvePersistedRunAsk(ctx context.Context, id session.Session return RunAskAcknowledgement{}, err } promoted = true + s.startExecutionRenewal(id, st, run) go func() { defer s.detachedControlWG.Done() s.relayDetachedControlRun(ownedLeaseCtx, id, run) @@ -6683,7 +6917,11 @@ func (s *Service) resumeFromAwaiting(ctx context.Context, id session.SessionID, if !leaseHeld() { return nil, fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) } - ctx = memory.WithWorkspace(port.WithRootSessionID(ctx, sess.ID), governanceRoot) + env, err = s.acquireExecution(ctx, st, sess, sess.RunID(), env) + if err != nil { + return nil, err + } + ctx = rootedRunContext(ctx, sess, governanceRoot) st.persistMu.Lock() accepted := resolution st.acceptedApproval = &accepted @@ -6698,6 +6936,7 @@ func (s *Service) resumeFromAwaiting(ctx context.Context, id session.SessionID, return nil, err } promoted = true + s.startExecutionRenewal(id, st, run) return run, nil } @@ -6797,7 +7036,7 @@ func (s *Service) approvePlan(ctx context.Context, id session.SessionID, targetM var resumedStop session.StopReason if resumed != nil { resumedStop = s.forwardRunEvents(ctx, id, resumed, out) - s.deregister(id, resumed) + s.FinishRun(id, resumed) } // (5) Atomic continuation (allow paths only). A deny leaves the session // in plan mode with no continuation run — the model re-plans on the next @@ -6831,7 +7070,7 @@ func (s *Service) approvePlan(ctx context.Context, id session.SessionID, targetM return } s.forwardRunEvents(ctx, id, cont, out) - s.deregister(id, cont) + s.FinishRun(id, cont) }() return out, nil @@ -7068,6 +7307,11 @@ func (s *Service) Persist(ctx context.Context, id session.SessionID) { return } s.persistRun(ctx, id, st) + if st.awaiting.Load() { + if ask, pending := st.sess.PendingAsk(); pending { + st.awaitingAskID = ask.AskID + } + } } // persistPermissionAsk is Persist with control-event correlation. A detached @@ -7100,6 +7344,9 @@ func (s *Service) persistPermissionAsk(ctx context.Context, id session.SessionID return } s.persistRun(ctx, id, st) + if st.awaiting.Load() { + st.awaitingAskID = askID + } } func (s *Service) persistRun(ctx context.Context, id session.SessionID, st *runState) { @@ -7155,7 +7402,54 @@ func (s *Service) finishRelayRun(ctx context.Context, id session.SessionID, run return } s.completeRelay(ctx, id, run) - s.deregister(id, run) + s.FinishRun(id, run) +} + +// admitRunEvent orders the durable projection against shutdown without holding +// persistMu across EventLog I/O. An admitted terminal or matching retraction +// invalidates the old handoff before drain can freeze it, regardless of append +// success. After the freeze, omit only the shutdown-local cancellation records; +// the live wire is unchanged. +func (s *Service) admitRunEvent(id session.SessionID, ev session.Event) bool { + if ev.RunID == "" { + return true + } + s.mu.Lock() + st := s.runs[id] + s.mu.Unlock() + if st == nil { + return true + } + st.persistMu.Lock() + defer st.persistMu.Unlock() + s.mu.Lock() + current := s.runs[id] == st && st.run != nil && st.run.RunID() == ev.RunID + s.mu.Unlock() + if !current { + return true + } + if st.preserveDurable.Load() { + switch ev.Type { + case session.EvResult: + if ev.Result != nil && ev.Result.Stop == session.StopCancelled { + return false + } + case session.EvPermissionRetract: + if ev.Ask != nil && ev.Ask.AskID == st.preservedAskID { + return false + } + } + } else { + switch ev.Type { + case session.EvResult: + st.terminalEventAdmitted = true + case session.EvPermissionRetract: + if ev.Ask != nil && ev.Ask.AskID != "" && ev.Ask.AskID == st.awaitingAskID { + st.resolvedAskID = ev.Ask.AskID + } + } + } + return true } // appendEvent durably records one projected relay event to the configured @@ -7176,6 +7470,10 @@ func (s *Service) appendEvent(ctx context.Context, id session.SessionID, ev sess if s.cfg.EventLog == nil { return nil } + + if !s.admitRunEvent(id, ev) { + return nil + } if !s.mutationLeaseHeld(id) { return fmt.Errorf("%w: %q", ErrSessionLeasedElsewhere, id) } @@ -7256,6 +7554,7 @@ func (s *Service) relayEvent(ctx context.Context, id session.SessionID, ev sessi if st != nil { st.persistMu.Lock() st.awaiting.Store(false) + st.awaitingAskID = "" st.persistMu.Unlock() } } @@ -7464,7 +7763,7 @@ func (s *Service) autoApproveContinuation(ctx context.Context, id session.Sessio recorder.Observe(ev) } recorder.Close() - s.deregister(id, cont) + s.FinishRun(id, cont) } // autoApproveWaitTimeout bounds how long autoApproveContinuation waits for the @@ -8285,6 +8584,7 @@ func (s *Service) cleanupRunAdmission(id session.SessionID, st *runState, promot return } s.cancelRegisteredRunState(id, st, nil, true) + s.teardownExecution(st) s.removeRunState(id, st) } @@ -8399,6 +8699,131 @@ func (s *Service) deregister(id session.SessionID, run *agent.Run) { } } +func (s *Service) acquireExecution(ctx context.Context, st *runState, sess *session.Session, runID string, fallback tool.Environment) (tool.Environment, error) { + access := s.cfg.ExecutionAccess + if access == nil || !access.Applies(sess.EnvironmentRef) { + return fallback, nil + } + handle, err := access.AcquireRun(ctx, ExecutionRunRequest{Ref: sess.EnvironmentRef, Principal: sess.Owner, BindingID: sess.ID, RunID: runID}) + if err != nil { + return tool.Environment{}, err + } + env := handle.Environment() + if env.Ref() != sess.EnvironmentRef || env.Workspace() == nil { + _ = handle.Release(context.WithoutCancel(ctx)) + return tool.Environment{}, ErrInvalidPlacementBinding + } + st.executionMu.Lock() + if st.executionClosed { + st.executionMu.Unlock() + _ = handle.Release(context.WithoutCancel(ctx)) + return tool.Environment{}, ErrPlacementUnavailable + } + st.execution = handle + st.executionMu.Unlock() + return env, nil +} + +type executionRenewalDeadline interface { + RenewalDeadline() time.Time +} + +func executionRenewDelay(handle ExecutionRunHandle, now time.Time) (time.Duration, bool) { + deadlineHandle, ok := handle.(executionRenewalDeadline) + if !ok { + return 20 * time.Second, true + } + remaining := deadlineHandle.RenewalDeadline().Sub(now) + if remaining <= 300*time.Millisecond { + return 0, false + } + delay := remaining / 3 + if delay > 20*time.Second { + delay = 20 * time.Second + } + if delay < 100*time.Millisecond { + delay = 100 * time.Millisecond + } + return delay, delay < remaining +} + +func (s *Service) startExecutionRenewal(_ session.SessionID, st *runState, run *agent.Run) { + st.executionMu.Lock() + if st.execution == nil || st.executionClosed { + st.executionMu.Unlock() + return + } + ctx, cancel := context.WithCancel(context.Background()) + st.executionRenewCancel = cancel + st.executionRenewDone = make(chan struct{}) + done := st.executionRenewDone + handle := st.execution + st.executionMu.Unlock() + go func() { + defer close(done) + for { + delay, usable := executionRenewDelay(handle, time.Now()) + if !usable { + s.logDiscoveryError(context.Background(), "execution ownership renewal failed", errors.New("execution grant has insufficient time remaining")) + run.Cancel() + return + } + timer := time.NewTimer(delay) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return + case <-timer.C: + remaining := 10 * time.Second + if deadlineHandle, ok := handle.(executionRenewalDeadline); ok { + remaining = time.Until(deadlineHandle.RenewalDeadline()) / 2 + if remaining <= 0 { + run.Cancel() + return + } + } + renewCtx, stop := context.WithTimeout(ctx, remaining) + err := handle.Renew(renewCtx) + stop() + if err != nil { + s.logDiscoveryError(context.Background(), "execution ownership renewal failed", err) + run.Cancel() + return + } + } + } + }() +} + +func (s *Service) teardownExecution(st *runState) { + if st == nil { + return + } + st.executionTeardown.Do(func() { + st.executionMu.Lock() + st.executionClosed = true + cancelRenew := st.executionRenewCancel + done := st.executionRenewDone + handle := st.execution + st.executionMu.Unlock() + if cancelRenew != nil { + cancelRenew() + } + if done != nil { + <-done + } + if handle != nil { + releaseCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if err := handle.Release(releaseCtx); err != nil { + s.logDiscoveryError(context.Background(), "execution ownership release failed", err) + } + cancel() + } + }) +} + // FinishRun removes run from the in-flight registry for id. It is the EXPORTED // counterpart of register that every wire adapter must call (typically via // `defer`) once it has finished draining run.Events(), so a completed run does @@ -8410,9 +8835,10 @@ func (s *Service) FinishRun(id session.SessionID, run *agent.Run) { return } s.mu.Lock() - st, ok := s.runs[id] + st := s.runs[id] + matches := st != nil && st.run == run s.mu.Unlock() - if !ok || st.run != run { + if !matches { return } parked := run.Outcome() == agent.RunOutcomeAuthorizationPending @@ -8421,6 +8847,7 @@ func (s *Service) FinishRun(id session.SessionID, run *agent.Run) { if parked && st.sess != nil { pending, pendingOK = st.sess.PendingAuthorization() } + s.teardownExecution(st) s.removeRunState(id, st) s.clearGuardrailReviewDetails(id) if parked { diff --git a/internal/adapter/server/team_test.go b/internal/adapter/server/team_test.go index cbbfa417ec..14c73d54b6 100644 --- a/internal/adapter/server/team_test.go +++ b/internal/adapter/server/team_test.go @@ -54,9 +54,9 @@ func (r *teamLivenessTracker) IsLive(id session.SessionID) bool { } // teamService builds a team-enabled Service whose per-member Engine uses the -// supplied mockllm provider (shared by all members) and carries that member's +// supplied provider (shared by all members) and carries that member's // coordination tools. -func teamService(t *testing.T, llm *mockllm.Provider) *server.Service { +func teamService(t *testing.T, llm port.LLMProvider) *server.Service { t.Helper() allow := permpolicy.NewPolicy(permpolicy.AllowAllFloorRules(), nil) memberEngine := func(tm *team.Team, spec agent.MemberSpec, _ string) agent.MemberBuild { diff --git a/internal/app/build.go b/internal/app/build.go index 17596e67a4..0a52cb5f57 100644 --- a/internal/app/build.go +++ b/internal/app/build.go @@ -164,6 +164,9 @@ type Config struct { MicroVMReadinessObserver microvmmanager.ReadinessObserver MicroVMReadinessFailed func(microvmmanager.ReadinessStage) MicroVMReadinessFailureHint string + // RemoteExecution selects the native remote-environment catalog attenuation and + // model-visible posture. It is set only by an explicit composition root. + RemoteExecution bool // ClientMCPOnCreate permits client-provided MCP servers on a session-creating // API request (issue #821, ADR 0237 applied to outbound MCP). It is a // deployment policy the cmd/ main decides from its listener topology and Build passes through verbatim; the zero value fails @@ -2301,18 +2304,25 @@ func Build(ctx context.Context, cfg Config) (*Built, error) { sessionReadLedger = redisBackend.ReadLedger } worktreeLister := buildWorktreeLister(cfg) + selectorIssuer, err := server.NewWorktreeSelectorIssuer(placementSelectorKey[:]) + if err != nil { + return nil, fmt.Errorf("initialize placement selector issuer: %w", err) + } + localProvider := &localPlacementProvider{ + scope: placementScope, root: cfg.Workspace, workspace: workspaceFactory, + runnerForRoot: func(root string) tool.CommandRunner { + return buildCommandRunnerForRoot(cfg, root) + }, + worktrees: worktreeLister, selectors: selectorIssuer, + } if placementProvider == nil { - selectorIssuer, err := server.NewWorktreeSelectorIssuer(placementSelectorKey[:]) - if err != nil { - return nil, fmt.Errorf("initialize placement selector issuer: %w", err) - } - placementProvider = &localPlacementProvider{ - scope: placementScope, root: cfg.Workspace, workspace: workspaceFactory, - runnerForRoot: func(root string) tool.CommandRunner { - return buildCommandRunnerForRoot(cfg, root) - }, - worktrees: worktreeLister, selectors: selectorIssuer, + placementProvider = localProvider + } else if cfg.RemoteExecution { + profiledProvider := &profilePlacementProvider{remote: placementProvider, local: localProvider} + if err := profiledProvider.ValidatePlacement(ctx); err != nil { + return nil, fmt.Errorf("validate default placement: %w", err) } + placementProvider = profiledProvider } attempts := startAttemptRecovery(ctx, cfg, reg, store, eventLog, assets.attemptRepository, assets.reflectionRepository, assets, placementProvider, placementScope) if attempts != nil { @@ -2395,6 +2405,8 @@ func Build(ctx context.Context, cfg Config) (*Built, error) { PlacementProvider: placementProvider, PlacementScope: placementScope, + ExecutionAccess: executionAccess(placementProvider), + ReferenceIntents: referenceIntentLifecycle(placementProvider), SessionReadLedger: sessionReadLedger, RootAuthority: func(kind session.SessionKind) session.Authority { return mintRuntimeRootAuthority(assets.rootCatalog, assets.mcpRuntimes, kind) @@ -2900,6 +2912,7 @@ func Build(ctx context.Context, cfg Config) (*Built, error) { // subagent-*/parallel-*/team-* children the run-entry funnel's own repair // (Step 3) never sees. See internal/app/session_reconcile.go. staleSessionReconcileClose := startStaleSessionReconcile(cfg, svc) + referenceIntentReconcileClose := startReferenceIntentReconcile(ctx, svcCfg.ReferenceIntents, svc) // Close tears down the main MCP manager AND any per-session client-MCP engines // still registered (svc.Close), so a process exit leaks neither. It also cancels @@ -2910,6 +2923,7 @@ func Build(ctx context.Context, cfg Config) (*Built, error) { if assets.reflectionLifecycle != nil { assets.reflectionLifecycle.close() } + referenceIntentReconcileClose() staleSessionReconcileClose() childGCClose() managedTempWorkerClose() @@ -3256,6 +3270,16 @@ func sessionEngineFactory( } } +func remoteSessionConfiguration(cfg Config, profile server.SessionProfile, workspace string, instructions prompt.InstructionAssembler, policy port.PermissionPolicy) (string, bool, prompt.InstructionAssembler, port.PermissionPolicy) { + if !cfg.RemoteExecution || profile == server.ProfileNoFS { + return workspace, false, instructions, policy + } + if variants, ok := policy.(sessionPermissionPolicies); ok { + policy = variants.remote + } + return "", true, instructions, policy +} + func sessionEngineFactoryWithTools( cfg Config, reg *providerRegistry, @@ -3282,7 +3306,8 @@ func sessionEngineFactoryWithTools( // is scoped to this one session's assembly; the SHARED engine keeps the // build-time pin over cfg.Workspace (the server's own root — the one // the shared engine was assembled for). - cfg.childPermResolver = childPermResolverFor(cfg, workspace) + projectWorkspace, remote, sessionInstructions, sessionPolicy := remoteSessionConfiguration(cfg, profile, workspace, instructions, policy) + cfg.childPermResolver = childPermResolverFor(cfg, projectWorkspace) // The NO-FS profile (issue #55): the service routes every no-fs session // through this factory unconditionally (the shared engine has the FS tools // baked in), and the profile selects the no-FS catalog assembly + the @@ -3432,7 +3457,7 @@ func sessionEngineFactoryWithTools( // Caller-scoped lazy hydration: rebuild this authenticated session's global // and admitted project generations from durable state before binding the // Skill tool. A failed authoritative read clears only these partitions. - skillPartitions := hydrateLearnedSkillPartitions(ctx, cfg, runtimeAssets, workspace) + skillPartitions := hydrateLearnedSkillPartitions(ctx, cfg, runtimeAssets, projectWorkspace) cat, closeFn, clientToolNames := assembleCatalog(ctx, cfg, reg, store, hooks, &runtimeAssets, catalogSession{ provider: resolvedProvider, @@ -3441,6 +3466,7 @@ func sessionEngineFactoryWithTools( clientMgr: mgr, narrate: false, noFS: noFS, + remote: remote, mode: mode, skillPartitions: skillPartitions, sessionTools: sessionTools, @@ -3452,13 +3478,14 @@ func sessionEngineFactoryWithTools( // the shared wiring, so no collaborator is silently dropped and a non-default // provider never contaminates compaction/counting. learningCfg := cfg - learningCfg.Workspace = workspace - learningCfg.LearningMode, learningCfg.LearningSensitivity, learningCfg.SkillActivationPolicy = learningPolicyForWorkspace(cfg, workspace) + learningCfg.Workspace = projectWorkspace + learningCfg.LearningMode, learningCfg.LearningSensitivity, learningCfg.SkillActivationPolicy = learningPolicyForWorkspace(cfg, projectWorkspace) learningCfg.Model = resolvedModel learningCfg.attemptRepository = runtimeAssets.attemptRepository learningCfg.automaticAdmissionLedger = runtimeAssets.automaticAdmissionLedger learningCfg.learningSourceStore = store - deps := engineDepsForProvider(cfg, resolvedProvider, resolvedModel, windowFn, store, policy, hooks, mcpProvider, instructions) + deps := engineDepsForProvider(cfg, resolvedProvider, resolvedModel, windowFn, store, sessionPolicy, hooks, mcpProvider, sessionInstructions) + deps = applyRemoteExecutionPosture(deps, remote) attachOperatorProfile(&deps, runtimeAssets.userModelStore) deps.LearningMode = learningCfg.LearningMode deps.LearningObserver = bindMaterializationLifecycle(buildReflectionObserver(learningCfg, resolvedProvider, learningCfg.Model, runtimeAssets.userModelStore, runtimeAssets.memStore, runtimeAssets.reflectionRepository, runtimeAssets.reflectionCoordinator, runtimeAssets.learningAdmissionGate, buildProcedureProcessor(learningCfg, runtimeAssets)), runtimeAssets.reflectionLifecycle) @@ -4381,7 +4408,12 @@ func buildEngine(ctx context.Context, cfg Config, reg *providerRegistry, provide // the SAME contextual ToolReviewer route; the option is a // no-op at any non-auto posture or with no checker, so yolo/strict/trusted // and the un-knobbed auto stay byte-identical. - sharedPolicy := escapePolicyForConfig(cfg, reg, provider, policy) + sharedPolicy := sessionPermissionPolicies{ + standard: escapePolicyForConfig(cfg, reg, provider, policy), + // Pin no workspace: retain operator permissions without project discovery. + remote: escapePolicyForConfig(cfg, reg, provider, + permpolicy.NewPolicyWithResolver(mainRules(cfg), learned, childPermResolverFor(cfg, ""), mainEvaluatorOptions(cfg)...)), + } var learningAdmissionGate *learningAdmissionGate if cfg.operatorLearningMode != learning.Off { learningAdmissionGate = newLearningAdmissionGate(cfg.LearningAdmissionInterval) @@ -4516,6 +4548,18 @@ func attachOperatorProfile(deps *agent.Deps, store tool.MemoryStore) { } } +type sessionPermissionPolicies struct { + standard port.PermissionPolicy + remote port.PermissionPolicy +} + +func (p sessionPermissionPolicies) Evaluate(ctx context.Context, id session.SessionID, mode session.PermissionMode, call session.ToolCall, ws tool.WorkspaceReader) governance.PermissionDecision { + return p.standard.Evaluate(ctx, id, mode, call, ws) +} +func (p sessionPermissionPolicies) Learn(id session.SessionID, call session.ToolCall) { + p.standard.Learn(id, call) +} + // buildInstructionAssembler composes the ephemeral turn-0 instruction fragments: // RootAssembler (project instructions), rules, soul, then the project memory // index. userModelStore remains in the internal signature to keep existing @@ -5612,7 +5656,7 @@ func compactionDecision(cfg Config) diagFact { // here with the composition-resolved provider (or the not-configured sentinel), // NOT as a zero-value tools.All()/NoFS() entry — an only-when-configured tool that // vanishes would be the silent-disable this harness avoids. -func registerCoreTools(cfg Config, cat *tool.Catalog, log, noFS bool, searchProvider tool.SearchProvider) { +func registerCoreTools(cfg Config, cat *tool.Catalog, log, noFS, remote bool, searchProvider tool.SearchProvider) { if noFS { for _, t := range tools.NoFS() { cat.MustRegister(t) @@ -5624,7 +5668,8 @@ func registerCoreTools(cfg Config, cat *tool.Catalog, log, noFS bool, searchProv cat.MustRegister(t) } cat.MustRegister(refsearch.NewWebSearchTool(searchProvider)) - if runner := buildCommandRunner(cfg); runner != nil { + runner := buildCommandRunner(cfg) + if runner != nil || remote { // The AGENT-loop Shell tool (not the fstools one): foreground byte-identical, // plus the `background: true` detach over the run's child registry. Its // companion ShellStatus — the SOLE status/collect/cancel channel for those @@ -5634,7 +5679,11 @@ func registerCoreTools(cfg Config, cat *tool.Catalog, log, noFS bool, searchProv cat.MustRegister(agent.NewShellTool()) cat.MustRegister(agent.NewShellStatusTool()) if log { - cfg.diag().Log(context.Background(), port.LevelInfo, "Shell tool ENABLED", "shell", cfg.Shell, "cwd", cfg.Workspace) + if remote { + cfg.diag().Log(context.Background(), port.LevelInfo, "Shell tool ENABLED", "shell", "remote execution") + } else { + cfg.diag().Log(context.Background(), port.LevelInfo, "Shell tool ENABLED", "shell", cfg.Shell, "cwd", cfg.Workspace) + } } } else if log { cfg.diag().Log(context.Background(), port.LevelInfo, "Shell tool DISABLED (shell-less mode): the agent has no command execution", @@ -5949,6 +5998,7 @@ func buildCatalog(ctx context.Context, cfg Config, reg *providerRegistry, provid provider: provider, providerID: reg.Default(), model: cfg.Model, + remote: cfg.RemoteExecution, narrate: true, }) // Aggregate the build-time Subagent per-def INLINE MCP managers' teardown into @@ -8544,6 +8594,21 @@ func applyRedisWorkspacePosture(pc prompt.Config, enabled bool) prompt.Config { const workspaceRootForPrompt = "/workspace" +const remoteExecutionPostureNote = "This session uses a persistent remote Kubernetes workspace. Use the filesystem tools and foreground Shell for work in /workspace. Local project instructions, rules, project-scoped skills and commands, schedules, SkillDraft, Parallel, Team, and Subagent delegation are unavailable; explicitly selected independent commands, operator-global skills, MCP, memory, and web tools remain available. Never assume harness-local files are part of this workspace." + +func applyRemoteExecutionPosture(deps agent.Deps, enabled bool) agent.Deps { + if !enabled { + return deps + } + pc := &deps.PromptConfig + pc.Env.Cwd, pc.Env.Shell, pc.Env.GitStatus = workspaceRootForPrompt, "remote foreground shell", "" + if pc.Role == "" { + pc.Role = prompt.DefaultRole() + } + pc.Role += "\n\n" + remoteExecutionPostureNote + return deps +} + func applyDebugSessionPosture(pc prompt.Config, target session.SessionID, selectedServers []string) prompt.Config { if pc.Role == "" { pc.Role = prompt.DefaultRole() @@ -8827,7 +8892,9 @@ const ( "the terminal client, which starts an embedded server by default or attaches to a remote one via " + "`mecatui connect ADDRESS`; mecated the general-purpose gRPC and HTTP/SSE server; mecak8s the " + "Kubernetes-native server keeping session state in Redis; mecatequi a one-shot CI task returning " + - "a patch." + "a patch; mecatl-execution-provider the authenticated Kubernetes control plane that owns persistent " + + "execution-environment lifecycle; mecatl-executor the credential-free workload helper that performs " + + "one confined filesystem or command operation inside an executor Pod." // selfKnowledgePostureAxes is the load-bearing content clause: the three safety // axes, kept distinct. Conflating the per-session permission mode with the diff --git a/internal/app/catalog.go b/internal/app/catalog.go index 5f02a3367b..672f6b7ac5 100644 --- a/internal/app/catalog.go +++ b/internal/app/catalog.go @@ -177,7 +177,8 @@ type catalogSession struct { // EXACTLY as in the default profile — guarded by TestNoFSCatalogProfile, // which pins the EXACT name-set delta. Always false for the build-time shared // catalog (a process always has a default-profile shared engine). - noFS bool + noFS bool + remote bool // mode is the session's permission mode (the per-session factory passes the // session's resolved mode; the build-time shared catalog leaves it // ModeDefault). The Schedule registration reads it to register the @@ -224,7 +225,7 @@ func assembleCatalog(ctx context.Context, cfg Config, reg *providerRegistry, sto classified := newClassifiedCatalog() cat := classified.catalog classified.captureEach(coreToolClassification, func() { - registerCoreTools(cfg, cat, s.narrate, s.noFS, a.searchProvider) + registerCoreTools(cfg, cat, s.narrate, s.noFS, s.remote, a.searchProvider) }) for _, sessionTool := range s.sessionTools { // A direct global manager retains ownership of its existing query wrapper. @@ -295,27 +296,31 @@ func assembleCatalog(ctx context.Context, cfg Config, reg *providerRegistry, sto refMgr = s.clientMgr } var subagentClose func() error - classified.captureEach(delegationToolClassification, func() { - subagentClose = registerSubagentTrio(ctx, cfg, cat, reg, store, hooks, *a, s, refMgr) - }) - // Parallel is ABSENT under the no-FS profile (not merely disarmed): every - // branch is a force-copy filesystem fork and the deliverable is a preserved - // fork PATH — both meaningless without a filesystem. - if !s.noFS { + if !s.remote { + classified.captureEach(delegationToolClassification, func() { + subagentClose = registerSubagentTrio(ctx, cfg, cat, reg, store, hooks, *a, s, refMgr) + }) + } + // Parallel is ABSENT under no-FS and remote execution profiles. + if !s.noFS && !s.remote { classified.captureEach(delegationToolClassification, func() { registerParallelTool(ctx, cfg, cat, reg, store, hooks, *a, s) }) } - classified.captureEach(delegationToolClassification, func() { - registerTeamTools(ctx, cfg, cat, reg, store, *a, s, refMgr) - }) + if !s.remote { + classified.captureEach(delegationToolClassification, func() { + registerTeamTools(ctx, cfg, cat, reg, store, *a, s, refMgr) + }) + } classified.capture(server.ClassificationEntry{Kind: server.KindCallerOwned, Rationale: "memory tools resolve the verified caller through the caller-partitioned store"}, func() { registerMemoryFamilies(ctx, cfg, cat, *a) }) - classified.captureEach(scheduleToolClassification, func() { - registerScheduleTool(ctx, cfg, cat, a, s) - }) + if !s.remote { + classified.captureEach(scheduleToolClassification, func() { + registerScheduleTool(ctx, cfg, cat, a, s) + }) + } classified.captureEach(func(t tool.Tool) (server.ClassificationEntry, bool) { return skillToolClassification(t, *a, s) }, func() { @@ -656,7 +661,7 @@ func registerSkillFamily(ctx context.Context, cfg Config, cat *tool.Catalog, a c cfg.diag().Log(ctx, port.LevelWarn, "registering skills failed; Skill tool disabled", "err", err) } } - if !s.noFS { + if !s.noFS && !s.remote { if a.learnedSkills != nil && cfg.SkillsDraftDir != "" { inventory := make([]learning.SkillInventoryItem, 0, len(a.skills)) for _, meta := range a.skills { diff --git a/internal/app/placement.go b/internal/app/placement.go index 188eb27fa6..2e4c19b0ad 100644 --- a/internal/app/placement.go +++ b/internal/app/placement.go @@ -18,6 +18,105 @@ const ( defaultPlacementScope = server.PlacementScope("deployment") ) +func referenceIntentLifecycle(provider server.PlacementProvider) server.ReferenceIntentLifecycle { + if profiled, ok := provider.(*profilePlacementProvider); ok { + lifecycle, _ := profiled.remote.(server.ReferenceIntentLifecycle) + return lifecycle + } + lifecycle, _ := provider.(server.ReferenceIntentLifecycle) + return lifecycle +} + +func executionAccess(provider server.PlacementProvider) server.ExecutionAccess { + access, _ := provider.(server.ExecutionAccess) + return access +} + +type profilePlacementProvider struct { + remote server.PlacementProvider + local *localPlacementProvider +} + +func (p *profilePlacementProvider) ValidatePlacement(ctx context.Context) error { + if validator, ok := p.remote.(server.PlacementValidator); ok { + return validator.ValidatePlacement(ctx) + } + return nil +} +func (p *profilePlacementProvider) Bind(ctx context.Context, req server.PlacementBindRequest) (server.PlacementBinding, error) { + if req.Selector.IsNoFS() { + return p.local.Bind(ctx, req) + } + return p.remote.Bind(ctx, req) +} +func (p *profilePlacementProvider) Reattach(ctx context.Context, req server.PlacementReattachRequest) (server.PlacementBinding, error) { + if req.Ref.Kind == session.EnvKindNoFS { + return p.local.Reattach(ctx, req) + } + remote, ok := p.remote.(server.PlacementReattacher) + if !ok { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + return remote.Reattach(ctx, req) +} + +func (p *profilePlacementProvider) PrepareReferenceDelete(ctx context.Context, req server.PlacementSuccessorRequest) (server.ReferenceDeleteHandle, error) { + lifecycle, ok := p.remote.(server.ReferenceLifecycle) + if !ok || !lifecycle.Applies(req.Ref) { + return nil, server.ErrPlacementUnavailable + } + return lifecycle.PrepareReferenceDelete(ctx, req) +} + +func (p *profilePlacementProvider) ListReferenceIntents(ctx context.Context, limit int) ([]server.ReferenceIntent, error) { + lifecycle, ok := p.remote.(server.ReferenceIntentLifecycle) + if !ok { + return nil, server.ErrPlacementUnavailable + } + return lifecycle.ListReferenceIntents(ctx, limit) +} +func (p *profilePlacementProvider) CommitReferenceIntent(ctx context.Context, intent server.ReferenceIntent) error { + lifecycle, ok := p.remote.(server.ReferenceIntentLifecycle) + if !ok { + return server.ErrPlacementUnavailable + } + return lifecycle.CommitReferenceIntent(ctx, intent) +} +func (p *profilePlacementProvider) ConfirmReferenceIntentDelete(ctx context.Context, intent server.ReferenceIntent) error { + lifecycle, ok := p.remote.(server.ReferenceIntentLifecycle) + if !ok { + return server.ErrPlacementUnavailable + } + return lifecycle.ConfirmReferenceIntentDelete(ctx, intent) +} +func (p *profilePlacementProvider) CancelReferenceIntentDelete(ctx context.Context, intent server.ReferenceIntent) error { + lifecycle, ok := p.remote.(server.ReferenceIntentLifecycle) + if !ok { + return server.ErrPlacementUnavailable + } + return lifecycle.CancelReferenceIntentDelete(ctx, intent) +} + +func (p *profilePlacementProvider) ReserveSuccessor(ctx context.Context, req server.PlacementSuccessorRequest) (server.PlacementBinding, error) { + reservoir, ok := p.remote.(server.PlacementSuccessorReservoir) + if !ok { + return server.PlacementBinding{}, server.ErrPlacementUnavailable + } + return reservoir.ReserveSuccessor(ctx, req) +} + +func (p *profilePlacementProvider) Applies(ref session.EnvironmentRef) bool { + access, ok := p.remote.(server.ExecutionAccess) + return ok && access.Applies(ref) +} +func (p *profilePlacementProvider) AcquireRun(ctx context.Context, req server.ExecutionRunRequest) (server.ExecutionRunHandle, error) { + access, ok := p.remote.(server.ExecutionAccess) + if !ok || !access.Applies(req.Ref) { + return nil, server.ErrPlacementUnavailable + } + return access.AcquireRun(ctx, req) +} + // localPlacementProvider is the trusted composition default. It owns exactly // one configured local record plus the no-FS attenuation; it has no inventory // registry or path-derived public identifier. diff --git a/internal/app/project_ingestion.go b/internal/app/project_ingestion.go index 19ca62f0ee..ef365fc40a 100644 --- a/internal/app/project_ingestion.go +++ b/internal/app/project_ingestion.go @@ -14,7 +14,9 @@ package app // shell also reads cfg.TrustProject directly because both decisions intentionally // express the same operator vouch for the workspace and its .git. func projectIngestionAdmitted(cfg Config) bool { - return cfg.TrustProject + // Local project trust cannot authorize source ingestion for a remote + // deployment; its explicit no-FS attenuation has no project source either. + return cfg.TrustProject && !cfg.RemoteExecution } // projectIngestionAdmittedForRoot binds the single project-ingestion decision to diff --git a/internal/app/reference_intent_reconcile.go b/internal/app/reference_intent_reconcile.go new file mode 100644 index 0000000000..03fd722a8a --- /dev/null +++ b/internal/app/reference_intent_reconcile.go @@ -0,0 +1,36 @@ +package app + +import ( + "context" + "time" + + "github.com/stacklok/mecatl/internal/adapter/server" +) + +const referenceIntentReconcileInterval = time.Minute + +func startReferenceIntentReconcile(ctx context.Context, lifecycle server.ReferenceIntentLifecycle, svc *server.Service) func() { + if lifecycle == nil || svc == nil { + return func() {} + } + workerCtx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + defer close(done) + svc.ReconcileReferenceIntents(workerCtx) + ticker := time.NewTicker(referenceIntentReconcileInterval) + defer ticker.Stop() + for { + select { + case <-workerCtx.Done(): + return + case <-ticker.C: + svc.ReconcileReferenceIntents(workerCtx) + } + } + }() + return func() { + cancel() + <-done + } +} diff --git a/internal/app/remote_create_test.go b/internal/app/remote_create_test.go new file mode 100644 index 0000000000..b9fd6352b7 --- /dev/null +++ b/internal/app/remote_create_test.go @@ -0,0 +1,116 @@ +package app + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stacklok/mecatl/engine/adapter/memfs" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/permconfig" + "github.com/stacklok/mecatl/internal/adapter/server" +) + +func TestRemoteHTTPCreateRealOpenRouterDoesNotWaitForDiscoveryOrInfer(t *testing.T) { + fakeRulesEnv(t, t.TempDir(), t.TempDir()) + t.Setenv("XDG_DATA_HOME", t.TempDir()) + t.Setenv("XDG_STATE_HOME", t.TempDir()) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + started, release := make(chan struct{}), make(chan struct{}) + var releaseOnce, startedOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer unblock() + live := &http.Client{Transport: nativeRoundTripFunc(func(r *http.Request) (*http.Response, error) { + startedOnce.Do(func() { close(started) }) + select { + case <-release: + return &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(`{"data":[]}`))}, nil + case <-r.Context().Done(): + return nil, r.Context().Err() + } + })} + var inference atomic.Int64 + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + inference.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer endpoint.Close() + ref := session.EnvironmentRef{Kind: "kubernetes", ID: "private-ref-sentinel", Revision: "private-revision-sentinel"} + env := tool.MustEnvironment(ref, memfs.NewWorkspace("/workspace"), memledger.New(), nil) + diag := &attrCapturingDiag{} + binds := 0 + built, err := buildIsolated(t, ctx, Config{ + // No selector, no launch root and no mock: /workspace must use the real + // per-session factory, exactly like the native live create request. + RemoteExecution: true, PlacementScope: "remote", PlacementProvider: remoteFactoryPlacement{env: env, binds: &binds}, + DefaultProvider: "openrouter", Model: "anthropic/claude-haiku-4.5", NoSoul: true, NoUserModel: true, + OpenRouterKey: "synthetic-offline-key", envDetector: fakeEnv(nil), Diagnostics: diag, + ProviderOverrides: permconfig.ProviderOverrides{providerOpenRouter: {BaseURL: endpoint.URL + "/v1"}}, + liveModelHTTPClient: live, + }) + if err != nil { + t.Fatal(err) + } + defer built.Close() + // Release discovery before closing Build, including the failure path. + defer unblock() + select { + case <-started: + case <-ctx.Done(): + t.Fatal("actual registry did not start live discovery") + } + principal := &session.Principal{Issuer: "private-issuer-sentinel", Subject: "private-owner-sentinel", GrantType: session.GrantTypeUser} + req := httptest.NewRequest(http.MethodPost, "/v1/sessions", strings.NewReader(`{"mode":"default","limits":{"max_turns":8,"max_tool_calls":20,"max_consecutive_failures":3}}`)) + req = req.WithContext(session.WithPrincipal(ctx, principal)) + response := httptest.NewRecorder() + done := make(chan struct{}) + go func() { server.NewHTTPHandler(built.Service).ServeHTTP(response, req); close(done) }() + select { + case <-done: + case <-ctx.Done(): + unblock() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("create did not unwind after discovery release and cancellation") + } + t.Fatalf("create blocked behind discovery; stages: %s", diag.dump()) + } + if response.Code != http.StatusCreated || binds != 1 { + t.Fatalf("create status=%d binds=%d", response.Code, binds) + } + var result struct { + ID string `json:"session_id"` + Model struct { + Provider string `json:"provider_id"` + } `json:"resolved_model"` + } + if err := json.Unmarshal(response.Body.Bytes(), &result); err != nil || result.Model.Provider != "openrouter" || result.ID == "" { + t.Fatal("not a real OpenRouter session response") + } + persisted, err := built.Service.GetSession(req.Context(), session.SessionID(result.ID)) + if err != nil || persisted.EnvironmentRef != ref || len(persisted.Conversation.Messages) != 0 || inference.Load() != 0 { + t.Fatal("create inferred, recorded a prompt, or lost exact placement") + } + stages := diag.dump() + for _, contract := range []string{"stage http_handler reason begin", "stage session_id_probe reason ok", "stage engine_factory reason ok", "stage session_persist reason ok", "stage http_response reason ok"} { + if !strings.Contains(stages, contract) { + t.Errorf("actual built factory missing diagnostic %q", contract) + } + } + for _, line := range strings.Split(stages, "\n") { + if strings.Contains(line, "remote create stage") && (strings.Contains(line, "sentinel") || strings.Contains(line, "/workspace") || strings.Contains(line, "synthetic-offline-key")) { + t.Fatal("private create data reached diagnostics") + } + } +} diff --git a/internal/app/remote_execution_test.go b/internal/app/remote_execution_test.go new file mode 100644 index 0000000000..638e0cbc25 --- /dev/null +++ b/internal/app/remote_execution_test.go @@ -0,0 +1,330 @@ +package app + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/stacklok/mecatl/engine/adapter/memfs" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/adapter/mockllm" + "github.com/stacklok/mecatl/engine/governance" + "github.com/stacklok/mecatl/engine/port" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/server" +) + +type remoteFactoryPlacement struct { + env tool.Environment + binds *int + validateErr error +} + +func (p remoteFactoryPlacement) ValidatePlacement(context.Context) error { return p.validateErr } +func (p remoteFactoryPlacement) Bind(_ context.Context, req server.PlacementBindRequest) (server.PlacementBinding, error) { + if p.binds != nil { + *p.binds++ + } + if req.BindingID == "" || req.Principal == nil { + return server.PlacementBinding{}, server.ErrInvalidPlacementSelection + } + return server.PlacementBinding{Environment: p.env, Ref: p.env.Ref(), Metadata: server.PlacementMetadata{Label: "Remote Kubernetes workspace"}}, nil +} +func (p remoteFactoryPlacement) Reattach(_ context.Context, req server.PlacementReattachRequest) (server.PlacementBinding, error) { + if req.BindingID == "" || req.Ref != p.env.Ref() { + return server.PlacementBinding{}, server.ErrPlacementNotFound + } + return server.PlacementBinding{Environment: p.env, Ref: p.env.Ref()}, nil +} + +func TestRemoteExecutionPreflightNeverAllocates(t *testing.T) { + fakeRulesEnv(t, t.TempDir(), t.TempDir()) + for _, validation := range []error{nil, server.ErrInvalidPlacementBinding, server.ErrPlacementUnavailable} { + binds := 0 + built, err := buildIsolated(t, t.Context(), Config{UseMock: true, NoSoul: true, RemoteExecution: true, PlacementScope: "remote", PlacementProvider: remoteFactoryPlacement{binds: &binds, validateErr: validation}}) + if built != nil { + built.Close() + } + if validation == nil && err != nil || validation != nil && !errors.Is(err, validation) { + t.Fatalf("validation=%v startup error=%v", validation, err) + } + if binds != 0 { + t.Fatalf("preflight allocated %d environments", binds) + } + } +} + +func TestRemoteDeploymentNoFSUsesLocalAttenuationWithoutProviderCall(t *testing.T) { + fakeRulesEnv(t, t.TempDir(), t.TempDir()) + binds := 0 + remoteWS := memfs.NewWorkspace("/workspace") + remoteRef := session.EnvironmentRef{Kind: "kubernetes", ID: "env-1", Revision: "rev-1"} + remoteEnv := tool.MustEnvironment(remoteRef, remoteWS, memledger.New(), nil) + var captured port.LLMRequest + provider := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { captured = req })}, + mockllm.ToolCallTurn(session.ToolCall{ID: "read", Name: "Read", Args: json.RawMessage(`{"path":"secret"}`)}), + mockllm.TextTurn("done"), + ) + built, err := buildIsolated(t, context.Background(), Config{MockProvider: provider, PlacementProvider: remoteFactoryPlacement{env: remoteEnv, binds: &binds}, PlacementScope: "remote", RemoteExecution: true, NoSoul: true, SchedulerEnabled: false}) + if err != nil { + t.Fatal(err) + } + defer built.Close() + ctx := session.WithPrincipal(context.Background(), &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser}) + sess, err := built.Service.CreateSessionWithProfile(ctx, session.ModeAccept, session.Limits{}, server.ProviderSelector{}, server.ProfileNoFS) + if err != nil { + t.Fatal(err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "no files") + if err != nil { + t.Fatal(err) + } + for range run.Events() { + } + built.Service.FinishRun(sess.ID, run) + if binds != 0 { + t.Fatalf("remote provider Bind calls = %d, want zero", binds) + } + if !strings.Contains(captured.System.StablePrefix, noFSPostureNote) || strings.Contains(captured.System.StablePrefix, remoteExecutionPostureNote) { + t.Fatalf("no-fs posture was replaced by remote posture: %q", captured.System.StablePrefix) + } + for _, spec := range captured.Tools { + if spec.Name == "Read" || spec.Name == "Shell" { + t.Fatalf("no-fs catalog exposed %q", spec.Name) + } + } +} + +type remotePermissionRunner struct { + *memfs.CommandRunner + root string +} + +func (r remotePermissionRunner) BoundWorkspaceRoot() string { return r.root } + +func TestRemoteExecutionPreservesOperatorPermissionsUnderAuto(t *testing.T) { + for _, source := range []string{"global", "explicit"} { + t.Run(source, func(t *testing.T) { + xdg := t.TempDir() + fakeRulesEnv(t, xdg, t.TempDir()) + t.Setenv("XDG_STATE_HOME", t.TempDir()) + operatorPath := filepath.Join(xdg, "mecatl", "settings.yaml") + if source == "explicit" { + operatorPath = filepath.Join(t.TempDir(), "operator.yaml") + } + writeProjectFile(t, operatorPath, "permissions:\n deny:\n - 'Shell(echo denied)'\n ask:\n - 'Shell(echo asked)'\n") + project := t.TempDir() + projectRules := "permissions:\n deny:\n - 'Shell(echo allowed)'\n ask:\n - 'Shell(echo project-ask)'\n allow:\n - 'Shell(echo denied)'\n - 'Shell(echo asked)'\n" + mkdirProjectSettings(t, project, projectRules) + ws := memfs.NewWorkspace(project) + if _, err := ws.CreateFile(t.Context(), ".mecatl/settings.yaml", []byte(projectRules)); err != nil { + t.Fatal(err) + } + runner := memfs.NewCommandRunner() + runner.SetResult(&tool.CommandResult{Stdout: "EXECUTED"}, nil) + ref := session.EnvironmentRef{Kind: "kubernetes", ID: "permissions", Revision: "v1"} + env := tool.MustEnvironment(ref, ws, memledger.New(), remotePermissionRunner{runner, project}) + llm := mockllm.New( + mockllm.ToolCallTurn( + session.NewToolCall("denied", "Shell", json.RawMessage(`{"command":"echo denied"}`)), + session.NewToolCall("asked", "Shell", json.RawMessage(`{"command":"echo asked"}`)), + session.NewToolCall("allowed", "Shell", json.RawMessage(`{"command":"echo allowed"}`)), + session.NewToolCall("project-ask", "Shell", json.RawMessage(`{"command":"echo project-ask"}`)), + ), mockllm.TextTurn("done")) + cfg := Config{MockProvider: llm, Workspace: project, Posture: PostureAuto, PermissionsConventional: true, NoSoul: true, RemoteExecution: true, PlacementScope: "remote", PlacementProvider: remoteFactoryPlacement{env: env}} + if source == "explicit" { + cfg.PermissionConfigs = []string{operatorPath} + } + built, err := buildIsolated(t, t.Context(), cfg) + if err != nil { + t.Fatal(err) + } + defer built.Close() + ctx := session.WithPrincipal(t.Context(), &session.Principal{Issuer: "issuer", Subject: "alice", GrantType: session.GrantTypeUser}) + sess, err := built.Service.CreateSession(ctx, session.ModeDefault, session.Limits{}) + if err != nil { + t.Fatal(err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "check permissions") + if err != nil { + t.Fatal(err) + } + asks := 0 + results := map[session.ToolCallID]session.ToolResult{} + for ev := range run.Events() { + if ev.Ask != nil { + asks++ + if ev.Ask.Call != "asked" || ev.Ask.AskProvenance != governance.AskProvenanceConfigured { + t.Errorf("unexpected permission ask: %+v", ev.Ask) + } + if _, err := built.Service.ApproveRun(ctx, sess.ID, ev.Ask.AskID, session.VerdictDeny, ""); err != nil { + t.Fatal(err) + } + } + if ev.ToolResult != nil { + results[ev.ToolResult.CallID] = *ev.ToolResult + } + } + built.Service.FinishRun(sess.ID, run) + if asks != 1 { + t.Errorf("configured asks = %d, want 1", asks) + } + for _, id := range []session.ToolCallID{"denied", "asked", "allowed", "project-ask"} { + result, ok := results[id] + blocked := id == "denied" || id == "asked" + if !ok || result.IsError != blocked || strings.Contains(result.Content, "EXECUTED") == blocked { + t.Errorf("%s result = %+v, present=%t, want blocked=%t", id, result, ok, blocked) + } + } + }) + } +} + +func TestRemoteExecutionRealFactoryCarriesPostureAndAttenuatedCatalog(t *testing.T) { + xdg := t.TempDir() + fakeRulesEnv(t, xdg, t.TempDir()) + t.Setenv("XDG_STATE_HOME", t.TempDir()) + writeUserRule(t, xdg, "operator-rule", "OPERATOR_RULE_MARKER\n") + writeSkill(t, filepath.Join(xdg, "mecatl/skills"), "operator-skill", "Operator skill", "OPERATOR_SKILL_BODY") + localProject := t.TempDir() + initTestRepo(t, localProject) + if err := os.WriteFile(filepath.Join(localProject, "AGENTS.md"), []byte("LOCAL_PROJECT_MARKER_DO_NOT_LOAD\n"), 0o600); err != nil { + t.Fatal(err) + } + writeProjectRule(t, localProject, "project-rule", "LOCAL_RULE_MARKER_DO_NOT_LOAD\n") + localSnapshotSeen, localProjectSeen, localRuleSeen := false, false, false + localLLM := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { + localSnapshotSeen = strings.Contains(req.System.Render(), "initial commit") && strings.Contains(req.System.Render(), "") + for _, message := range req.Messages { + localProjectSeen = localProjectSeen || strings.Contains(message.Text, "LOCAL_PROJECT_MARKER") + localRuleSeen = localRuleSeen || strings.Contains(message.Text, "LOCAL_RULE_MARKER") + } + })}, mockllm.TextTurn("local done")) + local, err := buildIsolated(t, t.Context(), Config{MockProvider: localLLM, Workspace: localProject, TrustProject: true, NoSoul: true, Shell: "/bin/sh"}) + if err != nil { + t.Fatal(err) + } + defer local.Close() + localSession, err := local.Service.CreateSession(t.Context(), session.ModeDefault, session.Limits{}) + if err != nil { + t.Fatal(err) + } + localRun, err := local.Service.StartRun(t.Context(), localSession.ID, "inspect local") + if err != nil { + t.Fatal(err) + } + for range localRun.Events() { + } + local.Service.FinishRun(localSession.ID, localRun) + if !localSnapshotSeen || !localProjectSeen || !localRuleSeen { + t.Fatal("local factory omitted the positive-control Git snapshot, instructions, or rules") + } + ws := memfs.NewWorkspace(localProject) + if _, err := ws.CreateFile(context.Background(), "AGENTS.md", []byte("REMOTE_PROJECT_MARKER_DO_NOT_LOAD")); err != nil { + t.Fatal(err) + } + if _, err := ws.CreateFile(context.Background(), "main.go", []byte("package main\n")); err != nil { + t.Fatal(err) + } + ref := session.EnvironmentRef{Kind: session.EnvironmentKind("kubernetes"), ID: "env-1", Revision: "rev-1"} + env := tool.MustEnvironment(ref, ws, memledger.New(), nil) + var mu sync.Mutex + var captured port.LLMRequest + provider := mockllm.NewWith([]mockllm.Option{mockllm.WithRequestObserver(func(req port.LLMRequest) { mu.Lock(); captured = req; mu.Unlock() })}, + mockllm.ToolCallTurn( + session.ToolCall{ID: "Subagent", Name: "Subagent", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "Parallel", Name: "Parallel", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "Team", Name: "Team", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "SkillDraft", Name: "SkillDraft", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "Schedule", Name: "Schedule", Args: json.RawMessage(`{}`)}, + session.ToolCall{ID: "project-skill", Name: "Skill", Args: json.RawMessage(`{"name":"project-only"}`)}, + session.ToolCall{ID: "operator-skill", Name: "Skill", Args: json.RawMessage(`{"name":"operator-skill"}`)}, + ), + mockllm.ToolCallTurn(session.ToolCall{ID: "read", Name: "Read", Args: json.RawMessage(`{"path":"main.go"}`)}), + mockllm.TextTurn(""), + mockllm.TextTurn("done"), + ) + writeSkill(t, filepath.Join(localProject, ".mecatl/skills"), "project-only", "LOCAL_SKILL_MARKER_DO_NOT_LOAD", "LOCAL_SKILL_BODY_DO_NOT_LOAD") + commandDir := filepath.Join(localProject, ".claude", "commands") + if err := os.MkdirAll(commandDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(commandDir, "project-only.md"), []byte("LOCAL_COMMAND_MARKER_DO_NOT_LOAD"), 0o600); err != nil { + t.Fatal(err) + } + built, err := buildIsolated(t, context.Background(), Config{MockProvider: provider, PlacementProvider: remoteFactoryPlacement{env: env}, PlacementScope: "remote", RemoteExecution: true, Workspace: localProject, TrustProject: true, AllowAllTools: true, EnableCommands: true, SkillsConventional: true, NoSoul: true, SchedulerEnabled: false}) + if err != nil { + t.Fatal(err) + } + defer built.Close() + principal := &session.Principal{Issuer: "https://issuer.example", Subject: "alice", GrantType: session.GrantTypeUser} + ctx := session.WithPrincipal(context.Background(), principal) + sess, err := built.Service.CreateSession(ctx, session.ModeAccept, session.Limits{}) + if err != nil { + t.Fatal(err) + } + if commands, err := built.Service.ListCommandsForSession(ctx, sess.ID); err != nil || len(commands) != 1 || commands[0].Name != "operator-skill" { + t.Fatalf("remote command discovery must retain only the operator-global skill: %v, %v", commands, err) + } + if _, err := built.Service.ListWorktreesForSession(ctx, sess.ID); !errors.Is(err, server.ErrPlacementUnavailable) { + t.Fatalf("remote worktree discovery did not reject unsupported placement: %v", err) + } + run, err := built.Service.StartRun(ctx, sess.ID, "/project-only") + if err != nil { + t.Fatal(err) + } + for range run.Events() { + } + built.Service.FinishRun(sess.ID, run) + mu.Lock() + defer mu.Unlock() + if !strings.Contains(captured.System.StablePrefix, remoteExecutionPostureNote) { + t.Fatal("real per-session factory omitted remote execution posture") + } + if strings.Contains(captured.System.Render(), "") { + t.Fatal("remote factory ingested the local Git snapshot") + } + requestJSON, err := json.Marshal(captured) + if err != nil { + t.Fatal(err) + } + for _, marker := range []string{"LOCAL_PROJECT_MARKER", "REMOTE_PROJECT_MARKER", "LOCAL_RULE_MARKER", "LOCAL_SKILL_MARKER", "LOCAL_SKILL_BODY", "LOCAL_COMMAND_MARKER", "initial commit"} { + if strings.Contains(string(requestJSON), marker) { + t.Errorf("remote request ingested forbidden project source %s", marker) + } + } + if !strings.Contains(string(requestJSON), "OPERATOR_RULE_MARKER") || !strings.Contains(string(requestJSON), "OPERATOR_SKILL_BODY") { + t.Fatal("remote request dropped operator-global rules or skills") + } + failed := map[string]bool{} + for _, message := range captured.Messages { + if result := message.ToolResult; result != nil { + failed[string(result.CallID)] = result.IsError + } + } + for _, id := range []string{"Subagent", "Parallel", "Team", "SkillDraft", "Schedule", "project-skill"} { + if !failed[id] { + t.Errorf("unsupported invocation %s did not return a model-visible error", id) + } + } + names := map[string]bool{} + for _, spec := range captured.Tools { + names[spec.Name] = true + } + for _, name := range []string{"Read", "Write", "WebFetch"} { + if !names[name] { + t.Errorf("remote catalog omitted compatible tool %q", name) + } + } + for _, name := range []string{"Subagent", "SubagentStatus", "InspectSubagent", "Parallel", "Team", "InspectMember", "SkillDraft", "Schedule"} { + if names[name] { + t.Errorf("remote catalog advertised unsupported tool %q", name) + } + } +} diff --git a/internal/creatediag/create.go b/internal/creatediag/create.go new file mode 100644 index 0000000000..8eca29be54 --- /dev/null +++ b/internal/creatediag/create.go @@ -0,0 +1,59 @@ +// Package creatediag records content-free, request-scoped remote-create progress. +package creatediag + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "time" + + "github.com/stacklok/mecatl/engine/port" +) + +type key struct{} +type trace struct { + diag port.Diagnostics + start time.Time + session string +} + +// Start is used only by the remote HTTP create handler, after authentication. +func Start(ctx context.Context, diag port.Diagnostics) context.Context { + return context.WithValue(ctx, key{}, &trace{diag: diag, start: time.Now()}) +} + +// Session hashes even caller-supplied IDs; no owner or placement identity is logged. +// The create request is sequential; the trace never escapes its request context. +func Session(ctx context.Context, id string) { + if t, ok := ctx.Value(key{}).(*trace); ok { + sum := sha256.Sum256([]byte(id)) + t.session = hex.EncodeToString(sum[:16]) + } +} + +// Note accepts only harness-owned stage/reason literals, never producer text. +func Note(ctx context.Context, stage, reason string, calls int) { + if t, ok := ctx.Value(key{}).(*trace); ok { + t.diag.Log(ctx, port.LevelDebug, "remote create stage", "stage", stage, + "reason", reason, "elapsed_ms", time.Since(t.start).Milliseconds(), + "calls", calls, "session", t.session) + } +} + +// Begin emits both sides of a blocking boundary without exposing its error text. +func Begin(ctx context.Context, stage string) func(error) { + Note(ctx, stage, "begin", 0) + return func(err error) { + reason := "ok" + switch { + case errors.Is(err, context.Canceled): + reason = "cancelled" + case errors.Is(err, context.DeadlineExceeded): + reason = "deadline" + case err != nil: + reason = "error" + } + Note(ctx, stage, reason, 1) + } +} diff --git a/internal/executionenv/errors.go b/internal/executionenv/errors.go new file mode 100644 index 0000000000..6a65285a82 --- /dev/null +++ b/internal/executionenv/errors.go @@ -0,0 +1,126 @@ +package executionenv + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "strings" +) + +// ErrorCode is a stable private-protocol error classification. +type ErrorCode string + +// Private-protocol error codes. +const ( + CodeInvalidArgument ErrorCode = "invalid_argument" + CodeUnauthenticated ErrorCode = "unauthenticated" + CodePermissionDenied ErrorCode = "permission_denied" + CodeNotFound ErrorCode = "not_found" + CodeAlreadyExists ErrorCode = "already_exists" + CodeConflict ErrorCode = "conflict" + CodeVersionMismatch ErrorCode = "version_mismatch" + CodeDirectoryNotEmpty ErrorCode = "directory_not_empty" + CodeNotReady ErrorCode = "not_ready" + CodeFenceUnknown ErrorCode = "fence_unknown" + CodeResourceExhausted ErrorCode = "resource_exhausted" + CodeInternal ErrorCode = "internal" +) + +// Valid reports whether c is part of the closed private-protocol vocabulary. +func (c ErrorCode) Valid() bool { + switch c { + case CodeInvalidArgument, CodeUnauthenticated, CodePermissionDenied, CodeNotFound, + CodeAlreadyExists, CodeConflict, CodeVersionMismatch, CodeDirectoryNotEmpty, CodeNotReady, CodeFenceUnknown, + CodeResourceExhausted, CodeInternal: + return true + default: + return false + } +} + +// Error is a bounded structured private-protocol failure. +type Error struct { + Code ErrorCode `json:"code"` + Message string `json:"message"` + Retryable bool `json:"retryable,omitempty"` +} + +func (e *Error) Error() string { return string(e.Code) + ": " + e.Message } + +// DecodeStrict decodes exactly one bounded JSON value for the credential-free +// provider-to-workload stdin protocol, without unknown or duplicate fields. +func DecodeStrict(data []byte, dst any) error { + if len(data) > MaxJSONBody { + return &Error{Code: CodeResourceExhausted, Message: "request body exceeds limit"} + } + if err := rejectDuplicateKeys(data); err != nil { + return &Error{Code: CodeInvalidArgument, Message: "invalid JSON: " + err.Error()} + } + d := json.NewDecoder(strings.NewReader(string(data))) + d.DisallowUnknownFields() + if err := d.Decode(dst); err != nil { + return &Error{Code: CodeInvalidArgument, Message: "invalid JSON: " + err.Error()} + } + if err := d.Decode(&struct{}{}); !errors.Is(err, io.EOF) { + return &Error{Code: CodeInvalidArgument, Message: "request must contain exactly one JSON value"} + } + return nil +} + +func rejectDuplicateKeys(data []byte) error { + d := json.NewDecoder(strings.NewReader(string(data))) + if err := walkJSON(d); err != nil { + return err + } + if _, err := d.Token(); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("trailing JSON value") + } + return err + } + return nil +} +func walkJSON(d *json.Decoder) error { + tok, err := d.Token() + if err != nil { + return err + } + delim, ok := tok.(json.Delim) + if !ok { + return nil + } + switch delim { + case '{': + seen := map[string]struct{}{} + for d.More() { + t, err := d.Token() + if err != nil { + return err + } + k, ok := t.(string) + if !ok { + return errors.New("object key is not a string") + } + if _, ok := seen[k]; ok { + return fmt.Errorf("duplicate field %q", k) + } + seen[k] = struct{}{} + if err := walkJSON(d); err != nil { + return err + } + } + _, err = d.Token() + return err + case '[': + for d.More() { + if err := walkJSON(d); err != nil { + return err + } + } + _, err = d.Token() + return err + default: + return errors.New("unexpected delimiter") + } +} diff --git a/internal/executionenv/grant.go b/internal/executionenv/grant.go new file mode 100644 index 0000000000..7da228d2e2 --- /dev/null +++ b/internal/executionenv/grant.go @@ -0,0 +1,181 @@ +package executionenv + +import ( + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strings" + "time" +) + +const grantAlgorithm = "Ed25519" + +var ( + // ErrGrantExpired identifies the only grant failure eligible for a read-only refresh. + ErrGrantExpired = errors.New("grant expired") + // ErrGrantNotYetValid identifies a grant whose validity window has not started. + ErrGrantNotYetValid = errors.New("grant not yet valid") +) + +type grantHeader struct { + Algorithm string `json:"alg"` + KeyID string `json:"kid"` + Type string `json:"typ"` +} + +// GrantClaims binds a short-lived capability to one client, owner, environment, and epoch. +type GrantClaims struct { + KeyID string `json:"-"` + Issuer string `json:"iss"` + Audience string `json:"aud"` + Client string `json:"client"` + OwnerHash string `json:"owner"` + BindingID string `json:"binding_id"` + RunID string `json:"run_id"` + ClaimID string `json:"claim_id"` + Environment EnvironmentRef `json:"environment"` + Epoch uint64 `json:"epoch"` + GrantGeneration uint64 `json:"grant_generation"` + Operations []Operation `json:"operations"` + NotBefore time.Time `json:"not_before"` + ExpiresAt time.Time `json:"expires_at"` + Nonce string `json:"nonce"` +} + +// GrantExpectation defines the exact binding and operation required by a request. +type GrantExpectation struct { + Client string + OwnerHash string + BindingID string + RunID string + ClaimID string + Environment EnvironmentRef + Epoch uint64 + GrantGeneration uint64 + Operation Operation +} + +// GrantVerifier verifies signed grants against configured trust and revocation state. +type GrantVerifier struct { + Keys map[string]ed25519.PublicKey + RevokedKeys map[string]struct{} + RevokedNonces map[string]struct{} + Issuer string + Audience string + MaxLifetime time.Duration + Now func() time.Time +} + +// SignGrant validates and signs claims with Ed25519. +func SignGrant(key ed25519.PrivateKey, c GrantClaims) (string, error) { + if len(key) != ed25519.PrivateKeySize { + return "", errors.New("invalid Ed25519 private key") + } + if err := validateClaims(c); err != nil { + return "", err + } + h, _ := json.Marshal(grantHeader{Algorithm: grantAlgorithm, KeyID: c.KeyID, Type: "MECATL-GRANT"}) + p, _ := json.Marshal(c) + unsigned := rawURL(h) + "." + rawURL(p) + sig := ed25519.Sign(key, []byte(unsigned)) + return unsigned + "." + rawURL(sig), nil +} + +// Verify validates a grant and its exact request binding. +func (v GrantVerifier) Verify(token string, e GrantExpectation) (GrantClaims, error) { //nolint:gocyclo // Fail-closed claim validation is intentionally linear. + var c GrantClaims + parts := strings.Split(token, ".") + if len(parts) != 3 { + return c, errors.New("malformed grant") + } + hb, err := decodeURL(parts[0]) + if err != nil { + return c, errors.New("malformed grant header") + } + var h grantHeader + if err := DecodeStrict(hb, &h); err != nil { + return c, errors.New("malformed grant header") + } + if h.Algorithm != grantAlgorithm || h.Type != "MECATL-GRANT" || h.KeyID == "" { + return c, errors.New("unsupported grant header") + } + if _, ok := v.RevokedKeys[h.KeyID]; ok { + return c, errors.New("grant key revoked") + } + key, ok := v.Keys[h.KeyID] + if !ok || len(key) != ed25519.PublicKeySize { + return c, errors.New("unknown grant key") + } + sig, err := decodeURL(parts[2]) + if err != nil || !ed25519.Verify(key, []byte(parts[0]+"."+parts[1]), sig) { + return c, errors.New("invalid grant signature") + } + pb, err := decodeURL(parts[1]) + if err != nil { + return c, errors.New("malformed grant claims") + } + if err := DecodeStrict(pb, &c); err != nil { + return c, errors.New("malformed grant claims") + } + c.KeyID = h.KeyID + if err := validateClaims(c); err != nil { + return c, err + } + now := time.Now().UTC() + if v.Now != nil { + now = v.Now().UTC() + } + if c.Issuer != v.Issuer || c.Audience != v.Audience { + return c, errors.New("grant issuer or audience mismatch") + } + if now.Before(c.NotBefore) { + return c, ErrGrantNotYetValid + } + if !now.Before(c.ExpiresAt) { + return c, ErrGrantExpired + } + maxLifetime := v.MaxLifetime + if maxLifetime <= 0 { + maxLifetime = 5 * time.Minute + } + if c.ExpiresAt.Sub(c.NotBefore) > maxLifetime { + return c, errors.New("grant lifetime exceeds limit") + } + if _, ok := v.RevokedNonces[c.Nonce]; ok { + return c, errors.New("grant revoked") + } + if c.Client != e.Client || c.OwnerHash != e.OwnerHash || c.BindingID != e.BindingID || c.RunID != e.RunID || c.ClaimID != e.ClaimID || c.Environment != e.Environment || c.Epoch != e.Epoch || c.GrantGeneration != e.GrantGeneration { + return c, errors.New("grant binding mismatch") + } + for _, op := range c.Operations { + if op == e.Operation { + return c, nil + } + } + return c, errors.New("grant does not authorize operation") +} + +//nolint:gocyclo // Complete fail-closed claim validation is deliberately linear. +func validateClaims(c GrantClaims) error { + if c.KeyID == "" || c.Issuer == "" || c.Audience == "" || c.Client == "" || c.OwnerHash == "" || c.BindingID == "" || c.RunID == "" || c.ClaimID == "" || c.Environment.ID == "" || c.Environment.Revision == "" || c.Epoch == 0 || c.GrantGeneration == 0 || c.Nonce == "" || c.NotBefore.IsZero() || c.ExpiresAt.IsZero() || !c.ExpiresAt.After(c.NotBefore) { + return errors.New("incomplete grant claims") + } + if len(c.Operations) == 0 || len(c.Operations) > 32 { + return errors.New("invalid grant operations") + } + seen := map[Operation]struct{}{} + for _, op := range c.Operations { + if !op.Valid() { + return fmt.Errorf("invalid grant operation %q", op) + } + if _, ok := seen[op]; ok { + return errors.New("duplicate grant operation") + } + seen[op] = struct{}{} + } + return nil +} +func rawURL(b []byte) string { return base64.RawURLEncoding.EncodeToString(b) } +func decodeURL(s string) ([]byte, error) { return base64.RawURLEncoding.Strict().DecodeString(s) } diff --git a/internal/executionenv/protocol_test.go b/internal/executionenv/protocol_test.go new file mode 100644 index 0000000000..862780f625 --- /dev/null +++ b/internal/executionenv/protocol_test.go @@ -0,0 +1,67 @@ +package executionenv + +import ( + "crypto/ed25519" + "crypto/rand" + "testing" + "time" +) + +func TestGrantRoundTripAndExactBindings(t *testing.T) { + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal(err) + } + now := time.Unix(1_800_000_000, 0).UTC() + claims := GrantClaims{KeyID: "k1", Issuer: "execution-provider", Audience: "mecatl-execution", Client: "spiffe://cluster/ns/mecak8s", OwnerHash: "owner", BindingID: "binding", RunID: "run", ClaimID: "claim", Environment: EnvironmentRef{ID: "env", Revision: "rev"}, Epoch: 7, GrantGeneration: 1, Operations: []Operation{OpFileRead}, NotBefore: now.Add(-time.Second), ExpiresAt: now.Add(time.Minute), Nonce: "n1"} + token, err := SignGrant(priv, claims) + if err != nil { + t.Fatal(err) + } + v := GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: claims.Issuer, Audience: claims.Audience, MaxLifetime: 5 * time.Minute, Now: func() time.Time { return now }} + if _, err := v.Verify(token, GrantExpectation{Client: claims.Client, OwnerHash: "owner", BindingID: claims.BindingID, RunID: claims.RunID, ClaimID: claims.ClaimID, Environment: claims.Environment, Epoch: 7, GrantGeneration: claims.GrantGeneration, Operation: OpFileRead}); err != nil { + t.Fatalf("verify: %v", err) + } + for name, mutate := range map[string]func(*GrantExpectation){ + "client": func(e *GrantExpectation) { e.Client = "spiffe://other" }, + "owner": func(e *GrantExpectation) { e.OwnerHash = "other" }, + "binding": func(e *GrantExpectation) { e.BindingID = "other" }, + "revision": func(e *GrantExpectation) { e.Environment.Revision = "other" }, + "epoch": func(e *GrantExpectation) { e.Epoch++ }, + "operation": func(e *GrantExpectation) { e.Operation = OpFileReplace }, + } { + t.Run(name, func(t *testing.T) { + e := GrantExpectation{Client: claims.Client, OwnerHash: claims.OwnerHash, BindingID: claims.BindingID, RunID: claims.RunID, ClaimID: claims.ClaimID, Environment: claims.Environment, Epoch: claims.Epoch, GrantGeneration: claims.GrantGeneration, Operation: OpFileRead} + mutate(&e) + if _, err := v.Verify(token, e); err == nil { + t.Fatal("expected rejection") + } + }) + } +} + +func TestGrantRejectsExpiryRevocationAndUnknownFields(t *testing.T) { + pub, priv, _ := ed25519.GenerateKey(rand.Reader) + now := time.Unix(1_800_000_000, 0).UTC() + c := GrantClaims{KeyID: "k1", Issuer: "i", Audience: "a", Client: "spiffe://c", OwnerHash: "o", BindingID: "b", RunID: "run", ClaimID: "claim", Environment: EnvironmentRef{ID: "e", Revision: "r"}, Epoch: 1, GrantGeneration: 1, Operations: []Operation{OpAttach}, NotBefore: now.Add(-time.Minute), ExpiresAt: now.Add(-time.Second), Nonce: "n"} + tok, _ := SignGrant(priv, c) + v := GrantVerifier{Keys: map[string]ed25519.PublicKey{"k1": pub}, Issuer: "i", Audience: "a", Now: func() time.Time { return now }} + if _, err := v.Verify(tok, GrantExpectation{Client: c.Client, OwnerHash: c.OwnerHash, BindingID: c.BindingID, RunID: c.RunID, ClaimID: c.ClaimID, Environment: c.Environment, Epoch: 1, GrantGeneration: c.GrantGeneration, Operation: OpAttach}); err == nil { + t.Fatal("expired grant accepted") + } + c.ExpiresAt = now.Add(time.Minute) + tok, _ = SignGrant(priv, c) + v.RevokedNonces = map[string]struct{}{"n": {}} + if _, err := v.Verify(tok, GrantExpectation{Client: c.Client, OwnerHash: c.OwnerHash, BindingID: c.BindingID, RunID: c.RunID, ClaimID: c.ClaimID, Environment: c.Environment, Epoch: 1, GrantGeneration: c.GrantGeneration, Operation: OpAttach}); err == nil { + t.Fatal("revoked nonce accepted") + } +} + +func TestExecutorStdinDecodeStrictRejectsUnknownTrailingAndOversize(t *testing.T) { + var req ExecutorRequest + for _, body := range []string{`{"operation":"file.read","extra":true}`, `{"operation":"file.read"}{}`, `{"path":"` + string(make([]byte, MaxJSONBody)) + `"}`} { + if err := DecodeStrict([]byte(body), &req); err == nil { + t.Fatalf("accepted invalid body") + } + } +} diff --git a/internal/executionenv/types.go b/internal/executionenv/types.go new file mode 100644 index 0000000000..b451344cf1 --- /dev/null +++ b/internal/executionenv/types.go @@ -0,0 +1,325 @@ +// Package executionenv defines the private, versioned host-side execution-provider protocol. +// It is intentionally not part of the public Harness API. +// +//nolint:revive // Private protocol types are exported only across internal adapter packages. +package executionenv + +import "time" + +// Private protocol bounds. +const ( + ProtocolVersion = "execution-grpc/1" + MaxMessageBytes = 8 << 20 + MaxJSONBody = 8 << 20 // credential-free provider-to-workload stdin framing only + MaxFileBytes = 5 << 20 + MaxCommandBytes = 1 << 20 + MaxPathBytes = 4096 + MaxIdentityBytes = 1024 + MaxBindingBytes = 253 + MaxGrantBytes = 16 << 10 + MaxListEntries = 10_000 +) + +// Operation identifies one provider or workload-helper operation. +type Operation string + +// Supported private-protocol operations. +const ( + OpAttach Operation = "attach" + OpAcquireRun Operation = "run.acquire" + OpRenewRun Operation = "run.renew" + OpReleaseRun Operation = "run.release" + OpReferenceCommit Operation = "reference.commit" + OpReferenceAbort Operation = "reference.abort" + OpReferenceReserve Operation = "reference.reserve" + OpReferenceDeletePrepare Operation = "reference.delete.prepare" + OpReferenceDeleteConfirm Operation = "reference.delete.confirm" + OpReferenceDeleteCancel Operation = "reference.delete.cancel" + OpReferenceRelease Operation = "reference.release" + OpRetire Operation = "retire" + OpFileRead Operation = "file.read" + OpFileResolveAuthority Operation = "file.resolve_authority" + OpFileStat Operation = "file.stat" + OpFileCreate Operation = "file.create" + OpFileReplace Operation = "file.replace" + OpFileList Operation = "file.list" + OpFileRemove Operation = "file.remove" + OpFileRename Operation = "file.rename" + OpFileCopy Operation = "file.copy" + OpFileGlob Operation = "file.glob" + OpFileGrep Operation = "file.grep" + OpCommandStart Operation = "command.start" + OpCommandStatus Operation = "command.status" + OpCommandCancel Operation = "command.cancel" + OpCommandStream Operation = "command.stream" +) + +// Valid reports whether the operation belongs to the closed protocol vocabulary. +func (o Operation) Valid() bool { + switch o { + case OpAttach, OpAcquireRun, OpRenewRun, OpReleaseRun, OpReferenceCommit, OpReferenceAbort, OpReferenceReserve, OpReferenceDeletePrepare, OpReferenceDeleteConfirm, OpReferenceDeleteCancel, OpReferenceRelease, OpRetire, OpFileRead, OpFileResolveAuthority, OpFileStat, OpFileCreate, OpFileReplace, OpFileList, OpFileRemove, OpFileRename, OpFileCopy, OpFileGlob, OpFileGrep, OpCommandStart, OpCommandStatus, OpCommandCancel, OpCommandStream: + return true + } + return false +} + +// EnvironmentRef is the provider-private durable environment identity. +type EnvironmentRef struct { + ID string `json:"id"` + Revision string `json:"revision"` +} + +// Owner is an authenticated principal attestation. +type Owner struct { + Issuer string `json:"issuer"` + Subject string `json:"subject"` +} + +// RequestContext carries the exact authorization binding for an operation. +type RequestContext struct { + Environment EnvironmentRef `json:"environment"` + Owner Owner `json:"owner"` + BindingID string `json:"binding_id"` + RunID string `json:"run_id"` + ClaimID string `json:"claim_id"` + Epoch uint64 `json:"epoch"` + GrantGeneration uint64 `json:"grant_generation"` + Grant string `json:"grant"` +} + +// ValidateProfileRequest selects an operator-defined profile for validation. +type ValidateProfileRequest struct { + Profile string `json:"profile"` +} + +// ValidateProfileResponse reports immutable profile capabilities and bounds. +type ValidateProfileResponse struct { + Profile string `json:"profile"` + Digest string `json:"digest"` + Capabilities []string `json:"capabilities"` + MaxFileBytes int64 `json:"max_file_bytes"` + MaxCommandBytes int64 `json:"max_command_bytes"` + MaxCommandDurationMillis int64 `json:"max_command_duration_ms"` +} + +// EnsureEnvironmentRequest requests an idempotent environment allocation. +type EnsureEnvironmentRequest struct { + BindingID string `json:"binding_id"` + Profile string `json:"profile"` + Owner Owner `json:"owner"` +} + +// EnsureEnvironmentResponse returns allocation identity, readiness, and a short-lived grant. +type EnsureEnvironmentResponse struct { + Environment EnvironmentRef `json:"environment"` + Epoch uint64 `json:"epoch"` + Ready bool `json:"ready"` + GrantGeneration uint64 `json:"grant_generation"` + Grant string `json:"grant"` + GrantExpiresAt time.Time `json:"grant_expires_at"` +} + +// AttachEnvironmentRequest requests exact reattachment to an existing environment. +type AttachEnvironmentRequest struct { + Context RequestContext `json:"context"` + Purpose string `json:"purpose"` +} + +// PurposeSession is the supported attachment purpose. +const PurposeSession = "session" + +// AttachEnvironmentResponse returns exact attachment state and a refreshed grant. +type AttachEnvironmentResponse struct { + Environment EnvironmentRef `json:"environment"` + Epoch uint64 `json:"epoch"` + Ready bool `json:"ready"` + GrantGeneration uint64 `json:"grant_generation"` + Grant string `json:"grant"` + GrantExpiresAt time.Time `json:"grant_expires_at"` +} + +const ( + MinRunTTL = 30 * time.Second + MaxRunTTL = 5 * time.Minute + DefaultRunTTL = time.Minute + DefaultRenewInterval = 20 * time.Second +) + +type RunClaimRequest struct { + Environment EnvironmentRef + Owner Owner + BindingID string + RunID string + ClaimID string + Epoch uint64 + GrantGeneration uint64 + OperationID string + TTL time.Duration +} + +type RunClaim struct { + Environment EnvironmentRef + BindingID string + RunID string + ClaimID string + Epoch uint64 + GrantGeneration uint64 + Grant string + ExpiresAt time.Time +} + +type ReferenceState string + +const ( + ReferencePendingCreate ReferenceState = "PendingCreate" + ReferencePublished ReferenceState = "Published" + ReferencePendingDelete ReferenceState = "PendingDelete" +) + +type ReferenceRequest struct { + Environment EnvironmentRef + Owner Owner + BindingID string + SourceBindingID string + OperationID string +} + +type ReferenceIntent struct { + Environment EnvironmentRef + Owner Owner + BindingID string + State ReferenceState + OperationID string + SourceBindingID string + CreatedAt time.Time +} + +// ReferenceReleaseRequest releases one durable binding reference. +type ReferenceReleaseRequest struct { + Context RequestContext `json:"context"` +} + +// RetireEnvironmentRequest requests controlled retirement without PVC deletion. +type RetireEnvironmentRequest struct { + Environment EnvironmentRef `json:"environment"` + Owner Owner `json:"owner"` + ExpectedEpoch uint64 `json:"expected_execution_epoch"` + ExpectedPodUID string `json:"expected_pod_uid"` + ExpectedPVCUID string `json:"expected_pvc_uid"` + OperationID string `json:"operation_id"` +} + +// EmptyResponse is the successful response for operations without a payload. +type EmptyResponse struct{} + +// FileRequest carries one bounded, authorized filesystem operation. +type FileRequest struct { + Context RequestContext `json:"context"` + Operation Operation `json:"operation"` + Path string `json:"path"` + Destination string `json:"destination,omitempty"` + Pattern string `json:"pattern,omitempty"` + Data []byte `json:"data,omitempty"` + Version string `json:"version,omitempty"` + Limit int `json:"limit,omitempty"` +} + +// FileInfo is bounded filesystem metadata returned by the helper. +type FileInfo struct { + Name string `json:"name"` + Size int64 `json:"size"` + Mode uint32 `json:"mode"` + ModTime time.Time `json:"mod_time"` + IsDir bool `json:"is_dir"` +} + +// GrepMatch is one bounded textual grep result. +type GrepMatch struct { + Path string `json:"path"` + Line int `json:"line"` + Text string `json:"text"` +} + +// FileResponse carries the result of one filesystem operation. +type FileResponse struct { + Data []byte `json:"data,omitempty"` + Version string `json:"version,omitempty"` + Info *FileInfo `json:"info,omitempty"` + Entries []FileInfo `json:"entries,omitempty"` + Paths []string `json:"paths,omitempty"` + Matches []GrepMatch `json:"matches,omitempty"` + AuthorityTarget string `json:"authority_target,omitempty"` + AuthorityWorkspace string `json:"authority_workspace,omitempty"` +} + +// CommandStartRequest starts one bounded foreground shell command. +type CommandStartRequest struct { + Context RequestContext `json:"context"` + Command string `json:"command"` + TimeoutMillis int64 `json:"timeout_ms,omitempty"` +} + +// CommandStartResponse returns the command identity and terminal result. +type CommandStartResponse struct { + CommandID string `json:"command_id"` + State CommandState `json:"state"` + Result CommandStatusResponse `json:"result"` +} + +// CommandState classifies command lifecycle and fencing outcomes. +type CommandState string + +// Command states. +const ( + CommandRunning CommandState = "running" + CommandSucceeded CommandState = "succeeded" + CommandFailed CommandState = "failed" + CommandCancelled CommandState = "cancelled" + CommandFenceUnknown CommandState = "fence_unknown" +) + +// CommandQueryRequest identifies one command for status or cancellation. +type CommandQueryRequest struct { + Context RequestContext `json:"context"` + CommandID string `json:"command_id"` + Offset int64 `json:"offset,omitempty"` +} + +// CommandStatusResponse carries bounded output and terminal proof. +type CommandStatusResponse struct { + CommandID string `json:"command_id"` + State CommandState `json:"state"` + ExitCode int `json:"exit_code,omitempty"` + Stdout []byte `json:"stdout,omitempty"` + Stderr []byte `json:"stderr,omitempty"` + NextOffset int64 `json:"next_offset,omitempty"` + Truncated bool `json:"truncated,omitempty"` + TerminalReceipt string `json:"terminal_receipt,omitempty"` +} + +// ExecutorRequest is the credential-free provider-to-workload stdin protocol. +// It deliberately cannot select an environment, root, Pod, image, argv, or process environment. +type ExecutorRequest struct { + Operation Operation `json:"operation"` + Path string `json:"path,omitempty"` + Destination string `json:"destination,omitempty"` + Pattern string `json:"pattern,omitempty"` + Data []byte `json:"data,omitempty"` + Version string `json:"version,omitempty"` + Limit int `json:"limit,omitempty"` + Command string `json:"command,omitempty"` + CommandID string `json:"command_id,omitempty"` + TimeoutMillis int64 `json:"timeout_ms,omitempty"` +} + +// ExecutorResponse carries one credential-free helper result. +type ExecutorResponse struct { + FileResponse + Command *CommandStatusResponse `json:"command,omitempty"` +} + +// ExecutorEnvelope provides a terminal, credential-free result over pods/exec. +type ExecutorEnvelope struct { + Response *ExecutorResponse `json:"response,omitempty"` + Error *Error `json:"error,omitempty"` +} diff --git a/internal/executionexecutor/command_linux.go b/internal/executionexecutor/command_linux.go new file mode 100644 index 0000000000..8d431da744 --- /dev/null +++ b/internal/executionexecutor/command_linux.go @@ -0,0 +1,169 @@ +//go:build linux + +package executionexecutor + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "sync/atomic" + "syscall" + "time" + + "golang.org/x/sys/unix" +) + +const descendantCleanupTimeout = 2 * time.Second + +var subreaperEnabled atomic.Bool + +type commandResult struct { + stdout []byte + stderr []byte + exitCode int + truncated bool +} + +// EnableCommandExecution makes this process a Linux child subreaper. It must be +// called by the dedicated executor helper before it starts any other process. +func EnableCommandExecution() error { + if err := unix.Prctl(unix.PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0); err != nil { + return fmt.Errorf("enable executor child subreaper: %w", err) + } + subreaperEnabled.Store(true) + return nil +} + +func runIsolatedCommand(ctx context.Context, dir, command string, outputLimit int) (commandResult, bool, error) { + if !subreaperEnabled.Load() { + return commandResult{}, false, errors.New("command execution requires the dedicated subreaper helper") + } + var stdout, stderr boundedWriter + stdout.limit = outputLimit + stderr.limit = outputLimit + cmd := exec.Command("/bin/sh", "-c", command) // #nosec G204 -- the command is the authenticated tool payload. + cmd.Dir = dir + cmd.Env = []string{"HOME=/workspace", "PATH=/usr/local/go/bin:/usr/local/bin:/usr/bin:/bin", "TMPDIR=/tmp", "GOTMPDIR=/tmp"} + cmd.Stdout = &stdout + cmd.Stderr = &stderr + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + cmd.WaitDelay = 250 * time.Millisecond + if err := cmd.Start(); err != nil { + return commandResult{}, true, err + } + + waited := make(chan error, 1) + go func() { waited <- cmd.Wait() }() + var waitErr error + select { + case waitErr = <-waited: + case <-ctx.Done(): + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + _ = cmd.Process.Kill() + waitErr = <-waited + } + + // The shell may have exited successfully while background descendants remain. + // Kill its original process group, then repeatedly kill every process reparented + // to this dedicated subreaper. ECHILD is the only clean-completion proof. + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + cleanupCtx, cancel := context.WithTimeout(context.Background(), descendantCleanupTimeout) + defer cancel() + proven := reapAllChildren(cleanupCtx) + + result := commandResult{stdout: stdout.Bytes(), stderr: stderr.Bytes(), exitCode: exitCode(waitErr), truncated: stdout.truncated || stderr.truncated} + if ctx.Err() != nil { + return result, proven, ctx.Err() + } + return result, proven, waitErr +} + +func reapAllChildren(ctx context.Context) bool { + for { + for _, pid := range directChildren() { + _ = syscall.Kill(pid, syscall.SIGKILL) + } + for { + var status syscall.WaitStatus + pid, err := syscall.Wait4(-1, &status, syscall.WNOHANG, nil) + switch { + case errors.Is(err, syscall.ECHILD): + return true + case err != nil: + return false + case pid <= 0: + goto wait + } + } + wait: + select { + case <-ctx.Done(): + return false + case <-time.After(5 * time.Millisecond): + } + } +} + +func directChildren() []int { + tasks, err := os.ReadDir("/proc/self/task") + if err != nil { + return nil + } + seen := make(map[int]struct{}) + for _, task := range tasks { + data, err := os.ReadFile(filepath.Join("/proc/self/task", task.Name(), "children")) + if err != nil { + continue + } + for _, field := range strings.Fields(string(data)) { + pid, err := strconv.Atoi(field) + if err == nil && pid > 0 { + seen[pid] = struct{}{} + } + } + } + out := make([]int, 0, len(seen)) + for pid := range seen { + out = append(out, pid) + } + return out +} + +func exitCode(err error) int { + if err == nil { + return 0 + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode() + } + return -1 +} + +type boundedWriter struct { + buf bytes.Buffer + limit int + truncated bool +} + +func (w *boundedWriter) Write(p []byte) (int, error) { + original := len(p) + if remaining := w.limit - w.buf.Len(); remaining > 0 { + if len(p) > remaining { + p = p[:remaining] + w.truncated = true + } + _, _ = w.buf.Write(p) + } else if original > 0 { + w.truncated = true + } + return original, nil +} + +func (w *boundedWriter) Bytes() []byte { return append([]byte(nil), w.buf.Bytes()...) } diff --git a/internal/executionexecutor/command_unsupported.go b/internal/executionexecutor/command_unsupported.go new file mode 100644 index 0000000000..177168a4fb --- /dev/null +++ b/internal/executionexecutor/command_unsupported.go @@ -0,0 +1,24 @@ +//go:build !linux + +package executionexecutor + +import ( + "context" + "errors" +) + +type commandResult struct { + stdout []byte + stderr []byte + exitCode int + truncated bool +} + +// EnableCommandExecution reports that the workload helper is Linux-only. +func EnableCommandExecution() error { + return errors.New("command execution requires Linux child-subreaper support") +} + +func runIsolatedCommand(context.Context, string, string, int) (commandResult, bool, error) { + return commandResult{}, false, errors.New("command execution requires Linux child-subreaper support") +} diff --git a/internal/executionexecutor/confinement_test.go b/internal/executionexecutor/confinement_test.go new file mode 100644 index 0000000000..e7db9a73b5 --- /dev/null +++ b/internal/executionexecutor/confinement_test.go @@ -0,0 +1,76 @@ +package executionexecutor + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestFileOperationsNeverTraverseExternalSymlink(t *testing.T) { + root, outside := t.TempDir(), t.TempDir() + sentinel := filepath.Join(outside, "file") + if err := os.WriteFile(sentinel, []byte("outside-secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil { + t.Fatal(err) + } + x, err := New(root, Limits{}) + if err != nil { + t.Fatal(err) + } + defer x.Close() + for _, op := range []executionenv.Operation{executionenv.OpFileRead, executionenv.OpFileResolveAuthority, executionenv.OpFileStat, executionenv.OpFileCreate, executionenv.OpFileReplace, executionenv.OpFileList, executionenv.OpFileRemove, executionenv.OpFileRename, executionenv.OpFileCopy} { + t.Run(string(op), func(t *testing.T) { + req := executionenv.ExecutorRequest{Operation: op, Path: "escape/file", Destination: "destination", Version: "version"} + if op == executionenv.OpFileList { + req.Path = "escape" + } + if op == executionenv.OpFileCreate { + req.Path = "escape/new" + } + if _, err := x.Execute(t.Context(), req); err == nil { + t.Fatal("external symlink traversal accepted") + } + if op == executionenv.OpFileCopy || op == executionenv.OpFileRename { + if _, err := x.Execute(t.Context(), executionenv.ExecutorRequest{Operation: executionenv.OpFileCreate, Path: string(op), Data: []byte("local")}); err != nil { + t.Fatal(err) + } + req.Path = string(op) + req.Destination = "escape/new" + if _, err := x.Execute(t.Context(), req); err == nil { + t.Fatal("external destination accepted") + } + } + }) + } + for _, op := range []executionenv.Operation{executionenv.OpFileGlob, executionenv.OpFileGrep} { + req := executionenv.ExecutorRequest{Operation: op, Pattern: "**/*", Limit: 100} + if op == executionenv.OpFileGrep { + req.Path = "**/*" + req.Pattern = "outside-secret" + } + result, err := x.Execute(t.Context(), req) + if err != nil { + continue + } // Honest confinement errors are also safe. + for _, path := range result.Paths { + if strings.HasPrefix(path, "escape/") { + t.Fatalf("glob escaped: %q", path) + } + } + if len(result.Matches) != 0 { + t.Fatalf("grep leaked external content: %+v", result.Matches) + } + } + data, err := os.ReadFile(sentinel) + if err != nil || string(data) != "outside-secret" { + t.Fatalf("external file mutated: %q %v", data, err) + } + if _, err := os.Stat(filepath.Join(outside, "new")); !os.IsNotExist(err) { + t.Fatalf("external file created: %v", err) + } +} diff --git a/internal/executionexecutor/executor.go b/internal/executionexecutor/executor.go new file mode 100644 index 0000000000..1c85956a53 --- /dev/null +++ b/internal/executionexecutor/executor.go @@ -0,0 +1,246 @@ +// Package executionexecutor is the fixed, credential-free workload helper. +package executionexecutor + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io/fs" + "strings" + "time" + "unicode/utf8" + + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/adapter/osfs" + "github.com/stacklok/mecatl/internal/executionenv" +) + +// Limits bounds one executor operation. +type Limits struct { + MaxFileBytes int + MaxEntries int + CommandTimeout time.Duration +} + +// Executor serves one credential-free workload-helper process. +type Executor struct { + workspace *osfs.Workspace + limits Limits +} + +// New constructs an executor confined to root. +func New(root string, limits Limits) (*Executor, error) { + if limits.MaxFileBytes <= 0 || limits.MaxFileBytes > executionenv.MaxFileBytes { + limits.MaxFileBytes = executionenv.MaxFileBytes + } + if limits.MaxEntries <= 0 || limits.MaxEntries > executionenv.MaxListEntries { + limits.MaxEntries = executionenv.MaxListEntries + } + if limits.CommandTimeout <= 0 { + limits.CommandTimeout = 30 * time.Second + } + ws, err := osfs.NewWorkspace(root) + if err != nil { + return nil, err + } + return &Executor{workspace: ws, limits: limits}, nil +} + +// Close releases executor resources. +func (*Executor) Close() error { return nil } + +// Execute performs one bounded executor operation. +func (e *Executor) Execute(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { //nolint:gocyclo // The operation switch keeps validation and dispatch in one boundary. + if !q.Operation.Valid() { + return executionenv.ExecutorResponse{}, bad("unsupported executor operation") + } + if len(q.Data) > e.limits.MaxFileBytes || len(q.Path) > executionenv.MaxPathBytes || len(q.Destination) > executionenv.MaxPathBytes || len(q.Pattern) > executionenv.MaxPathBytes || q.Limit < 0 || q.Limit > executionenv.MaxListEntries { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "executor request exceeds configured bounds") + } + switch q.Operation { + case executionenv.OpFileRead: + return e.read(ctx, q) + case executionenv.OpFileResolveAuthority: + target, workspace, err := e.workspace.AuthorityResourcePath(q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{AuthorityTarget: target, AuthorityWorkspace: workspace}}, nil + case executionenv.OpFileStat: + return e.stat(ctx, q) + case executionenv.OpFileCreate: + return e.create(ctx, q) + case executionenv.OpFileReplace: + return e.replace(ctx, q) + case executionenv.OpFileList: + return e.list(ctx, q) + case executionenv.OpFileRemove: + return executionenv.ExecutorResponse{}, mapErr(e.workspace.Remove(ctx, q.Path)) + case executionenv.OpFileRename: + return executionenv.ExecutorResponse{}, mapErr(e.workspace.Rename(ctx, q.Path, q.Destination)) + case executionenv.OpFileCopy: + v, err := e.workspace.CopyFile(ctx, q.Path, q.Destination) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + s, _ := tool.EncodeFileVersion(v) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Version: s}}, nil + case executionenv.OpFileGlob: + paths, err := e.workspace.Glob(ctx, q.Pattern) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + if len(paths) > e.resultLimit(q.Limit) { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "glob result exceeds configured limit") + } + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Paths: paths}}, nil + case executionenv.OpFileGrep: + matches, err := e.workspace.Grep(ctx, q.Pattern, q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + if len(matches) > e.resultLimit(q.Limit) { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "grep result exceeds configured limit") + } + out := make([]executionenv.GrepMatch, len(matches)) + for i, m := range matches { + out[i] = executionenv.GrepMatch{Path: m.Path, Line: m.Line, Text: strings.ToValidUTF8(m.Text, "�")} + } + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Matches: out}}, nil + case executionenv.OpCommandStart: + return e.command(ctx, q) + default: + return executionenv.ExecutorResponse{}, bad("operation is handled by provider, not workload") + } +} +func (e *Executor) read(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + b, v, err := e.workspace.ReadVersion(ctx, q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + if len(b) > e.limits.MaxFileBytes { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "file exceeds configured limit") + } + s, _ := tool.EncodeFileVersion(v) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Data: b, Version: s}}, nil +} +func (e *Executor) stat(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + i, err := e.workspace.Stat(ctx, q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + x := convertInfo(i) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Info: &x}}, nil +} +func (e *Executor) create(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + v, err := e.workspace.CreateFile(ctx, q.Path, q.Data) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + s, _ := tool.EncodeFileVersion(v) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Version: s}}, nil +} +func (e *Executor) replace(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + v, err := e.workspace.ReplaceFile(ctx, q.Path, tool.DecodeFileVersion(q.Version), q.Data) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + s, _ := tool.EncodeFileVersion(v) + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Version: s}}, nil +} +func (e *Executor) list(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + items, err := e.workspace.ReadDir(ctx, q.Path) + if err != nil { + return executionenv.ExecutorResponse{}, mapErr(err) + } + if len(items) > e.resultLimit(q.Limit) { + return executionenv.ExecutorResponse{}, coded(executionenv.CodeResourceExhausted, "directory exceeds configured limit") + } + out := make([]executionenv.FileInfo, len(items)) + for i, v := range items { + out[i] = convertInfo(v) + } + return executionenv.ExecutorResponse{FileResponse: executionenv.FileResponse{Entries: out}}, nil +} +func (e *Executor) command(ctx context.Context, q executionenv.ExecutorRequest) (executionenv.ExecutorResponse, error) { + if q.CommandID == "" || len(q.Command) == 0 || len(q.Command) > executionenv.MaxCommandBytes { + return executionenv.ExecutorResponse{}, bad("command id and bounded command are required") + } + d := e.limits.CommandTimeout + if q.TimeoutMillis > 0 { + requested := time.Duration(q.TimeoutMillis) * time.Millisecond + if requested < d { + d = requested + } + } + runCtx, cancel := context.WithTimeout(ctx, d) + defer cancel() + result, cleanupProven, err := runIsolatedCommand(runCtx, e.workspace.Root(), q.Command, e.limits.MaxFileBytes) + stdout, stderr, truncated := capStreams(result.stdout, result.stderr, e.limits.MaxFileBytes) + truncated = truncated || result.truncated + if !cleanupProven { + return executionenv.ExecutorResponse{Command: &executionenv.CommandStatusResponse{CommandID: q.CommandID, State: executionenv.CommandFenceUnknown, Stdout: stdout, Stderr: stderr, Truncated: truncated}}, nil + } + state := executionenv.CommandSucceeded + if errors.Is(runCtx.Err(), context.Canceled) || errors.Is(runCtx.Err(), context.DeadlineExceeded) { + state = executionenv.CommandCancelled + } else if err != nil || result.exitCode != 0 { + state = executionenv.CommandFailed + } + receiptInput := append(append([]byte(q.CommandID+"\x00"), stdout...), 0) + receiptHash := sha256.Sum256(append(receiptInput, stderr...)) + return executionenv.ExecutorResponse{Command: &executionenv.CommandStatusResponse{CommandID: q.CommandID, State: state, ExitCode: result.exitCode, Stdout: stdout, Stderr: stderr, Truncated: truncated, TerminalReceipt: hex.EncodeToString(receiptHash[:])}}, nil +} +func convertInfo(i tool.FileInfo) executionenv.FileInfo { + return executionenv.FileInfo{Name: i.Name, Size: i.Size, Mode: uint32(i.Mode), ModTime: i.ModTime, IsDir: i.IsDir} +} +func (e *Executor) resultLimit(requested int) int { + if requested > 0 && requested < e.limits.MaxEntries { + return requested + } + return e.limits.MaxEntries +} +func capStreams(stdout, stderr []byte, n int) ([]byte, []byte, bool) { + var stdoutClipped, stderrClipped bool + stdout, stdoutClipped = capOutput(stdout, n) + remaining := n - len(stdout) + stderr, stderrClipped = capOutput(stderr, remaining) + return stdout, stderr, stdoutClipped || stderrClipped +} + +func capOutput(b []byte, n int) ([]byte, bool) { + b = []byte(strings.ToValidUTF8(string(b), "�")) + clipped := len(b) > n + if clipped { + b = b[:n] + for !utf8.Valid(b) { + b = b[:len(b)-1] + } + } + return b, clipped +} +func bad(msg string) error { return coded(executionenv.CodeInvalidArgument, msg) } +func coded(code executionenv.ErrorCode, msg string) error { + return &executionenv.Error{Code: code, Message: msg} +} +func mapErr(err error) error { + if err == nil { + return nil + } + var mismatch *tool.VersionMismatchError + switch { + case errors.As(err, &mismatch): + return coded(executionenv.CodeVersionMismatch, "file version mismatch") + case errors.Is(err, fs.ErrNotExist): + return coded(executionenv.CodeNotFound, "path not found") + case errors.Is(err, fs.ErrExist): + return coded(executionenv.CodeAlreadyExists, "destination already exists") + case errors.Is(err, tool.ErrDirectoryNotEmpty): + return coded(executionenv.CodeDirectoryNotEmpty, "directory is not empty") + case errors.Is(err, osfs.ErrPathEscape): + return coded(executionenv.CodePermissionDenied, "path is outside workspace") + default: + return err + } +} diff --git a/internal/executionexecutor/executor_test.go b/internal/executionexecutor/executor_test.go new file mode 100644 index 0000000000..b4739a61af --- /dev/null +++ b/internal/executionexecutor/executor_test.go @@ -0,0 +1,239 @@ +package executionexecutor + +import ( + "context" + "encoding/json" + "errors" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/stacklok/mecatl/engine/adapter/fstools" + "github.com/stacklok/mecatl/engine/adapter/memledger" + "github.com/stacklok/mecatl/engine/session" + "github.com/stacklok/mecatl/engine/tool" + "github.com/stacklok/mecatl/internal/executionenv" +) + +func TestFileOperationsAreConfinedAndVersioned(t *testing.T) { + root := t.TempDir() + x, err := New(root, Limits{MaxFileBytes: 1024, MaxEntries: 100, CommandTimeout: time.Second}) + if err != nil { + t.Fatal(err) + } + defer x.Close() + ctx := context.Background() + created, err := x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileCreate, Path: "a/file.txt", Data: []byte("one")}) + if err != nil { + t.Fatal(err) + } + read, err := x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileRead, Path: "a/file.txt"}) + if err != nil || string(read.Data) != "one" || read.Version != created.Version { + t.Fatalf("read=%+v err=%v", read, err) + } + resolved, err := x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileResolveAuthority, Path: "a/file.txt"}) + if err != nil || resolved.AuthorityWorkspace != root || resolved.AuthorityTarget != filepath.Join(root, "a/file.txt") { + t.Fatalf("resolved authority=%+v err=%v", resolved.FileResponse, err) + } + _, err = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileReplace, Path: "a/file.txt", Data: []byte("two"), Version: "wrong"}) + var pe *executionenv.Error + if !errors.As(err, &pe) || pe.Code != executionenv.CodeVersionMismatch { + t.Fatalf("expected version mismatch: %v", err) + } + _, err = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileRead, Path: "../../etc/passwd"}) + if err == nil { + t.Fatal("escape accepted") + } + outside := filepath.Join(root, "..", "outside") + if err := os.WriteFile(outside, []byte("secret"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(root, "escape")); err != nil { + t.Fatal(err) + } + _, err = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileRead, Path: "escape"}) + if err == nil || errors.Is(err, fs.ErrNotExist) { + t.Fatalf("symlink escape not distinctly rejected: %v", err) + } + _, err = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileResolveAuthority, Path: "escape"}) + var denied *executionenv.Error + if !errors.As(err, &denied) || denied.Code != executionenv.CodePermissionDenied { + t.Fatalf("authority resolver accepted symlink escape: %v", err) + } +} + +func TestRenameAndCopyNeverClobber(t *testing.T) { + x, _ := New(t.TempDir(), Limits{MaxFileBytes: 1024, MaxEntries: 100}) + defer x.Close() + ctx := context.Background() + _, _ = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileCreate, Path: "a", Data: []byte("a")}) + _, _ = x.Execute(ctx, executionenv.ExecutorRequest{Operation: executionenv.OpFileCreate, Path: "b", Data: []byte("b")}) + for _, op := range []executionenv.Operation{executionenv.OpFileCopy, executionenv.OpFileRename} { + if _, err := x.Execute(ctx, executionenv.ExecutorRequest{Operation: op, Path: "a", Destination: "b"}); err == nil { + t.Fatalf("%s clobbered", op) + } else { + var pe *executionenv.Error + if !errors.As(err, &pe) || pe.Code != executionenv.CodeAlreadyExists { + t.Fatalf("%s wrong error: %v", op, err) + } + } + } +} + +func TestForegroundCommandReturnsReceiptAndTimeoutTerminatesDescendants(t *testing.T) { + if testing.Short() { + t.Skip("subprocess regression test") + } + root := t.TempDir() + ok := runExecutorHelper(t, root, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "c1", Command: "printf hello", TimeoutMillis: 500}) + if ok.Command == nil || ok.Command.State != executionenv.CommandSucceeded || ok.Command.TerminalReceipt == "" || string(ok.Command.Stdout) != "hello" { + t.Fatalf("response=%+v", ok) + } + timed := runExecutorHelper(t, root, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "c2", Command: "sleep 10 & wait", TimeoutMillis: 20}) + if timed.Command == nil || timed.Command.State != executionenv.CommandCancelled || timed.Command.TerminalReceipt == "" { + t.Fatalf("response=%+v", timed) + } +} + +func TestSuccessfulDetachedChildIsKilledBeforeTerminalReceipt(t *testing.T) { + root := t.TempDir() + result := filepath.Join(root, "result") + response := runExecutorHelper(t, root, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "detached", Command: "nohup sh -c 'sleep 0.2; echo late > result' >/dev/null 2>&1 &", TimeoutMillis: 1000}) + if response.Command == nil || response.Command.State != executionenv.CommandSucceeded || response.Command.TerminalReceipt == "" { + t.Fatalf("response=%+v", response) + } + timer := time.NewTimer(400 * time.Millisecond) + defer timer.Stop() + <-timer.C + if _, err := os.Stat(result); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("detached writer survived terminal receipt: %v", err) + } +} + +func TestNativeShellCapsCombinedOutputAndRepairsUTF8(t *testing.T) { + for _, tc := range []struct { + name, command string + wantStdout, wantStderr int + }{ + {"stdout-exact-cap-repair", `i=0; while [ "$i" -lt 1020 ]; do printf x; i=$((i+1)); done; printf '\377abc'`, 1024, 0}, + {"stderr-exact-cap-repair", `i=0; while [ "$i" -lt 1020 ]; do printf x >&2; i=$((i+1)); done; printf '\377abc' >&2`, 0, 1024}, + {"combined-exact-cap-repair", `printf xxxx; i=0; while [ "$i" -lt 1016 ]; do printf x >&2; i=$((i+1)); done; printf '\377abc' >&2`, 4, 1020}, + {"stdout", `i=0; while [ "$i" -lt 2048 ]; do printf x; i=$((i+1)); done`, 1024, 0}, + {"stderr", `i=0; while [ "$i" -lt 2048 ]; do printf x >&2; i=$((i+1)); done`, 0, 1024}, + {"split-malformed", `printf '\377'; printf '\376' >&2; i=0; while [ "$i" -lt 700 ]; do printf x; printf y >&2; i=$((i+1)); done`, 703, 321}, + } { + t.Run(tc.name, func(t *testing.T) { + response := runExecutorHelper(t, t.TempDir(), executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "bounded", Command: tc.command}) + result := response.Command + if result == nil || result.State != executionenv.CommandSucceeded || result.TerminalReceipt == "" { + t.Fatalf("native command failed: %+v", response) + } + if !result.Truncated || len(result.Stdout) != tc.wantStdout || len(result.Stderr) != tc.wantStderr || len(result.Stdout)+len(result.Stderr) > 1024 { + t.Fatalf("stdout=%d stderr=%d truncated=%t", len(result.Stdout), len(result.Stderr), result.Truncated) + } + if !utf8.Valid(result.Stdout) || !utf8.Valid(result.Stderr) { + t.Fatal("native command result contains malformed UTF-8") + } + if strings.HasSuffix(tc.name, "exact-cap-repair") { + stream := result.Stdout + if tc.wantStderr != 0 { + stream = result.Stderr + } + if !strings.HasSuffix(string(stream), "�a") { + t.Fatal("repair-expansion clip did not retain the expected prefix") + } + } + if tc.name == "split-malformed" && (!strings.HasPrefix(string(result.Stdout), "�") || !strings.HasPrefix(string(result.Stderr), "�")) { + t.Fatal("malformed stream bytes were not repaired") + } + }) + } +} + +func TestNativeShellWriteInvalidatesRecordedRead(t *testing.T) { + root := t.TempDir() + x, err := New(root, Limits{}) + if err != nil { + t.Fatal(err) + } + defer x.Close() + env := tool.MustEnvironment(session.EnvironmentRef{Kind: "kubernetes", ID: "shell-cas", Revision: "v1"}, x.workspace, memledger.New(), nil) + if _, err := x.workspace.CreateFile(t.Context(), "file.txt", []byte("before")); err != nil { + t.Fatal(err) + } + read, err := (fstools.ReadTool{}).Execute(t.Context(), session.NewToolCall("read", "Read", json.RawMessage(`{"path":"file.txt"}`)), env) + if err != nil || read.IsError { + t.Fatalf("record read: %+v, %v", read, err) + } + response := runExecutorHelper(t, root, executionenv.ExecutorRequest{Operation: executionenv.OpCommandStart, CommandID: "write", Command: "printf shell-change > file.txt"}) + if response.Command == nil || response.Command.State != executionenv.CommandSucceeded { + t.Fatalf("Shell write failed: %+v", response) + } + edit, err := (fstools.EditTool{}).Execute(t.Context(), session.NewToolCall("edit", "Edit", json.RawMessage(`{"path":"file.txt","old_string":"shell-change","new_string":"lost"}`)), env) + if err != nil || !edit.IsError || !strings.Contains(edit.Content, "changed since") { + t.Fatalf("Edit must reject the stale recorded version, not an exact-match failure: %+v, %v", edit, err) + } + data, err := x.workspace.Read(t.Context(), "file.txt") + if err != nil || string(data) != "shell-change" { + t.Fatalf("Shell change overwritten: %q, %v", data, err) + } +} + +func runExecutorHelper(t *testing.T, root string, request executionenv.ExecutorRequest) executionenv.ExecutorResponse { + t.Helper() + cmd := exec.Command(os.Args[0], "-test.run=^TestExecutorCommandHelper$") + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + root, "MECATL_EXECUTOR_TEST_HELPER=1", "MECATL_EXECUTOR_TEST_ROOT=" + root} + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + if err := json.NewEncoder(stdin).Encode(request); err != nil { + t.Fatal(err) + } + _ = stdin.Close() + var response executionenv.ExecutorResponse + if err := json.NewDecoder(stdout).Decode(&response); err != nil { + t.Fatal(err) + } + if err := cmd.Wait(); err != nil { + t.Fatalf("executor helper: %v", err) + } + return response +} + +func TestExecutorCommandHelper(_ *testing.T) { + if os.Getenv("MECATL_EXECUTOR_TEST_HELPER") != "1" { + return + } + if err := EnableCommandExecution(); err != nil { + os.Exit(2) + } + x, err := New(os.Getenv("MECATL_EXECUTOR_TEST_ROOT"), Limits{MaxFileBytes: 1024, MaxEntries: 100, CommandTimeout: time.Second}) + if err != nil { + os.Exit(2) + } + var request executionenv.ExecutorRequest + if err := json.NewDecoder(os.Stdin).Decode(&request); err != nil { + os.Exit(2) + } + response, err := x.Execute(context.Background(), request) + if err != nil { + os.Exit(2) + } + if err := json.NewEncoder(os.Stdout).Encode(response); err != nil { + os.Exit(2) + } + os.Exit(0) +} diff --git a/sdk/typescript/src/gen/mecatl/v1/harness_pb.ts b/sdk/typescript/src/gen/mecatl/v1/harness_pb.ts index 1b4dac9451..68f4dbadd6 100644 --- a/sdk/typescript/src/gen/mecatl/v1/harness_pb.ts +++ b/sdk/typescript/src/gen/mecatl/v1/harness_pb.ts @@ -10947,11 +10947,13 @@ export const HarnessService: GenService<{ output: typeof ListAgentsResponseSchema; }, /** - * ListCommands returns slash commands discovered for one owned session's exact - * server-bound placement. It authorizes and reattaches that session before - * discovery and accepts no workspace/root input. It powers the client's command - * palette; command expansion remains a run-path concern. A no-FS session or a - * server with no command expander returns an empty list. + * ListCommands returns slash commands from the configured, resolved sources + * for one authorized, owned session. It runs independently of execution and + * accepts no client workspace/root input. A source that needs execution files + * acquires only that session's exact backend. It powers the client's command + * palette; command expansion remains a run-path concern. A server with no + * configured command sources returns an empty list; a no-FS session can retain + * independently configured sources. * * @generated from rpc mecatl.v1.HarnessService.ListCommands */ diff --git a/user-docs/building/deployment/mecak8s.md b/user-docs/building/deployment/mecak8s.md index ade4b1034a..29b454de2f 100644 --- a/user-docs/building/deployment/mecak8s.md +++ b/user-docs/building/deployment/mecak8s.md @@ -154,6 +154,488 @@ delegation cannot add filesystem access that the parent lacks. See [Execution environments](/features/execution-environments.md) for the shared placement, no-FS, child-environment, and reattachment model. +### Kubernetes execution provider + +**DRAFT candidate, not released or approved for production.** These instructions +apply to the optional native provider, not to the maturity of `mecak8s` itself. +Use an operator-controlled cluster. This slice supports foreground commands, +has no force-takeover recovery, and does not promise hostile multi-tenant +isolation. + +The optional execution provider runs as a separate service and controller with its +own `mecatl-execution` chart. `mecak8s` remains a client and receives no Pod, +PVC, custom-resource, or controller management permissions. + +The provider supplies run-wide ownership, transactional references, controlled +replacement and retirement, durable grant revocation, and reloadable TLS and +signing material. Production isolation requires a CNI that enforces NetworkPolicy +and a RuntimeClass that supplies the isolation promised by your platform. See +[Native Kubernetes lifecycle and retention](/features/execution-environments.md#native-kubernetes-lifecycle-and-retention) +for restart recovery, fencing, and placement-capacity behavior. + +Before installation, provide these values through your trusted image and Secret +delivery system: + +- Digest-pinned provider and workload images. The workload image must contain + `/mecatl-executor`, `/bin/sh`, and the toolchain used by foreground commands. +- A projected security Secret with the keys `grant-k1.pem`, `tls.crt`, `tls.key`, + and `clients.pem`. The Secret bytes remain externally managed. Mount rotation + uses the projected volume's `..data` link, without `subPath`. +- A `mecak8s` client mTLS Secret containing `ca.crt`, `tls.crt`, and `tls.key`. +- Explicit provider-client pod and namespace selectors, API-server CIDRs, and DNS + resolver CIDRs. + +The provider validates every configured RuntimeClass and StorageClass with +cluster-scoped `get` requests before it becomes ready. The chart grants those +requests only for the names present in `profiles`; it grants no cluster-wide list +or watch access. + +The following profile shows every required chart key. Save it as +`execution-values.yaml` and replace each placeholder: + +```yaml +fullnameOverride: mecatl-execution +provider: + image: @sha256: + imagePullPolicy: IfNotPresent + replicas: 2 + securitySecretName: + securityManifest: | + { + "version": 1, + "generation": 42, + "issuer": "https://execution.example.com", + "audience": "mecatl-execution", + "activeKeyID": "k1", + "grantTTL": "1m", + "clockSkew": "5s", + "keys": [{ + "id": "k1", + "version": 1, + "file": "grant-k1.pem", + "publicKeySHA256": "0000000000000000000000000000000000000000000000000000000000000000", + "activateAt": "2027-01-01T00:00:00Z", + "verifyUntil": "2027-01-02T00:00:00Z", + "state": "active" + }], + "tls": { + "certificateFile": "tls.crt", + "privateKeyFile": "tls.key", + "clientCAFile": "clients.pem" + }, + "clients": [{ + "uri": "spiffe://cluster.example.com/ns/mecatl/sa/mecak8s", + "mayAttestOwner": true, + "administrator": false + }, { + "uri": "spiffe://cluster.example.com/ns/mecatl/sa/execution-admin", + "mayAttestOwner": false, + "administrator": true, + "administratorFor": ["spiffe://cluster.example.com/ns/mecatl/sa/mecak8s"] + }] + } + clientIngressSelectors: + - namespaceLabels: {kubernetes.io/metadata.name: } + podLabels: {app.kubernetes.io/name: mecak8s} + apiServerCIDRs: [/32] + dnsCIDRs: [/32] + +service: + port: 8443 + +profiles: + coding: + image: @sha256: + storageClass: + storageSize: 2Gi + cpuRequest: 100m + memoryRequest: 128Mi + cpuLimit: "1" + memoryLimit: 1Gi + ephemeralStorageRequest: 64Mi + ephemeralStorageLimit: 1Gi + tmpSizeLimit: 256Mi + runtimeClassName: + maxFileBytes: 5242880 + maxCommandBytes: 1048576 + maxCommandDuration: 5m + maxEnvironments: 100 + +networkPolicy: + enabled: true + dnsPorts: [53] + apiServerPorts: [443] + workloadProfiles: {} # default deny; add explicit coding egress only if required + +resourceGovernance: + enabled: true + pods: "200" + persistentVolumeClaims: "100" + executionEnvironments: "100" + requestsCPU: "20" + requestsMemory: 40Gi + requestsStorage: 500Gi + requestsEphemeralStorage: 40Gi + limitsCPU: "40" + limitsMemory: 80Gi + limitsEphemeralStorage: 80Gi +``` + +Replace the all-zero `publicKeySHA256` with the lowercase hexadecimal SHA-256 +hash of the **raw 32-byte Ed25519 public key** corresponding to `grant-k1.pem`, +not the PEM text or DER encoding. Replace the sample activation +and verification dates with a current, reviewed rotation window. Increase +`generation` for every authority change, including a CA, client policy, +issuer/audience, key state, key window, or TLS identity change. + +Install the provider chart separately from `mecak8s`, before allocating any +execution environments. Helm 3.16 is the minimum supported version. Helm 4 uses +server-side apply by default, so each chart lifecycle command selects client-side +mode when needed. + +Use a namespace dedicated to this provider release: + +```sh +HELM_APPLY_MODE= +case "$(helm version --template '{{.Version}}')" in + v4.*) HELM_APPLY_MODE=--server-side=false ;; +esac +helm install mecatl-execution ./deploy/helm/mecatl-execution \ + --namespace \ + --values execution-values.yaml \ + $HELM_APPLY_MODE +kubectl rollout status deployment/mecatl-execution --namespace +``` + +Add the following block to the existing `mecak8s` values. The endpoint is a +`host:port` gRPC target (no URL scheme) and must use the provider certificate's +DNS identity. The private `mecatl.execution.v1.ExecutionProviderService` uses +protocol `execution-grpc/1` with mandatory mTLS. Keep OIDC caller authentication enabled; +the remote binding is scoped to the verified issuer and subject. + +```yaml +execution: + enabled: true + endpoint: mecatl-execution..svc:8443 + profile: coding + tlsSecret: + caKey: ca.crt + certKey: tls.crt + keyKey: tls.key +``` + +An enabled client conflicts with `workspace`, `redis.filesystem.enabled`, +Parallel, and Team. Remote sessions receive the filesystem tools and foreground +Shell, but do not ingest project instructions, rules, skills, or source from the +remote PVC. Schedules, SkillDraft, background Shell, and delegated filesystem +execution are outside this draft. + +If allocation reports `Ready=false` with `PVCUnavailable` or +`ExecutorUnavailable`, inspect the namespace's quota and Kubernetes admission +failures. Restore the failed prerequisite and wait for `Ready=True`. The +controller keeps retrying through its rate-limited queue while the allocation +exists; no reference edit, restart, or manual reconciliation is needed. Missing +authoritative PVCs or Pods and ownership mismatches remain fail-closed and are +never repaired by creating a replacement. + +Run claims and signed grants renew before the issued grant expiry, including when +an operator configures a short grant TTL. Renewal operation receipts retain the +newest 32 identities. Retrying an exact retained identity preserves the claim and +original expiry (the replacement signature may carry a new nonce); conflicting or +expired receipt replay is denied and never extends or resurrects the claim. + +The provider retains one PVC per logical environment. Session deletion and chart +uninstall do not delete committed workspace data. Retirement requires no live +references and a terminal executor; even then, this candidate retains the PVC. +A missing executor or lost terminal receipt moves the environment to +`FenceUnknown` and requires external operator fencing. The built-in recovery RPC +accepts only a still-observable Pod in `Succeeded` or `Failed` phase with every +container terminated and the exact PVC still present. If that proof is missing, +use your platform's external fencing runbook; there is no acknowledgement flag. + +The built-in recovery RPC requires the same exact identity and an independently +stable operation ID. It does not read a Secret and has no force or +acknowledgement field: + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedExecutionEpoch":"","expectedPodUid":"","expectedPvcUid":"","operationId":"recover-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/RecoverEnvironment +``` + +#### Rotate execution-provider authority + +Give every changed material file a new, generation-specific basename. This applies +to signing keys, server certificates, server private keys, and client-CA bundles. +For example, a second bundle can use `grant-k2.pem`, `server-g2.crt`, +`server-g2.key`, and `clients-g2.pem`. Keep each name's bytes immutable. + +1. Add the new files to the operator-managed security Secret, retaining the files + referenced by the current manifest. Stage the material before changing + `provider.securityManifest`. +2. Publish a higher-generation manifest whose existing `file`, `certificateFile`, + `privateKeyFile`, and `clientCAFile` fields reference those names. For CA + rotation, first publish a separately named overlap bundle, move clients and + server trust, then publish another higher generation that removes old trust. +3. Verify that the authority ConfigMap has reached the intended generation and + use a current claim to read known workspace content. Pod readiness alone can + still reflect the previous generation. A replica whose snapshot lags the + ledger rejects requests before dispatch with structured `not_ready` and + `retryable=true`. Bound any read-only verification poll and stop on wrong + content, a nonretryable error, or another error code. +4. Retain overlap files until no live or in-flight manifest references them. + Retain verification keys through their grant windows before revoking them. + Never reuse a retired name for different bytes. + +Kubernetes projects Secret and ConfigMap updates independently. A manifest that +arrives before its new files fails closed until they arrive; staged files leave +an older manifest unchanged. Overwriting referenced TLS or CA files can instead +publish a mixed bundle's digest permanently at the new generation. If that has +happened, publish a complete, immutable bundle at a **higher** generation. +Repeated requests or a restart cannot repair same-generation digest drift; +preserve the authority ledger rather than resetting it. The provider confines +file access to the mounted security directory, but immutable publication remains +your secret-management procedure's responsibility. + +#### Upgrade, uninstall, and reinstall the execution provider + +The supported lifecycle keeps the **same Helm release name, namespace, resource +names, profiles, network policy configuration, and security Secret name**. Keep +`execution-values.yaml` and the current nonsecret authority manifest in your +operator configuration store. Retain the operator-owned Secret and its key +history independently; the chart neither owns nor reads Secret contents. + +Finish and verify any external authority rotation before starting a chart +upgrade. Do not rotate the authority ConfigMap while Helm is writing chart +resources. Each Helm lifecycle command below initializes `HELM_APPLY_MODE` for +the current shell. Helm 4 selects client-side field ownership, while Helm 3.16 +leaves the value empty. Continue to use reviewed current values and the quiescence +procedure below. Do not use `--force-conflicts` or `--take-ownership` as a blanket +takeover. + +**Before quiescing:** verify that the retained profiles ConfigMap has a nonempty +`data["lifetime.json"]` and that the authority/capacity ledgers and original +release ownership are intact. Pre-retention installations without this history +cannot be automatically adopted. Stop this procedure and recover trusted retained +history; do not synthesize it from proposed values or reset authority. The chart +reports missing history separately from an incompatible lifetime configuration. + +For a compatible provider upgrade: + +1. Stop new client traffic and finish or explicitly fence active work. Stop all + provider replicas before Helm reads the ledgers: + + ```sh + kubectl --namespace scale deployment/mecatl-execution --replicas=0 + kubectl --namespace wait --for=delete pod \ + --selector app.kubernetes.io/name=mecatl-execution --timeout=2m + ``` + +2. Apply the reviewed CRD schema **before** starting the upgraded provider. Helm + does not upgrade existing CRDs: + + ```sh + kubectl apply -f deploy/helm/mecatl-execution/crds/executionenvironment.yaml + kubectl wait --for=condition=Established \ + crd/executionenvironments.execution.mecatl.dev --timeout=60s + ``` + +3. Upgrade with the preserved lifetime configuration and current authority + manifest. Helm reads the existing ConfigMaps and includes their actual ledger + data, rather than empty bootstrap data, in the new release: + + ```sh + HELM_APPLY_MODE= + case "$(helm version --template '{{.Version}}')" in + v4.*) HELM_APPLY_MODE=--server-side=false ;; + esac + helm upgrade mecatl-execution ./deploy/helm/mecatl-execution \ + --namespace --values execution-values.yaml \ + --wait --timeout=4m $HELM_APPLY_MODE + ``` + +4. Complete any supported, explicit environment-schema migration while client + traffic remains quiesced. Verify readiness, exact environment/PVC UIDs, data, + and network confinement before resuming traffic. Unknown schema versions and + mixed-version provider operation are unsupported. + +Default uninstall removes the provider but keeps runtime CRs, PVCs, surviving +executors, workload default-deny and profile NetworkPolicies, both authority and +capacity ledgers, and the profile/security-manifest ConfigMaps. Uninstall is not +executor termination proof or storage disposal: + +```sh +helm uninstall mecatl-execution --namespace +``` + +Reinstall with the same identity and preserved values: + +```sh +HELM_APPLY_MODE= +case "$(helm version --template '{{.Version}}')" in + v4.*) HELM_APPLY_MODE=--server-side=false ;; +esac +helm install mecatl-execution ./deploy/helm/mecatl-execution \ + --namespace --values execution-values.yaml \ + --wait --timeout=4m $HELM_APPLY_MODE +``` + +Helm adopts retained resources only when their managed-by label and release-name +and release-namespace annotations match. The chart rejects missing or empty +ledgers while allocations or capacity reservations survive, and rejects changes +to its retained lifetime configuration. Profile removal or egress edits are deliberately outside this +upgrade path: retained allow policies are additive, so leaving an obsolete policy +could widen access. The provider rejects an older authority manifest against the +retained high-water generation and key history; never reset that ledger to make +readiness pass. + +Use live Helm install/upgrade for this lifecycle. Offline `helm template` cannot +perform ownership or history lookups and is not an adoption mechanism. Keep +provider writers stopped for upgrades; lookup plus apply is not a cross-resource +transaction. Rendering rejects a nonzero existing provider Deployment or any +remaining provider Pod, including a terminating Pod. Changed release/namespace +adoption, chart rollback, `--force-conflicts`, `--take-ownership`, CRD or +namespace deletion with +retained resources, and force-finalizer cleanup are unsupported. + +Final infrastructure decommission is not automated by this candidate. Supported +`RetireEnvironment` and `DeleteRetiredEnvironment` operations can terminate and +dispose of eligible individual environments; they do not remove the retained +NetworkPolicies, configuration, ledgers, or CRD. There is no supported final +infrastructure-cleanup procedure here. Manual destructive cleanup is outside the +supported lifecycle. Keep retention defaults and authority history, and never +reset ledgers or remove finalizers to bypass a failed safety check. + +#### Run an administrative lifecycle operation + +Administrative RPCs require `administrator: true`. For a distinct operations +identity, set its `administratorFor` list to the canonical URI of each client +that created the environments it may administer, as in the values example above. +An absent or empty list permits only self-administration. Keep +`mayAttestOwner: false` for an operations-only identity; administrative scope +confers no filesystem, Shell, attach, run, or reference access. See the +[scope constraints](../../features/execution-environments.md#production-security-material). + +Before first publishing `administratorFor`, quiesce client traffic and upgrade +all provider replicas to a version that understands the field. Older strict +manifest decoders reject it; mixed-version rolling operation is unsupported. +Preserve the authority high-water ConfigMap throughout the upgrade. Publish the +reviewed manifest with a higher `generation`, verify provider readiness, then +resume client traffic. + +Every administrative request still requires the original exact owner and +revision plus the operation's epoch, UID, schema, or generation preconditions. +The scope names creators, not owners; it can include a creator whose login has +been removed. After maintenance, remove its scope entry and increase +`generation` again. Subsequent requests, including receipt retries on established +connections, are denied; already admitted lifecycle operations may finish safe +reconciliation. + +An administrative `not_found` response deliberately does not distinguish a +missing environment from a wrong owner, revision, or creator scope. Check the +exact environment ID/revision and original owner against your authorized +operations record, then check that `administratorFor` names the original +creator's exact canonical client URI. If the scope is wrong, publish a reviewed +manifest at a higher `generation` and verify readiness before retrying. The RPC +will not disclose another creator's data to diagnose a scope mismatch. + +First capture the private identity while the environment still has a reference: + +```sh +kubectl --namespace get executionenvironment \ + -o jsonpath='{.spec.revision}{"\n"}{.spec.ownerIssuer}{"\n"}{.spec.ownerSubject}{"\n"}{.status.epoch}{"\n"}{.status.pod.uid}{"\n"}{.status.pvc.uid}{"\n"}{.status.grantGeneration}{"\n"}' +``` + +Record those seven lines as ``, ``, ``, +``, ``, ``, and ``. The owner hash is +not reversible, so retain the bounded `spec.ownerIssuer` and `spec.ownerSubject` +attestation in your authorized operations record before removing the final +reference. + +Set file references to trusted, mounted mTLS material. Keep private-key bytes out +of shell arguments and manifests: + +```sh +EXECUTION_ENDPOINT=mecatl-execution..svc:8443 +CA_FILE=/var/run/secrets/mecatl-admin/ca.crt +CERT_FILE=/var/run/secrets/mecatl-admin/tls.crt +KEY_FILE=/var/run/secrets/mecatl-admin/tls.key +PROTO=contracts/proto/mecatl/execution/v1/execution.proto +``` + +Replace one executor by reusing the same operation ID for every retry: + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedExecutionEpoch":"","expectedPodUid":"","expectedPvcUid":"","operationId":"replace-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/ReplaceExecutor +``` + +To retire and then delete retained storage, call `RetireEnvironment` with the same +identity fields and a new stable operation ID. Poll until +`.status.conditions[?(@.type=="Retired")].status` is `True`, then call +`DeleteRetiredEnvironment` with the retained PVC UID and another stable operation +ID. The provider refuses either request while references, claims, identity proof, +or UID checks are incomplete. + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedExecutionEpoch":"","expectedPodUid":"","expectedPvcUid":"","operationId":"retire-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/RetireEnvironment +kubectl --namespace wait executionenvironment/ \ + --for='jsonpath={.status.conditions[?(@.type=="Retired")].status}=True' \ + --timeout=10m +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedPvcUid":"","operationId":"delete-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/DeleteRetiredEnvironment +``` + +Revoke grants with the current generation and a stable operation ID. The response +returns the new generation; retrying the identical request returns the same +receipt. An old grant is denied after this CAS succeeds. + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedGrantGeneration":"","operationId":"revoke-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/RevokeEnvironment +``` + +Migration requires proto presence for `expectedSchemaVersion`. Supply exactly `0` +or `1`, plus the exact observable Pod and PVC UIDs. Omission, an unknown version, +or missing proof is refused without changing the resource. + +```sh +grpcurl -cacert "$CA_FILE" -cert "$CERT_FILE" -key "$KEY_FILE" \ + -import-path contracts/proto -proto "$PROTO" \ + -d '{"environment":{"id":"","revision":""},"owner":{"issuer":"","subject":""},"expectedSchemaVersion":1,"expectedPodUid":"","expectedPvcUid":"","operationId":"migrate-"}' \ + "$EXECUTION_ENDPOINT" mecatl.execution.v1.ExecutionProviderService/MigrateEnvironment +``` + +Foreground command cancellation is cooperative and bounded. The helper attempts +to terminate the command process group and reports a terminal receipt. If it +cannot prove termination, the environment is fenced instead of admitting more +work. There is no detached command status or cancellation API in this draft. + +The focused qualification task is `task e2e:k8s:execution`. It requires Go, +`ko`, Docker or rootless Podman, Kind, Helm, and `kubectl`. Optional toolbox use +requires explicit `MECATL_EXECUTION_DEV_TOOLBOX` and +`MECATL_EXECUTION_K8S_TOOLBOX` values. The task uses a synthetic OIDC issuer and +the mock model provider, creates a unique state directory and Kind cluster, and +retains both for inspection. CI invokes +`task e2e:k8s:execution:production` and automatically removes only its uniquely +owned cluster. The production target is the home for the enforcing-CNI network +and lifecycle fixture; until that fixture passes, Kind qualification is not +NetworkPolicy isolation evidence. + +When `execution.enabled` is `false`, the `mecak8s` chart mounts no execution mTLS +Secret and passes no execution-provider flags. The separate provider chart and +any environments it owns continue independently. + The no-FS default is intentional. A standard mecak8s pod is storage-free and has no authoritative filesystem root, so the server binds omitted/default profile to its configured no-FS placement. Clients never send a workspace path; explicit diff --git a/user-docs/features/execution-environments.md b/user-docs/features/execution-environments.md index d0a4d58fa7..0641c66298 100644 --- a/user-docs/features/execution-environments.md +++ b/user-docs/features/execution-environments.md @@ -86,6 +86,297 @@ reattached only when the deployment supplies an `EnvironmentResolver`; a missing resolver, mismatched identity, or nil workspace returns an error instead of using a local workspace. +## Native Kubernetes lifecycle and retention + +**DRAFT candidate, not released or approved for production.** The native provider +is intended for an operator-controlled cluster and foreground commands only. +It has no force-takeover recovery and is not a hostile multi-tenancy boundary. + +Remote deployments retain operator-global prompt rules and skills. A deployment can +programmatically compose independently configured harness context sources for +instructions and slash commands without acquiring native execution files. PVC-backed +context remains unselected in this slice. Permission rules +from enabled user-global settings and explicit permission files also apply, +including configured Deny and Ask rules under `auto` posture. Project permission +files are excluded. Local project trust cannot admit host-local `AGENTS.md`, +rules, skills, commands, or Git context into a remote session. Explicit `no-fs` +sessions keep their file-less catalog, allocate no execution environment, and +can retain independently configured harness context sources. + +The remote filesystem preserves the existing Read/Edit/Write version checks and +non-clobbering Copy/Move behavior. Copy, Move, and Remove do not require prior +content reads; Remove remains non-recursive. + +The optional Kubernetes execution provider stores environment ownership in a +namespaced `ExecutionEnvironment`. Provider replicas coordinate through +Kubernetes resource-version compare-and-swap; a replica restart does not clear +another replica's operation. When a caller resolves a persisted ordinary +approval after a server restart, the server reacquires native run ownership, +renews it while the resumed run drains, and releases it after the drain. A lost +or cancelled operation can still leave the environment in `FenceUnknown`, which +requires the documented manual recovery path. There is no force takeover. + +Operation lease expiry fences the environment and retains the unresolved +operation identity for administrator recovery. A `503` response with +`placement_unavailable` is generic: check provider readiness, the configured +profile's capacity, and retained allocations. Use the existing +[administrative lifecycle operation](../building/deployment/mecak8s.md#run-an-administrative-lifecycle-operation) +to retire an eligible environment and delete its retained storage when you need +to free capacity. + +Workspace PVCs are retained by default. Removing the last session reference, +deleting a session, uninstalling the chart, or deleting the provider does not +delete a workspace PVC. Executor replacement and retirement are private, +mutually authenticated administrator operations. They require exact environment, +execution-epoch, Pod-UID, and PVC-UID values. The provider waits for a kubelet +terminal Pod phase and terminated state for every container before removing its +Pod finalizer. A missing Pod or unreachable kubelet is not termination proof and +leaves the environment fenced. + +Retirement stops the executor but retains the PVC. PVC deletion requires the +separate `DeleteRetiredEnvironment` administrator RPC with the exact retained +PVC UID and no references or claims. Deleting the custom resource outside this +provider workflow is unsupported. Do not remove the retention or executor +finalizers manually; follow the operator's external-fencing runbook when the +provider cannot independently observe terminal compute. + +Persisted prototype resources use an explicit administrator migration. Normal +operations reject schema versions other than 2. `MigrateEnvironment` accepts +only recognized versions 0 and 1, verifies the exact live Pod and PVC identities, +and requires a healthy, idle environment. Unknown or malformed state is retained +unchanged rather than reset. Completed migration receipts expire when executor +replacement publishes a new Pod identity. After replacement, retrying the old +migration operation returns a conflict with either the original or replacement UID. + +### Production security material + +The production chart requires one projected Secret containing the TLS identity, +client CA bundle, and Ed25519 grant keys, plus `provider.securityManifest`. The +chart does not generate keys or certificates. The manifest is strict JSON with +this shape: + +```json +{ + "version": 1, + "generation": 42, + "issuer": "https://execution.example.com", + "audience": "mecatl-execution", + "activeKeyID": "k1", + "grantTTL": "1m", + "clockSkew": "5s", + "keys": [ + { + "id": "k1", + "version": 1, + "file": "grant-k1.pem", + "publicKeySHA256": "0000000000000000000000000000000000000000000000000000000000000000", + "activateAt": "2027-01-01T00:00:00Z", + "verifyUntil": "2027-01-02T00:00:00Z", + "state": "active" + } + ], + "tls": { + "certificateFile": "tls.crt", + "privateKeyFile": "tls.key", + "clientCAFile": "clients.pem" + }, + "clients": [ + { + "uri": "spiffe://cluster.example.com/ns/mecatl/sa/mecak8s", + "mayAttestOwner": true, + "administrator": false + }, + { + "uri": "spiffe://cluster.example.com/ns/mecatl/sa/execution-admin", + "mayAttestOwner": false, + "administrator": true, + "administratorFor": ["spiffe://cluster.example.com/ns/mecatl/sa/mecak8s"] + } + ] +} +``` + +The all-zero fingerprint and 2027 dates are non-secret example values. Replace +the fingerprint with the lowercase hexadecimal SHA-256 hash of the raw 32-byte +Ed25519 public key, not PEM text or DER encoding, and use a reviewed active window. + +`administratorFor` permits administration of environments created by the listed +client URIs, alongside the administrator's own environments. A nonempty list +requires `administrator: true` and accepts at most 256 unique canonical URIs: +nonempty scheme and host, lowercase scheme and host, and no userinfo, query, +fragment, or wildcard. Matching is exact, with no namespace or prefix grant. +An absent or empty list preserves self-administration only. The creator need +not remain in the client allowlist. + +The scope applies to `RetireEnvironment`, `ReplaceExecutor`, `RecoverEnvironment`, +`DeleteRetiredEnvironment`, `MigrateEnvironment`, and `RevokeEnvironment`. It +preserves each operation's exact owner and identity checks and grants no +`mayAttestOwner`, attach, file, command, run, or reference authority. Scope order +does not change the authority digest; adding or removing a creator does. Follow +the [administrative runbook](../building/deployment/mecak8s.md#run-an-administrative-lifecycle-operation) +for quiesced upgrades and scope removal. + +Use only basename file names. The projected Secret keys in this example are +`grant-k1.pem`, `tls.crt`, `tls.key`, and `clients.pem`; an external secret manager +owns their bytes. Kubernetes projected-volume `..data` symlinks are supported, +but paths escaping the mounted directory are rejected. Keep every referenced +filename immutable and use a new name for changed signing, TLS, or CA material. +Stage those files before publishing a higher-generation manifest and retain the +overlap files. Secret and ConfigMap projections are independent; see the +[authority rotation procedure](../building/deployment/mecak8s.md#rotate-execution-provider-authority) +for publication, verification, and recovery from mixed-material digest drift. +Increase `generation` for every authority change. Key IDs and `(id, version)` fingerprints cannot be +reused; the provider persists a bounded high-water ledger in its authority +ConfigMap. Keep retired keys as `verify-only` until all grants expire, then mark +them `revoked`. Invalid, incomplete, rolled-back, or newly expired material makes +readiness fail and denies new RPC authorization until corrected. The provider +re-verifies the peer certificate and URI policy against the current client CA on +every RPC, including RPCs on an existing HTTP/2 connection. Profile-resource or +controller-cache startup failures terminate the provider with that bounded reason +class before it accepts traffic. A failed `/ready` response reports +`security-authority-or-expiry`. Inspect provider logs and the named RuntimeClass, +StorageClass, and authority ConfigMap metadata; the endpoint never returns key or +certificate contents. + +`RevokeEnvironment` is an administrator-only, exact-reference CAS. Supply the +current positive grant generation; success increments it without changing the +execution epoch. Old grants then cannot authorize another operation or renewal. +An operation already accepted may still finish and clean up, so revocation is not +termination proof and a new run needs a fresh claim after the old claim is +released or fenced. + +The gRPC port remains the only execution data plane. `/live` and `/ready` are +separate, unauthenticated operational HTTP endpoints intended only for Kubernetes +probes; do not expose that port outside the cluster. Readiness requires current +security material, the authoritative generation, synchronized controller caches, +and Kubernetes access. Liveness reports process health only. + +Production values require two or more provider replicas, explicit provider-client +selectors, API-server and DNS CIDRs, and finite ResourceQuota/LimitRange values. +Executor pods are default-deny for ingress and egress. Add only profile-specific +CIDR and port egress under `networkPolicy.workloadProfiles`; there is no implicit +DNS, metadata-service, API-server, provider, or peer access. Each profile requires +an explicit RuntimeClass supplied by the cluster, but do not treat a RuntimeClass +name alone as a hostile-workload isolation guarantee. CPU, memory, ephemeral +storage, and `/tmp` are explicitly bounded per profile. Each profile also requires +`maxEnvironments`; the namespace quota is the final global bound. + +Preserve the authority ConfigMap and current manifest across upgrades and +same-release reinstalls. A missing ledger with retained allocations is not a new +installation: do not bootstrap it at generation 1. See the +[provider lifecycle procedure](/building/deployment/mecak8s.md#upgrade-uninstall-and-reinstall-the-execution-provider) +for ownership checks, retained network protection, and quiesced CRD/provider +upgrade order. Schema-2 environment migration remains an explicit operation; +complete it before enabling new sessions. + +## Live qualification (experimental) + +The repository keeps real-provider qualification separate from the default mock +suite. It is explicit opt-in, uses an already-owned retained Kind cluster, and is +not part of `task test`: + +```sh +MECATL_EXECUTION_CREDENTIAL_FILE=/absolute/path/to/provider-key \ +MECATL_EXECUTION_QUAL_STATE=/absolute/path/to/owned-state \ +task e2e:k8s:execution:live +``` + +The credential file must be a private regular file (no group or other access). +A trusted helper loads it only at runtime and creates a run-scoped Kubernetes +Secret through the API; it never renders the value into a manifest or reads the +Secret back. Only the `mecak8s` harness container receives the OpenRouter key. +The execution provider, controller, OIDC fixture, and executor workloads do not. +The task first runs the deterministic mock qualification, then runs one bounded +real-model coding smoke against `https://openrouter.ai/api/v1`. It verifies +positive token usage, required file and shell tool calls, file contents, and a +successful `go test` independently through the typed gRPC execution service. +Finally it restores the mock deployment and removes only its run-scoped Secret; +the owned cluster and execution workspaces remain for inspection. + +### Run the native qualification in GitHub Actions + +After independent review of the exact candidate, a maintainer can dispatch the +existing `e2e-live.yml` workflow in `stacklok/mecatl`: + +```sh +gh workflow run e2e-live.yml --repo stacklok/mecatl --ref \ + -f native_execution=true -f expected_sha= +``` + +`native_execution` defaults to `false`. Setting it to `true` runs only the native +job, avoiding duplicate paid inference in the ordinary live jobs. Schedules and +labelled PR runs retain their ordinary behavior and never select this job. +`expected_sha` is optional, but supply it for reviewed qualification: checkout +uses the dispatch event SHA, verifies exact equality before credential staging, +and records that SHA in the job summary. A moved branch fails the check. + +The trust gates are maintainer dispatch, the `stacklok/mecatl` repository check, +and access to the existing `OPENROUTER_API_KEY` repository secret. There is no +GitHub environment-approval gate. Review the workflow and all executed source +before dispatch; the SHA check establishes identity, not code safety. Checkout +retains no Git credentials, the token has read-only contents access, and the +native job uses no build cache. + +The same runner first completes the production Kind+Calico qualification without +a provider key or automatic cluster deletion. Only a successful production step +allows credential staging; a production ownership record alone is insufficient. +The workflow stages the key in a new 0700 CI directory and 0600 file, then removes +it from the environment before `live.sh` validates the retained owned state, +reruns mock qualification, and invokes the trusted credential loader. A missing +secret fails qualification rather than skipping successfully. The 100-minute job +has explicit step ceilings: setup 9 minutes, production 50, credential staging 1, +live 25, cleanup 10, and status/artifact reporting 3, leaving 2 minutes of overhead. +Production and live subprocess groups receive TERM after 49 and 24 minutes, +respectively, then KILL after a 15-second grace. These stage ceilings are +intentionally smaller than the sum of all subordinate build, rollout, and test +bounds. Hitting one fails qualification; it does not prove completion. Per-agent +token limits remain unchanged and are not a hard dollar cap. + +The live phase preserves the qualified cluster's local-path storage helper and +security configuration; rebuilding the executor image does not replace the +storage provisioner's helper image. + +The live script attempts to restore mock configuration and delete its Secret by +recorded UID using an independent cleanup context. A signal or stage deadline can +interrupt that attempt. An always-run workflow step has a separate 10-minute +ceiling to remove the CI-created credential file and delete the owned cluster. +Production publishes its state path and cluster identity after collision checks +and before cluster creation; live and cleanup use those fixed step outputs, not +the mutable local `current` pointer. Cleanup validates the recorded owner, name, +namespace, profile, container label, private kubeconfig, and context against that +identity. Ownership drift fails cleanup without selecting another cluster. +Failure before kubeconfig creation also fails cleanup without deleting a cluster. +Cleanup is bounded best effort: hard cancellation or runner loss can prevent it. +Hosted-runner disposal is the fallback, not evidence that cleanup succeeded. + +Artifacts are retained for seven days: a size-bounded `live-summary.json` when +available and a sanitized `qualification-status.txt`. On live failure, +`live-diagnostics.jsonl` captures the real process before mock restoration and +Secret cleanup. If this capture is missing or incomplete, or production fails, +workflow cleanup collects separate `production-diagnostics.jsonl` evidence before +cluster deletion. `diagnostics-status.txt` distinguishes pre-restoration and +fallback collection outcomes. Collection failure warns and still attempts cleanup. + +Each collector has a 45-second bound, 5-second API deadlines, and a 1 MiB output +limit. Resource evidence contains known condition/reason classes, deletion and +UID-match booleans, known finalizers, Pod phases, container exit reasons, +lease-expiry status, quota key names, and event reason counts. Log evidence passes +through a strict JSON allowlist in memory; only create stages, fixed reason +classes, elapsed milliseconds, call counts, and hashed session correlation survive. +Raw logs, stacks, PKI, kubeconfigs, Secret receipts, Helm values, resource manifests, +transcripts, and surrounding scratch directories are excluded from artifacts. + +The fixture enables `--log-level=debug`; native-execution debug diagnostics use +JSON. Create stages distinguish handler entry after authentication, session-ID +storage probing, remote Ensure, Attach polling, engine construction, persistence, +reference publication, and HTTP response construction. Polling logs its first +state, state changes, and final count rather than every retry. The live test first +checks authenticated HTTP reachability separately, then reports when create +headers were sent and a response started. A last `begin` stage without a matching +completion identifies the boundary to investigate; it does not establish the root +cause. Check live, diagnostics, and cleanup outcomes before treating the recorded +commit as qualified. + ## Limitations - No-FS sessions cannot use local file tools, shell commands, workspace forks, diff --git a/user-docs/mecatui/sessions.md b/user-docs/mecatui/sessions.md index 86ade1748e..8efd8eb948 100644 --- a/user-docs/mecatui/sessions.md +++ b/user-docs/mecatui/sessions.md @@ -22,12 +22,13 @@ mecatui connect 127.0.0.1:8080 --resume-latest ``` On servers that provide session activity inventory, `--resume-latest` considers -only active main chats and verifies each candidate's authoritative transcript. -It does not select drafts or unknown legacy rows automatically. Older servers -use their mixed inventory but still reject an empty authoritative transcript. If -no chat qualifies, `mecatui` starts a new one. Storage and listing failures -still return an error. You can combine either resume option with `--prompt` to -send a task after the transcript loads. +only main chats marked active in that inventory and verifies each candidate's +authoritative transcript. It excludes chats awaiting approval and does not select drafts or +unknown legacy rows automatically. Older servers use their mixed inventory but +still reject an empty authoritative transcript. If no chat qualifies, `mecatui` +starts a new one. Storage and listing failures still return an error. You can +combine either resume option with `--prompt` to send a task after an ordinary +transcript load; pending-approval recovery keeps it as a draft. A resumed chat is the stored chat, not a copy. It keeps its model, exact server-owned placement, cumulative token totals, and latest saved context meter. @@ -36,6 +37,39 @@ accounting. A chat with no saved context measurement shows an unknown value unti a completed turn establishes one. See [Session continuity](/features/session-continuity.md) for the storage and recovery behavior behind resume. +### Recover a pending approval + +Use an exact session ID to reopen an owned main chat that stopped while waiting +for an ordinary tool approval: + +```sh +mecatui --resume +``` + +When the server provides a complete durable event watch, `mecatui` displays the +original tool, arguments, and reason. Choose **Allow once** or **Deny**. The +client follows the same run and displays its later model messages and tool +results. **Allow always** is unavailable during recovery. + +`--resume-latest` continues to exclude chats that are waiting for approval. A +`--prompt` value remains in the composer as a draft. `mecatui` does not send it +before your choice or automatically send it after the recovered run finishes. + +Press `esc` before choosing to leave the approval pending. After you choose, +pressing `ctrl+c` twice can still exit while the acknowledgement is pending. +The outcome may then be unknown: quitting cannot undo the decision, and +`mecatui` does not resend the verdict automatically. Check the session before +taking another action. + +If the event history has a gap, the ask changed, or the session is no longer +waiting, first check that the session still has the expected pending approval and +that durable storage and watches are healthy. Retry with the exact ID only after +those checks. A server without durable watch support cannot recover the approval; +the operator must enable supported durable storage and watch support, or you must +use another supported client or control path. Repeating the unchanged command +will not add watch support. Plan approvals and guardrail reviews use their existing +dedicated flows and cannot be recovered this way. + To get the active session ID, run `/session` and press `c` to copy it. On a normal exit, `mecatui` also writes a machine-readable handoff to standard error: diff --git a/user-docs/reference/grpc-schema.md b/user-docs/reference/grpc-schema.md index bc08af6d16..f64d4819e1 100644 --- a/user-docs/reference/grpc-schema.md +++ b/user-docs/reference/grpc-schema.md @@ -60,7 +60,7 @@ the bidi Converse stream that drives one agent run. | **RPC:** `ListSessionMcpConnectors`
**Request:** `ListSessionMcpConnectorsRequest`
**Response:** `ListSessionMcpConnectorsResponse` | Client streamingNo | Server streamingNo | DescriptionListSessionMcpConnectors inspects the owned session's broker-local catalogue. This read neither probes upstreams nor progresses enrollment. | | **RPC:** `ListToolHiveGroups`
**Request:** `ListToolHiveGroupsRequest`
**Response:** `ListToolHiveGroupsResponse` | Client streamingNo | Server streamingNo | DescriptionListToolHiveGroups returns the distinct, non-empty ToolHive groups present in the resolved source inventory. Derived from the snapshot — it does NOT call ToolHive. | | **RPC:** `ListAgents`
**Request:** `ListAgentsRequest`
**Response:** `ListAgentsResponse` | Client streamingNo | Server streamingNo | DescriptionListAgents returns the resolved agent-definition registry snapshot: each discovered agent def's routing metadata (name/description), its resolved model, its effective read-only tool scope, permission mode, and UX color. Derived from the snapshot taken at startup; it performs no live discovery. | -| **RPC:** `ListCommands`
**Request:** `ListCommandsRequest`
**Response:** `ListCommandsResponse` | Client streamingNo | Server streamingNo | DescriptionListCommands returns slash commands discovered for one owned session's exact server-bound placement. It authorizes and reattaches that session before discovery and accepts no workspace/root input. It powers the client's command palette; command expansion remains a run-path concern. A no-FS session or a server with no command expander returns an empty list. | +| **RPC:** `ListCommands`
**Request:** `ListCommandsRequest`
**Response:** `ListCommandsResponse` | Client streamingNo | Server streamingNo | DescriptionListCommands returns slash commands from the configured, resolved sources for one authorized, owned session. It runs independently of execution and accepts no client workspace/root input. A source that needs execution files acquires only that session's exact backend. It powers the client's command palette; command expansion remains a run-path concern. A server with no configured command sources returns an empty list; a no-FS session can retain independently configured sources. | | **RPC:** `ListWorktrees`
**Request:** `ListWorktreesRequest`
**Response:** `ListWorktreesResponse` | Client streamingNo | Server streamingNo | DescriptionListWorktrees discovers eligible alternatives for one owned source session's exactly reattached placement. Results carry bounded display metadata and an opaque source-scoped selector accepted only by ClearSession or ForkSession. The server re-enumerates and matches current choices on use; selectors are not paths, are not persisted, and expire on restart. A no-FS session or a server with no worktree lister returns an empty list. | | **RPC:** `StreamSessionEvents`
**Request:** `StreamSessionEventsRequest`
**Response:** `Event` | Client streamingNo | Server streamingYes | DescriptionStreamSessionEvents replays a session's durable event log as a server stream of `Event` envelopes (cloud-native Phase 3a read-back). It is the client-tier surface over the SAME `port.EventLog.Read` the operator-tier 3c EventLogService.Read serves; the loop itself stays storage-agnostic (it only emits — persistence lives at the relay, never in `engine/agent`). The log stores ALREADY-REDACTED events and inherits the stream's redaction, so this path adds none (gauntlet #7: no raw args/deny-reason bodies/child content ever cross). UNLIKE the live `Converse` relay — which SKIPS the three log-only event kinds (`approval`/`compaction_archive`/`user_prompt`) on the client wire because they are persistence-only — StreamSessionEvents relays ALL events, including those three: a client opening a PAST session wants the verdicts and user prompts, as they ARE the transcript. They are metadata-only/redacted by construction, so no extra filter applies here. Read-only: no live model call, no mutation, no Converse run is started. An unknown session id yields an EMPTY stream (absence is data — a never-created or pruned id is indistinguishable, by design). A server with NO durable EventLog wired returns `UNIMPLEMENTED` (HTTP 501). There is intentionally NO ServerCapabilities bit for this feature: the capability is RPC-discoverable (UNIMPLEMENTED vs. an empty stream degrade honestly). | | **RPC:** `StreamSessionLive`
**Request:** `StreamSessionLiveRequest`
**Response:** `Event` | Client streamingNo | Server streamingYes | DescriptionStreamSessionLive is the LIVE per-session event stream (ADR 0075 fire-result-delivery Scenario 6 / Wave 3): a server-streaming RPC backed by the in-process per-session subscription registry (Service.Subscribe / PublishSessionEvent, List 1 row 35). It is the UNIFIED transport for BOTH embedded and remote clients — the embedded mecatui dials its in-process server over a real gRPC UNIX socket, so the in-process Subscribe registry is unreachable from the TUI without a wire transport; this RPC is that transport, and it serves a remote mecated identically (ONE proto, ONE TUI consumption path). UNLIKE StreamSessionEvents (the durable-log READ-BACK, which relays ALL events including the three log-only kinds), StreamSessionLive is a LIVE stream and applies the SAME live-wire relay discipline as the live `Converse` relay, with ONE narrow exception: a fire-result delivery note (an EvUserPrompt whose text starts with the renderFireDelivery provenance header "[scheduled task ") is RELAYED so the connected client renders the delivery card as it happens (AC6.2). The other two log-only kinds (EvApproval, EvCompactionArchive) stay SKIPPED on the live wire (they are persistence-only; a client holds its own verdict/compaction view); a non-delivery EvUserPrompt stays SKIPPED too (the client already holds its own prompt). The delivery note is metadata-only/redacted by construction (gauntlet #7 — it is the fenced-untrusted harness note the engine recorded). The stream stays open until the client cancels or disconnects. The handler is drain-to-discard: a dead client never wedges the delivery run (a full subscriber channel drops the event, AC6.3). The durable log records the tail regardless (it is appended by the relay/loop, independent of this stream). There is intentionally NO ServerCapabilities bit — the capability is RPC-discoverable (UNIMPLEMENTED on an older server, an empty stream on a server with no subscribers). An unknown session id yields an empty stream (absence is data — a never-created id is indistinguishable from a session with no live events yet). |