diff --git a/.github/workflows/validate-manifests.yml b/.github/workflows/validate-manifests.yml index 9e9748d9f..5e4f5178a 100644 --- a/.github/workflows/validate-manifests.yml +++ b/.github/workflows/validate-manifests.yml @@ -80,6 +80,15 @@ jobs: # workflows that keep the generated files in sync. scripts/materialize-openclaw-platform-skills.py --check + - name: Validate agent platform catalog + run: | + python3 -m pip install --quiet pyyaml jsonschema + # --selftest also replays scripts/tests/fixtures/agent-platform/ + # {valid,invalid} so a change that silently weakens a check (e.g. + # a reference no longer failing closed) fails CI even though the + # real platform-gitops/agent-platform/ catalog itself is fine. + scripts/validate-agent-platform.py --selftest + - name: Validate Argo local workdir invariants run: scripts/validate-local-workdir.py diff --git a/platform-gitops/agent-platform/README.md b/platform-gitops/agent-platform/README.md new file mode 100644 index 000000000..6712e42e4 --- /dev/null +++ b/platform-gitops/agent-platform/README.md @@ -0,0 +1,134 @@ +# Agent platform catalog + +GitOps half of ADR 007 (`mctlhq/mctl-agents` +`docs/adr/007-agent-definition-execution-profile-contract.md`). This +directory holds the reviewed-in-Git `ExecutionProfile` catalog, the +platform policy that every profile and release intent is checked against, +and non-promotable `ReleaseBindingIntent` fixtures. Everything here is +validated by `scripts/validate-agent-platform.py`, wired into +`.github/workflows/validate-manifests.yml`. + +This directory does **not** own, mirror, or replace: + +- the canonical `AgentDefinition` body (prompt, triggers, identity) -- + that stays at `mctl-agents/agents/_manifests//agent.yaml`, and is + referenced here only via an exact `sourceManifest {repo, path, gitSha}`; +- immutable published versions, environment `ReleaseBinding` history, or + `lifecycleState` -- all three live in the mctl-api registry + (`mctl_publish_agent_version` / `mctl_promote_agent` / + `mctl_resolve_agent` / `mctl_rollback_agent`); +- the per-run `ExecutionPlan`/`ExecutionRecord` -- that is the runtime + resolver's job (mctl-agents issue #227), not implemented here. + +## Layout + +```text +platform-gitops/agent-platform/ + policy.yaml policy ceilings + reference catalogs + execution-profiles//profile.yaml agents.mctl.ai/v1alpha2 ExecutionProfile + releases//.yaml agents.mctl.ai/v1alpha2 ReleaseBindingIntent + schemas/execution-profile.schema.json + schemas/release-binding-intent.schema.json +``` + +## Source-of-truth boundaries (ADR 007, four layers) + +| Layer | Owns | Example | +|---|---|---| +| `mctl-agents` Git | Canonical `AgentDefinition`: identity, prompt sources, triggers, `executionProfileRef` | `agents/_manifests/implementer/agent.yaml` | +| `mctl-gitops` Git (this directory) | `ExecutionProfile` drafts, policy ceilings, `ReleaseBindingIntent` review fixtures | `execution-profiles/implementer-default/profile.yaml` | +| mctl-api registry | Immutable published versions, per-environment `ReleaseBinding` history, lifecycle (published/deprecated/disabled) | `mctl_publish_agent_version`, `mctl_promote_agent`, `mctl_resolve_agent` | +| Runtime resolver / Temporal / Argo | One frozen `ExecutionPlan` per run; the actual execution | `orchestrator/resolver.py` (mctl-agents #227), Argo Workflow object | + +A file existing under `execution-profiles/` or `releases/` means only +"drafted, reviewed in Git" -- the `draft` state. Nothing in this +directory can ever mark a version `published`, `active`, `deprecated`, or +`disabled`; that authority belongs entirely to the mctl-api registry, and +`active` is itself derived from an environment binding, never stored as a +field. This is why neither JSON Schema in `schemas/` permits a +`lifecycleState` property anywhere -- see +`scripts/tests/fixtures/agent-platform/invalid/global-lifecycle-state/` +for the negative fixture that proves it fails closed. + +## Effective-value extraction + +The three initial profiles (`issue-investigator-default`, +`implementer-default`, `shepherd-default`) are migrated to be +behavior-preserving, not re-litigated. Their `budgetUsd`/`timeoutSeconds` +come from the values actually enforced in production today, which is not +always the `orchestrator/options.py` Python default: + +| Agent | budgetUsd | timeoutSeconds | Source | +|---|---|---|---| +| issue-investigator | 3.00 | 7200 | `cwft-mctl-agents-investigate.yaml` `ISSUE_INVESTIGATOR_BUDGET_USD`; no per-op timeout override exists, so the CWFT's `activeDeadlineSeconds: 7200` is the effective ceiling | +| implementer | 20.00 | 2400 | `cwft-mctl-agents-implement.yaml` `IMPLEMENTER_BUDGET_USD` / `IMPLEMENTER_TIMEOUT_SECONDS` -- these CWFT overrides win over the Python defaults ($3 / 900s) | +| shepherd | 5.00 | 7200 | `cwft-mctl-agents-shepherd.yaml` `SHEPHERD_BUDGET_USD` (raised from the Tier 3 spec's $1.00 default); no per-tick timeout override exists, so `activeDeadlineSeconds: 7200` is the effective ceiling | + +Any future profile bump must state which side (Python default vs. deployed +CWFT override) it is changing, and why -- silently reverting to the Python +default would be a behavior change this issue is explicitly scoped not to +make. + +## Production dependency + +Every `ReleaseBindingIntent` fixture under `releases/` today has +`spec.bindingSource: compatibility-fixture` and `spec.promotable: false`. +There is no real mctl-api-published v1alpha2 version of any of these three +agents or profiles yet -- that publish step, and the runtime resolver that +would actually consume a real binding (mctl-agents #227), are both +follow-up work. Until they land: + +- these fixtures can never be promoted to production by anything reading + this directory (`scripts/validate-agent-platform.py` rejects any + `compatibility-fixture` binding with `promotable: true` as inconsistent); +- no running agent, CWFT, or mctl-api schema changes as a result of this + catalog existing; +- the `history`/`rollbackOf` fields on each intent exercise exact-pair + rollback semantics against a local fixture ledger only, standing in for + what will eventually be real mctl-api `ReleaseBinding` history once + registry reconciliation exists. + +## Validation + +```bash +python3 -m pip install --quiet pyyaml jsonschema +scripts/validate-agent-platform.py # real catalog only +scripts/validate-agent-platform.py --selftest # + replay scripts/tests/fixtures/agent-platform/ +``` + +`--selftest` asserts every fixture under +`scripts/tests/fixtures/agent-platform/valid/` passes and every fixture +under `.../invalid/` fails, one fixture per ADR 007 validation +expectation: + +| ADR 007 expectation | Fixture | +|---|---| +| owner required | `invalid/missing-owner/` | +| policyRef required | `invalid/missing-policy-ref/` | +| permissions required | `invalid/missing-permissions/` | +| bounded budget/timeout present | `invalid/missing-bounds/` | +| budget ceiling enforced | `invalid/budget-exceeds-ceiling/` | +| unknown tool fails closed | `invalid/unknown-tool/` | +| unknown skill fails closed | `invalid/unknown-skill/` | +| unknown model-policy task fails closed | `invalid/unknown-model-policy-task/` | +| mutation requires scope + approval | `invalid/mutation-without-approval/` | +| unapproved sandbox rejected | `invalid/unapproved-sandbox/` | +| no global lifecycleState field | `invalid/global-lifecycle-state/` | +| unknown release-intent profile reference fails closed | `invalid/release-missing-profile/` | +| ambiguous profile version rejected | `invalid/release-ambiguous-profile/` | +| incompatible profile version rejected | `invalid/release-incompatible-version/` | +| disabled version rejected | `invalid/release-disabled-version/` | +| independent-half rollback rejected | `invalid/rollback-independent-half/` | +| exact-pair rollback accepted | `valid/rollback-replay/` | + +## Rollback + +This catalog is additive and not runtime-load-bearing -- nothing resolves +against it yet. Reverting the commit that introduced it removes the +catalog, schemas, validator, and CI step with no effect on any running +agent, CWFT, or mctl-api state. Once real registry-backed bindings exist, +operational rollback always selects the exact previous registry tuple +(`mctl_rollback_agent`'s existing "revert to from_version" semantics), +never an independently chosen pair -- exactly what +`invalid/rollback-independent-half/` and `valid/rollback-replay/` pin down +here ahead of that integration. diff --git a/platform-gitops/agent-platform/execution-profiles/implementer-default/profile.yaml b/platform-gitops/agent-platform/execution-profiles/implementer-default/profile.yaml new file mode 100644 index 000000000..32367e88a --- /dev/null +++ b/platform-gitops/agent-platform/execution-profiles/implementer-default/profile.yaml @@ -0,0 +1,64 @@ +# ExecutionProfile for implementer (ADR 007). Preserves today's effective +# values: $20.00 budget and 2400s timeout are the production +# cwft-mctl-agents-implement.yaml overrides (IMPLEMENTER_BUDGET_USD / +# IMPLEMENTER_TIMEOUT_SECONDS), not the orchestrator/options.py Python +# defaults ($3 / 900s) -- see requirements.md's "Migration is +# behavior-preserving: effective deployed CWFT overrides win over Python +# defaults." This is the only one of the three initial profiles with +# repository.commit/pullRequestCreate: true, matching +# docs/agent-inventory.yaml's riskLevel: high / "the only agent that +# authors code" note (mctl-agents issue-226 design.md). merge stays false -- +# the implementer opens PRs, it does not merge them. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: implementer-default + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: + - Read + - Write + - Edit + - Glob + - Grep + - Bash + - WebSearch + - WebFetch + - "mcp__mctl__*" + policyRef: production-code-author + permissions: + repository: + read: true + branchCreate: true + commit: true + pullRequestCreate: true + merge: false + kubernetes: none + network: approved-providers-only + mutationScopes: + - proposal-status + - target-repository-branch + - pull-request-create + budgetUsd: 20.00 + timeoutSeconds: 2400 + runtime: + entrypoint: orchestrator.run_implementer:implement + optionsBuilder: orchestrator.options:build_implementer_options + sandbox: + backend: argo + clusterWorkflowTemplate: mctl-agents-implement + approved: true + approval: + requiredBefore: + - code-authoring + evidence: + required: + - proposal + - diff + - tests + - pull-request diff --git a/platform-gitops/agent-platform/execution-profiles/issue-investigator-default/profile.yaml b/platform-gitops/agent-platform/execution-profiles/issue-investigator-default/profile.yaml new file mode 100644 index 000000000..80c7676d6 --- /dev/null +++ b/platform-gitops/agent-platform/execution-profiles/issue-investigator-default/profile.yaml @@ -0,0 +1,53 @@ +# ExecutionProfile for issue-investigator (ADR 007). Preserves today's +# effective values: $3.00 budget (cwft-mctl-agents-investigate.yaml +# ISSUE_INVESTIGATOR_BUDGET_USD) and the CWFT's 7200s activeDeadlineSeconds +# (no per-op ISSUE_INVESTIGATOR_TIMEOUT_SECONDS override exists today, so the +# pod-level deadline is the effective timeout). Read-only against the target +# repository: repository.commit/branchCreate/pullRequestCreate/merge are all +# false, and the only mutation scopes are writing the proposal triplet and +# commenting on the source issue -- neither requires a human approval gate. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: issue-investigator-default + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: issue_investigator + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: + - Read + - Grep + - Glob + - WebSearch + - WebFetch + - "mcp__mctl__*" + policyRef: read-only-investigation + permissions: + repository: + read: true + branchCreate: false + commit: false + pullRequestCreate: false + merge: false + kubernetes: none + network: approved-providers-only + mutationScopes: + - proposal-directory-write + - issue-comment + budgetUsd: 3.00 + timeoutSeconds: 7200 + runtime: + entrypoint: orchestrator.run_issue_investigator:investigate + optionsBuilder: orchestrator.options:build_issue_investigator_options + sandbox: + backend: argo + clusterWorkflowTemplate: mctl-agents-investigate + approved: true + approval: + requiredBefore: [] + evidence: + required: + - proposal diff --git a/platform-gitops/agent-platform/execution-profiles/shepherd-default/profile.yaml b/platform-gitops/agent-platform/execution-profiles/shepherd-default/profile.yaml new file mode 100644 index 000000000..7fc16db0c --- /dev/null +++ b/platform-gitops/agent-platform/execution-profiles/shepherd-default/profile.yaml @@ -0,0 +1,55 @@ +# ExecutionProfile for shepherd (ADR 007). Preserves today's effective +# values: $5.00 per-tick budget (cwft-mctl-agents-shepherd.yaml +# SHEPHERD_BUDGET_USD, raised from the Tier 3 spec's $1.00 default) and the +# CWFT's 7200s activeDeadlineSeconds (no per-tick SHEPHERD_TIMEOUT_SECONDS +# override exists today). No mcp__mctl__* tool and network stays +# approved-providers-only-scoped to GitHub, matching +# docs/agent-inventory.yaml's "mcp_servers={} / no mctl MCP access" note for +# this agent (mctl-agents issue-226 design.md). merge: true and the +# pull-request-merge mutation scope are the one high-risk capability this +# profile grants -- gated by the "merge" approval rule below. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: shepherd-default + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: shepherd + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: + - Read + - Grep + - Glob + - Bash + - WebFetch + policyRef: pr-shepherd + permissions: + repository: + read: true + branchCreate: false + commit: false + pullRequestCreate: false + merge: true + kubernetes: none + network: approved-providers-only + mutationScopes: + - proposal-status + - pull-request-merge + budgetUsd: 5.00 + timeoutSeconds: 7200 + runtime: + entrypoint: orchestrator.run_shepherd:tick + optionsBuilder: orchestrator.options:build_shepherd_options + sandbox: + backend: argo + clusterWorkflowTemplate: mctl-agents-shepherd + approved: true + approval: + requiredBefore: + - merge + evidence: + required: + - pull-request diff --git a/platform-gitops/agent-platform/policy.yaml b/platform-gitops/agent-platform/policy.yaml new file mode 100644 index 000000000..4d8635233 --- /dev/null +++ b/platform-gitops/agent-platform/policy.yaml @@ -0,0 +1,139 @@ +# Agent platform policy ceilings and reference catalogs (ADR 007, +# mctlhq/mctl-agents docs/adr/007-agent-definition-execution-profile-contract.md). +# +# This is the single source of "what an ExecutionProfile or +# ReleaseBindingIntent is allowed to claim" for scripts/validate-agent-platform.py. +# Every reference an ExecutionProfile or ReleaseBindingIntent makes (tool, +# skill, policyRef, mutation scope, model-policy task, runtime, sandbox) is +# checked against a list here. An entry not present in the matching list +# fails validation closed — see requirements.md's "Unknown tool/action/ +# policy/permission/model/skill/runtime references fail closed." +# +# Tools are never authorization by themselves (a tool grants SDK-level +# capability; a permission scope plus its approval rule grants the right to +# use it mutably). See knownTools[].category below — "mutating" tools still +# require the profile to declare a matching permission scope AND approval +# rule before they can touch anything outside the sandbox pod. + +apiVersion: agents.mctl.ai/v1alpha2 +kind: AgentPlatformPolicy +metadata: + name: platform-default + owner: platform + +spec: + # Hard ceilings. An ExecutionProfile's budgetUsd/timeoutSeconds must be + # <= these. Raising a ceiling is itself a reviewed mctl-gitops PR. + limits: + maxBudgetUsd: 25.00 + maxTimeoutSeconds: 7200 + + # Every SDK tool name an ExecutionProfile.spec.tools entry may reference. + # "category" is informational classification only (read/write/exec/mcp) — + # it does not grant capability. Actual mutation capability comes from + # spec.permissions + spec.approval, checked independently below. + knownTools: + - name: Read + category: read + - name: Grep + category: read + - name: Glob + category: read + - name: WebSearch + category: network + - name: WebFetch + category: network + - name: Write + category: write + - name: Edit + category: write + - name: Bash + category: exec + - name: "mcp__mctl__*" + category: mcp + + # Platform-skill references an ExecutionProfile.spec.skills entry may + # name. Source of truth for skill *content* stays + # platform-gitops/platform-skills/catalog/ (see + # scripts/validate-platform-skills.py) — this list is only the subset + # sanctioned for agent-platform execution profiles to reference. + knownSkills: + - mctl-platform + - git-flow + + # model-policy "task" names an ExecutionProfile.spec.modelPolicyRef.task + # may reference. The model-policy versions/profiles themselves are owned + # by mctl-agents config/model-policy.yaml; this catalog only enumerates + # which task keys mctl-gitops profiles are allowed to point at, so a typo + # fails closed here instead of resolving to an unintended model tier at + # runtime. + knownModelPolicyTasks: + - issue_investigator + - service_agent + - shepherd + + # policyRef values an ExecutionProfile may declare. Each is a named, + # externally-enforced behavior policy (prompt/guardrail scope) — this + # catalog is the GitOps-reviewed registry of policy *names*, not their + # content. + knownPolicies: + - read-only-investigation + - production-code-author + - pr-shepherd + + # Mutation scopes an ExecutionProfile.spec.permissions.mutationScopes + # entry may declare. "mutating: true" scopes require the profile's + # spec.approval.requiredBefore to include the mapped approval gate below + # — see requirements.md's "Mutation requires both a scoped permission and + # the applicable approval rule." + knownMutationScopes: + - name: proposal-status + mutating: true + requiresApproval: null # gitops-agents writes its own status; no human gate + - name: proposal-directory-write + mutating: true + requiresApproval: null # proposal drafting, not yet code/merge + - name: issue-comment + mutating: true + requiresApproval: null + - name: target-repository-branch + mutating: true + requiresApproval: code-authoring + - name: pull-request-create + mutating: true + requiresApproval: code-authoring + - name: pull-request-merge + mutating: true + requiresApproval: merge + - name: gitops-main-write + mutating: true + requiresApproval: gitops-main-write + + # Approval gate names an ExecutionProfile.spec.approval.requiredBefore + # entry may reference. + knownApprovalGates: + - code-authoring + - merge + - gitops-main-write + + # Approved sandbox backends and, per backend, the exact + # ClusterWorkflowTemplates a profile's spec.runtime.sandbox may name. + # Mirrors the templates actually deployed under + # platform-gitops/argo-workflows/cluster-templates/. An + # ExecutionProfile with sandbox.approved: true but a CWFT not listed + # here fails closed. + approvedSandboxes: + - backend: argo + clusterWorkflowTemplates: + - mctl-agents-investigate + - mctl-agents-implement + - mctl-agents-shepherd + + # Evidence kinds an ExecutionProfile.spec.evidence.required entry may + # name. + knownEvidenceKinds: + - proposal + - diff + - tests + - pull-request + - issue-comment diff --git a/platform-gitops/agent-platform/releases/shadow/implementer.yaml b/platform-gitops/agent-platform/releases/shadow/implementer.yaml new file mode 100644 index 000000000..4ef996dc8 --- /dev/null +++ b/platform-gitops/agent-platform/releases/shadow/implementer.yaml @@ -0,0 +1,34 @@ +# ReleaseBindingIntent fixture -- NOT a real registry release. See +# issue-investigator.yaml's header comment for why (no real mctl-api +# v1alpha2 rows exist yet). gitSha is a placeholder. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ReleaseBindingIntent +metadata: + agent: implementer + environment: shadow +spec: + sourceManifest: + repo: mctlhq/mctl-agents + path: agents/_manifests/implementer/agent.yaml + gitSha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + bindingSource: compatibility-fixture + promotable: false + registryLifecycle: + definition: published + profile: published + definition: + name: implementer + version: "3" + profileCompatibility: ">=1.0.0 <2.0.0" + profile: + name: implementer-default + version: "1.0.0" + bindingRevision: 12 + previousBindingRevision: 11 + history: + - bindingRevision: 11 + definition: {name: implementer, version: "2"} + profile: {name: implementer-default, version: "1.0.0"} + - bindingRevision: 10 + definition: {name: implementer, version: "2"} + profile: {name: implementer-default, version: "1.0.0"} diff --git a/platform-gitops/agent-platform/releases/shadow/issue-investigator.yaml b/platform-gitops/agent-platform/releases/shadow/issue-investigator.yaml new file mode 100644 index 000000000..fc87a7ed4 --- /dev/null +++ b/platform-gitops/agent-platform/releases/shadow/issue-investigator.yaml @@ -0,0 +1,34 @@ +# ReleaseBindingIntent fixture -- NOT a real registry release. There is no +# mctl-api-published v1alpha2 issue-investigator definition or profile +# version yet (that is #227/runtime-resolver + real registry rows, both out +# of this issue's scope). bindingSource/promotable mark this so it can never +# be read as production activation state. gitSha is a placeholder, not a +# real mctlhq/mctl-agents commit. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ReleaseBindingIntent +metadata: + agent: issue-investigator + environment: shadow +spec: + sourceManifest: + repo: mctlhq/mctl-agents + path: agents/_manifests/issue-investigator/agent.yaml + gitSha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + bindingSource: compatibility-fixture + promotable: false + registryLifecycle: + definition: published + profile: published + definition: + name: issue-investigator + version: "1" + profileCompatibility: ">=1.0.0 <2.0.0" + profile: + name: issue-investigator-default + version: "1.0.0" + bindingRevision: 6 + previousBindingRevision: 5 + history: + - bindingRevision: 5 + definition: {name: issue-investigator, version: "1"} + profile: {name: issue-investigator-default, version: "1.0.0"} diff --git a/platform-gitops/agent-platform/releases/shadow/shepherd.yaml b/platform-gitops/agent-platform/releases/shadow/shepherd.yaml new file mode 100644 index 000000000..8f9d5a00b --- /dev/null +++ b/platform-gitops/agent-platform/releases/shadow/shepherd.yaml @@ -0,0 +1,31 @@ +# ReleaseBindingIntent fixture -- NOT a real registry release. See +# issue-investigator.yaml's header comment for why (no real mctl-api +# v1alpha2 rows exist yet). gitSha is a placeholder. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ReleaseBindingIntent +metadata: + agent: shepherd + environment: shadow +spec: + sourceManifest: + repo: mctlhq/mctl-agents + path: agents/_manifests/shepherd/agent.yaml + gitSha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + bindingSource: compatibility-fixture + promotable: false + registryLifecycle: + definition: published + profile: published + definition: + name: shepherd + version: "1" + profileCompatibility: ">=1.0.0 <2.0.0" + profile: + name: shepherd-default + version: "1.0.0" + bindingRevision: 3 + previousBindingRevision: 2 + history: + - bindingRevision: 2 + definition: {name: shepherd, version: "1"} + profile: {name: shepherd-default, version: "1.0.0"} diff --git a/platform-gitops/agent-platform/schemas/execution-profile.schema.json b/platform-gitops/agent-platform/schemas/execution-profile.schema.json new file mode 100644 index 000000000..b8f034943 --- /dev/null +++ b/platform-gitops/agent-platform/schemas/execution-profile.schema.json @@ -0,0 +1,138 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://mctl.ai/schemas/agents/v1alpha2/execution-profile.schema.json", + "title": "ExecutionProfile", + "description": "agents.mctl.ai/v1alpha2 ExecutionProfile, per ADR 007 (mctlhq/mctl-agents docs/adr/007-agent-definition-execution-profile-contract.md). Reusable, independently versioned execution-shape constraints referenced by an AgentDefinition's executionProfileRef. Does not carry AgentDefinition identity/prompt/trigger data, and does not carry a global lifecycleState field -- registry publish/environment-release state lives in mctl-api, not here.", + "type": "object", + "required": ["apiVersion", "kind", "metadata", "spec"], + "additionalProperties": false, + "properties": { + "apiVersion": { "const": "agents.mctl.ai/v1alpha2" }, + "kind": { "const": "ExecutionProfile" }, + "metadata": { + "type": "object", + "required": ["name", "owner"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$" }, + "owner": { "type": "string", "minLength": 1 } + } + }, + "spec": { + "type": "object", + "required": [ + "version", + "modelPolicyRef", + "skills", + "tools", + "policyRef", + "permissions", + "budgetUsd", + "timeoutSeconds", + "runtime", + "approval", + "evidence" + ], + "additionalProperties": false, + "properties": { + "version": { + "type": "string", + "description": "Semver-ish content version of this profile file. Bumped on any spec change. Independent of the mctl-api registry's own publish version number.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "modelPolicyRef": { + "type": "object", + "required": ["task", "compatibility"], + "additionalProperties": false, + "properties": { + "task": { "type": "string", "minLength": 1 }, + "compatibility": { "type": "string", "minLength": 1 } + } + }, + "skills": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true + }, + "tools": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true, + "description": "SDK tool names only. Never authorization by itself -- see spec.permissions." + }, + "policyRef": { "type": "string", "minLength": 1 }, + "permissions": { + "type": "object", + "required": ["repository", "kubernetes", "network", "mutationScopes"], + "additionalProperties": false, + "properties": { + "repository": { + "type": "object", + "required": ["read", "branchCreate", "commit", "pullRequestCreate", "merge"], + "additionalProperties": false, + "properties": { + "read": { "type": "boolean" }, + "branchCreate": { "type": "boolean" }, + "commit": { "type": "boolean" }, + "pullRequestCreate": { "type": "boolean" }, + "merge": { "type": "boolean" } + } + }, + "kubernetes": { "enum": ["none", "read-only", "namespace-scoped"] }, + "network": { "enum": ["none", "approved-providers-only"] }, + "mutationScopes": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true + } + } + }, + "budgetUsd": { "type": "number", "exclusiveMinimum": 0 }, + "timeoutSeconds": { "type": "integer", "exclusiveMinimum": 0 }, + "runtime": { + "type": "object", + "required": ["entrypoint", "optionsBuilder", "sandbox"], + "additionalProperties": false, + "properties": { + "entrypoint": { "type": "string", "minLength": 1 }, + "optionsBuilder": { "type": "string", "minLength": 1 }, + "sandbox": { + "type": "object", + "required": ["backend", "clusterWorkflowTemplate", "approved"], + "additionalProperties": false, + "properties": { + "backend": { "type": "string", "minLength": 1 }, + "clusterWorkflowTemplate": { "type": "string", "minLength": 1 }, + "approved": { "type": "boolean" } + } + } + } + }, + "approval": { + "type": "object", + "required": ["requiredBefore"], + "additionalProperties": false, + "properties": { + "requiredBefore": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true + } + } + }, + "evidence": { + "type": "object", + "required": ["required"], + "additionalProperties": false, + "properties": { + "required": { + "type": "array", + "items": { "type": "string" }, + "uniqueItems": true + } + } + } + } + } + } +} diff --git a/platform-gitops/agent-platform/schemas/release-binding-intent.schema.json b/platform-gitops/agent-platform/schemas/release-binding-intent.schema.json new file mode 100644 index 000000000..1e9e78557 --- /dev/null +++ b/platform-gitops/agent-platform/schemas/release-binding-intent.schema.json @@ -0,0 +1,124 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://mctl.ai/schemas/agents/v1alpha2/release-binding-intent.schema.json", + "title": "ReleaseBindingIntent", + "description": "agents.mctl.ai/v1alpha2 ReleaseBindingIntent, per ADR 007. This is reviewed intent, not release history: mctl-api's registry remains the sole authority for immutable published versions and atomic environment ReleaseBinding history. A ReleaseBindingIntent pins exactly one compatible (definition, profile) pair plus the sourceManifest that grounds the canonical AgentDefinition. Every fixture checked in under this schema before real registry-backed versions exist must be marked bindingSource: compatibility-fixture and promotable: false so it can never be read as production activation state.", + "type": "object", + "required": ["apiVersion", "kind", "metadata", "spec"], + "additionalProperties": false, + "properties": { + "apiVersion": { "const": "agents.mctl.ai/v1alpha2" }, + "kind": { "const": "ReleaseBindingIntent" }, + "metadata": { + "type": "object", + "required": ["agent", "environment"], + "additionalProperties": false, + "properties": { + "agent": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$" }, + "environment": { "enum": ["shadow", "production"] } + } + }, + "spec": { + "type": "object", + "required": [ + "sourceManifest", + "bindingSource", + "promotable", + "registryLifecycle", + "definition", + "profile", + "bindingRevision" + ], + "additionalProperties": false, + "properties": { + "sourceManifest": { + "type": "object", + "required": ["repo", "path", "gitSha"], + "additionalProperties": false, + "properties": { + "repo": { "type": "string", "minLength": 1 }, + "path": { "type": "string", "minLength": 1 }, + "gitSha": { "type": "string", "pattern": "^[0-9a-f]{40}$" } + } + }, + "bindingSource": { + "enum": ["compatibility-fixture", "registry"], + "description": "compatibility-fixture = checked-in test/shadow fixture (this issue). registry = resolved from a real mctl-api published version (future work, #227/runtime resolver)." + }, + "promotable": { + "type": "boolean", + "description": "Must be false for any bindingSource: compatibility-fixture. Only a bindingSource: registry intent, resolved from real immutable versions, may ever be true -- and even then this file is reviewed intent, not the promotion action itself." + }, + "registryLifecycle": { + "type": "object", + "required": ["definition", "profile"], + "additionalProperties": false, + "properties": { + "definition": { "enum": ["published", "deprecated", "disabled"] }, + "profile": { "enum": ["published", "deprecated", "disabled"] } + } + }, + "definition": { + "type": "object", + "required": ["name", "version", "profileCompatibility"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 }, + "profileCompatibility": { + "type": "string", + "minLength": 1, + "description": "The canonical AgentDefinition's executionProfileRef.compatibility constraint, mirrored here because mctl-gitops CI cannot read the mctl-agents source file at sourceManifest.path directly. Checked against profile.version -- compatibility is never read from the profile side." + } + } + }, + "profile": { + "type": "object", + "required": ["name", "version"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 } + } + }, + "bindingRevision": { "type": "integer", "minimum": 1 }, + "previousBindingRevision": { "type": "integer", "minimum": 1 }, + "history": { + "type": "array", + "description": "Deterministic local fixture of prior binding revisions for this (agent, environment), used only to validate exact-pair rollback semantics before a real mctl-api registry exists. Not itself a release-history store -- see requirements.md's 'must not create a second AgentDefinition body or release database.'", + "items": { + "type": "object", + "required": ["bindingRevision", "definition", "profile"], + "additionalProperties": false, + "properties": { + "bindingRevision": { "type": "integer", "minimum": 1 }, + "definition": { + "type": "object", + "required": ["name", "version"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 } + } + }, + "profile": { + "type": "object", + "required": ["name", "version"], + "additionalProperties": false, + "properties": { + "name": { "type": "string", "minLength": 1 }, + "version": { "type": "string", "minLength": 1 } + } + } + } + } + }, + "rollbackOf": { + "type": "integer", + "minimum": 1, + "description": "Set only when this intent represents a rollback. Must equal previousBindingRevision, and this intent's (definition, profile) pair must exactly match the history entry recorded under that revision -- an independently chosen or synthetic mix of the two halves is rejected." + } + } + } + } +} diff --git a/scripts/tests/fixtures/agent-platform/invalid/budget-exceeds-ceiling/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/budget-exceeds-ceiling/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..8f9d77bf1 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/budget-exceeds-ceiling/execution-profiles/example/profile.yaml @@ -0,0 +1,29 @@ +# Invalid: budgetUsd 999.00 exceeds policy.yaml spec.limits.maxBudgetUsd. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 999.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/global-lifecycle-state/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/global-lifecycle-state/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..49cd8ae88 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/global-lifecycle-state/execution-profiles/example/profile.yaml @@ -0,0 +1,34 @@ +# Invalid: spec.lifecycleState is not a recognized field. +# execution-profile.schema.json sets additionalProperties: false on spec -- +# no global mutable lifecycleState field is allowed here (ADR 007: +# registry version owns published/deprecated/disabled, environment binding +# derives active). +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + lifecycleState: active + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/missing-bounds/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/missing-bounds/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..882938897 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/missing-bounds/execution-profiles/example/profile.yaml @@ -0,0 +1,27 @@ +# Invalid: spec.budgetUsd and spec.timeoutSeconds are required bounds. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/missing-owner/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/missing-owner/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..b0fa317b8 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/missing-owner/execution-profiles/example/profile.yaml @@ -0,0 +1,28 @@ +# Invalid: metadata.owner is required by execution-profile.schema.json. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/missing-permissions/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/missing-permissions/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..9bbd7ff57 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/missing-permissions/execution-profiles/example/profile.yaml @@ -0,0 +1,24 @@ +# Invalid: spec.permissions is required by execution-profile.schema.json. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + policyRef: read-only-investigation + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/missing-policy-ref/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/missing-policy-ref/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..612829b87 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/missing-policy-ref/execution-profiles/example/profile.yaml @@ -0,0 +1,28 @@ +# Invalid: spec.policyRef is required by execution-profile.schema.json. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/mutation-without-approval/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/mutation-without-approval/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..39f47661d --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/mutation-without-approval/execution-profiles/example/profile.yaml @@ -0,0 +1,32 @@ +# Invalid: mutationScopes includes target-repository-branch, which +# policy.yaml maps to the "code-authoring" approval gate, but +# approval.requiredBefore is empty. Mutation requires both a scoped +# permission and the applicable approval rule. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read, Write, Bash] + policyRef: production-code-author + permissions: + repository: {read: true, branchCreate: true, commit: true, pullRequestCreate: true, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [target-repository-branch] + budgetUsd: 20.00 + timeoutSeconds: 2400 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-implement, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal, diff] diff --git a/scripts/tests/fixtures/agent-platform/invalid/policy-missing-ceiling/policy.yaml b/scripts/tests/fixtures/agent-platform/invalid/policy-missing-ceiling/policy.yaml new file mode 100644 index 000000000..a5bf97939 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/policy-missing-ceiling/policy.yaml @@ -0,0 +1,24 @@ +# Invalid: spec.limits.maxBudgetUsd is dropped (only maxTimeoutSeconds is +# declared). A missing ceiling must fail closed at policy load rather than +# leaving the budget check unconstrained -- see require_limit() in +# scripts/validate-agent-platform.py. +# +# The selftest picks up this file instead of the real policy.yaml because it +# sits at the fixture case root (case_policy_path()). +apiVersion: agents.mctl.ai/v1alpha2 +kind: AgentPlatformPolicy +metadata: + name: platform-default + owner: platform + +spec: + limits: + maxTimeoutSeconds: 7200 + knownTools: [] + knownSkills: [] + knownModelPolicyTasks: [] + knownPolicies: [] + knownEvidenceKinds: [] + knownMutationScopes: [] + knownApprovalGates: [] + approvedSandboxes: [] diff --git a/scripts/tests/fixtures/agent-platform/invalid/release-ambiguous-profile/execution-profiles/a/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/release-ambiguous-profile/execution-profiles/a/profile.yaml new file mode 100644 index 000000000..827a51c14 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/release-ambiguous-profile/execution-profiles/a/profile.yaml @@ -0,0 +1,31 @@ +# Invalid together with execution-profiles/b/profile.yaml: both declare +# metadata.name "dup-profile", making the release intent's profile +# reference ambiguous. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: dup-profile + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/release-ambiguous-profile/execution-profiles/b/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/release-ambiguous-profile/execution-profiles/b/profile.yaml new file mode 100644 index 000000000..00fc63fc8 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/release-ambiguous-profile/execution-profiles/b/profile.yaml @@ -0,0 +1,31 @@ +# Invalid together with execution-profiles/a/profile.yaml: both declare +# metadata.name "dup-profile", making the release intent's profile +# reference ambiguous. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: dup-profile + owner: platform +spec: + version: "2.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/release-ambiguous-profile/releases/shadow/example.yaml b/scripts/tests/fixtures/agent-platform/invalid/release-ambiguous-profile/releases/shadow/example.yaml new file mode 100644 index 000000000..f777e582e --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/release-ambiguous-profile/releases/shadow/example.yaml @@ -0,0 +1,23 @@ +apiVersion: agents.mctl.ai/v1alpha2 +kind: ReleaseBindingIntent +metadata: + agent: example + environment: shadow +spec: + sourceManifest: + repo: mctlhq/mctl-agents + path: agents/_manifests/example/agent.yaml + gitSha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + bindingSource: compatibility-fixture + promotable: false + registryLifecycle: + definition: published + profile: published + definition: + name: example + version: "1" + profileCompatibility: ">=1.0.0 <2.0.0" + profile: + name: dup-profile + version: "1.0.0" + bindingRevision: 1 diff --git a/scripts/tests/fixtures/agent-platform/invalid/release-disabled-version/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/release-disabled-version/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..4fe9bccfe --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/release-disabled-version/execution-profiles/example/profile.yaml @@ -0,0 +1,28 @@ +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/release-disabled-version/releases/shadow/example.yaml b/scripts/tests/fixtures/agent-platform/invalid/release-disabled-version/releases/shadow/example.yaml new file mode 100644 index 000000000..ab8b3c95e --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/release-disabled-version/releases/shadow/example.yaml @@ -0,0 +1,25 @@ +# Invalid: registryLifecycle.profile is disabled -- resolve must treat a +# disabled version as unresolvable regardless of everything else matching. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ReleaseBindingIntent +metadata: + agent: example + environment: shadow +spec: + sourceManifest: + repo: mctlhq/mctl-agents + path: agents/_manifests/example/agent.yaml + gitSha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + bindingSource: compatibility-fixture + promotable: false + registryLifecycle: + definition: published + profile: disabled + definition: + name: example + version: "1" + profileCompatibility: ">=1.0.0 <2.0.0" + profile: + name: example + version: "1.0.0" + bindingRevision: 1 diff --git a/scripts/tests/fixtures/agent-platform/invalid/release-incompatible-version/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/release-incompatible-version/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..4fe9bccfe --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/release-incompatible-version/execution-profiles/example/profile.yaml @@ -0,0 +1,28 @@ +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/release-incompatible-version/releases/shadow/example.yaml b/scripts/tests/fixtures/agent-platform/invalid/release-incompatible-version/releases/shadow/example.yaml new file mode 100644 index 000000000..302523156 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/release-incompatible-version/releases/shadow/example.yaml @@ -0,0 +1,26 @@ +# Invalid: the catalog profile is version 1.0.0, but the definition's +# profileCompatibility here demands >=2.0.0 <3.0.0 -- no version satisfies +# both "pinned to 1.0.0" and this constraint. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ReleaseBindingIntent +metadata: + agent: example + environment: shadow +spec: + sourceManifest: + repo: mctlhq/mctl-agents + path: agents/_manifests/example/agent.yaml + gitSha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + bindingSource: compatibility-fixture + promotable: false + registryLifecycle: + definition: published + profile: published + definition: + name: example + version: "1" + profileCompatibility: ">=2.0.0 <3.0.0" + profile: + name: example + version: "1.0.0" + bindingRevision: 1 diff --git a/scripts/tests/fixtures/agent-platform/invalid/release-missing-profile/releases/shadow/example.yaml b/scripts/tests/fixtures/agent-platform/invalid/release-missing-profile/releases/shadow/example.yaml new file mode 100644 index 000000000..25e131f5d --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/release-missing-profile/releases/shadow/example.yaml @@ -0,0 +1,26 @@ +# Invalid: profile.name "does-not-exist" has no matching +# execution-profiles/*/profile.yaml in this catalog root (this fixture +# directory deliberately has no execution-profiles/ dir at all). +apiVersion: agents.mctl.ai/v1alpha2 +kind: ReleaseBindingIntent +metadata: + agent: example + environment: shadow +spec: + sourceManifest: + repo: mctlhq/mctl-agents + path: agents/_manifests/example/agent.yaml + gitSha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + bindingSource: compatibility-fixture + promotable: false + registryLifecycle: + definition: published + profile: published + definition: + name: example + version: "1" + profileCompatibility: ">=1.0.0 <2.0.0" + profile: + name: does-not-exist + version: "1.0.0" + bindingRevision: 1 diff --git a/scripts/tests/fixtures/agent-platform/invalid/rollback-independent-half/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/rollback-independent-half/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..4fe9bccfe --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/rollback-independent-half/execution-profiles/example/profile.yaml @@ -0,0 +1,28 @@ +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/rollback-independent-half/releases/shadow/example.yaml b/scripts/tests/fixtures/agent-platform/invalid/rollback-independent-half/releases/shadow/example.yaml new file mode 100644 index 000000000..a74095e56 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/rollback-independent-half/releases/shadow/example.yaml @@ -0,0 +1,33 @@ +# Invalid: rollbackOf 5 replays history revision 5's profile half +# (example@1.0.0) but pairs it with a definition version ("2") that +# revision 5 never recorded (it recorded "1") -- an independently chosen +# mix of the two halves. Independent rollback of either half is rejected. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ReleaseBindingIntent +metadata: + agent: example + environment: shadow +spec: + sourceManifest: + repo: mctlhq/mctl-agents + path: agents/_manifests/example/agent.yaml + gitSha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + bindingSource: compatibility-fixture + promotable: false + registryLifecycle: + definition: published + profile: published + definition: + name: example + version: "2" + profileCompatibility: ">=1.0.0 <2.0.0" + profile: + name: example + version: "1.0.0" + bindingRevision: 6 + previousBindingRevision: 5 + rollbackOf: 5 + history: + - bindingRevision: 5 + definition: {name: example, version: "1"} + profile: {name: example, version: "1.0.0"} diff --git a/scripts/tests/fixtures/agent-platform/invalid/unapproved-sandbox/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/unapproved-sandbox/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..2a98303da --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/unapproved-sandbox/execution-profiles/example/profile.yaml @@ -0,0 +1,29 @@ +# Invalid: runtime.sandbox.approved is false. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: false} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/unknown-model-policy-task/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/unknown-model-policy-task/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..3fb592a96 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/unknown-model-policy-task/execution-profiles/example/profile.yaml @@ -0,0 +1,29 @@ +# Invalid: "made-up-task" is not in policy.yaml knownModelPolicyTasks. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: made-up-task + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/unknown-skill/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/unknown-skill/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..9b9f542a6 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/unknown-skill/execution-profiles/example/profile.yaml @@ -0,0 +1,29 @@ +# Invalid: "made-up-skill" is not in policy.yaml knownSkills. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [made-up-skill] + tools: [Read] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/invalid/unknown-tool/execution-profiles/example/profile.yaml b/scripts/tests/fixtures/agent-platform/invalid/unknown-tool/execution-profiles/example/profile.yaml new file mode 100644 index 000000000..7190ab911 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/invalid/unknown-tool/execution-profiles/example/profile.yaml @@ -0,0 +1,29 @@ +# Invalid: "Frobnicate" is not in policy.yaml knownTools. +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: [Read, Frobnicate] + policyRef: read-only-investigation + permissions: + repository: {read: true, branchCreate: false, commit: false, pullRequestCreate: false, merge: false} + kubernetes: none + network: approved-providers-only + mutationScopes: [proposal-directory-write] + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: {backend: argo, clusterWorkflowTemplate: mctl-agents-investigate, approved: true} + approval: + requiredBefore: [] + evidence: + required: [proposal] diff --git a/scripts/tests/fixtures/agent-platform/valid/minimal-profile-only/execution-profiles/valid-example/profile.yaml b/scripts/tests/fixtures/agent-platform/valid/minimal-profile-only/execution-profiles/valid-example/profile.yaml new file mode 100644 index 000000000..7f623bdc0 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/valid/minimal-profile-only/execution-profiles/valid-example/profile.yaml @@ -0,0 +1,40 @@ +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: valid-example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: + - Read + - Grep + policyRef: read-only-investigation + permissions: + repository: + read: true + branchCreate: false + commit: false + pullRequestCreate: false + merge: false + kubernetes: none + network: approved-providers-only + mutationScopes: + - proposal-directory-write + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: + backend: argo + clusterWorkflowTemplate: mctl-agents-investigate + approved: true + approval: + requiredBefore: [] + evidence: + required: + - proposal diff --git a/scripts/tests/fixtures/agent-platform/valid/rollback-replay/execution-profiles/rollback-example/profile.yaml b/scripts/tests/fixtures/agent-platform/valid/rollback-replay/execution-profiles/rollback-example/profile.yaml new file mode 100644 index 000000000..5901d5c40 --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/valid/rollback-replay/execution-profiles/rollback-example/profile.yaml @@ -0,0 +1,39 @@ +apiVersion: agents.mctl.ai/v1alpha2 +kind: ExecutionProfile +metadata: + name: rollback-example + owner: platform +spec: + version: "1.0.0" + modelPolicyRef: + task: service_agent + compatibility: ">=1.0.0 <2.0.0" + skills: [] + tools: + - Read + policyRef: read-only-investigation + permissions: + repository: + read: true + branchCreate: false + commit: false + pullRequestCreate: false + merge: false + kubernetes: none + network: approved-providers-only + mutationScopes: + - proposal-directory-write + budgetUsd: 3.00 + timeoutSeconds: 900 + runtime: + entrypoint: orchestrator.run_example:main + optionsBuilder: orchestrator.options:build_example_options + sandbox: + backend: argo + clusterWorkflowTemplate: mctl-agents-investigate + approved: true + approval: + requiredBefore: [] + evidence: + required: + - proposal diff --git a/scripts/tests/fixtures/agent-platform/valid/rollback-replay/releases/shadow/rollback-example.yaml b/scripts/tests/fixtures/agent-platform/valid/rollback-replay/releases/shadow/rollback-example.yaml new file mode 100644 index 000000000..6905b3c7e --- /dev/null +++ b/scripts/tests/fixtures/agent-platform/valid/rollback-replay/releases/shadow/rollback-example.yaml @@ -0,0 +1,29 @@ +apiVersion: agents.mctl.ai/v1alpha2 +kind: ReleaseBindingIntent +metadata: + agent: rollback-example + environment: shadow +spec: + sourceManifest: + repo: mctlhq/mctl-agents + path: agents/_manifests/rollback-example/agent.yaml + gitSha: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef" + bindingSource: compatibility-fixture + promotable: false + registryLifecycle: + definition: published + profile: published + definition: + name: rollback-example + version: "1" + profileCompatibility: ">=1.0.0 <2.0.0" + profile: + name: rollback-example + version: "1.0.0" + bindingRevision: 6 + previousBindingRevision: 5 + rollbackOf: 5 + history: + - bindingRevision: 5 + definition: {name: rollback-example, version: "1"} + profile: {name: rollback-example, version: "1.0.0"} diff --git a/scripts/validate-agent-platform.py b/scripts/validate-agent-platform.py new file mode 100755 index 000000000..1e59c0209 --- /dev/null +++ b/scripts/validate-agent-platform.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python3 +"""Validate the agent platform catalog (ADR 007) under +platform-gitops/agent-platform/. + +ADR 007 (mctlhq/mctl-agents docs/adr/007-agent-definition-execution-profile-contract.md) +splits agent execution into a canonical AgentDefinition (mctl-agents Git), +an independently versioned ExecutionProfile (this repo, reviewed in Git), +immutable published versions and atomic environment ReleaseBinding history +(mctl-api registry), and one immutable ExecutionPlan/ExecutionRecord per run +(runtime resolver, #227). This script owns only the mctl-gitops half: the +ExecutionProfile catalog, the ReleaseBindingIntent fixtures that pin an +exact (definition, profile) pair plus the canonical AgentDefinition's +sourceManifest, and platform-wide policy ceilings. It does not, and must +not, become a second AgentDefinition body or a release-history database -- +see requirements.md's fixed decisions. + +Every check here fails closed: an unrecognized tool/skill/policy/mutation- +scope/approval-gate/model-policy-task/sandbox reference, a budget or +timeout above the policy ceiling, a mutating scope with no matching +approval gate, a release intent that is not exactly one compatible +(definition, profile) pair, or a rollback that does not exactly replay a +previously recorded pair, is a validation error -- never a silent +fallback. + +Usage: + scripts/validate-agent-platform.py validate the real catalog + scripts/validate-agent-platform.py --selftest also replay the fixtures + under + scripts/tests/fixtures/agent-platform/ + and assert every valid/ + case passes and every + invalid/ case fails +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import re +import sys + +import jsonschema +import yaml + +ROOT = pathlib.Path(__file__).resolve().parents[1] +CATALOG = ROOT / "platform-gitops" / "agent-platform" +SCHEMAS = CATALOG / "schemas" +FIXTURES_ROOT = ROOT / "scripts" / "tests" / "fixtures" / "agent-platform" + +NAME_RE = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}[a-z0-9]$") +COMPAT_RE = re.compile(r"(>=|<=|==|>|<)\s*([0-9]+(?:\.[0-9]+)*)") + + +class CatalogValidationError(Exception): + """Raised for a structural problem that stops validation of one file.""" + + +def load_yaml(path: pathlib.Path): + with path.open("r", encoding="utf-8") as f: + return yaml.safe_load(f) + + +def load_json(path: pathlib.Path): + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def parse_version(version: str) -> tuple: + parts = version.split(".") + out = [] + for part in parts: + if not part.isdigit(): + raise CatalogValidationError(f"non-numeric version component {part!r} in {version!r}") + out.append(int(part)) + return tuple(out) + + +def parse_compatibility(constraint: str): + matches = COMPAT_RE.findall(constraint) + if not matches: + raise CatalogValidationError(f"unparseable compatibility constraint {constraint!r}") + return [(op, parse_version(bound)) for op, bound in matches] + + +def _pad(a: tuple, b: tuple): + n = max(len(a), len(b)) + return a + (0,) * (n - len(a)), b + (0,) * (n - len(b)) + + +def version_satisfies(version: str, constraint: str) -> bool: + version_tuple = parse_version(version) + for op, bound in parse_compatibility(constraint): + a, b = _pad(version_tuple, bound) + if op == ">=" and not a >= b: + return False + if op == "<=" and not a <= b: + return False + if op == ">" and not a > b: + return False + if op == "<" and not a < b: + return False + if op == "==" and not a == b: + return False + return True + + +# -------------------------------------------------------------------------- +# Policy +# -------------------------------------------------------------------------- + + +def require_limit(limits: dict, key: str): + """Read a policy ceiling, failing closed when it is missing or non-numeric. + + policy.yaml has no JSON Schema, so a mistyped or dropped ceiling would + otherwise leave the limit unset and skip the ceiling check entirely -- + the one place this validator could fail *open*. Every other policy list + degrades to an empty collection, which correctly makes every reference + "unknown"; these two scalars must raise instead. + """ + value = limits.get(key) + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise CatalogValidationError( + f"spec.limits.{key} is missing or non-numeric ({value!r});" + " policy ceilings must be declared explicitly (fail closed)" + ) + return value + + +class Policy: + def __init__(self, doc: dict): + spec = doc.get("spec") or {} + limits = spec.get("limits") or {} + self.max_budget_usd = require_limit(limits, "maxBudgetUsd") + self.max_timeout_seconds = require_limit(limits, "maxTimeoutSeconds") + self.known_tools = {t["name"] for t in spec.get("knownTools") or []} + self.known_skills = set(spec.get("knownSkills") or []) + self.known_model_policy_tasks = set(spec.get("knownModelPolicyTasks") or []) + self.known_policies = set(spec.get("knownPolicies") or []) + self.known_evidence_kinds = set(spec.get("knownEvidenceKinds") or []) + self.mutation_scopes = { + s["name"]: s for s in spec.get("knownMutationScopes") or [] + } + self.known_approval_gates = set(spec.get("knownApprovalGates") or []) + self.approved_sandboxes = { + entry["backend"]: set(entry.get("clusterWorkflowTemplates") or []) + for entry in spec.get("approvedSandboxes") or [] + } + + @classmethod + def load(cls, path: pathlib.Path) -> "Policy": + return cls(load_yaml(path)) + + +def load_policy(errors: list, path: pathlib.Path | None = None) -> Policy | None: + path = path if path is not None else CATALOG / "policy.yaml" + if not path.exists(): + errors.append(f"{path}: missing policy.yaml") + return None + try: + return Policy.load(path) + except Exception as exc: # noqa: BLE001 - report and continue + errors.append(f"{path}: {exc}") + return None + + +# -------------------------------------------------------------------------- +# ExecutionProfile +# -------------------------------------------------------------------------- + + +def validate_profile_file(path: pathlib.Path, schema: dict, policy: Policy, errors: list): + """Returns (name, version) on success, or None on any error (appended to errors).""" + try: + doc = load_yaml(path) + except Exception as exc: # noqa: BLE001 + errors.append(f"{path}: {exc}") + return None + + try: + jsonschema.validate(doc, schema) + except jsonschema.ValidationError as exc: + errors.append(f"{path}: schema: {exc.message} at {'/'.join(str(p) for p in exc.absolute_path)}") + return None + + spec = doc["spec"] + name = doc["metadata"]["name"] + + # -- reference checks: fail closed on anything not in policy.yaml -- + unknown_tools = set(spec["tools"]) - policy.known_tools + for tool in sorted(unknown_tools): + errors.append(f"{path}: unknown tool {tool!r} (not in policy.yaml knownTools)") + + unknown_skills = set(spec["skills"]) - policy.known_skills + for skill in sorted(unknown_skills): + errors.append(f"{path}: unknown skill {skill!r} (not in policy.yaml knownSkills)") + + if spec["policyRef"] not in policy.known_policies: + errors.append(f"{path}: unknown policyRef {spec['policyRef']!r} (not in policy.yaml knownPolicies)") + + model_task = spec["modelPolicyRef"]["task"] + if model_task not in policy.known_model_policy_tasks: + errors.append(f"{path}: unknown modelPolicyRef.task {model_task!r} (not in policy.yaml knownModelPolicyTasks)") + + try: + parse_compatibility(spec["modelPolicyRef"]["compatibility"]) + except CatalogValidationError as exc: + errors.append(f"{path}: modelPolicyRef.compatibility: {exc}") + + for kind in spec["evidence"]["required"]: + if kind not in policy.known_evidence_kinds: + errors.append(f"{path}: unknown evidence kind {kind!r} (not in policy.yaml knownEvidenceKinds)") + + # -- budget / timeout ceilings -- + if spec["budgetUsd"] > policy.max_budget_usd: + errors.append( + f"{path}: budgetUsd {spec['budgetUsd']} exceeds policy ceiling {policy.max_budget_usd}" + ) + if spec["timeoutSeconds"] > policy.max_timeout_seconds: + errors.append( + f"{path}: timeoutSeconds {spec['timeoutSeconds']} exceeds policy ceiling {policy.max_timeout_seconds}" + ) + + # -- sandbox approval -- + sandbox = spec["runtime"]["sandbox"] + approved_cwfts = policy.approved_sandboxes.get(sandbox["backend"], set()) + if not sandbox["approved"]: + errors.append(f"{path}: runtime.sandbox.approved is false") + elif sandbox["clusterWorkflowTemplate"] not in approved_cwfts: + errors.append( + f"{path}: unapproved sandbox {sandbox['backend']}/{sandbox['clusterWorkflowTemplate']}" + " (not in policy.yaml approvedSandboxes)" + ) + + # -- mutation requires scope + approval -- + approval_gates = set(spec["approval"]["requiredBefore"]) + for gate in approval_gates: + if gate not in policy.known_approval_gates: + errors.append(f"{path}: unknown approval gate {gate!r} (not in policy.yaml knownApprovalGates)") + + mutation_scopes = spec["permissions"]["mutationScopes"] + for scope_name in mutation_scopes: + scope = policy.mutation_scopes.get(scope_name) + if scope is None: + errors.append(f"{path}: unknown mutationScope {scope_name!r} (not in policy.yaml knownMutationScopes)") + continue + required_gate = scope.get("requiresApproval") + if scope.get("mutating") and required_gate and required_gate not in approval_gates: + errors.append( + f"{path}: mutationScope {scope_name!r} requires approval gate {required_gate!r}" + " in spec.approval.requiredBefore, but it is absent" + " (mutation requires both a scoped permission and the applicable approval rule)" + ) + + return name, spec["version"] + + +# -------------------------------------------------------------------------- +# ReleaseBindingIntent +# -------------------------------------------------------------------------- + + +def validate_release_file( + path: pathlib.Path, + schema: dict, + policy: Policy, + profiles_by_name: dict, + errors: list, +): + try: + doc = load_yaml(path) + except Exception as exc: # noqa: BLE001 + errors.append(f"{path}: {exc}") + return + + try: + jsonschema.validate(doc, schema) + except jsonschema.ValidationError as exc: + errors.append(f"{path}: schema: {exc.message} at {'/'.join(str(p) for p in exc.absolute_path)}") + return + + spec = doc["spec"] + + if spec["bindingSource"] == "compatibility-fixture" and spec["promotable"] is not False: + errors.append( + f"{path}: bindingSource: compatibility-fixture requires promotable: false" + " (a fixture must never be interpreted as production activation state)" + ) + + if spec["registryLifecycle"]["definition"] == "disabled": + errors.append(f"{path}: definition {spec['definition']['name']!r} registryLifecycle is disabled") + if spec["registryLifecycle"]["profile"] == "disabled": + errors.append(f"{path}: profile {spec['profile']['name']!r} registryLifecycle is disabled") + + # -- exact-pair resolution: profile must resolve to exactly one known + # (name, version) in the profile catalog this validation run loaded. -- + profile_name = spec["profile"]["name"] + profile_version = spec["profile"]["version"] + candidates = profiles_by_name.get(profile_name) + if candidates is None: + errors.append(f"{path}: release references unknown profile {profile_name!r}") + elif len(candidates) > 1: + errors.append( + f"{path}: profile name {profile_name!r} is ambiguous" + f" ({len(candidates)} catalog entries share this name)" + ) + elif candidates[0] != profile_version: + errors.append( + f"{path}: release pins profile version {profile_version!r} but the catalog" + f" entry for {profile_name!r} is version {candidates[0]!r}" + ) + + # -- compatibility: definition's constraint against the concrete + # selected profile version. Never read from the profile side. -- + try: + if not version_satisfies(profile_version, spec["definition"]["profileCompatibility"]): + errors.append( + f"{path}: profile version {profile_version!r} does not satisfy definition" + f" profileCompatibility {spec['definition']['profileCompatibility']!r}" + ) + except CatalogValidationError as exc: + errors.append(f"{path}: {exc}") + + # -- exact-pair rollback: rollbackOf must replay one recorded prior + # pair verbatim, never an independently chosen or synthetic mix. -- + rollback_of = spec.get("rollbackOf") + if rollback_of is not None: + history = {entry["bindingRevision"]: entry for entry in spec.get("history") or []} + recorded = history.get(rollback_of) + if recorded is None: + errors.append( + f"{path}: rollbackOf {rollback_of} has no matching entry in spec.history" + " (rollback must select a prior recorded pair, not an independently chosen one)" + ) + else: + # spec.definition also carries profileCompatibility, which a + # history entry does not -- compare only name/version so that + # field doesn't produce a false mismatch. + current_definition = { + "name": spec["definition"]["name"], + "version": spec["definition"]["version"], + } + if recorded["definition"] != current_definition or recorded["profile"] != spec["profile"]: + errors.append( + f"{path}: rollbackOf {rollback_of} does not exactly match its recorded" + " (definition, profile) pair -- independent rollback of either half is rejected" + ) + if spec.get("previousBindingRevision") != rollback_of: + errors.append( + f"{path}: rollbackOf {rollback_of} must equal previousBindingRevision" + f" ({spec.get('previousBindingRevision')!r})" + ) + + +# -------------------------------------------------------------------------- +# Catalog walk +# -------------------------------------------------------------------------- + + +def validate_catalog( + root: pathlib.Path, + profile_schema: dict, + release_schema: dict, + policy: Policy, + errors: list, +): + profiles_by_name: dict[str, list[str]] = {} + + profiles_dir = root / "execution-profiles" + for profile_path in sorted(profiles_dir.glob("*/profile.yaml")) if profiles_dir.is_dir() else []: + result = validate_profile_file(profile_path, profile_schema, policy, errors) + if result is None: + continue + name, version = result + profiles_by_name.setdefault(name, []).append(version) + + # profiles_by_name maps name -> list of versions seen across every + # execution-profiles/*/profile.yaml in this catalog root; more than one + # entry for the same name is exactly the "ambiguous version" case, + # handled in validate_release_file. + releases_dir = root / "releases" + for release_path in sorted(releases_dir.glob("*/*.yaml")) if releases_dir.is_dir() else []: + validate_release_file(release_path, release_schema, policy, profiles_by_name, errors) + + +def run(root: pathlib.Path) -> list: + errors: list = [] + policy = load_policy(errors) + if policy is None: + return errors + profile_schema = load_json(SCHEMAS / "execution-profile.schema.json") + release_schema = load_json(SCHEMAS / "release-binding-intent.schema.json") + validate_catalog(root, profile_schema, release_schema, policy, errors) + return errors + + +def case_policy_path(case_dir: pathlib.Path) -> pathlib.Path: + """A fixture may ship its own policy.yaml to exercise policy loading.""" + fixture_policy = case_dir / "policy.yaml" + return fixture_policy if fixture_policy.exists() else CATALOG / "policy.yaml" + + +def run_selftest() -> list: + problems: list = [] + profile_schema = load_json(SCHEMAS / "execution-profile.schema.json") + release_schema = load_json(SCHEMAS / "release-binding-intent.schema.json") + + for case_dir in sorted((FIXTURES_ROOT / "valid").glob("*")) if (FIXTURES_ROOT / "valid").is_dir() else []: + if not case_dir.is_dir(): + continue + errors: list = [] + policy = load_policy(errors, case_policy_path(case_dir)) + if policy is not None: + validate_catalog(case_dir, profile_schema, release_schema, policy, errors) + if errors: + problems.append(f"valid fixture {case_dir.name} unexpectedly failed:") + problems.extend(f" {e}" for e in errors) + + for case_dir in sorted((FIXTURES_ROOT / "invalid").glob("*")) if (FIXTURES_ROOT / "invalid").is_dir() else []: + if not case_dir.is_dir(): + continue + errors = [] + policy = load_policy(errors, case_policy_path(case_dir)) + if policy is not None: + validate_catalog(case_dir, profile_schema, release_schema, policy, errors) + if not errors: + problems.append(f"invalid fixture {case_dir.name} unexpectedly passed (expected at least one error)") + + return problems + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--selftest", + action="store_true", + help="also replay scripts/tests/fixtures/agent-platform/{valid,invalid} and assert expected pass/fail", + ) + args = parser.parse_args() + + errors = run(CATALOG) + if errors: + for error in errors: + print(error, file=sys.stderr) + print(f"agent platform catalog: {len(errors)} error(s)", file=sys.stderr) + return 1 + profile_count = len(list((CATALOG / "execution-profiles").glob("*/profile.yaml"))) + release_count = len(list((CATALOG / "releases").glob("*/*.yaml"))) + print(f"validated {profile_count} execution profile(s), {release_count} release intent(s)") + + if args.selftest: + problems = run_selftest() + if problems: + for problem in problems: + print(problem, file=sys.stderr) + print(f"agent platform selftest: {len(problems)} problem(s)", file=sys.stderr) + return 1 + valid_count = len(list((FIXTURES_ROOT / "valid").glob("*"))) if (FIXTURES_ROOT / "valid").is_dir() else 0 + invalid_count = len(list((FIXTURES_ROOT / "invalid").glob("*"))) if (FIXTURES_ROOT / "invalid").is_dir() else 0 + print(f"selftest ok: {valid_count} valid fixture(s), {invalid_count} invalid fixture(s)") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())