diff --git a/plugins/autonomy/.claude-plugin/plugin.json b/plugins/autonomy/.claude-plugin/plugin.json index fb895f8ba..5c8f670e8 100644 --- a/plugins/autonomy/.claude-plugin/plugin.json +++ b/plugins/autonomy/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "autonomy", - "version": "0.7.2", + "version": "0.7.3", "description": "Governed autonomous agent operation: role-topology, binding-seam, wiring-vs-advisor, telemetry, return-accounting, trigger-dispatch, per-work-class guardrail-matrix, standing-routine-catalog, and design-only runner-charter contracts for climbing the AI-adoption ladder, plus a guided-setup skill that discovers an adopting org's state, writes its schema-versioned binding, wires standards-pinned OTLP emission with a zero-cost file-artifact default, wires human-attested return capture at the task boundary, wires signal adapters with one governed dispatch entrypoint, binds the five-class guardrail matrix to an org's isolation substrates with an in-boundary live-validation probe before recording each fail-closed binding, and stands up standing-routine-catalog classes as scheduled temporal signal adapters behind the one governed queue with free scheduling defaults wired as reviewable changes and each routine's work-class mapping homed on the security surface.", "author": { "name": "Melodic Software", diff --git a/plugins/autonomy/CHANGELOG.md b/plugins/autonomy/CHANGELOG.md index fb5d45e98..60ffe3c97 100644 --- a/plugins/autonomy/CHANGELOG.md +++ b/plugins/autonomy/CHANGELOG.md @@ -6,6 +6,19 @@ All notable changes to the `autonomy` plugin are documented here. Format follows Versions 0.1.0–0.7.0 predate this file (introduced with 0.7.1); their history lives in the merged work-package PRs (#333, #343, #356, #372, #377, #600, #676). +## [0.7.3] + +### Added + +- **Security-binding golden suite is graded (`#662`).** A table-driven runner + (`check-security-binding.fixtures.test.mjs` + its co-located expectations manifest) runs + every fixture under `evals/fixtures/security-binding/` through + `check-security-binding.mjs` and asserts exit code + defect-naming findings: 109 fixtures + (14 pass-expected, 95 reject-expected), zero quarantined, with the fixtures' 67 probe + transcripts enumerated as suite inputs. Self-policing in both directions — an ungraded + new fixture, an unlisted transcript, or a manifest entry whose file vanished all fail the + suite, and the repo's orphaned-fixture gate no longer grandfathers the set. + ## [0.7.2] ### Changed diff --git a/plugins/autonomy/skills/setup/scripts/check-security-binding.fixtures.test.manifest.json b/plugins/autonomy/skills/setup/scripts/check-security-binding.fixtures.test.manifest.json new file mode 100644 index 000000000..1b1c933dc --- /dev/null +++ b/plugins/autonomy/skills/setup/scripts/check-security-binding.fixtures.test.manifest.json @@ -0,0 +1,710 @@ +{ + "_comment": "Graded-fixture manifest for check-security-binding.fixtures.test.mjs. Keyed by top-level fixture filename under evals/fixtures/security-binding/. Each entry: expected checker exit (0 valid, 1 findings) and defect-naming substrings the runner asserts (stdout when exit 0, stderr when exit 1). A binding+evidence pair grades evaluation mode with the named event-array input; egress_hosts pins the --egress-hosts allow-list seam. quarantined lists any fixture whose real checker behavior contradicts its name (none currently).", + "fixtures": { + "admission-weakened-no-justification.json": { + "exit": 1, + "findings_substrings": [ + "admission.rules[2]: disposition \"autonomous-eligible\" is more permissive than the shipped default for C3" + ] + }, + "ambiguous-runtime-markers.json": { + "exit": 1, + "findings_substrings": [ + "isolation_bindings: runtime_markers of surfaces \"ci-pool-a\" and \"ci-pool-b\" are jointly satisfiable" + ] + }, + "as112-v6-valid.json": { + "exit": 0, + "findings_substrings": [] + }, + "c2-auto-unratified.json": { + "exit": 1, + "findings_substrings": [ + "merge_policy.C2: \"auto\" without a ratified promotion_state.C2-auto-merge entry (state \"promoted\")" + ] + }, + "c3-blocking-unratified.json": { + "exit": 1, + "findings_substrings": [ + "verification_blocking.ai-review.C3: \"blocking\" without a ratified promotion_state.C3-ai-review-blocking entry (state \"promoted\")" + ] + }, + "c4-auto-merge.json": { + "exit": 1, + "findings_substrings": [ + "merge_policy.C4: \"auto\" exceeds the matrix cap" + ] + }, + "equal-specificity-tie.json": { + "exit": 1, + "findings_substrings": [ + "admission.rules[5] and admission.rules[6]: equal specificity (2 bound axes) with different dispositions and a jointly matchable triple" + ] + }, + "escalation-ack-invalid.json": { + "exit": 1, + "findings_substrings": [ + "escalation_ack.staleness_window: must be a positive integer (>= 1)" + ] + }, + "escalation-ack-loop.json": { + "exit": 1, + "findings_substrings": [ + "escalation_ack.reescalation_cap: 2 exceeds the contract cap" + ] + }, + "escalation-push-without-channel.json": { + "exit": 1, + "findings_substrings": [ + "escalation_severity_routes.urgent: binds a push leg with no channel leg" + ] + }, + "escalation-runner-route-missing.json": { + "exit": 1, + "findings_substrings": [ + "escalation_severity.runner-needs-human: severity is bound for a runner event class with no escalation_routes.runner-needs-human entry" + ] + }, + "escalation-severity-bad-token.json": { + "exit": 1, + "findings_substrings": [ + "escalation_severity.gate-failure: \"critical\" is not one of notice | attention | urgent" + ] + }, + "escalation-severity-tracker-only-valid.json": { + "exit": 0, + "findings_substrings": [] + }, + "escalation-severity-valid.json": { + "exit": 0, + "findings_substrings": [] + }, + "evidence-contrary.json": { + "binding": "promoted-c2.json", + "evidence": "evidence-contrary.json", + "exit": 0, + "findings_substrings": [ + "ceiling lowered by contrary evidence" + ] + }, + "evidence-stale.json": { + "binding": "promoted-c2.json", + "evidence": "evidence-stale.json", + "exit": 0, + "findings_substrings": [ + "pre-epoch contrary event(s) ignored" + ] + }, + "hosted-ephemeral-l3-valid.json": { + "exit": 0, + "findings_substrings": [] + }, + "human-gated-only-no-l2.json": { + "exit": 0, + "findings_substrings": [] + }, + "l3-ratified-class-container.json": { + "exit": 1, + "findings_substrings": [ + "isolation_bindings.ci-pool-a.L3.substrate_class: \"container\" ratified for an L3 entry" + ] + }, + "marker-prototype-key.json": { + "exit": 1, + "findings_substrings": [ + "isolation_bindings.ci-pool-a: runtime_markers of level entries L2 and L3 are jointly satisfiable" + ] + }, + "missing-l2-autonomous.json": { + "exit": 1, + "findings_substrings": [ + "isolation_bindings: no bound surface reaches L2 under dispatch_posture autonomous-enabled" + ] + }, + "missing-probe-evidence.json": { + "exit": 1, + "findings_substrings": [ + "isolation_bindings.ci-pool-a.L2.probe_evidence: missing or empty" + ] + }, + "missing-substrate-class.json": { + "exit": 1, + "findings_substrings": [ + "isolation_bindings.ci-pool-a.L2.substrate_class: undefined" + ] + }, + "multi-host-egress-valid.json": { + "egress_hosts": "example.com,iana.org", + "exit": 0, + "findings_substrings": [] + }, + "probe-evidence-6to4-relay.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"192.88.99.1\"" + ] + }, + "probe-evidence-6to4-v6.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"2002:101:101::1\"" + ] + }, + "probe-evidence-bare-metadata.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"http://metadata/computeMetadata\"" + ] + }, + "probe-evidence-benchmarking-v6.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"2001:2::7\"" + ] + }, + "probe-evidence-bogus-credpath.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"/definitely-not-a-host-credential\"" + ] + }, + "probe-evidence-bogus-metadata.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"http://metadata.invalid/not-a-credential\"" + ] + }, + "probe-evidence-cred-outer-missing.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.outer_exit_code undefined for 1 probed path entry" + ] + }, + "probe-evidence-cred-outer-nonzero.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.outer_exit_code entry \"1\" for path \"/root/.ssh/id_rsa\"" + ] + }, + "probe-evidence-credpath-code-count.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.exit_code \"1\" for 2 probed path entries" + ] + }, + "probe-evidence-credpath-partial-zero.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.exit_code entry \"0\" for path \"http://169.254.169.254/metadata/instance\"" + ] + }, + "probe-evidence-discard-only.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"100::1\"" + ] + }, + "probe-evidence-doc-3fff.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"3fff::1\"" + ] + }, + "probe-evidence-docnet-host.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"2001:db8::1\"" + ] + }, + "probe-evidence-dot-traversal.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"/root/../var/empty/.ssh/id_rsa\"" + ] + }, + "probe-evidence-dummy-prefix.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"100:0:0:1::1\"" + ] + }, + "probe-evidence-encoded-loopback.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"2130706433\"" + ] + }, + "probe-evidence-etc-descendant-credpath.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"/etc/example/credentials\"" + ] + }, + "probe-evidence-expanded-loopback.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"0:0:0:0:0:0:0:1\"" + ] + }, + "probe-evidence-expansion-ephemeral.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.host_expanded entry \"/tmp/.ssh/id_rsa\" for path \"$HOME/.ssh/id_rsa\"" + ] + }, + "probe-evidence-expansion-tail-mismatch.json": { + "exit": 1, + "findings_substrings": [ + "the host-side expansion is not tail-consistent" + ] + }, + "probe-evidence-generic-env-token.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"$TOKEN\"" + ] + }, + "probe-evidence-generic-token.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"/tmp/token\"" + ] + }, + "probe-evidence-home-descendant-credpath.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"$HOME/scratch/credentials\"" + ] + }, + "probe-evidence-host-home-mount.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"/host-home/.netrc\"" + ] + }, + "probe-evidence-imds-http-error.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.transport_outcome entry \"read-denied\" for metadata-URL path \"http://169.254.169.254/metadata/instance\"" + ] + }, + "probe-evidence-invalid-tld.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"does-not-exist.invalid\"" + ] + }, + "probe-evidence-invented-env-token.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"$NOT_A_TOKEN\"" + ] + }, + "probe-evidence-invented-home-user.json": { + "exit": 1, + "findings_substrings": [ + "definitely-not-a-host-user" + ] + }, + "probe-evidence-invented-token.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"$HOME/definitely_not_a_token\"" + ] + }, + "probe-evidence-ipv4-compat.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"::127.0.0.1\"" + ] + }, + "probe-evidence-known-hosts.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"$HOME/.ssh/known_hosts\"" + ] + }, + "probe-evidence-l3-container.json": { + "exit": 1, + "findings_substrings": [ + "isolation_bindings.ci-pool-a.L3.substrate_class: \"container\" ratified for an L3 entry" + ] + }, + "probe-evidence-level-mismatch.json": { + "exit": 1, + "findings_substrings": [ + "records level \"L2\", not this binding entry's level \"L3\"" + ] + }, + "probe-evidence-local-host.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"localhost\"" + ] + }, + "probe-evidence-malformed-dns.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"egress..example.com\"" + ] + }, + "probe-evidence-metadata-port.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"http://169.254.169.254:1/metadata\"" + ] + }, + "probe-evidence-metadata-route-boundary.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"http://169.254.169.254/metadata-not-a-credential\"" + ] + }, + "probe-evidence-metadata-route.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"http://169.254.169.254/not-a-credential\"" + ] + }, + "probe-evidence-metadata-scheme.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"file://169.254.169.254/metadata\"" + ] + }, + "probe-evidence-missing-host-expanded.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.host_expanded undefined for 1 probed path entry" + ] + }, + "probe-evidence-missing-outer-exit.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.outer_exit_code undefined for 1 probed host entry" + ] + }, + "probe-evidence-missing-probed-at.json": { + "exit": 1, + "findings_substrings": [ + "records probed_at undefined" + ] + }, + "probe-evidence-missing-target.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host undefined" + ] + }, + "probe-evidence-missing-transport.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.transport_outcome undefined for 1 probed path entry" + ] + }, + "probe-evidence-multi-host-private.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"10.0.0.1\"" + ] + }, + "probe-evidence-nat64-local.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"64:ff9b:1::1\"" + ] + }, + "probe-evidence-nat64-private-embed.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"64:ff9b::10.0.0.1\"" + ] + }, + "probe-evidence-outer-egress-failed.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.outer_exit_code entry \"6\" for host \"definitely-no-such-probe-target-983475.example.net\"" + ] + }, + "probe-evidence-protocol-assignment.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"192.0.0.1\"" + ] + }, + "probe-evidence-ratified-class-mismatch.json": { + "exit": 1, + "findings_substrings": [ + "records substrate_class \"container\", not this binding entry's ratified substrate_class \"vm-microvm\"" + ] + }, + "probe-evidence-relative-credpath.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"home/.ssh\"" + ] + }, + "probe-evidence-root-escape.json": { + "exit": 1, + "findings_substrings": [ + "via \"..\" traversal" + ] + }, + "probe-evidence-site-local-v6.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"fec0::1\"" + ] + }, + "probe-evidence-srv6-sid.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"5f00::1\"" + ] + }, + "probe-evidence-ssh-dir.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"$HOME/.ssh\"" + ] + }, + "probe-evidence-teredo.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"2001::5\"" + ] + }, + "probe-evidence-token-substring.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"/definitely-not-a-host-token\"" + ] + }, + "probe-evidence-transport-cross-type.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.transport_outcome entry \"connect-failed\" for path \"$HOME/.netrc\"" + ] + }, + "probe-evidence-unanchored-credpath.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"/tmp/.ssh\"" + ] + }, + "probe-evidence-unanchored-envvar.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"$UNRELATED/.ssh\"" + ] + }, + "probe-evidence-unc-etc.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.credentials_absent.path entry \"//etc/credentials\"" + ] + }, + "probe-evidence-uri-host.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.host \"http://127.0.0.1\"" + ] + }, + "probe-evidence-zero-exit.json": { + "exit": 1, + "findings_substrings": [ + "records assertions.egress_denied.exit_code entry \"0\" for host \"example.com\"" + ] + }, + "promoted-c2.json": { + "exit": 0, + "findings_substrings": [] + }, + "promotion-nonpromotable.json": { + "exit": 1, + "findings_substrings": [ + "promotion_state.C4-merge: not a promotable cell" + ] + }, + "ratified-at-calendar-invalid.json": { + "exit": 1, + "findings_substrings": [ + "promotion_state.C2-auto-merge.ratified_at: missing, not strict ISO 8601 with explicit offset (Z or +/-hh:mm), or not a calendar-valid instant" + ] + }, + "ratified-at-non-iso.json": { + "exit": 1, + "findings_substrings": [ + "promotion_state.C2-auto-merge.ratified_at: missing, not strict ISO 8601 with explicit offset (Z or +/-hh:mm), or not a calendar-valid instant" + ] + }, + "ratified-at-offset-invalid.json": { + "exit": 1, + "findings_substrings": [ + "promotion_state.C2-auto-merge.ratified_at: missing, not strict ISO 8601 with explicit offset (Z or +/-hh:mm), or not a calendar-valid instant" + ] + }, + "ratified-at-submilli.json": { + "exit": 1, + "findings_substrings": [ + "promotion_state.C2-auto-merge.ratified_at: missing, not strict ISO 8601 with explicit offset (Z or +/-hh:mm), or not a calendar-valid instant" + ] + }, + "same-surface-level-ambiguity.json": { + "exit": 1, + "findings_substrings": [ + "isolation_bindings.ci-pool-a: runtime_markers of level entries L2 and L3 are jointly satisfiable" + ] + }, + "ssh-id-rsa-valid.json": { + "exit": 0, + "findings_substrings": [] + }, + "temporal-classification-bare-string.json": { + "exit": 1, + "findings_substrings": [ + "admission.classification.temporal.\"doc-freshness-sweep/advisory\": \"C1\" is a bare work class" + ] + }, + "temporal-classification-valid.json": { + "exit": 0, + "findings_substrings": [] + }, + "temporal-duplicate-producer-identity.json": { + "exit": 1, + "findings_substrings": [ + "admission.classification.temporal: routine identities \"doc-freshness-sweep/advisory\" and \"ci-health-review\" both declare producer_identity \"ci/workflows/shared-scheduler.yml\"" + ] + }, + "temporal-empty-source-surface.json": { + "exit": 1, + "findings_substrings": [ + "admission.classification.temporal.\"ci-health-review\".source_surface: missing or empty" + ] + }, + "temporal-identity-grammar.json": { + "exit": 1, + "findings_substrings": [ + "admission.classification.temporal.\"Doc-Freshness-Sweep/advisory\": \"Doc-Freshness-Sweep/advisory\" is not a routine identity" + ] + }, + "temporal-missing-producer-identity.json": { + "exit": 1, + "findings_substrings": [ + "admission.classification.temporal.\"ci-health-review\".producer_identity: missing or empty" + ] + }, + "temporal-missing-run-link-prefix.json": { + "exit": 1, + "findings_substrings": [ + "admission.classification.temporal.\"ci-health-review\".run_link_prefix: missing or empty" + ] + }, + "temporal-shared-run-link-prefix-valid.json": { + "exit": 0, + "findings_substrings": [] + }, + "temporal-shared-source-surface.json": { + "exit": 1, + "findings_substrings": [ + "admission.classification.temporal: routine identities \"doc-freshness-sweep/advisory\" and \"ci-health-review\" both bind source_surface \"ci-cron-doc-freshness-advisory\"" + ] + }, + "two-level-surface-valid.json": { + "exit": 0, + "findings_substrings": [] + }, + "unverifiable-probe-evidence.json": { + "exit": 1, + "findings_substrings": [ + "is not readable JSON" + ] + }, + "valid.json": { + "exit": 0, + "findings_substrings": [] + }, + "vendor-hosted-auto.json": { + "exit": 1, + "findings_substrings": [ + "merge_policy.C2: \"auto\" with executor_class vendor-hosted" + ] + }, + "verification-below-floor.json": { + "exit": 1, + "findings_substrings": [ + "verification_blocking.deterministic.C2: \"advisory\" weakens the shipped floor \"blocking\"" + ] + } + }, + "quarantined": {}, + "probe_transcripts": [ + "ci-pool-a-l2-6to4-relay.json", + "ci-pool-a-l2-6to4-v6.json", + "ci-pool-a-l2-as112-v6.json", + "ci-pool-a-l2-bare-metadata.json", + "ci-pool-a-l2-benchmarking-v6.json", + "ci-pool-a-l2-bogus-credpath.json", + "ci-pool-a-l2-bogus-metadata.json", + "ci-pool-a-l2-cred-outer-missing.json", + "ci-pool-a-l2-cred-outer-nonzero.json", + "ci-pool-a-l2-credpath-code-count.json", + "ci-pool-a-l2-credpath-partial-zero.json", + "ci-pool-a-l2-discard-only.json", + "ci-pool-a-l2-doc-3fff.json", + "ci-pool-a-l2-docnet.json", + "ci-pool-a-l2-dot-traversal.json", + "ci-pool-a-l2-dummy-prefix.json", + "ci-pool-a-l2-encoded-loopback.json", + "ci-pool-a-l2-etc-descendant-credpath.json", + "ci-pool-a-l2-expanded-loopback.json", + "ci-pool-a-l2-expansion-ephemeral.json", + "ci-pool-a-l2-expansion-tail-mismatch.json", + "ci-pool-a-l2-generic-env-token.json", + "ci-pool-a-l2-generic-token.json", + "ci-pool-a-l2-home-descendant-credpath.json", + "ci-pool-a-l2-host-home-mount.json", + "ci-pool-a-l2-id-rsa.json", + "ci-pool-a-l2-imds-http-error.json", + "ci-pool-a-l2-invalid-tld.json", + "ci-pool-a-l2-invented-env-token.json", + "ci-pool-a-l2-invented-home-user.json", + "ci-pool-a-l2-invented-token.json", + "ci-pool-a-l2-ipv4-compat.json", + "ci-pool-a-l2-known-hosts.json", + "ci-pool-a-l2-localhost.json", + "ci-pool-a-l2-malformed-dns.json", + "ci-pool-a-l2-metadata-port.json", + "ci-pool-a-l2-metadata-route-boundary.json", + "ci-pool-a-l2-metadata-route.json", + "ci-pool-a-l2-metadata-scheme.json", + "ci-pool-a-l2-multi-host-private.json", + "ci-pool-a-l2-multi-host.json", + "ci-pool-a-l2-nat64-local.json", + "ci-pool-a-l2-nat64-private-embed.json", + "ci-pool-a-l2-no-host-expanded.json", + "ci-pool-a-l2-no-host.json", + "ci-pool-a-l2-no-outer-exit.json", + "ci-pool-a-l2-no-probed-at.json", + "ci-pool-a-l2-no-transport.json", + "ci-pool-a-l2-outer-egress-failed.json", + "ci-pool-a-l2-protocol-assignment.json", + "ci-pool-a-l2-relative-credpath.json", + "ci-pool-a-l2-site-local-v6.json", + "ci-pool-a-l2-srv6-sid.json", + "ci-pool-a-l2-ssh-dir.json", + "ci-pool-a-l2-teredo.json", + "ci-pool-a-l2-token-substring.json", + "ci-pool-a-l2-transport-cross-type.json", + "ci-pool-a-l2-unanchored-credpath.json", + "ci-pool-a-l2-unanchored-envvar.json", + "ci-pool-a-l2-unc-etc.json", + "ci-pool-a-l2-uri-host.json", + "ci-pool-a-l2-zero-exit.json", + "ci-pool-a-l2.json", + "ci-pool-a-l3-container.json", + "ci-pool-a-l3-hosted.json", + "ci-pool-a-l3.json", + "ci-pool-b-l2.json" + ] +} diff --git a/plugins/autonomy/skills/setup/scripts/check-security-binding.fixtures.test.mjs b/plugins/autonomy/skills/setup/scripts/check-security-binding.fixtures.test.mjs new file mode 100644 index 000000000..66e46096c --- /dev/null +++ b/plugins/autonomy/skills/setup/scripts/check-security-binding.fixtures.test.mjs @@ -0,0 +1,124 @@ +// Table-driven graded-fixture suite for check-security-binding.mjs. Every +// top-level fixture under evals/fixtures/security-binding/ is graded: the +// co-located manifest pins each fixture's expected checker exit code and the +// substrings the finding (or evaluation-mode verdict) must contain, and this +// runner FAILS if any top-level fixture lacks a manifest (or quarantine) entry +// — so a future ungraded fixture cannot be added silently. +// +// The checker verifies L2/L3 probe evidence only against a protected root, so +// every fixture runs with --probe-evidence-root pointed at the fixtures dir. +// Event-array inputs (evidence-*.json) are not bindings; their entry names the +// valid binding they pair with and grades evaluation mode via --evidence. +// +// Runnable directly (`node check-security-binding.fixtures.test.mjs`) or through +// the co-located .test.sh wrapper that scripts/run-plugin-tests.sh discovers. + +import { spawnSync } from "node:child_process"; +import { readdirSync, readFileSync } from "node:fs"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import process from "node:process"; + +const scriptsDir = dirname(fileURLToPath(import.meta.url)); +const checker = join(scriptsDir, "check-security-binding.mjs"); +const fixturesDir = resolve(scriptsDir, "..", "evals", "fixtures", "security-binding"); +const manifestPath = join(scriptsDir, "check-security-binding.fixtures.test.manifest.json"); + +const manifest = JSON.parse(readFileSync(manifestPath, "utf8")); +const fixtures = manifest.fixtures ?? {}; +const quarantined = manifest.quarantined ?? {}; + +let failed = 0; +let cases = 0; +const pass = (name) => { + cases += 1; + process.stdout.write(`PASS: ${name}\n`); +}; +const fail = (name, detail) => { + cases += 1; + failed += 1; + process.stderr.write(`FAIL: ${name}\n detail: ${detail}\n`); +}; + +// --- Self-policing: every top-level fixture is graded or quarantined --------- +const topLevelFixtures = readdirSync(fixturesDir, { withFileTypes: true }) + .filter((entry) => entry.isFile() && entry.name.endsWith(".json")) + .map((entry) => entry.name) + .sort(); + +for (const name of topLevelFixtures) { + if (Object.hasOwn(fixtures, name) || Object.hasOwn(quarantined, name)) { + pass(`covered: ${name}`); + } else { + fail(`covered: ${name}`, "top-level fixture has no manifest (or quarantine) entry — add one so it is graded"); + } +} +// Probe transcripts are suite INPUTS (fixtures reference them as probe_evidence +// and the checker reads them through --probe-evidence-root), so the manifest +// enumerates them too — same self-policing in both directions: an unlisted +// on-disk transcript or a listed-but-missing one fails. +const transcriptsDir = join(fixturesDir, "probe-transcripts"); +const onDiskTranscripts = readdirSync(transcriptsDir, { withFileTypes: true }) + .filter((entry) => entry.isFile()) + .map((entry) => entry.name) + .sort(); +const listedTranscripts = manifest.probe_transcripts ?? []; +const listedSet = new Set(listedTranscripts); +for (const name of onDiskTranscripts) { + if (listedSet.has(name)) pass(`transcript-listed: ${name}`); + else fail(`transcript-listed: ${name}`, "probe transcript on disk but not in the manifest's probe_transcripts list"); +} +const onDiskSet = new Set(onDiskTranscripts); +for (const name of listedTranscripts) { + if (!onDiskSet.has(name)) fail(`transcript-exists: ${name}`, "manifest lists a probe transcript that is not on disk"); +} + +// Reverse drift: a manifest entry whose fixture (or paired binding/evidence) +// file has been removed. +const present = new Set(topLevelFixtures); +for (const [name, entry] of Object.entries(fixtures)) { + const referenced = [entry.binding ?? name, entry.evidence].filter(Boolean); + const missing = referenced.filter((file) => !present.has(file)); + if (missing.length === 0) pass(`manifest-refs-exist: ${name}`); + else fail(`manifest-refs-exist: ${name}`, `manifest references missing file(s): ${missing.join(", ")}`); +} + +// --- Grade each fixture against the checker ---------------------------------- +for (const [name, entry] of Object.entries(fixtures)) { + const bindingFile = entry.binding ?? name; + const args = [checker, join(fixturesDir, bindingFile), "--probe-evidence-root", fixturesDir]; + if (entry.evidence) args.push("--evidence", join(fixturesDir, entry.evidence)); + if (entry.egress_hosts) args.push("--egress-hosts", entry.egress_hosts); + + const result = spawnSync(process.execPath, args, { + encoding: "utf8", + maxBuffer: 10 * 1024 * 1024, + timeout: 30_000, + }); + if (result.error) { + fail(`grade: ${name}`, `could not launch checker: ${result.error.message}`); + continue; + } + + if (result.status !== entry.exit) { + fail(`grade: ${name}`, `expected exit ${entry.exit}, got ${result.status}\n stderr: ${result.stderr.trim()}`); + continue; + } + + // Findings print to stderr (exit 1); OK verdicts and evaluation-mode lines + // print to stdout (exit 0) — assert each substring against the right stream. + const stream = entry.exit === 0 ? result.stdout : result.stderr; + const missing = (entry.findings_substrings ?? []).filter((sub) => !stream.includes(sub)); + if (missing.length > 0) { + fail(`grade: ${name}`, `missing expected substring(s): ${JSON.stringify(missing)}\n actual: ${stream.trim()}`); + continue; + } + pass(`grade: ${name}`); +} + +if (failed === 0) { + process.stdout.write(`\nAll ${cases} checks passed (${Object.keys(fixtures).length} fixtures graded, ${Object.keys(quarantined).length} quarantined).\n`); + process.exit(0); +} +process.stderr.write(`\n${failed}/${cases} checks failed.\n`); +process.exit(1); diff --git a/plugins/autonomy/skills/setup/scripts/check-security-binding.fixtures.test.sh b/plugins/autonomy/skills/setup/scripts/check-security-binding.fixtures.test.sh new file mode 100755 index 000000000..7bb612d2a --- /dev/null +++ b/plugins/autonomy/skills/setup/scripts/check-security-binding.fixtures.test.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Discovery wrapper: scripts/run-plugin-tests.sh finds plugins/**/*.test.sh, so +# this hands off to the table-driven Node graded-fixture suite. SKIPs (exit 0) +# when Node is unavailable, matching the optional-tool skip idiom the other +# plugin contract tests use. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +if ! command -v node >/dev/null 2>&1; then + echo "SKIP: node not installed" >&2 + exit 0 +fi + +exec node "$SCRIPT_DIR/check-security-binding.fixtures.test.mjs" diff --git a/scripts/orphaned-fixtures-baseline.txt b/scripts/orphaned-fixtures-baseline.txt index cb8b38bad..5a95e94ff 100644 --- a/scripts/orphaned-fixtures-baseline.txt +++ b/scripts/orphaned-fixtures-baseline.txt @@ -12,9 +12,7 @@ # issue lands the fix. Do NOT add a plugin's fixtures here to dodge the gate on # NEW work — reference them from a grader instead. -# autonomy setup skill: the security-binding golden suite (referenced by no # eval files[] entry and graded by no test) plus the otlp-demo sample corpus, -# enumerated file-by-file below. The security-binding subset is tracked by # issue #662 (attach expected-verdict blocks and wire them into the grader, or # demote them to documentation exemplars out of evals/fixtures/); otlp-demo is # the same ungraded class in the same skill. autonomy is WP-lane-owned — the fix @@ -23,182 +21,6 @@ plugins/autonomy/skills/setup/evals/fixtures/otlp-demo/pipeline.jsonl plugins/autonomy/skills/setup/evals/fixtures/otlp-demo/session-metrics.jsonl plugins/autonomy/skills/setup/evals/fixtures/otlp-demo/session-spans.jsonl plugins/autonomy/skills/setup/evals/fixtures/otlp-demo/trigger.jsonl -plugins/autonomy/skills/setup/evals/fixtures/security-binding/admission-weakened-no-justification.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/ambiguous-runtime-markers.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/as112-v6-valid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/c2-auto-unratified.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/c3-blocking-unratified.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/c4-auto-merge.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/equal-specificity-tie.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-ack-invalid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-ack-loop.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-push-without-channel.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-runner-route-missing.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-severity-bad-token.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-severity-tracker-only-valid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-severity-valid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/evidence-contrary.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/evidence-stale.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/hosted-ephemeral-l3-valid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/human-gated-only-no-l2.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/l3-ratified-class-container.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/marker-prototype-key.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/missing-l2-autonomous.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/missing-probe-evidence.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/missing-substrate-class.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/multi-host-egress-valid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-6to4-relay.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-6to4-v6.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-bare-metadata.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-benchmarking-v6.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-bogus-credpath.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-bogus-metadata.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-cred-outer-missing.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-cred-outer-nonzero.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-credpath-code-count.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-credpath-partial-zero.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-discard-only.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-doc-3fff.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-docnet-host.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-dot-traversal.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-dummy-prefix.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-encoded-loopback.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-etc-descendant-credpath.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-expanded-loopback.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-expansion-ephemeral.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-expansion-tail-mismatch.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-generic-env-token.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-generic-token.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-home-descendant-credpath.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-host-home-mount.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-imds-http-error.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-invalid-tld.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-invented-env-token.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-invented-home-user.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-invented-token.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-ipv4-compat.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-known-hosts.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-l3-container.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-level-mismatch.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-local-host.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-malformed-dns.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-metadata-port.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-metadata-route-boundary.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-metadata-route.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-metadata-scheme.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-host-expanded.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-outer-exit.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-probed-at.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-target.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-transport.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-multi-host-private.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-nat64-local.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-nat64-private-embed.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-outer-egress-failed.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-protocol-assignment.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-ratified-class-mismatch.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-relative-credpath.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-root-escape.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-site-local-v6.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-srv6-sid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-ssh-dir.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-teredo.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-token-substring.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-transport-cross-type.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-unanchored-credpath.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-unanchored-envvar.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-unc-etc.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-uri-host.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-zero-exit.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-6to4-relay.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-6to4-v6.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-as112-v6.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-bare-metadata.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-benchmarking-v6.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-bogus-credpath.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-bogus-metadata.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-cred-outer-missing.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-cred-outer-nonzero.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-credpath-code-count.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-credpath-partial-zero.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-discard-only.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-doc-3fff.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-docnet.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-dot-traversal.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-dummy-prefix.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-encoded-loopback.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-etc-descendant-credpath.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-expanded-loopback.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-expansion-ephemeral.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-expansion-tail-mismatch.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-generic-env-token.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-generic-token.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-home-descendant-credpath.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-host-home-mount.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-id-rsa.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-imds-http-error.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-invalid-tld.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-invented-env-token.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-invented-home-user.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-invented-token.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-ipv4-compat.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-known-hosts.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-localhost.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-malformed-dns.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-metadata-port.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-metadata-route-boundary.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-metadata-route.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-metadata-scheme.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-multi-host-private.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-multi-host.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-nat64-local.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-nat64-private-embed.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-host-expanded.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-host.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-outer-exit.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-probed-at.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-transport.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-outer-egress-failed.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-protocol-assignment.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-relative-credpath.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-site-local-v6.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-srv6-sid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-ssh-dir.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-teredo.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-token-substring.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-transport-cross-type.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-unanchored-credpath.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-unanchored-envvar.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-unc-etc.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-uri-host.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-zero-exit.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l3-container.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l3-hosted.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l3.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-b-l2.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/promoted-c2.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/promotion-nonpromotable.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/ratified-at-calendar-invalid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/ratified-at-non-iso.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/ratified-at-offset-invalid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/ratified-at-submilli.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/same-surface-level-ambiguity.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/ssh-id-rsa-valid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-classification-bare-string.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-classification-valid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-duplicate-producer-identity.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-empty-source-surface.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-identity-grammar.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-missing-producer-identity.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-missing-run-link-prefix.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-shared-run-link-prefix-valid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-shared-source-surface.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/two-level-surface-valid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/unverifiable-probe-evidence.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/valid.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/vendor-hosted-auto.json -plugins/autonomy/skills/setup/evals/fixtures/security-binding/verification-below-floor.json # knowledge youtube-digest skill: this data file lives under evals/fixtures/ but # is named only from SKILL.md / vendor TUNING.md prose — no eval files[] entry