diff --git a/.gitattributes b/.gitattributes index 905610707a..15f2c81aa1 100644 --- a/.gitattributes +++ b/.gitattributes @@ -59,3 +59,11 @@ packages.lock.json -diff *.woff2 binary *.ttf binary *.otf binary + +############################################################################### +# Byte-exact fixtures. This one reproduces a measured Claude Code defect: a RAW +# control character inside a JSON string value, which `jq` rejects outright. Any +# line-ending normalization would rewrite that byte and the fixture would become +# valid JSON, silently turning its regression case into a test of nothing. +############################################################################### +plugins/claude-config/skills/audit-permission-state/evals/fixtures/automode-config-rawctl.json -text diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42d4106d48..810331aab2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -184,10 +184,23 @@ jobs: # the same corpus: it pins each fixture's trusted --credential-roots # (a real runner home) and expected not-under-root rejections, so it # necessarily carries those same host paths. + # audit-permission-state is the third case of the same kind: its + # C6-winPath check EXISTS to find Windows paths in permission rules, + # so its detector message quotes one, its criteria file quotes the + # settings page quoting one, and its fixtures must carry them or the + # check has nothing to detect. The oracle fixtures reproduce Claude + # Code's own [DEBUG] narration, which prints native Windows settings + # paths — rewriting those would make the fixture stop matching what + # the parser has to survive. The paths are invented (`alice`, `x`), + # not any operator's. exclude: >- :(exclude)plugins/guardrails/lib/path-detection/** :(exclude)plugins/autonomy/skills/setup/evals/fixtures/security-binding/** :(exclude)plugins/autonomy/skills/setup/scripts/check-security-binding.fixtures.test.manifest.json + :(exclude)plugins/claude-config/skills/audit-permission-state/reference/criteria.md + :(exclude)plugins/claude-config/skills/audit-permission-state/scripts/permission-plane-lint.sh + :(exclude)plugins/claude-config/skills/audit-permission-state/scripts/permission-plane-lint.test.sh + :(exclude)plugins/claude-config/skills/audit-permission-state/scripts/automode-entry-diff.test.sh - name: Check index-level EOL drift id: eol diff --git a/docs/CATALOG.md b/docs/CATALOG.md index 2b6d9cc3e7..586a60bfda 100644 --- a/docs/CATALOG.md +++ b/docs/CATALOG.md @@ -74,7 +74,7 @@ plugin manifests and kept in sync by CI — never hand-edit it; the category voc - [`desktop-notification`](../plugins/desktop-notification) — Alert you when Claude Code needs input — an audible terminal bell, an OSC 9 terminal notification, and an OS-native toast (macOS/Linux) on permission and idle prompts. - [`playbooks`](../plugins/playbooks) — Doctrine and knowledge playbooks as on-demand skills, plus a maintainer-facing update skill. boris — Boris Cherny's Claude Code workflow tips (howborisusesclaudecode.com); skill-authoring — Anthropic's internal skill-authoring playbook; fable-5 — Claude Fable 5's operating doctrine (self-authored, no upstream). The boris and skill-authoring packs vendor a verbatim upstream baseline; /playbooks:update drift-checks and syncs those baselines centrally (maintainers). -- [`claude-config`](../plugins/claude-config) — Eight configuration-health skills (plus setup) for a repo's Claude Code configuration: audit (settings.json / .mcp.json / hooks / plugins / permissions drift), audit-automation-gaps (evidence-gated verdicts on automation gaps), audit-permission-grants (allow-rule / allowed-tools grants for auto-mode durability and portability), audit-permission-state (which settings scopes exist and what rules each one holds — managed policy, user-global, project, local, and the pre-v2.1.211 start-directory copy), audit-instructions (locally-owned instruction surfaces vs current model capability — proposes removals/rewrites of instructions the model no longer needs, and detects cross-surface instruction conflicts), audit-prompting-postures (the additive lane — posture guidance the prompting guide says a component's purpose needs but the component does not carry), audit-pass (one coordinated, ordered, resumable pass over a named target — three-scope inventory, run-time-derived exclusion set, stable finding identity, suppression memory, resume, one human gate — delegating every check to the plugin that owns it), and unhobble (the empirical bare-baseline experiment: reversibly strip a repo's standing instructions, log real stumbles against the current model, re-add only what evidence earns). +- [`claude-config`](../plugins/claude-config) — Nine configuration-health skills (plus setup) for a repo's Claude Code configuration: audit (settings.json / .mcp.json / hooks / plugins / permissions drift), audit-automation-gaps (evidence-gated verdicts on automation gaps), audit-permission-grants (allow-rule / allowed-tools grants for auto-mode durability and portability), audit-permission-state (the permission rules actually in effect — every settings scope merged with per-rule provenance, what auto mode drops on entry, config written where nothing reads it, and which managed intents are enforced versus loosenable), draft-auto-mode-rules (interview and draft a paste-ready autoMode classifier block; prints only, never writes), audit-instructions (locally-owned instruction surfaces vs current model capability — proposes removals/rewrites of instructions the model no longer needs, and detects cross-surface instruction conflicts), audit-prompting-postures (the additive lane — posture guidance the prompting guide says a component's purpose needs but the component does not carry), audit-pass (one coordinated, ordered, resumable pass over a named target — three-scope inventory, run-time-derived exclusion set, stable finding identity, suppression memory, resume, one human gate — delegating every check to the plugin that owns it), and unhobble (the empirical bare-baseline experiment: reversibly strip a repo's standing instructions, log real stumbles against the current model, re-add only what evidence earns). - [`claude-memory`](../plugins/claude-memory) — Keeps a repo's Claude Code memory layer healthy and under your control, against criteria derived from official Claude Code documentation. The audit skill checks the instruction/memory layer (CLAUDE.md, CLAUDE.local.md, .claude/rules/, auto-memory) with a deterministic script-backed spine plus judgment-tier checks. The stateless skill inspects, disables, and (confirm-gated) purges Claude-written auto memory across all settings scopes. - [`claude-ops`](../plugins/claude-ops) — Claude Code operations toolkit. Ten skills: inventory (read-only enumeration of the complete invocable surface — every built-in CLI command with aliases and hidden/gated status, every bundled skill, and every component of every installed plugin across all marketplaces; reads the shipped binary because upstream publishes no built-in command list, and carries an integrity verdict so a drifted build reports counts as floors rather than silently short totals), audit-install-state (read-only audit of the machine-scope ~/.claude installation directory and ~/.claude.json — full inventory split into an authored surface and rolled-up bulk trees, product-managed retention vs genuinely unmanaged state, filename-scheme resolution before any process-liveness check, and deliberate/mid-experiment detection; reports, never deletes), audit-performance (read-only slowness-diagnostic capture run at the moment the machine or a session feels slow — CLI version, retention-sweep health including the silent unparsable-settings pause, a timed census walk of the install tree as a sweep-cost proxy, active-session and plugin-fleet counts, a process census, and a bundled known-performance-issues reference; separates the three documented suspects — accumulated state, version regression, component bloat — and routes remediation out; reports, never mutates), observability (read locally captured telemetry — OTEL store, collector, hook-event JSONL, ccusage — with trend reports and store pruning), known-issues (search known Claude product GitHub bugs, check service health, maintain a persistent tracked-issue registry), changelog (ingest Claude Code changelog entries and integrate them into the current repo), plugins (bring a machine's plugin fleet current on demand — marketplace refresh, effective-scope updates including in-repo project/local installs, new-plugin install per policy, scope-divergence detection and explicit convergence), morning-brief (read-only gh-based operator morning view — queue-label counts, merge-ready PRs, parked decisions with their RECOMMENDED lines, and loop-lane telemetry freshness), lanes (start/restart/stop/status loop lanes as named background Claude Code sessions seeded from canonical prompt files, with per-lane model/effort, a repo-pull + marketplace-refresh launch step, and a consume-restarts action — an OS-schedulable reader that relaunches stopped lanes whose telemetry carries a restart_request), and a re-runnable setup action that settles where the known-issues registry lives. Plus a family of seven advisory *-audit telemetry-emitter hooks (API errors, config changes, instruction loads, permission denials, pre-compaction, skill usage, tool failures) that emit the shared hook-telemetry envelope, and a reference sink that maps envelopes into the hook-events.jsonl the observability skill reads. - [`rate-limit-guard`](../plugins/rate-limit-guard) — Shared rate-limit guard for loop lanes: a statusline wrapper tees the subscription rate-limit windows to a fixed machine-scope file, a StopFailure hook records rate-limit stops reactively, and a reader contract fixes how consuming sessions pause and resume. diff --git a/docs/MIGRATION-PLAYBOOK.md b/docs/MIGRATION-PLAYBOOK.md index 249bdf26c2..7ed832dd59 100644 --- a/docs/MIGRATION-PLAYBOOK.md +++ b/docs/MIGRATION-PLAYBOOK.md @@ -683,6 +683,19 @@ plugins-reference, and hooks pages 2026-07-17; re-verify per the `CLAUDE.md` fre **advisory** (exits 0, never blocks) vs gating; no `eval` / `curl … | sh` / outbound network; untrusted input (file contents, tool args, PR/issue text) never flows unquoted into a shell; a kill switch (a per-hook `userConfig` boolean with a `default` of `true`) exists. + - **A skill's frontmatter `allowed-tools` is a prompt-free execution grant, and workspace trust does + not gate it.** Measured on Claude Code 2.1.225: a marketplace-installed skill's `allowed-tools` + entry takes effect at **user scope** in a **never-trusted** workspace, under `-p` where no trust + dialog can appear — the covered command ran without a prompt, the uncovered one blocked with + `This command requires approval`, and a no-grant baseline confirmed that shape blocks. Bounds on + the measurement: a local-directory marketplace, and user scope only. + **Consequence: the install-time plugin trust prompt is the only gate in front of such a grant — + there is no second, per-workspace one.** Review every `allowed-tools` entry with the scrutiny a + hook command gets, and deny by default anything broader than the specific command the skill's own + scripts invoke. A wildcard interpreter grant (`Bash(python*)`, `Bash(*)`, bare `Bash`) is a deny + outright: it is arbitrary code execution in a workspace the consumer never trusted. + `claude-config:audit-permission-grants` check P1 detects exactly these shapes and is the + mechanical half of this criterion. 2. **MCP servers — `.mcp.json` / inline in `plugin.json`.** `miro` is the only plugin that ships a **local** `stdio`, bundled server (see its §2 trust accept above); `dometrain` is the only plugin that ships a **remote** server (see its review record below), which remains the higher-scrutiny diff --git a/docs/OFFICIAL-DOCS.md b/docs/OFFICIAL-DOCS.md index f38c752922..255264d2e7 100644 --- a/docs/OFFICIAL-DOCS.md +++ b/docs/OFFICIAL-DOCS.md @@ -47,7 +47,7 @@ components are declared in, not a component, so it has no row. | Monitors (`monitors/monitors.json`) | | 2026-08-06 | | Channels (`channels` manifest field) | | 2026-08-06 | | Executables (`bin/`) | | 2026-08-06 | -| Settings (`settings.json` defaults) | | 2026-08-06 | +| Settings (`settings.json` defaults) | | 2026-08-12 | | Dependencies (`dependencies` manifest field) | | 2026-08-06 | ## Authoring @@ -104,14 +104,14 @@ SDK-based host. | Page | Official doc page | Verified date | |---|---|---| -| Settings | | 2026-08-06 | +| Settings | | 2026-08-12 | | Server-managed settings | | 2026-08-06 | | Control MCP server access for your organization | | 2026-08-06 | | Memory — CLAUDE.md, `.claude/rules/`, auto memory | | 2026-08-06 | | The `.claude` directory | | 2026-08-06 | -| Permissions | | 2026-08-06 | -| Permission modes | | 2026-08-06 | -| Configure auto mode (`autoMode`, `claude auto-mode`) | | 2026-08-09 | +| Permissions | | 2026-08-12 | +| Permission modes | | 2026-08-12 | +| Configure auto mode (`autoMode`, `claude auto-mode`) | | 2026-08-12 | | Environment variables | | 2026-08-10 | ## Prompting doctrine (platform docs) diff --git a/docs/SKILL-CHEAT-SHEET.md b/docs/SKILL-CHEAT-SHEET.md index 6a4f736603..dea5d49fb6 100644 --- a/docs/SKILL-CHEAT-SHEET.md +++ b/docs/SKILL-CHEAT-SHEET.md @@ -143,8 +143,9 @@ owned by [docs/CATALOG-TAXONOMY.md](CATALOG-TAXONOMY.md). | [`/claude-config:audit-instructions`](../plugins/claude-config/skills/audit-instructions/SKILL.md) | `claude-config` | Find instructions current models no longer need across CLAUDE.md, rules, and skill bodies | | [`/claude-config:audit-pass`](../plugins/claude-config/skills/audit-pass/SKILL.md) | `claude-config` | Run one coordinated, resumable audit pass over a repo with a single human gate | | [`/claude-config:audit-permission-grants`](../plugins/claude-config/skills/audit-permission-grants/SKILL.md) | `claude-config` | Audit permission grants for portability and auto-mode durability | -| [`/claude-config:audit-permission-state`](../plugins/claude-config/skills/audit-permission-state/SKILL.md) | `claude-config` | Report which permission scopes exist and what rules each one holds | +| [`/claude-config:audit-permission-state`](../plugins/claude-config/skills/audit-permission-state/SKILL.md) | `claude-config` | Report the permission rules actually in effect and what auto mode drops | | [`/claude-config:audit-prompting-postures`](../plugins/claude-config/skills/audit-prompting-postures/SKILL.md) | `claude-config` | Find posture guidance the prompting guide says a component needs but does not carry | +| [`/claude-config:draft-auto-mode-rules`](../plugins/claude-config/skills/draft-auto-mode-rules/SKILL.md) | `claude-config` | Interview and draft a paste-ready autoMode block, never writing settings | | [`/claude-config:unhobble`](../plugins/claude-config/skills/unhobble/SKILL.md) | `claude-config` | Strip instructions to a bare baseline, log real stumbles, re-add only what evidence earns | | [`/claude-memory:audit`](../plugins/claude-memory/skills/audit/SKILL.md) | `claude-memory` | Audit CLAUDE.md, rules, and auto-memory against the official-docs checklist | | [`/claude-memory:stateless`](../plugins/claude-memory/skills/stateless/SKILL.md) | `claude-memory` | Inspect, disable, or purge Claude Code's per-repo auto memory | diff --git a/docs/adr/0009-report-the-permission-plane-as-in-effect-and-never-write-to-it.md b/docs/adr/0009-report-the-permission-plane-as-in-effect-and-never-write-to-it.md new file mode 100644 index 0000000000..d49fc65e3e --- /dev/null +++ b/docs/adr/0009-report-the-permission-plane-as-in-effect-and-never-write-to-it.md @@ -0,0 +1,75 @@ +# Report the permission plane as in effect, and never write to it + +- Status: accepted +- Date: 2026-08-12 + +## Context + +Claude Code's permission plane is legible only to the harness. `/permissions` lists rules and the file +each came from, but nothing resolves which of two conflicting rules wins, nothing distinguishes a +scope that was empty from one that could not be read, and there is no machine-readable export. Auto +mode became the default permission mode for new sessions, and on entry it silently drops broad allow +rules — so a consumer's grants can stop taking effect with no signal at all. + +Research across roughly thirty third-party tools found nobody auditing an `autoMode` block, resolving +cross-scope precedence, or validating managed policy against the scopes beneath it. The highest-adoption +linter in the space carries one permission rule out of 447. The official marketplace ships 284 plugins +and none manage permission configuration. + +Two questions had to be decided before building anything, and both had a defensible opposite answer. + +## Decision 1 — report, never write + +**These skills write nothing, in any scope, under any flag.** No `--fix`, no "shall I apply this", +no follow-up offer. + +Enforcement was available: hook `deny` and exit-2 both work, measured. This is a posture choice, not a +capability limit. Editing a consumer's settings file is making a permission decision on their behalf, +and a plugin that will silently adjust what the agent may do is a worse failure than one that reports +a problem the human then fixes. The authoring lane draws the same line — it prints a block and the +human pastes it. + +The cost is real: an operator with 77 dead allow rules must fix them by hand. That is accepted. + +## Decision 2 — compute the merge locally, bounded by decidability + +**Claims follow from documented mechanics over readable inputs, each citing the mechanic it follows +from.** Anything resting on classifier judgment, runtime demotion state, or an open upstream +discrepancy becomes a **named caveat on the affected finding** — never a silent drop, never an +assertion. + +The alternative was to wait for an official export. Nothing suggests one is coming, and the local +merge is decidable for the part that matters. + +Two mechanics carry the result, and conflating them produces confident wrong answers: + +- **Permission rules merge across scopes rather than override.** A rule at two scopes has no winner — + both are live. Electing one would assert an override the documentation denies. +- **Kind is decided by evaluation order — deny, then ask, then allow — from any scope, in both + directions.** A user-level deny blocks a project-level allow just as the reverse. Scope rank does + not enter into it, and an implementation that ranked scopes here gets the low-scope-deny case + exactly backwards. + +Every effective-set claim states two standing bounds: the command-line scope outranks the files and +has none to read, and rules are compared by exact text, so a narrow allow blocked only by a broader +deny **pattern** is still reported effective. The error direction is known — over-reporting allow, +never over-reporting blocking. + +## Consequences + +- Reports are trustworthy about their own limits. `absent` means looked and found nothing; `skipped` + means could not look. A surface that could not be read is never reported as absence of policy — + an administrator reading silence as "no policy deployed" is the failure this exists to prevent. +- **Managed policy is read-only by construction**, not by restraint: those are admin-write OS + locations or a claude.ai Owner role. The conformance report says what a consumer's own policy does + and does not achieve, and ships no security floor of its own. +- Server-managed settings have no local path, so every managed claim is scoped to the local surfaces. +- One optional lane needs `python3`, because `claude auto-mode config` emits raw control characters + inside JSON string values that no line-oriented POSIX filter can repair. Absent it, that lane skips + visibly and everything else still runs. +- **Re-opens if** an official machine-readable export of resolved permission state ships, or if + `claude permissions` becomes a real subcommand — either would retire the local merge as the read + path. A debug-channel oracle already exists as corroboration but is not a read path: it costs a + session spawn and parses undocumented `[DEBUG]` strings with no stability contract. +- **The write posture re-opens only on an explicit decision**, not on a feature request. It is the + contract these skills are trusted under. diff --git a/plugins/claude-config/.claude-plugin/plugin.json b/plugins/claude-config/.claude-plugin/plugin.json index da4dff62a2..6f602ff861 100644 --- a/plugins/claude-config/.claude-plugin/plugin.json +++ b/plugins/claude-config/.claude-plugin/plugin.json @@ -1,8 +1,8 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "claude-config", - "version": "0.35.4", - "description": "Eight configuration-health skills (plus setup) for a repo's Claude Code configuration: audit (settings.json / .mcp.json / hooks / plugins / permissions drift), audit-automation-gaps (evidence-gated verdicts on automation gaps), audit-permission-grants (allow-rule / allowed-tools grants for auto-mode durability and portability), audit-permission-state (which settings scopes exist and what rules each one holds — managed policy, user-global, project, local, and the pre-v2.1.211 start-directory copy), audit-instructions (locally-owned instruction surfaces vs current model capability — proposes removals/rewrites of instructions the model no longer needs, and detects cross-surface instruction conflicts), audit-prompting-postures (the additive lane — posture guidance the prompting guide says a component's purpose needs but the component does not carry), audit-pass (one coordinated, ordered, resumable pass over a named target — three-scope inventory, run-time-derived exclusion set, stable finding identity, suppression memory, resume, one human gate — delegating every check to the plugin that owns it), and unhobble (the empirical bare-baseline experiment: reversibly strip a repo's standing instructions, log real stumbles against the current model, re-add only what evidence earns).", + "version": "0.36.0", + "description": "Nine configuration-health skills (plus setup) for a repo's Claude Code configuration: audit (settings.json / .mcp.json / hooks / plugins / permissions drift), audit-automation-gaps (evidence-gated verdicts on automation gaps), audit-permission-grants (allow-rule / allowed-tools grants for auto-mode durability and portability), audit-permission-state (the permission rules actually in effect — every settings scope merged with per-rule provenance, what auto mode drops on entry, config written where nothing reads it, and which managed intents are enforced versus loosenable), draft-auto-mode-rules (interview and draft a paste-ready autoMode classifier block; prints only, never writes), audit-instructions (locally-owned instruction surfaces vs current model capability — proposes removals/rewrites of instructions the model no longer needs, and detects cross-surface instruction conflicts), audit-prompting-postures (the additive lane — posture guidance the prompting guide says a component's purpose needs but the component does not carry), audit-pass (one coordinated, ordered, resumable pass over a named target — three-scope inventory, run-time-derived exclusion set, stable finding identity, suppression memory, resume, one human gate — delegating every check to the plugin that owns it), and unhobble (the empirical bare-baseline experiment: reversibly strip a repo's standing instructions, log real stumbles against the current model, re-add only what evidence earns).", "author": { "name": "Melodic Software", "email": "info@melodicsoftware.com" diff --git a/plugins/claude-config/CHANGELOG.md b/plugins/claude-config/CHANGELOG.md index 49b6b868f6..29eef487a9 100644 --- a/plugins/claude-config/CHANGELOG.md +++ b/plugins/claude-config/CHANGELOG.md @@ -3,6 +3,127 @@ All notable changes to the `claude-config` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.36.0] + +### Changed + +- **`audit-permission-grants` check P1 now sees user-global allow rules.** It scanned project and + local settings only, so an interpreter-wildcard rule in + `${CLAUDE_CONFIG_DIR:-$HOME/.claude}/settings.json` was invisible to it — and that is the scope + Claude Code's own "Always allow" path writes to, so it is where the broad rules auto mode drops + actually accumulate. Expect new findings on a repository whose own configuration did not change. + The user scope resolves through `CLAUDE_CONFIG_DIR` before `$HOME`, and a finding names the + resolved absolute path rather than `~/.claude/settings.json`, which would name the wrong file + whenever the config root has been relocated. +- **`audit`'s structure check now reports a start-directory `settings.local.json`.** A pre-v2.1.211 + Claude Code wrote the file to the directory the session started in, and the current one still + reads what an earlier version left there: the repository-root copy wins on a shared key, but + permission rules from **both** files stay in effect. The row appears only when the start directory + genuinely differs from the project root and a copy is there, so the same file is never counted as + two rule sources. +- **`audit`'s structure check names the managed surfaces it does not read.** On Windows the + `HKLM`/`HKCU\SOFTWARE\Policies\ClaudeCode` policy keys, on macOS the `com.anthropic.claudecode` + managed-preferences domain. A file-only reader that stays silent about them lets an absent + `managed-settings.json` read as "no managed policy deployed" while a policy is in force. + +### Added + +- **`audit-permission-state`** — a new skill reporting which permission rules are actually in effect + and where each comes from. `/permissions` lists your rules and the file each came from, but it does + not resolve which of two conflicting rules wins, cannot distinguish a scope that was empty from one + it could not read, and exists only inside a live session — there is no `claude permissions` + subcommand and no machine-readable export. The reader discovers managed policy, user-global, + project, local, and any + pre-v2.1.211 start-directory copy, and inventories each scope's `allow`/`ask`/`deny` rules with its + source named. Every scope and every managed surface emits a record on every OS, so a surface that + was never attempted can never be mistaken for one that is genuinely empty: `absent` means looked and + found nothing, `skipped` means could not look. Server-managed settings are disclosed as having no + local path rather than assumed absent. Report-only, and managed policy is read-only by construction. + A second pass merges those scopes into the set actually in force, each rule naming every scope that + contributes it and the documented mechanic that put it there. Permission rules merge across scopes + rather than override, so a rule written at two scopes has no winner and is never reported as one; + what a rule can lose is its kind, because deny is evaluated before ask and ask before allow from any + scope in either direction — a user-level deny blocks a project-level allow just as the reverse. The + beaten entry is reported as inert alongside the rule that beat it, which is the answer to "why is my + allow rule ignored". A rule that is a bare tool name reaches every call of that tool: a whole-tool + deny removes the tool from context entirely, so every other rule naming it is inert — including + other denies, which are moot rather than weakened — and a whole-tool ask prompts for every call, so + no scoped allow for that tool applies. `EndConversation` is exempt from removal, as documented. + A third pass answers what entering auto mode does to that set — which became urgent when auto mode + turned on by default for new sessions. Every effective allow rule is classified as dropped (with the + documented reason named: blanket, wildcarded interpreter, package-manager run, or `Agent`) or as + carried over, using the same shared pattern vocabulary `audit-permission-grants` check P1 scans + with. `autoMode.classifyAllShell` is read too, because when it is on it suspends every Bash and + PowerShell allow rule and a diff blind to it can be exactly wrong — and it is resolved only from the + scopes the classifier actually reads, so a project-scope copy is reported inert rather than obeyed. + An opt-in `--oracle` flag corroborates the prediction against the harness's own drop narration by + spawning a real `claude -p` session; it never fires without the flag, prints what it will leave + behind before spawning anything, and reports an empty capture as unavailable rather than as an + empty drop set. + A fourth pass lints the permission plane for configuration that is written but never read. Eight + checks: an `autoMode` section in a scope the classifier does not read, `defaultMode: "auto"` in + project or local settings, `useAutoModeDuringPlan` in shared project settings, `disableAutoMode` + typed as a boolean instead of the string `"disable"`, and four rule shapes that cannot match — + doubled-backslash Windows paths, parameter-form rules on a tool's primary content field, path rules + on a tool whose path rules are never consulted, and `:*` used anywhere but at the end of a pattern. + The three dead-config gates stay separate findings because they cover different scope sets and carry + different version histories; merging them would let an operator fix one and believe they had fixed + all three. The `disableAutoMode` check is the highest-consequence one — a boolean is valid JSON, is + accepted, and does nothing, so the operator believes auto mode is locked out when it is not — and it + is read at both documented key paths in every scope, since it is not managed-only. Several of these + also emit a startup warning upstream; the added value is reading every scope at once, before a + session, and naming the file. Advisory: it exits 0 whenever it ran, and exit 2 means it could not + run at all rather than that it found nothing. + A fifth lane reads the `autoMode` classifier block — a different surface again, four + natural-language sections rather than permission rules. It reports a customized section that omits + `"$defaults"` (which **replaces** the built-in list rather than adding to it, so the finding names + how many entries are discarded), the same subject appearing in both `allow` and a deny section, and + an entry an earlier `hard_deny` already forecloses. `claude auto-mode critique` is surfaced with + `--critique` rather than reimplemented — it owns the semantic judgment — but it is wrapped in + truncation and empty-output detection, because across three consecutive runs on one unchanged config + its output was truncated mid-sentence twice and empty once while exiting 0 every time. This lane + needs `python3`, because `claude auto-mode config` emits raw control characters inside JSON string + values that `jq` rejects outright and no line-oriented filter can repair; absent it, the lane prints + a visible skip notice and exits 0 while every other stage still runs. A capture that produced + nothing is reported as unavailable with an explicit "this is NOT a clean bill" — exit status is + never consulted, since it is 0 even when nothing came back. + A sixth lane reports which managed intents are actually enforced and which a developer can loosen. + A managed `permissions.deny` is the strongest thing an administrator can write and is reported + enforced; a managed `autoMode` section is **additive, not a policy boundary** — a developer cannot + remove entries it provides, but a developer-added `allow` can override an organization `soft_deny`, + because permissions, hooks, MCP, sandbox-filesystem and sandbox-network each have an exclusivity + lock and auto mode has none. The report also surfaces an interaction the precedence table alone does + not suggest: managed settings are the highest scope, but evaluation order applies from any scope, so + a lower-scope deny beats a managed allow without ever overriding it. It prescribes nothing — every + rule string it prints came from a file it read, and it ships no security floor of its own. Every run + bounds its own completeness: server-managed settings have no local path, and a managed surface that + could not be read is reported as such rather than left silent, since an administrator reading + silence as "no policy deployed" is the failure the report exists to prevent. Every run states the two bounds on the claim: the command-line scope + (`--settings`, `--allowedTools`, `--disallowedTools`) outranks the files and has none to read, and + rules are compared by exact text, so a narrow allow blocked only by a broader deny pattern is still + reported effective — the error direction is over-reporting allow, never over-reporting blocking. +- **`draft-auto-mode-rules`** — a second new skill, the authoring counterpart. It interviews you about + what should and should not be auto-approved, then prints a paste-ready `autoMode` block to stdout. + The entry shape is `claude auto-mode critique`'s own recommendation applied at authoring time rather + than reported afterwards: run against a real 66 KB hand-authored block, it found the classifier is + "an LLM doing a single pass under a 'default is ALLOW' instruction", so conditions buried in a + paragraph are missed at a materially higher rate than conditions in a bullet list. Entries are + therefore a label, bulleted COVERED / NOT COVERED, and one line of rationale. The interview pushes + back on conditions the classifier cannot evaluate from the command text — the same critique named + those the biggest weakness, since the classifier either allows blindly or blocks entirely with no + stated disposition. Every emitted section opens with `"$defaults"`, because customizing a section + replaces the built-in rule list rather than adding to it. It **writes nothing, in any scope, under + any flag** — editing a consumer's settings file would be making a permission decision on their + behalf, which is the one thing this plugin exists not to do. +- **`lib/permission-patterns.sh`** — the auto-mode drop vocabulary (blanket, wildcarded-interpreter, + package-manager-runner, and script-glob rule shapes, plus the top-level tool-token grammar) as a + define-only library. It was inline in the P1 detector, which self-executes and cannot be sourced, + so a second consumer had no way to reuse it without copying. +- **`lib/managed-scope.sh`** — the per-OS managed-policy surface enumeration (base JSON file, + `managed-settings.d/` drop-in directory, Windows policy registry keys, macOS preferences domain). + `claude-memory` carries a byte-identical copy, registered as a cross-plugin shared-source cluster + so the two cannot drift. + ## [0.35.4] ### Fixed diff --git a/plugins/claude-config/lib/permission-patterns.sh b/plugins/claude-config/lib/permission-patterns.sh index f10ff348dd..710d40635f 100644 --- a/plugins/claude-config/lib/permission-patterns.sh +++ b/plugins/claude-config/lib/permission-patterns.sh @@ -55,10 +55,21 @@ CCPERM_SCRIPT_BODY='py|sh|rb|js|ts|mjs|cjs|pl|php' # # Each alternative captures the whole Tool(...) spec (trailing [^)]*\) ) so a # driver reports the full offending rule, not a substring truncated at the *. -CCPERM_P1_ERE="(Bash|PowerShell)\\(\\*\\)" -CCPERM_P1_ERE="${CCPERM_P1_ERE}|(Bash|PowerShell)\\([\"' ]*([^)\"' ]*[/\\\\])?(${CCPERM_INTERP_BODY})([\"' :][^)]*)?\\*[^)]*\\)" -CCPERM_P1_ERE="${CCPERM_P1_ERE}|(Bash|PowerShell)\\([\"' ]*(${CCPERM_RUNNER_BODY})([\"' :][^)]*)?([\"' :])\\*[^)]*\\)" -CCPERM_P1_ERE="${CCPERM_P1_ERE}|(Bash|PowerShell)\\([\"' ]*\\*[^)]*\\.(${CCPERM_SCRIPT_BODY})[^)]*\\)" +# +# The four alternatives are also exposed individually so a driver that must NAME +# the class a rule fell into (the entry diff's drop reason) tests them one at a +# time instead of re-deriving the wrapping; the union stays the P1 detector's +# single match target and is composed from them, never written twice. +# +# The RUNNER alternative carries #2382's precision tightening -- the +# `([\"' :])` before the `\*` requires a separator, so `Bash(npmx *)` no longer +# matches the `npm` runner. Composed in rather than resolved away: this branch +# only restructured these patterns, it did not change what they match. +CCPERM_P1_BLANKET_ERE="(Bash|PowerShell)\\(\\*\\)" +CCPERM_P1_INTERP_ERE="(Bash|PowerShell)\\([\"' ]*([^)\"' ]*[/\\\\])?(${CCPERM_INTERP_BODY})([\"' :][^)]*)?\\*[^)]*\\)" +CCPERM_P1_RUNNER_ERE="(Bash|PowerShell)\\([\"' ]*(${CCPERM_RUNNER_BODY})([\"' :][^)]*)?([\"' :])\\*[^)]*\\)" +CCPERM_P1_SCRIPTGLOB_ERE="(Bash|PowerShell)\\([\"' ]*\\*[^)]*\\.(${CCPERM_SCRIPT_BODY})[^)]*\\)" +CCPERM_P1_ERE="${CCPERM_P1_BLANKET_ERE}|${CCPERM_P1_INTERP_ERE}|${CCPERM_P1_RUNNER_ERE}|${CCPERM_P1_SCRIPTGLOB_ERE}" # Splits rule text into top-level `Tool` / `Tool(...)` tokens. The greedy # `(\(...\))?` consumes a tool's whole parenthesized payload as one token, so a diff --git a/plugins/claude-config/skills/audit-pass/SKILL.md b/plugins/claude-config/skills/audit-pass/SKILL.md index 74eec97e70..a89122dfce 100644 --- a/plugins/claude-config/skills/audit-pass/SKILL.md +++ b/plugins/claude-config/skills/audit-pass/SKILL.md @@ -249,6 +249,23 @@ Dispatch, in inventory order, each invocation presence-gated with its fallback s belong to exactly one lane by construction rather than needing a deduplication rule downstream — and the per-class lanes drop the pair check, since dispatching it there is what created the overlap. +- **`/claude-config:audit-permission-state`** — sibling in this plugin, always available. It owns the + permission plane as it is *in effect*: the merged allow/ask/deny set with per-rule provenance, + what auto mode drops on entry, configuration written where nothing reads it, and which managed + intents are enforced versus loosenable. It takes an **action flag and no target**, so it is + **exactly one lane** covering all of that. + + **Its managed-scope reads belong to the pass's read-only managed inventory, not to a project lane.** + It reads managed policy on every OS and never writes anywhere, in any scope, under any flag — so it + is safe to dispatch under the pass's bare invocation. Its `--oracle` path spawns a real session and + is **never dispatched here**: the pass has no way to price that for the operator mid-run, and the + flag exists to make the cost an explicit choice. + + **Its optional lanes degrade rather than fail.** The `autoMode` block lane needs `python3` and + `claude` on PATH; absent either, that lane self-reports as skipped and the rest of the skill still + runs. Carry that skip into the report as **unchecked with its reason**, exactly as an absent plugin + would be — the distinction between "clean" and "not read" is this skill's whole contract and the + pass must not collapse it. - **`/claude-memory:audit`** — invoke when the `claude-memory` plugin is installed; it owns memory-layer hygiene and the within-memory-layer consistency check. It takes an **action verb and no surface filter**, so it is **exactly one lane** covering the whole memory layer. Not installed: diff --git a/plugins/claude-config/skills/audit-permission-state/SKILL.md b/plugins/claude-config/skills/audit-permission-state/SKILL.md index 11e4a7238f..5c0c51d02f 100644 --- a/plugins/claude-config/skills/audit-permission-state/SKILL.md +++ b/plugins/claude-config/skills/audit-permission-state/SKILL.md @@ -1,11 +1,11 @@ --- -description: "Report the Claude Code permission state actually in effect — discovers every settings scope (managed policy, user-global, project, local, and the pre-v2.1.211 start-directory copy) and inventories each one's allow/ask/deny rules with its source named. Use when: 'what permissions are actually in effect', 'which settings file is my rule coming from', 'why is my allow rule ignored', 'show me my effective permissions', 'is my managed policy being read', 'what scopes did you check', or before changing a permission rule you cannot locate. Report-only — never writes any settings file." -argument-hint: "[--scopes] — surface records only, no rule inventory" +description: "Report the Claude Code permission state actually in effect — discovers every settings scope (managed policy, user-global, project, local, and the pre-v2.1.211 start-directory copy), merges them into the effective allow/ask/deny set with each rule's source and precedence mechanic named, and classifies which allow rules auto mode drops on entry. Use when: 'what permissions are actually in effect', 'which settings file is my rule coming from', 'why is my allow rule ignored', 'show me my effective permissions', 'what does auto mode drop', 'which of my rules survive auto mode', 'is my managed policy being read', 'what scopes did you check', or before changing a permission rule you cannot locate. Report-only — never writes any settings file." +argument-hint: "[--scopes] surfaces only | [--entry-diff] what auto mode drops" user-invocable: true disable-model-invocation: false metadata: workflow-stage: anytime - summary: Report which permission scopes exist and what rules each one holds + summary: Report the permission rules actually in effect and what auto mode drops --- ## Purpose @@ -30,7 +30,11 @@ could actually open, and what each one holds. ## Report-only, permanently -This skill writes nothing, in any scope, under any flag. Managed policy is read-only by +**This skill writes no settings file, in any scope, under any flag** — that is the contract, and it +holds including under `--oracle`. It is not the same as writing nothing at all: `--oracle` spawns a +real `claude -p` session, and a session rewrites `~/.claude.json` and adds project, session-env, +security, subagent and backup state under your config directory. The flag prints that before it +spawns anything. Every other action writes nothing anywhere. Managed policy is read-only by construction — those are admin-write OS locations or a claude.ai Owner role, so a plugin could not author them even if it wanted to. @@ -40,7 +44,11 @@ Parse `$ARGUMENTS`: - `--scopes` — surface records only, no rule inventory. Use when the question is "which scopes exist and which could you read", not "what is in them". -- (no argument) — surfaces plus one record per allow/ask/deny rule. +- `--entry-diff` — run the full pipeline through to the auto-mode entry diff (Phase 3 below). +- `--oracle` — with `--entry-diff`, cross-check the prediction against the harness's own drop + narration. **Spawns a real `claude -p` session**; never fires without this flag. See its cost + notice, which the run prints before anything is spawned. +- (no argument) — surfaces plus one record per allow/ask/deny rule, then the merge. ## Phase 1: Discover and inventory @@ -93,15 +101,132 @@ Two mechanics decide those records, and conflating them produces confident wrong user-level allow. Scope rank does not enter into it. This is what answers "why is my allow rule ignored": the `inert` record names the rule that beat it. - **A rule that is a bare tool name reaches every call of that tool.** A whole-tool deny removes the - tool from context entirely, so every other rule naming it is inert — including other denies, which - are moot rather than weakened; `EndConversation` is the documented exception. A whole-tool ask - prompts for every call, so no scoped allow for that tool applies. Both print a `NOTE:` naming the - tool. + tool from context entirely, so every other rule naming it is inert — other denies included; + `EndConversation` is the documented exception. A whole-tool ask prompts for every call, so no scoped + allow for that tool applies. Both print a `NOTE:` naming the tool. `reference/criteria.md` maps every `precedence_basis` token to the sentence it follows from, and -states the two standing bounds the run prints — the command-line scope has no file to read, and rules -are compared by exact text, so a narrow allow blocked only by a broader deny **pattern** is still -reported effective. The error direction is over-reporting allow. +states the two standing bounds the run prints. + +## Phase 3: What entering auto mode drops + +Auto mode became the default permission mode for new sessions on 2026-08-14, and on entry it +**silently drops** broad allow rules. This stage says which of yours survive: + +```shell +bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-permission-state/scripts/permission-state.sh" | + bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-permission-state/scripts/permission-merge.sh" | + bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-permission-state/scripts/automode-entry-diff.sh" +``` + +```text +DIFF-NOTE: classifyAllShell state, bounds +entry-diff dropped class= scopes= dropped on entry +entry-diff suspended reason=classifyAllShell ... suspended while auto mode is active +entry-diff kept scopes= carries over +entry-diff summary allow_before= dropped= suspended= kept= +``` + +- **Only allow rules change on entry.** Deny and ask are evaluated before the classifier in every + mode, so they are not part of this diff — do not report them as "surviving". +- **`class` names the documented reason**: `blanket`, `interpreter-wildcard`, `package-manager-run`, + or `agent`, from `lib/permission-patterns.sh` — the vocabulary `audit-permission-grants` check P1 + also scans with. +- **`autoMode.classifyAllShell` inverts the answer wholesale.** When true it suspends *every* Bash and + PowerShell allow rule, so narrow rules do **not** carry over. It is resolved only from the scopes + the classifier reads, so a project- or local-scope copy is reported inert rather than obeyed. +- **`--oracle` is opt-in and priced.** It spawns a real `claude -p` session to corroborate the + prediction. Measured cost: your settings files are untouched, but `~/.claude.json` is rewritten and + project, session-env, security and subagent state appear under your config directory. A capture + that yields nothing is **unavailable**, never an empty drop set. + +## Phase 4: Configuration that is written but never read + +The permission plane accepts things it silently ignores. This finds them across every scope at once, +before a session starts: + +```shell +bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-permission-state/scripts/permission-state.sh" | + bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-permission-state/scripts/permission-plane-lint.sh" +``` + +```text +finding [] +lint summary findings= checks_run= +``` + +Nine checks: three `C2-*` dead-config gates, `C5-disableType`, and five `C6-*` rules-that-cannot-match. +`reference/criteria.md` maps each to the sentence it follows from and lists the legitimate rule shapes +the checks are written NOT to flag. + +- **`C5-disableType` is the one to read first.** `disableAutoMode` must be the **string** `"disable"`; + a boolean is valid JSON, is accepted, and does nothing — so the operator believes auto mode is + locked out when it is not. +- **The three `C2` gates stay separate findings.** Different scope sets, different version histories: + an operator who fixed one and saw the count drop would reasonably believe they had fixed all three. +- **Several of these also produce a startup warning.** The added value here is reading every scope at + once, before a session, and naming the file — not that the harness is silent. +- **Advisory: the lint always exits 0 when it ran.** Exit 2 means it could not run at all, never + "nothing found". + +## Phase 5: The `autoMode` classifier block + +A different surface from everything above: four natural-language sections an LLM classifier reads, +not permission rules the harness matches. Independent of the pipeline — it reads the CLI, not stdin: + +```shell +bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-permission-state/scripts/automode-block-lint.sh" [--critique] +``` + +- **`C4-defaults`** — a customized section that omits `"$defaults"`. Customizing **replaces** the + built-in list rather than adding to it, so the finding names how many built-in entries are gone. +- **`C2b-contradiction`** — the same subject in `allow` and in a deny section. +- **`C3-shadowed`** — an entry an earlier `hard_deny` already forecloses, so it can never fire. +- **`--critique` surfaces `claude auto-mode critique`, wrapped — never replaced.** It owns the + semantic judgment. What this adds is honesty about it: measured across three consecutive runs on one + unchanged config, output was truncated mid-sentence twice and empty once, **exiting 0 every time**. + A mid-sentence cut is reported as truncated; an empty result says "critique returned nothing; run it + yourself" rather than implying your rules are clean. + +**This lane is optional, and its prerequisite is nobody else's problem.** It needs `python3` because +`claude auto-mode config` emits raw control characters inside string values — `jq` rejects the output +outright, and no line-oriented POSIX filter can repair it, since the offending byte is a raw line feed +inside a string. Absent `python3` or `claude`, the lane prints a visible skip notice and exits 0; every +other stage still runs. + +**Exit status is never trusted here.** A run that exits 0 having produced nothing is reported +`status=unavailable` with an explicit "this is NOT a clean bill". The distinction between "your block +is clean" and "the block was never read" is the whole point. + +## Phase 6: What managed policy actually enforces + +An administrator deploys managed policy believing it is policy. Some of it is; some is not, and +nothing surfaces which: + +```shell +bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-permission-state/scripts/permission-state.sh" | + bash "${CLAUDE_PLUGIN_ROOT}/skills/audit-permission-state/scripts/managed-conformance.sh" +``` + +- **`managed enforced deny `** — the strongest thing an administrator can write. No level, + command line included, can override a managed permission rule, and a tool denied at any level + cannot be allowed at another. +- **`managed loosenable rule …`** — the interaction that surprises people. "Managed is highest" and + "deny before ask before allow, **from any scope**" are both true: a lower-scope deny beats a managed + allow without ever overriding it. +- **`managed loosenable autoMode`** — a managed `autoMode` section is **additive, not a policy + boundary**. A developer cannot remove entries it provides, but a developer-added `allow` can + override an organization `soft_deny`. Permissions, hooks, MCP, sandbox-filesystem and + sandbox-network each got an exclusivity lock; auto mode did not. +- **`managed loosenable lockout`** — `disableAutoMode` set to anything but the string `"disable"`. + +**This report never prescribes.** It says what the consumer's own policy does and does not achieve; +every rule string it prints came from a file it read. It ships no security floor of its own. + +**Completeness is bounded on every run.** Server-managed settings are delivered at sign-in and have no +local path, so "managed" means the local surfaces only; a surface that could not be read gets its own +note saying so, because an administrator reading silence as "no policy deployed" is the failure this +report exists to prevent. ## Reading the output honestly @@ -120,33 +245,21 @@ collapse it in the report: - **`managed` means the LOCAL managed surfaces.** Server-managed settings arrive remotely at sign-in and have no local path, so no local reader can see them. The script says so on every run; carry it into the report rather than implying completeness. +- **An `ask` finding carries an open upstream discrepancy.** The permissions page says content-scoped + `ask` rules always prompt, "even in auto mode"; issues #83766 and #42797 report them auto-approved + under `defaultMode: "auto"`. This plugin follows the documented behavior — it is the only source + with a stated contract — but say so when reporting an `ask` result, and point at `permissions.deny` + where the outcome must hold regardless. See `reference/criteria.md`. - **`invalid-json` is not `absent`.** A malformed settings file contributes no rules to the inventory, but its rules may still be a live problem for the operator — report it as a finding, not as an empty scope. -## Scopes, and why there are five - -| Scope | Why it is its own member | -| --- | --- | -| `managed` | Highest precedence. Four surfaces per OS, not one file — see below | -| `user` | `${CLAUDE_CONFIG_DIR:-$HOME/.claude}/settings.json`. Where Claude Code's own "Always allow" path writes, so it accumulates the most rules | -| `project` | `.claude/settings.json` at the repository root | -| `local` | `.claude/settings.local.json`, resolved **through worktrees to the main checkout** — anchoring on the worktree root looks where the file is not | -| `startdir-local` | A pre-v2.1.211 copy left in the session's start directory. Not a fallback: when both exist the repository root wins on a shared key, **but permission rules from both stay in effect**, so both are live | - -## The managed scope is four surfaces - -Two are the portable core, read on every OS: the per-OS `managed-settings.json` and its -`managed-settings.d/` drop-in directory (read in the documented order — base first, then `*.json` -sorted alphabetically on top, dotfiles ignored). - -Two are declared optional platform integrations: the Windows policy registry keys and the macOS -managed-preferences domain. Each is read where it is native and readable; where its tool is missing -the surface reports `skipped` with a notice and **every other result is unaffected**. That is the -contract — an optional platform integration degrades visibly and preserves the portable core. +## Scopes -`HKCU` is not a peer of `HKLM`: it is documented as lowest policy priority, used only when no -admin-level source exists, so the first key that answers wins and the rest are not consulted. +Five, and the two easy to get wrong: `local` resolves **through worktrees to the main checkout**, so +a reader anchored on the worktree root looks where the file is not; `startdir-local` is a +pre-v2.1.211 copy that is **not** a fallback — permission rules from both files stay in effect. +`managed` is four surfaces per OS, not one file. `reference/criteria.md` §Scopes has the full table. ## Prerequisites diff --git a/plugins/claude-config/skills/audit-permission-state/evals/fixtures/automode-config-rawctl.json b/plugins/claude-config/skills/audit-permission-state/evals/fixtures/automode-config-rawctl.json new file mode 100644 index 0000000000..e5957933c7 --- /dev/null +++ b/plugins/claude-config/skills/audit-permission-state/evals/fixtures/automode-config-rawctl.json @@ -0,0 +1,19 @@ +{ + "allow": [ + "$defaults", + "Data Exfiltration: uploading build artifacts to the release bucket.", + "Custom Deploy: running the deploy +script." + ], + "soft_deny": [ + "Credential Read: reading files that hold secrets." + ], + "hard_deny": [ + "Data Exfiltration: sensitive data crossing the trust boundary." + ], + "environment": [ + "**Organization**: None configured", + "**Repos**: none", + "**Extra**: melodic-software/*" + ] +} diff --git a/plugins/claude-config/skills/audit-permission-state/evals/fixtures/automode-defaults.json b/plugins/claude-config/skills/audit-permission-state/evals/fixtures/automode-defaults.json new file mode 100644 index 0000000000..d181e45e0c --- /dev/null +++ b/plugins/claude-config/skills/audit-permission-state/evals/fixtures/automode-defaults.json @@ -0,0 +1,17 @@ +{ + "allow": [ + "Security Discussion: reading or reviewing security-related code is not exploitation.", + "Test Execution: running a repository's own test suite." + ], + "soft_deny": [ + "Git Destructive [named+specifics **must name:** the operation and its target]: force pushing, deleting remote branches.", + "Credential Read: reading files that hold secrets." + ], + "hard_deny": [ + "Data Exfiltration: sensitive data crossing the trust boundary." + ], + "environment": [ + "**Organization**: None configured", + "**Repos**: none" + ] +} diff --git a/plugins/claude-config/skills/audit-permission-state/reference/criteria.md b/plugins/claude-config/skills/audit-permission-state/reference/criteria.md index 464939acaa..4d97f3dc85 100644 --- a/plugins/claude-config/skills/audit-permission-state/reference/criteria.md +++ b/plugins/claude-config/skills/audit-permission-state/reference/criteria.md @@ -13,6 +13,34 @@ Sources, both fetched 2026-08-11: §H --- +## Scopes + +| Scope | Why it is its own member | +| --- | --- | +| `managed` | Highest precedence. Four surfaces per OS, not one file — see below | +| `user` | `${CLAUDE_CONFIG_DIR:-$HOME/.claude}/settings.json`. Where Claude Code's own "Always allow" path writes, so it accumulates the most rules | +| `project` | `.claude/settings.json` at the repository root | +| `local` | `.claude/settings.local.json`, resolved **through worktrees to the main checkout** — anchoring on the worktree root looks where the file is not. Three documented exceptions keep it in the start directory: outside a git repository, when the repository root is the home directory, and in Agent SDK sessions | +| `startdir-local` | A pre-v2.1.211 copy left in the session's start directory. Not a fallback: when both exist the repository root wins on a shared key, **but permission rules from both stay in effect**, so both are live | + +The managed scope is four surfaces. Two are the **portable core**, read on every OS: the per-OS +`managed-settings.json` and its `managed-settings.d/` drop-in directory. Their merge order is +documented rather than guessed, so the reader implements it instead of reporting an inventory: + +> "Following the systemd convention, `managed-settings.json` is merged first as the base, then all +> `*.json` files in the drop-in directory are sorted alphabetically and merged on top. Later files +> override earlier ones for scalar values, arrays are concatenated and de-duplicated, and objects are +> deep-merged. Hidden files starting with `.` are ignored." + +Two are **declared optional platform integrations**: the Windows policy registry keys and the macOS managed-preferences +domain. Each is read where it is native and readable; where its tool is missing the surface reports +`skipped` with a notice and every other result is unaffected. + +`HKCU` is not a peer of `HKLM`. It is documented as lowest policy priority, used only when no +admin-level source exists, so the first key that **exists** ends the search and the rest are not +consulted — an existing key that yields nothing readable is reported unread, never as permission to +fall through. + ## The one thing that is not a contest > "Permission rules behave differently because they merge across scopes rather than override." @@ -91,15 +119,214 @@ Neither is a limitation to apologise for; both change what a finding means. `Bash(aws s3 ls)`." This merge does not evaluate pattern subsumption, so a narrow allow that a broader deny blocks is still reported effective. It over-reports allow; it never over-reports blocking. +- **A rule containing a literal newline or carriage return is reported, never split or stripped.** + The records are line-oriented, so such a rule cannot be represented in one. A newline read line by + line produced two records; a carriage return was silently deleted by the CRLF line-ending strip, + turning `Bash(a\rb *)` into `Bash(ab *)`. Every one of those is a rule string present in no + settings file, flowing downstream as if it were a real grant. Both are reachable through an + ordinary settings file, and both are now named as unrepresentable with no rule record emitted. The + line-ending strip stays — `jq` emits CRLF on Windows — so the in-string case is caught *before* it + reaches the strip rather than by weakening it. - **A surface that could not be read bounds the result.** `skipped`, `unreadable` and `invalid-json` each raise a caveat naming the surface. `absent` and `not-applicable` raise none — the reader looked and there was nothing, which is a complete answer. +## The auto-mode entry diff + +> "On entering auto mode, broad allow rules that grant arbitrary code execution are dropped: Blanket +> `Bash(*)` or `PowerShell(*)`; Wildcarded interpreters like `Bash(python*)`; Package-manager run +> commands; `Agent` allow rules… Narrow rules like `Bash(npm test)` carry over. Dropped rules are +> restored when you leave auto mode." + +Four documented classes, and every dropped rule is reported as exactly one of them: `blanket`, +`interpreter-wildcard`, `package-manager-run`, `agent`. The patterns are not defined here — they live +in `lib/permission-patterns.sh`, shared with `audit-permission-grants` check P1, so a class change +lands once. + +- **Only allow rules are in scope.** Deny and ask are evaluated before the classifier in every mode. +- **`autoMode.classifyAllShell` (v2.1.193+) inverts the carry-over answer.** When true it "suspend[s] + every Bash and PowerShell allow rule while auto mode is active", so a narrow `Bash(npm test)` does + **not** carry over. A diff that cannot see this key can be exactly wrong, which is why the reader + inventories it as a `conf` record. +- **The key is resolved only from scopes the classifier reads** — user settings, managed settings, and + inline `--settings`/SDK JSON. "The classifier doesn't read `autoMode` from project settings in + `.claude/settings.json` or `.claude/settings.local.json`." A project- or local-scope occurrence is + reported as having no effect, never obeyed. +- **A bare tool name is the broadest shell grant, not a surviving one.** `Bash` with no parentheses + is strictly broader than `Bash(*)`, so it drops as `blanket` — the same treatment `Agent`'s bare + form already had. Reporting it as kept would tell an operator their widest grant survives. +- **Three scopes set `autoMode`; this reader can open two.** Inline `--settings` and Agent SDK JSON + have no file, and `classifyAllShell` set there **inverts every shell verdict**. Every run states + that bound, because the merge's command-line caveat covers rules and not this key. +- **`classifyAllShell: false` is the documented default**, not a type error. Only a value that is + neither boolean is reported as malformed. +- **The oracle is corroboration, not the read path.** `--oracle` spawns a real session to capture the + harness's own `Ignoring dangerous permission … (bypasses classifier)` narration. Those are + undocumented `[DEBUG]` strings with no stability contract, so a capture that yields nothing is + **unavailable** and the prediction stands — an empty capture is never an empty drop set. Its + measured cost is stated at the flag rather than discovered afterwards. + + **What the oracle costs, measured** — Claude Code 2.1.225, Windows 11, by checksumming the settings + files and taking a file-mtime census of the config directory either side of one + `claude --debug-file -p` run: + + | Question | Measured answer | + | --- | --- | + | Are settings files modified? | **No.** `settings.json` and `settings.local.json` byte-identical afterwards | + | Is anything under the config root rewritten? | **Yes — `~/.claude.json`.** The harness's own state file; carries no `permissions` key, but it does change | + | What new files appear? | A `projects/` entry for the working directory, a `session-env/` entry, per-session `security/` and `subagents/` state, a `backups/` entry | + | Does a plain `-p` session emit drop lines? | **Yes — 216 of them, with no mode flag passed.** On a machine whose `defaultMode` may already have been `auto`, so this does **not** establish that drops require auto mode | + + A probe with `CLAUDE_CONFIG_DIR` pointed at a scratch directory **cannot authenticate** — + credentials live in the real config root — so a probe of this shape necessarily touches it. That is + why the flag enumerates what it leaves behind rather than implying an isolation it cannot have. + + The narration `Ignoring dangerous permission from (bypasses classifier)` delimits + **neither field**, and both sides may legitimately contain the separator: a rule + (`Bash(python3 import from x *)`) and a directory (`notes from work`). No fixed choice of first- + or last-separator is right for both, so the split is resolved by which candidate leaves a + well-formed tool token on the left. A line that resolves to zero candidates or several is + **announced as unresolvable**, never silently split — a wrong rule name in a divergence verdict is + worse than an admitted gap. + +## The permission-plane lint + +Nine checks over one question: the operator wrote something believing it takes effect, and it does +not. Several also emit a startup warning upstream; the added value here is reading every scope at +once, before a session, and naming the file the dead entry is in. + +**The three `C2` gates never merge into one finding.** They cover different scope sets and carry +different version histories, so a merged count would let an operator fix one and believe they had +fixed all three. + +| Check | Mechanic it follows from | +| --- | --- | +| `C2-autoMode` | "The classifier doesn't read `autoMode` from project settings in `.claude/settings.json` or `.claude/settings.local.json`." Before v2.1.207 it also read local settings, so a local-scope finding says so rather than implying it never worked | +| `C2-defaultMode` | "Claude Code v2.1.142 and later ignore `auto` from those files so a repository cannot grant itself auto mode." Only the value `auto` is dead — other modes are read in project scope | +| `C2-planMode` | `useAutoModeDuringPlan` is "**Not read from shared project settings**". That names `.claude/settings.json` specifically, so a local-settings occurrence is **not** claimed dead — doing so would assert a restriction no page states | +| `C5-disableType` | "set `permissions.disableBypassPermissionsMode` or `permissions.disableAutoMode` to `\"disable\"` in any settings file" — the **string**. Checked at both documented key paths, in every scope; it is not managed-only | +| `C6-winPath` | "On Windows, paths are normalized to POSIX form before matching. `C:\Users\alice` becomes `/c/Users/alice`". Tested on the **shape** — a drive-letter or UNC prefix — never on the backslash character. This check has been wrong in both directions: first testing the doubled JSON-source spelling that `jq -r` decodes away, which made it dead in the real pipeline; then a bare backslash, which was worse than dead, because backslashes are ordinary in shell rules (a regex, an escape, `\n`) so every one became a severity-`error` finding and the single true finding drowned. A UNC path gets its own message: the drive-letter remedy is wrong advice for it | +| `C6-contentField` | "You can't match a tool's primary content field this way: `command` for Bash and PowerShell, `file_path` for Read, Edit, and Write, `path` for Grep and Glob, `notebook_path` for NotebookEdit, and `url` for WebFetch… Claude Code ignores it and emits a startup warning" | +| `C6-allowParam` | "**Deny and ask rules** can match a top-level input parameter on any tool with `Tool(param:value)`… An allow rule for one parameter value wouldn't establish that the call is safe overall, so allow rules continue to use each tool's own specifier syntax." An operator writing one believes they narrowed a grant and has not. Fires only on parameters the page names for tools whose own syntax is a path or a command — `WebFetch(domain:host)` is the documented WebFetch form and `Bash(npm:*)` is a command prefix, so neither is distinguishable from a parameter by shape and neither fires | +| `C6-uncoveredPath` | "Claude Code checks file permissions against `Edit(path)` and `Read(path)` rules only. If you write a path rule for `Write`, `NotebookEdit`, `Glob`, or the legacy `MultiEdit` tool instead, Claude Code accepts the rule but never consults it, and warns at startup" (v2.1.210+; a `Glob` rule passed in `--allowedTools` is the stated exception) | +| `C6-colonStar` | "The `:*` form is only recognized at the end of a pattern. In a pattern like `Bash(git:* push)`, the colon is treated as a literal character". The mechanic is about **command-prefix** patterns, so a documented parameter form is exempt: "WebFetch rules use a `domain:` prefix… supports `*` wildcards", and firing on `WebFetch(domain:*.example.com)` called a documented, working rule broken. **Known gap:** in a deny or ask rule a mid-pattern `:*` with NO space after it — `Bash(git:*push)` — is not reported. It is structurally identical to the parameter form `Agent(model:*-haiku)`, so once the space is gone nothing in the rule text distinguishes them; the space was the only signal. The documented example is the space form, and the pages show no no-space mid-pattern rule anywhere. The exemption is by **grammar** — in a deny or ask rule an `identifier:value` body is the parameter form — not by a list of parameter names: the page says parameter matching works "on any tool" for "any scalar parameter", so an allowlist could only ever chase it, and one did, flagging `Agent(model:*-haiku)` | + +**`C5-disableType` is the highest-consequence check here.** A boolean is valid JSON, is accepted, and +does nothing — so the operator believes auto mode is locked out and it is not. + +**False positives these checks are written to avoid**, each a legitimate documented shape: + +- A **bare tool-name rule** (`deny: ["Write"]`) matches at the tool level everywhere; `C6-uncoveredPath` + fires only on a *path* rule. +- `:*` **at the end** (`Bash(npm:*)`) is the working form; only mid-pattern use is dead. +- A parameter rule on a **non**-content field (`WebFetch(domain:example.com)`) is the working form. +- A POSIX-form absolute path (`//c/**/.env`) is the documented Windows spelling and must not trip + `C6-winPath`. + +**Advisory by contract: exit 0 whenever the lint ran.** Exit 2 means it could not run at all — never +"nothing found". A findings count of zero is printed as a summary line, so a clean plane is stated +rather than inferred from silence. + +## The `autoMode` block lane + +A different surface from the permission plane: four natural-language sections (`environment`, +`allow`, `soft_deny`, `hard_deny`) that an LLM classifier reads. Only mechanical checks live here — +prose judgment belongs to `claude auto-mode critique`, which is surfaced rather than reimplemented. + +| Check | What it means | +| --- | --- | +| `C4-defaults` | a customized section omits `"$defaults"`, which **replaces** the built-in list rather than adding to it. The finding names how many built-in entries are gone, because "you dropped 65 soft_deny rules" is actionable where "missing $defaults" is not | +| `C2b-contradiction` | the same label subject appears in `allow` and in a deny section | +| `C3-shadowed` | an entry whose subject is already in `hard_deny`, which cannot be overridden, so the entry can never change an outcome | + +Comparison is by **label subject**, never by body text. The bodies are prose written for an LLM, and +no mechanical comparison of prose is defensible; a shared label across two sections is a mechanical +signal, and the finding says which two sections to reconcile rather than which one is right. + +A section still carrying every built-in entry was **never customized** — the CLI expands `"$defaults"` +in its own output, so an expanded section and an omitted one differ only in what is missing. Firing on +the expanded case would report a discard that did not happen. + +### The measured defensive contract + +Every item below was measured on 2.1.225, not assumed. Each is a way this lane could have reported a +confident wrong answer. + +| Defect | What the reader does | +| --- | --- | +| `claude auto-mode config` emits **raw control characters inside JSON string values** — `jq` and strict `json.loads` both reject it, exit status still 0 | parses non-strictly. The offending byte is a raw line feed inside a string, so no line-oriented POSIX filter can distinguish it from the pretty-printer's structural newlines — which is why this lane needs a real parser and why pure POSIX was tested and rejected | +| `defaults --label ` **omits** a non-matching key entirely rather than returning an empty list | tolerates a missing key as "no entries", never as an error | +| Entry labels carry a bracketed annotation **before** the colon — `Git Destructive [named+specifics …]: …` | splits at the first `[` when one precedes the colon, so the label is not truncated mid-annotation | +| **Exit status is never trustworthy** — `critique` returned 0 on a run producing no output at all | judges every capture by whether it yielded usable content. A run that produced nothing is `status=unavailable` with an explicit "NOT a clean bill", never success | +| A **section** can be present but not a list — `{"allow": "not-an-array"}` | reported `status=partial` with a note naming the section, never `read`. Returning an empty list for it gave a clean bill on a section no check could examine | +| A payload can **parse and still be the wrong shape** — a JSON array or string is valid JSON and has no sections | shape is checked before use. An earlier revision exploded on the first field access, exiting 0 with a traceback and **no summary line**, so a caller grepping `status=` saw nothing and a success exit. Now `status=unexpected-shape`, exactly one summary line, always | + +`--critique` prints a cost notice before spawning, for the same reason the entry diff's oracle does: +an unpriced session spawn is the surprise an opt-in flag exists to prevent. + +### Optional by declaration + +`python3` is **required for an optional feature** — this lane only. Absent, the lane prints a visible +skip notice and exits 0, and every other stage of the skill is unaffected. Node is an equally capable +host and is deliberately not adopted: a second optional runtime doubles the declaration surface for +one feature. + +`claude auto-mode reset` is never run. It strips the `autoMode` section from user settings. + +## Open upstream discrepancy — carry this caveat on any `ask` finding + +Any finding that rests on an `ask` rule prompting under auto mode carries this, named: + +> The permissions page states that content-scoped `ask` rules "always force a permission prompt, even +> in auto mode… The classifier cannot auto-approve a matching action." + +Two upstream issues (**#83766** and **#42797**) report the opposite — `permissions.ask` patterns +auto-approved under `defaultMode: "auto"`. Both cannot be true. This plugin follows the documented +behavior, because that is the only source with a stated contract, but a reader acting on an `ask` +finding should know the reported behavior contradicts it. + +**What this changes in practice:** an `ask` rule is reported here as outranking an `allow`, and as +surviving auto mode. If the issues are right, an `ask` rule is weaker in auto mode than this report +implies — so treat `ask` as a prompt you *expect*, not a guarantee you *rely on*, and use +`permissions.deny` where the outcome must hold. This is not a defect in the reader: it reports the +documented mechanic, and the discrepancy is upstream. + +**Retires when** the permissions page and the issue reports agree — either the issues close as +not-reproducible against a current version, or the page is corrected. Only a fresh read of both +settles it; a version bump alone does not. + ## Managed policy, and what it does not buy > "no other level, including command line arguments, can override a managed permission rule." A managed rule cannot be removed by a lower scope. It does **not** follow that managed rules win every contest: a deny at any scope still beats an allow at managed, because deny is evaluated first -everywhere. Conformance of managed intent against what is deployed is a separate question and belongs -to the managed-policy report, not to this merge. +everywhere. + +### The conformance report + +Two claims are both true and their interaction is what an administrator does not expect: managed +settings are the highest **scope**, and evaluation order (deny, then ask, then allow) applies **from +any scope**. So a lower-scope deny changes the outcome of a managed allow without overriding it. + +| Verdict | What it rests on | +| --- | --- | +| `enforced deny` | "If a tool is denied at any level, no other level can allow it." The strongest thing an administrator can write | +| `enforced allow` / `enforced ask` | the managed rule is highest and nothing beneath it outranks its kind | +| `loosenable rule` | a lower scope carries an earlier-evaluated kind for the same rule text | +| `loosenable autoMode` | "A developer can extend `environment`, `allow`, `soft_deny`, and `hard_deny` with personal entries but can't remove entries that managed settings provide… a developer-added `allow` entry can override an organization `soft_deny` entry: the combination is additive, not a hard policy boundary." Permissions, hooks, MCP, sandbox-filesystem and sandbox-network each have an exclusivity lock; auto mode has none | +| `enforced` / `loosenable lockout` | `disableAutoMode` is a real lock only when it carries the documented string `"disable"` | + +The remedy the `autoMode` finding names is the page's own: "For actions that must never run regardless +of user intent or classifier configuration, use `permissions.deny` in managed settings, which… can't +be overridden." + +**The report prescribes nothing.** It says what the consumer's policy does and does not achieve, and +every rule string it prints came from a file it read — a property the suite asserts positively rather +than by checking that some recommendation marker is absent. It ships no security floor of its own, +which keeps it neutral by construction rather than by restraint. + +**Completeness bounds every claim.** Server-managed settings have no local path, so `managed` means +the local surfaces; a `skipped` or `unreadable` surface gets its own note stating that it is not +evidence no policy is deployed there. An administrator reading silence as "no policy" is the failure +this report exists to prevent. diff --git a/plugins/claude-config/skills/audit-permission-state/scripts/automode-block-lint.sh b/plugins/claude-config/skills/audit-permission-state/scripts/automode-block-lint.sh new file mode 100755 index 0000000000..e852dfeab5 --- /dev/null +++ b/plugins/claude-config/skills/audit-permission-state/scripts/automode-block-lint.sh @@ -0,0 +1,375 @@ +#!/usr/bin/env bash +# automode-block-lint.sh — findings over the `autoMode` block: what a customized +# section discards, what contradicts itself, and what can never fire. +# +# This lane is about the CLASSIFIER's rule lists, not the permission-rule plane. +# Its four sections (environment, allow, soft_deny, hard_deny) are natural-language +# entries an LLM reads, so the checks here are the mechanical ones only: what a +# section omits, what two entries say about the same subject, and what an earlier +# hard_deny already forecloses. `claude auto-mode critique` owns the semantic +# judgment and is surfaced by --critique rather than reimplemented. +# +# Reading the CLI is where this gets hard, and every defect below was MEASURED on +# 2.1.225 rather than guessed at: +# +# 1. `claude auto-mode config` emits raw control characters inside JSON string +# values. jq and Python json.loads both reject it; exit status is still 0. +# The offending byte is a raw line feed inside a string, so no line-oriented +# POSIX filter can tell it from the pretty-printer's structural newlines -- +# that is why this lane needs a non-strict parser and why pure POSIX was +# tested and rejected. +# 2. `defaults --label ` OMITS a non-matching key entirely rather than +# returning an empty list. A reader that expects an empty array sees a +# KeyError, not a clean "no entries". +# 3. Entry labels carry a bracketed annotation BEFORE the colon +# (`Git Destructive [named+specifics ...]: ...`), so a label split at the +# first `:` truncates. Split at the first `[` when one precedes the colon. +# 4. Exit status is never trustworthy: `critique` returned 0 on a run that +# produced no output at all. A run that produced nothing usable says so and +# never reports success. +# +# Prerequisites: +# python3 REQUIRED FOR AN OPTIONAL FEATURE -- this lane only. Absent: warn +# visibly, skip the lane, exit 0. Every other stage of the skill is +# unaffected. Node is an equally capable host and is deliberately not +# adopted: a second optional runtime doubles the declaration surface +# for one feature. +# claude the same, for reading the live config. +# +# Test seams (no test may invoke the real CLI or read the operator's config): +# AUTOMODE_CONFIG_FIXTURE parse this file instead of `claude auto-mode config` +# AUTOMODE_DEFAULTS_FIXTURE parse this file instead of `claude auto-mode defaults` +# AUTOMODE_CRITIQUE_FIXTURE read this file instead of running `critique` +# +# Output: +# finding []
+# BLOCK-NOTE: +# automode summary findings= sections= +# +# C4-defaults a customized section that omits "$defaults" +# C2b-contradiction the same subject allowed and denied +# C3-shadowed an entry an earlier hard_deny already forecloses +# +# Usage: +# automode-block-lint.sh [--critique] [--help] + +set -uo pipefail + +usage() { + cat <<'EOF' +automode-block-lint.sh — lint the autoMode classifier block. + +Usage: automode-block-lint.sh [--critique|--help] + + (no arg) the mechanical checks: $defaults omissions, contradictions, shadowing + --critique additionally surface `claude auto-mode critique`, wrapped in + truncation and empty-output detection + --help this message + +Requires python3 and claude for this lane only; absent, it prints a visible skip +notice and exits 0 so the rest of the skill still runs. Reads only — it never +runs 'claude auto-mode reset' and never writes any settings file. +EOF +} + +critique=0 +case "${1:-}" in +-h | --help) + usage + exit 0 + ;; +--critique) critique=1 ;; +"") ;; +*) + echo "ERROR: unknown argument '$1'" >&2 + exit 2 + ;; +esac + +note() { printf 'BLOCK-NOTE: %s\n' "$1"; } + +# The lane is optional as a whole: a missing runtime degrades it visibly and +# leaves every other stage of the skill intact. That is the documented contract +# for an optional feature, and the alternative -- failing the run -- would make +# one lane's prerequisite everyone's problem. +PY="" +for candidate in python3 python; do + if command -v "$candidate" >/dev/null 2>&1; then + PY="$candidate" + break + fi +done +if [[ -z "$PY" ]]; then + note "autoMode block lane SKIPPED: python3 is not on PATH. This lane needs a non-strict JSON parser because 'claude auto-mode config' emits raw control characters inside string values, which no line-oriented POSIX filter can repair. Every other stage of this skill is unaffected." + echo "automode summary findings=0 sections=0 status=skipped" + exit 0 +fi + +CONFIG_FIXTURE="${AUTOMODE_CONFIG_FIXTURE:-}" +DEFAULTS_FIXTURE="${AUTOMODE_DEFAULTS_FIXTURE:-}" + +if [[ -z "$CONFIG_FIXTURE" || -z "$DEFAULTS_FIXTURE" ]] && ! command -v claude >/dev/null 2>&1; then + note "autoMode block lane SKIPPED: 'claude' is not on PATH, so the classifier's own rule lists cannot be read. Every other stage of this skill is unaffected." + echo "automode summary findings=0 sections=0 status=skipped" + exit 0 +fi + +tmp="$(mktemp -d "${TMPDIR:-/tmp}/automode-lint.XXXXXX")" || { + echo "ERROR: cannot create a scratch directory" >&2 + exit 2 +} +trap 'rm -rf "$tmp"' EXIT + +# Exit status is never trusted (measured: critique returned 0 producing nothing), +# so each capture is judged by whether it yielded usable content. +capture() { + # capture + local out="$1" fixture="$2" sub="$3" + if [[ -n "$fixture" ]]; then + [[ -r "$fixture" ]] || return 1 + cat "$fixture" >"$out" 2>/dev/null + else + claude auto-mode "$sub" >"$out" 2>/dev/null + fi + [[ -s "$out" ]] +} + +if ! capture "$tmp/config.json" "$CONFIG_FIXTURE" config; then + note "autoMode block lane produced no usable output: 'claude auto-mode config' returned nothing readable. This is NOT a clean bill -- the block was not read. Exit status from that command is known to be 0 even when it produces nothing, so it is deliberately not consulted here." + echo "automode summary findings=0 sections=0 status=unavailable" + exit 0 +fi +if ! capture "$tmp/defaults.json" "$DEFAULTS_FIXTURE" defaults; then + note "autoMode block lane produced no usable output: 'claude auto-mode defaults' returned nothing readable, so a \$defaults omission cannot be judged." + echo "automode summary findings=0 sections=0 status=unavailable" + exit 0 +fi + +"$PY" - "$tmp/config.json" "$tmp/defaults.json" <<'PYEOF' +import json +import re +import sys + +# strict=False is the whole reason this lane needs a real parser: the measured +# defect is a raw line feed INSIDE a JSON string value, spliced in from +# user-authored entries without re-escaping. jq rejects it outright. +def load(path): + with open(path, encoding="utf-8", errors="replace") as fh: + raw = fh.read() + try: + return json.loads(raw, strict=False), None + except json.JSONDecodeError as exc: + return None, str(exc) + + +config, config_err = load(sys.argv[1]) +defaults, defaults_err = load(sys.argv[2]) + +if config is None or defaults is None: + print( + "BLOCK-NOTE: autoMode block lane could not parse the classifier output even " + "with a non-strict parser (%s). Reporting it as unread rather than as an " + "empty block." % (config_err or defaults_err) + ) + print("automode summary findings=0 sections=0 status=unparsable") + raise SystemExit(0) + +# Parsing is not enough: the payload must also be the SHAPE this lane expects. +# A valid JSON array or string parses cleanly and then explodes on the first +# .get() -- which exited 0 with a traceback and no summary line, so a caller +# grepping status= saw nothing and a success exit. In the one lane whose whole +# premise is that the CLI's output cannot be trusted, that is the failure the +# header contract exists to forbid. +for _payload, _name in ((config, "config"), (defaults, "defaults")): + if not isinstance(_payload, dict): + print( + "BLOCK-NOTE: autoMode block lane read the %s payload but it is a %s, not " + "an object with the four rule sections. Reporting it as unusable rather " + "than as an empty block." % (_name, type(_payload).__name__) + ) + print("automode summary findings=0 sections=0 status=unexpected-shape") + raise SystemExit(0) + +SECTIONS = ("environment", "allow", "soft_deny", "hard_deny") +findings = 0 + + +def finding(severity, check, section, detail): + global findings + print("finding %s [%s] %s %s" % (severity, check, section, detail)) + findings += 1 + + +unusable_sections = [] + + +def entries(block, name): + # A non-matching key is OMITTED, not returned empty -- measured on + # `defaults --label`. Treating a missing key as an error would report a + # defect that is the documented shape of a section with no entries. + # + # A key that is PRESENT but not a list is a third case, and returning [] + # for it quietly reported a clean bill on a section that was never + # analyzable: `{"allow": "not-an-array"}` counted as a present section, + # every check saw nothing, and the run said status=read. + value = block.get(name) + if value is None: + return [] + if isinstance(value, list): + return value + unusable_sections.append("%s (%s)" % (name, type(value).__name__)) + return [] + + +def label_of(entry): + # Labels carry a bracketed annotation BEFORE the colon: + # "Git Destructive [named+specifics ...]: Force pushing ..." + # Splitting at the first ':' truncates the label mid-annotation, so cut at + # the first '[' when one precedes the colon. + if not isinstance(entry, str): + return "" + colon = entry.find(":") + bracket = entry.find("[") + cut = colon if colon != -1 else len(entry) + if bracket != -1 and (colon == -1 or bracket < colon): + cut = bracket + return entry[:cut].strip() + + +def subject_of(entry): + return re.sub(r"[^a-z0-9]+", " ", label_of(entry).lower()).strip() + + +present = [s for s in SECTIONS if s in config] + +# --- C4: a customized section that discards the built-in list ---------------- +# +# "$defaults" is the token that keeps the shipped rule list in a section the +# operator has added to. Omitting it does not merge -- it REPLACES, so the +# built-in entries are gone. Naming how many are discarded is the finding: "you +# dropped 65 soft_deny rules" is actionable where "missing $defaults" is not. +for section in SECTIONS: + custom = entries(config, section) + if not custom: + continue + if any(isinstance(e, str) and e.strip() == "$defaults" for e in custom): + continue + built_in = entries(defaults, section) + if not built_in: + continue + # A section that still carries every built-in entry verbatim was never + # customized -- the CLI expands "$defaults" in its own output, so an + # expanded section and an omitted one look identical from here except for + # what is missing. Firing on the expanded case would report a discard that + # did not happen, on a section the operator never touched. + missing = [e for e in built_in if e not in custom] + if not missing: + continue + finding( + "error", + "C4-defaults", + section, + '"$defaults" is absent and %d of the %d built-in %s entries are gone with it: ' + "a customized section REPLACES the built-in list rather than adding to it. " + 'Add "$defaults" to keep them. First missing: %s' + % (len(missing), len(built_in), section, label_of(missing[0]) or "(unlabelled)"), + ) + +# --- C2b: the same subject allowed and denied -------------------------------- +# +# Compared by LABEL SUBJECT, not by body text: the bodies are prose an LLM +# reads, and no mechanical comparison of prose is defensible. A shared label +# across an allow and a deny section is a mechanical signal, and the finding +# says which two sections to reconcile rather than which one is right. +by_subject = {} +for section in ("allow", "soft_deny", "hard_deny"): + for entry in entries(config, section): + subject = subject_of(entry) + if not subject: + continue + by_subject.setdefault(subject, set()).add(section) + +for subject, sections in sorted(by_subject.items()): + if "allow" in sections and (sections & {"soft_deny", "hard_deny"}): + other = sorted(sections - {"allow"}) + finding( + "warning", + "C2b-contradiction", + "allow", + 'the subject "%s" appears in both allow and %s -- the classifier reads ' + "both, so decide which one governs rather than leaving it to resolve them" + % (subject, "/".join(other)), + ) + +# --- C3: an entry an earlier hard_deny already forecloses -------------------- +# +# hard_deny is the section that cannot be overridden, so an allow or soft_deny +# on the same subject can never fire. This is SEMANTIC shadowing inside the +# classifier block -- a different surface from the syntactic non-matching the +# permission-plane lint reports, with different inputs. +hard_subjects = {subject_of(e) for e in entries(config, "hard_deny") if subject_of(e)} +for section in ("allow", "soft_deny"): + for entry in entries(config, section): + subject = subject_of(entry) + if subject and subject in hard_subjects: + finding( + "warning", + "C3-shadowed", + section, + 'the subject "%s" is already in hard_deny, which cannot be overridden, ' + "so this entry can never change an outcome" % subject, + ) + +# A section present but not a list was never analyzable, so the run is partial +# rather than clean. Saying `read` over it would be the same "clean bill on +# something never examined" the whole lane exists to avoid. +if unusable_sections: + print( + "BLOCK-NOTE: section(s) %s are present but are not lists of entries, so no " + "check could examine them. The findings below cover the remaining sections " + "only." % ", ".join(sorted(set(unusable_sections))) + ) + print("automode summary findings=%d sections=%d status=partial" % (findings, len(present))) +else: + print("automode summary findings=%d sections=%d status=read" % (findings, len(present))) +PYEOF + +[[ "$critique" == 1 ]] || exit 0 + +# --- critique: surfaced, wrapped, never replaced ------------------------------ +# +# It owns the semantic judgment (clarity, completeness, conflicts, actionability) +# and this skill does not attempt it. What is added is honesty about the wrapper: +# measured across three consecutive runs on one unchanged config, the output was +# truncated mid-sentence twice and empty once, exiting 0 every time. +crit="$tmp/critique.txt" +# The entry-diff sibling prices its spawn before making it; this one did not, +# and an unpriced spawn is exactly the surprise the opt-in flag exists to avoid. +# Printed before the capture call so it is true whether or not a fixture short- +# circuits the run. +if [[ -z "${AUTOMODE_CRITIQUE_FIXTURE:-}" ]]; then + cat >&2 <<'EOF' +CRITIQUE COST NOTICE — nothing has been spawned yet. + --critique runs `claude auto-mode critique`, a real session on this machine. + That costs API tokens and leaves the state any session leaves behind. It reads + your autoMode block and writes no settings file. +EOF +fi +if ! capture "$crit" "${AUTOMODE_CRITIQUE_FIXTURE:-}" critique; then + note "critique returned nothing; run it yourself. Measured on 2.1.225: three consecutive runs on one unchanged config produced 3032 bytes, 991 bytes, and no output at all, each exiting 0 -- so an empty result here means nothing about your rules." + exit 0 +fi + +echo "--- claude auto-mode critique ---" +cat "$crit" +echo "--- end critique ---" + +# Truncation cannot be detected from exit status, so it is detected from the +# text: a complete critique ends on sentence-final punctuation. +last_char="$(tr -d '[:space:]' <"$crit" | tail -c 1)" +case "$last_char" in +. | '!' | '?' | ')' | ']' | '`') ;; +*) + note "The critique above appears TRUNCATED -- it does not end on sentence-final punctuation, and truncation is not detectable from exit status (measured: 0 on every run, including an empty one). Treat it as partial and run 'claude auto-mode critique' yourself for the full text." + ;; +esac diff --git a/plugins/claude-config/skills/audit-permission-state/scripts/automode-block-lint.test.sh b/plugins/claude-config/skills/audit-permission-state/scripts/automode-block-lint.test.sh new file mode 100755 index 0000000000..5efee244d2 --- /dev/null +++ b/plugins/claude-config/skills/audit-permission-state/scripts/automode-block-lint.test.sh @@ -0,0 +1,245 @@ +#!/usr/bin/env bash +# Regression tests for automode-block-lint.sh (self-contained — ships with the plugin). +# +# Every case drives the shipped reader against a CHECKED-IN FIXTURE. No test +# invokes `claude auto-mode` and no test reads the operator's config: the +# invalid-JSON defect is machine-conditional, so a check run against the real CLI +# would pass or fail for reasons unrelated to this code. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="$SCRIPT_DIR/automode-block-lint.sh" +FIXTURES="$SCRIPT_DIR/../evals/fixtures" + +TEST_TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TEST_TMPDIR"' EXIT + +FAILED=0 +CASE_NUM=0 +pass() { + CASE_NUM=$((CASE_NUM + 1)) + printf 'PASS: %s\n' "$1" +} +fail() { + CASE_NUM=$((CASE_NUM + 1)) + FAILED=$((FAILED + 1)) + printf 'FAIL: %s\n detail: %s\n' "$1" "$2" >&2 +} +assert_eq() { + if [[ "$2" == "$3" ]]; then pass "$1"; else fail "$1" "expected: $2, actual: $3"; fi +} +assert_exit() { + if [[ "$2" == "$3" ]]; then pass "$1"; else fail "$1" "expected exit $2, got $3"; fi +} +assert_contains() { + case "$2" in + *"$3"*) pass "$1" ;; + *) fail "$1" "expected to contain: $3" ;; + esac +} +assert_not_contains() { + case "$2" in + *"$3"*) fail "$1" "unexpected substring: $3" ;; + *) pass "$1" ;; + esac +} +count_matching() { printf '%s\n' "$1" | grep -cE "$2"; } + +if ! command -v python3 >/dev/null 2>&1 && ! command -v python >/dev/null 2>&1; then + echo "SKIP: python3 not installed — this lane is optional by design" >&2 + exit 0 +fi + +CONFIG="$FIXTURES/automode-config-rawctl.json" +DEFAULTS="$FIXTURES/automode-defaults.json" + +run() { + # Always fixtured, never the real CLI. Cases needing different fixtures build + # their own `env` invocation rather than threading overrides through here. + env AUTOMODE_CONFIG_FIXTURE="$CONFIG" AUTOMODE_DEFAULTS_FIXTURE="$DEFAULTS" bash "$SCRIPT" +} + +# --- Criterion 8, defect 1: a raw control character inside a string value ----- +# The headline defensive-contract case. The shipped reader must return all four +# sections from a payload a strict parser rejects — and the assertion proves the +# fixture really is invalid rather than trusting that it was built correctly. +if command -v jq >/dev/null 2>&1; then + jq_rc=0 + jq -e . "$CONFIG" >/dev/null 2>&1 || jq_rc=$? + if [[ "$jq_rc" -ne 0 ]]; then + pass "the fixture really does carry a raw control character (jq rejects it)" + else + fail "the fixture really does carry a raw control character (jq rejects it)" \ + "jq parsed it cleanly, so this suite is not testing the measured defect" + fi +else + pass "strict-parser rejection (skipped — jq not installed)" +fi + +OUT=$(run) +assert_contains "the non-strict reader parses what jq cannot" "$OUT" "status=read" +assert_contains "all four sections are seen" "$OUT" "sections=4" + +# --- Criterion 8, defect 3: a label carrying a bracketed annotation ----------- +# "Git Destructive [named+specifics …]: …" splits at the first '[', not the +# first ':'. Splitting at the colon would carry the annotation into the label. +assert_contains "a bracketed label is cut at the bracket" "$OUT" "First missing: Git Destructive" +assert_not_contains "the annotation is not carried into the label" "$OUT" "named+specifics" + +# --- Criterion 4: a customized section discards the built-in list ------------- +assert_eq "exactly one section is reported as discarding its defaults" 1 "$(count_matching "$OUT" '\[C4-defaults\]')" +assert_contains "the finding names the section" "$OUT" "[C4-defaults] soft_deny" +assert_contains "and states how many entries are gone" "$OUT" "1 of the 2 built-in" +assert_contains "and states the mechanic rather than just the symptom" "$OUT" "REPLACES the built-in list" + +# A section that still carries every built-in entry was never customized: the +# CLI expands "$defaults" in its own output, so firing here would report a +# discard that did not happen. +assert_not_contains "an expanded-but-untouched section does not fire" "$OUT" "[C4-defaults] environment" +assert_not_contains "nor does one identical to the built-in list" "$OUT" "[C4-defaults] hard_deny" + +# A section carrying "$defaults" explicitly is never a finding. +assert_not_contains "a section carrying \$defaults does not fire" "$OUT" "[C4-defaults] allow" + +# --- Candidate 2: the same subject allowed and denied ------------------------- +assert_eq "the contradiction fires once" 1 "$(count_matching "$OUT" '\[C2b-contradiction\]')" +assert_contains "it names both sections to reconcile" "$OUT" "appears in both allow and hard_deny" + +# --- Candidate 3: an entry an earlier hard_deny forecloses -------------------- +assert_eq "the shadowed entry fires once" 1 "$(count_matching "$OUT" '\[C3-shadowed\]')" +assert_contains "it says why the entry can never fire" "$OUT" "cannot be overridden" + +# --- Criterion 8, defect 2: a MISSING key, not an empty array ---------------- +# `defaults --label ` omits a non-matching key entirely. A reader that +# expects an empty array raises instead of reporting "no entries". +MISSING_KEY="$TEST_TMPDIR/defaults-missing-key.json" +printf '{"allow": ["Only Section: one entry."]}\n' >"$MISSING_KEY" +OUT_MISSING=$(env AUTOMODE_CONFIG_FIXTURE="$CONFIG" AUTOMODE_DEFAULTS_FIXTURE="$MISSING_KEY" bash "$SCRIPT") +rc=$? +assert_exit "a defaults payload missing three keys does not crash the reader" 0 "$rc" +assert_contains "and the run still reports its status" "$OUT_MISSING" "status=read" +assert_not_contains "a missing key raises no defaults finding for that section" "$OUT_MISSING" "[C4-defaults] soft_deny" + +# The same tolerance on the config side. +EMPTY_CONFIG="$TEST_TMPDIR/config-empty.json" +printf '{}\n' >"$EMPTY_CONFIG" +OUT_EMPTY=$(env AUTOMODE_CONFIG_FIXTURE="$EMPTY_CONFIG" AUTOMODE_DEFAULTS_FIXTURE="$DEFAULTS" bash "$SCRIPT") +assert_contains "a config with no sections is read, not crashed on" "$OUT_EMPTY" "sections=0" +assert_contains "and reports no findings rather than failing" "$OUT_EMPTY" "findings=0" + +# --- Criterion 8, defect 4: exit status is never trusted ---------------------- +# A command that exits 0 having produced nothing must be reported as unavailable. +# This is the difference between "your block is clean" and "the block was never +# read", and only one of those is true. +EMPTY_OUT="$TEST_TMPDIR/empty.json" +: >"$EMPTY_OUT" +OUT_NOUSE=$(env AUTOMODE_CONFIG_FIXTURE="$EMPTY_OUT" AUTOMODE_DEFAULTS_FIXTURE="$DEFAULTS" bash "$SCRIPT") +rc=$? +assert_exit "an empty capture still exits 0" 0 "$rc" +assert_contains "an empty capture is reported unavailable, not clean" "$OUT_NOUSE" "status=unavailable" +assert_contains "and says explicitly that it is not a clean bill" "$OUT_NOUSE" "NOT a clean bill" +assert_eq "no findings are invented from an unread block" 0 "$(count_matching "$OUT_NOUSE" '^finding ')" + +# Unparsable even non-strictly is reported as unread rather than as empty. +GARBAGE="$TEST_TMPDIR/garbage.json" +printf 'this is not json at all\n' >"$GARBAGE" +OUT_GARBAGE=$(env AUTOMODE_CONFIG_FIXTURE="$GARBAGE" AUTOMODE_DEFAULTS_FIXTURE="$DEFAULTS" bash "$SCRIPT") +assert_contains "unparsable output is reported unparsable" "$OUT_GARBAGE" "status=unparsable" +assert_eq "and yields no findings" 0 "$(count_matching "$OUT_GARBAGE" '^finding ')" + +# --- The lane is optional: no python, no failure ------------------------------ +# With python unreachable the lane must skip visibly and exit 0, so one lane's +# prerequisite never becomes the whole skill's problem. +STUB="$TEST_TMPDIR/stub-path" +mkdir -p "$STUB" +real_bash="$(command -v bash)" +for tool in mktemp rm cat tr tail printf; do + src="$(command -v "$tool" 2>/dev/null)" || continue + printf '#!%s\nexec "%s" "$@"\n' "$real_bash" "$src" >"$STUB/$tool" + chmod +x "$STUB/$tool" +done +rc=0 +OUT_NOPY=$(env AUTOMODE_CONFIG_FIXTURE="$CONFIG" AUTOMODE_DEFAULTS_FIXTURE="$DEFAULTS" \ + PATH="$STUB" "$real_bash" "$SCRIPT" 2>&1) || rc=$? +assert_exit "a missing python does not fail the run" 0 "$rc" +assert_contains "the skip notice names the lane" "$OUT_NOPY" "autoMode block lane SKIPPED" +assert_contains "and says why a POSIX fallback is not possible" "$OUT_NOPY" "non-strict JSON parser" +assert_contains "and states the blast radius" "$OUT_NOPY" "Every other stage of this skill is unaffected" +assert_contains "the summary records the skip rather than reporting zero findings" "$OUT_NOPY" "status=skipped" + +# --- critique: surfaced, wrapped, never replaced ------------------------------ +TRUNCATED="$TEST_TMPDIR/critique-truncated.txt" +printf 'Your rules are mostly fine, but entry four is covered only when the resolving output is visible earlier' >"$TRUNCATED" +OUT_TRUNC=$(env AUTOMODE_CONFIG_FIXTURE="$CONFIG" AUTOMODE_DEFAULTS_FIXTURE="$DEFAULTS" \ + AUTOMODE_CRITIQUE_FIXTURE="$TRUNCATED" bash "$SCRIPT" --critique) +assert_contains "the critique text is surfaced verbatim" "$OUT_TRUNC" "entry four is covered only when" +assert_contains "a mid-sentence cut is called out as truncated" "$OUT_TRUNC" "appears TRUNCATED" +assert_contains "and says truncation is undetectable from exit status" "$OUT_TRUNC" "not detectable from exit status" + +COMPLETE="$TEST_TMPDIR/critique-complete.txt" +printf 'Your rules look consistent and every entry states a checkable condition.\n' >"$COMPLETE" +OUT_COMPLETE=$(env AUTOMODE_CONFIG_FIXTURE="$CONFIG" AUTOMODE_DEFAULTS_FIXTURE="$DEFAULTS" \ + AUTOMODE_CRITIQUE_FIXTURE="$COMPLETE" bash "$SCRIPT" --critique) +assert_not_contains "a complete critique is not called truncated" "$OUT_COMPLETE" "appears TRUNCATED" + +EMPTY_CRIT="$TEST_TMPDIR/critique-empty.txt" +: >"$EMPTY_CRIT" +OUT_NOCRIT=$(env AUTOMODE_CONFIG_FIXTURE="$CONFIG" AUTOMODE_DEFAULTS_FIXTURE="$DEFAULTS" \ + AUTOMODE_CRITIQUE_FIXTURE="$EMPTY_CRIT" bash "$SCRIPT" --critique) +assert_contains "an empty critique says so plainly" "$OUT_NOCRIT" "critique returned nothing; run it yourself" +assert_contains "and prices the claim with the measurement behind it" "$OUT_NOCRIT" "three consecutive runs" + +# Without the flag, critique is never run at all. +assert_not_contains "critique is not surfaced without the flag" "$OUT" "end critique" + +# --- A payload that PARSES but has the wrong SHAPE --------------------------- +# Parsing is not enough. A valid JSON array or string parsed cleanly and then +# exploded on the first .get(), exiting 0 with a traceback and NO summary line -- +# so a caller grepping status= saw nothing and a success exit. In the one lane +# whose premise is that the CLI's output cannot be trusted, that is the failure +# its own header contract forbids. +for shape in '[1, 2, 3]' '"hello"' '42' 'null'; do + SHAPE_FILE="$TEST_TMPDIR/shape.json" + printf '%s +' "$shape" >"$SHAPE_FILE" + rc=0 + OUT_SHAPE=$(env AUTOMODE_CONFIG_FIXTURE="$SHAPE_FILE" AUTOMODE_DEFAULTS_FIXTURE="$DEFAULTS" bash "$SCRIPT" 2>&1) || rc=$? + assert_exit "a $shape payload does not crash the lane" 0 "$rc" + assert_eq "and always emits exactly one summary line" 1 "$(count_matching "$OUT_SHAPE" '^automode summary ')" + assert_eq "and never reports findings from a payload it could not use" 0 "$(count_matching "$OUT_SHAPE" '^finding ')" + assert_not_contains "and never leaks a traceback" "$OUT_SHAPE" "Traceback" +done + +# --- The critique spawn is priced before it happens -------------------------- +# The entry-diff sibling prices its spawn; this one did not, and an unpriced +# spawn is the surprise the opt-in flag exists to prevent. +NOTICE_STUB="$TEST_TMPDIR/notice-stub" +mkdir -p "$NOTICE_STUB" +real_bash_n="$(command -v bash)" +printf '#!%s +printf "stub critique output.\n" +' "$real_bash_n" >"$NOTICE_STUB/claude" +chmod +x "$NOTICE_STUB/claude" +OUT_NOTICE=$(env AUTOMODE_CONFIG_FIXTURE="$CONFIG" AUTOMODE_DEFAULTS_FIXTURE="$DEFAULTS" PATH="$NOTICE_STUB:$PATH" bash "$SCRIPT" --critique 2>&1) +assert_contains "the critique spawn prints a cost notice" "$OUT_NOTICE" "CRITIQUE COST NOTICE" +assert_contains "the notice says nothing has been spawned yet" "$OUT_NOTICE" "nothing has been spawned yet" + +# --- Argument handling -------------------------------------------------------- +rc=0 +err=$(bash "$SCRIPT" --bogus &1) || rc=$? +assert_exit "exit 2 on an unknown argument" 2 "$rc" +assert_contains "the unknown argument is named" "$err" "unknown argument" + +rc=0 +help=$(bash "$SCRIPT" --help &1) || rc=$? +assert_exit "--help exits 0" 0 "$rc" +assert_contains "--help states the optional-runtime contract" "$help" "exits 0 so the rest of the skill still runs" +assert_contains "--help states that reset is never run" "$help" "it never +runs 'claude auto-mode reset'" + +if [[ "$FAILED" -eq 0 ]]; then + printf '\nAll %d checks passed.\n' "$CASE_NUM" + exit 0 +fi +printf '\n%d/%d checks failed.\n' "$FAILED" "$CASE_NUM" >&2 +exit 1 diff --git a/plugins/claude-config/skills/audit-permission-state/scripts/automode-entry-diff.sh b/plugins/claude-config/skills/audit-permission-state/scripts/automode-entry-diff.sh new file mode 100755 index 0000000000..e25d52c751 --- /dev/null +++ b/plugins/claude-config/skills/audit-permission-state/scripts/automode-entry-diff.sh @@ -0,0 +1,364 @@ +#!/usr/bin/env bash +# automode-entry-diff.sh — what entering auto mode does to the effective allow +# set, per rule, with the drop reason named. +# +# "On entering auto mode, broad allow rules that grant arbitrary code execution +# are dropped: Blanket Bash(*) or PowerShell(*); Wildcarded interpreters like +# Bash(python*); Package-manager run commands; Agent allow rules. Narrow rules +# like Bash(npm test) carry over." This script classifies every effective allow +# rule from the merge into exactly one of those documented classes or `kept`, +# using the same shared vocabulary (lib/permission-patterns.sh) that +# audit-permission-grants check P1 scans with — one definition, two consumers. +# +# `autoMode.classifyAllShell` inverts the carry-over answer wholesale: when +# true it "suspend[s] every Bash and PowerShell allow rule while auto mode is +# active", so narrow rules do NOT carry over. The classifier reads `autoMode` +# from user settings, managed settings, and inline --settings/SDK JSON only — +# never project or local settings — so this script resolves the key from the +# managed and user conf records alone and says so when a non-read scope sets it. +# +# Input: permission-merge.sh output on stdin (records pass through above the +# diff section unless --diff-only). With no piped input the sibling pipeline is +# run directly and its exit status is propagated. +# +# Output (diff section): +# DIFF-NOTE: classifyAllShell state, bounds +# entry-diff dropped class= scopes= one per dropped allow rule +# entry-diff suspended reason=classifyAllShell scopes= +# entry-diff kept scopes= one per carried-over allow rule +# entry-diff summary allow_before= dropped= suspended= kept= +# +# class blanket | interpreter-wildcard | package-manager-run | agent +# (the documented four; the vocabulary's script-glob alternative is an +# interpreter-wildcard shape and reports as that class) +# +# Only allow rules change on entry: deny and ask rules are evaluated before the +# classifier in every mode and are not part of this diff. +# +# --oracle (opt-in, explicitly priced): cross-check the prediction against the +# harness's own drop narration by spawning `claude --debug-file -p` and +# parsing `Ignoring dangerous permission from (bypasses +# classifier)` lines. The prediction stays the default read path: the oracle +# costs a real session spawn (tokens, and the session writes its own state — see +# the cost notice) and parses undocumented [DEBUG] strings with no stability +# contract. Never spawned without the flag; never spawned silently. Criterion +# 8's defensive contract binds this read too: exit status is never trusted, a +# missing or empty capture is reported as "oracle unavailable" and the +# prediction stands — an empty capture is NEVER read as an empty drop set. +# +# Test seams: +# ENTRY_DIFF_ORACLE_CAPTURE parse this capture file instead of spawning +# ENTRY_DIFF_ORACLE_PROMPT probe prompt (default "Reply with exactly: OK") +# +# Prerequisites: POSIX text tools only. The oracle additionally needs `claude` +# on PATH; absent, it degrades to "oracle unavailable" and the prediction stands. +# +# Usage: +# permission-state.sh | permission-merge.sh | automode-entry-diff.sh +# automode-entry-diff.sh [--diff-only] [--oracle] [--help] + +set -uo pipefail + +usage() { + cat <<'EOF' +automode-entry-diff.sh — classify what entering auto mode drops from the effective allow set. + +Usage: permission-state.sh | permission-merge.sh | automode-entry-diff.sh [--diff-only] [--oracle] + automode-entry-diff.sh [--diff-only] [--oracle] [--help] + + (no arg) input records pass through, then the diff section + --diff-only the diff section alone + --oracle ALSO spawn a claude session (real token cost, real side effects — + a cost notice prints before anything is spawned) and cross-check + the prediction against the harness's own drop narration + --help this message + +Records: "entry-diff dropped class= scopes= ", +"entry-diff suspended reason=classifyAllShell scopes= ", +"entry-diff kept scopes= ", a closing "entry-diff summary" count line, +and with --oracle one "oracle AGREES " or "oracle DIVERGES ..." per compared rule. + +Writes no settings file. Without --oracle it writes nothing at all; with it, the +capture goes to a scratch path AND the spawned session leaves its own state +under your config directory — the cost notice enumerates that. Exits 2 when the +input carries no records at all. +EOF +} + +diff_only=0 +oracle=0 +for arg in "$@"; do + case "$arg" in + -h | --help) + usage + exit 0 + ;; + --diff-only) diff_only=1 ;; + --oracle) oracle=1 ;; + *) + echo "ERROR: unknown argument '$arg'" >&2 + exit 2 + ;; + esac +done + +PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-$(cd "${BASH_SOURCE[0]%/*}/../../.." && pwd)}" +PATTERNS_LIB="$PLUGIN_ROOT/lib/permission-patterns.sh" +if [[ ! -r "$PATTERNS_LIB" ]]; then + echo "ERROR: cannot read $PATTERNS_LIB — the shared drop vocabulary is missing, so the drop set cannot be classified" >&2 + exit 2 +fi +# shellcheck source=../../../lib/permission-patterns.sh +source "$PATTERNS_LIB" + +if [[ -t 0 ]]; then + MERGE_SCRIPT="${BASH_SOURCE[0]%/*}/permission-merge.sh" + if [[ ! -r "$MERGE_SCRIPT" ]]; then + echo "ERROR: cannot read $MERGE_SCRIPT — nothing to diff" >&2 + exit 2 + fi + records="$(bash "$MERGE_SCRIPT")" || exit $? +else + records="$(cat)" +fi + +if [[ -z "$records" ]]; then + echo "ERROR: no records on input — automode-entry-diff.sh will not report a drop set it never read" >&2 + exit 2 +fi + +[[ "$diff_only" == 1 ]] || printf '%s\n' "$records" + +# --- classifyAllShell resolution -------------------------------------------- +# +# Managed is the highest settings scope, so a managed value wins over user. +# conf records from scopes the classifier does not read change nothing here and +# are called out instead of silently honored. +cas_value="" +cas_scope="" +cas_ignored="" +while read -r rec scope _surface key value _; do + [[ "$rec" == "conf" && "$key" == "classifyAllShell" ]] || continue + case "$scope" in + managed) + cas_value="$value" + cas_scope="managed" + ;; + user) + if [[ "$cas_scope" != "managed" ]]; then + cas_value="$value" + cas_scope="user" + fi + ;; + *) cas_ignored="${cas_ignored:+$cas_ignored, }$scope" ;; + esac +done <<<"$records" + +cas_active=0 +if [[ "$cas_value" == "true" ]]; then + cas_active=1 + echo "DIFF-NOTE: autoMode.classifyAllShell is true in $cas_scope settings — every Bash and PowerShell allow rule is suspended while auto mode is active, so narrow shell rules do NOT carry over. Requires Claude Code v2.1.193 or later; earlier versions ignore the key and carry narrow rules." +elif [[ "$cas_value" == "false" ]]; then + # `false` is the documented default and a perfectly valid setting. Describing + # it as a type error told an operator their correct configuration was wrong. + echo "DIFF-NOTE: autoMode.classifyAllShell is false in $cas_scope settings (the default), so narrow Bash and PowerShell allow rules carry over into auto mode as usual." +elif [[ -n "$cas_value" ]]; then + echo "DIFF-NOTE: autoMode.classifyAllShell in $cas_scope settings is $cas_value, which is neither of the documented boolean values — treated as inactive here; the harness's handling of a non-boolean value is undocumented." +fi +# The classifier reads autoMode from THREE sources and this reader can see two. +# Inline --settings / SDK JSON has no file to open, and it can invert the whole +# answer: classifyAllShell true there suspends every shell allow rule while this +# diff reports them kept. The merge's command-line caveat covers RULES, not this +# key, so it is stated here or nowhere. +echo "DIFF-NOTE: inline --settings and Agent SDK JSON are a third scope the classifier reads autoMode from, and they have no file for this reader to open. If classifyAllShell is set there, every Bash and PowerShell verdict below is inverted — this diff reflects the settings FILES only." +if [[ -n "$cas_ignored" ]]; then + echo "DIFF-NOTE: autoMode.classifyAllShell also appears in scope(s) the classifier does not read ($cas_ignored) — the classifier reads autoMode from user settings, managed settings, and inline --settings/SDK JSON only. Those entries have no effect and are not part of this diff." +fi + +# --- Per-rule classification -------------------------------------------------- + +n_before=0 n_dropped=0 n_suspended=0 n_kept=0 +predicted_dropped="" +diff_lines="" +while read -r rec kind scopes_field _basis rule; do + [[ "$rec" == "effective" && "$kind" == "allow" && -n "$rule" ]] || continue + n_before=$((n_before + 1)) + tool="${rule%%(*}" + verdict="" + if [[ "$rule" == "Agent" || "$rule" == "Agent("* ]]; then + verdict="dropped class=agent" + elif [[ "$rule" == "Bash" || "$rule" == "PowerShell" ]]; then + # A bare tool name is the WHOLE-TOOL grant -- strictly broader than + # Bash(*), which this same run classifies as blanket. Reporting it as + # surviving auto mode while dropping the narrower form would be backwards + # in the direction that matters: it tells an operator their broadest shell + # grant is safe. The Agent branch above already handles its own bare form; + # this is the same rule for the two shell tools. + verdict="dropped class=blanket" + elif printf '%s\n' "$rule" | grep -qE "$CCPERM_P1_BLANKET_ERE"; then + verdict="dropped class=blanket" + elif printf '%s\n' "$rule" | grep -qE "$CCPERM_P1_INTERP_ERE|$CCPERM_P1_SCRIPTGLOB_ERE"; then + verdict="dropped class=interpreter-wildcard" + elif printf '%s\n' "$rule" | grep -qE "$CCPERM_P1_RUNNER_ERE"; then + verdict="dropped class=package-manager-run" + elif [[ "$cas_active" == 1 && ("$tool" == "Bash" || "$tool" == "PowerShell") ]]; then + verdict="suspended reason=classifyAllShell" + fi + if [[ "$verdict" == dropped* ]]; then + n_dropped=$((n_dropped + 1)) + predicted_dropped="${predicted_dropped}${rule}"$'\n' + elif [[ "$verdict" == suspended* ]]; then + n_suspended=$((n_suspended + 1)) + predicted_dropped="${predicted_dropped}${rule}"$'\n' + else + verdict="kept" + n_kept=$((n_kept + 1)) + fi + diff_lines="${diff_lines}entry-diff $verdict $scopes_field $rule"$'\n' +done <<<"$records" + +printf '%s' "$diff_lines" +echo "entry-diff summary allow_before=$n_before dropped=$n_dropped suspended=$n_suspended kept=$n_kept" + +[[ "$oracle" == 1 ]] || exit 0 + +# --- Debug-channel oracle (opt-in, priced) ------------------------------------ + +capture="${ENTRY_DIFF_ORACLE_CAPTURE:-}" +if [[ -z "$capture" ]]; then + cat >&2 <<'EOF' +ORACLE COST NOTICE — nothing has been spawned yet. + --oracle starts a real `claude -p` session on this machine to capture the + harness's own drop narration. That costs API tokens, and the session writes + outside the scratch capture path. Measured on 2.1.225 / Windows 11 by + checksumming before and after a probe run: + - Your settings files are NOT modified: ~/.claude/settings.json and + ~/.claude/settings.local.json were byte-identical afterwards. + - ~/.claude.json IS rewritten. It is the harness's own state file, not a + settings file, and it carries no permission rules -- but it does change. + - New files appear under your config directory: a project entry for the + session's working directory, a session-env entry, per-session security + and subagent state, and a backup entry. + The debug capture itself goes to a scratch path, never to ~/.claude/debug/. + Run it from a directory you do not mind appearing in your project list. +EOF + if ! command -v claude >/dev/null 2>&1; then + echo "oracle UNAVAILABLE: 'claude' is not on PATH — the prediction above stands, uncorroborated." + exit 0 + fi + scratch="$(mktemp -d "${TMPDIR:-/tmp}/entry-diff-oracle.XXXXXX")" || { + echo "oracle UNAVAILABLE: could not create a scratch directory — the prediction above stands, uncorroborated." + exit 0 + } + capture="$scratch/capture.log" + # `--permission-mode auto` is passed so the probe does not depend on the + # consumer's own defaultMode. What was actually MEASURED is narrower than that + # framing suggests, and reference/criteria.md carries the measurement: a `-p` + # run with NO mode flag emitted 216 drop lines, on a machine whose defaultMode + # may already have been auto. So the flag is known-valid (`claude --help` + # lists `auto`) and known-harmless, but "drops appear only in auto mode" is + # NOT established — it is a plausible reading of one capture. The zero-drop + # path below degrades to `unavailable` precisely because this is unproven. + # + # Exit status is deliberately not consulted — the capture file is the evidence + # either way. + claude --debug-file "$capture" --permission-mode auto -p "${ENTRY_DIFF_ORACLE_PROMPT:-Reply with exactly: OK}" >/dev/null 2>&1 || true +fi + +if [[ ! -s "$capture" ]]; then + echo "oracle UNAVAILABLE: the capture at ${capture:-} is missing or empty — a session that produced no usable output is reported as exactly that, never as an empty drop set. The prediction above stands, uncorroborated." + exit 0 +fi +if ! grep -q 'Applying permission update' "$capture"; then + echo "oracle UNAVAILABLE: the capture carries no permission-merge narration (the undocumented [DEBUG] strings may have changed, or the session did not reach the merge) — the prediction above stands, uncorroborated." + exit 0 +fi + +# `Ignoring dangerous permission from (bypasses classifier)`. +# This strips the fixed prefix and suffix; splitting rule from path is the +# block below, which explains how. +oracle_raw="$(sed -n 's/^.*Ignoring dangerous permission \(.*\) (bypasses classifier).*$/\1/p' "$capture")" + +# Neither field is delimited, so a line carrying more than one " from " must be +# RESOLVED rather than guessed at. The discriminator is the RULE side, not the +# path: a permission rule is a tool token, optionally with one parenthesized +# payload, and that grammar already exists as CCPERM_TOOL_TOKEN_ERE in the shared +# vocabulary. Each candidate separator is tried and kept only when the text to +# its left is a well-formed rule. +# +# That distinguishes the two cases a single expression cannot. In +# `Bash(python3 import from x *) from ` the first candidate leaves +# `Bash(python3 import` -- unbalanced, rejected. In +# `Bash(uv run *) from C:\...\notes from work\...\settings.json` the second +# leaves `Bash(uv run *) from C:\...\notes` -- not a tool token, rejected. Each +# resolves to exactly one survivor, and the path may contain anything at all. +# The grammar is checked structurally rather than by regex: awk strips the +# backslashes out of an ERE passed through -v, which turns the vocabulary's +# escaped parens into grouping and matches nothing. Balance and shape are what +# matter here, and both are cheaper to check directly than to re-escape. +split_lines="$(printf '%s\n' "$oracle_raw" | awk ' + function trim(s) { sub(/^[ \t]+/, "", s); sub(/[ \t]+$/, "", s); return s } + # A rule is a tool token, optionally followed by ONE balanced parenthesized + # payload that runs to the end of the text. + function is_rule(s, open, i, c, depth) { + if (s !~ /^[A-Za-z_][A-Za-z0-9_]*(\(|$)/) return 0 + open = index(s, "(") + if (open == 0) return s ~ /^[A-Za-z_][A-Za-z0-9_]*$/ + depth = 0 + for (i = open; i <= length(s); i++) { + c = substr(s, i, 1) + if (c == "(") depth++ + else if (c == ")") { depth--; if (depth == 0) return i == length(s) } + } + return 0 + } + { + line = $0 + if (line !~ /[^ \t]/) next + n_ok = 0; rule = "" + start = 1 + while ((p = index(substr(line, start), " from ")) > 0) { + at = start + p - 1 + cand = trim(substr(line, 1, at - 1)) + tail = substr(line, at + 6) + if (is_rule(cand) && tail ~ /[^ \t]/) { n_ok++; rule = cand } + start = at + 1 + } + if (n_ok == 1) print "OK\t" rule + else print "AMBIGUOUS\t" line + }')" + +oracle_dropped="$(printf '%s\n' "$split_lines" | sed -n 's/^OK\t//p' | LC_ALL=C sort -u)" + +# Everything else carries a real harness drop this run cannot name. Saying so +# beats guessing, and beats dropping them in silence. +oracle_unparsed="$(printf '%s\n' "$split_lines" | sed -n 's/^AMBIGUOUS\t//p')" +if [[ -n "$oracle_unparsed" ]]; then + n_unparsed="$(printf '%s\n' "$oracle_unparsed" | grep -c .)" + echo "oracle NOTE: $n_unparsed drop line(s) could not be split into rule and source path — the narration delimits neither field, and no candidate split left a well-formed rule on the left (or more than one did). Those drops are real and are NOT reflected in the verdicts below; treat this comparison as incomplete rather than clean." +fi + +if [[ -z "$oracle_dropped" ]]; then + echo "oracle NOTE: the session narrated the permission merge but emitted zero drop lines. If this machine has rules the prediction above says are dropped, the session was likely not in auto mode; treat the oracle as unavailable rather than as an empty drop set." + [[ -z "$(printf '%s' "$predicted_dropped")" ]] || exit 0 +fi + +# Compare on rule text over the union of both sets: one verdict line per rule, +# exactly — a disagreement in EITHER direction is a finding. +predicted_sorted="$(printf '%s' "$predicted_dropped" | LC_ALL=C sort -u)" +while IFS= read -r rule; do + [[ -n "$rule" ]] || continue + in_pred=0 + in_orc=0 + grep -qxF "$rule" <<<"$predicted_sorted" && in_pred=1 + grep -qxF "$rule" <<<"$oracle_dropped" && in_orc=1 + if [[ "$in_pred" == 1 && "$in_orc" == 1 ]]; then + echo "oracle AGREES $rule" + elif [[ "$in_pred" == 1 ]]; then + echo "oracle DIVERGES prediction=dropped oracle=kept $rule" + else + echo "oracle DIVERGES prediction=kept oracle=dropped $rule" + fi +done < <(printf '%s\n%s\n' "$predicted_sorted" "$oracle_dropped" | grep -v '^$' | LC_ALL=C sort -u) + +exit 0 diff --git a/plugins/claude-config/skills/audit-permission-state/scripts/automode-entry-diff.test.sh b/plugins/claude-config/skills/audit-permission-state/scripts/automode-entry-diff.test.sh new file mode 100755 index 0000000000..00da13ffbb --- /dev/null +++ b/plugins/claude-config/skills/audit-permission-state/scripts/automode-entry-diff.test.sh @@ -0,0 +1,402 @@ +#!/usr/bin/env bash +# Regression tests for automode-entry-diff.sh (self-contained — ships with the plugin). +# +# Classification cases feed hand-written merge records on stdin. Oracle cases +# use a fixture capture (ENTRY_DIFF_ORACLE_CAPTURE) or a recording stub claude +# on PATH — no case ever spawns a real session, and no test reads the +# operator's real ~/.claude. +# +# portability-scope: the oracle fixture reproduces Claude Code's own [DEBUG] +# narration byte for byte, and that narration prints Windows settings paths in +# native form (C:\Users\...\.claude\settings.json — see the recorded capture in +# the permission-model research corpus). The backslash runs the gate reads as +# regex escapes are those paths inside heredoc DATA, not shell code. Rewriting +# them to POSIX form would make the fixture stop matching what the parser must +# survive on the plugin's primary platform, which is the whole point of the case. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="$SCRIPT_DIR/automode-entry-diff.sh" +STATE_SCRIPT="$SCRIPT_DIR/permission-state.sh" +MERGE_SCRIPT="$SCRIPT_DIR/permission-merge.sh" + +TEST_TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TEST_TMPDIR"' EXIT + +FAILED=0 +CASE_NUM=0 +pass() { + CASE_NUM=$((CASE_NUM + 1)) + printf 'PASS: %s\n' "$1" +} +fail() { + CASE_NUM=$((CASE_NUM + 1)) + FAILED=$((FAILED + 1)) + printf 'FAIL: %s\n detail: %s\n' "$1" "$2" >&2 +} +assert_eq() { + if [[ "$2" == "$3" ]]; then pass "$1"; else fail "$1" "expected: $2, actual: $3"; fi +} +assert_exit() { + if [[ "$2" == "$3" ]]; then pass "$1"; else fail "$1" "expected exit $2, got $3"; fi +} +assert_contains() { + case "$2" in + *"$3"*) pass "$1" ;; + *) fail "$1" "expected to contain: $3" ;; + esac +} +assert_not_contains() { + case "$2" in + *"$3"*) fail "$1" "unexpected substring: $3" ;; + *) pass "$1" ;; + esac +} +count_matching() { printf '%s\n' "$1" | grep -cE "$2"; } + +diff_only() { printf '%s\n' "$1" | bash "$SCRIPT" --diff-only; } + +# --- Case 1: one rule of each documented drop class, plus a narrow survivor --- +# "On entering auto mode, broad allow rules that grant arbitrary code execution +# are dropped: Blanket Bash(*) or PowerShell(*); Wildcarded interpreters like +# Bash(python*); Package-manager run commands; Agent allow rules. Narrow rules +# like Bash(npm test) carry over." +FOUR_CLASSES=$( + cat <<'EOF' +user settings present /fx/home/.claude/settings.json +effective allow scopes=user precedence_basis=uncontested Bash(*) +effective allow scopes=user precedence_basis=uncontested Bash(python3 *) +effective allow scopes=user precedence_basis=uncontested Bash(npx *) +effective allow scopes=user precedence_basis=uncontested Agent +effective allow scopes=user precedence_basis=uncontested Bash(git status) +EOF +) +OUT=$(diff_only "$FOUR_CLASSES") +assert_contains "blanket Bash(*) is dropped" "$OUT" "entry-diff dropped class=blanket scopes=user Bash(*)" +assert_contains "a wildcarded interpreter is dropped" "$OUT" "entry-diff dropped class=interpreter-wildcard scopes=user Bash(python3 *)" +assert_contains "a package-manager run command is dropped" "$OUT" "entry-diff dropped class=package-manager-run scopes=user Bash(npx *)" +assert_contains "a bare Agent allow rule is dropped" "$OUT" "entry-diff dropped class=agent scopes=user Agent" +assert_contains "a narrow exact rule carries over" "$OUT" "entry-diff kept scopes=user Bash(git status)" +assert_eq "all four classes appear in the dropped set" 4 "$(count_matching "$OUT" '^entry-diff dropped ')" +assert_contains "the summary reconciles" "$OUT" "entry-diff summary allow_before=5 dropped=4 suspended=0 kept=1" + +# A SCOPED Agent rule is dropped too: auto mode drops all Agent allow rules +# categorically, unlike Bash where narrow rules survive. +SCOPED_AGENT=$( + cat <<'EOF' +user settings present /fx/home/.claude/settings.json +effective allow scopes=user precedence_basis=uncontested Agent(model:haiku) +EOF +) +OUT=$(diff_only "$SCOPED_AGENT") +assert_contains "a scoped Agent allow rule is dropped like a bare one" "$OUT" "entry-diff dropped class=agent scopes=user Agent(model:haiku)" + +# Deny and ask rules are evaluated before the classifier in every mode; the +# entry diff must not touch them. +DENY_ASK=$( + cat <<'EOF' +user settings present /fx/home/.claude/settings.json +effective deny scopes=user precedence_basis=uncontested Bash(*) +effective ask scopes=user precedence_basis=uncontested Agent +EOF +) +OUT=$(diff_only "$DENY_ASK") +assert_eq "deny and ask rules produce no diff records" 0 "$(count_matching "$OUT" '^entry-diff (dropped|suspended|kept) ')" +assert_contains "the summary shows an empty allow set, not a missing one" "$OUT" "entry-diff summary allow_before=0" + +# --- Case 2: classifyAllShell inverts the carry-over answer ------------------- +# "suspend[s] every Bash and PowerShell allow rule while auto mode is active." +CAS_ON=$( + cat <<'EOF' +user settings present /fx/home/.claude/settings.json +conf user settings classifyAllShell true +effective allow scopes=user precedence_basis=uncontested Bash(git status) +effective allow scopes=user precedence_basis=uncontested PowerShell(Get-ChildItem *) +effective allow scopes=user precedence_basis=uncontested WebFetch(domain:example.com) +effective allow scopes=user precedence_basis=uncontested Bash(*) +EOF +) +OUT=$(diff_only "$CAS_ON") +assert_contains "a narrow Bash rule is suspended, not kept" "$OUT" "entry-diff suspended reason=classifyAllShell scopes=user Bash(git status)" +assert_contains "PowerShell rules are suspended too" "$OUT" "entry-diff suspended reason=classifyAllShell scopes=user PowerShell(Get-ChildItem *)" +assert_contains "a non-shell rule is untouched by classifyAllShell" "$OUT" "entry-diff kept scopes=user WebFetch(domain:example.com)" +assert_contains "a blanket rule still reports its drop class, not suspension" "$OUT" "entry-diff dropped class=blanket scopes=user Bash(*)" +assert_contains "the inversion is announced with its version gate" "$OUT" "v2.1.193" + +# Managed is the highest scope: a managed false beats a user true. +CAS_MANAGED=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +user settings present /fx/home/.claude/settings.json +conf managed file classifyAllShell false +conf user settings classifyAllShell true +effective allow scopes=user precedence_basis=uncontested Bash(git status) +EOF +) +OUT=$(diff_only "$CAS_MANAGED") +assert_eq "a managed false overrides a user true" 0 "$(count_matching "$OUT" '^entry-diff suspended ')" +assert_contains "the narrow rule stays kept" "$OUT" "entry-diff kept scopes=user Bash(git status)" + +# "The classifier doesn't read autoMode from project settings in +# .claude/settings.json or .claude/settings.local.json." +CAS_PROJECT=$( + cat <<'EOF' +project settings present /proj/.claude/settings.json +conf project settings classifyAllShell true +effective allow scopes=project precedence_basis=uncontested Bash(git status) +EOF +) +OUT=$(diff_only "$CAS_PROJECT") +assert_eq "a project-scope classifyAllShell suspends nothing" 0 "$(count_matching "$OUT" '^entry-diff suspended ')" +assert_contains "the ignored scope is named, not silently honored" "$OUT" "scope(s) the classifier does not read (project)" + +# A string "true" is not the documented boolean and must not activate. +CAS_STRING=$( + cat <<'EOF' +user settings present /fx/home/.claude/settings.json +conf user settings classifyAllShell "true" +effective allow scopes=user precedence_basis=uncontested Bash(git status) +EOF +) +OUT=$(diff_only "$CAS_STRING") +assert_eq "a string-typed value suspends nothing" 0 "$(count_matching "$OUT" '^entry-diff suspended ')" +assert_contains "the type mismatch is called out" "$OUT" "neither of the documented boolean values" + +# `false` is the DOCUMENTED DEFAULT, not a type error. An earlier revision +# lumped it in with malformed values and told operators their correct +# configuration was wrong. +CAS_FALSE=$( + cat <<'EOF' +user settings present /fx/home/.claude/settings.json +conf user settings classifyAllShell false +effective allow scopes=user precedence_basis=uncontested Bash(git status) +EOF +) +OUT=$(diff_only "$CAS_FALSE") +assert_eq "false suspends nothing" 0 "$(count_matching "$OUT" '^entry-diff suspended ')" +assert_contains "and is described as the default, not as malformed" "$OUT" "(the default)" +assert_not_contains "false is never called a type error" "$OUT" "neither of the documented boolean values" + +# The classifier reads autoMode from three scopes and this reader can see two. +# Inline --settings can invert every shell verdict below, so the bound is stated +# on every run rather than only when something else happens to be set. +OUT=$(diff_only "$CAS_FALSE") +assert_contains "the unreadable inline scope is disclosed" "$OUT" "inline --settings and Agent SDK JSON" +assert_contains "and the disclosure says what it would change" "$OUT" "every Bash and PowerShell verdict below is inverted" + +# --- Case 3: pass-through default, --diff-only suppression -------------------- +PASS_IN=$( + cat <<'EOF' +user settings present /fx/home/.claude/settings.json +NOTE: something the operator must know +effective allow scopes=user precedence_basis=uncontested Bash(git status) +EOF +) +OUT=$(printf '%s\n' "$PASS_IN" | bash "$SCRIPT") +assert_contains "input records pass through by default" "$OUT" "NOTE: something the operator must know" +assert_contains "the diff section follows" "$OUT" "entry-diff kept scopes=user Bash(git status)" +OUT=$(diff_only "$PASS_IN") +assert_not_contains "--diff-only drops the input records" "$OUT" "NOTE: something the operator" + +# --- Case 4: no records is an error, never an empty diff ---------------------- +rc=0 +err_out=$(printf '' | bash "$SCRIPT" --diff-only 2>&1) || rc=$? +assert_exit "exit 2 on empty input" 2 "$rc" +assert_contains "the empty-input error says why" "$err_out" "no records on input" + +rc=0 +err_out=$(bash "$SCRIPT" --bogus &1) || rc=$? +assert_exit "exit 2 on an unknown argument" 2 "$rc" + +rc=0 +help_out=$(bash "$SCRIPT" --help &1) || rc=$? +assert_exit "--help exits 0" 0 "$rc" +assert_contains "--help prices the oracle" "$help_out" "real token cost" + +# --- Case 5: oracle OFF by default — no spawn, no scratch file ---------------- +# A recording stub claude sits FIRST on PATH; if anything invoked it, the +# marker file would exist. +STUB="$TEST_TMPDIR/stub" +mkdir -p "$STUB" +cat >"$STUB/claude" <>"$TEST_TMPDIR/claude-invocations.txt" +exit 0 +EOF +chmod +x "$STUB/claude" + +OUT=$(printf '%s\n' "$FOUR_CLASSES" | PATH="$STUB:$PATH" bash "$SCRIPT" --diff-only 2>&1) +if [[ -e "$TEST_TMPDIR/claude-invocations.txt" ]]; then + fail "no flag, no spawn" "stub claude was invoked: $(cat "$TEST_TMPDIR/claude-invocations.txt")" +else + pass "no flag, no spawn" +fi +assert_not_contains "no flag, no oracle output" "$OUT" "oracle" + +# --- Case 6: oracle ON spawns only after the cost notice ---------------------- +# The stub writes an empty capture, so this also proves a dead spawn is +# reported as unavailable rather than as an empty drop set. +OUT=$(printf '%s\n' "$FOUR_CLASSES" | PATH="$STUB:$PATH" bash "$SCRIPT" --diff-only --oracle 2>&1) +assert_contains "the cost notice printed" "$OUT" "ORACLE COST NOTICE" +if [[ -s "$TEST_TMPDIR/claude-invocations.txt" ]]; then + pass "with the flag, the spawn happened" +else + fail "with the flag, the spawn happened" "stub claude was never invoked" +fi +assert_contains "the stub session is in auto mode explicitly" "$(cat "$TEST_TMPDIR/claude-invocations.txt")" "--permission-mode auto" +assert_contains "an empty capture is unavailable, not an empty drop set" "$OUT" "oracle UNAVAILABLE" +assert_not_contains "no verdicts from a dead capture" "$OUT" "oracle AGREES" +rm -f "$TEST_TMPDIR/claude-invocations.txt" + +# claude missing entirely: the notice still precedes the (refused) spawn. +# Wrappers exec the real binaries by absolute path (a copied MSYS binary loses +# the msys-2.0.dll beside it); bash is invoked by absolute path so the stub +# PATH cannot hide the interpreter itself. +NOCLAUDE="$TEST_TMPDIR/noclaude" +mkdir -p "$NOCLAUDE" +real_bash="$(command -v bash)" +for tool in cat grep sed sort mktemp rm tr; do + src="$(command -v "$tool" 2>/dev/null)" || continue + printf '#!%s\nexec "%s" "$@"\n' "$real_bash" "$src" >"$NOCLAUDE/$tool" + chmod +x "$NOCLAUDE/$tool" +done +OUT=$(printf '%s\n' "$FOUR_CLASSES" | PATH="$NOCLAUDE" "$real_bash" "$SCRIPT" --diff-only --oracle 2>&1) +assert_contains "notice before any spawn attempt" "$OUT" "ORACLE COST NOTICE" +assert_contains "a missing claude degrades to unavailable" "$OUT" "'claude' is not on PATH" + +# --- A bare tool name is the BROADEST shell grant, not a surviving one -------- +# `Bash` with no parens is strictly broader than `Bash(*)`, which this same run +# classifies as blanket. Reporting it kept told an operator their widest grant +# survives auto mode — wrong in the direction that costs them. +BARE_SHELL=$( + cat <<'EOF' +user settings present /fx/home/.claude/settings.json +effective allow scopes=user precedence_basis=uncontested Bash +effective allow scopes=user precedence_basis=uncontested PowerShell +effective allow scopes=user precedence_basis=uncontested Bash(*) +EOF +) +OUT=$(diff_only "$BARE_SHELL") +assert_eq "a bare shell grant is never reported kept" 0 "$(count_matching "$OUT" '^entry-diff kept ')" +assert_contains "bare Bash drops as blanket" "$OUT" "entry-diff dropped class=blanket scopes=user Bash" +assert_contains "bare PowerShell drops as blanket" "$OUT" "entry-diff dropped class=blanket scopes=user PowerShell" +assert_eq "all three shell grants drop" 3 "$(count_matching "$OUT" '^entry-diff dropped ')" + +# --- Case 7: oracle comparison — one verdict per compared rule, exactly ------- +CAPTURE="$TEST_TMPDIR/capture.log" +cat >"$CAPTURE" <<'EOF' +[DEBUG] Applying permission update: Adding 3 allow rule(s) to destination 'userSettings': [...] +[DEBUG] Ignoring dangerous permission Bash(*) from C:\Users\x\.claude\settings.json (bypasses classifier) +[DEBUG] Ignoring dangerous permission Bash(python3 *) from C:\Users\x\.claude\settings.json (bypasses classifier) +[DEBUG] Ignoring dangerous permission Bash(uv run *) from C:\Users\x\.claude\settings.json (bypasses classifier) +[DEBUG] Applying permission update: Removing 3 allow rule(s) from source 'userSettings' +EOF +# Prediction: Bash(*), Bash(python3 *), Bash(npx *), Agent dropped. Oracle: +# Bash(*), Bash(python3 *), Bash(uv run *). Union: 5 rules, 2 agreements, +# 2 predicted-only, 1 oracle-only. +OUT=$(printf '%s\n' "$FOUR_CLASSES" | ENTRY_DIFF_ORACLE_CAPTURE="$CAPTURE" bash "$SCRIPT" --diff-only --oracle 2>&1) +assert_eq "one verdict line per rule in the union, exactly" 5 "$(count_matching "$OUT" '^oracle (AGREES|DIVERGES) ')" +assert_contains "agreement is stated per rule" "$OUT" "oracle AGREES Bash(*)" +assert_contains "a predicted drop the oracle kept diverges" "$OUT" "oracle DIVERGES prediction=dropped oracle=kept Bash(npx *)" +assert_contains "an oracle drop the prediction kept diverges too" "$OUT" "oracle DIVERGES prediction=kept oracle=dropped Bash(uv run *)" + +# A rule containing the word "from" must not truncate itself. +CAPTURE_FROM="$TEST_TMPDIR/capture-from.log" +cat >"$CAPTURE_FROM" <<'EOF' +[DEBUG] Applying permission update: Adding 1 allow rule(s) to destination 'userSettings': [...] +[DEBUG] Ignoring dangerous permission Bash(python3 import from x *) from C:\Users\x\.claude\settings.json (bypasses classifier) +EOF +FROM_IN=$( + cat <<'EOF' +user settings present /fx/home/.claude/settings.json +effective allow scopes=user precedence_basis=uncontested Bash(python3 import from x *) +EOF +) +OUT=$(printf '%s\n' "$FROM_IN" | ENTRY_DIFF_ORACLE_CAPTURE="$CAPTURE_FROM" bash "$SCRIPT" --diff-only --oracle 2>&1) +assert_contains "a rule containing 'from' survives parsing intact" "$OUT" "oracle AGREES Bash(python3 import from x *)" + +# The mirror image, and the one a fixed split cannot also satisfy: the word is +# in the PATH, because a directory may be named anything. Cutting at the last +# separator here produced two false verdicts and a phantom rule. The split is +# resolved by which side parses as a rule, so both directions work. +CAPTURE_FROM_PATH="$TEST_TMPDIR/capture-from-path.log" +cat >"$CAPTURE_FROM_PATH" <<'EOF' +[DEBUG] Applying permission update: Adding 1 allow rule(s) to destination 'userSettings' +[DEBUG] Ignoring dangerous permission Bash(uv run *) from C:\Users\x\notes from work\.claude\settings.json (bypasses classifier) +EOF +FROM_PATH_IN=$( + cat <<'EOF' +user settings present /fx/home/.claude/settings.json +effective allow scopes=user precedence_basis=uncontested Bash(uv run *) +EOF +) +OUT=$(printf '%s\n' "$FROM_PATH_IN" | ENTRY_DIFF_ORACLE_CAPTURE="$CAPTURE_FROM_PATH" bash "$SCRIPT" --diff-only --oracle 2>&1) +assert_contains "a PATH containing 'from' does not truncate the rule" "$OUT" "oracle AGREES Bash(uv run *)" +assert_eq "and produces no phantom rule" 0 "$(count_matching "$OUT" 'oracle DIVERGES')" + +# A line no candidate split resolves is ANNOUNCED, not dropped. A real harness +# drop that silently vanishes turns an incomplete comparison into a clean one. +CAPTURE_UNSPLIT="$TEST_TMPDIR/capture-unsplit.log" +cat >"$CAPTURE_UNSPLIT" <<'EOF' +[DEBUG] Applying permission update: Adding 1 allow rule(s) to destination 'userSettings' +[DEBUG] Ignoring dangerous permission not a rule at all from somewhere (bypasses classifier) +[DEBUG] Ignoring dangerous permission Bash(npx *) from C:\Users\x\.claude\settings.json (bypasses classifier) +EOF +UNSPLIT_IN=$( + cat <<'EOF' +user settings present /fx/home/.claude/settings.json +effective allow scopes=user precedence_basis=uncontested Bash(npx *) +EOF +) +OUT=$(printf '%s\n' "$UNSPLIT_IN" | ENTRY_DIFF_ORACLE_CAPTURE="$CAPTURE_UNSPLIT" bash "$SCRIPT" --diff-only --oracle 2>&1) +assert_contains "an unresolvable drop line is announced" "$OUT" "could not be split into rule and source path" +assert_contains "and the comparison is called incomplete, not clean" "$OUT" "incomplete rather than clean" +assert_contains "the resolvable line is still compared" "$OUT" "oracle AGREES Bash(npx *)" + +# --- Case 8: capture without drop strings never becomes an empty drop set ----- +CAPTURE_NODROPS="$TEST_TMPDIR/capture-nodrops.log" +cat >"$CAPTURE_NODROPS" <<'EOF' +[DEBUG] Applying permission update: Adding 3 allow rule(s) to destination 'userSettings': [...] +EOF +OUT=$(printf '%s\n' "$FOUR_CLASSES" | ENTRY_DIFF_ORACLE_CAPTURE="$CAPTURE_NODROPS" bash "$SCRIPT" --diff-only --oracle 2>&1) +assert_contains "zero drop lines with predicted drops reads as unavailable" "$OUT" "treat the oracle as unavailable rather than as an empty drop set" +assert_eq "no verdict is emitted from it" 0 "$(count_matching "$OUT" '^oracle (AGREES|DIVERGES) ')" + +# A capture with no permission narration at all (the [DEBUG] strings changed, +# or the session died before the merge) is unavailable, with the reason named. +CAPTURE_ALIEN="$TEST_TMPDIR/capture-alien.log" +cat >"$CAPTURE_ALIEN" <<'EOF' +[DEBUG] something else entirely +EOF +OUT=$(printf '%s\n' "$FOUR_CLASSES" | ENTRY_DIFF_ORACLE_CAPTURE="$CAPTURE_ALIEN" bash "$SCRIPT" --diff-only --oracle 2>&1) +assert_contains "a capture without merge narration is unavailable" "$OUT" "no permission-merge narration" + +# --- Case 9: end to end — reader conf record reaches the diff ----------------- +if command -v jq >/dev/null 2>&1; then + FX="$TEST_TMPDIR/fx" + mkdir -p "$FX/proj/.claude" "$FX/home/.claude" + jq -n '{autoMode:{classifyAllShell:true},permissions:{allow:["Bash(npm test)","Bash(*)"]}}' >"$FX/home/.claude/settings.json" + jq -n '{permissions:{allow:["WebFetch(domain:example.com)"]}}' >"$FX/proj/.claude/settings.json" + + E2E=$(env -u CLAUDE_CONFIG_DIR \ + HOME="$FX/home" \ + PERMISSION_STATE_FIXTURE_DIR="$FX/proj" \ + PERMISSION_STATE_STARTDIR="$FX/proj" \ + PERMISSION_STATE_MANAGED_PATH="$FX/nonexistent/managed-settings.json" \ + PERMISSION_STATE_REGISTRY_KEYS="" \ + PERMISSION_STATE_PLIST_DOMAIN="" \ + bash "$STATE_SCRIPT" | bash "$MERGE_SCRIPT" | bash "$SCRIPT" --diff-only) + + assert_contains "the reader's conf record survives the merge pass-through" "$E2E" "entry-diff suspended reason=classifyAllShell scopes=user Bash(npm test)" + assert_contains "the blanket rule is dropped, not suspended" "$E2E" "entry-diff dropped class=blanket scopes=user Bash(*)" + assert_contains "the non-shell project rule is kept" "$E2E" "entry-diff kept scopes=project WebFetch(domain:example.com)" +else + pass "end-to-end conf record flow (skipped — jq not installed)" +fi + +if [[ "$FAILED" -eq 0 ]]; then + printf '\nAll %d checks passed.\n' "$CASE_NUM" + exit 0 +fi +printf '\n%d/%d checks failed.\n' "$FAILED" "$CASE_NUM" >&2 +exit 1 diff --git a/plugins/claude-config/skills/audit-permission-state/scripts/managed-conformance.sh b/plugins/claude-config/skills/audit-permission-state/scripts/managed-conformance.sh new file mode 100755 index 0000000000..d639c4f475 --- /dev/null +++ b/plugins/claude-config/skills/audit-permission-state/scripts/managed-conformance.sh @@ -0,0 +1,231 @@ +#!/usr/bin/env bash +# managed-conformance.sh — which managed intents are actually enforced, and +# which a developer can loosen. +# +# An administrator deploys managed policy believing it is policy. Some of it is: +# "no other level, including command line arguments, can override a managed +# permission rule", and "if a tool is denied at any level, no other level can +# allow it". Some of it is not: a managed `autoMode` section is additive, so a +# developer can extend it with a personal `allow` entry that overrides an +# organization `soft_deny` -- "the combination is additive, not a hard policy +# boundary". Nothing surfaces which is which. +# +# This reports that split. It reports ONLY what the consumer's own policy does +# and does not achieve; it never prescribes what a policy should contain. That +# keeps it neutral by construction rather than by restraint -- every rule string +# it prints comes from a file it read. +# +# Input: permission-state.sh records on stdin (surface + rule + conf records). +# With no piped input the reader is run directly and its status propagated. +# +# Output: +# MANAGED-NOTE: +# managed enforced cannot be overridden from below +# managed loosenable a lower scope can weaken this +# managed conformance summary enforced= loosenable= status= +# +# Prerequisites: POSIX text tools only. +# +# Usage: +# permission-state.sh | managed-conformance.sh +# managed-conformance.sh [--help] + +set -uo pipefail + +usage() { + cat <<'EOF' +managed-conformance.sh — report which managed intents are enforced vs loosenable. + +Usage: permission-state.sh | managed-conformance.sh + managed-conformance.sh [--help] + +Emits "managed enforced ", "managed loosenable ", +and a summary. Every rule string printed comes from a file that was read -- this +report never prescribes rules a policy should contain. + +Reads only. Managed policy is read-only by construction: those are admin-write +OS locations or a claude.ai Owner role, so a plugin could not author them. +EOF +} + +case "${1:-}" in +-h | --help) + usage + exit 0 + ;; +"") ;; +*) + echo "ERROR: unknown argument '$1'" >&2 + exit 2 + ;; +esac + +if [[ -t 0 ]]; then + STATE_SCRIPT="${BASH_SOURCE[0]%/*}/permission-state.sh" + if [[ ! -r "$STATE_SCRIPT" ]]; then + echo "ERROR: cannot read $STATE_SCRIPT — nothing to report on" >&2 + exit 2 + fi + records="$(bash "$STATE_SCRIPT")" || exit $? +else + records="$(cat)" +fi + +report="$(printf '%s\n' "$records" | awk ' +function text_of(start, i, s) { + s = $start + for (i = start + 1; i <= NF; i++) s = s " " $i + return s +} + +# The tool token: everything before the first "(". A rule that IS its own token +# is the whole-tool form and reaches every call of that tool. Same definition +# the merge uses, so the two stages agree about what beats what. +function tool_of(t, p) { p = index(t, "("); return p ? substr(t, 1, p - 1) : t } + +$1 == "rule" { + if ($2 == "managed") { + key = $4 SUBSEP text_of(5) + if (!(key in managed_seen)) { + managed_seen[key] = 1 + managed_order[++n_managed] = key + } + } else { + lower[$4 SUBSEP text_of(5)] = ($4 SUBSEP text_of(5)) in lower ? lower[$4 SUBSEP text_of(5)] "," $2 : $2 + } + next +} +$1 == "conf" { + conf[$2 SUBSEP $4] = text_of(5) + next +} +$1 == "NOTE:" { next } +NF >= 3 { + n_surfaces++ + if ($1 == "managed") { + n_managed_surfaces++ + status = $3 + if (status == "present") n_managed_present++ + # invalid-json belongs here, not with absent: a CORRUPT managed policy is a + # policy that exists and could not be read, and reporting it as "no policy + # deployed" is the single worst answer this report can give an + # administrator. The merge already groups the three; this now matches it. + else if (status == "skipped" || status == "unreadable" || status == "invalid-json") unread[++n_unread] = $2 " (" status ")" + } + next +} + +END { + if (n_surfaces == 0) exit 2 + + # A managed report that cannot say whether it read the managed scope is worse + # than no report: an administrator would read silence as "no policy deployed". + for (i = 1; i <= n_unread; i++) + print "MANAGED-NOTE: the managed surface " unread[i] " was NOT read, so this report is incomplete by that surface — it is not evidence that no policy is deployed there." + + print "MANAGED-NOTE: server-managed settings are delivered remotely at sign-in and have no local path, so no local reader can see them. \"Managed\" here means the LOCAL managed surfaces only; an intent enforced remotely will not appear below." + + # "No local policy" is claimed ONLY when every surface was actually looked at + # and found empty. A surface that could not be read -- corrupt, skipped, + # unreadable -- means a policy may well be deployed and this report could not + # see it, so the honest status is `incomplete`. Reporting that as + # `no-local-policy` told an administrator their corrupt policy file was an + # absent one. + if (n_managed_present == 0) { + if (n_unread > 0) { + print "MANAGED-NOTE: no managed surface could be read with content, and " n_unread " surface(s) above could not be read at all — this is NOT evidence that no policy is deployed, only that none could be read here." + print "managed conformance summary enforced=0 loosenable=0 status=incomplete" + } else { + print "MANAGED-NOTE: no local managed policy surface was readable with content, so there is nothing to report conformance against." + print "managed conformance summary enforced=0 loosenable=0 status=no-local-policy" + } + exit 0 + } + + # --- Enforced: what no lower scope can undo -------------------------------- + # + # "no other level, including command line arguments, can override a managed + # permission rule", and separately "if a tool is denied at any level, no other + # level can allow it". A managed deny is therefore the strongest thing an + # administrator can write, and it is the only category this report calls + # enforced without qualification. + for (i = 1; i <= n_managed; i++) { + split(managed_order[i], f, SUBSEP) + kind = f[1]; text = f[2] + if (kind == "deny") { + print "managed enforced deny " text + n_enforced++ + } else { + # ask and allow at managed scope still cannot be OVERRIDDEN, but they can + # be OUTRANKED: deny is evaluated before ask and ask before allow, from any + # scope. So a lower-scope deny changes the outcome without overriding the + # managed rule at all -- a distinction an administrator reading "managed is + # highest" would not expect. + # A lower scope beats this rule either by carrying the SAME TEXT in an + # earlier-evaluated kind, or by carrying a WHOLE-TOOL rule for its tool -- + # `deny Bash` reaches every Bash call, so it beats `allow Bash(npm test)` + # without sharing a character of its text. + # + # Testing exact text alone made this stage contradict the merge on the + # same records: the merge reported `inert allow … removed_by=deny@Bash` + # while this reported `enforced allow`. Of the two, the administrator- + # facing one over-claimed, which is the direction that matters. + tk = tool_of(text) + beaten = ""; beaten_key = "" + if (kind == "allow") { + if ((("deny") SUBSEP text) in lower) { beaten = "deny"; beaten_key = ("deny") SUBSEP text } + else if (tk != text && (("deny") SUBSEP tk) in lower) { beaten = "deny"; beaten_key = ("deny") SUBSEP tk } + else if ((("ask") SUBSEP text) in lower) { beaten = "ask"; beaten_key = ("ask") SUBSEP text } + else if (tk != text && (("ask") SUBSEP tk) in lower) { beaten = "ask"; beaten_key = ("ask") SUBSEP tk } + } else if (kind == "ask") { + if ((("deny") SUBSEP text) in lower) { beaten = "deny"; beaten_key = ("deny") SUBSEP text } + else if (tk != text && (("deny") SUBSEP tk) in lower) { beaten = "deny"; beaten_key = ("deny") SUBSEP tk } + } + if (beaten != "") { + split(beaten_key, bk, SUBSEP) + whole = (bk[2] == tk && tk != text) ? " whole-tool" : "" + print "managed loosenable rule the managed " kind " rule " text " is outranked by a" whole " " beaten " rule (" bk[2] ") in scope(s) " lower[beaten_key] " — evaluation order (deny, then ask, then allow) applies from any scope, so a lower scope changes the outcome without overriding the managed rule" + n_loosenable++ + } else { + print "managed enforced " kind " " text + n_enforced++ + } + } + } + + # --- Loosenable: the managed autoMode block -------------------------------- + # + # permissions, hooks, MCP, sandbox-filesystem and sandbox-network each have an + # exclusivity lock; auto mode has none. "A developer can extend environment, + # allow, soft_deny, and hard_deny with personal entries but cannot remove + # entries that managed settings provide… a developer-added allow entry can + # override an organization soft_deny entry: the combination is additive, not a + # hard policy boundary." + if ((("managed") SUBSEP "autoModePresent") in conf) { + print "managed loosenable autoMode a managed autoMode section is ADDITIVE, not a policy boundary: a developer cannot remove entries it provides, but a developer-added allow entry can override an organization soft_deny entry. For an action that must never run regardless of classifier configuration, use permissions.deny in managed settings, which cannot be overridden" + n_loosenable++ + } + + # disableAutoMode is the one auto-mode-adjacent lever that IS a lock -- and + # only when it carries the documented string. + k = ("managed") SUBSEP "disableAutoMode" + k2 = ("managed") SUBSEP "permissions.disableAutoMode" + for (kk in conf) { + split(kk, kf, SUBSEP) + if (kf[1] != "managed") continue + if (kf[2] != "disableAutoMode" && kf[2] != "permissions.disableAutoMode") continue + if (conf[kk] == "\"disable\"") + print "managed enforced lockout " kf[2] " is set to \"disable\" in managed settings, which prevents auto mode from being used at all" + else + print "managed loosenable lockout " kf[2] " is " conf[kk] " in managed settings, but the documented value is the STRING \"disable\" — any other value is accepted and silently does nothing, so auto mode is NOT disabled" + if (conf[kk] == "\"disable\"") n_enforced++; else n_loosenable++ + } + + print "managed conformance summary enforced=" n_enforced + 0 " loosenable=" n_loosenable + 0 " status=read" +} +')" || { + echo "ERROR: no scope records on input — managed-conformance.sh will not report on a managed policy it never read" >&2 + exit 2 +} + +printf '%s\n' "$report" diff --git a/plugins/claude-config/skills/audit-permission-state/scripts/managed-conformance.test.sh b/plugins/claude-config/skills/audit-permission-state/scripts/managed-conformance.test.sh new file mode 100755 index 0000000000..2df5d00261 --- /dev/null +++ b/plugins/claude-config/skills/audit-permission-state/scripts/managed-conformance.test.sh @@ -0,0 +1,285 @@ +#!/usr/bin/env bash +# Regression tests for managed-conformance.sh (self-contained — ships with the plugin). +# +# Every case feeds hand-written reader records on stdin. No test reads the +# operator's real ~/.claude, and none touches the machine's real managed policy +# — deploying policy to a developer's machine to test a report about policy +# would change every session's permission behavior on that machine. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="$SCRIPT_DIR/managed-conformance.sh" + +TEST_TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TEST_TMPDIR"' EXIT + +FAILED=0 +CASE_NUM=0 +pass() { + CASE_NUM=$((CASE_NUM + 1)) + printf 'PASS: %s\n' "$1" +} +fail() { + CASE_NUM=$((CASE_NUM + 1)) + FAILED=$((FAILED + 1)) + printf 'FAIL: %s\n detail: %s\n' "$1" "$2" >&2 +} +assert_eq() { + if [[ "$2" == "$3" ]]; then pass "$1"; else fail "$1" "expected: $2, actual: $3"; fi +} +assert_exit() { + if [[ "$2" == "$3" ]]; then pass "$1"; else fail "$1" "expected exit $2, got $3"; fi +} +assert_contains() { + case "$2" in + *"$3"*) pass "$1" ;; + *) fail "$1" "expected to contain: $3" ;; + esac +} +assert_not_contains() { + case "$2" in + *"$3"*) fail "$1" "unexpected substring: $3" ;; + *) pass "$1" ;; + esac +} +count_matching() { printf '%s\n' "$1" | grep -cE "$2"; } + +report() { printf '%s\n' "$1" | bash "$SCRIPT"; } + +# --- The phase's own sanity check: one deny, one autoMode rule --------------- +# The deny is enforced; the autoMode section is loosenable. Getting this pair +# backwards is the failure an administrator would act on. +FIXTURE=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +user settings present /fx/home/.claude/settings.json +project settings present /proj/.claude/settings.json +conf managed file autoModePresent true +rule managed file deny Read(./.env) +rule user settings allow Bash(npm test) +EOF +) +OUT=$(report "$FIXTURE") +assert_contains "a managed deny is reported enforced" "$OUT" "managed enforced deny Read(./.env)" +assert_contains "a managed autoMode section is reported loosenable" "$OUT" "managed loosenable autoMode" +assert_contains "and the loosenable finding names the remedy" "$OUT" "use permissions.deny in managed settings" +assert_contains "the summary counts both sides" "$OUT" "enforced=1 loosenable=1" + +# --- Lane neutrality, asserted as a POSITIVE property ------------------------- +# Every rule string in the report must appear in the input. Asserting the +# absence of some marker string would pass unconditionally and prove nothing. +NEUTRAL=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +user settings present /fx/home/.claude/settings.json +rule managed file deny Read(./.env) +rule managed file deny WebFetch +rule managed file allow Bash(git status) +rule user settings allow Bash(npm test) +EOF +) +OUT=$(report "$NEUTRAL") +leaked=0 +while IFS= read -r line; do + case "$line" in + "managed enforced deny "*) + rule="${line#managed enforced deny }" + printf '%s\n' "$NEUTRAL" | grep -qF -- "$rule" || { + leaked=1 + printf ' leaked rule not present in input: %s\n' "$rule" >&2 + } + ;; + *) ;; + esac +done < <(printf '%s\n' "$OUT") +assert_eq "every rule the report prints came from a file it read" 0 "$leaked" +assert_not_contains "the report never prescribes a rule to add" "$OUT" "should contain" +assert_not_contains "nor recommends one" "$OUT" "recommend" + +# --- Managed is highest, but it is not evaluated first ------------------------ +# "Managed settings highest" and "deny before ask before allow, from any scope" +# are both true, and their interaction is the thing an administrator would not +# expect: a lower-scope deny beats a managed allow without overriding it. +OUTRANKED=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +user settings present /fx/home/.claude/settings.json +rule managed file allow Bash(npm test) +rule user settings deny Bash(npm test) +EOF +) +OUT=$(report "$OUTRANKED") +assert_contains "a managed allow beaten by a lower deny is loosenable" "$OUT" "managed loosenable rule the managed allow rule Bash(npm test)" +assert_contains "the finding names the scope that beat it" "$OUT" "in scope(s) user" +assert_contains "and cites the mechanic rather than asserting it" "$OUT" "evaluation order (deny, then ask, then allow)" +assert_not_contains "it is not also reported enforced" "$OUT" "managed enforced allow Bash(npm test)" + +# A managed ask beaten by a lower deny is the same shape. +ASK_BEATEN=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +project settings present /proj/.claude/settings.json +rule managed file ask WebFetch +rule project settings deny WebFetch +EOF +) +OUT=$(report "$ASK_BEATEN") +assert_contains "a managed ask beaten by a lower deny is loosenable" "$OUT" "managed loosenable rule the managed ask rule WebFetch" + +# An unbeaten managed allow IS enforced — nothing below it changes the outcome. +UNBEATEN=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +user settings present /fx/home/.claude/settings.json +rule managed file allow Bash(git status) +rule user settings allow Bash(npm test) +EOF +) +OUT=$(report "$UNBEATEN") +assert_contains "an unbeaten managed allow is enforced" "$OUT" "managed enforced allow Bash(git status)" + +# A managed deny is never loosenable, whatever a lower scope says. +DENY_CHALLENGED=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +user settings present /fx/home/.claude/settings.json +rule managed file deny Read(./.env) +rule user settings allow Read(./.env) +EOF +) +OUT=$(report "$DENY_CHALLENGED") +assert_contains "a managed deny stays enforced against a lower allow" "$OUT" "managed enforced deny Read(./.env)" +assert_eq "and nothing about it is called loosenable" 0 "$(count_matching "$OUT" 'loosenable rule.*Read')" + +# --- A WHOLE-TOOL rule beneath a managed rule beats it ----------------------- +# `lower[]` keyed on exact text alone made this stage contradict the merge on +# the same records: the merge said `inert allow … removed_by=deny@Bash`, this +# said `enforced allow`. The administrator-facing one over-claimed. +WHOLE_TOOL=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +user settings present /fx/home/.claude/settings.json +rule managed file allow Bash(npm test) +rule user settings deny Bash +EOF +) +OUT=$(report "$WHOLE_TOOL") +assert_contains "a whole-tool deny beneath a managed allow makes it loosenable" "$OUT" "managed loosenable rule the managed allow rule Bash(npm test)" +assert_contains "and the finding names it as whole-tool" "$OUT" "whole-tool deny rule (Bash)" +assert_not_contains "it is never also reported enforced" "$OUT" "managed enforced allow Bash(npm test)" + +WHOLE_ASK=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +user settings present /fx/home/.claude/settings.json +rule managed file allow WebFetch(domain:example.com) +rule user settings ask WebFetch +EOF +) +OUT=$(report "$WHOLE_ASK") +assert_contains "a whole-tool ask does the same" "$OUT" "whole-tool ask rule (WebFetch)" + +# An unrelated whole-tool rule must not beat a managed rule for another tool. +UNRELATED=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +user settings present /fx/home/.claude/settings.json +rule managed file allow Bash(git status) +rule user settings deny WebFetch +EOF +) +OUT=$(report "$UNRELATED") +assert_contains "a whole-tool deny for another tool changes nothing" "$OUT" "managed enforced allow Bash(git status)" + +# --- The lock-out switch, at both key paths ---------------------------------- +LOCKED=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +conf managed file disableAutoMode "disable" +EOF +) +OUT=$(report "$LOCKED") +assert_contains "a correctly-typed lock-out is enforced" "$OUT" "managed enforced lockout disableAutoMode" + +MISTYPED=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +conf managed file permissions.disableAutoMode true +EOF +) +OUT=$(report "$MISTYPED") +assert_contains "a mistyped lock-out is loosenable, not enforced" "$OUT" "managed loosenable lockout permissions.disableAutoMode" +assert_contains "and says plainly that auto mode is not disabled" "$OUT" "auto mode is NOT disabled" + +# --- Completeness bounds every claim ----------------------------------------- +# An administrator reading silence as "no policy deployed" is the failure this +# report exists to prevent, so it never stays quiet about what it could not read. +UNREAD=$( + cat <<'EOF' +managed file present /policy/managed-settings.json +managed registry skipped - +managed plist unreadable com.anthropic.claudecode +rule managed file deny Read(./.env) +EOF +) +OUT=$(report "$UNREAD") +assert_eq "one note per surface that could not be read" 2 "$(count_matching "$OUT" 'MANAGED-NOTE: the managed surface')" +assert_contains "an unread surface is not evidence of absence" "$OUT" "not evidence that no policy is deployed" +assert_contains "server-managed settings are disclosed on every run" "$OUT" "no local path" + +# No local policy at all is stated, not implied by an empty report. +NO_POLICY=$( + cat <<'EOF' +managed file absent /policy/managed-settings.json +managed dropin-dir absent /policy/managed-settings.d +managed registry not-applicable - +managed plist not-applicable - +user settings present /fx/home/.claude/settings.json +rule user settings allow Bash(npm test) +EOF +) +OUT=$(report "$NO_POLICY") +assert_contains "no local policy is stated explicitly" "$OUT" "status=no-local-policy" +assert_eq "and no conformance claim is made" 0 "$(count_matching "$OUT" '^managed enforced')" + +# --- A CORRUPT managed policy is not an absent one --------------------------- +# invalid-json means a policy exists and could not be read. Reporting that as +# "no local policy" is the single worst answer this report can give an +# administrator, and the merge already grouped the three statuses correctly. +CORRUPT=$( + cat <<'EOF' +managed file invalid-json /policy/managed-settings.json +managed dropin-dir absent /policy/managed-settings.d +managed registry not-applicable - +managed plist not-applicable - +user settings present /fx/home/.claude/settings.json +rule user settings allow Bash(npm test) +EOF +) +OUT=$(report "$CORRUPT") +assert_contains "a corrupt managed file is reported unread" "$OUT" "the managed surface file (invalid-json) was NOT read" +assert_contains "and the status says incomplete, not absent" "$OUT" "status=incomplete" +assert_contains "and explicitly not as absence of policy" "$OUT" "not evidence that no policy is deployed" +assert_not_contains "it is never reported as no local policy" "$OUT" "status=no-local-policy" + +# --- Fail-loud ---------------------------------------------------------------- +rc=0 +err=$(printf '' | bash "$SCRIPT" 2>&1) || rc=$? +assert_exit "exit 2 on empty input" 2 "$rc" +assert_contains "the error refuses to report on an unread policy" "$err" "never read" + +rc=0 +err=$(bash "$SCRIPT" --bogus &1) || rc=$? +assert_exit "exit 2 on an unknown argument" 2 "$rc" + +rc=0 +help=$(bash "$SCRIPT" --help &1) || rc=$? +assert_exit "--help exits 0" 0 "$rc" +assert_contains "--help states the read-only construction" "$help" "read-only by construction" + +if [[ "$FAILED" -eq 0 ]]; then + printf '\nAll %d checks passed.\n' "$CASE_NUM" + exit 0 +fi +printf '\n%d/%d checks failed.\n' "$FAILED" "$CASE_NUM" >&2 +exit 1 diff --git a/plugins/claude-config/skills/audit-permission-state/scripts/no-writes.test.sh b/plugins/claude-config/skills/audit-permission-state/scripts/no-writes.test.sh new file mode 100755 index 0000000000..1ff1e34bd7 --- /dev/null +++ b/plugins/claude-config/skills/audit-permission-state/scripts/no-writes.test.sh @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +# no-writes.test.sh — criterion 9, asserted mechanically rather than by reading +# the code. +# +# Both skills claim to write nothing, in any scope, under any flag. That claim is +# the reason they are safe to point at a consumer's real configuration, so it is +# checked by RUNNING every action of both and diffing the filesystem either side +# — a fixture tree AND a fixture HOME, since the settings files these skills are +# about live in the second one. +# +# The oracle path is exercised deliberately. Running only the defaults would +# prove nothing about the one code path that spawns a process capable of writing +# outside the tree; a stub `claude` on PATH stands in for the real CLI so the +# spawn happens without a session, a token, or a network call. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +STATE="$SCRIPT_DIR/permission-state.sh" +MERGE="$SCRIPT_DIR/permission-merge.sh" +ENTRY_DIFF="$SCRIPT_DIR/automode-entry-diff.sh" +PLANE_LINT="$SCRIPT_DIR/permission-plane-lint.sh" +BLOCK_LINT="$SCRIPT_DIR/automode-block-lint.sh" +CONFORMANCE="$SCRIPT_DIR/managed-conformance.sh" +DRAFTER="$SCRIPT_DIR/../../draft-auto-mode-rules/scripts/draft-automode-block.sh" +FIXTURES="$SCRIPT_DIR/../evals/fixtures" + +TEST_TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TEST_TMPDIR"' EXIT + +FAILED=0 +CASE_NUM=0 +pass() { + CASE_NUM=$((CASE_NUM + 1)) + printf 'PASS: %s\n' "$1" +} +fail() { + CASE_NUM=$((CASE_NUM + 1)) + FAILED=$((FAILED + 1)) + printf 'FAIL: %s\n detail: %s\n' "$1" "$2" >&2 +} +assert_eq() { + if [[ "$2" == "$3" ]]; then pass "$1"; else fail "$1" "expected: $2, actual: $3"; fi +} + +if ! command -v jq >/dev/null 2>&1; then + echo "SKIP: jq not installed" >&2 + exit 0 +fi + +# --- The tree every action runs against -------------------------------------- +FX="$TEST_TMPDIR/fx" +mkdir -p "$FX/proj/.claude" "$FX/home/.claude" "$FX/policy/managed-settings.d" "$FX/startdir/.claude" +jq -n '{permissions:{allow:["Bash(git status)","Bash(python*)"],deny:["WebFetch"]}}' >"$FX/proj/.claude/settings.json" +jq -n '{permissions:{allow:["Bash(npm test)"]}}' >"$FX/proj/.claude/settings.local.json" +jq -n '{autoMode:{classifyAllShell:true},permissions:{allow:["Bash(*)"],disableAutoMode:true}}' >"$FX/home/.claude/settings.json" +jq -n '{permissions:{deny:["Read(./.env)"]}}' >"$FX/policy/managed-settings.json" +jq -n '{permissions:{ask:["Bash(rm *)"]}}' >"$FX/policy/managed-settings.d/10-first.json" +jq -n '{permissions:{allow:["Bash(ls)"]}}' >"$FX/startdir/.claude/settings.local.json" + +# A stub `claude` so the oracle's spawn path executes without a real session. +# It writes into the debug file it is handed, which is the scratch path the +# oracle chose — proving the capture lands there and nowhere else. +STUB="$TEST_TMPDIR/stub-bin" +mkdir -p "$STUB" +real_bash="$(command -v bash)" +printf '#!%s\n' "$real_bash" >"$STUB/claude" +# A quoted heredoc: the body is the STUB's source, so its parameters must reach +# the file verbatim rather than expanding against this harness's values. +cat >>"$STUB/claude" <<'STUB_CLAUDE' +debug="" +while [[ $# -gt 0 ]]; do + case "$1" in + --debug-file) + debug="$2" + shift 2 + ;; + *) shift ;; + esac +done +if [[ -n "$debug" ]]; then + printf '%s\n' "[DEBUG] Applying permission update: Adding 1 allow rule(s) to destination 'userSettings'" >"$debug" +fi +printf 'OK\n' +STUB_CLAUDE +chmod +x "$STUB/claude" +# Only `claude` is stubbed. An earlier revision wrapped every tool the scripts +# use, which deadlocked: a wrapper for `env` re-entered itself through the very +# PATH it was setting up, and the harness hung before its first assertion. The +# real tools are reached through the inherited PATH appended below. + +snapshot() { + # Path, size and full content of every file under the given roots. Mtime is + # deliberately excluded: a read can update atime on some filesystems, and this + # is a test about writes. + # + # `find -exec` rather than a `while read` loop: on Git Bash the loop's subshell + # deadlocked against the pipeline feeding it, and the harness hung before its + # first assertion. `-exec … +` needs no subshell at all. + local root + for root in "$@"; do + find "$root" -type f -exec wc -c {} + 2>/dev/null | LC_ALL=C sort + find "$root" -type f -exec cat {} + 2>/dev/null + done +} + +fixture_env() { + env -u CLAUDE_CONFIG_DIR \ + HOME="$FX/home" \ + PERMISSION_STATE_FIXTURE_DIR="$FX/proj" \ + PERMISSION_STATE_STARTDIR="$FX/startdir" \ + PERMISSION_STATE_MANAGED_PATH="$FX/policy/managed-settings.json" \ + PERMISSION_STATE_REGISTRY_KEYS="" \ + PERMISSION_STATE_PLIST_DOMAIN="" \ + AUTOMODE_CONFIG_FIXTURE="$FIXTURES/automode-config-rawctl.json" \ + AUTOMODE_DEFAULTS_FIXTURE="$FIXTURES/automode-defaults.json" \ + AUTOMODE_CRITIQUE_FIXTURE="$TEST_TMPDIR/critique.txt" \ + PATH="$STUB:$PATH" \ + "$@" +} + +printf 'Your rules look consistent.\n' >"$TEST_TMPDIR/critique.txt" + +ANSWERS=$( + cat <<'EOF' +section allow +label Test Execution +covered running the test suite +not installing dependencies +why the runner is named in the command +EOF +) + +# --- Every action of both skills, oracle ON ----------------------------------- +before="$(snapshot "$FX")" + +CASE_NUM_ACTIONS=0 +run_action() { + # run_action