diff --git a/spec/audit.py b/spec/audit.py index c6aa520c..9ddfbf80 100644 --- a/spec/audit.py +++ b/spec/audit.py @@ -118,20 +118,65 @@ def applies(applies_to, sel): _HEADING = re.compile(r"^#{1,6}\s+(.*?)\s*$") +def _section_spec(elt): + """Normalize a sections[] entry to (name, appliesTo, fidelity). A bare string is appliesTo `*`, intent.""" + if isinstance(elt, str): + return elt, "*", "intent" + return elt.get("name", ""), elt.get("appliesTo", "*"), elt.get("fidelity", "intent") + + def required_sections(item, sel): - """Section names this repo must carry from a baseline entry, filtered by each section's own appliesTo. + """Section names to presence-check (heading grep) for this repo - the non-verbatim sections. - A bare-string section is appliesTo `*`; an object section carries its own selector. The entry's own - appliesTo is assumed already matched by the caller (the file is carried at all). + A bare-string section is appliesTo `*`, intent; an object section carries its own selector and fidelity. + A verbatim section is checked byte-for-byte instead (verbatim_sections), so it is excluded here to avoid a + redundant presence finding. The entry's own appliesTo is assumed already matched by the caller. """ out = [] for elt in item.get("sections", []): - name, sec = (elt, "*") if isinstance(elt, str) else (elt.get("name", ""), elt.get("appliesTo", "*")) - if name and applies(sec, sel): + name, sec, fid = _section_spec(elt) + if name and fid != "verbatim" and applies(sec, sel): + out.append(name) + return out + + +def verbatim_sections(item, sel): + """Section names marked fidelity verbatim for this repo - checked byte-for-byte against the hub canonical.""" + out = [] + for elt in item.get("sections", []): + name, sec, fid = _section_spec(elt) + if name and fid == "verbatim" and applies(sec, sel): out.append(name) return out +def extract_section(text, heading): + """The `## ` H2 section including its heading line, up to the next sibling H2 or EOF; None if absent. + + EOL-normalized to `\\n`. The match that locates the heading is by its parsed text (the text after the `## ` + marker, case- and whitespace-folded), so a re-cased or re-spaced heading is still found rather than read as + a missing section. The heading line's exact bytes are then part of the hashed region, so that re-casing or + re-spacing surfaces as drift. A nested `###` stays inside the body. A `## ` line inside a fenced code block + (``` or ~~~) is not a boundary, so a code sample cannot truncate the region and hide drift after it. + """ + want = heading.strip().lower() + out, capturing, fenced = [], False, False + for ln in normalize(text).split("\n"): + stripped = ln.strip() + if stripped.startswith("```") or stripped.startswith("~~~"): + fenced = not fenced + elif not fenced and stripped.startswith("## "): + if capturing: + break # a sibling H2 ends the section + if stripped[2:].strip().lower() == want: # parsed heading text after the "## " marker + capturing = True + out.append(ln) # include the heading so its exact bytes are part of the hash + continue + if capturing: + out.append(ln) + return "\n".join(out) if capturing else None + + def heading_texts(markdown): """Lowercased heading texts in a markdown document, for case-insensitive section-presence matching.""" return {m.group(1).strip().lower() for line in markdown.splitlines() for m in (_HEADING.match(line),) if m} @@ -479,6 +524,7 @@ def audit_repo(entry, spec): # is DRIFT (a hint to verify), never a LETTER. sel = repo_selectors(entry, spec["registry"].get("defaults", {})) wanted_sections = {} # path -> set of required section names, unioned across applicable entries + verbatim_secs = {} # path -> set of section names checked byte-for-byte against the hub canonical check_item = {} # path -> entry, for a fidelity interface/verbatim entry (last applicable wins per path) path_order = [] for item in spec["files"]["baseline"]: @@ -487,8 +533,10 @@ def audit_repo(entry, spec): path = item["path"] if path not in wanted_sections: wanted_sections[path] = set() + verbatim_secs[path] = set() path_order.append(path) wanted_sections[path].update(required_sections(item, sel)) + verbatim_secs[path].update(verbatim_sections(item, sel)) if item.get("fidelity") in ("interface", "verbatim"): check_item[path] = item for path in path_order: @@ -526,7 +574,8 @@ def audit_repo(entry, spec): # Heading-based presence is only meaningful for markdown. A "section" named on a non-md file (e.g. a # tasks.json task group) is an intent marker judged per AUDIT.md, not a heading grep. needed = wanted_sections[path] - if needed and path.endswith(".md"): + verbatim_needed = verbatim_secs[path] + if (needed or verbatim_needed) and path.endswith(".md"): if text is None: # Fail loud rather than skip silently: the contents API returned no inline content (an # oversized file, a symlink, a submodule), so the section check could not run - surface that @@ -537,6 +586,12 @@ def audit_repo(entry, spec): for name in sorted(needed): if name.strip().lower() not in present: findings.append(("DRIFT", f"section: '{name}' not found as a heading in {path} on {ground} (renamed or missing; verify intent per AUDIT.md section 7)")) + # A verbatim section must match the hub's canonical byte-for-byte (EOL-normalized), like a + # verbatim file but scoped to the one `## ` region - so a universal rule block cannot + # drift or fall behind a newly added rule while its heading still passes the presence check. + for name in sorted(verbatim_needed): + findings.extend(check_verbatim(f"{path} section '{name}'", text, path, + extract=lambda t, n=name: extract_section(t, n))) # --- Registry driftNotes freshness --- # Gated on everything else passing: a clean repo has no outstanding work for a pending-marker note to @@ -628,6 +683,21 @@ def _selftest(): print(" FAIL verbatim: forked github-release region should hash differently") else: print(" ok verbatim: a forked github-release region hashes differently from the canonical") + # Section-region extraction: the region includes the heading line, keeps a nested ### and a fenced ## inside + # the body, ends at the next sibling H2, is None if absent, and rehashes when the heading is re-cased - the + # per-section verbatim check depends on every one of these. + md = "# Title\n\n## Alpha\n\nbody a\n\n```\n## not a heading\n```\n\n### nested\nstill alpha\n\n## Beta\n\nbody b\n" + a, b, gone = extract_section(md, "Alpha"), extract_section(md, "Beta"), extract_section(md, "Gamma") + spaced = extract_section("## Alpha\n\nbody a\n", "Alpha") # extra marker-gap whitespace still locates + if (a is None or not a.startswith("## Alpha") or "body a" not in a or "## not a heading" not in a + or "still alpha" not in a or "body b" in a + or b is None or not b.startswith("## Beta") or "body b" not in b or "body a" in b or gone is not None + or spaced is None or not spaced.startswith("## Alpha") + or content_hash(a) == content_hash(extract_section(md.replace("## Alpha", "## alpha"), "Alpha"))): + ok = False + print(" FAIL section: extract_section region/hash behaviour") + else: + print(" ok section: heading in region, fenced ## kept, sibling H2 ends, None if absent, whitespace-tolerant locate, re-cased heading rehashes") print("SELFTEST PASS" if ok else "SELFTEST FAIL") return 0 if ok else 1 diff --git a/spec/fidelity-model.md b/spec/fidelity-model.md index 83f3c764..e8f53f23 100644 --- a/spec/fidelity-model.md +++ b/spec/fidelity-model.md @@ -12,14 +12,14 @@ Each [`spec/files.json`][files] entry declares one `fidelity`, defaulting to `pr - **presence** - the unit exists (a file, or a markdown section heading). The audit's baseline check. - **intent** - carried faithfully but judged by meaning, not bytes. A downstream copy legitimately differs (a governed divergence or a paraphrase), and equivalence is a human call via `intentRef`. The audit asserts nothing beyond presence. -- **verbatim** - byte-identical to the hub's canonical after line-ending normalization. The audit content-hashes the downstream copy against canonical. It applies to a whole file or a workflow job region (a job selected by key). +- **verbatim** - byte-identical to the hub's canonical after line-ending normalization. The audit content-hashes the downstream copy against canonical. It applies to a whole file, a workflow job region (a job selected by key), or a markdown section region (a `## heading` block selected by name). The section granularity lets one file be **intent overall while a few of its sections are verbatim** - a universal rule block stays byte-identical fleet-wide even though the rest of the document is a repo-adapted paraphrase, so a stale section or a missing rule is caught while its heading still passes the presence check. - **interface** - an overridable body that must honor a named contract. The audit checks the contract by name and wiring, never the body. Fidelity is a declared field defaulting to `presence`, never inferred from `whole`/`placeholders`. `.editorconfig` and `.markdownlint-cli2.jsonc` are both whole with no placeholders yet sit at opposite fidelity, because the discriminator is governance, not field shape. ## Why Each Unit Sits Where It Does -- **verbatim** - `.markdownlint-cli2.jsonc` (fleet-generic, no governed divergence), and the `github-release` job region of the release task (the canonical orchestration a repo must not fork). +- **verbatim** - `.markdownlint-cli2.jsonc` (fleet-generic, no governed divergence), the `github-release` job region of the release task (the canonical orchestration a repo must not fork), and the universal rule sections of `AGENTS.md` (`Repository Boundaries and Write Safety`, `Git and Commit Rules`, `Verification Discipline`) - fleet-law with no repo-specific content (no SHAs, no `ptr727/` references), where a paraphrase or a missing rule is a defect, not an adaptation. The rest of `AGENTS.md` stays intent because it carries repo-specific content (the `Branching Model` cites this repo's own historical SHAs, others carry project-type examples). - **interface** - the release and PR workflows. Their fixed contract is the job and check names plus the artifact handoff, while the leaf build jobs are owned. See the override seam in [`AGENTS.md`][agents]. - **intent** - `.editorconfig` and `.gitattributes` (the `[*] end_of_line` default and path pins vary by platform), `cspell.json` (the words list and file scope vary), `CODESTYLE.md` / `WORKFLOW.md` / `AUDIT.md` / `.github/copilot-instructions.md` (carried docs judged by meaning), and the ruleset payloads (whose live state is diffed separately). - **presence** - `README.md`, `HISTORY.md`, `.gitignore`, and the per-repo config that only needs to exist. diff --git a/spec/files.json b/spec/files.json index 1833489c..b7c8c21d 100644 --- a/spec/files.json +++ b/spec/files.json @@ -2,7 +2,7 @@ "$schema": "./files.schema.json", "note": "The standardization baseline: files and sections a fleet repo is expected to carry, and their intent authority. The audit mechanically checks presence (letter). Equivalence (intent) is judged by hand, and a section for an absent language or target is N/A. Each entry, and each section, carries an appliesTo selector - see spec/scope-model.md for the scope model and selector vocabulary. Each entry also has a fidelity (presence by default, or intent, verbatim, interface) governing how faithfully the content is checked - see spec/fidelity-model.md.", "baseline": [ - { "path": "AGENTS.md", "fidelity": "intent", "sections": ["Repository Boundaries and Write Safety", "Git and Commit Rules", "Branching Model", "Release Model", { "name": "Operational Repositories", "appliesTo": ["operational"] }, "Pull Request Title and Commit Message Conventions", "Documentation Style Conventions", "Verification Discipline", "PR Review Etiquette", "Workflow YAML Conventions"], "intentRef": "AGENTS.md", "appliesTo": "*" }, + { "path": "AGENTS.md", "fidelity": "intent", "sections": [{ "name": "Repository Boundaries and Write Safety", "fidelity": "verbatim" }, { "name": "Git and Commit Rules", "fidelity": "verbatim" }, "Branching Model", "Release Model", { "name": "Operational Repositories", "appliesTo": ["operational"] }, "Pull Request Title and Commit Message Conventions", "Documentation Style Conventions", { "name": "Verification Discipline", "fidelity": "verbatim" }, "PR Review Etiquette", "Workflow YAML Conventions"], "intentRef": "AGENTS.md", "appliesTo": "*" }, { "path": "CODESTYLE.md", "fidelity": "intent", "whole": true, "placeholders": ["InternalsVisibleTo project names"], "intentRef": "CODESTYLE.md", "appliesTo": "*" }, { "path": "WORKFLOW.md", "fidelity": "intent", "whole": true, "intentRef": "WORKFLOW.md", "appliesTo": "*" }, { "path": "README.md", "appliesTo": "*" }, diff --git a/spec/files.schema.json b/spec/files.schema.json index 6cc49e64..11aad28f 100644 --- a/spec/files.schema.json +++ b/spec/files.schema.json @@ -27,7 +27,8 @@ "additionalProperties": false, "properties": { "name": { "type": "string" }, - "appliesTo": { "type": ["string", "array"], "items": { "type": "string" }, "minItems": 1 } + "appliesTo": { "type": ["string", "array"], "items": { "type": "string" }, "minItems": 1 }, + "fidelity": { "enum": ["intent", "verbatim"] } } } ] diff --git a/spec/validate.py b/spec/validate.py index ed0032e7..06c60227 100644 --- a/spec/validate.py +++ b/spec/validate.py @@ -286,6 +286,14 @@ def check_selector(where, applies_to): for elt in sections: if isinstance(elt, dict): check_selector(f"{path} section '{elt.get('name', '?')}'", elt.get("appliesTo", "*")) + # A section may carry its own fidelity (intent default, or verbatim for a universal rule block + # checked byte-for-byte). verbatim is meaningful only on a markdown file, where the heading + # delimits the region. The hub's own file is the canonical, so no reference is needed. + sfid = elt.get("fidelity", "intent") + if sfid not in ("intent", "verbatim"): + errors.append(f"files.json: {path} section '{elt.get('name', '?')}' fidelity '{sfid}' invalid (expected intent or verbatim)") + elif sfid == "verbatim" and not path.endswith(".md"): + errors.append(f"files.json: {path} section '{elt.get('name', '?')}' is verbatim but {path} is not markdown (heading regions apply to .md only)") elif not isinstance(elt, str): errors.append(f"files.json: {path} section entry {elt!r} must be a string or object")