Add per-section verbatim fidelity for markdown files

spec/audit.py

@@ -118,20 +118,65 @@ def applies(applies_to, sel):
 _HEADING = re.compile(r"^#{1,6}\s+(.*?)\s*$")
 
 
+def _section_spec(elt):
+    """Normalize a sections[] entry to (name, appliesTo, fidelity). A bare string is appliesTo `*`, intent."""
+    if isinstance(elt, str):
+        return elt, "*", "intent"
+    return elt.get("name", ""), elt.get("appliesTo", "*"), elt.get("fidelity", "intent")
+
+
 def required_sections(item, sel):
-    """Section names this repo must carry from a baseline entry, filtered by each section's own appliesTo.
+    """Section names to presence-check (heading grep) for this repo - the non-verbatim sections.
 
-    A bare-string section is appliesTo `*`; an object section carries its own selector. The entry's own
-    appliesTo is assumed already matched by the caller (the file is carried at all).
+    A bare-string section is appliesTo `*`, intent; an object section carries its own selector and fidelity.
+    A verbatim section is checked byte-for-byte instead (verbatim_sections), so it is excluded here to avoid a
+    redundant presence finding. The entry's own appliesTo is assumed already matched by the caller.
     """
     out = []
     for elt in item.get("sections", []):
-        name, sec = (elt, "*") if isinstance(elt, str) else (elt.get("name", ""), elt.get("appliesTo", "*"))
-        if name and applies(sec, sel):
+        name, sec, fid = _section_spec(elt)
+        if name and fid != "verbatim" and applies(sec, sel):
+            out.append(name)
+    return out
+
+
+def verbatim_sections(item, sel):
+    """Section names marked fidelity verbatim for this repo - checked byte-for-byte against the hub canonical."""
+    out = []
+    for elt in item.get("sections", []):
+        name, sec, fid = _section_spec(elt)
+        if name and fid == "verbatim" and applies(sec, sel):
             out.append(name)
     return out
 
 
+def extract_section(text, heading):
+    """The `## <heading>` H2 section including its heading line, up to the next sibling H2 or EOF; None if absent.
+
+    EOL-normalized to `\\n`. The match that locates the heading is by its parsed text (the text after the `## `
+    marker, case- and whitespace-folded), so a re-cased or re-spaced heading is still found rather than read as
+    a missing section. The heading line's exact bytes are then part of the hashed region, so that re-casing or
+    re-spacing surfaces as drift. A nested `###` stays inside the body. A `## ` line inside a fenced code block
+    (``` or ~~~) is not a boundary, so a code sample cannot truncate the region and hide drift after it.
+    """
+    want = heading.strip().lower()
+    out, capturing, fenced = [], False, False
+    for ln in normalize(text).split("\n"):
+        stripped = ln.strip()
+        if stripped.startswith("```") or stripped.startswith("~~~"):
+            fenced = not fenced
+        elif not fenced and stripped.startswith("## "):
+            if capturing:
+                break  # a sibling H2 ends the section
+            if stripped[2:].strip().lower() == want:  # parsed heading text after the "## " marker
+                capturing = True
+                out.append(ln)  # include the heading so its exact bytes are part of the hash
+            continue
+        if capturing:
+            out.append(ln)
+    return "\n".join(out) if capturing else None
+
+
 def heading_texts(markdown):
     """Lowercased heading texts in a markdown document, for case-insensitive section-presence matching."""
     return {m.group(1).strip().lower() for line in markdown.splitlines() for m in (_HEADING.match(line),) if m}
@@ -479,6 +524,7 @@ def audit_repo(entry, spec):
     # is DRIFT (a hint to verify), never a LETTER.
     sel = repo_selectors(entry, spec["registry"].get("defaults", {}))
     wanted_sections = {}  # path -> set of required section names, unioned across applicable entries
+    verbatim_secs = {}  # path -> set of section names checked byte-for-byte against the hub canonical
     check_item = {}  # path -> entry, for a fidelity interface/verbatim entry (last applicable wins per path)
     path_order = []
     for item in spec["files"]["baseline"]:
@@ -487,8 +533,10 @@ def audit_repo(entry, spec):
         path = item["path"]
         if path not in wanted_sections:
             wanted_sections[path] = set()
+            verbatim_secs[path] = set()
             path_order.append(path)
         wanted_sections[path].update(required_sections(item, sel))
+        verbatim_secs[path].update(verbatim_sections(item, sel))
         if item.get("fidelity") in ("interface", "verbatim"):
             check_item[path] = item
     for path in path_order:
@@ -526,7 +574,8 @@ def audit_repo(entry, spec):
         # Heading-based presence is only meaningful for markdown. A "section" named on a non-md file (e.g. a
         # tasks.json task group) is an intent marker judged per AUDIT.md, not a heading grep.
         needed = wanted_sections[path]
-        if needed and path.endswith(".md"):
+        verbatim_needed = verbatim_secs[path]
+        if (needed or verbatim_needed) and path.endswith(".md"):
             if text is None:
                 # Fail loud rather than skip silently: the contents API returned no inline content (an
                 # oversized file, a symlink, a submodule), so the section check could not run - surface that
@@ -537,6 +586,12 @@ def audit_repo(entry, spec):
                 for name in sorted(needed):
                     if name.strip().lower() not in present:
                         findings.append(("DRIFT", f"section: '{name}' not found as a heading in {path} on {ground} (renamed or missing; verify intent per AUDIT.md section 7)"))
+                # A verbatim section must match the hub's canonical byte-for-byte (EOL-normalized), like a
+                # verbatim file but scoped to the one `## <heading>` region - so a universal rule block cannot
+                # drift or fall behind a newly added rule while its heading still passes the presence check.
+                for name in sorted(verbatim_needed):
+                    findings.extend(check_verbatim(f"{path} section '{name}'", text, path,
+                                                   extract=lambda t, n=name: extract_section(t, n)))
 
     # --- Registry driftNotes freshness ---
     # Gated on everything else passing: a clean repo has no outstanding work for a pending-marker note to
@@ -628,6 +683,21 @@ def _selftest():
         print("  FAIL verbatim: forked github-release region should hash differently")
     else:
         print("  ok   verbatim: a forked github-release region hashes differently from the canonical")
+    # Section-region extraction: the region includes the heading line, keeps a nested ### and a fenced ## inside
+    # the body, ends at the next sibling H2, is None if absent, and rehashes when the heading is re-cased - the
+    # per-section verbatim check depends on every one of these.
+    md = "# Title\n\n## Alpha\n\nbody a\n\n```\n## not a heading\n```\n\n### nested\nstill alpha\n\n## Beta\n\nbody b\n"
+    a, b, gone = extract_section(md, "Alpha"), extract_section(md, "Beta"), extract_section(md, "Gamma")
+    spaced = extract_section("##   Alpha\n\nbody a\n", "Alpha")  # extra marker-gap whitespace still locates
+    if (a is None or not a.startswith("## Alpha") or "body a" not in a or "## not a heading" not in a
+            or "still alpha" not in a or "body b" in a
+            or b is None or not b.startswith("## Beta") or "body b" not in b or "body a" in b or gone is not None
+            or spaced is None or not spaced.startswith("##   Alpha")
+            or content_hash(a) == content_hash(extract_section(md.replace("## Alpha", "## alpha"), "Alpha"))):
+        ok = False
+        print("  FAIL section: extract_section region/hash behaviour")
+    else:
+        print("  ok   section: heading in region, fenced ## kept, sibling H2 ends, None if absent, whitespace-tolerant locate, re-cased heading rehashes")
 
     print("SELFTEST PASS" if ok else "SELFTEST FAIL")
     return 0 if ok else 1

spec/fidelity-model.md

@@ -12,14 +12,14 @@ Each [`spec/files.json`][files] entry declares one `fidelity`, defaulting to `pr
 
 - **presence** - the unit exists (a file, or a markdown section heading). The audit's baseline check.
 - **intent** - carried faithfully but judged by meaning, not bytes. A downstream copy legitimately differs (a governed divergence or a paraphrase), and equivalence is a human call via `intentRef`. The audit asserts nothing beyond presence.
-- **verbatim** - byte-identical to the hub's canonical after line-ending normalization. The audit content-hashes the downstream copy against canonical. It applies to a whole file or a workflow job region (a job selected by key).
+- **verbatim** - byte-identical to the hub's canonical after line-ending normalization. The audit content-hashes the downstream copy against canonical. It applies to a whole file, a workflow job region (a job selected by key), or a markdown section region (a `## heading` block selected by name). The section granularity lets one file be **intent overall while a few of its sections are verbatim** - a universal rule block stays byte-identical fleet-wide even though the rest of the document is a repo-adapted paraphrase, so a stale section or a missing rule is caught while its heading still passes the presence check.
 - **interface** - an overridable body that must honor a named contract. The audit checks the contract by name and wiring, never the body.
 
 Fidelity is a declared field defaulting to `presence`, never inferred from `whole`/`placeholders`. `.editorconfig` and `.markdownlint-cli2.jsonc` are both whole with no placeholders yet sit at opposite fidelity, because the discriminator is governance, not field shape.
 
 ## Why Each Unit Sits Where It Does
 
-- **verbatim** - `.markdownlint-cli2.jsonc` (fleet-generic, no governed divergence), and the `github-release` job region of the release task (the canonical orchestration a repo must not fork).
+- **verbatim** - `.markdownlint-cli2.jsonc` (fleet-generic, no governed divergence), the `github-release` job region of the release task (the canonical orchestration a repo must not fork), and the universal rule sections of `AGENTS.md` (`Repository Boundaries and Write Safety`, `Git and Commit Rules`, `Verification Discipline`) - fleet-law with no repo-specific content (no SHAs, no `ptr727/<repo>` references), where a paraphrase or a missing rule is a defect, not an adaptation. The rest of `AGENTS.md` stays intent because it carries repo-specific content (the `Branching Model` cites this repo's own historical SHAs, others carry project-type examples).
 - **interface** - the release and PR workflows. Their fixed contract is the job and check names plus the artifact handoff, while the leaf build jobs are owned. See the override seam in [`AGENTS.md`][agents].
 - **intent** - `.editorconfig` and `.gitattributes` (the `[*] end_of_line` default and path pins vary by platform), `cspell.json` (the words list and file scope vary), `CODESTYLE.md` / `WORKFLOW.md` / `AUDIT.md` / `.github/copilot-instructions.md` (carried docs judged by meaning), and the ruleset payloads (whose live state is diffed separately).
 - **presence** - `README.md`, `HISTORY.md`, `.gitignore`, and the per-repo config that only needs to exist.

spec/files.json

@@ -2,7 +2,7 @@
     "$schema": "./files.schema.json",
     "note": "The standardization baseline: files and sections a fleet repo is expected to carry, and their intent authority. The audit mechanically checks presence (letter). Equivalence (intent) is judged by hand, and a section for an absent language or target is N/A. Each entry, and each section, carries an appliesTo selector - see spec/scope-model.md for the scope model and selector vocabulary. Each entry also has a fidelity (presence by default, or intent, verbatim, interface) governing how faithfully the content is checked - see spec/fidelity-model.md.",
     "baseline": [
-        { "path": "AGENTS.md", "fidelity": "intent", "sections": ["Repository Boundaries and Write Safety", "Git and Commit Rules", "Branching Model", "Release Model", { "name": "Operational Repositories", "appliesTo": ["operational"] }, "Pull Request Title and Commit Message Conventions", "Documentation Style Conventions", "Verification Discipline", "PR Review Etiquette", "Workflow YAML Conventions"], "intentRef": "AGENTS.md", "appliesTo": "*" },
+        { "path": "AGENTS.md", "fidelity": "intent", "sections": [{ "name": "Repository Boundaries and Write Safety", "fidelity": "verbatim" }, { "name": "Git and Commit Rules", "fidelity": "verbatim" }, "Branching Model", "Release Model", { "name": "Operational Repositories", "appliesTo": ["operational"] }, "Pull Request Title and Commit Message Conventions", "Documentation Style Conventions", { "name": "Verification Discipline", "fidelity": "verbatim" }, "PR Review Etiquette", "Workflow YAML Conventions"], "intentRef": "AGENTS.md", "appliesTo": "*" },
         { "path": "CODESTYLE.md", "fidelity": "intent", "whole": true, "placeholders": ["InternalsVisibleTo project names"], "intentRef": "CODESTYLE.md", "appliesTo": "*" },
         { "path": "WORKFLOW.md", "fidelity": "intent", "whole": true, "intentRef": "WORKFLOW.md", "appliesTo": "*" },
         { "path": "README.md", "appliesTo": "*" },

spec/files.schema.json

@@ -27,7 +27,8 @@
                                     "additionalProperties": false,
                                     "properties": {
                                         "name": { "type": "string" },
-                                        "appliesTo": { "type": ["string", "array"], "items": { "type": "string" }, "minItems": 1 }
+                                        "appliesTo": { "type": ["string", "array"], "items": { "type": "string" }, "minItems": 1 },
+                                        "fidelity": { "enum": ["intent", "verbatim"] }
                                     }
                                 }
                             ]

spec/validate.py

@@ -286,6 +286,14 @@ def check_selector(where, applies_to):
         for elt in sections:
             if isinstance(elt, dict):
                 check_selector(f"{path} section '{elt.get('name', '?')}'", elt.get("appliesTo", "*"))
+                # A section may carry its own fidelity (intent default, or verbatim for a universal rule block
+                # checked byte-for-byte). verbatim is meaningful only on a markdown file, where the heading
+                # delimits the region. The hub's own file is the canonical, so no reference is needed.
+                sfid = elt.get("fidelity", "intent")
+                if sfid not in ("intent", "verbatim"):
+                    errors.append(f"files.json: {path} section '{elt.get('name', '?')}' fidelity '{sfid}' invalid (expected intent or verbatim)")
+                elif sfid == "verbatim" and not path.endswith(".md"):
+                    errors.append(f"files.json: {path} section '{elt.get('name', '?')}' is verbatim but {path} is not markdown (heading regions apply to .md only)")
             elif not isinstance(elt, str):
                 errors.append(f"files.json: {path} section entry {elt!r} must be a string or object")