Normalize a Dependabot Pin Bump Out of the Intent Staleness Advisory

spec/audit.py

@@ -1714,42 +1714,80 @@ def classify_verbatim(down_text, canon_text, past_texts):
     return "modified"
 
 
-_HISTORY_CACHE: dict[
-    str, list[str]
-] = {}  # rel_path -> past revision contents, cached because one canonical is compared against every audited repo
-
+@functools.cache
+def _git_revisions(rel_path):
+    """Every commit that touched rel_path in the hub's history, newest first, as (date, sha, text).
 
-def git_file_history(rel_path):
-    """Every past revision's content of a hub-tracked file (to tell a stale copy from a modified one), cached per rel_path."""
-    if rel_path in _HISTORY_CACHE:
-        return _HISTORY_CACHE[rel_path]
-    out = []
-    # Decode as UTF-8 with replacement to match the downstream and canonical reads.
-    # A divergent decode would fabricate a mismatch.
+    `text` is None where `git show` failed (rare: a permission or encoding fluke, not absence,
+    since the commit came from `git log -- rel_path` and so the path existed at that revision).
+    Cached because one canonical's history is read once per fidelity/staleness check, then reused
+    for every audited repo's copy.
+    """
     r = subprocess.run(
-        ["git", "log", "--format=%H", "--", rel_path],
+        ["git", "log", "--format=%cI %H", "--", rel_path],
         cwd=ROOT,
         capture_output=True,
-        encoding="utf-8",
-        errors="replace",
+        text=True,
         check=False,
     )
-    if r.returncode == 0:
-        for sha in r.stdout.split():
-            s = subprocess.run(
-                ["git", "show", f"{sha}:{rel_path}"],
-                cwd=ROOT,
-                capture_output=True,
-                encoding="utf-8",
-                errors="replace",
-                check=False,
-            )
-            if s.returncode == 0:
-                out.append(s.stdout)
-    _HISTORY_CACHE[rel_path] = out
+    if r.returncode != 0:
+        # A real command failure (not a git repo, a corrupt object) must not read as "no
+        # history": that silently clears the intent-staleness advisory and drops verbatim's
+        # past-revision list, both misreporting a tool fault as a clean audit.
+        raise RuntimeError(f"git log failed for {rel_path}: {r.stderr.strip()}")
+    if not r.stdout.strip():
+        return []
+    out = []
+    for line in r.stdout.splitlines():
+        date, sha = line.split(" ", 1)
+        # Decode as UTF-8 with replacement to match the downstream and canonical reads.
+        # A divergent decode would fabricate a mismatch.
+        s = subprocess.run(
+            ["git", "show", f"{sha}:{rel_path}"],
+            cwd=ROOT,
+            capture_output=True,
+            encoding="utf-8",
+            errors="replace",
+            check=False,
+        )
+        out.append((date, sha, s.stdout if s.returncode == 0 else None))
     return out
 
 
+def git_file_history(rel_path):
+    """Every past revision's content of a hub-tracked file (to tell a stale copy from a modified one)."""
+    return [text for _, _, text in _git_revisions(rel_path) if text is not None]
+
+
+def _last_effective_change(revisions):
+    """The (date, sha) of the newest revision in `revisions` (newest-first (date, sha, text)
+    triples for one file) whose content differs from its predecessor after normalize(), or the
+    oldest (creation) revision if every later one differs from its predecessor only by normalized
+    churn (spec/fidelity-model.md "Normalization": EOL, a Dependabot action-pin bump, a pruned
+    `needs:` list). None if `revisions` is empty.
+
+    The creation revision always counts as effective: it has no predecessor, and normalize()
+    applied to only one side proves nothing. A revision with unreadable text (None) also counts as
+    effective rather than being silently skipped, since a real difference cannot be ruled out.
+
+    `revisions` is assumed newest-first with each entry the immediate predecessor of the one
+    before it (verified empirically over every intent-fidelity canonical file's full history,
+    ptr727/ProjectTemplate#1014): this repo's own branching is forward-only (no back-merges from
+    main into develop) with feature branches squash-merged one at a time, so a canonical file's
+    per-path `git log` is a single line, not a graph with siblings to mis-order. A canonical file
+    reached through a genuinely branching history (a direct hotfix to main alongside independent
+    develop work touching the same path) would need each revision compared against its actual git
+    parent(s) instead of its list neighbor.
+    """
+    for i, (date, sha, text) in enumerate(revisions):
+        if i + 1 == len(revisions):
+            return date, sha
+        older_text = revisions[i + 1][2]
+        if text is None or older_text is None or normalize(text) != normalize(older_text):
+            return date, sha
+    return None
+
+
 @functools.cache
 def git_blob_in_file_history(rel_path, blob_sha):
     """Whether a blob occurred in a path's hub history."""
@@ -1765,21 +1803,22 @@ def git_blob_in_file_history(rel_path, blob_sha):
 
 @functools.cache
 def hub_last_change(rel_path):
-    """The hub checkout's last commit touching rel_path, as (iso_date, short_sha), or None if untracked.
+    """The hub checkout's most recent EFFECTIVE change to rel_path, as (iso_date, short_sha), or
+    None if untracked.
+
+    "Effective" excludes normalized churn (spec/fidelity-model.md "Normalization": EOL, a
+    Dependabot action-pin bump, a pruned `needs:` list) the same way the verbatim check already
+    does, via _last_effective_change over the file's full history. A raw last-commit date treated
+    every one of those bumps as the copy "trailing", even though the fidelity model already
+    classifies that class as governed per-repo drift rather than a deviation (ptr727/ProjectTemplate#735).
 
     Cached because one canonical's date is compared against every audited repo's copy.
     """
-    r = subprocess.run(
-        ["git", "log", "-1", "--format=%cI %h", "--", rel_path],
-        cwd=ROOT,
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    if r.returncode != 0 or not r.stdout.strip():
+    change = _last_effective_change(_git_revisions(rel_path))
+    if change is None:
         return None
-    date, sha = r.stdout.strip().split(" ", 1)
-    return date, sha
+    date, sha = change
+    return date, sha[:7]
 
 
 def intent_canonical_rel(item, path):
@@ -1810,8 +1849,10 @@ def check_intent_staleness(slug, ground, path, canonical_rel, down_text):
     (spec/fidelity-model.md), which is how a copy trailed the hub by many revisions while every
     check read clean. No reconciliation record exists anywhere, so the implementable proxy is
     when each side last changed: the hub canonical changing after the repo's copy marks the copy
-    as possibly trailing. Advisory only, DRIFT and never a failure, and honest about its blind
-    spot: a copy touched after the hub change without actually reconciling reads current.
+    as possibly trailing. "Changed" excludes normalized churn (hub_last_change), so a Dependabot
+    action-pin bump or a needs-list prune does not by itself mark every carrier as trailing.
+    Advisory only, DRIFT and never a failure, and honest about its blind spot: a copy touched
+    after the hub change without actually reconciling reads current.
 
     A copy content-identical to the canonical cannot trail it, so that case is skipped however
     old the copy's last commit is. It is also the promotion candidate spec/fidelity_honesty.py
@@ -3158,6 +3199,48 @@ def _selftest():
             "  ok   needs-mask: pruned needs (inline, block, scalar) normalizes equal, forked step differs, next key preserved"
         )
 
+    # _last_effective_change must skip a normalized-only bump, per ptr727/ProjectTemplate#735.
+    d3, d2, d1 = (
+        "2024-03-01T00:00:00+00:00",
+        "2024-02-01T00:00:00+00:00",
+        "2024-01-01T00:00:00+00:00",
+    )
+    lec_cases = [
+        (
+            "every bump back to creation is pin-only -> creation wins",
+            [(d3, "sha3", pin_b), (d2, "sha2", pin_a), (d1, "sha1", pin_a)],
+            (d1, "sha1"),
+        ),
+        (
+            "newest differs for real -> newest wins",
+            [(d3, "sha3", pin_struct), (d2, "sha2", pin_a), (d1, "sha1", pin_a)],
+            (d3, "sha3"),
+        ),
+        (
+            "newest is a pin-only bump over a real change -> the real change wins",
+            [(d3, "sha3", pin_b), (d2, "sha2", pin_a), (d1, "sha1", pin_struct)],
+            (d2, "sha2"),
+        ),
+        (
+            "one revision (creation) -> that revision wins, nothing to compare against",
+            [(d1, "sha1", "only revision\n")],
+            (d1, "sha1"),
+        ),
+        (
+            "unreadable newest text -> treated as effective, never silently skipped",
+            [(d2, "sha2", None), (d1, "sha1", "whatever\n")],
+            (d2, "sha2"),
+        ),
+        ("no history -> None", [], None),
+    ]
+    for label, revisions, want in lec_cases:
+        got = _last_effective_change(revisions)
+        if got != want:
+            ok = False
+        print(
+            f"  {'ok  ' if got == want else 'FAIL'} want={want!s:<24} got={got!s:<24}  _last_effective_change: {label}"
+        )
+
     # Region extraction and hashing: a forked github-release block must hash differently from the canonical.
     region = split_jobs(rel_ok).get("github-release")
     forked_region = split_jobs(

spec/fidelity-model.md

@@ -11,7 +11,7 @@ Carried content is a class with virtual functions. The **fixed** part is the int
 Each [`spec/files.json`][files] entry declares one `fidelity`, defaulting to `presence`.
 
 - **presence** - the unit exists (a file, or a Markdown section heading). The audit's baseline check.
-- **intent** - carried faithfully but judged by meaning, not bytes. A downstream copy legitimately differs (a governed divergence or a paraphrase), and equivalence is a human call via `intentRef`. The audit asserts presence, plus a last-modified staleness advisory at drift: a hub canonical changing after the copy's own last commit marks the copy as possibly trailing, a hint rather than proof, and content is never judged.
+- **intent** - carried faithfully but judged by meaning, not bytes. A downstream copy legitimately differs (a governed divergence or a paraphrase), and equivalence is a human call via `intentRef`. The audit asserts presence, plus a last-modified staleness advisory at drift: a hub canonical changing after the copy's own last commit marks the copy as possibly trailing, a hint rather than proof, and content is never judged. That "changing" walks past a revision whose diff is normalized-only (the same line-ending, action-pin, and job-needs normalization as verbatim below), so a Dependabot pin bump on a workflow file does not by itself mark every carrier as trailing (ptr727/ProjectTemplate#735).
 - **verbatim** - byte-identical to the hub's canonical after line-ending, action-pin, and job-needs normalization. The audit content-hashes the downstream copy against canonical. It applies to a whole file, a workflow job region (a job selected by key), or a Markdown section region (a `## heading` block selected by name). The section granularity lets one file be **intent overall while a few of its sections are verbatim**. A universal rule block stays byte-identical fleet-wide even though the rest of the document is a repo-adapted paraphrase, so a stale section or a missing rule is caught while its heading still passes the presence check.
 - **interface** - an overridable body that must honor a named contract. The audit checks the contract by name and wiring, never the body.