Read dupword from the comments, not from the source lines

scripts/README.md

@@ -34,7 +34,9 @@ Whole-tree (`python3 scripts/prose_lint.py .`) reports the legacy backlog as wel
 
 The `spelling` rule covers the US English convention where cspell does not reach. That gate reads README and HISTORY only, deliberately, because gating every markdown file would mean endlessly padding `cspell.json` with technical terms, so a British spelling anywhere else in the tree had nothing checking it. The banned words are generated from stems rather than listed one by one, since an inflected spelling is as wrong as its base and a hand-listed family drifts as soon as one form is added without the others. Two words are deliberately absent: `analyses` is the US plural of `analysis` as much as it is a British verb form, and `cancelled` is a GitHub Actions job status rather than prose.
 
-**Outside markdown the rule reads the comments, not the source lines**, reusing the extraction the `comment-wrap` rule already does. An identifier, a string literal, or a lookup table is code, and judging it as prose would make this script report its own table of banned words.
+**Outside markdown `spelling` and `dupword` read the comments, not the source lines**, reusing the extraction the `comment-wrap` rule already does. An identifier, a string literal, or a lookup table is code, and judging it as prose would make this script report its own table of banned words. Each comment on a line is judged on its own rather than joined with its neighbors, because two comments are two sentences and joining them reads the second's opening word as a repeat of the first's last.
+
+`dupword` gates CI, so its scope decides what a correct file is allowed to contain. A repeated token outside a comment is usually correct authoring rather than a typo: `class="gallery gallery-cols-1"` is the ordinary way two CSS class names share a prefix, and `rel`, `srcset`, `sizes` and the `data-*` attributes all take value lists of the same shape. There is no edit that satisfies the rule without changing the rendered page, so a blocking gate that reads those lines rejects correct work. The cost of the narrower scope is stated plainly rather than hidden: a duplicated word in HTML body text, or in a YAML or JSON string value, is no longer caught. Narrowing to the comment is preferred over exempting an attribute, since an exemption list covers only the attributes its author thought of.
 
 **Scope** is every text file git tracks, binaries skipped by a NUL-byte check, with no extension allowlist: an allowlist covers what its author thought of and silently stops covering whatever is added next, which is the same reason the line-endings rule already requires `git ls-files` over a raw `find`. `--list-files` prints the discovered set for auditing.
 

scripts/prose_lint.py

@@ -788,10 +788,11 @@ def check_file(path: Path, rules: set[str]) -> list[tuple[int, str, str]]:
     lines = raw.split('\n')
     if {'comment-wrap', 'comment-case'} & rules:
         out.extend(f for f in comment_wrap_findings(path, raw, lines) if f[1] in rules)
-    # Outside markdown the prose lives in the comments, and the rule judges prose rather than code.
-    # Reading the source line itself would flag an identifier, or in this file the rule's own table.
+    # Outside markdown the prose lives in the comments, and both rules judge prose, not code.
+    # A source line holds identifiers and literals, and an attribute value may legally repeat.
+    # Reading it rejects correct work, `class="gallery gallery-cols-1"` being the reported case.
     comments: dict[int, list[str]] = {}
-    if 'spelling' in rules and path.suffix != '.md':
+    if {'spelling', 'dupword'} & rules and path.suffix != '.md':
         for ln, text, _ in extracted_comments(path, lines):
             comments.setdefault(ln, []).append(text)
     in_fence = False
@@ -837,10 +838,15 @@ def check_file(path: Path, rules: set[str]) -> list[tuple[int, str, str]]:
                                 'spaced hyphen -> a comma, two sentences, or parentheses'))
 
         if 'dupword' in rules:
-            for m in DUPWORD.finditer(prose):
-                if m.group(0).lower() in DUP_ALLOW:
-                    continue
-                out.append((i, 'dupword', f"duplicated word '{m.group(1)}'"))
+            # Each comment on the line is judged on its own rather than joined with its neighbors.
+            # Joining them would read the second's opening word as a repeat of the first's last.
+            texts = ([prose] if path.suffix == '.md'
+                     else [strip_inline_code(c) for c in comments.get(i, [])])
+            for text in texts:
+                for m in DUPWORD.finditer(text):
+                    if m.group(0).lower() in DUP_ALLOW:
+                        continue
+                    out.append((i, 'dupword', f"duplicated word '{m.group(1)}'"))
 
         if 'spelling' in rules:
             texts = [prose] if path.suffix == '.md' else comments.get(i, [])

scripts/test_prose_lint.py

@@ -21,6 +21,9 @@
 # A file full of rejected input would otherwise report itself.
 DUP = 'the ' + 'the'
 SPLICE_BAIT = 'It runs on push; ' + 'it gates the merge'
+# Attribute values whose repetition is correct authoring, assembled for the same reason.
+DUP_CLASS = 'gallery ' + 'gallery-cols-1'
+DUP_REL = 'nofollow ' + 'nofollow-ugc'
 
 
 class BaitCase(unittest.TestCase):
@@ -166,6 +169,13 @@ def test_the_tiers_do_not_overlap(self) -> None:
 
 
 class TestDupword(BaitCase):
+    """A doubled word, read from markdown prose and from the comments of every other syntax.
+
+    The scope matters more here than for the other prose rules, because this one gates CI. Outside
+    markdown a repeated token is far more often correct code than a typo: `class="gallery
+    gallery-cols-1"` is the ordinary HTML idiom, and no edit fixes it without changing the page.
+    """
+
     def test_every_allowlist_entry_is_permitted(self) -> None:
         for phrase in prose_lint.DUP_ALLOW:
             with self.subTest(phrase=phrase):
@@ -181,6 +191,55 @@ def test_a_word_joining_character_does_not_start_a_repetition(self) -> None:
         self.assertEqual([], self.kinds('either/or or must-pair inputs\n', {'dupword'}))
         self.assertEqual([], self.kinds('a must-pair pair of inputs\n', {'dupword'}))
 
+    def test_a_repetition_in_a_comment_is_flagged_in_every_syntax(self) -> None:
+        """Narrowing the rule to comments has to leave it enforcing in all of them."""
+        for name, comment in (('bait.py', f'# {DUP.capitalize()} thing.'),
+                              ('bait.sh', f'# {DUP.capitalize()} thing.'),
+                              ('bait.yml', f'# {DUP.capitalize()} thing.'),
+                              ('bait.cs', f'// {DUP.capitalize()} thing.'),
+                              ('bait.cs', f'/* {DUP.capitalize()} thing. */'),
+                              ('bait.html', f'<!-- {DUP.capitalize()} thing. -->')):
+            with self.subTest(file=name, comment=comment):
+                self.assertEqual(['dupword'], self.kinds(f'{comment}\n', {'dupword'}, name=name))
+
+    def test_a_trailing_comment_is_read_and_the_code_before_it_is_not(self) -> None:
+        """The comment is prose wherever it sits on the line, and the statement stays code."""
+        self.assertEqual(['dupword'],
+                         self.kinds(f'x = 1  # {DUP.capitalize()} thing.\n', {'dupword'},
+                                    name='bait.py'))
+
+    def test_code_is_not_prose_outside_markdown(self) -> None:
+        """A repeated token in code is the author's, and often the only spelling that works.
+
+        The class attribute is the reported case: two class names sharing a prefix is how CSS is
+        written, and `rel`, `srcset` and `data-*` all take value lists with the same shape.
+        """
+        for name, line in (('bait.html', f'<div class="{DUP_CLASS}">'),
+                           ('bait.html', f'<a rel="{DUP_REL}" href="#">x</a>'),
+                           ('bait.html', f'<p>{DUP} thing</p>'),
+                           ('bait.py', f'x = "{DUP}"'),
+                           ('bait.yml', f'key: {DUP}'),
+                           ('bait.json', f'{{ "a": "{DUP}" }}')):
+            with self.subTest(file=name, line=line):
+                self.assertEqual([], self.kinds(f'{line}\n', {'dupword'}, name=name))
+
+    def test_two_comments_on_one_line_are_judged_separately(self) -> None:
+        """Joining them would read the second comment's opening word as a repeat of the first's."""
+        word = DUP.split()[0]
+        pair = f'<!-- Ends with {word} --><!-- {word.capitalize()} opens -->\n'
+        self.assertEqual([], self.kinds(pair, {'dupword'}, name='bait.html'))
+
+    def test_inline_code_is_not_prose(self) -> None:
+        """A backticked token is quoted, the same exemption every other prose rule takes."""
+        self.assertEqual([], self.kinds(f'The `{DUP}` field.\n', {'dupword'}))
+        self.assertEqual([], self.kinds(f'# The `{DUP}` field.\n', {'dupword'}, name='bait.py'))
+
+    def test_the_repo_is_clean_of_duplicated_words(self) -> None:
+        """The rule gates in CI, so the tree it gates has to pass it today and not eventually."""
+        found = [f'{prose_lint.rel(p)}:{ln}' for p in prose_lint.discover(['.'])
+                 for ln, kind, _ in prose_lint.check_file(p, {'dupword'}) if kind == 'dupword']
+        self.assertEqual([], found)
+
 
 class TestSemicolon(BaitCase):
     def test_a_splice_is_flagged(self) -> None: