From 439d8f06310ce1b4ee82ff52a329791111d8c090 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Thu, 30 Jul 2026 17:20:49 -0700 Subject: [PATCH 1/4] Carry a verbatim string across the lines it spans Blanking quoted spans per line suits a language whose strings end on the line they start. The C# verbatim string does not, so a marker on any later line of one was read as a comment and its prose linted as code. `strip_strings` now reports whether it ended inside a verbatim string, and the extractor carries that state as it does an unclosed block. A line wholly inside one is skipped, and the line that closes it still gives back whatever follows the quote. The README claimed markers in string literals are never comments, which was true only within a line. It now says what is carried and names the heredoc and the block scalar as forms that are not, rather than leaving the reader to find out. Reported by Copilot on promotion #460 as a wording gap. The wording was the smaller half. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/README.md | 2 +- scripts/prose_lint.py | 16 +++++++++++----- scripts/test_prose_lint.py | 18 +++++++++++++++++- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 5f7c7816..fd0685b5 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -41,7 +41,7 @@ The `semicolon` and `dash` rules ban a construction rather than a detectable sub The `comment-wrap` rule covers comments in every syntax the fleet's project types carry, not only the hash ones: `//` and `/* */` for C#, C, C++ and JSONC, `/* */` alone for CSS, `` for XML, csproj and markdown, `<# #>` for PowerShell, `;` for INI, and `#` for Python, shell, YAML and TOML. -JSON is treated as JSONC, because that is what ships: VS Code tasks, launch, devcontainer and workspace files all carry comments under a plain `.json` name. A marker inside a string literal is not a comment, so each line is scanned with quoted spans blanked first, and Python uses `tokenize` so a trailing comment is seen exactly. A documentation comment (`///`, `/**`, a docstring) is left to CODESTYLE, which permits the paragraphs this rule forbids. +JSON is treated as JSONC, because that is what ships: VS Code tasks, launch, devcontainer and workspace files all carry comments under a plain `.json` name. A marker inside a string literal is not a comment, so each line is scanned with quoted spans blanked first, and Python uses `tokenize` so a trailing comment is seen exactly. Blanking is per line, which suits a language whose strings end on the line they start. The C# verbatim string is carried across lines because it does not, while a heredoc or a block scalar in the shell and YAML syntaxes is not yet, so a marker inside one of those still reads as a comment. A documentation comment (`///`, `/**`, a docstring) is left to CODESTYLE, which permits the paragraphs this rule forbids. A comment sentence also has to start with a capital, which `comment-case` checks. A lowercase opening reads as the continuation of the line above it, so the two rules are read together: a wrapped sentence reports as `comment-wrap`, and a lowercase opening that is not a continuation reports as `comment-case`. Where the first word is a tool whose own casing is lowercase, the fix is to restructure rather than to capitalize the name against CODESTYLE's tooling-casing rule. diff --git a/scripts/prose_lint.py b/scripts/prose_lint.py index 037c1b71..de16b65c 100644 --- a/scripts/prose_lint.py +++ b/scripts/prose_lint.py @@ -250,17 +250,19 @@ def syntax_for(path: Path) -> Syntax | None: return HASH if not suffix else None -def strip_strings(line: str, quotes: str, verbatim: bool = False) -> str: +def strip_strings(line: str, quotes: str, verbatim: bool = False, + carried: bool = False) -> tuple[str, bool]: """Blank quoted spans so a comment marker inside a string is not read as one. Length-preserving, so an offset into the result is an offset into the line. A verbatim string takes its own rules where the syntax has one. There the backslash is an ordinary character and a doubled quote is the escape, so reading a backslash as an escape consumes the closing quote and blanks the rest of the line. + It also spans lines, so `carried` opens one and the second return says it is still open. """ out = list(line) - quote = '' - inside_verbatim = False + quote = '"' if carried else '' + inside_verbatim = carried escaped = False i = 0 while i < len(line): @@ -292,7 +294,7 @@ def strip_strings(line: str, quotes: str, verbatim: bool = False) -> str: start -= 1 inside_verbatim = verbatim and ch == '"' and '@' in line[start:i] i += 1 - return ''.join(out) + return ''.join(out), inside_verbatim # A pragma, shebang, or divider is machinery rather than prose. @@ -399,9 +401,13 @@ def extracted_comments(path: Path, lines: list[str]) -> list[tuple[int, str, boo out: list[tuple[int, str, bool]] = [] closing = '' doc_closing = '' + in_string = False for n, raw in enumerate(lines, 1): line = raw.rstrip('\r') - masked = strip_strings(line, spec['quotes'], spec['verbatim']) + was_inside = in_string + masked, in_string = strip_strings(line, spec['quotes'], spec['verbatim'], in_string) + if was_inside and in_string: # the whole line is string content + continue pos = 0 if doc_closing: # CODESTYLE owns every line until it closes end = line.find(doc_closing) diff --git a/scripts/test_prose_lint.py b/scripts/test_prose_lint.py index 005d9b76..2240d71a 100644 --- a/scripts/test_prose_lint.py +++ b/scripts/test_prose_lint.py @@ -324,9 +324,25 @@ def test_verbatim_rules_apply_to_the_double_quoted_form_only(self) -> None: Under verbatim rules the doubled quote is one escaped character and both are blanked, so counting what survives tells the two readings apart. """ - masked = prose_lint.strip_strings("var c = @'a''b'; // t", '"\'', True) + masked, _ = prose_lint.strip_strings("var c = @'a''b'; // t", '"\'', True) self.assertEqual(4, masked.count("'")) + def test_a_verbatim_string_spans_lines(self) -> None: + """It is the one string form here that carries, so masking per line invents comments. + + The line that closes it still gives back what follows the quote. + """ + # The marker on the second line is string content, so nothing is reported. + self.assertEqual([], self.flag('a.cs', 'var s = @"line one\n' + '// Two things. Here.\n' + 'line three";\n')) + # The line that closes it still gives back the comment after the quote. + self.assertEqual(['comment-wrap'], self.flag('a.cs', 'var s = @"line one\n' + 'line two"; // Two things. Here.\n')) + # A plain string ends on its own line, so the next line is ordinary code. + self.assertEqual(['comment-wrap'], self.flag('a.cs', 'var s = "line one";\n' + '// Two things. Here.\n')) + def test_a_format_with_no_comment_syntax_is_skipped(self) -> None: for name in ('a.lock', 'a.csv', 'a.txt'): with self.subTest(file=name): From ff4eca984eea41288085084155f4abd7d7f4dca6 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Thu, 30 Jul 2026 17:25:00 -0700 Subject: [PATCH 2/4] Drop the line-skip that a reopened string turned into a miss `was_inside and in_string` was read as "the whole line is string content", and it is not. A line can close the carried string, hold real code and comments, and open another that stays unclosed. Both ends are inside a string while the middle is not, so the skip dropped whatever sat between them. The skip was also redundant. A line genuinely inside a carried string is blanked whole by the masker, so the scan below it finds nothing there either way. Taking it out fixes the one case and changes neither of the others. Reported by Copilot on #465. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/prose_lint.py | 5 ++--- scripts/test_prose_lint.py | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/prose_lint.py b/scripts/prose_lint.py index de16b65c..a753c494 100644 --- a/scripts/prose_lint.py +++ b/scripts/prose_lint.py @@ -404,10 +404,9 @@ def extracted_comments(path: Path, lines: list[str]) -> list[tuple[int, str, boo in_string = False for n, raw in enumerate(lines, 1): line = raw.rstrip('\r') - was_inside = in_string + # A line inside a carried string is blanked whole, so the scan below finds nothing in it. + # Skipping the line instead would drop a string that closes and reopens around real code. masked, in_string = strip_strings(line, spec['quotes'], spec['verbatim'], in_string) - if was_inside and in_string: # the whole line is string content - continue pos = 0 if doc_closing: # CODESTYLE owns every line until it closes end = line.find(doc_closing) diff --git a/scripts/test_prose_lint.py b/scripts/test_prose_lint.py index 2240d71a..14b2b8fb 100644 --- a/scripts/test_prose_lint.py +++ b/scripts/test_prose_lint.py @@ -342,6 +342,11 @@ def test_a_verbatim_string_spans_lines(self) -> None: # A plain string ends on its own line, so the next line is ordinary code. self.assertEqual(['comment-wrap'], self.flag('a.cs', 'var s = "line one";\n' '// Two things. Here.\n')) + # Closing one and opening another leaves real code between them, which is not string content. + self.assertEqual(['comment-wrap'], + self.flag('a.cs', 'var s = @"start\n' + 'end"; /* Two things. Here. */ var t = @"open again\n' + 'still string";\n')) def test_a_format_with_no_comment_syntax_is_skipped(self) -> None: for name in ('a.lock', 'a.csv', 'a.txt'): From 0a6f6b8cac100c61df711eb6248e662af643c1b2 Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Thu, 30 Jul 2026 17:32:30 -0700 Subject: [PATCH 3/4] Mask code rather than the whole line, so a comment cannot open a string A quote in comment text is prose. Masking the line in one pass read it as a string opener, which blanked every marker after it. Within a line that cost the block its closer: `/* note @"x */ code(); // tail` found no `*/`, so the comment ran to end of line and carried into the lines below. Across lines the carried string state made it worse, because the opener stayed open and blanked markers until something closed it. Masking now runs from the scan position, and only the code before a marker advances the string state, so a comment contributes nothing to it. Block closers are found in the raw line for the same reason. The within-line half predates the string carry and reproduces on develop. The cross-line half arrived with the carry in this branch. The second assertion was written expecting two findings where the input has three, the extra one being the block comment's own wrapped sentence. Rephrased so each block line is a sentence and the two findings are the recovered comments, rather than fitting the number to the output. Reported by Copilot on #465, in the low-confidence block. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/prose_lint.py | 14 +++++++++----- scripts/test_prose_lint.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/scripts/prose_lint.py b/scripts/prose_lint.py index a753c494..fefe0671 100644 --- a/scripts/prose_lint.py +++ b/scripts/prose_lint.py @@ -404,9 +404,6 @@ def extracted_comments(path: Path, lines: list[str]) -> list[tuple[int, str, boo in_string = False for n, raw in enumerate(lines, 1): line = raw.rstrip('\r') - # A line inside a carried string is blanked whole, so the scan below finds nothing in it. - # Skipping the line instead would drop a string that closes and reopens around real code. - masked, in_string = strip_strings(line, spec['quotes'], spec['verbatim'], in_string) pos = 0 if doc_closing: # CODESTYLE owns every line until it closes end = line.find(doc_closing) @@ -424,6 +421,10 @@ def extracted_comments(path: Path, lines: list[str]) -> list[tuple[int, str, boo # Scan left to right and take whichever marker comes first. # A ceiling can only describe the first comment, so a later one was unreachable. while pos < len(line): + # Mask from here rather than once per line, so comment text never sets string state. + # A quote in a comment is prose, and reading it as a string blanks the markers after it. + tail, tail_state = strip_strings(line[pos:], spec['quotes'], spec['verbatim'], in_string) + masked = ' ' * pos + tail found: str | tuple[str, str] | None = None at = len(line) for marker in spec['line']: @@ -435,13 +436,16 @@ def extracted_comments(path: Path, lines: list[str]) -> list[tuple[int, str, boo if 0 <= where < at: at, found = where, (opener, closer) if found is None: + in_string = tail_state # the rest of the line is code break + # Only the code before the marker advances the string state. + _, in_string = strip_strings(line[pos:at], spec['quotes'], spec['verbatim'], in_string) # CODESTYLE owns a documentation comment, so this rule skips over it. # A line one runs to end of line, while a closed block one gives the rest back. if any(line[at:].startswith(d) for d in spec['doc']): if isinstance(found, str): break - end = masked.find(found[1], at + len(found[0])) + end = line.find(found[1], at + len(found[0])) if end < 0: doc_closing = found[1] # it carries on into the lines below break @@ -454,7 +458,7 @@ def extracted_comments(path: Path, lines: list[str]) -> list[tuple[int, str, boo out.append((n, body, leading)) break opener, closer = found - end = masked.find(closer, at + len(opener)) + end = line.find(closer, at + len(opener)) # a quote in the comment is prose body = (line[at + len(opener):end if end >= 0 else None]).strip().lstrip('*').strip() if body: out.append((n, body, leading)) diff --git a/scripts/test_prose_lint.py b/scripts/test_prose_lint.py index 14b2b8fb..b7103fb0 100644 --- a/scripts/test_prose_lint.py +++ b/scripts/test_prose_lint.py @@ -348,6 +348,20 @@ def test_a_verbatim_string_spans_lines(self) -> None: 'end"; /* Two things. Here. */ var t = @"open again\n' 'still string";\n')) + def test_a_quote_in_comment_text_is_prose_rather_than_a_string(self) -> None: + """Masking the comment too lets its quote open a string that blanks the markers after it. + + Within the line that costs the block its closer, and across lines the state carries and + blanks every marker below until something closes it. + """ + self.assertEqual(['comment-wrap'], + self.flag('a.cs', 'code(); /* note @"x */ code2(); // Two things. Here.\n')) + # Each block line is its own sentence, so the two findings are the recovered comments. + self.assertEqual(['comment-wrap', 'comment-wrap'], + self.flag('a.cs', '/* A note about @"paths.\n' + ' And more. */ // Two things. Here.\n' + 'var x = 1; // Two things. Here.\n')) + def test_a_format_with_no_comment_syntax_is_skipped(self) -> None: for name in ('a.lock', 'a.csv', 'a.txt'): with self.subTest(file=name): From b997508b181c7f0bb60156dc42fd4504f7996f7f Mon Sep 17 00:00:00 2001 From: Pieter Viljoen Date: Thu, 30 Jul 2026 17:37:39 -0700 Subject: [PATCH 4/4] Name the whole multi-line string gap, not two examples of it The previous wording put the C# verbatim string on one side and a heredoc or a block scalar on the other, which reads as a list of two remaining cases. The carry is keyed on the syntax, and C# is the only syntax that sets it, so every other one scans a line at a time and any string spanning lines leaks its markers. Checked rather than reasoned: a shell quoted string, a PowerShell here-string, and a YAML block scalar each report a `#` inside them as a comment. Naming the rule rather than two instances of it means the sentence stays true when a syntax is added. Reported by Copilot on #465, in the low-confidence block. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/README.md b/scripts/README.md index fd0685b5..913a1c74 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -41,7 +41,7 @@ The `semicolon` and `dash` rules ban a construction rather than a detectable sub The `comment-wrap` rule covers comments in every syntax the fleet's project types carry, not only the hash ones: `//` and `/* */` for C#, C, C++ and JSONC, `/* */` alone for CSS, `` for XML, csproj and markdown, `<# #>` for PowerShell, `;` for INI, and `#` for Python, shell, YAML and TOML. -JSON is treated as JSONC, because that is what ships: VS Code tasks, launch, devcontainer and workspace files all carry comments under a plain `.json` name. A marker inside a string literal is not a comment, so each line is scanned with quoted spans blanked first, and Python uses `tokenize` so a trailing comment is seen exactly. Blanking is per line, which suits a language whose strings end on the line they start. The C# verbatim string is carried across lines because it does not, while a heredoc or a block scalar in the shell and YAML syntaxes is not yet, so a marker inside one of those still reads as a comment. A documentation comment (`///`, `/**`, a docstring) is left to CODESTYLE, which permits the paragraphs this rule forbids. +JSON is treated as JSONC, because that is what ships: VS Code tasks, launch, devcontainer and workspace files all carry comments under a plain `.json` name. A marker inside a string literal is not a comment, so each line is scanned with quoted spans blanked first, and Python uses `tokenize` so a trailing comment is seen exactly. Blanking is per line, which suits a string that ends on the line it starts. The C# verbatim string is the one form carried across lines. Every other syntax is scanned a line at a time, so any string that spans lines leaves its markers readable, and an ordinary quoted string in shell, a PowerShell here-string, a YAML block scalar, and a heredoc all report a marker inside them as a comment. A documentation comment (`///`, `/**`, a docstring) is left to CODESTYLE, which permits the paragraphs this rule forbids. A comment sentence also has to start with a capital, which `comment-case` checks. A lowercase opening reads as the continuation of the line above it, so the two rules are read together: a wrapped sentence reports as `comment-wrap`, and a lowercase opening that is not a continuation reports as `comment-case`. Where the first word is a tool whose own casing is lowercase, the fix is to restructure rather than to capitalize the name against CODESTYLE's tooling-casing rule.