Identity hook: a word is read in linear time however long it is

docs/decisions/0040-the-identity-hook-reads-where-the-shell-may-stand-and-a-program-it-does-not-read-may-write.md

@@ -44,4 +44,5 @@ that a hook pull request ends: a script, not a reviewer, decides what a regressi
 | A program the hook does not read may write what it is given and where it stands; one of `DATA_ONLY` writes nothing; find's and tree's output options are destinations | `tests/test_hooks.py::test_a_program_the_hook_does_not_read_may_write_what_it_is_given_and_where_it_stands` |
 | Ordinary session commands are not refused by the identity hook | `tests/test_hooks.py::test_ordinary_session_commands_pass` |
 | The identity `git var` gives, a credential in a URL and an scp-like remote are read by string methods in one pass, not by the three regexes SonarCloud S8786 flagged as super-linear, so a long command cannot stall the guard, "Identity hook: three super-linear regexes bounded, so a long command cannot stall the guard (SonarCloud S8786)" (issue 239) | `tests/test_hooks.py::test_the_readings_sonarcloud_flagged_are_read_in_linear_time`, `tests/test_hooks.py::test_the_readings_sonarcloud_flagged_read_what_their_regexes_read` |
+| A word is read in linear time however long it is: the lexer grows it in pieces, as shlex reads it, and the hook's own reading grows it in place, "Identity hook: escaped backticks in double quotes are read in more than linear time" (issue 260) | `tests/test_hooks.py::test_the_lexer_reads_what_shlex_reads_and_grows_a_word_in_pieces`, `tests/test_hooks.py::test_the_readings_issue_174_adds_run_in_linear_time` |
 | Each of these rules has a mutant that a test kills | `tools/mutants.py` |

tests/support.py

@@ -82,16 +82,15 @@ def over(spent: Spent, budget: float) -> str:
 
 def faster_than_linear(quarter: Spent, half: Spent, whole: Spent) -> str:
     """Why a reading grew faster than linearly over its `sizes`, or "": what the whole costs past the quarter may be four
-    and a half times what the half costs past it, and a tenth of a second over for the noise of a measure. Past the
-    quarter, linear growth gives three times, n to the 1.6 four, n to the 1.8 four and a half and quadratic five, and
-    what does not grow, such as the git the hook runs, falls away. The line is at n to the 1.8 because one reading the
-    hook has, escaped backticks in double quotes, grows as n to the 1.6 ("Identity hook: escaped backticks in double
-    quotes are read in more than linear time", issue 260); a quarter, not size 0, because a reading of nothing may take another path, as an arithmetic the hook refuses
+    times what the half costs past it, and a tenth of a second over for the noise of a measure. Past the quarter, linear
+    growth gives three times and quadratic five, and what does not grow, such as the git the hook runs, falls away. The
+    line was at four and a half while the hook read escaped backticks in double quotes as n to the 1.6 ("Identity hook:
+    escaped backticks in double quotes are read in more than linear time", issue 260); a quarter, not size 0, because a reading of nothing may take another path, as an arithmetic the hook refuses
     does not at size 0. All three are measured in the same run, so a loaded machine slows them alike (issue 253)."""
     if half.cpu < quarter.cpu:    # the measures disagree, as a warm-up the quarter paid makes them: no growth is told
         return ""
     grew, grew_half = whole.cpu - quarter.cpu, half.cpu - quarter.cpu
-    if grew >= 4.5 * grew_half + 0.1:
+    if grew >= 4 * grew_half + 0.1:
         return "cost %.2f s of CPU at its size, against %.2f s at half and %.2f s at a quarter" % (whole.cpu, half.cpu,
                                                                                                  quarter.cpu)
     return ""

tests/test_hooks.py

@@ -169,16 +169,15 @@ def three(*cpus):
         self.assertEqual(faster_than_linear(*three(0.1, 0.12, 0.15)), "")                # what does not grow
         self.assertIn("at half", faster_than_linear(*three(0.25, 1.0, 4.0)))             # quadratic
         self.assertIn("at half", faster_than_linear(*three(0.1, 0.4, 1.6)))              # quadratic, and small
-        self.assertEqual(faster_than_linear(*three(0.24, 0.63, 1.80)), "")              # n to the 1.6, as the hook has
         # A second of what does not grow must not hide a quadratic past it, and a first run that costs more than the
         # half, as warming what the others reuse does, is not growth.
         self.assertIn("at half", faster_than_linear(*three(1.1, 1.4, 2.6)))
         self.assertEqual(faster_than_linear(*three(0.3, 0.1, 0.15)), "")
         self.assertEqual(faster_than_linear(*three(0.07, 0.03, 0.04)), "")
         self.assertEqual(faster_than_linear(*three(1.3, 1.0, 1.5)), "")         # linear, after a warm-up the quarter paid
-        # At the edge: exactly four and a half times and the noise is past it.
-        self.assertIn("at half", faster_than_linear(*three(0.0, 1.0, 4.6)))
-        self.assertEqual(faster_than_linear(*three(0.0, 1.0, 4.59)), "")
+        # At the edge: exactly four times and the noise is past it.
+        self.assertIn("at half", faster_than_linear(*three(0.0, 1.0, 4.1)))
+        self.assertEqual(faster_than_linear(*three(0.0, 1.0, 4.09)), "")
         # The sizes a reading is run at: a quarter, half and whole, so the three spends are of three sizes on one path.
         self.assertEqual(sizes(16000), (4000, 8000, 16000))
         self.assertEqual(sizes(901), (225, 450, 901))
@@ -1337,6 +1336,39 @@ def test_ordinary_session_commands_pass(self):
                 "# commit the change, then git push it\ngit commit -m x"):
             self.assertEqual(self.codes(command), "", command)
 
+    def test_the_lexer_reads_what_shlex_reads_and_grows_a_word_in_pieces(self):
+        # Issue 260: the hook's lexer gives every word shlex gives, in the same order, with the same settings; and it
+        # grows a word by appending its pieces, so a long word is read in linear time and not copied whole at each one.
+        settings = dict(posix=True, punctuation_chars=shell.PUNCTUATION)
+        for text in ('git commit -m "' + "\\`x\\` " * 50 + '"', "a 'b c' \"d\\\"e\" f\\ g; h|i>j", "x=(1 2); y=$((3))",
+                     "''", '""', "a\\", "a#b c", "  ", "", "é ü", "a;;b&&c||d", "'\n'", '"\\$x"'):
+            ours, theirs = shell.Lexer(text, **settings), shlex.shlex(text, **settings)
+            for lex in (ours, theirs):
+                lex.whitespace, lex.whitespace_split, lex.commenters = " \t\r", True, ""
+            try:
+                expected = list(theirs)
+            except ValueError:
+                with self.assertRaises(ValueError, msg=text):
+                    list(ours)
+                continue
+            got = list(ours)
+            self.assertEqual(got, expected, text)
+            self.assertTrue(all(type(word) is str for word in got), text)     # the lexer's words, as strings
+        grown = []
+        real = shell._Token.__iadd__
+        with mock.patch.object(shell._Token, "__iadd__", autospec=True,
+                               side_effect=lambda token, text: grown.append(text) or real(token, text)):
+            read = shell.commands("echo " + "a" * 1000)
+        self.assertIn("a" * 1000, str(read))
+        self.assertGreaterEqual(len(grown), 999)
+        # The hook's own reading keeps the first WORD_KEPT characters of a word and one more, enough to tell a keyword,
+        # so a long word, or one dense with substitutions, is never copied whole (Muse on PR 262).
+        kept = "a" * shell.WORD_KEPT
+        self.assertEqual(shell.grown("ab", "c"), "abc")
+        self.assertEqual(shell.grown(kept, "b"), kept + "b")
+        self.assertEqual(shell.grown(kept + "b", "c"), kept + "b")
+        self.assertTrue(all(len(word) <= shell.WORD_KEPT for word in shell.AGAIN | {"in", "case"}))
+
     @wall_clock
     def test_the_readings_issue_174_adds_run_in_linear_time(self):
         # Each new reading is one pass: a comment, a $'...' refused at its backslash, escaped backticks, ( and ${...}.
@@ -1348,6 +1380,8 @@ def test_the_readings_issue_174_adds_run_in_linear_time(self):
                                  (lambda n: "for f in a; do cp /tmp/x " + "${f%}" * n + "; done; git push", 5000,
                                   "git_directory_written"),
                                  (lambda n: "alias x='cd .git;'\n" + "x ls\n" * n + "git push", 200, ""),
+                                 # A word dense with substitutions: each ) gave back the word before it (Muse on PR 262).
+                                 (lambda n: "echo " + "a$(x)" * n + "; git push", 2000, ""),
                                  # A run of stars, or of expansions read as one, before a tail that does not match.
                                  (lambda n: "x=a; cp /tmp/x " + "${x%}" * n + "Q/y; git push", 320, ""),
                                  (lambda n: "cp /tmp/x " + "*" * n + "Q/y; git push", 320, ""),

tools/hooks/shell.py

@@ -98,6 +98,48 @@ def simple_commands(text: str) -> list:
     return [(words, [target for _, target in redirections]) for words, redirections in commands(text)]
 
 
+class _Token:
+    """A word shlex grows a character at a time, held as its pieces so that it grows in linear time. shlex writes
+    `self.token += c`, which CPython copies whole for a string held in an attribute, so a word of n characters cost n
+    squared over two copies: a commit message of 120,000 took 1.8 s, past the reading budget near 80,000 pairs of
+    escaped backticks ("Identity hook: escaped backticks in double quotes are read in more than linear time", issue 260).
+    """
+    __slots__ = ("pieces",)
+    __hash__ = None
+
+    def __init__(self, text: str = ""):
+        self.pieces = [text] if text else []
+
+    def __iadd__(self, text: str):
+        self.pieces.append(text)
+        return self
+
+    def __bool__(self) -> bool:
+        return any(self.pieces)
+
+    def __eq__(self, other) -> bool:
+        return str(self) == other
+
+    def __str__(self) -> str:
+        return "".join(self.pieces)
+
+
+class Lexer(shlex.shlex):
+    """shlex, its words grown as `_Token`s and given back as strings, so that it reads in linear time what shlex reads."""
+
+    @property
+    def token(self):
+        return self._token
+
+    @token.setter
+    def token(self, value):
+        self._token = value if isinstance(value, _Token) else _Token(value)
+
+    def get_token(self):
+        word = super().get_token()
+        return str(word) if isinstance(word, _Token) else word
+
+
 def commands(text: str, spend=None, texts: list = None, scopes: bool = False, marked: bool = False) -> list:
     """Each simple command in `text`, as (words, [(operator, target)]), read as simple_commands reads it.
 
@@ -114,7 +156,7 @@ def commands(text: str, spend=None, texts: list = None, scopes: bool = False, ma
     if texts is not None:
         texts.append(text)
     held = _held(without_heredocs(text))     # only an unquoted ; ) or > is punctuation
-    lex = shlex.shlex(held, posix=True, punctuation_chars=PUNCTUATION)
+    lex = Lexer(held, posix=True, punctuation_chars=PUNCTUATION)
     lex.whitespace, lex.whitespace_split, lex.commenters = " \t\r", True, ""
     found, words, targets, operator = [], [], [], None
     # A case's patterns are data, in every arm: from `in`, and from each ;; or ;&, to the ) that ends the pattern. Each is
@@ -244,6 +286,15 @@ def _marks(token: str) -> list:
 WORD_ENDS = " \t\r" + ENDS
 # Reserved words after which the next word is a command again: a ( there opens a subshell or an arithmetic command.
 AGAIN = {"if", "then", "else", "elif", "while", "until", "do", "!", "{", "time", "for"}
+# What _held keeps of the word it is reading: it compares it only with AGAIN, "in" and "case", so the first
+# WORD_KEPT characters tell it all, and a longer word is never copied or hashed whole (issue 260; Muse on PR 262).
+WORD_KEPT = 64
+
+
+def grown(word: str, more: str) -> str:
+    """`word` with `more` after it, kept to WORD_KEPT characters and one more, so a longer word is still not empty and
+    equals no keyword."""
+    return word + more if len(word) <= WORD_KEPT else word
 # How a $'...' with a backslash is refused: the hook decodes none, so bash and zsh cannot read it differently from the
 # hook (issue 174, ruling R1). Its message says to write any message to a file.
 ANSI_REFUSED = "a $'...' holds a backslash, which the hook does not decode; write any message to a file and use -F"
@@ -316,7 +367,8 @@ def _held(text: str) -> str:
                 out.append(ESCAPED[after])
             elif after != "\n":     # a backslash before a newline is a line continuation: the two lines are one
                 out.append(c + HOLDS.get(after, after))
-            starts, comments, word = starts and after == "\n", comments and after == "\n", word + c
+            starts, comments = starts and after == "\n", comments and after == "\n"
+            word = grown(word, c)
             i += 2
             continue
         if quote is None:
@@ -329,14 +381,16 @@ def _held(text: str) -> str:
             if i < braces and (c in PUNCTUATION or c == "`" or text.startswith("$(", i)):
                 # In a ${...} each is part of the word, zsh's ${(L)f} too, and a substitution in it runs: ${x:-`cmd`}.
                 out.append(LIVE[c] if c == "`" or c == "$" else HOLDS[c])
-                starts, comments, word = False, False, word + c
+                starts, comments = False, False
+                word = grown(word, c)
                 i += 1
                 continue
             if c == "$" and text.startswith("$'", i):
                 # $'...' with a backslash is refused (ansi_end, ruling R1); with none it is a single-quoted word.
                 end = ansi_end(text, i + 2)
                 out.append(_literal(text[i + 2:end - 1]))
-                starts, comments, word = False, False, word + "'"
+                starts, comments = False, False
+                word = grown(word, "'")
                 i = end
                 continue
             if c == "$" and text.startswith('$"', i):
@@ -354,7 +408,7 @@ def _held(text: str) -> str:
             elif c == ")" and opened:
                 kept = opened.pop()
             if kept is not None:
-                words, first, word = kept[0], kept[1], kept[2] + c
+                words, first, word = kept[0], kept[1], grown(kept[2], c)
             else:
                 if c in WORD_ENDS and word:
                     words += 1
@@ -365,7 +419,7 @@ def _held(text: str) -> str:
                 if c in ENDS:
                     words, first = 0, None
                 elif c not in " \t\r<>":
-                    word += c
+                    word = grown(word, c)
             quote = c if c in "'\"" else None
             out.append(c)
             starts = c in BREAKS and c != ")"       # a ) may end a substitution inside a word: $(x)#y is one word

tools/mutants.py

@@ -3365,7 +3365,7 @@ def moved_after(rel: str, block: str, anchor: str) -> tuple:
      "    return 2.0 * max(1.0, load / cores())\n", "    return 2.0\n",
      ("tests.test_hooks.Budgets.test_a_slow_hook_fails_and_a_loaded_machine_does_not",)),
     ("a reading may grow faster than linearly", "tests/support.py",
-     "    if grew >= 4.5 * grew_half + 0.1:\n", "    if False:\n",
+     "    if grew >= 4 * grew_half + 0.1:\n", "    if False:\n",
      ("tests.test_hooks.Budgets.test_a_reading_that_grows_faster_than_its_size_fails",)),
     ("the machine's cores stand for the process's", "tests/support.py",
      "        return max(1, len(os.sched_getaffinity(0)))\n", "        return max(1, os.cpu_count() or 1)\n",
@@ -3395,6 +3395,17 @@ def moved_after(rel: str, block: str, anchor: str) -> tuple:
     ("a message about who asked may carry a credential", "tools/check_trials.py",
      "    return [hidden(message) for message in out]\n", "    return out\n",
      ("tests.test_tools.Trials.test_a_credential_in_a_fixture_is_caught",)),
+    # Issue 260: the hook's lexer grows a word in pieces.
+    ("the hook reads with shlex's own lexer, copying a word at each character", "tools/hooks/shell.py",
+     "    lex = Lexer(held, posix=True, punctuation_chars=PUNCTUATION)\n",
+     "    lex = shlex.shlex(held, posix=True, punctuation_chars=PUNCTUATION)\n",
+     (IDENTITY_TESTS + "test_the_lexer_reads_what_shlex_reads_and_grows_a_word_in_pieces",)),
+    ("the hook's reading keeps a word whole however long", "tools/hooks/shell.py",
+     "    return word + more if len(word) <= WORD_KEPT else word\n", "    return word + more\n",
+     (IDENTITY_TESTS + "test_the_lexer_reads_what_shlex_reads_and_grows_a_word_in_pieces",)),
+    ("a lexer's word is given back in pieces", "tools/hooks/shell.py",
+     "        return str(word) if isinstance(word, _Token) else word\n", "        return word\n",
+     (IDENTITY_TESTS + "test_the_lexer_reads_what_shlex_reads_and_grows_a_word_in_pieces",)),
 ]