diff --git a/docs/decisions/0040-the-identity-hook-reads-where-the-shell-may-stand-and-a-program-it-does-not-read-may-write.md b/docs/decisions/0040-the-identity-hook-reads-where-the-shell-may-stand-and-a-program-it-does-not-read-may-write.md index c78ee5a5..c538b170 100644 --- a/docs/decisions/0040-the-identity-hook-reads-where-the-shell-may-stand-and-a-program-it-does-not-read-may-write.md +++ b/docs/decisions/0040-the-identity-hook-reads-where-the-shell-may-stand-and-a-program-it-does-not-read-may-write.md @@ -44,4 +44,5 @@ that a hook pull request ends: a script, not a reviewer, decides what a regressi | A program the hook does not read may write what it is given and where it stands; one of `DATA_ONLY` writes nothing; find's and tree's output options are destinations | `tests/test_hooks.py::test_a_program_the_hook_does_not_read_may_write_what_it_is_given_and_where_it_stands` | | Ordinary session commands are not refused by the identity hook | `tests/test_hooks.py::test_ordinary_session_commands_pass` | | The identity `git var` gives, a credential in a URL and an scp-like remote are read by string methods in one pass, not by the three regexes SonarCloud S8786 flagged as super-linear, so a long command cannot stall the guard, "Identity hook: three super-linear regexes bounded, so a long command cannot stall the guard (SonarCloud S8786)" (issue 239) | `tests/test_hooks.py::test_the_readings_sonarcloud_flagged_are_read_in_linear_time`, `tests/test_hooks.py::test_the_readings_sonarcloud_flagged_read_what_their_regexes_read` | +| A word is read in linear time however long it is: the lexer grows it in pieces, as shlex reads it, and the hook's own reading grows it in place, "Identity hook: escaped backticks in double quotes are read in more than linear time" (issue 260) | `tests/test_hooks.py::test_the_lexer_reads_what_shlex_reads_and_grows_a_word_in_pieces`, `tests/test_hooks.py::test_the_readings_issue_174_adds_run_in_linear_time` | | Each of these rules has a mutant that a test kills | `tools/mutants.py` | diff --git a/tests/support.py b/tests/support.py index c8a4afac..69e4d3d5 100644 --- a/tests/support.py +++ b/tests/support.py @@ -82,16 +82,15 @@ def over(spent: Spent, budget: float) -> str: def faster_than_linear(quarter: Spent, half: Spent, whole: Spent) -> str: """Why a reading grew faster than linearly over its `sizes`, or "": what the whole costs past the quarter may be four - and a half times what the half costs past it, and a tenth of a second over for the noise of a measure. Past the - quarter, linear growth gives three times, n to the 1.6 four, n to the 1.8 four and a half and quadratic five, and - what does not grow, such as the git the hook runs, falls away. The line is at n to the 1.8 because one reading the - hook has, escaped backticks in double quotes, grows as n to the 1.6 ("Identity hook: escaped backticks in double - quotes are read in more than linear time", issue 260); a quarter, not size 0, because a reading of nothing may take another path, as an arithmetic the hook refuses + times what the half costs past it, and a tenth of a second over for the noise of a measure. Past the quarter, linear + growth gives three times and quadratic five, and what does not grow, such as the git the hook runs, falls away. The + line was at four and a half while the hook read escaped backticks in double quotes as n to the 1.6 ("Identity hook: + escaped backticks in double quotes are read in more than linear time", issue 260); a quarter, not size 0, because a reading of nothing may take another path, as an arithmetic the hook refuses does not at size 0. All three are measured in the same run, so a loaded machine slows them alike (issue 253).""" if half.cpu < quarter.cpu: # the measures disagree, as a warm-up the quarter paid makes them: no growth is told return "" grew, grew_half = whole.cpu - quarter.cpu, half.cpu - quarter.cpu - if grew >= 4.5 * grew_half + 0.1: + if grew >= 4 * grew_half + 0.1: return "cost %.2f s of CPU at its size, against %.2f s at half and %.2f s at a quarter" % (whole.cpu, half.cpu, quarter.cpu) return "" diff --git a/tests/test_hooks.py b/tests/test_hooks.py index ec2d055f..7a68e0dd 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -169,16 +169,15 @@ def three(*cpus): self.assertEqual(faster_than_linear(*three(0.1, 0.12, 0.15)), "") # what does not grow self.assertIn("at half", faster_than_linear(*three(0.25, 1.0, 4.0))) # quadratic self.assertIn("at half", faster_than_linear(*three(0.1, 0.4, 1.6))) # quadratic, and small - self.assertEqual(faster_than_linear(*three(0.24, 0.63, 1.80)), "") # n to the 1.6, as the hook has # A second of what does not grow must not hide a quadratic past it, and a first run that costs more than the # half, as warming what the others reuse does, is not growth. self.assertIn("at half", faster_than_linear(*three(1.1, 1.4, 2.6))) self.assertEqual(faster_than_linear(*three(0.3, 0.1, 0.15)), "") self.assertEqual(faster_than_linear(*three(0.07, 0.03, 0.04)), "") self.assertEqual(faster_than_linear(*three(1.3, 1.0, 1.5)), "") # linear, after a warm-up the quarter paid - # At the edge: exactly four and a half times and the noise is past it. - self.assertIn("at half", faster_than_linear(*three(0.0, 1.0, 4.6))) - self.assertEqual(faster_than_linear(*three(0.0, 1.0, 4.59)), "") + # At the edge: exactly four times and the noise is past it. + self.assertIn("at half", faster_than_linear(*three(0.0, 1.0, 4.1))) + self.assertEqual(faster_than_linear(*three(0.0, 1.0, 4.09)), "") # The sizes a reading is run at: a quarter, half and whole, so the three spends are of three sizes on one path. self.assertEqual(sizes(16000), (4000, 8000, 16000)) self.assertEqual(sizes(901), (225, 450, 901)) @@ -1337,6 +1336,39 @@ def test_ordinary_session_commands_pass(self): "# commit the change, then git push it\ngit commit -m x"): self.assertEqual(self.codes(command), "", command) + def test_the_lexer_reads_what_shlex_reads_and_grows_a_word_in_pieces(self): + # Issue 260: the hook's lexer gives every word shlex gives, in the same order, with the same settings; and it + # grows a word by appending its pieces, so a long word is read in linear time and not copied whole at each one. + settings = dict(posix=True, punctuation_chars=shell.PUNCTUATION) + for text in ('git commit -m "' + "\\`x\\` " * 50 + '"', "a 'b c' \"d\\\"e\" f\\ g; h|i>j", "x=(1 2); y=$((3))", + "''", '""', "a\\", "a#b c", " ", "", "é ü", "a;;b&&c||d", "'\n'", '"\\$x"'): + ours, theirs = shell.Lexer(text, **settings), shlex.shlex(text, **settings) + for lex in (ours, theirs): + lex.whitespace, lex.whitespace_split, lex.commenters = " \t\r", True, "" + try: + expected = list(theirs) + except ValueError: + with self.assertRaises(ValueError, msg=text): + list(ours) + continue + got = list(ours) + self.assertEqual(got, expected, text) + self.assertTrue(all(type(word) is str for word in got), text) # the lexer's words, as strings + grown = [] + real = shell._Token.__iadd__ + with mock.patch.object(shell._Token, "__iadd__", autospec=True, + side_effect=lambda token, text: grown.append(text) or real(token, text)): + read = shell.commands("echo " + "a" * 1000) + self.assertIn("a" * 1000, str(read)) + self.assertGreaterEqual(len(grown), 999) + # The hook's own reading keeps the first WORD_KEPT characters of a word and one more, enough to tell a keyword, + # so a long word, or one dense with substitutions, is never copied whole (Muse on PR 262). + kept = "a" * shell.WORD_KEPT + self.assertEqual(shell.grown("ab", "c"), "abc") + self.assertEqual(shell.grown(kept, "b"), kept + "b") + self.assertEqual(shell.grown(kept + "b", "c"), kept + "b") + self.assertTrue(all(len(word) <= shell.WORD_KEPT for word in shell.AGAIN | {"in", "case"})) + @wall_clock def test_the_readings_issue_174_adds_run_in_linear_time(self): # Each new reading is one pass: a comment, a $'...' refused at its backslash, escaped backticks, ( and ${...}. @@ -1348,6 +1380,8 @@ def test_the_readings_issue_174_adds_run_in_linear_time(self): (lambda n: "for f in a; do cp /tmp/x " + "${f%}" * n + "; done; git push", 5000, "git_directory_written"), (lambda n: "alias x='cd .git;'\n" + "x ls\n" * n + "git push", 200, ""), + # A word dense with substitutions: each ) gave back the word before it (Muse on PR 262). + (lambda n: "echo " + "a$(x)" * n + "; git push", 2000, ""), # A run of stars, or of expansions read as one, before a tail that does not match. (lambda n: "x=a; cp /tmp/x " + "${x%}" * n + "Q/y; git push", 320, ""), (lambda n: "cp /tmp/x " + "*" * n + "Q/y; git push", 320, ""), diff --git a/tools/hooks/shell.py b/tools/hooks/shell.py index 486982b7..08820247 100644 --- a/tools/hooks/shell.py +++ b/tools/hooks/shell.py @@ -98,6 +98,48 @@ def simple_commands(text: str) -> list: return [(words, [target for _, target in redirections]) for words, redirections in commands(text)] +class _Token: + """A word shlex grows a character at a time, held as its pieces so that it grows in linear time. shlex writes + `self.token += c`, which CPython copies whole for a string held in an attribute, so a word of n characters cost n + squared over two copies: a commit message of 120,000 took 1.8 s, past the reading budget near 80,000 pairs of + escaped backticks ("Identity hook: escaped backticks in double quotes are read in more than linear time", issue 260). + """ + __slots__ = ("pieces",) + __hash__ = None + + def __init__(self, text: str = ""): + self.pieces = [text] if text else [] + + def __iadd__(self, text: str): + self.pieces.append(text) + return self + + def __bool__(self) -> bool: + return any(self.pieces) + + def __eq__(self, other) -> bool: + return str(self) == other + + def __str__(self) -> str: + return "".join(self.pieces) + + +class Lexer(shlex.shlex): + """shlex, its words grown as `_Token`s and given back as strings, so that it reads in linear time what shlex reads.""" + + @property + def token(self): + return self._token + + @token.setter + def token(self, value): + self._token = value if isinstance(value, _Token) else _Token(value) + + def get_token(self): + word = super().get_token() + return str(word) if isinstance(word, _Token) else word + + def commands(text: str, spend=None, texts: list = None, scopes: bool = False, marked: bool = False) -> list: """Each simple command in `text`, as (words, [(operator, target)]), read as simple_commands reads it. @@ -114,7 +156,7 @@ def commands(text: str, spend=None, texts: list = None, scopes: bool = False, ma if texts is not None: texts.append(text) held = _held(without_heredocs(text)) # only an unquoted ; ) or > is punctuation - lex = shlex.shlex(held, posix=True, punctuation_chars=PUNCTUATION) + lex = Lexer(held, posix=True, punctuation_chars=PUNCTUATION) lex.whitespace, lex.whitespace_split, lex.commenters = " \t\r", True, "" found, words, targets, operator = [], [], [], None # A case's patterns are data, in every arm: from `in`, and from each ;; or ;&, to the ) that ends the pattern. Each is @@ -244,6 +286,15 @@ def _marks(token: str) -> list: WORD_ENDS = " \t\r" + ENDS # Reserved words after which the next word is a command again: a ( there opens a subshell or an arithmetic command. AGAIN = {"if", "then", "else", "elif", "while", "until", "do", "!", "{", "time", "for"} +# What _held keeps of the word it is reading: it compares it only with AGAIN, "in" and "case", so the first +# WORD_KEPT characters tell it all, and a longer word is never copied or hashed whole (issue 260; Muse on PR 262). +WORD_KEPT = 64 + + +def grown(word: str, more: str) -> str: + """`word` with `more` after it, kept to WORD_KEPT characters and one more, so a longer word is still not empty and + equals no keyword.""" + return word + more if len(word) <= WORD_KEPT else word # How a $'...' with a backslash is refused: the hook decodes none, so bash and zsh cannot read it differently from the # hook (issue 174, ruling R1). Its message says to write any message to a file. ANSI_REFUSED = "a $'...' holds a backslash, which the hook does not decode; write any message to a file and use -F" @@ -316,7 +367,8 @@ def _held(text: str) -> str: out.append(ESCAPED[after]) elif after != "\n": # a backslash before a newline is a line continuation: the two lines are one out.append(c + HOLDS.get(after, after)) - starts, comments, word = starts and after == "\n", comments and after == "\n", word + c + starts, comments = starts and after == "\n", comments and after == "\n" + word = grown(word, c) i += 2 continue if quote is None: @@ -329,14 +381,16 @@ def _held(text: str) -> str: if i < braces and (c in PUNCTUATION or c == "`" or text.startswith("$(", i)): # In a ${...} each is part of the word, zsh's ${(L)f} too, and a substitution in it runs: ${x:-`cmd`}. out.append(LIVE[c] if c == "`" or c == "$" else HOLDS[c]) - starts, comments, word = False, False, word + c + starts, comments = False, False + word = grown(word, c) i += 1 continue if c == "$" and text.startswith("$'", i): # $'...' with a backslash is refused (ansi_end, ruling R1); with none it is a single-quoted word. end = ansi_end(text, i + 2) out.append(_literal(text[i + 2:end - 1])) - starts, comments, word = False, False, word + "'" + starts, comments = False, False + word = grown(word, "'") i = end continue if c == "$" and text.startswith('$"', i): @@ -354,7 +408,7 @@ def _held(text: str) -> str: elif c == ")" and opened: kept = opened.pop() if kept is not None: - words, first, word = kept[0], kept[1], kept[2] + c + words, first, word = kept[0], kept[1], grown(kept[2], c) else: if c in WORD_ENDS and word: words += 1 @@ -365,7 +419,7 @@ def _held(text: str) -> str: if c in ENDS: words, first = 0, None elif c not in " \t\r<>": - word += c + word = grown(word, c) quote = c if c in "'\"" else None out.append(c) starts = c in BREAKS and c != ")" # a ) may end a substitution inside a word: $(x)#y is one word diff --git a/tools/mutants.py b/tools/mutants.py index 900139f1..0a7586b3 100644 --- a/tools/mutants.py +++ b/tools/mutants.py @@ -3365,7 +3365,7 @@ def moved_after(rel: str, block: str, anchor: str) -> tuple: " return 2.0 * max(1.0, load / cores())\n", " return 2.0\n", ("tests.test_hooks.Budgets.test_a_slow_hook_fails_and_a_loaded_machine_does_not",)), ("a reading may grow faster than linearly", "tests/support.py", - " if grew >= 4.5 * grew_half + 0.1:\n", " if False:\n", + " if grew >= 4 * grew_half + 0.1:\n", " if False:\n", ("tests.test_hooks.Budgets.test_a_reading_that_grows_faster_than_its_size_fails",)), ("the machine's cores stand for the process's", "tests/support.py", " return max(1, len(os.sched_getaffinity(0)))\n", " return max(1, os.cpu_count() or 1)\n", @@ -3395,6 +3395,17 @@ def moved_after(rel: str, block: str, anchor: str) -> tuple: ("a message about who asked may carry a credential", "tools/check_trials.py", " return [hidden(message) for message in out]\n", " return out\n", ("tests.test_tools.Trials.test_a_credential_in_a_fixture_is_caught",)), + # Issue 260: the hook's lexer grows a word in pieces. + ("the hook reads with shlex's own lexer, copying a word at each character", "tools/hooks/shell.py", + " lex = Lexer(held, posix=True, punctuation_chars=PUNCTUATION)\n", + " lex = shlex.shlex(held, posix=True, punctuation_chars=PUNCTUATION)\n", + (IDENTITY_TESTS + "test_the_lexer_reads_what_shlex_reads_and_grows_a_word_in_pieces",)), + ("the hook's reading keeps a word whole however long", "tools/hooks/shell.py", + " return word + more if len(word) <= WORD_KEPT else word\n", " return word + more\n", + (IDENTITY_TESTS + "test_the_lexer_reads_what_shlex_reads_and_grows_a_word_in_pieces",)), + ("a lexer's word is given back in pieces", "tools/hooks/shell.py", + " return str(word) if isinstance(word, _Token) else word\n", " return word\n", + (IDENTITY_TESTS + "test_the_lexer_reads_what_shlex_reads_and_grows_a_word_in_pieces",)), ]