fix(refraction): bound the three quadratic tool regexes (#3222)

gitgalaxy/tools/cobol_to_cobol/cobol_dag_architect.py

@@ -24,6 +24,10 @@
 from gitgalaxy.tools.cobol_to_cobol.cobol_graveyard_finder import unit_header
 
 _OPEN_MODES = frozenset({"INPUT", "OUTPUT", "I-O", "EXTEND"})
+# #3222: anchor only. The operand run used to be `[^.]*\.` inside the pattern,
+# which rescans to the next period from every OPEN -- the #3205 shape. The run
+# is now sliced with str.find, which visits each character once.
+_OPEN_ANCHOR = re.compile(r"\bOPEN\s+(?=(?:INPUT|OUTPUT|I-O|EXTEND)\b)")
 
 
 def extract_lineage(filepath: Path, dead_paras: Optional[set] = None) -> Optional[dict]:
@@ -93,9 +97,20 @@ def extract_lineage(filepath: Path, dead_paras: Optional[set] = None) -> Optiona
     # We run this on the safe_content where unreachable logic is invisible.
     # One OPEN can carry several modes (`OPEN INPUT A B OUTPUT C D.`), so the
     # operand list is walked and the mode switches at each mode keyword (#3204).
-    for match in re.finditer(r"\bOPEN\s+((?:INPUT|OUTPUT|I-O|EXTEND)\b[^.]*)\.", safe_content):
+    consumed = 0
+    for match in _OPEN_ANCHOR.finditer(safe_content):
+        # Keep the old pattern's non-overlapping scan: an anchor inside the
+        # operand run of a statement we already read is not a second OPEN.
+        if match.start() < consumed:
+            continue
+        stop = safe_content.find(".", match.end())
+        if stop == -1:
+            # `[^.]*\.` required the terminator, so an unterminated tail matched
+            # nothing. Unchanged here.
+            continue
+        consumed = stop + 1
         mode = None
-        for internal_file in match.group(1).replace(",", " ").split():
+        for internal_file in safe_content[match.end() : stop].replace(",", " ").split():
             if internal_file in _OPEN_MODES:
                 mode = internal_file
             elif internal_file in file_map:

gitgalaxy/tools/cobol_to_cobol/cobol_graveyard_finder.py

@@ -60,6 +60,16 @@
 # attention key), so a unit named in one is reached with no PERFORM or GO TO.
 _CICS_HANDLE = re.compile(r"\bEXEC\s+CICS\s+HANDLE\s+(?:ABEND|CONDITION|AID)\b(.{0,600}?)\bEND-EXEC", re.S)
 _CICS_LABEL = re.compile(rf"\(\s*({_NAME})\s*\)")
+# One `REPLACING ==A== BY ==B==` pair. #3222: the leading `(?<!...)` only lets a
+# pair start at a token boundary. Without it every position inside a long name
+# is a candidate start, each one consuming the rest of the name before failing
+# on the required `BY` -- quadratic in the clause length. It changes no pair:
+# the name run is greedy and cannot stop inside a token, so a match that starts
+# mid-token always captures the same names as the boundary start does.
+_REPLACING_PAIR = re.compile(
+    r"(?<![A-Z0-9_\-])(?:==)?([A-Z0-9_\-]+)(?:==)?\s+BY\s+(?:==)?([A-Z0-9_\-]+)(?:==)?",
+    re.IGNORECASE,
+)
 
 
 @lru_cache(maxsize=8)
@@ -305,11 +315,7 @@ def replacer(match):
             # ==============================================================
             if replacing_clause:
                 # Extracts pairs, ignoring the optional == delimiters
-                pairs = re.findall(
-                    r"(?:==)?([A-Z0-9_\-]+)(?:==)?\s+BY\s+(?:==)?([A-Z0-9_\-]+)(?:==)?",
-                    replacing_clause,
-                    re.IGNORECASE,
-                )
+                pairs = _REPLACING_PAIR.findall(replacing_clause)
                 for old_val, new_val in pairs:
                     # Use negative lookarounds so we don't accidentally replace partial words with hyphens
                     cpy_content = re.sub(

gitgalaxy/tools/cobol_to_cobol/cobol_jcl_forge.py

@@ -20,6 +20,11 @@
 from pathlib import Path
 from typing import Any, Optional
 
+_FILE_ASSIGN_ANCHOR = re.compile(
+    r"SELECT\s+([A-Z0-9\-]+)\s+ASSIGN\s+(?:TO\s+)?([A-Z0-9\-]+)",
+    re.IGNORECASE,
+)
+
 
 def analyze_cobol_intent(filepath: Path) -> dict:
     """Extracts execution intent and data boundaries from legacy source code."""
@@ -52,11 +57,19 @@ def analyze_cobol_intent(filepath: Path) -> dict:
             intent["program_id"] = filepath.stem.upper()
 
         # 3. BATCH I/O: Extract all File Assignments
-        file_assign_pattern = re.compile(
-            r"SELECT\s+([A-Z0-9\-]+)\s+ASSIGN\s+(?:TO\s+)?([A-Z0-9\-]+)[^.]*\.",
-            re.IGNORECASE,
-        )
-        for match in file_assign_pattern.finditer(monolith_code):
+        # #3222: the pattern used to carry the rest of the statement as
+        # `[^.]*\.`, which rescans to the next period from every SELECT (the
+        # #3205 shape). The terminator is still required -- an ASSIGN with no
+        # period after it is not a statement -- but it is now found with
+        # str.find, which visits each character once.
+        consumed = 0
+        for match in _FILE_ASSIGN_ANCHOR.finditer(monolith_code):
+            if match.start() < consumed:
+                continue
+            stop = monolith_code.find(".", match.end())
+            if stop == -1:
+                continue
+            consumed = stop + 1
             internal_name = match.group(1).strip()
             raw_dd = match.group(2).strip()
             clean_dd = re.sub(r"^(?:UT|UR)-S-", "", raw_dd)

tests/cobol_mainframe/test_cobol_dag_architect.py

@@ -1,4 +1,5 @@
 import sys
+import time
 from unittest.mock import patch
 
 import pytest
@@ -200,3 +201,45 @@ def test_masking_covers_a_dead_section_in_sequence_numbered_source(tmp_path):
 
     assert lineage["inputs"] == {"LIVEDD"}
     assert lineage["outputs"] == set()
+
+
+# ==============================================================================
+# #3222: the OPEN operand run is sliced, not rescanned
+# ==============================================================================
+def test_open_operand_run_is_linear_on_unterminated_input(tmp_path):
+    """`[^.]*\\.` inside the pattern rescanned to the end of the program from every
+    OPEN. 20k unterminated OPENs cost 24s before, ~1ms after."""
+    pgm = tmp_path / "BIG.cbl"
+    pgm.write_text(
+        "       PROGRAM-ID. BIG.\n       PROCEDURE DIVISION.\n" + "           OPEN INPUT WS-F\n" * 20000,
+        encoding="utf-8",
+    )
+
+    start = time.perf_counter()
+    lineage = dag_module.extract_lineage(pgm)
+    elapsed = time.perf_counter() - start
+
+    # No period anywhere, so no OPEN is a statement -- the old pattern's rule, kept.
+    assert not lineage["inputs"] and not lineage["outputs"]
+    assert elapsed < 5.0, f"OPEN scan took {elapsed:.2f}s on 20k unterminated OPENs"
+
+
+def test_open_statements_still_read_every_mode_after_anchoring(tmp_path):
+    """The anchored scan keeps #3204's multi-mode walk and stops at the period."""
+    pgm = tmp_path / "IO.cbl"
+    pgm.write_text(
+        "       PROGRAM-ID. IO.\n"
+        "       FILE-CONTROL.\n"
+        "           SELECT F-IN ASSIGN TO DDIN.\n"
+        "           SELECT F-OUT ASSIGN TO DDOUT.\n"
+        "           SELECT F-UPD ASSIGN TO DDUPD.\n"
+        "       PROCEDURE DIVISION.\n"
+        "           OPEN INPUT F-IN OUTPUT F-OUT.\n"
+        "           OPEN I-O F-UPD.\n",
+        encoding="utf-8",
+    )
+
+    lineage = dag_module.extract_lineage(pgm)
+
+    assert sorted(lineage["inputs"]) == ["DDIN", "DDUPD"]
+    assert sorted(lineage["outputs"]) == ["DDOUT", "DDUPD"]

tests/cobol_mainframe/test_cobol_graveyard_finder.py

@@ -1,4 +1,5 @@
 import sys
+import time
 from unittest.mock import patch
 
 # IMPORTANT: Adjust this path to match exactly where your file is located
@@ -343,3 +344,56 @@ def test_cics_handle_labels_and_comments(tmp_path):
         "       COMMENTED-OUT.",
         "           DISPLAY 'NEVER'.",
     ) == {"COMMENTED-OUT"}
+
+
+# ==============================================================================
+# #3222: the REPLACING pair scan starts only at a token boundary
+# ==============================================================================
+def test_replacing_pairs_are_linear_in_the_clause_length():
+    """Without the boundary lookbehind, every position inside a long name is a
+    candidate start that consumes the rest of the name before failing on the
+    required `BY`. 20k name characters cost 10s before, ~1ms after."""
+    start = time.perf_counter()
+    pairs = graveyard_module._REPLACING_PAIR.findall("A" * 20000)
+    elapsed = time.perf_counter() - start
+
+    assert pairs == []
+    assert elapsed < 5.0, f"REPLACING pair scan took {elapsed:.2f}s on a 20k-character name"
+
+
+def test_replacing_pairs_are_unchanged_by_the_boundary_anchor():
+    """The lookbehind captures the same pairs: a name run is greedy and cannot stop
+    inside a token, so a mid-token start would report the same two names anyway."""
+    clause = " ==TAG== BY ==WS-CUST== LEADING BY TRAILING ==A-1== BY ==B_2== "
+
+    assert graveyard_module._REPLACING_PAIR.findall(clause) == [
+        ("TAG", "WS-CUST"),
+        ("LEADING", "TRAILING"),
+        ("A-1", "B_2"),
+    ]
+    # Pre-existing and unchanged: `:` is outside the name class, so a `:TAG:`
+    # pseudo-text name matches neither the old pattern nor this one.
+    assert graveyard_module._REPLACING_PAIR.findall("==:TAG:== BY ==WS-CUST==") == []
+
+
+def test_replacing_clause_still_substitutes_into_the_copybook(tmp_path):
+    """End to end: the extracted pair renames the whole token, so the aliased field
+    is the one the dead-code X-ray sees referenced."""
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    (repo / "TAGCOPY.cpy").write_text("       01 TAG-BALANCE PIC 9(5).\n", encoding="utf-8")
+    pgm = repo / "PGM.cbl"
+    pgm.write_text(
+        "       PROGRAM-ID. PGM.\n"
+        "       WORKING-STORAGE SECTION.\n"
+        "           COPY TAGCOPY REPLACING ==TAG-BALANCE== BY ==WS-CUST-BALANCE==.\n"
+        "       PROCEDURE DIVISION.\n"
+        "       A010.\n"
+        "           MOVE 1 TO WS-CUST-BALANCE.\n",
+        encoding="utf-8",
+    )
+
+    resolved = graveyard_module.resolve_copybooks(pgm.read_text(), pgm, copybook_root=repo)
+    assert "01 WS-CUST-BALANCE" in resolved
+    assert "01 TAG-BALANCE" not in resolved
+    assert graveyard_module.x_ray_dead_code(pgm, copybook_root=repo)["orphaned_vars"] == set()

tests/cobol_mainframe/test_cobol_jcl_forge.py

@@ -1,4 +1,5 @@
 import sys
+import time
 from unittest.mock import patch
 
 # IMPORTANT: Adjust this path to match exactly where your file is located
@@ -163,3 +164,42 @@ def test_unterminated_exec_on_large_input_is_linear(tmp_path):
     pgm.write_text("       PROCEDURE DIVISION.\n" + "           EXEC CICS X\n" * 20000, encoding="utf-8")
 
     assert forge_module.analyze_cobol_intent(pgm)["cics_calls"] == 20000
+
+
+# ==============================================================================
+# #3222: the SELECT/ASSIGN statement tail is sliced, not rescanned
+# ==============================================================================
+def test_select_assign_is_linear_on_unterminated_input(tmp_path):
+    """`[^.]*\\.` after ASSIGN rescanned to the next period from every SELECT -- the
+    #3205 shape. 20k period-less SELECTs cost 21s before, ~1ms after."""
+    pgm = tmp_path / "BIG.cbl"
+    pgm.write_text(
+        "       FILE-CONTROL.\n" + "           SELECT F1 ASSIGN TO DD1\n" * 20000,
+        encoding="utf-8",
+    )
+
+    start = time.perf_counter()
+    intent = forge_module.analyze_cobol_intent(pgm)
+    elapsed = time.perf_counter() - start
+
+    # The terminator is still required, so none of these is a statement.
+    assert intent["files_requested"] == []
+    assert elapsed < 5.0, f"SELECT scan took {elapsed:.2f}s on 20k unterminated SELECTs"
+
+
+def test_select_assign_still_reads_every_statement_after_anchoring(tmp_path):
+    """The anchored scan keeps the UT-/UR- strip and the one-statement-per-period rule."""
+    pgm = tmp_path / "IO.cbl"
+    pgm.write_text(
+        "       FILE-CONTROL.\n"
+        "           SELECT F-IN ASSIGN TO UT-S-DDIN\n"
+        "               ORGANIZATION IS SEQUENTIAL.\n"
+        "           SELECT F-OUT ASSIGN DDOUT\n"
+        "               FILE STATUS IS WS-ST.\n",
+        encoding="utf-8",
+    )
+
+    assert forge_module.analyze_cobol_intent(pgm)["files_requested"] == [
+        {"internal": "F-IN", "dd_name": "DDIN"},
+        {"internal": "F-OUT", "dd_name": "DDOUT"},
+    ]

tests/cobol_mainframe/tool_regex_redos_baseline.json

@@ -1,37 +1,4 @@
 {
-  "offenders": [
-    {
-      "module": "gitgalaxy/tools/cobol_to_cobol/cobol_dag_architect.py",
-      "pattern": "\\bOPEN\\s+((?:INPUT|OUTPUT|I-O|EXTEND)\\b[^.]*)\\.",
-      "flags": 0,
-      "method": "finditer",
-      "reason": "slow",
-      "unit": "OPEN INPUT ",
-      "seconds": 0.4557,
-      "chars": 40000,
-      "note": "Runs over the whole masked program. `[^.]*` stops only at a period, so a program with many OPENs and no period between them (or a long unterminated tail) rescans to the end from every OPEN. Fix tracked in #3222."
-    },
-    {
-      "module": "gitgalaxy/tools/cobol_to_cobol/cobol_graveyard_finder.py",
-      "pattern": "(?:==)?([A-Z0-9_\\-]+)(?:==)?\\s+BY\\s+(?:==)?([A-Z0-9_\\-]+)(?:==)?",
-      "flags": 2,
-      "method": "findall",
-      "reason": "slow",
-      "unit": "A",
-      "seconds": 0.3783,
-      "chars": 4000,
-      "note": "Runs over one COPY statement's REPLACING clause, so real inputs are short; the unbounded name before a required `BY` is still quadratic in the clause length. Fix tracked in #3222."
-    },
-    {
-      "module": "gitgalaxy/tools/cobol_to_cobol/cobol_jcl_forge.py",
-      "pattern": "SELECT\\s+([A-Z0-9\\-]+)\\s+ASSIGN\\s+(?:TO\\s+)?([A-Z0-9\\-]+)[^.]*\\.",
-      "flags": 2,
-      "method": "findall",
-      "reason": "quadratic (15.7x for 4x the input)",
-      "unit": "SELECT X ASSIGN X ",
-      "seconds": 0.6363,
-      "chars": 160000,
-      "note": "Runs over the whole program. `[^.]*\\\\.` after ASSIGN rescans to the next period from every SELECT: the #3205 shape with a smaller constant. Fix tracked in #3222."
-    }
-  ]
+  "offenders": [],
+  "about": "#3214 ReDoS sweep baseline: known-slow refraction-tool regexes, so `--ci` fails only on a new one. Empty since #3222 bounded the last three (OPEN operands, SELECT/ASSIGN, REPLACING pairs). Add an entry only with an issue tracking its fix."
 }