Fix #4266: Extract SYNC, JUSTIFIED, BLANK WHEN ZERO, RENAMES, and resolve qualified names in layout model

gitgalaxy/core/mainframe_boundary.py

@@ -481,6 +481,38 @@
     + r"){0,15})",
     re.I,
 )
+# #4266: the clauses that shape an item without naming storage -- `SYNCHRONIZED [LEFT|RIGHT]` (slack bytes
+# before a binary / floating / pointer item), `JUSTIFIED [RIGHT]` and `BLANK [WHEN] ZERO[S|ES]` -- and a 66
+# entry's `RENAMES a [THRU b]` (qualifiers kept, `IN` read as `OF`). Each is read from the entry's own text
+# (before its period, literals blanked) and bounded by COBOL-name boundaries, so `WS-SYNC-FLAG` is no clause.
+# A COBOL-word character (national letters and #3991's full-width digits / hyphens included), so a clause
+# keyword inside a longer name (`WS-SYNC-FLAG`, `#SYNC`) is not read as the clause.
+_CLAUSE_EDGE = r"[A-Z" + NATIONAL + WIDE_DIGITS + WIDE_HYPHENS + r"0-9-]"
+_SYNC_CLAUSE = re.compile(
+    "(?<!" + _CLAUSE_EDGE + r")SYNC(?:HRONIZED)?(?:[ \t\n\u3000]+(LEFT|RIGHT))?(?!" + _CLAUSE_EDGE + ")", re.I
+)
+_JUSTIFIED_CLAUSE = re.compile("(?<!" + _CLAUSE_EDGE + r")JUST(?:IFIED)?(?!" + _CLAUSE_EDGE + ")", re.I)
+_BLANK_WHEN_ZERO_CLAUSE = re.compile(
+    "(?<!" + _CLAUSE_EDGE + r")BLANK[ \t\n\u3000]+(?:WHEN[ \t\n\u3000]+)?ZERO(?:E?S)?(?!" + _CLAUSE_EDGE + ")", re.I
+)
+_RENAMES_OPERAND = _COBOL_NAME + r"(?:[ \t\n\u3000]+(?:OF|IN)[ \t\n\u3000]+" + _COBOL_NAME + r"){0,15}"
+_RENAMES_CLAUSE = re.compile(
+    "(?<!"
+    + _CLAUSE_EDGE
+    + r")RENAMES[ \t\n\u3000]+("
+    + _RENAMES_OPERAND
+    + r")(?:[ \t\n\u3000]+(?:THRU|THROUGH)[ \t\n\u3000]+("
+    + _RENAMES_OPERAND
+    + r"))?",
+    re.I,
+)
+
+
+def _renames_operand(text: str) -> str:
+    """`A IN B` / `A  OF\n B` -> `A OF B`: one spelling, as galaxy_ir's operand reader splits it."""
+    return " ".join("OF" if w == "IN" else w for w in text.upper().split())
+
+
 # `REDEFINES <name>` -- the storage-overlay pointer.
 _REDEFINES_CLAUSE = re.compile(
     r"\bREDEFINES[ \t\n\u3000]+([A-Z" + NATIONAL + r"][A-Z" + NATIONAL + WIDE_DIGITS + WIDE_HYPHENS + r"0-9-]*)", re.I
@@ -1233,6 +1265,15 @@ def _context(offset: int) -> tuple[Optional[str], Optional[str]]:
             depending = " ".join(dep_match.group(1).upper().split()) if dep_match else None
         redefines_match = _REDEFINES_CLAUSE.search(window)
         redefines = redefines_match.group(1).upper() if redefines_match else None
+        # #4266: SYNC / JUSTIFIED / BLANK WHEN ZERO, and a 66 entry's RENAMES range
+        sync_match = _SYNC_CLAUSE.search(entry)
+        sync = ("SYNC " + sync_match.group(1).upper() if sync_match.group(1) else "SYNC") if sync_match else None
+        renames_match = _RENAMES_CLAUSE.search(entry) if level == 66 else None
+        renames = None
+        if renames_match:
+            renames = _renames_operand(renames_match.group(1))
+            if renames_match.group(2):
+                renames += " THRU " + _renames_operand(renames_match.group(2))
         value_match = _VALUE_CLAUSE.search(window)
         value = None
         if value_match:
@@ -1305,6 +1346,11 @@ def _context(offset: int) -> tuple[Optional[str], Optional[str]]:
                 # #3694: presence-keyed likewise -- only a SEPARATE sign is recorded.
                 # (1 a TRAILING separate sign, 2 a LEADING one)
                 **({"sign_separate": _sign_separate(window)} if _sign_separate(window) else {}),
+                # #4266: presence-keyed likewise -- an entry without the clause keeps its pre-#4266 shape.
+                **({"sync": sync} if sync else {}),
+                **({"justified": True} if _JUSTIFIED_CLAUSE.search(entry) else {}),
+                **({"blank_when_zero": True} if _BLANK_WHEN_ZERO_CLAUSE.search(entry) else {}),
+                **({"renames": renames} if renames else {}),
             }
         )
     if section_copies is not None:

gitgalaxy/core/state_rehydrator.py

@@ -578,6 +578,13 @@ def load_state(self, repo_name: str, commit_hash: Optional[str] = None) -> Optio
                     if _has_table(cursor, "record_data") and _has_column(cursor, "record_data", "copy_replacing")
                     else "NULL"
                 )
+                # #4266: SYNC / JUSTIFIED / BLANK WHEN ZERO / RENAMES likewise.
+                layout_cols = ", ".join(
+                    f"rd.{c} AS {c}"
+                    if _has_table(cursor, "record_data") and _has_column(cursor, "record_data", c)
+                    else f"NULL AS {c}"
+                    for c in ("sync", "justified", "blank_when_zero", "renames")
+                )
                 records_by_file = _restore_child_table(
                     cursor,
                     repo_name,
@@ -587,7 +594,8 @@ def load_state(self, repo_name: str, commit_hash: Optional[str] = None) -> Optio
                     "rd.level_number AS level, rd.item_name AS name, rd.pic, rd.usage, rd.occurs_min, "
                     "rd.occurs_max, rd.occurs_depending_on, rd.redefines, rd.value_literal AS value, "
                     f"rd.line_number AS line, {attributes_col} AS attributes, {copy_col} AS copy_members, "
-                    f"{sign_col} AS sign_separate, {lib_col} AS copy_libraries, {repl_col} AS copy_replacing "
+                    f"{sign_col} AS sign_separate, {lib_col} AS copy_libraries, {repl_col} AS copy_replacing, "
+                    f"{layout_cols} "
                     "FROM record_data rd JOIN file_data fd ON rd.file_id = fd.id "
                     "WHERE fd.repo_name = ? AND fd.commit_hash = ? ORDER BY rd.file_id, rd.ordinal",
                     lambda r: {
@@ -610,6 +618,10 @@ def load_state(self, repo_name: str, commit_hash: Optional[str] = None) -> Optio
                         **({"sign_separate": int(r["sign_separate"])} if r["sign_separate"] else {}),
                         **({"copy_libraries": r["copy_libraries"]} if r["copy_libraries"] else {}),
                         **({"copy_replacing": r["copy_replacing"]} if r["copy_replacing"] else {}),
+                        **({"sync": r["sync"]} if r["sync"] else {}),
+                        **({"justified": True} if r["justified"] else {}),
+                        **({"blank_when_zero": True} if r["blank_when_zero"] else {}),
+                        **({"renames": r["renames"]} if r["renames"] else {}),
                     },
                 )
                 # #3211-followup: the CSD transaction definitions, restored per

gitgalaxy/recorders/record_keeper.py

@@ -1096,6 +1096,10 @@ def record_mission(
                 sign_separate INTEGER,
                 copy_libraries TEXT,
                 copy_replacing TEXT,
+                sync TEXT,
+                justified INTEGER,
+                blank_when_zero INTEGER,
+                renames TEXT,
                 FOREIGN KEY(file_id) REFERENCES file_data(id) ON DELETE CASCADE
             )
         """)
@@ -1119,6 +1123,13 @@ def record_mission(
         # pseudo-text as written minus its `==` delimiters. NULL when no COPY on the entry replaces
         # anything. The copybook keeps its own (tagged) names; the reader applies the REPLACING.
         _ensure_columns(cursor, "record_data", ["copy_replacing TEXT"])
+        # #4266: `sync` -- 'SYNC' / 'SYNC LEFT' / 'SYNC RIGHT' when a COBOL item codes SYNCHRONIZED (slack
+        # bytes before it are the reader's job, galaxy_ir); `justified` / `blank_when_zero` -- 1 when it codes
+        # JUSTIFIED / BLANK WHEN ZERO; `renames` -- a 66 entry's `A [THRU B]` (qualifiers as `X OF Y`).
+        # All NULL otherwise.
+        _ensure_columns(
+            cursor, "record_data", ["sync TEXT", "justified INTEGER", "blank_when_zero INTEGER", "renames TEXT"]
+        )
         cursor.execute("CREATE INDEX IF NOT EXISTS idx_record_file_id ON record_data(file_id);")
         cursor.execute("CREATE INDEX IF NOT EXISTS idx_record_snapshot ON record_data(repo_name, commit_hash);")
 
@@ -3054,6 +3065,10 @@ def _seq_base(table: str) -> int:
                 "sign_separate",
                 "copy_libraries",
                 "copy_replacing",
+                "sync",
+                "justified",
+                "blank_when_zero",
+                "renames",
             ),
             "record_layouts",
             lambda it: (
@@ -3076,6 +3091,10 @@ def _seq_base(table: str) -> int:
                 int(it["sign_separate"]) if it.get("sign_separate") else None,  # #3694: 1 trailing, 2 leading
                 it.get("copy_libraries"),  # #4265
                 it.get("copy_replacing"),  # #4265
+                it.get("sync"),  # #4266
+                1 if it.get("justified") else None,  # #4266
+                1 if it.get("blank_when_zero") else None,  # #4266
+                it.get("renames"),  # #4266
             ),
         )
 

gitgalaxy/tools/cobol_to_cobol/galaxy_ir.py

@@ -349,6 +349,13 @@ class EngineDataItem:
     # before the digits when `sign_leading` (SIGN LEADING SEPARATE), else after them.
     sign_separate: bool = False
     sign_leading: bool = False
+    # #4266: SYNCHRONIZED ('SYNC' / 'SYNC LEFT' / 'SYNC RIGHT'; None without it) -- a binary, floating or
+    # pointer item then starts on its boundary, after slack bytes (_sync_boundary); JUSTIFIED and BLANK
+    # WHEN ZERO as coded; a 66 entry's RENAMES operands (`A` / `A THRU B`, qualifiers as `X OF Y`).
+    sync: str | None = None
+    justified: bool = False
+    blank_when_zero: bool = False
+    renames: str | None = None
     # #3820: {symbol: currency string} for each declared currency symbol the PIC
     # uses; a string of None means the estate declares that symbol with different
     # strings, so its width is unknown. Empty when the PIC uses none.
@@ -2231,6 +2238,7 @@ def _walk(
                                 size += wide - region[1]
                                 region[1] = wide
                         continue
+                    size += _slack(kid, usage, offset + size)  # #4266: SYNCHRONIZED
                     kid_width = _walk(kid_file, kid, offset + size, kid_depth, sink, usage)
                     region = [offset + size, kid_width, kid.name.upper()]
                     size += kid_width
@@ -4924,6 +4932,8 @@ def walk(
                             size = None if size is None else size + wide - region[1]
                             region[1] = wide
                         continue
+                    if size is not None:
+                        size += _slack(kid, usage, offset + size)  # #4266: SYNCHRONIZED
                     at[kid.name.upper()] = offset + (size or 0)
                     width = walk(kid_file, kid, key, offset + (size or 0), depth + 1, None, (it.name, *path), usage)
                     region = [kid.name.upper(), width]
@@ -4978,12 +4988,57 @@ def fill(it: EngineDataItem, path: tuple, lay=lay, key=key, children=children) -
                 if id(root) not in spans:
                     ext = self._copy_extension(ef, cb) if roots[-1:] == [root] else None
                     walk(cb, root, (cb.file_path, root.name, None), 0, 0, ext, ())
+        self._renames_spans(ef, spans, paths, placed)
         cache[(ef.file_path, id(ef))] = (spans, ef)  # ef held so its id is never reused
         self.__dict__.setdefault("_span_paths", {})[ef.file_path] = paths
         self.__dict__.setdefault("_span_items", {})[ef.file_path] = items
         self.__dict__.setdefault("_span_placed", {})[ef.file_path] = placed
         return spans
 
+    def _renames_spans(self, ef: EngineFile, spans: dict, paths: dict, placed: dict) -> None:
+        """#4266: lay out each 66 RENAMES entry `ef` can see as the byte range it names -- from the first
+        byte of `A` to the last of `B` (`RENAMES A THRU B`), or `A`'s own bytes (`RENAMES A`) -- in the
+        record the entry ends (the 01 it follows). Each operand must name exactly one placed item of
+        that record (its qualifiers matched against the storage ancestors, as `_operand_span` does);
+        otherwise the entry stays unplaced. The entry answers to its own name, qualified by its record."""
+        owners = [ef] + [
+            cb for cb in self._copy_files(ef) if nfc(Path(cb.file_path).stem.upper()) not in ef.replaced_members
+        ]
+        for owner in owners:
+            entries = [*owner.data_items, *owner.copied_items]
+            by_ordinal = {it.ordinal: it for it in entries}
+            for it in entries:
+                if it.level != 66 or not it.renames:
+                    continue
+                root, hops = by_ordinal.get(it.parent_ordinal), 0
+                while root is not None and root.parent_ordinal is not None and hops < 64:
+                    root, hops = by_ordinal.get(root.parent_ordinal), hops + 1
+                if root is None or id(root) not in spans:
+                    continue
+                key = spans[id(root)][0]
+                ends = []
+                for operand in it.renames.split(" THRU "):
+                    parts = operand.split(" OF ")
+                    hits = {
+                        sp
+                        for at in placed.values()
+                        for x, sp, path in at
+                        if x.name == parts[0]
+                        and x.level not in (66, 88)
+                        and sp[0] == key
+                        and _in_order(parts[1:], path)
+                    }
+                    if len(hits) != 1:
+                        break
+                    ends.append(next(iter(hits)))
+                else:
+                    (_, start, first, _), (_, last_at, last, _) = ends[0], ends[-1]
+                    width = None if first is None or last is None or last_at < start else last_at + last - start
+                    span = (key, start, width, width)
+                    spans[id(it)] = span
+                    paths[id(it)] = (root.name,)
+                    placed.setdefault(id(it), []).append((it, span, (root.name,)))
+
     def _name_at(self, file_path: str, span: dict) -> Optional[str]:
         """The most specific item of program `file_path` at `span`: the deepest one
         with exactly its offset and width, else the smallest one containing it."""
@@ -5037,7 +5092,7 @@ def visible(sp: Optional[tuple]) -> bool:
                     (it, sp)
                     for at in self.__dict__["_span_placed"][ef.file_path].values()
                     for it, sp, path in at
-                    if it.name == parts[0] and it.level not in (66, 88) and _in_order(parts[1:], path) and visible(sp)
+                    if it.name == parts[0] and it.level != 88 and _in_order(parts[1:], path) and visible(sp)
                 ]
             if program:
                 # Every placement in the program's own storage: a copybook COPYed by several
@@ -5047,8 +5102,16 @@ def visible(sp: Optional[tuple]) -> bool:
                 for _, it, _ in self._find_item(ef, parts[0], None, program, exact=True):
                     at = [sp for _, sp, _ in placed.get(id(it), []) if visible(sp)]
                     out += [(it, sp) for sp in at] if at else ([] if placed.get(id(it)) else [(it, None)])
-                return out
-            return [(it, spans.get(id(it))) for _, it, _ in self._find_item(ef, parts[0], None)]
+                cands = out
+            else:
+                cands = [(it, spans.get(id(it))) for _, it, _ in self._find_item(ef, parts[0], None)]
+            # #4266: a 66 RENAMES entry `_storage_spans` placed answers to its name too.
+            return cands or [
+                (it, sp)
+                for at in self.__dict__["_span_placed"][ef.file_path].values()
+                for it, sp, _ in at
+                if it.level == 66 and it.name == parts[0] and visible(sp)
+            ]
 
         found: list = []
         for scope in self._program_scope(ef, program):  # the program, then its GLOBAL candidates
@@ -5628,6 +5691,32 @@ def _item_class(item: EngineDataItem, inherited: str | None = None) -> str:
     return "9"
 
 
+def _sync_boundary(item: EngineDataItem, usage: str | None) -> int:
+    """#4266: the boundary a SYNCHRONIZED elementary item starts on, in bytes from the start of its record
+    (which is doubleword-aligned), per IBM Enterprise COBOL's SYNCHRONIZED clause: a binary item of up to 4
+    digits a halfword, of more a fullword (an 8-byte S9(10)-S9(18) one too); COMP-1, INDEX and the
+    pointers a fullword (LP(32)); COMP-2 a doubleword. 1 -- no slack -- for any other item (DISPLAY,
+    packed, national: SYNC is documentation there), a group, PL/I, or no SYNC at all."""
+    if not item.sync or item.attributes is not None or not _is_elementary(item, usage):
+        return 1
+    u = (usage or "DISPLAY").upper()
+    if u in ("COMP-2", "COMPUTATIONAL-2"):
+        return 8
+    if u in ("COMP-1", "COMPUTATIONAL-1", "INDEX", "POINTER", "PROCEDURE-POINTER", "FUNCTION-POINTER"):
+        return 4
+    if item.pic and _item_class(item, usage) == "B":
+        width = _elementary_bytes(item, usage)
+        return 1 if width is None else 2 if width <= 2 else 4
+    return 1
+
+
+def _slack(item: EngineDataItem, inherited: str | None, at: int) -> int:
+    """#4266: the slack bytes the compiler puts before `item` when it would start at record offset
+    `at` (0 unless it is SYNCHRONIZED and `at` is off its boundary). `inherited` is the enclosing
+    group's USAGE (#4525). Slack between the occurrences of a table is not modelled."""
+    return -at % _sync_boundary(item, _usage_under(item, inherited))
+
+
 def _elementary_bytes(item: EngineDataItem, inherited: str | None = None) -> int | None:
     """One occurrence's storage width of an elementary item, or None when unknown.
     `inherited` is the enclosing group's USAGE (#4525), used when the item has none."""
@@ -6009,6 +6098,11 @@ def load_galaxy_ir(db_path: Path, repo_name: Optional[str] = None) -> GalaxyIR:
             # #4265: `copy_libraries` likewise.
             lib_col = "copy_libraries" if _has_column(cur, "record_data", "copy_libraries") else "NULL"
             repl_col = "copy_replacing" if _has_column(cur, "record_data", "copy_replacing") else "NULL"
+            # #4266: SYNC / JUSTIFIED / BLANK WHEN ZERO / RENAMES likewise (an older DB lays out no slack bytes).
+            layout_cols = ", ".join(
+                c if _has_column(cur, "record_data", c) else "NULL"
+                for c in ("sync", "justified", "blank_when_zero", "renames")
+            )
             for (
                 file_id,
                 ordinal,
@@ -6030,10 +6124,14 @@ def load_galaxy_ir(db_path: Path, repo_name: Optional[str] = None) -> GalaxyIR:
                 sign_sep,
                 copy_libs,
                 copy_repl,
+                sync,
+                justified,
+                blank_zero,
+                renames,
             ) in cur.execute(
                 "SELECT file_id, ordinal, parent_ordinal, level_number, item_name, section, fd_name, pic, "  # noqa: S608 -- attributes_col is one of two literals; values are bound
                 "usage, occurs_min, occurs_max, occurs_depending_on, redefines, value_literal, line_number, "
-                f"{attributes_col}, {copy_col}, {sign_col}, {lib_col}, {repl_col} FROM record_data "
+                f"{attributes_col}, {copy_col}, {sign_col}, {lib_col}, {repl_col}, {layout_cols} FROM record_data "
                 "WHERE repo_name = ? AND commit_hash = ? "
                 "ORDER BY file_id, ordinal",
                 (repo_name, commit_hash),
@@ -6062,6 +6160,10 @@ def load_galaxy_ir(db_path: Path, repo_name: Optional[str] = None) -> GalaxyIR:
                         sign_leading=sign_sep == 2,
                         copy_libraries=copy_libs,
                         copy_replacing=copy_repl,
+                        sync=sync or None,
+                        justified=bool(justified),
+                        blank_when_zero=bool(blank_zero),
+                        renames=renames or None,
                     )
                 )
             # Thread children onto parents and collect the roots. `data_items` is