diff --git a/gitgalaxy/core/mainframe_boundary.py b/gitgalaxy/core/mainframe_boundary.py index 150790b65e..399ea867f3 100644 --- a/gitgalaxy/core/mainframe_boundary.py +++ b/gitgalaxy/core/mainframe_boundary.py @@ -481,6 +481,38 @@ + r"){0,15})", re.I, ) +# #4266: the clauses that shape an item without naming storage -- `SYNCHRONIZED [LEFT|RIGHT]` (slack bytes +# before a binary / floating / pointer item), `JUSTIFIED [RIGHT]` and `BLANK [WHEN] ZERO[S|ES]` -- and a 66 +# entry's `RENAMES a [THRU b]` (qualifiers kept, `IN` read as `OF`). Each is read from the entry's own text +# (before its period, literals blanked) and bounded by COBOL-name boundaries, so `WS-SYNC-FLAG` is no clause. +# A COBOL-word character (national letters and #3991's full-width digits / hyphens included), so a clause +# keyword inside a longer name (`WS-SYNC-FLAG`, `#SYNC`) is not read as the clause. +_CLAUSE_EDGE = r"[A-Z" + NATIONAL + WIDE_DIGITS + WIDE_HYPHENS + r"0-9-]" +_SYNC_CLAUSE = re.compile( + "(? str: + """`A IN B` / `A OF\n B` -> `A OF B`: one spelling, as galaxy_ir's operand reader splits it.""" + return " ".join("OF" if w == "IN" else w for w in text.upper().split()) + + # `REDEFINES ` -- the storage-overlay pointer. _REDEFINES_CLAUSE = re.compile( r"\bREDEFINES[ \t\n\u3000]+([A-Z" + NATIONAL + r"][A-Z" + NATIONAL + WIDE_DIGITS + WIDE_HYPHENS + r"0-9-]*)", re.I @@ -1233,6 +1265,15 @@ def _context(offset: int) -> tuple[Optional[str], Optional[str]]: depending = " ".join(dep_match.group(1).upper().split()) if dep_match else None redefines_match = _REDEFINES_CLAUSE.search(window) redefines = redefines_match.group(1).upper() if redefines_match else None + # #4266: SYNC / JUSTIFIED / BLANK WHEN ZERO, and a 66 entry's RENAMES range + sync_match = _SYNC_CLAUSE.search(entry) + sync = ("SYNC " + sync_match.group(1).upper() if sync_match.group(1) else "SYNC") if sync_match else None + renames_match = _RENAMES_CLAUSE.search(entry) if level == 66 else None + renames = None + if renames_match: + renames = _renames_operand(renames_match.group(1)) + if renames_match.group(2): + renames += " THRU " + _renames_operand(renames_match.group(2)) value_match = _VALUE_CLAUSE.search(window) value = None if value_match: @@ -1305,6 +1346,11 @@ def _context(offset: int) -> tuple[Optional[str], Optional[str]]: # #3694: presence-keyed likewise -- only a SEPARATE sign is recorded. # (1 a TRAILING separate sign, 2 a LEADING one) **({"sign_separate": _sign_separate(window)} if _sign_separate(window) else {}), + # #4266: presence-keyed likewise -- an entry without the clause keeps its pre-#4266 shape. + **({"sync": sync} if sync else {}), + **({"justified": True} if _JUSTIFIED_CLAUSE.search(entry) else {}), + **({"blank_when_zero": True} if _BLANK_WHEN_ZERO_CLAUSE.search(entry) else {}), + **({"renames": renames} if renames else {}), } ) if section_copies is not None: diff --git a/gitgalaxy/core/state_rehydrator.py b/gitgalaxy/core/state_rehydrator.py index 009702cc14..32c5c3ca7e 100644 --- a/gitgalaxy/core/state_rehydrator.py +++ b/gitgalaxy/core/state_rehydrator.py @@ -578,6 +578,13 @@ def load_state(self, repo_name: str, commit_hash: Optional[str] = None) -> Optio if _has_table(cursor, "record_data") and _has_column(cursor, "record_data", "copy_replacing") else "NULL" ) + # #4266: SYNC / JUSTIFIED / BLANK WHEN ZERO / RENAMES likewise. + layout_cols = ", ".join( + f"rd.{c} AS {c}" + if _has_table(cursor, "record_data") and _has_column(cursor, "record_data", c) + else f"NULL AS {c}" + for c in ("sync", "justified", "blank_when_zero", "renames") + ) records_by_file = _restore_child_table( cursor, repo_name, @@ -587,7 +594,8 @@ def load_state(self, repo_name: str, commit_hash: Optional[str] = None) -> Optio "rd.level_number AS level, rd.item_name AS name, rd.pic, rd.usage, rd.occurs_min, " "rd.occurs_max, rd.occurs_depending_on, rd.redefines, rd.value_literal AS value, " f"rd.line_number AS line, {attributes_col} AS attributes, {copy_col} AS copy_members, " - f"{sign_col} AS sign_separate, {lib_col} AS copy_libraries, {repl_col} AS copy_replacing " + f"{sign_col} AS sign_separate, {lib_col} AS copy_libraries, {repl_col} AS copy_replacing, " + f"{layout_cols} " "FROM record_data rd JOIN file_data fd ON rd.file_id = fd.id " "WHERE fd.repo_name = ? AND fd.commit_hash = ? ORDER BY rd.file_id, rd.ordinal", lambda r: { @@ -610,6 +618,10 @@ def load_state(self, repo_name: str, commit_hash: Optional[str] = None) -> Optio **({"sign_separate": int(r["sign_separate"])} if r["sign_separate"] else {}), **({"copy_libraries": r["copy_libraries"]} if r["copy_libraries"] else {}), **({"copy_replacing": r["copy_replacing"]} if r["copy_replacing"] else {}), + **({"sync": r["sync"]} if r["sync"] else {}), + **({"justified": True} if r["justified"] else {}), + **({"blank_when_zero": True} if r["blank_when_zero"] else {}), + **({"renames": r["renames"]} if r["renames"] else {}), }, ) # #3211-followup: the CSD transaction definitions, restored per diff --git a/gitgalaxy/recorders/record_keeper.py b/gitgalaxy/recorders/record_keeper.py index 405cf4dedd..1443d0d66d 100644 --- a/gitgalaxy/recorders/record_keeper.py +++ b/gitgalaxy/recorders/record_keeper.py @@ -1096,6 +1096,10 @@ def record_mission( sign_separate INTEGER, copy_libraries TEXT, copy_replacing TEXT, + sync TEXT, + justified INTEGER, + blank_when_zero INTEGER, + renames TEXT, FOREIGN KEY(file_id) REFERENCES file_data(id) ON DELETE CASCADE ) """) @@ -1119,6 +1123,13 @@ def record_mission( # pseudo-text as written minus its `==` delimiters. NULL when no COPY on the entry replaces # anything. The copybook keeps its own (tagged) names; the reader applies the REPLACING. _ensure_columns(cursor, "record_data", ["copy_replacing TEXT"]) + # #4266: `sync` -- 'SYNC' / 'SYNC LEFT' / 'SYNC RIGHT' when a COBOL item codes SYNCHRONIZED (slack + # bytes before it are the reader's job, galaxy_ir); `justified` / `blank_when_zero` -- 1 when it codes + # JUSTIFIED / BLANK WHEN ZERO; `renames` -- a 66 entry's `A [THRU B]` (qualifiers as `X OF Y`). + # All NULL otherwise. + _ensure_columns( + cursor, "record_data", ["sync TEXT", "justified INTEGER", "blank_when_zero INTEGER", "renames TEXT"] + ) cursor.execute("CREATE INDEX IF NOT EXISTS idx_record_file_id ON record_data(file_id);") cursor.execute("CREATE INDEX IF NOT EXISTS idx_record_snapshot ON record_data(repo_name, commit_hash);") @@ -3054,6 +3065,10 @@ def _seq_base(table: str) -> int: "sign_separate", "copy_libraries", "copy_replacing", + "sync", + "justified", + "blank_when_zero", + "renames", ), "record_layouts", lambda it: ( @@ -3076,6 +3091,10 @@ def _seq_base(table: str) -> int: int(it["sign_separate"]) if it.get("sign_separate") else None, # #3694: 1 trailing, 2 leading it.get("copy_libraries"), # #4265 it.get("copy_replacing"), # #4265 + it.get("sync"), # #4266 + 1 if it.get("justified") else None, # #4266 + 1 if it.get("blank_when_zero") else None, # #4266 + it.get("renames"), # #4266 ), ) diff --git a/gitgalaxy/tools/cobol_to_cobol/galaxy_ir.py b/gitgalaxy/tools/cobol_to_cobol/galaxy_ir.py index 565c83cf61..9f1599db6a 100644 --- a/gitgalaxy/tools/cobol_to_cobol/galaxy_ir.py +++ b/gitgalaxy/tools/cobol_to_cobol/galaxy_ir.py @@ -349,6 +349,13 @@ class EngineDataItem: # before the digits when `sign_leading` (SIGN LEADING SEPARATE), else after them. sign_separate: bool = False sign_leading: bool = False + # #4266: SYNCHRONIZED ('SYNC' / 'SYNC LEFT' / 'SYNC RIGHT'; None without it) -- a binary, floating or + # pointer item then starts on its boundary, after slack bytes (_sync_boundary); JUSTIFIED and BLANK + # WHEN ZERO as coded; a 66 entry's RENAMES operands (`A` / `A THRU B`, qualifiers as `X OF Y`). + sync: str | None = None + justified: bool = False + blank_when_zero: bool = False + renames: str | None = None # #3820: {symbol: currency string} for each declared currency symbol the PIC # uses; a string of None means the estate declares that symbol with different # strings, so its width is unknown. Empty when the PIC uses none. @@ -2231,6 +2238,7 @@ def _walk( size += wide - region[1] region[1] = wide continue + size += _slack(kid, usage, offset + size) # #4266: SYNCHRONIZED kid_width = _walk(kid_file, kid, offset + size, kid_depth, sink, usage) region = [offset + size, kid_width, kid.name.upper()] size += kid_width @@ -4924,6 +4932,8 @@ def walk( size = None if size is None else size + wide - region[1] region[1] = wide continue + if size is not None: + size += _slack(kid, usage, offset + size) # #4266: SYNCHRONIZED at[kid.name.upper()] = offset + (size or 0) width = walk(kid_file, kid, key, offset + (size or 0), depth + 1, None, (it.name, *path), usage) region = [kid.name.upper(), width] @@ -4978,12 +4988,57 @@ def fill(it: EngineDataItem, path: tuple, lay=lay, key=key, children=children) - if id(root) not in spans: ext = self._copy_extension(ef, cb) if roots[-1:] == [root] else None walk(cb, root, (cb.file_path, root.name, None), 0, 0, ext, ()) + self._renames_spans(ef, spans, paths, placed) cache[(ef.file_path, id(ef))] = (spans, ef) # ef held so its id is never reused self.__dict__.setdefault("_span_paths", {})[ef.file_path] = paths self.__dict__.setdefault("_span_items", {})[ef.file_path] = items self.__dict__.setdefault("_span_placed", {})[ef.file_path] = placed return spans + def _renames_spans(self, ef: EngineFile, spans: dict, paths: dict, placed: dict) -> None: + """#4266: lay out each 66 RENAMES entry `ef` can see as the byte range it names -- from the first + byte of `A` to the last of `B` (`RENAMES A THRU B`), or `A`'s own bytes (`RENAMES A`) -- in the + record the entry ends (the 01 it follows). Each operand must name exactly one placed item of + that record (its qualifiers matched against the storage ancestors, as `_operand_span` does); + otherwise the entry stays unplaced. The entry answers to its own name, qualified by its record.""" + owners = [ef] + [ + cb for cb in self._copy_files(ef) if nfc(Path(cb.file_path).stem.upper()) not in ef.replaced_members + ] + for owner in owners: + entries = [*owner.data_items, *owner.copied_items] + by_ordinal = {it.ordinal: it for it in entries} + for it in entries: + if it.level != 66 or not it.renames: + continue + root, hops = by_ordinal.get(it.parent_ordinal), 0 + while root is not None and root.parent_ordinal is not None and hops < 64: + root, hops = by_ordinal.get(root.parent_ordinal), hops + 1 + if root is None or id(root) not in spans: + continue + key = spans[id(root)][0] + ends = [] + for operand in it.renames.split(" THRU "): + parts = operand.split(" OF ") + hits = { + sp + for at in placed.values() + for x, sp, path in at + if x.name == parts[0] + and x.level not in (66, 88) + and sp[0] == key + and _in_order(parts[1:], path) + } + if len(hits) != 1: + break + ends.append(next(iter(hits))) + else: + (_, start, first, _), (_, last_at, last, _) = ends[0], ends[-1] + width = None if first is None or last is None or last_at < start else last_at + last - start + span = (key, start, width, width) + spans[id(it)] = span + paths[id(it)] = (root.name,) + placed.setdefault(id(it), []).append((it, span, (root.name,))) + def _name_at(self, file_path: str, span: dict) -> Optional[str]: """The most specific item of program `file_path` at `span`: the deepest one with exactly its offset and width, else the smallest one containing it.""" @@ -5037,7 +5092,7 @@ def visible(sp: Optional[tuple]) -> bool: (it, sp) for at in self.__dict__["_span_placed"][ef.file_path].values() for it, sp, path in at - if it.name == parts[0] and it.level not in (66, 88) and _in_order(parts[1:], path) and visible(sp) + if it.name == parts[0] and it.level != 88 and _in_order(parts[1:], path) and visible(sp) ] if program: # Every placement in the program's own storage: a copybook COPYed by several @@ -5047,8 +5102,16 @@ def visible(sp: Optional[tuple]) -> bool: for _, it, _ in self._find_item(ef, parts[0], None, program, exact=True): at = [sp for _, sp, _ in placed.get(id(it), []) if visible(sp)] out += [(it, sp) for sp in at] if at else ([] if placed.get(id(it)) else [(it, None)]) - return out - return [(it, spans.get(id(it))) for _, it, _ in self._find_item(ef, parts[0], None)] + cands = out + else: + cands = [(it, spans.get(id(it))) for _, it, _ in self._find_item(ef, parts[0], None)] + # #4266: a 66 RENAMES entry `_storage_spans` placed answers to its name too. + return cands or [ + (it, sp) + for at in self.__dict__["_span_placed"][ef.file_path].values() + for it, sp, _ in at + if it.level == 66 and it.name == parts[0] and visible(sp) + ] found: list = [] for scope in self._program_scope(ef, program): # the program, then its GLOBAL candidates @@ -5628,6 +5691,32 @@ def _item_class(item: EngineDataItem, inherited: str | None = None) -> str: return "9" +def _sync_boundary(item: EngineDataItem, usage: str | None) -> int: + """#4266: the boundary a SYNCHRONIZED elementary item starts on, in bytes from the start of its record + (which is doubleword-aligned), per IBM Enterprise COBOL's SYNCHRONIZED clause: a binary item of up to 4 + digits a halfword, of more a fullword (an 8-byte S9(10)-S9(18) one too); COMP-1, INDEX and the + pointers a fullword (LP(32)); COMP-2 a doubleword. 1 -- no slack -- for any other item (DISPLAY, + packed, national: SYNC is documentation there), a group, PL/I, or no SYNC at all.""" + if not item.sync or item.attributes is not None or not _is_elementary(item, usage): + return 1 + u = (usage or "DISPLAY").upper() + if u in ("COMP-2", "COMPUTATIONAL-2"): + return 8 + if u in ("COMP-1", "COMPUTATIONAL-1", "INDEX", "POINTER", "PROCEDURE-POINTER", "FUNCTION-POINTER"): + return 4 + if item.pic and _item_class(item, usage) == "B": + width = _elementary_bytes(item, usage) + return 1 if width is None else 2 if width <= 2 else 4 + return 1 + + +def _slack(item: EngineDataItem, inherited: str | None, at: int) -> int: + """#4266: the slack bytes the compiler puts before `item` when it would start at record offset + `at` (0 unless it is SYNCHRONIZED and `at` is off its boundary). `inherited` is the enclosing + group's USAGE (#4525). Slack between the occurrences of a table is not modelled.""" + return -at % _sync_boundary(item, _usage_under(item, inherited)) + + def _elementary_bytes(item: EngineDataItem, inherited: str | None = None) -> int | None: """One occurrence's storage width of an elementary item, or None when unknown. `inherited` is the enclosing group's USAGE (#4525), used when the item has none.""" @@ -6009,6 +6098,11 @@ def load_galaxy_ir(db_path: Path, repo_name: Optional[str] = None) -> GalaxyIR: # #4265: `copy_libraries` likewise. lib_col = "copy_libraries" if _has_column(cur, "record_data", "copy_libraries") else "NULL" repl_col = "copy_replacing" if _has_column(cur, "record_data", "copy_replacing") else "NULL" + # #4266: SYNC / JUSTIFIED / BLANK WHEN ZERO / RENAMES likewise (an older DB lays out no slack bytes). + layout_cols = ", ".join( + c if _has_column(cur, "record_data", c) else "NULL" + for c in ("sync", "justified", "blank_when_zero", "renames") + ) for ( file_id, ordinal, @@ -6030,10 +6124,14 @@ def load_galaxy_ir(db_path: Path, repo_name: Optional[str] = None) -> GalaxyIR: sign_sep, copy_libs, copy_repl, + sync, + justified, + blank_zero, + renames, ) in cur.execute( "SELECT file_id, ordinal, parent_ordinal, level_number, item_name, section, fd_name, pic, " # noqa: S608 -- attributes_col is one of two literals; values are bound "usage, occurs_min, occurs_max, occurs_depending_on, redefines, value_literal, line_number, " - f"{attributes_col}, {copy_col}, {sign_col}, {lib_col}, {repl_col} FROM record_data " + f"{attributes_col}, {copy_col}, {sign_col}, {lib_col}, {repl_col}, {layout_cols} FROM record_data " "WHERE repo_name = ? AND commit_hash = ? " "ORDER BY file_id, ordinal", (repo_name, commit_hash), @@ -6062,6 +6160,10 @@ def load_galaxy_ir(db_path: Path, repo_name: Optional[str] = None) -> GalaxyIR: sign_leading=sign_sep == 2, copy_libraries=copy_libs, copy_replacing=copy_repl, + sync=sync or None, + justified=bool(justified), + blank_when_zero=bool(blank_zero), + renames=renames or None, ) ) # Thread children onto parents and collect the roots. `data_items` is diff --git a/tests/cobol_mainframe/test_sync_renames_4266.py b/tests/cobol_mainframe/test_sync_renames_4266.py new file mode 100644 index 0000000000..15f410e150 --- /dev/null +++ b/tests/cobol_mainframe/test_sync_renames_4266.py @@ -0,0 +1,203 @@ +"""#4266: SYNCHRONIZED, JUSTIFIED, BLANK WHEN ZERO and 66 RENAMES in the layout model. + +The extractor had no handling of any of them: a SYNC binary item was laid out with no slack bytes +before it, so every offset after it (and the record's length) was short, and a 66 RENAMES entry +was attached to the item above it with no storage. Now: +- the COBOL record extractor records `sync`, `justified`, `blank_when_zero` and a 66 entry's + `renames` (presence-keyed, like sign_separate); +- record_data carries them as columns, healed into old tables; the rehydrator restores them; +- GalaxyIR puts slack bytes before a SYNC item (record_layout and the storage spans: IBM Enterprise + COBOL's SYNCHRONIZED clause, the record doubleword-aligned), and lays a RENAMES entry out as the + byte range it names, so a MOVE from / to it resolves in data_flows like any other item. +A DB written before the columns existed still loads, with no slack bytes and no RENAMES ranges. +""" + +import shutil +import sqlite3 + +import pytest + +from gitgalaxy.core.mainframe_boundary import extract_boundary +from gitgalaxy.tools.cobol_to_cobol.galaxy_ir import load_galaxy_ir, scan_to_db + +PROGRAM = """\ + IDENTIFICATION DIVISION. + PROGRAM-ID. SYNCREN. + DATA DIVISION. + WORKING-STORAGE SECTION. + 01 UNSYNCED. + 05 U-FLAG PIC X. + 05 U-HALF PIC S9(4) COMP. + 05 U-NAME PIC X(3). + 05 U-FULL PIC S9(9) COMP. + 05 U-DBL PIC S9(18) COMP. + 05 U-FLT COMP-2. + 01 SYNCED. + 05 S-FLAG PIC X. + 05 S-HALF PIC S9(4) COMP SYNC. + 05 S-NAME PIC X(3). + 05 S-FULL PIC S9(9) BINARY SYNCHRONIZED LEFT. + 05 S-DBL PIC S9(18) COMP-5 Sync. + 05 S-FLT COMP-2 SYNC. + 05 S-TEXT PIC X(3) SYNC. + 05 S-LAST PIC 9. + 01 NESTED COMP. + 05 N-A PIC XX DISPLAY. + 05 N-GRP. + 10 N-B PIC X DISPLAY. + 10 N-C PIC S9(4) SYNC. + 88 N-C-ZERO VALUE 0. + 01 WS-SYNC-FLAG PIC X VALUE 'SYNC'. + 01 CUSTOMER. + 05 CUST-ID PIC X(6) JUSTIFIED RIGHT. + 05 CUST-NAME PIC X(10) JUST. + 05 CUST-BAL PIC 9(5) BLANK WHEN ZERO. + 05 CUST-LIMIT PIC 9(5) BLANK ZEROES. + 05 CUST-ZIP PIC X(5). + 66 CUST-KEY RENAMES CUST-ID THRU CUST-NAME. + 66 CUST-AMOUNTS RENAMES CUST-BAL THROUGH CUST-LIMIT. + 66 CUST-POSTAL RENAMES CUST-ZIP IN CUSTOMER. + 01 WS-KEY PIC X(16). + 01 WS-AMOUNTS PIC X(10). + PROCEDURE DIVISION. + MOVE CUST-KEY TO WS-KEY. + MOVE WS-AMOUNTS TO CUST-AMOUNTS. + MOVE CUST-POSTAL OF CUSTOMER TO WS-KEY. + GOBACK. +""" + + +@pytest.fixture(scope="module") +def scanned(tmp_path_factory): + base = tmp_path_factory.mktemp("sync_renames_4266") + (base / "repo").mkdir() + (base / "repo" / "SYNCREN.cbl").write_text(PROGRAM, encoding="utf-8") + return scan_to_db(base / "repo", base / "scan") + + +def _rows(): + return {r["name"]: r for r in extract_boundary("cobol", PROGRAM)["records"]} + + +def test_the_extractor_records_the_clauses(): + rows = _rows() + assert rows["S-HALF"]["sync"] == "SYNC" + assert rows["S-FULL"]["sync"] == "SYNC LEFT" + assert rows["S-DBL"]["sync"] == "SYNC" # lower case, and the period right after the clause + assert rows["S-FLT"]["sync"] == "SYNC" + assert rows["N-C"]["sync"] == "SYNC" + assert rows["CUST-ID"]["justified"] is True and rows["CUST-NAME"]["justified"] is True + assert rows["CUST-BAL"]["blank_when_zero"] is True and rows["CUST-LIMIT"]["blank_when_zero"] is True + # presence-keyed: an entry without a clause keeps its pre-#4266 shape + for name in ("U-HALF", "S-FLAG", "WS-SYNC-FLAG", "CUST-ZIP"): + assert not {"sync", "justified", "blank_when_zero", "renames"} & set(rows[name]), name + + +def test_the_extractor_records_renames_ranges(): + rows = _rows() + assert rows["CUST-KEY"]["renames"] == "CUST-ID THRU CUST-NAME" + assert rows["CUST-AMOUNTS"]["renames"] == "CUST-BAL THRU CUST-LIMIT" # THROUGH spelled THRU + assert rows["CUST-POSTAL"]["renames"] == "CUST-ZIP OF CUSTOMER" # IN read as OF + assert rows["CUST-KEY"]["redefines"] is None # RENAMES is not a REDEFINES + + +def _layout(ir, name): + ef = ir.files["SYNCREN.cbl"] + return ir.record_layout(ef, next(it for it in ef.records if it.name == name)) + + +def _fields(layout): + return [(f["name"], f["offset"], f["bytes"]) for f in layout["fields"]] + + +def test_an_unsynced_record_has_no_slack(scanned): + layout = _layout(load_galaxy_ir(scanned), "UNSYNCED") + assert _fields(layout) == [ + ("U-FLAG", 0, 1), ("U-HALF", 1, 2), ("U-NAME", 3, 3), ("U-FULL", 6, 4), ("U-DBL", 10, 8), ("U-FLT", 18, 8)] # fmt: skip + assert layout["bytes"] == 26 + + +def test_a_synced_record_takes_its_slack_bytes(scanned): + layout = _layout(load_galaxy_ir(scanned), "SYNCED") + assert _fields(layout) == [ + ("S-FLAG", 0, 1), + ("S-HALF", 2, 2), # halfword: 1 slack byte + ("S-NAME", 4, 3), + ("S-FULL", 8, 4), # fullword: 1 slack byte + ("S-DBL", 12, 8), # an 8-byte binary is fullword-aligned (IBM): none + ("S-FLT", 24, 8), # COMP-2 is doubleword-aligned: 4 slack bytes + ("S-TEXT", 32, 3), # SYNC on a DISPLAY item adds none + ("S-LAST", 35, 1), + ] + assert layout["bytes"] == 36 # 30 without the slack: the synced record is longer + + +def test_slack_is_counted_from_the_record_and_inside_the_group(scanned): + ir = load_galaxy_ir(scanned) + layout = _layout(ir, "NESTED") + assert _fields(layout) == [("N-A", 0, 2), ("N-B", 2, 1), ("N-C", 4, 2)] # N-C's usage is NESTED's COMP + assert layout["bytes"] == 6 + ef = ir.files["SYNCREN.cbl"] + spans = ir._storage_spans(ef) + grp = next(it for it in ef.data_items if it.name == "N-GRP") + assert spans[id(grp)][1:3] == (2, 4) # the slack byte belongs to the group holding the item + + +def test_storage_spans_agree_with_the_layout(scanned): + ir = load_galaxy_ir(scanned) + ef = ir.files["SYNCREN.cbl"] + spans = ir._storage_spans(ef) + for name, size in (("UNSYNCED", 26), ("SYNCED", 36), ("NESTED", 6), ("CUSTOMER", 31)): + root = next(it for it in ef.records if it.name == name) + assert spans[id(root)][2] == size, name + for name, offset in (("S-HALF", 2), ("S-FULL", 8), ("S-DBL", 12), ("S-FLT", 24), ("S-LAST", 35)): + item = next(it for it in ef.data_items if it.name == name) + assert spans[id(item)][1] == offset, name + + +def test_the_ir_keeps_the_clauses(scanned): + ef = load_galaxy_ir(scanned).files["SYNCREN.cbl"] + items = {it.name: it for it in ef.data_items} + assert items["S-FULL"].sync == "SYNC LEFT" and items["U-FULL"].sync is None + assert items["CUST-ID"].justified and not items["CUST-ZIP"].justified + assert items["CUST-BAL"].blank_when_zero and not items["CUST-ID"].blank_when_zero + assert items["CUST-KEY"].renames == "CUST-ID THRU CUST-NAME" + + +def test_renames_entries_are_byte_ranges(scanned): + ir = load_galaxy_ir(scanned) + ef = ir.files["SYNCREN.cbl"] + spans = ir._storage_spans(ef) + items = {it.name: it for it in ef.data_items} + assert spans[id(items["CUST-KEY"])][1:3] == (0, 16) # CUST-ID (6) through CUST-NAME (10) + assert spans[id(items["CUST-AMOUNTS"])][1:3] == (16, 10) + assert spans[id(items["CUST-POSTAL"])][1:3] == (26, 5) + assert spans[id(items["CUST-KEY"])][0] == spans[id(items["CUSTOMER"])][0] # in the record it follows + + +def test_a_move_through_a_renames_entry_resolves(scanned): + flows = {(f["source"], f["target"]): f for f in load_galaxy_ir(scanned).data_flows()} + key = flows[("CUST-KEY", "WS-KEY")] + assert key["status"] == "resolved" + assert (key["source_span"]["record"], key["source_span"]["offset"], key["source_span"]["bytes"]) == ( + "CUSTOMER", 0, 16) # fmt: skip + assert key["truncates"] is False + amounts = flows[("WS-AMOUNTS", "CUST-AMOUNTS")] + assert amounts["status"] == "resolved" + assert (amounts["target_span"]["offset"], amounts["target_span"]["bytes"]) == (16, 10) + postal = flows[("CUST-POSTAL OF CUSTOMER", "WS-KEY")] # qualified by its record + assert postal["status"] == "resolved" + assert (postal["source_span"]["offset"], postal["source_span"]["bytes"]) == (26, 5) + + +def test_a_pre_4266_db_loads_with_no_slack_and_no_renames(scanned, tmp_path): + old = tmp_path / "old.db" + shutil.copy(scanned, old) + with sqlite3.connect(old) as conn: + for column in ("sync", "justified", "blank_when_zero", "renames"): + conn.execute(f"ALTER TABLE record_data DROP COLUMN {column}") + ir = load_galaxy_ir(old) + assert _layout(ir, "SYNCED")["bytes"] == 30 + ef = ir.files["SYNCREN.cbl"] + key = next(it for it in ef.data_items if it.name == "CUST-KEY") + assert key.renames is None and id(key) not in ir._storage_spans(ef) diff --git a/tests/golden_master_audit/7_mainframe_skeleton_completeness/channels.json b/tests/golden_master_audit/7_mainframe_skeleton_completeness/channels.json index 6ab9fc1c99..9da42bf8bc 100644 --- a/tests/golden_master_audit/7_mainframe_skeleton_completeness/channels.json +++ b/tests/golden_master_audit/7_mainframe_skeleton_completeness/channels.json @@ -33,10 +33,10 @@ }, "data flows": { "Gaps": { - "unresolved operand": 99 + "unresolved operand": 95 }, - "Ratio": "99.6%", - "Resolved": 27040, + "Ratio": "99.7%", + "Resolved": 27044, "System Names (Not Gaps)": 750, "Total": 27139 }, diff --git a/tests/golden_master_zero_dep_audit/7_mainframe_skeleton_completeness/channels.json b/tests/golden_master_zero_dep_audit/7_mainframe_skeleton_completeness/channels.json index 6ab9fc1c99..9da42bf8bc 100644 --- a/tests/golden_master_zero_dep_audit/7_mainframe_skeleton_completeness/channels.json +++ b/tests/golden_master_zero_dep_audit/7_mainframe_skeleton_completeness/channels.json @@ -33,10 +33,10 @@ }, "data flows": { "Gaps": { - "unresolved operand": 99 + "unresolved operand": 95 }, - "Ratio": "99.6%", - "Resolved": 27040, + "Ratio": "99.7%", + "Resolved": 27044, "System Names (Not Gaps)": 750, "Total": 27139 },