DRAFT: BNK1CCA + DELCUS cases; screen compare at every OCCURS occurrence (Part of #5076)

docs/language_status/register/Q3.md

@@ -16,8 +16,10 @@ reached: "CICS: yes (CBSA XFRFUN); batch: yes (synthetic UOWDEMO, `tests/equival
   at the SYNCPOINT ROLLBACK and goes on in the same transaction, so the SQL that follows is committed with the task, as
   the COBOL side's Db2 commits it. Reached: CBSA INQACCCU's cursor failing inside CREACC's task (its SYNCPOINT
   ROLLBACK, then CREACC's CONTROL updates and INSERTs: cbsa-creacc, cbsa-bnk1cac). A recoverable FILE changed after a
-  SYNCPOINT ROLLBACK is still undone with the task on the Java side (the transaction is marked rollback-only): no case
-  reaches one, and it would show as a difference.
+  SYNCPOINT ROLLBACK is still undone with the task on the Java side (the transaction is marked rollback-only), and it
+  shows as a difference: CBSA DELCUS deleting the CUSTOMER record after INQACCCU's rollback (cbsa-delcus: customer 3's
+  NULL statement date, and the derived SQL faults at INQACCCU lines 246 and 466) -- listed in det_sweep_baseline.json,
+  every other output of those scenarios equal.
 - **CICS, EXEC SQL COMMIT / ROLLBACK.** IBM: both are invalid under CICS -- Db2 for z/OS answers -925 / -926 (SQLSTATE
   2D521; Db2 for z/OS Codes) and the task's unit of work is CICS's (SYNCPOINT). The det translator REFUSES a CICS
   program's ROLLBACK by name. A CICS program's COMMIT stays a reset of the SQLCA on the Java side and a real commit on

gitgalaxy/tools/cobol_to_java/det/cics.py

@@ -439,7 +439,14 @@ def entity_key(self, entity: str, prop: str | None) -> tuple[int, int]:
 
     def screen_fields(self, cls: str) -> list[str]:
         text = self._file("dto/screen", cls).read_text(encoding="utf-8")
-        return re.findall(r'values\.put\("([A-Z0-9-]+)",', text)
+        return re.findall(r'values\.put\("([A-Z0-9-]+)"(?:,| \+ )', text)  # #5076: a repeated field: "NAME" + "." + i
+
+    def screen_occurs(self, cls: str) -> dict[str, int]:
+        """#5076: the BMS fields a map repeats (DFHMDF OCCURS=n), by name -> n, read off the screen class's LAYOUT
+        (`new ScreenField(name, ..., occurs, justify)`); a field without OCCURS is absent."""
+        text = self._file("dto/screen", cls).read_text(encoding="utf-8")
+        rows = re.findall(r'new ScreenField\("([A-Z0-9-]+)",.*, (\d+), (?:null|"[A-Z,]+")\)\)?[,;]?\s*$', text, re.M)
+        return {name: int(n) for name, n in rows if int(n) > 1}
 
     def screen_justify(self, cls: str) -> dict[str, str]:
         """#5082: a named field's DFHMDF JUSTIFY ("RIGHT", "RIGHT,ZERO", "LEFT,ZERO" ...), read off the screen class's
@@ -2485,17 +2492,18 @@ def ts_queue(self, verb: str, opts: dict, ind: str) -> list[str]:
             )
         return out + self.outcome(opts, f"DetCics.resp({r}.resp())", "0", ind)
 
-    def symbolic(self, map_name: str, suffix: str, name: str, record: str | None) -> str | None:
-        """The symbolic map item <name><suffix> under `record` (COSGN0AI / COSGN0AO), or None."""
+    def symbolic(self, map_name: str, suffix: str, name: str, record: str | None, k: int | None = None) -> str | None:
+        """The symbolic map item <name><suffix> under `record` (COSGN0AI / COSGN0AO), or None. #5076 `k`: occurrence k
+        of a repeated field (ACCOUNTO(3))."""
         for rec in [*([record] if record else []), map_name + "I", map_name + "O"]:
-            found = self._field_or_none(name + suffix, rec)
+            found = self._field_or_none(name + suffix, rec, k)
             if found is not None:
                 return found
         return None
 
-    def _field_or_none(self, name: str, rec: str) -> str | None:
+    def _field_or_none(self, name: str, rec: str, subscript: int | None = None) -> str | None:
         try:
-            return self.g.field_expr(E.Ref(name, [rec]))
+            return self.g.field_expr(E.Ref(name, [rec], [] if subscript is None else [E.Lit(E.NumLit(str(subscript)))]))
         except Exception:
             return None
 
@@ -2559,6 +2567,14 @@ def map_not_found(self, m: str, ms: str, ind: str) -> list[str] | None:
                 f"{ind}if ({lbl} == null) throw abended();",
                 f"{ind}if (true) {self.g.jump(f'paragraph({lbl})')}"]  # fmt: skip
 
+    def screen_slots(self, cls: str) -> list[tuple[str, int | None]]:
+        """#5076: a screen class's fields as (name, occurrence): a repeated field once per occurrence (ACCOUNT, 3), any
+        other once (COMPANY, None)."""
+        repeated = self.gp.screen_occurs(cls)
+        return [
+            (n, k) for n in self.gp.screen_fields(cls) for k in (range(1, repeated[n] + 1) if n in repeated else [None])
+        ]
+
     def no_screen(self, m: str) -> CicsError:
         """Why map `m` has no generated screen, by name (a map of no BMS source we hold)."""
         return CicsError(f"no generated screen for map {m}")
@@ -2576,15 +2592,16 @@ def send_map(self, opts: dict, ind: str) -> list[str]:
         v, sub, scr = self.g.tmpname("values"), self.g.tmpname("sub"), self.g.tmpname("screen")
         out = [*guard, f"{ind}java.util.Map<String, String> {v} = new java.util.LinkedHashMap<>();",
                f"{ind}CicsTask.MapSubfields {sub} = new CicsTask.MapSubfields();"]  # fmt: skip
-        for name in self.gp.screen_fields(cls):
-            o = self.symbolic(m, "O", name, frm)
+        for name, k in self.screen_slots(cls):  # #5076: a repeated field's every occurrence, keyed NAME.k
+            key = name if k is None else f"{name}.{k}"
+            o = self.symbolic(m, "O", name, frm, k)
             if o is None:
-                raise CicsError(f"map {m}: no {name}O")
-            out.append(f"{ind}{v}.put({G_jstr(name)}, Cobol.text({o}, CS));")
-            parts = [self.symbolic(m, s, name, None) for s in ("L", "A", "C", "H")]
+                raise CicsError(f"map {m}: no {name}O" + ("" if k is None else f"({k})"))
+            out.append(f"{ind}{v}.put({G_jstr(key)}, Cobol.text({o}, CS));")
+            parts = [self.symbolic(m, s, name, None, k) for s in ("L", "A", "C", "H")]
             if parts[1] is None:
-                parts[1] = self.symbolic(m, "F", name, None)
-            out.append(f"{ind}DetCics.subfields({sub}, {G_jstr(name)}, "
+                parts[1] = self.symbolic(m, "F", name, None, k)
+            out.append(f"{ind}DetCics.subfields({sub}, {G_jstr(key)}, "
                        f"{', '.join(p or 'null' for p in parts)}, CS);")  # fmt: skip
         if opts.get("CURSOR"):
             out.append(f"{ind}{sub}.cursorAt({self.int_(opts['CURSOR'])});")
@@ -2615,16 +2632,17 @@ def receive_map(self, opts: dict, ind: str) -> list[str]:
                "(byte) 0);",
                f"{ind}    java.util.Map<String, String> {vals} = {r}.get().screenValues();"]  # fmt: skip
         justify = self.gp.screen_justify(cls)
-        for name in self.gp.screen_fields(cls):
-            i = self.symbolic(m, "I", name, into.name)
-            ln = self.symbolic(m, "L", name, into.name)
+        for name, k in self.screen_slots(cls):  # #5076: every occurrence of a repeated field, keyed NAME.k
+            key = name if k is None else f"{name}.{k}"
+            i = self.symbolic(m, "I", name, into.name, k)
+            ln = self.symbolic(m, "L", name, into.name, k)
             if i is None:
                 continue
-            out.append(f"{ind}    if ({vals}.get({G_jstr(name)}) != null) {{")
+            out.append(f"{ind}    if ({vals}.get({G_jstr(key)}) != null) {{")
             # #5082: IBM DFHMDF JUSTIFY -- the input right-justified / zero-filled as the field says; none: as typed
             tail = f", {G_jstr(justify[name])}" if name in justify else ""
             call = f"{self.package}.cobolrt.cics.DetJustify.place" if name in justify else "DetCics.typed"
-            out.append(f"{ind}        {call}({i}, {ln or 'null'}, {vals}.get({G_jstr(name)}), CS{tail});")
+            out.append(f"{ind}        {call}({i}, {ln or 'null'}, {vals}.get({G_jstr(key)}), CS{tail});")
             out.append(f"{ind}    }}")
         out.append(f"{ind}}}")
         return out + self.input_outcome(opts, resp, ind)

tests/cobol_mainframe/test_call_compare_4778.py

@@ -11,7 +11,9 @@
 - a det DTO whose record ends in a FILLER no property names (CBSA's CUSTCTRL DFHCOMMAREA, GenApp LGICVS01's) spans
   its declared record (Dto.size), as a table's (#4765) does; a COMMAREA's FILLER is compared as bytes when both
   sides are bytes, and declared not compared when the Java side is a DTO.
-- a symbolic map with OCCURS is refused by name (screen_layout): screens are compared at a single-occurrence layout.
+- a symbolic map with OCCURS is compared at every occurrence by subscript (screen_layout, screen_key: ACCOUNT.3, the
+  key the generated screen's screenValues() writes), as a COMMAREA is (#5076); one with OCCURS DEPENDING ON, and a
+  RECEIVE typed into one occurrence, are refused by name.
 """
 
 from __future__ import annotations
@@ -173,30 +175,106 @@ def test_a_det_call_entry_takes_the_items_bytes(tmp_path):
     assert "byte[][] areas = callAreas;" in body and body.count("System.arraycopy(") == 2
 
 
-# ---- screens: a symbolic map with OCCURS is refused by name -------------------------------------------------------
+# ---- screens: a symbolic map with OCCURS is compared at every occurrence -----------------------------------------
 MAP = """\
        01 MAPI.
           02 FILLER                  PIC X(12).
           02 ROWL                    COMP PIC S9(4).
           02 ROWF                    PIC X.
           02 ROWI                    PIC X(5).
+       01 MAPO REDEFINES MAPI.
+          02 FILLER                  PIC X(12).
+          02 FILLER                  PIC X(3).
+          02 ROWC                    PIC X.
+          02 ROWO                    PIC X(5).
 """
 MAP_OCCURS = """\
        01 MAPI.
           02 FILLER                  PIC X(12).
-          02 LINES OCCURS 3.
+          02 LINESD OCCURS 3.
              03 ROWL                 COMP PIC S9(4).
              03 ROWF                 PIC X.
              03 ROWI                 PIC X(5).
+       01 MAPO REDEFINES MAPI.
+          02 FILLER                  PIC X(12).
+          02 LINESD OCCURS 3.
+             03 FILLER               PIC X(3).
+             03 ROWC                 PIC X.
+             03 ROWO                 PIC X(5).
 """
+MAP_ODO = """\
+       01 MAPI.
+          02 N                       PIC 9.
+          02 ROWI                    PIC X(5) OCCURS 1 TO 3 DEPENDING ON N.
+"""
+
+
+def _names(layout):
+    return [f["name"] for f in layout]
+
+
+def test_a_symbolic_map_with_occurs_lists_every_occurrence(tmp_path):
+    (tmp_path / "map.cpy").write_text(MAP, encoding="utf-8")
+    (tmp_path / "occ.cpy").write_text(MAP_OCCURS, encoding="utf-8")
+    assert _names(ec.screen_layout(tmp_path, "map.cpy", "MAPI")) == ["FILLER", "ROWL", "ROWF", "ROWI"]
+    out = ec.screen_layout(tmp_path, "occ.cpy", "MAPO")
+    assert [n for n in _names(out) if n.startswith("ROWO")] == ["ROWO(1)", "ROWO(2)", "ROWO(3)"]
+    assert [f["offset"] for f in out if f["name"].startswith("ROWO")] == [16, 25, 34]  # one 9-byte row each
 
 
-def test_a_symbolic_map_with_occurs_is_refused(tmp_path):
+def test_screen_key_names_an_occurrence_as_the_generated_screen_does(tmp_path):
+    (tmp_path / "occ.cpy").write_text(MAP_OCCURS, encoding="utf-8")
     (tmp_path / "map.cpy").write_text(MAP, encoding="utf-8")
+    keys = [ec.screen_key(f) for f in ec.screen_layout(tmp_path, "occ.cpy", "MAPO")]
+    assert ("ROW.2", "O") in keys and ("ROW.3", "C") in keys
+    assert ec.screen_key(ec.screen_layout(tmp_path, "map.cpy", "MAPO")[-1]) == ("ROW", "O")
+
+
+def test_a_second_occurrence_that_differs_is_a_difference(tmp_path):
+    """The old single-occurrence layout read ROW.1 only: a second line one side left out compared equal."""
+    (tmp_path / "occ.cpy").write_text(MAP_OCCURS, encoding="utf-8")
+    layout = ec.screen_layout(tmp_path, "occ.cpy", "MAPO")
+    keys = {f["name"]: ec.screen_key(f) for f in layout}
+
+    def screen(rows):
+        data = bytearray(b" " * 40)
+        for i, text in enumerate(rows):
+            at = 16 + 9 * i
+            data[at : at + 5] = text.encode()
+        vals = ec.decode_record(bytes(data), layout, "latin-1")
+        return {keys[k][0]: v for k, v in vals.items() if keys[k][1] == "O"}
+
+    a, b = screen(["AAAAA", "BBBBB", "CCCCC"]), screen(["AAAAA", "BBBBB", "CCCCX"])
+    assert sorted(a) == ["ROW.1", "ROW.2", "ROW.3"]
+    cob = [{"event": "SEND-MAP", "map": "M", "screen": a, "subfields": {}, "options": [], "cursor": None}]
+    jav = [{"event": "SEND-MAP", "map": "M", "screen": b, "subfields": {}, "options": [], "cursor": None}]
+    diffs = ec.compare_events(cob, jav)["diffs"]
+    assert [(f["field"], f["cobol"], f["java"]) for d in diffs for f in d["fields"]] == [
+        ("screen.ROW.3", "CCCCC", "CCCCX")
+    ]
+
+
+def test_an_occurrences_attribute_is_a_subfield_of_its_own(tmp_path):
+    (tmp_path / "occ.cpy").write_text(MAP_OCCURS, encoding="utf-8")
+    case = {"screens": {"M": {"copybook": "occ.cpy", "input": "MAPI", "output": "MAPO"}}}
+    data = bytearray(b"\x00" * 40)
+    data[15 + 9] = 0xF1  # ROWC(2): the colour byte of the second line
+    sub = ec.map_subfields(tmp_path, case, "M", bytes(data), "latin-1")
+    assert sub == {"ROW.2": {"color": 0xF1}}
+
+
+def test_a_symbolic_map_with_occurs_depending_on_is_refused(tmp_path):
+    (tmp_path / "odo.cpy").write_text(MAP_ODO, encoding="utf-8")
+    with pytest.raises(ec.Unsupported, match="OCCURS DEPENDING ON"):
+        ec.screen_layout(tmp_path, "odo.cpy", "MAPI")
+
+
+def test_a_receive_typed_into_one_occurrence_is_refused(tmp_path):
     (tmp_path / "occ.cpy").write_text(MAP_OCCURS, encoding="utf-8")
-    assert [f["name"] for f in ec.screen_layout(tmp_path, "map.cpy", "MAPI")] == ["FILLER", "ROWL", "ROWF", "ROWI"]
-    with pytest.raises(ec.Unsupported, match="symbolic map with OCCURS"):
-        ec.screen_layout(tmp_path, "occ.cpy", "MAPI")
+    layout = ec.screen_layout(tmp_path, "occ.cpy", "MAPI")
+    assert len(ec.map_input(layout, {}, "latin-1")) == 12 + 3 * 8
+    with pytest.raises(ec.Unsupported, match="one occurrence"):
+        ec.map_input(layout, {"ROWI(2)": "TYPED"}, "latin-1")
 
 
 def _corpora() -> Path | None:
@@ -206,23 +284,32 @@ def _corpora() -> Path | None:
     return p if p.is_dir() else None
 
 
-def test_no_case_map_has_occurs_today():
-    """The screens' single-occurrence compare holds for every case's map; the first map with OCCURS fails here (and
-    is refused by its proof) until the screen compare names every occurrence."""
+def test_every_case_map_resolves_and_bnk1cca_has_occurs():
+    """Every case's map lays out (a map with OCCURS DEPENDING ON would be refused here), and CBSA's BNK1ACC is read at
+    each of its ten ACCOUNT occurrences (#5076)."""
     root = _corpora()
     if root is None:
         pytest.skip("no mainframe corpora")
     import equivalence as eq
 
-    seen = 0
+    seen = occurs = 0
     for path in sorted(eq.CASES.glob("*/case.json")):
         case = json.loads(path.read_text(encoding="utf-8"))
         corpus = root / case.get("corpus", "")
         if not case.get("screens") or not corpus.is_dir():
             continue
         for name in case["screens"]:
             for side in ("input", "output"):
-                ec.screen_fields(corpus, case, name, side)  # raises Unsupported on an OCCURS
+                layout = ec.screen_fields(corpus, case, name, side)  # raises Unsupported on an ODO
                 seen += 1
+                if path.parent.name == "cbsa-bnk1cca" and side == "output":
+                    keys = {ec.screen_key(f) for f in layout}
+                    assert {(f"ACCOUNT.{i}", "O") for i in range(1, 11)} <= keys
+                    occurs += 1
     if not seen:
         pytest.skip("no case's corpus is checked out")
+    assert (
+        occurs
+        or not (eq.CASES / "cbsa-bnk1cca").is_dir()
+        or not (root / "cics-banking-sample-application-cbsa").is_dir()
+    )

tests/cobol_mainframe/test_det_screen_occurs_5076.py

@@ -0,0 +1,75 @@
+"""#5076: SEND MAP / RECEIVE MAP over a map whose DFHMDF repeats (OCCURS=n).
+
+The screen forge writes a repeated field's value as `values.put("NAME" + "." + (i + 1), ...)`, which the translator's
+reader of the screen class did not match: SEND MAP left every occurrence out of the port's screen, silently. Whole
+operand class: no OCCURS, OCCURS=n, an OCCURS beside a JUSTIFY field, and an OCCURS=1 (not a repeat).
+"""
+
+from __future__ import annotations
+
+import pytest
+
+pytest.importorskip("tree_sitter_language_pack")
+
+from gitgalaxy.tools.cobol_to_java.det import cics as C  # noqa: E402
+
+_SCREEN = """class LstScreen {
+    public static final List<ScreenField> LAYOUT = List.of(
+            new ScreenField(null, 1, 1, 6, false, false, false, false, false, "Rows", null, 1, null),
+            new ScreenField("HEAD", 2, 1, 8, true, true, false, false, true, null, null, 1, "RIGHT,ZERO"),
+            new ScreenField("ROW", 3, 1, 79, false, false, false, false, false, null, "NEUTRAL", 10, null),
+            new ScreenField("PAIR", 14, 1, 5, false, false, false, false, false, null, null, 2, "LEFT"),
+            new ScreenField("ONE", 20, 1, 5, false, false, false, false, false, null, null, 1, null),
+            new ScreenField("MSG", 23, 1, 79, false, false, true, false, false, null, "YELLOW", 1, null));
+    public Map<String, String> screenValues() {
+        values.put("HEAD", head);
+        if (row != null) {
+            for (int i = 0; i < row.size(); i++) {
+                values.put("ROW" + "." + (i + 1), row.get(i));
+            }
+        }
+        if (pair != null) {
+            for (int i = 0; i < pair.size(); i++) {
+                values.put("PAIR" + "." + (i + 1), pair.get(i));
+            }
+        }
+        values.put("ONE", one);
+        values.put("MSG", msg);
+    }
+}
+"""
+
+
+def _gp(tmp_path):
+    d = tmp_path / "src/main/java/x/dto/screen"
+    d.mkdir(parents=True)
+    (d / "LstScreen.java").write_text(_SCREEN, encoding="utf-8")
+    return C.Generated(tmp_path, "")
+
+
+def test_every_field_is_listed_a_repeated_one_too(tmp_path):
+    assert _gp(tmp_path).screen_fields("LstScreen") == ["HEAD", "ROW", "PAIR", "ONE", "MSG"]
+
+
+def test_the_repeats_are_read_off_the_layout(tmp_path):
+    assert _gp(tmp_path).screen_occurs("LstScreen") == {"ROW": 10, "PAIR": 2}  # OCCURS=1 and no OCCURS: absent
+
+
+def test_a_screen_without_repeats_has_none(tmp_path):
+    d = tmp_path / "src/main/java/x/dto/screen"
+    d.mkdir(parents=True)
+    (d / "OldScreen.java").write_text(
+        'new ScreenField("AMT", 2, 10, 8, true, true, false, false, true, null, null, 1, null));\n values.put("AMT", a);\n',
+        encoding="utf-8",
+    )
+    gp = C.Generated(tmp_path, "")
+    assert gp.screen_occurs("OldScreen") == {} and gp.screen_fields("OldScreen") == ["AMT"]
+
+
+def test_a_slot_per_occurrence(tmp_path):
+    c = C.Cics.__new__(C.Cics)
+    c.gp = _gp(tmp_path)
+    slots = c.screen_slots("LstScreen")
+    assert slots[0] == ("HEAD", None) and slots[1] == ("ROW", 1) and slots[10] == ("ROW", 10)
+    assert slots[11:13] == [("PAIR", 1), ("PAIR", 2)] and slots[-2:] == [("ONE", None), ("MSG", None)]
+    assert len(slots) == 1 + 10 + 2 + 2