det: TS items in the region's code page; DTO copybooks in their declared page (#4528)

docs/language_status/det_port_design.md

@@ -365,6 +365,17 @@ Only generator output, never a test case:
   SQL an apostrophe is a string, a quotation mark a name; the grammar never reads them). When column 72 of a
   re-wrapped row would split a doubled quote, the row ends a column early. Newly translating: CardDemo CBSTM03A, DSF
   FO04F1X1, estate-crucible RPTHDR and LNCALC; CBSTM03A's cross-check found the engine's group USAGE gap (#4525).
+- #4528: a TS item is the bytes the program wrote "in the region's code page" (CicsTask), and the COBOL side's
+  region keeps it in CCSID 037 (cics-crucible SPEC 2; the stub transcodes its Latin-1 storage at the boundary). The
+  det port handed CicsTask its storage's bytes (CS, CobolRecords.charset()) as they were: WS-ONE VALUE 'W' reached
+  the queue as X'57' ('ï'), an item 'A' came back as 'Á'. WRITEQ / READQ TS now move each byte between CS and
+  `REGION` (`DetCics.toRegion` / `fromRegion`): the estate's declared code page for the program when it is EBCDIC
+  (`EngineCopies.page`, #4462), else CCSID 037; the system property `gitgalaxy.cics.charset` names another. NL is
+  NEL (X'15') and LF X'25' as in CDRA and Python's cp037, not the JDK's LF for both; a byte one page cannot carry
+  stops the run by name. TD (`Cobol.text(.., CS)`), SEND TEXT, terminal RECEIVE and the COMMAREA DTO codecs already
+  went through characters. `Cics.declared` reads a DTO's copybook in its declared page too (it read Latin-1).
+  Newly passing on the det port (cics-crucible): hc-perform-range 4/4 (`length-trap` byte for byte), hc-abend-link
+  push-pop, pushed-abend, sub-own-exit, sub-resp, ca-link-lengths no-commarea.
 
 ### Keyed reads
 

gitgalaxy/tools/cobol_to_java/det/cics.py

@@ -53,6 +53,29 @@
     pass
 
 
+# #4528: CICS's default CCSID, the region's page when the estate declares no EBCDIC one (cics-crucible SPEC 2: "every
+# byte in every area is CCSID 037")
+REGION_PAGE = "cp037"
+
+
+def region_page(declared: str | None) -> str:
+    """#4528: the JDK name of the region's code page -- the page a TS item's bytes are in on the COBOL side: the
+    estate's declared code page for the program when it is an EBCDIC one (cp273, cp277 ...: EngineCopies.page, the
+    page the engine decoded it with), else CCSID 037. An ASCII-family declaration says how the source was
+    transferred, not which page the region runs."""
+    from gitgalaxy.core.ebcdic_codecs import java_charset_name, register
+
+    register()
+    page = REGION_PAGE
+    if declared:
+        try:
+            if " ".encode(declared) == b"\x40":
+                page = declared
+        except (LookupError, UnicodeError):
+            pass
+    return java_charset_name(page)
+
+
 # #4411: every option each modelled command accepts. An option outside its command's set is refused by name (the
 # statement becomes a hole), never accepted and ignored: a silently divergent port is worse than a visible hole.
 # Options accepted with no code of their own say why they cannot change what the program sees.
@@ -392,6 +415,10 @@
         # handler state is stacked); set by program._translate from the source before any statement is translated
         self.handle_aid = False
         self.push_handle = False
+        # #4528: the region's code page (region_page), set by program._translate; `region_used`: the port declares
+        # REGION (a TS command moves bytes between the program's storage and the region)
+        self.region = region_page(None)
+        self.region_used = False
 
     # -- operands
     def operand(self, text: str):
@@ -562,15 +589,16 @@
 
     def declared(self, leaf: Leaf) -> L.Item:
         """A DTO field's item as the copybook the generator read it from declares it."""
-        from gitgalaxy.tools.cobol_to_java.det.source import logical_lines
+        from gitgalaxy.tools.cobol_to_java.det.source import _raw_lines, logical_lines
 
         name = Path(leaf.source or "").name
         path = next((d / name for d in self.g.copy_dirs if name and (d / name).is_file()), None)
         if path is None:
             raise CicsError(f"{leaf.cobol}: {leaf.size} bytes, PIC {leaf.pic} is not; its copybook {name!r} not found")
         raw = ["       IDENTIFICATION DIVISION.", "       PROGRAM-ID. GGDTO.", "       DATA DIVISION.",
                "       WORKING-STORAGE SECTION.", "       01 GG-DTO-RECORD."]  # fmt: skip
-        raw += path.read_text(encoding="latin-1").splitlines()
+        # #4528: decoded with the code page the estate declares for it, as the engine and the source reader do
+        raw += _raw_lines(path, self.g.engine)
         for rec in L.parse(logical_lines(raw, str(path))):
             for it in rec.walk():
                 if it.name == leaf.cobol and it.size == leaf.size:
@@ -892,19 +920,22 @@
         return out + self.input_outcome(opts, f"DetCics.resp({r}.resp())", ind)
 
     def ts_queue(self, verb: str, opts: dict, ind: str) -> list[str]:
-        """WRITEQ TS / READQ TS on the task's temporary storage (CicsTask): an item is the program's own bytes.
-        (The proofs cover no TS command yet: declared in docs/language_status/det_port_design.md.)"""
+        """WRITEQ TS / READQ TS on the task's temporary storage (CicsTask): an item is the program's own bytes in the
+        region's code page (#4528: REGION, region_page) -- written from the storage's page (CS) and read back into
+        it, character by character, as the COBOL side's region moves them."""
         q = opts.get("QUEUE") or opts.get("QNAME")
         if q is None:
             raise CicsError(f"{verb} TS without QUEUE / QNAME")
         queue = self.name(q)
         g = self.g
         r = g.tmpname("ts")
         out: list[str] = []
+        self.region_used = True
         if verb == "WRITEQ":
             f = self.read_field(_arg(opts.get("FROM")))
             n = self.int_(_arg(opts["LENGTH"])) if opts.get("LENGTH") else f"{f}.length()"
-            data = f"DetCics.bytes({f}, {n})"
+            # #4528: the item in the region's page, as the COBOL side's region keeps it
+            data = f"DetCics.toRegion(DetCics.bytes({f}, {n}), CS, REGION)"
             if "REWRITE" in opts:
                 out.append(
                     f"{ind}CicsTask.TsResult {r} = task.rewriteqTs({queue}, {self.int_(_arg(opts.get('ITEM')))}, {data});"
@@ -924,7 +955,7 @@
                 out.append(
                     f"{ind}CicsTask.TsResult {r} = task.readqTs({queue}, {self.int_(_arg(opts['ITEM']))}, {maxlen});"
                 )
-            out.append(f"{ind}if ({r}.data() != null) DetCics.put({into}, {r}.data());")
+            out.append(f"{ind}if ({r}.data() != null) DetCics.put({into}, DetCics.fromRegion({r}.data(), REGION, CS));")
             if opts.get("LENGTH"):
                 out.append(f"{ind}if ({r}.length() >= 0) Cobol.store({self.field(_arg(opts['LENGTH']))}, "
                            f"BigDecimal.valueOf({r}.length()), false, CS);")  # fmt: skip

gitgalaxy/tools/cobol_to_java/det/cobolrt/cics/DetCics.java

@@ -198,6 +198,78 @@ public static int readInto(Field into, byte[] record, int max, boolean update) {
         return lengerr ? 22 : 0;
     }
 
+    /** #4528: the region's code page -- the one TS items are in (CicsTask: "the bytes the program wrote, in the
+     *  region's code page"). `declared` is the estate's EBCDIC page for the program, else CCSID 037; the system
+     *  property gitgalaxy.cics.charset names another. A deployment fact, like CobolRecords.charset(). */
+    public static Charset region(String declared) {
+        return Charset.forName(System.getProperty("gitgalaxy.cics.charset", declared));
+    }
+
+    /** #4528: the program's bytes (in `cs`, its storage's page) as the region's (`region`), byte by byte: how the
+     *  COBOL side's region moves a TS item between its storage and the queue. Each byte must be one character of
+     *  both pages: anything else stops the run by name, never a substituted byte. */
+    public static byte[] toRegion(byte[] data, Charset cs, Charset region) {
+        return transcode(data, cs, region);
+    }
+
+    /** #4528: a region's bytes (a TS item read) in the program's page: toRegion's inverse. */
+    public static byte[] fromRegion(byte[] data, Charset region, Charset cs) {
+        return transcode(data, region, cs);
+    }
+
+    private static byte[] transcode(byte[] data, Charset from, Charset to) {
+        if (data == null || from.equals(to)) {
+            return data;
+        }
+        java.nio.charset.CharsetEncoder enc = to.newEncoder()
+                .onMalformedInput(java.nio.charset.CodingErrorAction.REPORT)
+                .onUnmappableCharacter(java.nio.charset.CodingErrorAction.REPORT);
+        byte[] out = new byte[data.length];
+        for (int i = 0; i < data.length; i++) {
+            char c = character(from, data[i]);
+            if (usesLfForNl(to) && (c == '\n' || c == '\u0085')) {
+                out[i] = (byte) (c == '\n' ? 0x25 : 0x15);  // CDRA's LF and NEL, as the COBOL side's cp037
+                continue;
+            }
+            try {
+                java.nio.ByteBuffer b = enc.encode(java.nio.CharBuffer.wrap(new char[] {c}));
+                if (b.remaining() != 1) {
+                    throw new java.nio.charset.CharacterCodingException();
+                }
+                out[i] = b.get();
+            } catch (java.nio.charset.CharacterCodingException e) {
+                throw new UnsupportedOperationException(String.format("byte %02X in %s is U+%04X, not one byte in %s:"
+                        + " not modelled", data[i] & 0xFF, from, (int) c, to), e);
+            }
+        }
+        return out;
+    }
+
+    /** Whether `page` is one of the JDK's EBCDIC pages that take NL (X'15') for LF -- decoding both X'15' and X'25'
+     *  as LF and encoding LF as X'15' (a USS convenience). CDRA's -- and the COBOL side's (Python's cp037) -- NL is
+     *  NEL (U+0085) and LF is X'25': transcode keeps those. */
+    private static boolean usesLfForNl(Charset page) {
+        return new String(new byte[] {0x15}, page).equals("\n") && new String(new byte[] {0x25}, page).equals("\n");
+    }
+
+    /** The one character a byte is in `page` (X'15' NEL on a page usesLfForNl). */
+    private static char character(Charset page, byte b) {
+        java.nio.charset.CharsetDecoder dec = page.newDecoder()
+                .onMalformedInput(java.nio.charset.CodingErrorAction.REPORT)
+                .onUnmappableCharacter(java.nio.charset.CodingErrorAction.REPORT);
+        try {
+            java.nio.CharBuffer c = dec.decode(java.nio.ByteBuffer.wrap(new byte[] {b}));
+            if (c.remaining() != 1) {
+                throw new java.nio.charset.CharacterCodingException();
+            }
+            char ch = c.get();
+            return b == 0x15 && usesLfForNl(page) ? '\u0085' : ch;
+        } catch (java.nio.charset.CharacterCodingException e) {
+            throw new UnsupportedOperationException(String.format("byte %02X is no character of %s on its own: not"
+                    + " modelled", b & 0xFF, page), e);
+        }
+    }
+
     /** Text into the first bytes of a field, the rest left as it was (ASSIGN SYSID: 4 bytes into an 8-byte area). */
     public static void putText(Field f, String text, Charset cs) {
         put(f, text.getBytes(cs));

gitgalaxy/tools/cobol_to_java/det/gen.py

@@ -20,6 +20,7 @@
 
 if TYPE_CHECKING:
     from gitgalaxy.tools.cobol_to_java.det.cics import Cics
+    from gitgalaxy.tools.cobol_to_java.det.source import EngineCopies
 
 
 class Untranslatable(Exception):
@@ -147,6 +148,7 @@ def __init__(self, prog: Program, structured: bool = False):
         self.cur = 0
         self.cics: Cics | None = None
         self.copy_dirs: list = []
+        self.engine: EngineCopies | None = None  # #4528: the estate's declared code pages
         self.lifted: dict[int, str] = {}  # id(item) -> "X" (a String of its length) | "BIN" (a long) -- B3
         self.violations: set[str] = set()  # lifted items used through their bytes: this translation is discarded
         # typed groups: a group holding lifted items keeps its bytes for whole-group uses -- packed from the typed

gitgalaxy/tools/cobol_to_java/det/program.py

@@ -466,12 +466,14 @@ def _translate(program: Path, copy_dirs: list[Path], stub: str, package: str, es
         gen.lifted = liftable(records, excluded | {"GG-SORT-RETURN"}, rc)
     gen.alphabets, gen.program_collating = alphabets(lines)
     gen.copy_dirs = [program.parent, *copy_dirs, C.COPY]
+    gen.engine = engine
     gen.java_root = (project / "src/main/java") if project is not None else None
     gen.clock = "clock.currentDate()" if batch else "Funcs.currentDate(java.time.LocalDateTime.now())"
     if is_cics:
         if project is None:
             raise ValueError("a CICS program needs the generated project")
         gen.cics = C.Cics(gen, C.Generated(project, stub), package)
+        gen.cics.region = C.region_page(engine.page(program) if engine is not None else None)  # #4528
         # #4414: HANDLE AID / PUSH HANDLE anywhere in the program, before its input commands are translated
         execs = [s.text for p in proc.paragraphs for s in S.walk(p.body) if s.kind == "EXEC"]
         gen.cics.handle_aid = any(re.match(r"(?is)\s*EXEC\s+CICS\s+HANDLE\s+AID\b", t) for t in execs)
@@ -1134,6 +1136,8 @@ def _cics_parts(gen: G.Gen, records: list, roots: dict, proc: S.Procedure,
     store_cases = [f'            case "{n}" -> {e};' for n, e in cx.stores.items()]
     members = [
         "    private CicsTask task;",
+        # #4528: the region's code page, which TS items are in (only in a program that issues a TS command)
+        *([f'    private static final Charset REGION = DetCics.region("{cx.region}");'] if cx.region_used else []),
         # HANDLE CONDITION labels; -1: IGNORE CONDITION (#4414)
         "    private final java.util.Map<String, Integer> handlers = new java.util.HashMap<>();",
         # #4414: HANDLE AID labels, and PUSH HANDLE's saved states -- only in a program that issues them