det facade: a DTO-only answer of a RETURN COMMAREA refuses bytes the DTO cannot hold (Part of #4817)

docs/language_status/register/X51.md

@@ -24,4 +24,9 @@ reached: "no (unit-proven on CicsTask for RETURN, RETURN IMMEDIATE and XCTL, eve
   no layout, or bytes shorter than its record, are answered as before: an `Object` facade keeps the `byte[]`, a typed one is
   refused by name (`returned(Dto.class)`; CBSA BNK1CCS, BNK1TFN, BNK1UAC, #5076); a facade with no DTO of its own (`Object
   request`) keeps the `byte[]`; a forge-generated (model-written) facade is unchanged (#4695).
+- **DTO-only answer (#4817).** A DTO holds a number it decoded, so a DTO-only answer of a RETURN (`returned(..)` of a task whose
+  `returnedBytes()` was not read: the one-arg `handleTransaction`, the REST endpoint over it) is refused by name when the DTO's
+  layout bytes (`toCommarea`) differ from a byte the RETURN passed, unless the DTO's byte is a space (FILLER, an unset field;
+  0x20 or 0x40, the page being unknown to CicsTask). The byte-carrying `handleTransaction(.., byte[])` reads `returnedBytes()` first
+  and answers as before. A DTO with no layout, or a RETURN with no bytes beside it, is unchanged. The check is a lower bound: a loss under a space byte of the DTO's layout is not seen.
 - **Status:** ASSUMED. The oracle's (GnuCOBOL) storage past the LENGTH of a copied COMMAREA is not IBM's either; no program may rely on it.

gitgalaxy/tools/cobol_to_java/cobol_to_java_transaction_forge.py

@@ -315,6 +315,7 @@ def commarea_alternative_todos(commarea: dict) -> list[str]:
     private Runnable rollbackHook;                                          // how a rollback undoes (the root's)
     private final Map<String, Integer> faultSeen = new HashMap<>();
     private Object returnedArea;                                             // #4343: the level-1 RETURN's COMMAREA
+    private boolean bytesTaken;                                              // #4817: returnedBytes() was read
     private byte[] returnedBytes;                                            // #4817: its raw bytes
     private Integer returnedLength;                                          // #4449: its LENGTH (null: the whole record)
 
@@ -5059,9 +5060,41 @@ def commarea_alternative_todos(commarea: dict) -> list[str]:
      *  (#4695: `returned(Object.class)`, so a facade's handleTransaction answers that byte[] by design: the Object
      *  answer is the COMMAREA as the next task is given it, and `returned(Dto.class)` refuses it by name). */
     public byte[] returnedBytes() {
+        root().bytesTaken = true;
         return returnedBytes == null ? null : java.util.Arrays.copyOf(returnedBytes, returnedBytes.length);
     }
 
+    /** #4817: a DTO-only answer must not change what the RETURN passed. IBM hands the next task the COMMAREA's bytes
+     *  as they were; the DTO holds a number it decoded (a numeric DISPLAY field of spaces and digits, invalid content),
+     *  and answering it would put digits where the task left something else. When the task's bytes were not taken
+     *  (returnedBytes(): the byte-carrying handleTransaction does), the DTO's layout bytes (toCommarea) must hold
+     *  every byte the RETURN passed: a byte toCommarea writes that is not a space (FILLER and an unset field are
+     *  spaces, in either page) and differs from the passed one is refused by name. A DTO with no layout, or a RETURN
+     *  that passed no bytes beside it, is as it was. */
+    private void lossless(Object area) {
+        byte[] passed = root().returnedBytes;
+        if (passed == null) {
+            return;
+        }
+        byte[] laid;
+        try {
+            java.lang.reflect.Method to = area.getClass().getMethod("toCommarea");
+            if (to.getReturnType() != byte[].class) {
+                return;
+            }
+            laid = (byte[]) to.invoke(area);
+        } catch (NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e) {
+            return;
+        }
+        for (int i = 0; i < Math.min(laid.length, passed.length); i++) {
+            if (laid[i] != passed[i] && laid[i] != 0x20 && laid[i] != 0x40) {
+                throw new IllegalStateException("the task RETURNed a COMMAREA whose byte " + i + " the " + area.getClass().getName()
+                        + " DTO cannot hold as it was (a number it decoded): answer it through the byte-carrying"
+                        + " handleTransaction(.., byte[]) and lastAnswerBytes() (#4817)");
+            }
+        }
+    }
+
     /** The COMMAREA the task's level-1 RETURN passed on (#4343: what handleTransaction answers), as `type`; null when
      *  the task RETURNed none, or has not RETURNed. A COMMAREA of another class is converted by layout (#4449) when
      *  both are laid out as bytes (the generator writes toCommarea / fromCommarea into the DTOs a facade converts
@@ -5070,6 +5103,9 @@ def commarea_alternative_todos(commarea: dict) -> list[str]:
      *  fewer bytes than `type`'s record (never passed). */
     public <T> T returned(Class<T> type) {
         Object area = root().returnedArea;
+        if (area != null && !root().bytesTaken) {
+            lossless(area);
+        }
         if (area == null || type.isInstance(area)) {
             return type.cast(area);
         }

tests/cobol_mainframe/test_return_dto_lossless_4817.py

@@ -0,0 +1,149 @@
+"""#4817: a DTO-only answer of a RETURN COMMAREA never silently changes the bytes the RETURN passed.
+
+IBM hands the next task the COMMAREA's bytes as they were. A DTO holds a number it decoded, so the DTO-only
+handleTransaction (and the REST endpoint over it) refuses by name a RETURN whose bytes the DTO cannot hold; the
+byte-carrying overload (returnedBytes) answers them. The whole input class: RETURN and RETURN IMMEDIATE, answered as
+the DTO's class and as Object, bytes taken first or not, a LENGTH cut, FILLER, a DTO with no layout, no bytes beside.
+"""
+
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+from gitgalaxy.tools.cobol_to_java.cobol_to_java_transaction_forge import CICS_SPEC_JAVA, CICS_TASK_JAVA
+
+
+def _jdk() -> Path | None:
+    jdk = os.environ.get("JDK_17") or os.environ.get("JAVA_HOME")
+    if jdk and (Path(jdk) / "bin" / "javac").exists():
+        return Path(jdk)
+    javac = shutil.which("javac")
+    return Path(javac).resolve().parent.parent if javac else None
+
+
+MAIN = """
+package com.squid.gitgalaxy.cics;
+
+import java.nio.charset.StandardCharsets;
+
+public class Main {
+    /** A DTO with a layout: toCommarea answers the bytes it was built to write. */
+    public static class Dto {
+        final byte[] laid;
+        Dto(String laid) { this.laid = laid.getBytes(StandardCharsets.US_ASCII); }
+        public byte[] toCommarea() { return laid.clone(); }
+    }
+
+    /** A DTO with no layout. */
+    public static class Bare { }
+
+    static byte[] b(String s) { return s.getBytes(StandardCharsets.US_ASCII); }
+
+    static String answer(String name, boolean immediate, Object dto, String passed, Integer length, boolean takeBytes,
+                         Class<?> type) {
+        CicsTask.Region region = new CicsTask.LocalRegion(null, "T001", null);
+        CicsTask task = region.transaction("T1", null);
+        if (immediate) {
+            task.returnImmediate("T2", dto, length, b(passed));
+        } else {
+            task.returnTransid("T2", dto, length, b(passed));
+        }
+        if (takeBytes) {
+            task.returnedBytes();
+        }
+        try {
+            Object got = task.returned(type);
+            return name + ": answered " + (got == dto);
+        } catch (IllegalStateException e) {
+            return name + ": refused " + e.getMessage().contains("#4817");
+        }
+    }
+
+    public static void main(String[] args) {
+        for (boolean imm : new boolean[] {false, true}) {
+            String k = imm ? "imm " : "ret ";
+            // a number of spaces and digits the DTO decoded to digits: refused, as the DTO's class and as Object
+            System.out.println(answer(k + "lossy", imm, new Dto("1202"), "1 2 ", null, false, Dto.class));
+            System.out.println(answer(k + "lossy-object", imm, new Dto("1202"), "1 2 ", null, false, Object.class));
+            // the same, the bytes taken first (the byte-carrying handleTransaction): answered
+            System.out.println(answer(k + "taken", imm, new Dto("1202"), "1 2 ", null, true, Dto.class));
+            // the DTO holds every byte: answered
+            System.out.println(answer(k + "clean", imm, new Dto("1234"), "1234", null, false, Dto.class));
+            // FILLER and an unset field are spaces in the DTO's bytes, anything in the passed ones: answered
+            System.out.println(answer(k + "filler", imm, new Dto("12  "), "12XY", null, false, Dto.class));
+            // EBCDIC page spaces
+            System.out.println(answer(k + "ebcdic-space", imm, new Dto("12@@"), "12XY", null, false, Dto.class));
+            // a loss past LENGTH is not passed: answered; one inside it is refused
+            System.out.println(answer(k + "cut-past", imm, new Dto("1299"), "1234", 2, false, Dto.class));
+            System.out.println(answer(k + "cut-inside", imm, new Dto("1999"), "1234", 2, false, Dto.class));
+            // a DTO with no layout: as it was
+            System.out.println(answer(k + "bare", imm, new Bare(), "1234", null, false, Bare.class));
+        }
+        // a RETURN with no bytes beside the DTO (the 3-arg call): as it was
+        CicsTask.Region region = new CicsTask.LocalRegion(null, "T001", null);
+        CicsTask task = region.transaction("T1", null);
+        Dto none = new Dto("1202");
+        task.returnTransid("T2", none, null);
+        System.out.println("nobytes: answered " + (task.returned(Dto.class) == none));
+        // a task that RETURNed nothing
+        System.out.println("nothing: " + (region.transaction("T3", null).returned(Dto.class) == null));
+    }
+}
+"""
+
+EXPECT = [
+    "lossy: refused true",
+    "lossy-object: refused true",
+    "taken: answered true",
+    "clean: answered true",
+    "filler: answered true",
+    "ebcdic-space: answered true",
+    "cut-past: answered true",
+    "cut-inside: refused true",
+    "bare: answered true",
+]
+
+
+@pytest.mark.skipif(_jdk() is None, reason="needs a JDK 17 (JAVA_HOME / JDK_17)")
+def test_a_dto_only_answer_refuses_bytes_the_dto_cannot_hold(tmp_path):
+    jdk = _jdk()
+    src = tmp_path / "src" / "com" / "squid" / "gitgalaxy" / "cics"
+    src.mkdir(parents=True)
+    pkg = "package com.squid.gitgalaxy.cics;"
+    (src / "CicsTask.java").write_text(
+        CICS_TASK_JAVA.replace("package __PACKAGE__.cics;", pkg).replace("__ZONE__", "UTC"), encoding="utf-8"
+    )
+    (src / "CicsSpec.java").write_text(CICS_SPEC_JAVA.replace("package __PACKAGE__.cics;", pkg), encoding="utf-8")
+    (src / "Main.java").write_text(MAIN, encoding="utf-8")
+    subprocess.run(
+        [
+            str(jdk / "bin" / "javac"),
+            "-encoding",
+            "UTF-8",
+            "-d",
+            str(tmp_path / "classes"),
+            *map(str, src.glob("*.java")),
+        ],
+        check=True,
+    )  # noqa: S603
+    res = subprocess.run(
+        [
+            str(jdk / "bin" / "java"),
+            "-Dfile.encoding=UTF-8",
+            "-Dstdout.encoding=UTF-8",
+            "-cp",
+            str(tmp_path / "classes"),
+            "com.squid.gitgalaxy.cics.Main",
+        ],
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        check=True,
+    )  # noqa: S603
+    lines = res.stdout.splitlines()
+    for prefix in ("ret ", "imm "):
+        assert [ln for ln in lines if ln.startswith(prefix)] == [prefix + e for e in EXPECT]
+    assert "nobytes: answered true" in lines
+    assert "nothing: true" in lines