diff --git a/gitgalaxy/tools/cobol_to_cobol/cobol_jcl_forge.py b/gitgalaxy/tools/cobol_to_cobol/cobol_jcl_forge.py index 3ffa27083..530bd9fc8 100644 --- a/gitgalaxy/tools/cobol_to_cobol/cobol_jcl_forge.py +++ b/gitgalaxy/tools/cobol_to_cobol/cobol_jcl_forge.py @@ -26,8 +26,14 @@ # character, ASCII exactly as before or any non-ASCII letter or digit. Missing such a SELECT turned the program # from batch into transactional and its controller named a DTO nothing generates, so the Java did not compile. _W = r"(?:[A-Z0-9\-]|[^\W\x00-\x7f])" +# #3992: the ASSIGN operand may be a literal (`ASSIGN "TEST-FILE"`, `ASSIGN TO "./input.txt"`: GnuCOBOL, +# opensourcecobol4j, Micro Focus) and the file OPTIONAL, as the engine's own SELECT reader has them +# (mainframe_boundary._SELECT_ASSIGN). Missing the literal form made a batch program transactional (a DTO nothing +# generates); `ASSIGN TO "x"` backtracked to read `TO` as the DD name. A literal is bounded, so an unclosed quote +# cannot run on; the word form never takes the `TO` keyword itself. _FILE_ASSIGN_ANCHOR = re.compile( - rf"SELECT\s+({_W}+)\s+ASSIGN\s+(?:TO\s+)?({_W}+)", + rf"SELECT\s+(?:OPTIONAL\s+)?({_W}+)\s+ASSIGN\s+(?:TO\s+)?" + rf"(?:\"([^\"]{{1,1024}})\"|'([^']{{1,1024}})'|(?!TO\s)({_W}+))", re.IGNORECASE, ) @@ -86,7 +92,9 @@ def analyze_cobol_intent(filepath: Path, declared: Optional[str] = None) -> dict continue consumed = stop + 1 internal_name = match.group(1).strip() - raw_dd = match.group(2).strip() + raw_dd = next(g for g in match.groups()[1:] if g is not None).strip() + if not raw_dd: + continue clean_dd = re.sub(r"^(?:UT|UR)-S-", "", raw_dd) intent["files_requested"].append({"internal": internal_name, "dd_name": clean_dd}) diff --git a/gitgalaxy/tools/cobol_to_java/cobol_to_java_api_contract_forge.py b/gitgalaxy/tools/cobol_to_java/cobol_to_java_api_contract_forge.py index e1f77b44f..575d22930 100644 --- a/gitgalaxy/tools/cobol_to_java/cobol_to_java_api_contract_forge.py +++ b/gitgalaxy/tools/cobol_to_java/cobol_to_java_api_contract_forge.py @@ -24,6 +24,8 @@ _WORD_BREAKS, capitalize_name, java_class_base, + java_legal_chars, + java_start_ok, java_url_segment, lower_name, output_key, @@ -54,6 +56,11 @@ def generate_rest_controller( files_requested = base_intent.get("files_requested", []) is_cics = base_intent.get("is_cics", False) + # #3992: a non-CICS program that opens files is batch. Its files are its SELECTs (files_requested); when the + # intent reader read none (a SELECT form it does not know) the files it OPENs for input (the lineage) stand in, + # so its files arrive as uploads -- never as a request DTO, which no generator writes. + if not files_requested and not is_cics: + files_requested = [{"dd_name": i} for i in inputs] is_batch = len(files_requested) > 0 and not is_cics java = [] @@ -106,6 +113,11 @@ def generate_rest_controller( if not dd_parts: dd_parts = ["unknown"] safe_dd_name = lower_name(dd_parts[0]) + "".join(capitalize_name(word) for word in dd_parts[1:]) + # #3992: a literal ASSIGN (`"01.DAT"`, `"KUNDE§NR"`) may start with a digit or hold a character Java + # rejects; such a name is made legal, a legal one is unchanged. + safe_dd_name = java_legal_chars(safe_dd_name) + if not java_start_ok(safe_dd_name[0]): + safe_dd_name = f"file{safe_dd_name}" base_var_name = f"{safe_dd_name}File" # Enforce unique variable names and Spring request params @@ -135,16 +147,11 @@ def generate_rest_controller( java.append(' @PostMapping("/execute")') java.append(f" public ResponseEntity execute{camel_prog}(") - params = [] + # #3992: the controller names only classes the run generates. No generator writes a DTO per input file + # (a CICS program's COMMAREA contract comes from its skeleton, forges.cics.controller), so a file input is + # listed for the porter, not bound as an `@RequestBody DTO` that cannot compile. if inputs: - for i in inputs: - words = [w for w in _WORD_BREAKS.split(i) if w] - safe_class = "".join(capitalize_name(word) for word in words) if words else "Unknown" - safe_var = lower_name(safe_class[0]) + safe_class[1:] if safe_class else "unknown" - params.append(f"@RequestBody {safe_class}DTO {safe_var}Data") - - if params: - java.append(" " + ",\n ".join(params)) + java.append(f" // Input files: {', '.join(inputs)} (no request DTO is generated for them)") else: java.append(" /* No external data dependencies detected */") diff --git a/tests/cobol_mainframe/test_no_jcl_batch_controller.py b/tests/cobol_mainframe/test_no_jcl_batch_controller.py new file mode 100644 index 000000000..eb34e7a21 --- /dev/null +++ b/tests/cobol_mainframe/test_no_jcl_batch_controller.py @@ -0,0 +1,196 @@ +"""#3992: a batch program with no JCL -- a PC dialect (GnuCOBOL, opensourcecobol4j, Micro Focus) that ASSIGNs its +files to literals -- gets a REST controller that compiles: its files are uploads (MultipartFile), and the controller +names no class the run does not generate (it used to bind `@RequestBody TestFileDTO`, which nothing writes).""" + +from __future__ import annotations + +import os +import re +import shutil +import subprocess +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +import gitgalaxy.cobol_refractor_controller as refractor +import gitgalaxy.cobol_to_java_controller as java_controller +from gitgalaxy.tools.cobol_to_cobol.cobol_jcl_forge import analyze_cobol_intent +from gitgalaxy.tools.cobol_to_cobol.galaxy_ir import scan_to_db +from gitgalaxy.tools.cobol_to_java.cobol_to_java_api_contract_forge import generate_rest_controller + + +def _program(pgm: str, select: str, fd: str, open_: str) -> str: + return ( + " IDENTIFICATION DIVISION.\n" + f" PROGRAM-ID. {pgm}.\n" + " ENVIRONMENT DIVISION.\n" + " INPUT-OUTPUT SECTION.\n" + " FILE-CONTROL.\n" + f" {select}\n" + " ORGANIZATION SEQUENTIAL\n" + " FILE STATUS F-STATUS.\n" + " DATA DIVISION.\n" + " FILE SECTION.\n" + f" FD {fd}.\n" + " 01 TEST-RECORD PIC X(10).\n" + " WORKING-STORAGE SECTION.\n" + " 77 F-STATUS PIC X(02).\n" + " PROCEDURE DIVISION.\n" + f" OPEN {open_} {fd}.\n" + " DISPLAY F-STATUS.\n" + f" CLOSE {fd}.\n" + " STOP RUN.\n" + ) + + +# The issue's reproduction, and the other literal / OPTIONAL shapes of the opensourcecobol4j jp-compat suite +ESTATE = { + "ASGPATH.cbl": _program("ASGPATH", 'SELECT TEST-FILE ASSIGN "TEST-FILE"', "TEST-FILE", "I-O"), + "ASGDOT.cbl": _program("ASGDOT", 'SELECT TEST-FILE ASSIGN "./TEST-FILE"', "TEST-FILE", "EXTEND"), + "ASGTO.cbl": _program("ASGTO", 'SELECT IN-FILE ASSIGN TO "./input.txt"', "IN-FILE", "INPUT"), + "ASGDIGIT.cbl": _program("ASGDIGIT", "SELECT OPTIONAL F1 ASSIGN TO '01.DAT'", "F1", "INPUT"), +} + + +# --------------------------------------------------------------------------------------------------------------- +# The intent reader: the SELECT forms the engine reads (mainframe_boundary._SELECT_ASSIGN) +# --------------------------------------------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("select", "internal", "dd_name"), + [ + ('SELECT TEST-FILE ASSIGN "TEST-FILE"', "TEST-FILE", "TEST-FILE"), + ('SELECT TEST-FILE ASSIGN "./TEST-FILE"', "TEST-FILE", "./TEST-FILE"), + ('SELECT IN-FILE ASSIGN TO "./input.txt"', "IN-FILE", "./input.txt"), + ("SELECT IN-FILE ASSIGN TO 'S01'", "IN-FILE", "S01"), + ("SELECT OPTIONAL F1 ASSIGN TO UT-S-DD1", "F1", "DD1"), + ("SELECT F2 ASSIGN TO DD2", "F2", "DD2"), # the word forms read as before + ("SELECT F3 ASSIGN DD3", "F3", "DD3"), + ], +) +def test_select_assign_reads_literals_and_optional(tmp_path, select, internal, dd_name): + pgm = tmp_path / "IO.cbl" + pgm.write_text(f" FILE-CONTROL.\n {select}\n ORGANIZATION SEQUENTIAL.\n") + assert analyze_cobol_intent(pgm)["files_requested"] == [{"internal": internal, "dd_name": dd_name}] + + +def test_assign_to_a_literal_never_reads_the_to_keyword(tmp_path): + """`ASSIGN TO "x"` backtracked past the optional TO and read `TO` itself as the DD (`@RequestParam("toFile")`); + an empty or unclosed literal is no file, not `TO`.""" + pgm = tmp_path / "IO.cbl" + pgm.write_text( + " FILE-CONTROL.\n" + ' SELECT F-EMPTY ASSIGN TO "".\n' + ' SELECT F-OPEN ASSIGN TO "' + "X" * 2000 + ".\n" + ) + assert analyze_cobol_intent(pgm)["files_requested"] == [] + + +# --------------------------------------------------------------------------------------------------------------- +# The controller +# --------------------------------------------------------------------------------------------------------------- + + +def _ir(inputs, files_requested=(), is_cics=False, outputs=()): + return { + "metadata": {"file_name": "ASGPATH.cbl"}, + "analysis": { + "base_intent": {"files_requested": list(files_requested), "is_cics": is_cics}, + "lineage": {"inputs": list(inputs), "outputs": list(outputs)}, + }, + } + + +def test_a_non_cics_program_with_input_files_is_batch_without_selects(): + """The issue's IR: lineage.inputs = ['TEST-FILE'], files_requested = [], is_cics = False.""" + java = generate_rest_controller(_ir(["TEST-FILE"], outputs=["TEST-FILE"]), "com.acme") + assert '@PostMapping(value = "/execute-batch", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)' in java + assert '@RequestParam("testFileFile") MultipartFile testFileFile' in java + assert "import org.springframework.web.multipart.MultipartFile;" in java + assert "DTO" not in java and "@RequestBody" not in java + + +def test_the_selects_still_decide_the_upload_parameters(): + ir = _ir(["INDD"], files_requested=[{"internal": "IN-FILE", "dd_name": "INDD"}, {"dd_name": "OUTDD"}]) + java = generate_rest_controller(ir, "com.acme") + assert '@RequestParam("inddFile") MultipartFile inddFile' in java + assert '@RequestParam("outddFile") MultipartFile outddFile' in java + + +def test_a_cics_program_names_no_request_dto_for_its_files(): + """The transactional branch lists a file input for the porter; it never binds a DTO nothing generates.""" + java = generate_rest_controller(_ir(["CUSTFILE"], is_cics=True), "com.acme") + assert '@PostMapping("/execute")' in java + assert "// Input files: CUSTFILE (no request DTO is generated for them)" in java + assert "DTO " not in java.replace("request DTO is", "") and "@RequestBody" not in java + + +@pytest.mark.parametrize( + ("dd_name", "var"), + [("01.DAT", "file01DatFile"), ("./input.txt", "inputTxtFile"), ("KUNDE§NR", "kunde_nrFile")], +) +def test_a_literal_dd_name_becomes_a_legal_java_identifier(dd_name, var): + java = generate_rest_controller(_ir([], files_requested=[{"dd_name": dd_name}]), "com.acme") + assert f'@RequestParam("{var}") MultipartFile {var}' in java + + +# --------------------------------------------------------------------------------------------------------------- +# End to end: scan, refract and generate the no-JCL estate; every class a controller names is generated, and +# (with a JDK 17 and Maven that resolve offline) the project compiles. +# --------------------------------------------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def generated(tmp_path_factory): + base = tmp_path_factory.mktemp("no_jcl_batch") + repo = base / "estate" + repo.mkdir() + for name, text in ESTATE.items(): + (repo / name).write_text(text, encoding="utf-8") + db = scan_to_db(repo, base / "scan") + with patch.object(sys, "argv", ["refract", str(repo), "--galaxy-db", str(db)]): + refractor.main() + (clean,) = base.glob("estate_gitgalaxy_clean_*") + (base / "none.txt").write_text("", encoding="utf-8") + with patch.object(sys, "argv", ["cobol-to-java", str(clean), "--header", str(base / "none.txt")]): + java_controller.main() + (java,) = base.glob("estate_gitgalaxy_java_spring_*") + return java + + +_TYPE_USE = re.compile(r"@Request(?:Body|Param\([^)]*\))\s+([A-Za-z_$][\w$]*)\s") +_JDK_TYPES = {"MultipartFile", "String"} + + +def test_every_controller_is_a_batch_upload_endpoint(generated): + controllers = sorted((generated / "src/main/java/com/gitgalaxy/modernized/controller").glob("*.java")) + assert {p.stem for p in controllers} == {"AsgpathController", "AsgdotController", "AsgtoController", + "AsgdigitController"} # fmt: skip + for p in controllers: + code = p.read_text(encoding="utf-8") + assert "MultipartFile" in code and "@RequestBody" not in code, p.name + assert '"toFile"' not in code, p.name + + +def test_controllers_name_only_generated_classes(generated): + classes = {p.stem for p in generated.rglob("*.java")} | _JDK_TYPES + for p in (generated / "src/main/java/com/gitgalaxy/modernized/controller").glob("*.java"): + for used in _TYPE_USE.findall(p.read_text(encoding="utf-8")): + assert used in classes, f"{p.name} names {used}, which the run did not generate" + + +JDK17 = Path(os.environ.get("JDK_17") or "/usr/lib/jvm/java-17-openjdk-amd64") + + +@pytest.mark.skipif(not shutil.which("mvn") or not (JDK17 / "bin" / "javac").is_file(), reason="no Maven + JDK 17") +def test_the_generated_project_compiles(generated): + env = {**os.environ, "JAVA_HOME": str(JDK17)} + proc = subprocess.run(["mvn", "-q", "-o", "compile"], cwd=generated, env=env, capture_output=True, text=True, + timeout=600) # fmt: skip + out = proc.stdout + proc.stderr + if proc.returncode and "COMPILATION ERROR" not in out and re.search(r"offline|resolve|Could not", out): + pytest.skip("Maven cannot resolve the Spring Boot build offline here") + assert proc.returncode == 0, out[-4000:] diff --git a/tests/tools_recorders/test_golden_forge.py b/tests/tools_recorders/test_golden_forge.py index 309158a53..4a3a016d9 100644 --- a/tests/tools_recorders/test_golden_forge.py +++ b/tests/tools_recorders/test_golden_forge.py @@ -46,6 +46,8 @@ # ============================================================================== # GOLDEN IMAGES (The "Perfect" Expected Outputs) # ============================================================================== +# #3992: a non-CICS program that opens files is batch even when no SELECT was read -- its input files are +# uploads, never `@RequestBody DTO`, a class nothing generates. GOLDEN_CONTROLLER = """package com.gitgalaxy.modernized.controller; import org.springframework.web.bind.annotation.*; @@ -53,6 +55,9 @@ import lombok.RequiredArgsConstructor; import com.gitgalaxy.modernized.service.ProcessPayrollService; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.http.MediaType; + @RestController @RequestMapping("/api/v1/process-payroll") @RequiredArgsConstructor @@ -60,13 +65,14 @@ private final ProcessPayrollService processPayrollService; - @PostMapping("/execute") - public ResponseEntity executeProcessPayroll( - @RequestBody EmployeeRecordDTO employeeRecordData, - @RequestBody TimecardDataDTO timecardDataData + @PostMapping(value = "/execute-batch", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ResponseEntity executeProcessPayrollBatch( + @RequestParam("employeeRecordFile") MultipartFile employeeRecordFile, + @RequestParam("timecardDataFile") MultipartFile timecardDataFile ) { - // TRANSACTIONAL PARADIGM DETECTED - processPayrollService.executeProcessPayroll(/* pass DTOs here */); + // BATCH PARADIGM DETECTED + // Pass the InputStream directly to the Service layer. + processPayrollService.executeProcessPayroll(/* pass streams here */); // Expected Outputs: PAYROLL-RECEIPT return ResponseEntity.ok().build(); }