Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions gitgalaxy/tools/cobol_to_cobol/cobol_jcl_forge.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,14 @@
# character, ASCII exactly as before or any non-ASCII letter or digit. Missing such a SELECT turned the program
# from batch into transactional and its controller named a DTO nothing generates, so the Java did not compile.
_W = r"(?:[A-Z0-9\-]|[^\W\x00-\x7f])"
# #3992: the ASSIGN operand may be a literal (`ASSIGN "TEST-FILE"`, `ASSIGN TO "./input.txt"`: GnuCOBOL,
# opensourcecobol4j, Micro Focus) and the file OPTIONAL, as the engine's own SELECT reader has them
# (mainframe_boundary._SELECT_ASSIGN). Missing the literal form made a batch program transactional (a DTO nothing
# generates); `ASSIGN TO "x"` backtracked to read `TO` as the DD name. A literal is bounded, so an unclosed quote
# cannot run on; the word form never takes the `TO` keyword itself.
_FILE_ASSIGN_ANCHOR = re.compile(
rf"SELECT\s+({_W}+)\s+ASSIGN\s+(?:TO\s+)?({_W}+)",
rf"SELECT\s+(?:OPTIONAL\s+)?({_W}+)\s+ASSIGN\s+(?:TO\s+)?"
rf"(?:\"([^\"]{{1,1024}})\"|'([^']{{1,1024}})'|(?!TO\s)({_W}+))",
re.IGNORECASE,
)

Expand Down Expand Up @@ -86,7 +92,9 @@ def analyze_cobol_intent(filepath: Path, declared: Optional[str] = None) -> dict
continue
consumed = stop + 1
internal_name = match.group(1).strip()
raw_dd = match.group(2).strip()
raw_dd = next(g for g in match.groups()[1:] if g is not None).strip()
if not raw_dd:
continue
clean_dd = re.sub(r"^(?:UT|UR)-S-", "", raw_dd)
intent["files_requested"].append({"internal": internal_name, "dd_name": clean_dd})

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@
_WORD_BREAKS,
capitalize_name,
java_class_base,
java_legal_chars,
java_start_ok,
java_url_segment,
lower_name,
output_key,
Expand Down Expand Up @@ -54,6 +56,11 @@ def generate_rest_controller(
files_requested = base_intent.get("files_requested", [])
is_cics = base_intent.get("is_cics", False)

# #3992: a non-CICS program that opens files is batch. Its files are its SELECTs (files_requested); when the
# intent reader read none (a SELECT form it does not know) the files it OPENs for input (the lineage) stand in,
# so its files arrive as uploads -- never as a request DTO, which no generator writes.
if not files_requested and not is_cics:
files_requested = [{"dd_name": i} for i in inputs]
is_batch = len(files_requested) > 0 and not is_cics

java = []
Expand Down Expand Up @@ -106,6 +113,11 @@ def generate_rest_controller(
if not dd_parts:
dd_parts = ["unknown"]
safe_dd_name = lower_name(dd_parts[0]) + "".join(capitalize_name(word) for word in dd_parts[1:])
# #3992: a literal ASSIGN (`"01.DAT"`, `"KUNDE§NR"`) may start with a digit or hold a character Java
# rejects; such a name is made legal, a legal one is unchanged.
safe_dd_name = java_legal_chars(safe_dd_name)
if not java_start_ok(safe_dd_name[0]):
safe_dd_name = f"file{safe_dd_name}"
base_var_name = f"{safe_dd_name}File"

# Enforce unique variable names and Spring request params
Expand Down Expand Up @@ -135,16 +147,11 @@ def generate_rest_controller(
java.append(' @PostMapping("/execute")')
java.append(f" public ResponseEntity<?> execute{camel_prog}(")

params = []
# #3992: the controller names only classes the run generates. No generator writes a DTO per input file
# (a CICS program's COMMAREA contract comes from its skeleton, forges.cics.controller), so a file input is
# listed for the porter, not bound as an `@RequestBody <File>DTO` that cannot compile.
if inputs:
for i in inputs:
words = [w for w in _WORD_BREAKS.split(i) if w]
safe_class = "".join(capitalize_name(word) for word in words) if words else "Unknown"
safe_var = lower_name(safe_class[0]) + safe_class[1:] if safe_class else "unknown"
params.append(f"@RequestBody {safe_class}DTO {safe_var}Data")

if params:
java.append(" " + ",\n ".join(params))
java.append(f" // Input files: {', '.join(inputs)} (no request DTO is generated for them)")
else:
java.append(" /* No external data dependencies detected */")

Expand Down
196 changes: 196 additions & 0 deletions tests/cobol_mainframe/test_no_jcl_batch_controller.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
"""#3992: a batch program with no JCL -- a PC dialect (GnuCOBOL, opensourcecobol4j, Micro Focus) that ASSIGNs its
files to literals -- gets a REST controller that compiles: its files are uploads (MultipartFile), and the controller
names no class the run does not generate (it used to bind `@RequestBody TestFileDTO`, which nothing writes)."""

from __future__ import annotations

import os
import re
import shutil
import subprocess
import sys
from pathlib import Path
from unittest.mock import patch

import pytest

import gitgalaxy.cobol_refractor_controller as refractor
import gitgalaxy.cobol_to_java_controller as java_controller
from gitgalaxy.tools.cobol_to_cobol.cobol_jcl_forge import analyze_cobol_intent
from gitgalaxy.tools.cobol_to_cobol.galaxy_ir import scan_to_db
from gitgalaxy.tools.cobol_to_java.cobol_to_java_api_contract_forge import generate_rest_controller


def _program(pgm: str, select: str, fd: str, open_: str) -> str:
return (
" IDENTIFICATION DIVISION.\n"
f" PROGRAM-ID. {pgm}.\n"
" ENVIRONMENT DIVISION.\n"
" INPUT-OUTPUT SECTION.\n"
" FILE-CONTROL.\n"
f" {select}\n"
" ORGANIZATION SEQUENTIAL\n"
" FILE STATUS F-STATUS.\n"
" DATA DIVISION.\n"
" FILE SECTION.\n"
f" FD {fd}.\n"
" 01 TEST-RECORD PIC X(10).\n"
" WORKING-STORAGE SECTION.\n"
" 77 F-STATUS PIC X(02).\n"
" PROCEDURE DIVISION.\n"
f" OPEN {open_} {fd}.\n"
" DISPLAY F-STATUS.\n"
f" CLOSE {fd}.\n"
" STOP RUN.\n"
)


# The issue's reproduction, and the other literal / OPTIONAL shapes of the opensourcecobol4j jp-compat suite
ESTATE = {
"ASGPATH.cbl": _program("ASGPATH", 'SELECT TEST-FILE ASSIGN "TEST-FILE"', "TEST-FILE", "I-O"),
"ASGDOT.cbl": _program("ASGDOT", 'SELECT TEST-FILE ASSIGN "./TEST-FILE"', "TEST-FILE", "EXTEND"),
"ASGTO.cbl": _program("ASGTO", 'SELECT IN-FILE ASSIGN TO "./input.txt"', "IN-FILE", "INPUT"),
"ASGDIGIT.cbl": _program("ASGDIGIT", "SELECT OPTIONAL F1 ASSIGN TO '01.DAT'", "F1", "INPUT"),
}


# ---------------------------------------------------------------------------------------------------------------
# The intent reader: the SELECT forms the engine reads (mainframe_boundary._SELECT_ASSIGN)
# ---------------------------------------------------------------------------------------------------------------


@pytest.mark.parametrize(
("select", "internal", "dd_name"),
[
('SELECT TEST-FILE ASSIGN "TEST-FILE"', "TEST-FILE", "TEST-FILE"),
('SELECT TEST-FILE ASSIGN "./TEST-FILE"', "TEST-FILE", "./TEST-FILE"),
('SELECT IN-FILE ASSIGN TO "./input.txt"', "IN-FILE", "./input.txt"),
("SELECT IN-FILE ASSIGN TO 'S01'", "IN-FILE", "S01"),
("SELECT OPTIONAL F1 ASSIGN TO UT-S-DD1", "F1", "DD1"),
("SELECT F2 ASSIGN TO DD2", "F2", "DD2"), # the word forms read as before
("SELECT F3 ASSIGN DD3", "F3", "DD3"),
],
)
def test_select_assign_reads_literals_and_optional(tmp_path, select, internal, dd_name):
pgm = tmp_path / "IO.cbl"
pgm.write_text(f" FILE-CONTROL.\n {select}\n ORGANIZATION SEQUENTIAL.\n")
assert analyze_cobol_intent(pgm)["files_requested"] == [{"internal": internal, "dd_name": dd_name}]


def test_assign_to_a_literal_never_reads_the_to_keyword(tmp_path):
"""`ASSIGN TO "x"` backtracked past the optional TO and read `TO` itself as the DD (`@RequestParam("toFile")`);
an empty or unclosed literal is no file, not `TO`."""
pgm = tmp_path / "IO.cbl"
pgm.write_text(
" FILE-CONTROL.\n"
' SELECT F-EMPTY ASSIGN TO "".\n'
' SELECT F-OPEN ASSIGN TO "' + "X" * 2000 + ".\n"
)
assert analyze_cobol_intent(pgm)["files_requested"] == []


# ---------------------------------------------------------------------------------------------------------------
# The controller
# ---------------------------------------------------------------------------------------------------------------


def _ir(inputs, files_requested=(), is_cics=False, outputs=()):
return {
"metadata": {"file_name": "ASGPATH.cbl"},
"analysis": {
"base_intent": {"files_requested": list(files_requested), "is_cics": is_cics},
"lineage": {"inputs": list(inputs), "outputs": list(outputs)},
},
}


def test_a_non_cics_program_with_input_files_is_batch_without_selects():
"""The issue's IR: lineage.inputs = ['TEST-FILE'], files_requested = [], is_cics = False."""
java = generate_rest_controller(_ir(["TEST-FILE"], outputs=["TEST-FILE"]), "com.acme")
assert '@PostMapping(value = "/execute-batch", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)' in java
assert '@RequestParam("testFileFile") MultipartFile testFileFile' in java
assert "import org.springframework.web.multipart.MultipartFile;" in java
assert "DTO" not in java and "@RequestBody" not in java


def test_the_selects_still_decide_the_upload_parameters():
ir = _ir(["INDD"], files_requested=[{"internal": "IN-FILE", "dd_name": "INDD"}, {"dd_name": "OUTDD"}])
java = generate_rest_controller(ir, "com.acme")
assert '@RequestParam("inddFile") MultipartFile inddFile' in java
assert '@RequestParam("outddFile") MultipartFile outddFile' in java


def test_a_cics_program_names_no_request_dto_for_its_files():
"""The transactional branch lists a file input for the porter; it never binds a <File>DTO nothing generates."""
java = generate_rest_controller(_ir(["CUSTFILE"], is_cics=True), "com.acme")
assert '@PostMapping("/execute")' in java
assert "// Input files: CUSTFILE (no request DTO is generated for them)" in java
assert "DTO " not in java.replace("request DTO is", "") and "@RequestBody" not in java


@pytest.mark.parametrize(
("dd_name", "var"),
[("01.DAT", "file01DatFile"), ("./input.txt", "inputTxtFile"), ("KUNDE§NR", "kunde_nrFile")],
)
def test_a_literal_dd_name_becomes_a_legal_java_identifier(dd_name, var):
java = generate_rest_controller(_ir([], files_requested=[{"dd_name": dd_name}]), "com.acme")
assert f'@RequestParam("{var}") MultipartFile {var}' in java


# ---------------------------------------------------------------------------------------------------------------
# End to end: scan, refract and generate the no-JCL estate; every class a controller names is generated, and
# (with a JDK 17 and Maven that resolve offline) the project compiles.
# ---------------------------------------------------------------------------------------------------------------


@pytest.fixture(scope="module")
def generated(tmp_path_factory):
base = tmp_path_factory.mktemp("no_jcl_batch")
repo = base / "estate"
repo.mkdir()
for name, text in ESTATE.items():
(repo / name).write_text(text, encoding="utf-8")
db = scan_to_db(repo, base / "scan")
with patch.object(sys, "argv", ["refract", str(repo), "--galaxy-db", str(db)]):
refractor.main()
(clean,) = base.glob("estate_gitgalaxy_clean_*")
(base / "none.txt").write_text("", encoding="utf-8")
with patch.object(sys, "argv", ["cobol-to-java", str(clean), "--header", str(base / "none.txt")]):
java_controller.main()
(java,) = base.glob("estate_gitgalaxy_java_spring_*")
return java


_TYPE_USE = re.compile(r"@Request(?:Body|Param\([^)]*\))\s+([A-Za-z_$][\w$]*)\s")
_JDK_TYPES = {"MultipartFile", "String"}


def test_every_controller_is_a_batch_upload_endpoint(generated):
controllers = sorted((generated / "src/main/java/com/gitgalaxy/modernized/controller").glob("*.java"))
assert {p.stem for p in controllers} == {"AsgpathController", "AsgdotController", "AsgtoController",
"AsgdigitController"} # fmt: skip
for p in controllers:
code = p.read_text(encoding="utf-8")
assert "MultipartFile" in code and "@RequestBody" not in code, p.name
assert '"toFile"' not in code, p.name


def test_controllers_name_only_generated_classes(generated):
classes = {p.stem for p in generated.rglob("*.java")} | _JDK_TYPES
for p in (generated / "src/main/java/com/gitgalaxy/modernized/controller").glob("*.java"):
for used in _TYPE_USE.findall(p.read_text(encoding="utf-8")):
assert used in classes, f"{p.name} names {used}, which the run did not generate"


JDK17 = Path(os.environ.get("JDK_17") or "/usr/lib/jvm/java-17-openjdk-amd64")


@pytest.mark.skipif(not shutil.which("mvn") or not (JDK17 / "bin" / "javac").is_file(), reason="no Maven + JDK 17")
def test_the_generated_project_compiles(generated):
env = {**os.environ, "JAVA_HOME": str(JDK17)}
proc = subprocess.run(["mvn", "-q", "-o", "compile"], cwd=generated, env=env, capture_output=True, text=True,
timeout=600) # fmt: skip
out = proc.stdout + proc.stderr
if proc.returncode and "COMPILATION ERROR" not in out and re.search(r"offline|resolve|Could not", out):
pytest.skip("Maven cannot resolve the Spring Boot build offline here")
assert proc.returncode == 0, out[-4000:]
18 changes: 12 additions & 6 deletions tests/tools_recorders/test_golden_forge.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,27 +46,33 @@
# ==============================================================================
# GOLDEN IMAGES (The "Perfect" Expected Outputs)
# ==============================================================================
# #3992: a non-CICS program that opens files is batch even when no SELECT was read -- its input files are
# uploads, never `@RequestBody <File>DTO`, a class nothing generates.
GOLDEN_CONTROLLER = """package com.gitgalaxy.modernized.controller;

import org.springframework.web.bind.annotation.*;
import org.springframework.http.ResponseEntity;
import lombok.RequiredArgsConstructor;
import com.gitgalaxy.modernized.service.ProcessPayrollService;

import org.springframework.web.multipart.MultipartFile;
import org.springframework.http.MediaType;

@RestController
@RequestMapping("/api/v1/process-payroll")
@RequiredArgsConstructor
public class ProcessPayrollController {

private final ProcessPayrollService processPayrollService;

@PostMapping("/execute")
public ResponseEntity<?> executeProcessPayroll(
@RequestBody EmployeeRecordDTO employeeRecordData,
@RequestBody TimecardDataDTO timecardDataData
@PostMapping(value = "/execute-batch", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
public ResponseEntity<?> executeProcessPayrollBatch(
@RequestParam("employeeRecordFile") MultipartFile employeeRecordFile,
@RequestParam("timecardDataFile") MultipartFile timecardDataFile
) {
// TRANSACTIONAL PARADIGM DETECTED
processPayrollService.executeProcessPayroll(/* pass DTOs here */);
// BATCH PARADIGM DETECTED
// Pass the InputStream directly to the Service layer.
processPayrollService.executeProcessPayroll(/* pass streams here */);
// Expected Outputs: PAYROLL-RECEIPT
return ResponseEntity.ok().build();
}
Expand Down
Loading