From aadead9fd3a8629fd1cf2d70fbdca6abb53cd0e7 Mon Sep 17 00:00:00 2001 From: squid-protocol Date: Sun, 4 Oct 2026 20:37:47 -0400 Subject: [PATCH 1/2] test(cobol): referee panel -- independent COBOL parsers scored per fact channel (#4377) tests/tools/referees/: a common referee-facts/1 format, one adapter per referee (Cobrix, Spantree tree-sitter-cobol-enterprise, Eclipse Che4z's analysis CLI, mapa's CallTree), the engine and estate-crucible keys as fact sources, and score.py, which scores every source against the answer keys and against the engine and groups the disagreements for adjudication. Test tooling only: no referee is vendored, and nothing under gitgalaxy/ changes. Part of #4377. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FNWHcUSn5p7eMxFTsB5rFo --- tests/cobol_mainframe/test_referee_score.py | 164 ++++++++++ tests/tools/referees/README.md | 97 ++++++ tests/tools/referees/che4z_adapter.py | 233 ++++++++++++++ tests/tools/referees/cobrix_adapter.py | 131 ++++++++ tests/tools/referees/engine_adapter.py | 116 +++++++ tests/tools/referees/estate_key.py | 118 +++++++ tests/tools/referees/facts.py | 276 +++++++++++++++++ tests/tools/referees/java/CobrixFacts.java | 91 ++++++ tests/tools/referees/mapa_adapter.py | 155 ++++++++++ tests/tools/referees/score.py | 288 ++++++++++++++++++ tests/tools/referees/treesitter_adapter.py | 321 ++++++++++++++++++++ 11 files changed, 1990 insertions(+) create mode 100644 tests/cobol_mainframe/test_referee_score.py create mode 100644 tests/tools/referees/README.md create mode 100644 tests/tools/referees/che4z_adapter.py create mode 100644 tests/tools/referees/cobrix_adapter.py create mode 100644 tests/tools/referees/engine_adapter.py create mode 100644 tests/tools/referees/estate_key.py create mode 100644 tests/tools/referees/facts.py create mode 100644 tests/tools/referees/java/CobrixFacts.java create mode 100644 tests/tools/referees/mapa_adapter.py create mode 100644 tests/tools/referees/score.py create mode 100644 tests/tools/referees/treesitter_adapter.py diff --git a/tests/cobol_mainframe/test_referee_score.py b/tests/cobol_mainframe/test_referee_score.py new file mode 100644 index 0000000000..4040d2e906 --- /dev/null +++ b/tests/cobol_mainframe/test_referee_score.py @@ -0,0 +1,164 @@ +"""The referee panel's scoring logic (#4377): tests/tools/referees/{facts,score}.py. + +No referee is run here -- the panel's tools (JVM / tree-sitter builds) live outside the repo and +run on demand. These tests pin the arithmetic the scorecard reports. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools" / "referees")) + +import cobrix_adapter +import facts as F +import score as S + + +def _key() -> dict: + raw = { + "corpus": "toy", + "ref": "0123456789abcdef", + "programs": { + "A.cbl": { + "program_id": "PROGA", + "units": [ + {"name": "P1", "kind": "paragraph", "line": 10, "end": 12, + "edges": [{"verb": "PERFORM_THRU", "target": "P2", "thru": "P2-EXIT", "line": 11}]}, + {"name": "P2", "kind": "paragraph", "line": 14, "end": 15, + "edges": [{"verb": "GO_TO", "target": "P1", "line": 15}]}, + ], + "main_line": {"line": 8, "end": 9, "edges": [{"verb": "PERFORM", "target": "P1", "line": 9}]}, + "siblings": {"PROGB": {"program_id": "PROGB", "units": [ + {"name": "MAINLINE", "kind": "section", "line": 30, "end": 31, "edges": []}]}}, + "calls": [ + {"verb": "CALL", "form": "literal", "operand": "SUBX", "target": "SUBX", "line": 12}, + {"verb": "LINK", "form": "identifier", "operand": "WS-PGM", "target": "PROGC", "line": 13}, + ], + "copybooks": [{"name": "CPY1"}], + "records": [ + {"line": 5, "level": 1, "name": "WS-A", "pic": "x(3)", "usage": None, "occurs_min": None, + "occurs_max": None, "occurs_depending_on": None, "redefines": None, "value": "abc"}, + {"line": 6, "level": 5, "name": "WS-N", "pic": "S9(4)", "usage": "BINARY", "occurs_min": 1, + "occurs_max": 5, "occurs_depending_on": "WS-K", "redefines": None, "value": None}, + ], + }, + "B.cbl": {"program_id": "PROGD", "units": [], "calls": [], "copybooks": [], "records": []}, + }, + "cics_resources": {"A.cbl": {"operations": [ + {"verb": "READ", "kind": "FILE", "name": "ACCTS", "line": 20}, + {"verb": "SEND", "kind": "MAP", "name": "M1", "line": 21}, + ]}}, + "cics_tasks": {"A.cbl": {"operations": [{"verb": "START ATTACH", "line": 22}]}}, + "sql_access": {"B.cbl": {"accesses": ["read T1"]}}, + "copybook_layouts": {"C.cpy": {"units": ["R/F1 @0+3", "R/F2 @3+2"]}}, + } # fmt: skip + return F.key_doc(raw) + + +def _doc(source: str, files: dict, channels=F.CHANNELS) -> dict: + doc = F.new_doc(source, "v", "toy", channels) + for rel, (status, facts) in files.items(): + F.add_file(doc, rel, facts, status=status, seconds=0.5) + return doc + + +def test_key_doc_shapes() -> None: + k = _key()["files"]["A.cbl"]["facts"] + assert k["program_ids"] == ["PROGA", "PROGB"] + assert "PROGB:MAINLINE" in k["units"] and "P1" in k["units"] + assert "(procedure division) L8-9" in k["unit_extents"] + assert set(k["edges"]) == {"P1 -> PERFORM P2", "P2 -> GO TO P1", "(procedure division) -> PERFORM P1"} + assert set(k["calls"]) == {"CALL 'SUBX'", "LINK WS-PGM"} + assert set(k["call_targets"]) == {"SUBX", "PROGC"} + assert set(k["cics_commands"]) == {"L13 LINK", "L20 READ", "L21 SEND", "L22 START"} + assert k["cics_files"] == ["READ ACCTS"] + assert "L6 WS-N COMP" in k["usage"] # BINARY folds to COMP + assert "L6 WS-N 1..5 DEPENDING ON WS-K" in k["occurs"] + assert "L5 WS-A X(3)" in k["pic"] and "L5 WS-A = ABC" in k["value"] + assert _key()["files"]["C.cpy"]["facts"] == {"layouts": ["R/F1 @0+3", "R/F2 @3+2"]} + + +@pytest.mark.parametrize( + "raw,norm", + [("' ab '", "AB"), ("+12", "12"), ("ZERO", "ZEROES"), ("spaces", "SPACES"), ("SPACE", "SPACES"), ("-3", "-3")], +) +def test_norm_value(raw: str, norm: str) -> None: + assert F.norm_value(raw) == norm + + +def test_depending_on_qualifier_is_dropped() -> None: + assert F.item_values(4, 5, "V", occurs_min=0, occurs_max=9, depending_on="LEN OF GRP")["occurs"] == "L4 V 0..9 DEPENDING ON LEN" + + +def test_call_value_strips_cics_padding() -> None: + assert F.call_value("LINK", "literal", "'INQCUST '") == "LINK 'INQCUST'" + assert F.call_value("CALL", "identifier", "ws-pgm") == "CALL WS-PGM" + + +def test_universe_programs_and_copybooks() -> None: + key = _key() + assert S.universe(key, "units") == ["A.cbl", "B.cbl"] + assert S.universe(key, "layouts") == ["C.cpy"] + + +def test_score_channel_counts_and_failed_files() -> None: + key = _key() + ref = _doc("ref", { + "A.cbl": ("ok", {"units": ["P1", "BOGUS"]}), + "B.cbl": ("fail", {}), + }) # fmt: skip + sc = S.score_channel(key, ref, "units") + assert sc == {"tp": 1, "reported": 2, "true": 3, "true_parsed": 3, "files": 2, "files_parsed": 1} + # B.cbl has no units, so its failure costs nothing; program_ids shows the cost of a failed file + pid = S.score_channel(key, _doc("ref", {"A.cbl": ("ok", {"program_ids": ["PROGA"]}), "B.cbl": ("fail", {})}), "program_ids") + assert (pid["tp"], pid["true"], pid["true_parsed"]) == (1, 3, 2) + + +def test_channel_a_source_cannot_produce_is_na() -> None: + assert S.score_channel(_key(), _doc("ref", {}, channels=["units"]), "layouts") is None + assert S._cell(None) == "n/a" + + +def test_cics_commands_scored_over_key_verbs() -> None: + key = _key() + ref = _doc("ref", {"A.cbl": ("ok", {"cics_commands": ["L20 READ", "L23 ASKTIME", "L24 SEND-SEND_TEXT"]})}) + sc = S.score_channel(key, ref, "cics_commands") + assert (sc["tp"], sc["reported"]) == (1, 1) # ASKTIME / SEND TEXT are not censused: not false positives + + +def test_agreement_and_disagreements_only_on_files_both_parsed() -> None: + key = _key() + eng = _doc("engine", {"A.cbl": ("ok", {"units": ["P1", "P2"]}), "B.cbl": ("ok", {"units": ["X"]})}) + ref = _doc("ref", {"A.cbl": ("partial", {"units": ["P1", "P9"]}), "B.cbl": ("fail", {})}) + assert S.agreement(key, eng, ref, "units") == {"both": 1, "union": 3} + dis = S.disagreements(key, eng, ref, "units") + assert {(d["value"], d["only"], d["in_key"]) for d in dis} == {("P2", "engine", True), ("P9", "ref", False)} + + +def test_scorecard_markdown() -> None: + key = _key() + corpora = {"toy": {"key": key, "sources": { + "engine": _doc("engine", {"A.cbl": ("ok", {"units": ["P1", "P2", "PROGB:MAINLINE"]}), "B.cbl": ("ok", {})}), + "ref": _doc("ref", {"A.cbl": ("ok", {"units": ["P1"]}), "B.cbl": ("fail", {})}, channels=["units"]), + }}} # fmt: skip + md, data, dis = S.scorecard(corpora) + assert "| units | P 100.0% · R 100.0% | P 100.0% · R 33.3% (R parsed 33.3%) |" in md + assert "| layouts | P — · R 0.0% (R parsed —) | n/a |" in md # a keyed copybook the engine never read + assert data["sources"]["ref"]["fail"] == 1 and data["sources"]["ref"]["ok"] == 1 + assert dis["ref"]["units"]["engine-only, in key"]["count"] == 2 + + +def test_cobrix_layout_units_skip_overlays_and_fillers() -> None: + fields = [ + {"root": "R", "name": "R", "level": 1, "group": True, "offset": 0, "size": 10}, + {"root": "R", "name": "A", "level": 5, "group": False, "offset": 0, "size": 4, "pic": "X(4)"}, + {"root": "R", "name": "B", "level": 5, "group": True, "offset": 0, "size": 4, "redefines": "A"}, + {"root": "R", "name": "B1", "level": 10, "group": False, "offset": 0, "size": 4, "pic": "9(4)"}, + {"root": "R", "name": "FILLER", "level": 5, "group": False, "offset": 4, "size": 2, "pic": "X(2)", "filler": True}, + {"root": "R", "name": "C", "level": 5, "group": False, "offset": 6, "size": 4, "pic": "X(2)"}, + ] + assert cobrix_adapter.layout_units(fields) == {"R/A @0+4", "R/C @6+4"} diff --git a/tests/tools/referees/README.md b/tests/tools/referees/README.md new file mode 100644 index 0000000000..868d915d26 --- /dev/null +++ b/tests/tools/referees/README.md @@ -0,0 +1,97 @@ +# COBOL / CICS referee panel (#4377) + +Independent open-source COBOL parsers ("referees") scored against the engine and against the +hand-verified answer keys, per fact channel. Test / validation tooling only: nothing here is +imported by `gitgalaxy/`, and no referee's code is vendored. Every referee is installed outside +the repository and invoked as an external, unmodified tool. + +| file | role | +|---|---| +| `facts.py` | the common `referee-facts/1` format, the canonical value shapes, and the answer key as a fact source | +| `engine_adapter.py` | the engine: a scanned corpus's master DB through GalaxyIR (+ the unit graph `cobol_answer_key.py` reads) | +| `cobrix_adapter.py` + `java/CobrixFacts.java` | Cobrix: copybook record layouts (offsets, sizes) | +| `treesitter_adapter.py` | Spantree tree-sitter-cobol-enterprise: units, extents, edges, calls, COPY, data items, typed EXEC CICS / SQL | +| `che4z_adapter.py` | Eclipse Che4z COBOL LSP's headless `analysis` CLI: units, extents, edges, COPY, data items | +| `mapa_adapter.py` | mapa's CallTree: PROGRAM-IDs, COPY / INCLUDE, CALL / LINK / XCTL targets, DB2 tables, CICS file commands | +| `estate_key.py` | estate-crucible's generated key (#4317) in the answer-key shape, so every adapter runs on it unchanged | +| `score.py` | the scorecard: each source vs the key, agreement with the engine, disagreements for adjudication | + +The scoring rules are in `score.py`'s docstring and pinned by `tests/cobol_mainframe/test_referee_score.py`. + +## Installing the referees + +Put them anywhere outside the checkout (the examples use `$REF`). Record the versions you ran: the +scorecard prints each source's version. + +**Cobrix** (Apache-2.0, AbsaOSS/cobrix; JDK 11+ and Maven): + +```sh +git clone https://github.com/AbsaOSS/cobrix $REF/cobrix +cd $REF/cobrix +# the parent POM still targets Java 7, which current JDKs refuse +mvn -q -pl cobol-parser -am -DskipTests -Dmaven.compiler.source=8 -Dmaven.compiler.target=8 package +mvn -q -pl cobol-parser dependency:build-classpath -Dmdep.outputFile=$REF/cobrix.cp +export COBRIX_CLASSPATH=$(cat $REF/cobrix.cp):$(ls $REF/cobrix/cobol-parser/target/cobol-parser_2.12-*-SNAPSHOT.jar | grep -v javadoc) +``` + +The adapter compiles `java/CobrixFacts.java` into `$REFEREES_CACHE` (default +`~/.cache/gitgalaxy-referees`) on first use. + +**Spantree tree-sitter-cobol-enterprise** (MIT; node, a C compiler, py-tree-sitter >= 0.22): + +```sh +git clone https://github.com/Spantree/tree-sitter-cobol-enterprise $REF/tree-sitter-cobol-enterprise +(cd $REF && npm install tree-sitter-cli@0.25) # src/parser.c is not committed upstream +cd $REF/tree-sitter-cobol-enterprise && $REF/node_modules/.bin/tree-sitter generate # ~5 min +cc -O2 -fPIC -shared -I src src/parser.c src/scanner.c -o $REF/cobol_enterprise.so +pip install tree-sitter +export TS_COBOL_ENTERPRISE_LIB=$REF/cobol_enterprise.so TS_COBOL_ENTERPRISE_REPO=$REF/tree-sitter-cobol-enterprise +``` + +**Eclipse Che4z COBOL Language Support** (EPL-2.0; JDK 17+). The server jar ships in the VS Code +extension; it is used unmodified, as an external process: + +```sh +gh release download 2.5.1 -R eclipse-che4z/che-che4z-lsp-for-cobol -p 'cobol-language-support-2.5.1.vsix' -D $REF/che4z +unzip -q $REF/che4z/cobol-language-support-2.5.1.vsix -d $REF/che4z/vsix +export CHE4Z_SERVER_JAR=$REF/che4z/vsix/extension/server/jar/server.jar +``` + +Known limits of 2.5.1's `analysis` CLI (see the scorecard): it prints no result and exits 1 on any +program containing a `CALL` statement, its `--ast` JSON writer overflows / runs out of memory on +large CICS programs, and its serialised OCCURS clause carries no DEPENDING ON. Driving the server +over LSP (`textDocument/documentSymbol`) would avoid all three; that is #4377's next phase. + +**mapa** (MIT, cschneid-the-elder/mapa; JDK 21). `cobol/CallTree.jar` is committed upstream: + +```sh +git clone https://github.com/cschneid-the-elder/mapa $REF/mapa +export MAPA_CALLTREE_JAR=$REF/mapa/cobol/CallTree.jar +``` + +## Running the panel + +```sh +# 1. engine facts: scan with the checkout under test (cached per engine commit), then extract +python tests/tools/mainframe_corpus.py scan +python tests/tools/referees/engine_adapter.py --db "$(python tests/tools/mainframe_corpus.py path --db)" \ + --corpus --key tests/cobol_mainframe/answer_key/.json --out $FACTS//engine.json + +# 2. each referee, on the same members (the key's programs / keyed copybooks) +for a in cobrix treesitter che4z mapa; do + python tests/tools/referees/${a}_adapter.py --corpus --root "$(python tests/tools/mainframe_corpus.py path )" \ + --key tests/cobol_mainframe/answer_key/.json --out $FACTS//$a.json +done + +# estate-crucible: convert its key once, scan with tests/tools/estate_crucible.py --scan-dir, then as above +python tests/tools/referees/estate_key.py --crucible ../estate-crucible \ + --out $WORK/estate-crucible.answer_key.json --facts $FACTS/estate-crucible/key.json + +# 3. the scorecard +python tests/tools/referees/score.py --facts $FACTS --md scorecard.md --json scorecard.json \ + --disagreements disagreements.json +``` + +`disagreements.json` groups every engine / referee difference by pattern +(`engine-only, in key`, `-only, not in key`, ...) with a seeded sample of each, for +adjudication against the source: engine wrong, referee wrong, definitional, or key wrong. diff --git a/tests/tools/referees/che4z_adapter.py b/tests/tools/referees/che4z_adapter.py new file mode 100644 index 0000000000..ea5dc4d910 --- /dev/null +++ b/tests/tools/referees/che4z_adapter.py @@ -0,0 +1,233 @@ +r"""Eclipse Che4z COBOL Language Support referee (#4377): the language server's headless +`analysis` command (EPL-2.0; run as an external, unmodified tool, never vendored), as a +referee-facts/1 document: PROGRAM-IDs, paragraphs / sections with extents, PERFORM / GO TO edges, +COPY members and the DATA DIVISION items with their clauses. + + python tests/tools/referees/che4z_adapter.py --corpus --root \ + --key tests/cobol_mainframe/answer_key/.json --out facts//che4z.json + +Needs the environment variable CHE4Z_SERVER_JAR: server/jar/server.jar unpacked from the +cobol-language-support VSIX release (README.md). Each member runs as +`java -jar server.jar analysis -s -cf= --ast`; the AST's first stage +(`START`) is read. Che4z's lines are 0-based. A member whose AST cannot be serialised (the server's +JSON writer overflows its stack on some trees) falls back to a run without `--ast`: it then counts +as `fail` for facts but its diagnostics still say whether it parsed. Per-member time is the +server's own `timings.total` (JVM start-up excluded; it is reported separately as wall time). +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any, Optional + +HERE = Path(__file__).resolve().parent +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +import facts as F + +JAR_ENV = "CHE4Z_SERVER_JAR" +COPY_EXTS = (".cpy", ".CPY", ".cbl", ".CBL", ".dcl", ".DCL", ".copy", ".cob") +CHANNELS = ["program_ids", "units", "unit_extents", "edges", "copybooks", + "data_items", "pic", "usage", "occurs", "redefines", "value"] # fmt: skip +UNIT_TYPES = ("PARAGRAPH", "PROCEDURE_SECTION") + + +def che4z_version(jar: Path) -> str: + out = subprocess.run(["unzip", "-p", str(jar), "META-INF/MANIFEST.MF"], capture_output=True, text=True) + for ln in out.stdout.splitlines(): + if ln.startswith("Version:"): + return ln.split(":", 1)[1].strip() + return jar.name + + +def _line(loc: dict[str, Any], end: bool = False) -> int: + return int(loc["range"]["end" if end else "start"]["line"]) + 1 + + +def _in_file(node: dict[str, Any], uri: str) -> bool: + return node.get("locality", {}).get("uri", "").rstrip("/") == uri + + +def _name(node: dict[str, Any]) -> Optional[str]: + if node.get("name"): + return str(node["name"]).upper() + for c in node.get("children", []): + if c.get("nodeType") in ("PARAGRAPH_NAME_NODE", "SECTION_NAME_NODE") and c.get("name"): + return str(c["name"]).upper() + return None + + +def _first(clauses: list[Any]) -> Optional[str]: + if not clauses: + return None + c = clauses[0] + if isinstance(c, dict): + for k in ("name", "value", "usage", "format"): + if c.get(k): + return str(c[k]) + return None + return str(c) + + +def _usage(clauses: list[Any]) -> Optional[str]: + u = _first(clauses) + return u.replace("_", "-") if u else None # Che4z writes the enum name, COMP_3 + + +def _occurs(clauses: list[Any]) -> tuple[Optional[int], Optional[int], Optional[str]]: + if not clauses or not isinstance(clauses[0], dict): + return None, None, None + c = clauses[0] + lo, hi = c.get("from"), c.get("to") + if hi is None: + hi = lo + # The serialised clause carries no DEPENDING ON object (2.5.1): an ODO table scores as a miss. + dep = c.get("dependingOn") + if isinstance(dep, dict): + dep = dep.get("name") + return lo, hi, (str(dep) if dep else None) + + +def _value(clauses: list[Any]) -> Optional[str]: + if not clauses or not isinstance(clauses[0], dict): + return None + iv = clauses[0].get("valueIntervals") or [] + return str(iv[0].get("from")) if iv and iv[0].get("from") is not None else None + + +def ast_facts(doc: dict[str, Any], path: Path) -> dict[str, set[str]]: + uri = path.resolve().as_uri().replace("file:///", "file:/") + root = doc["asts"]["START"] + f: dict[str, set[str]] = {ch: set() for ch in CHANNELS} + programs: list[str] = [] + spans: dict[str, list[int]] = {} + + def visit(n: dict[str, Any], unit: Optional[str], in_proc: bool, prog: int) -> None: + t = n.get("nodeType") + if t == "PROGRAM_ID" and n.get("programId"): + programs.append(str(n["programId"]).upper()) + f["program_ids"].add(programs[-1]) + if t == "PROGRAM" and programs: + prog = len(programs) + if t == "COPY" and n.get("name"): + f["copybooks"].add(str(n["name"]).upper()) + if t == "VARIABLE_DEFINITION" and _in_file(n, uri): + name = (n.get("variableName") or {}).get("name") or "FILLER" + ln = _line(n.get("levelLocality") or n["locality"]) + lo, hi, dep = _occurs(n.get("occursClauses") or []) + F.merge_item(f, F.item_values(ln, int(n.get("level") or 0), str(name), _first(n.get("picClauses") or []), + _usage(n.get("usageClauses") or []), lo, hi, dep, + _first(n.get("redefinesClauses") or []), _value(n.get("valueClauses") or []))) # fmt: skip + if t == "DIVISION" and n.get("divisionType") == "PROCEDURE_DIVISION" and _in_file(n, uri): + in_proc = True + spans.setdefault(F.MAIN_LINE, [_line(n["locality"]), 0]) + unit = F.MAIN_LINE + if t in UNIT_TYPES and _in_file(n, uri): + name = _name(n) + if name: + pid = programs[-1] if programs else None + unit = F.unit_name(max(len(programs) - 1, 0), pid, name) + f["units"].add(unit) + start = _line(n["locality"]) + spans[unit] = [start, start] # the header; its own sentences extend it below + if t == "SENTENCE" and unit in spans and _in_file(n, uri): + # a unit ends at its last sentence: trailing comments and the sentences of a + # section's paragraphs are not its own (the answer keys' extent contract) + spans[unit][1] = max(spans[unit][1], _line(n["locality"], end=True)) + if unit is not None and _in_file(n, uri): + if t == "PERFORM" and (n.get("target") or {}).get("name"): + f["edges"].add(F.edge_value(unit, "PERFORM", str(n["target"]["name"]))) + if t == "GO_TO": + for tgt in n.get("targets") or []: + f["edges"].add(F.edge_value(unit, "GO_TO", str(tgt))) + for c in n.get("children", []): + visit(c, unit, in_proc, prog) + + visit(root, None, False, 0) + for name, (s, e) in spans.items(): + if e and e >= s: + f["unit_extents"].add(f"{name} L{s}-{e}") + return f + + +def copy_dirs(root: Path) -> list[str]: + return sorted({str(p.parent) for p in root.rglob("*") if p.is_file() and ".git" not in p.parts and p.suffix in COPY_EXTS}) + + +def run_member(jar: Path, path: Path, dirs: list[str], work: Path, ast: bool) -> tuple[Optional[dict[str, Any]], str]: + cmd = ["java", "-Xss64m", "-jar", str(jar), "analysis", "-s", str(path)] + cmd += [f"-cf={d}" for d in dirs] + [f"-ce={e.lstrip('.')}" for e in COPY_EXTS] + (["--ast"] if ast else []) + out = work / "out.json" + with out.open("wb") as fh: + proc = subprocess.run(cmd, stdout=fh, stderr=subprocess.PIPE, timeout=600) + try: + return json.loads(out.read_text(encoding="utf-8")), "" + except (json.JSONDecodeError, UnicodeDecodeError): + return None, (proc.stderr.decode("utf-8", "backslashreplace").splitlines() or ["no output"])[0][:200] + finally: + out.unlink(missing_ok=True) + + +def che4z_doc(corpus: str, root: Path, key: dict[str, Any]) -> dict[str, Any]: + jar = Path(os.environ.get(JAR_ENV) or "") + if not jar.is_file(): + raise SystemExit(f"{JAR_ENV} does not name the Che4z server.jar (see tests/tools/referees/README.md)") + doc = F.new_doc("che4z", che4z_version(jar), corpus, CHANNELS) + dirs = copy_dirs(root) + scratch = os.environ.get("REFEREES_CACHE") + if scratch: + Path(scratch).mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix="che4z-", dir=scratch)) + wall = 0.0 + for rel in sorted(key.get("programs", {})): + path = root / rel + t0 = time.perf_counter() + out, err = run_member(jar, path, dirs, work, ast=True) + if out is None: + diag, err2 = run_member(jar, path, dirs, work, ast=False) + wall += time.perf_counter() - t0 + syntax = [d for d in (diag or {}).get("diagnostics", []) if d.get("severity") == "ERROR" and d.get("code") != "missing copybook"] # fmt: skip + note = "parsed clean" if diag is not None and not syntax else f"{len(syntax)} error diagnostic(s)" + F.add_file(doc, rel, {}, status="fail", seconds=(diag or {}).get("timings", {}).get("total"), + error=f"AST not serialisable ({err}); without --ast: {note if diag else err2}") # fmt: skip + continue + wall += time.perf_counter() - t0 + if "asts" not in out: + # 2.5.1's CLI prints only {uri, language} and exits 1 on a program with a CALL statement + F.add_file(doc, rel, {}, status="fail", error="analysis returned no AST and no diagnostics (CLI crash)") + continue + syntax = [d for d in out.get("diagnostics", []) if d.get("severity") == "ERROR" and d.get("code") != "missing copybook"] # fmt: skip + try: + facts = ast_facts(out, path) + except (KeyError, TypeError, ValueError) as e: + F.add_file(doc, rel, {}, status="fail", error=f"AST walk: {type(e).__name__}: {e}") + continue + F.add_file(doc, rel, facts, status="partial" if syntax else "ok", seconds=out.get("timings", {}).get("total"), + error=(syntax[0].get("suggestion") or syntax[0].get("message") or "")[:160] if syntax else None) # fmt: skip + work.rmdir() + doc["wall_seconds"] = wall + return doc + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--corpus", required=True) + ap.add_argument("--root", type=Path, required=True) + ap.add_argument("--key", type=Path, required=True) + ap.add_argument("--out", type=Path, required=True) + args = ap.parse_args() + key = json.loads(args.key.read_text(encoding="utf-8")) + F.dump(che4z_doc(args.corpus, args.root, key), args.out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tools/referees/cobrix_adapter.py b/tests/tools/referees/cobrix_adapter.py new file mode 100644 index 0000000000..7aa2ee7af3 --- /dev/null +++ b/tests/tools/referees/cobrix_adapter.py @@ -0,0 +1,131 @@ +r"""Cobrix referee (#4377): copybook record layouts -- offsets and sizes -- from AbsaOSS/cobrix's +CopybookParser (Apache-2.0), as a referee-facts/1 document with the `layouts` channel. + + python tests/tools/referees/cobrix_adapter.py --corpus --root \ + --key tests/cobol_mainframe/answer_key/.json --out facts//cobrix.json + +Needs the environment variable COBRIX_CLASSPATH: the cobol-parser jar plus its runtime +dependencies, built outside this repository (README.md). The small Java driver +java/CobrixFacts.java is compiled on first use into $REFEREES_CACHE (default ~/.cache/gitgalaxy-referees). + +The layout contract is the engine's and the key's (#3602): `ROOT/NAME @offset+bytes` per named +elementary item with a PICTURE, offsets relative to the 01; an item under a REDEFINES (or a root +that REDEFINES) is an overlay and left out; an item inside an OCCURS group is listed once, at its +first occurrence, its size covering its own OCCURS. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +import facts as F + +DRIVER = HERE / "java" / "CobrixFacts.java" + + +def cache_dir() -> Path: + return Path(os.environ.get("REFEREES_CACHE") or Path.home() / ".cache" / "gitgalaxy-referees") + + +def classpath() -> str: + cp = os.environ.get("COBRIX_CLASSPATH") + if not cp: + raise SystemExit("COBRIX_CLASSPATH is not set (see tests/tools/referees/README.md)") + return cp + + +def compile_driver(cp: str) -> Path: + out = cache_dir() / "cobrix-driver" + cls = out / "CobrixFacts.class" + if not cls.is_file() or cls.stat().st_mtime < DRIVER.stat().st_mtime: + out.mkdir(parents=True, exist_ok=True) + subprocess.run(["javac", "-d", str(out), "-cp", cp, str(DRIVER)], check=True) + return out + + +def cobrix_version(cp: str) -> str: + for part in cp.split(os.pathsep): + name = Path(part).name + if name.startswith("cobol-parser"): + return name.removesuffix(".jar") + return "unknown" + + +def layout_units(fields: list[dict[str, Any]]) -> set[str]: + """The #3602 layout units of one Cobrix-parsed copybook (see the module docstring).""" + out: set[str] = set() + root_at: dict[str, int] = {} + stack: list[tuple[int, bool]] = [] # (level, overlaid) of the open ancestors + for f in fields: + while stack and stack[-1][0] >= f["level"]: + stack.pop() + overlaid = bool(f.get("redefines")) or (bool(stack) and stack[-1][1]) + if f["level"] == 1 or not stack: + root_at[f["root"]] = f["offset"] + if f["group"]: + stack.append((f["level"], overlaid)) + continue + if overlaid or f.get("filler") or not f.get("pic") or f["name"].upper() == "FILLER": + continue + at = f["offset"] - root_at.get(f["root"], 0) + out.add(f"{f['root'].upper()}/{f['name'].upper()} @{at}+{f['size']}") + return out + + +def run(paths: list[Path], cp: str) -> list[dict[str, Any]]: + driver = compile_driver(cp) + proc = subprocess.run( + ["java", "-cp", cp + os.pathsep + str(driver), "CobrixFacts"], + input="\n".join(str(p) for p in paths) + "\n", + capture_output=True, + text=True, + check=True, + ) + return [json.loads(line) for line in proc.stdout.splitlines() if line.startswith("{")] + + +def cobrix_doc(corpus: str, root: Path, key: dict[str, Any]) -> dict[str, Any]: + cp = classpath() + rels = sorted(key.get("copybook_layouts", {})) + t0 = time.perf_counter() + rows = run([root / r for r in rels], cp) + wall = time.perf_counter() - t0 + doc = F.new_doc("cobrix", cobrix_version(cp), corpus, ["layouts"]) + doc["wall_seconds"] = wall + by_path = {Path(r["path"]).resolve(): r for r in rows} + for rel in rels: + r = by_path.get((root / rel).resolve()) + if r is None: + F.add_file(doc, rel, {}, status="fail", error="no output from the driver") + elif not r["ok"]: + F.add_file(doc, rel, {}, status="fail", seconds=r["seconds"], error=r["error"]) + else: + F.add_file(doc, rel, {"layouts": layout_units(r["fields"])}, seconds=r["seconds"]) + return doc + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--corpus", required=True) + ap.add_argument("--root", type=Path, required=True) + ap.add_argument("--key", type=Path, required=True) + ap.add_argument("--out", type=Path, required=True) + args = ap.parse_args() + key = json.loads(args.key.read_text(encoding="utf-8")) + F.dump(cobrix_doc(args.corpus, args.root, key), args.out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tools/referees/engine_adapter.py b/tests/tools/referees/engine_adapter.py new file mode 100644 index 0000000000..6d70b8123e --- /dev/null +++ b/tests/tools/referees/engine_adapter.py @@ -0,0 +1,116 @@ +r"""The engine's side of the referee panel (#4377): a scanned corpus's master DB, read through +GalaxyIR plus the unit graph tests/tools/cobol_answer_key.py reads, as a referee-facts/1 document. + + python tests/tools/referees/engine_adapter.py --db _galaxy_master.db --corpus \ + --key tests/cobol_mainframe/answer_key/.json --out facts//engine.json + +The scan itself is `tests/tools/mainframe_corpus.py scan ` from the checkout under test. +Only the files the key covers are emitted (the key's programs and its keyed copybook layouts). +""" + +from __future__ import annotations + +import argparse +import json +import sqlite3 +import subprocess +import sys +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parents[2] +for p in (str(REPO_ROOT), str(HERE.parent), str(HERE)): + if p not in sys.path: + sys.path.insert(0, p) + +import cobol_answer_key as ak +import facts as F + + +def engine_version() -> str: + """The engine checkout's HEAD (the scan is cached per engine commit, mainframe_corpus.py).""" + out = subprocess.run(["git", "rev-parse", "--short=12", "HEAD"], cwd=REPO_ROOT, capture_output=True, text=True) + return out.stdout.strip() or "unknown" + + +def _raw_imports(db: Path, repo_name: str, commit_hash: str) -> dict[str, set[str]]: + """file -> every COPY / INCLUDE member the scan saw, resolved or not (file_data.raw_imports); + GalaxyIR's copy_deps keeps only the resolved ones.""" + con = sqlite3.connect(f"file:{db}?mode=ro", uri=True) + try: + rows = con.execute( + "SELECT file_path, raw_imports FROM file_data WHERE repo_name = ? AND commit_hash = ?", (repo_name, commit_hash) + ).fetchall() + finally: + con.close() + return {(p or "").replace("\\", "/"): {str(x).upper() for x in json.loads(raw or "[]")} for p, raw in rows} + + +def engine_doc(db: Path, corpus: str, key: dict[str, Any]) -> dict[str, Any]: + from gitgalaxy.tools.cobol_to_cobol.galaxy_ir import load_galaxy_ir + + ir = load_galaxy_ir(db) + graph = ak.engine_unit_graph(db, ir.repo_name, ir.commit_hash) + access: dict[str, set[str]] = {} + for row in ir.sql_table_access(): + access.setdefault(row["file"], set()).update(f"{a} {row['table']}" for a in row["accesses"]) + doc = F.new_doc("engine", engine_version(), corpus, F.CHANNELS) + raw_imports = _raw_imports(db, ir.repo_name, ir.commit_hash) + for rel, prog in key.get("programs", {}).items(): + ef = ir.files.get(rel) + if ef is None: + F.add_file(doc, rel, {}, status="fail", error="file not in the master DB") + continue + f: dict[str, set[str]] = {ch: set() for ch in F.CHANNELS if ch not in F.COPYBOOK_CHANNELS} + f["program_ids"] = {p.upper() for p in ef.program_ids} + units = graph.get(rel, []) + call_names = {n.upper() for c in ef.calls for n in (c.operand, c.target) if n} + f["unit_extents"] = ak.engine_unit_extents(prog, units) + f["units"] = {ak.keyed_unit_name(prog, u["name"], u["start"]) for u in units if not u["synthetic"]} + f["edges"] = ak.engine_unit_edges(prog, units, call_names) + for c in ef.calls: + if c.verb.upper() not in ("CALL", "LINK", "XCTL"): + continue + v = F.call_value(c.verb, c.form, c.operand) + if v: + f["calls"].add(v) + if c.target: + f["call_targets"].add(c.target.upper()) + if c.verb.upper() in ("LINK", "XCTL"): + f["cics_commands"].add(f"L{c.line} {c.verb.upper()}") + f["copybooks"] = raw_imports.get(rel, set()) + for it in ef.data_items: + F.merge_item(f, F.item_values(it.line, it.level, it.name, it.pic, it.usage, it.occurs_min, + it.occurs_max, it.occurs_depending_on, it.redefines, it.value)) # fmt: skip + f["sql_access"] = access.get(rel, set()) + for r in ef.cics_resources: + f["cics_commands"].add(F.cics_command(r.line, r.verb)) + if r.kind == "FILE" and r.name: + f["cics_files"].add(f"{F.cics_verb(r.verb)} {r.name.upper()}") + for t in ef.cics_tasks: + f["cics_commands"].add(F.cics_command(t.line, t.verb)) + F.add_file(doc, rel, f) # per-member time is the scan's, recorded by score.py --engine-seconds + for rel in key.get("copybook_layouts", {}): + units = ak.engine_copybook_units(ir, rel) + if units is None: + F.add_file(doc, rel, {}, status="fail", error="copybook not in the master DB") + else: + F.add_file(doc, rel, {"layouts": units}) + return doc + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--db", type=Path, required=True) + ap.add_argument("--corpus", required=True) + ap.add_argument("--key", type=Path, required=True) + ap.add_argument("--out", type=Path, required=True) + args = ap.parse_args() + key = json.loads(args.key.read_text(encoding="utf-8")) + F.dump(engine_doc(args.db, args.corpus, key), args.out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tools/referees/estate_key.py b/tests/tools/referees/estate_key.py new file mode 100644 index 0000000000..ee24e611b1 --- /dev/null +++ b/tests/tools/referees/estate_key.py @@ -0,0 +1,118 @@ +r"""estate-crucible's generated key (format `estate-crucible-key/1`, #4317) in the answer-key shape the +referee adapters read (tests/cobol_mainframe/answer_key/*.json: `programs`, `cics_resources`, +`sql_access`), so that the same adapters and score.py run on the synthetic estate unchanged. + + python tests/tools/referees/estate_key.py --crucible ../estate-crucible \ + --out /estate-crucible.answer_key.json [--facts /estate-crucible/key.json] + +Only COBOL members with a PROGRAM-ID are carried. The estate key's layouts are per program record +(COPY-expanded), not per copybook, so they are not mapped onto the copybook `layouts` channel. +A `mainline` unit and the main-line edges (`from: null`) become the `(procedure division)` +pseudo-unit, as in the answer keys. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +import facts as F + +_EDGE_VERB = {"perform": "PERFORM", "perform_thru": "PERFORM_THRU", "goto": "GO_TO", "go_to": "GO_TO"} +_PROGRAM_CALLS = ("CALL", "LINK", "XCTL") + + +def _occurs(raw: Any) -> tuple[Any, Any, Any]: + if raw is None: + return None, None, None + if isinstance(raw, int): + return raw, raw, None + if isinstance(raw, dict): + hi = raw.get("max", raw.get("times")) + return raw.get("min", hi), hi, raw.get("depending_on") + return None, None, None + + +def convert(crucible: Path) -> dict[str, Any]: + manifest = json.loads((crucible / "key" / "manifest.json").read_text(encoding="utf-8")) + out: dict[str, Any] = {"corpus": "estate-crucible", "ref": manifest.get("generator", {}).get("version", "") + if isinstance(manifest.get("generator"), dict) else str(manifest.get("generator", "")), + "programs": {}, "cics_resources": {}, "sql_access": {}} # fmt: skip + for app in sorted((crucible / "key" / "apps").glob("*.json")): + for rel, m in json.loads(app.read_text(encoding="utf-8"))["members"].items(): + if m.get("language") != "cobol" or not m.get("programs"): + continue + units = [] + main_edges = [] + by_unit: dict[str, list[dict[str, Any]]] = {} + for e in m.get("edges", []): + if e.get("kind") not in _EDGE_VERB: + continue + edge = {"verb": _EDGE_VERB[e["kind"]], "target": e["target"], "line": e.get("line")} + (main_edges if not e.get("from") else by_unit.setdefault(e["from"], [])).append(edge) + main_span: dict[str, Any] = {"line": None, "end": None} + for u in m.get("units", []): + if u.get("kind") == "mainline" or not u.get("name"): + main_span = {"line": u["start_line"], "end": u["end_line"]} + continue + units.append({"name": u["name"], "kind": u.get("kind", "paragraph"), "line": u["start_line"], + "end": u["end_line"], "edges": by_unit.get(u["name"], [])}) # fmt: skip + progs = sorted(m["programs"], key=lambda p: p.get("line") or 0) + starts = [p.get("line") or 0 for p in progs] + ends = [nxt - 1 for nxt in starts[1:]] + [10**9] + sibling_units: dict[str, list[dict[str, Any]]] = {p["program_id"]: [] for p in progs[1:]} + first_units = [] + for u in units: # a second program's units go to its own block (#4206 naming) + owner = next((p["program_id"] for p, s, e in zip(progs[1:], starts[1:], ends[1:]) if s <= u["line"] <= e), None) + (sibling_units[owner] if owner else first_units).append(u) + units = first_units + records = [] + for d in m.get("data_items", []): + lo, hi, dep = _occurs(d.get("occurs")) + records.append({"line": d["line"], "level": d["level"], "name": d.get("name") or "FILLER", + "pic": d.get("pic"), "usage": d.get("usage"), "occurs_min": lo, "occurs_max": hi, + "occurs_depending_on": dep, "redefines": d.get("redefines"), "value": d.get("value")}) # fmt: skip + out["programs"][rel] = { + "program_id": progs[0]["program_id"], + "siblings": {p["program_id"]: {"program_id": p["program_id"], "units": sibling_units[p["program_id"]], + "line": st, "end_line": en} + for p, st, en in zip(progs[1:], starts[1:], ends[1:])}, # fmt: skip + "units": units, + "main_line": {**main_span, "edges": main_edges} if main_edges or main_span["line"] else None, + "calls": [{"verb": c["verb"], "form": c.get("form", "literal"), "operand": c.get("operand"), + "target": c.get("target"), "line": c.get("line")} + for c in m.get("call_sites", []) if c.get("verb") in _PROGRAM_CALLS], # fmt: skip + "copybooks": [{"name": c["member"]} for c in m.get("copies", [])], + "records": records, + } + if m.get("cics_resources"): + out["cics_resources"][rel] = {"operations": m["cics_resources"]} + acc = sorted({f"{s['access']} {s['table']}" for s in m.get("sql_statements", []) if s.get("table") and s.get("access")}) + if acc: + out["sql_access"][rel] = {"accesses": acc} + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--crucible", type=Path, required=True) + ap.add_argument("--out", type=Path, required=True) + ap.add_argument("--facts", type=Path, help="also write the converted key as a referee-facts/1 `key` document") + args = ap.parse_args() + key = convert(args.crucible) + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(json.dumps(key, indent=1, sort_keys=True) + "\n", encoding="utf-8") + if args.facts: + F.dump(F.key_doc(key), args.facts) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tools/referees/facts.py b/tests/tools/referees/facts.py new file mode 100644 index 0000000000..07f834e67e --- /dev/null +++ b/tests/tools/referees/facts.py @@ -0,0 +1,276 @@ +r"""Common fact format for the COBOL / CICS referee panel (#4377). + +Every source of facts -- the engine (master DB via GalaxyIR), an independent open-source parser +(a "referee"), and the hand-verified answer key (ground truth) -- is reduced to the same shape: + + { + "schema": "referee-facts/1", + "source": "engine" | "key" | "", + "version": "", + "corpus": "", + "channels": [], + "files": { + "": { + "status": "ok" | "partial" | "fail", # clean parse / parsed with recovered errors / no facts + "seconds": , + "error": , + "facts": {"": ["", ...], ...} + } + } + } + +A channel value is a canonical string, so that comparing two sources is plain set arithmetic. The +shapes follow the answer key's own score pairs (tests/tools/cobol_answer_key.py) wherever one +exists, so the engine-vs-key numbers here reproduce that tool's: + + program_ids PROGRAM-ID + units paragraph / section name (`PROG:NAME` for a second program in a file) + unit_extents `NAME L-`, the main line as `(procedure division)` + edges `FROM -> PERFORM TARGET` / `FROM -> GO TO TARGET` (THRU scores as PERFORM) + calls `CALL 'LIT'` (static form) / `CALL IDENT` (dynamic form), also LINK / XCTL + call_targets the program a CALL / LINK / XCTL names, through a VALUE for an identifier + copybooks COPY / EXEC SQL INCLUDE member name + data_items `L NAME` of every data description in the program's own source + pic, usage, occurs, redefines, value `L NAME ` per item that carries it + layouts `ROOT/NAME @offset+bytes` per elementary PIC item of a copybook (#3602 contract) + sql_access ` TABLE` + cics_commands `L VERB` of an EXEC CICS command (scored over the verbs the key censuses) + cics_files `VERB DATASET` of an EXEC CICS file-control command, the name resolved +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any, Optional +from collections.abc import Iterable + +SCHEMA = "referee-facts/1" + +CHANNELS = ( + "program_ids", + "units", + "unit_extents", + "edges", + "calls", + "call_targets", + "copybooks", + "data_items", + "pic", + "usage", + "occurs", + "redefines", + "value", + "layouts", + "sql_access", + "cics_commands", + "cics_files", +) + +# Channels whose universe of files is the key's copybook layouts rather than its programs. +COPYBOOK_CHANNELS = frozenset({"layouts"}) +MAIN_LINE = "(procedure division)" +CICS_FILE_VERBS = frozenset({"READ", "WRITE", "REWRITE", "DELETE", "STARTBR", "READNEXT", "READPREV", "ENDBR", "RESETBR", "UNLOCK"}) # fmt: skip + +_USAGE_ALIASES = { + "COMPUTATIONAL": "COMP", + "COMPUTATIONAL-1": "COMP-1", + "COMPUTATIONAL-2": "COMP-2", + "COMPUTATIONAL-3": "COMP-3", + "COMPUTATIONAL-4": "COMP-4", + "COMPUTATIONAL-5": "COMP-5", + "PACKED-DECIMAL": "COMP-3", + "COMP-4": "COMP", + "BINARY": "COMP", +} +_FIGURATIVE = { + "SPACE": "SPACES", + "ZERO": "ZEROES", + "ZEROS": "ZEROES", + "HIGH-VALUE": "HIGH-VALUES", + "LOW-VALUE": "LOW-VALUES", + "QUOTE": "QUOTES", +} + + +def new_doc(source: str, version: str, corpus: str, channels: Iterable[str]) -> dict[str, Any]: + return {"schema": SCHEMA, "source": source, "version": version, "corpus": corpus, + "channels": sorted(channels), "files": {}} # fmt: skip + + +def add_file( + doc: dict[str, Any], + rel: str, + facts: dict[str, Iterable[str]], + status: str = "ok", + seconds: Optional[float] = None, + error: Optional[str] = None, +) -> None: + doc["files"][rel] = { + "status": status, + "seconds": seconds, + "error": error, + "facts": {ch: sorted(set(v)) for ch, v in facts.items() if ch in CHANNELS}, + } + + +def dump(doc: dict[str, Any], path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(doc, indent=1, sort_keys=True) + "\n", encoding="utf-8") + + +def load(path: Path) -> dict[str, Any]: + doc = json.loads(path.read_text(encoding="utf-8")) + if doc.get("schema") != SCHEMA: + raise ValueError(f"{path}: not a {SCHEMA} document") + return doc + + +# ------------------------------------------------------------------------------ +# Canonical value builders, shared by every adapter so the shapes cannot drift. +# ------------------------------------------------------------------------------ +def norm_usage(usage: Optional[str]) -> Optional[str]: + if not usage: + return None + u = re.sub(r"\s+", " ", usage.upper().strip()) + return _USAGE_ALIASES.get(u, u) + + +def norm_value(value: Optional[str]) -> Optional[str]: + """A VALUE clause's first literal: quotes dropped, figurative-constant synonyms folded, + `ALL` kept, numerics as written.""" + if value is None: + return None + v = value.strip() + if len(v) >= 2 and v[0] in "'\"" and v[-1] == v[0]: + v = v[1:-1] + elif re.fullmatch(r"[+-]?[0-9.,]+", v): + v = v.lstrip("+") # the key reads `+12` as `12` + v = v.upper().strip() # the key folds case and trims a literal's blanks (definitional, #4377) + return _FIGURATIVE.get(v, v) + + +def norm_pic(pic: Optional[str]) -> Optional[str]: + return re.sub(r"\s+", "", pic.upper()) if pic else None + + +def item_values( + line: int, + level: int, + name: str, + pic: Optional[str] = None, + usage: Optional[str] = None, + occurs_min: Optional[int] = None, + occurs_max: Optional[int] = None, + depending_on: Optional[str] = None, + redefines: Optional[str] = None, + value: Optional[str] = None, +) -> dict[str, str]: + """The per-item channel values of one data description entry.""" + name = (name or "FILLER").upper() + out = {"data_items": f"L{line} {level:02d} {name}"} + if pic: + out["pic"] = f"L{line} {name} {norm_pic(pic)}" + u = norm_usage(usage) + if u and u != "DISPLAY": + out["usage"] = f"L{line} {name} {u}" + if occurs_max is not None: + lo = occurs_min if occurs_min is not None else occurs_max + # the key names the DEPENDING ON item unqualified (`X OF Y` -> `X`) + dep = f" DEPENDING ON {depending_on.split()[0].upper()}" if depending_on else "" + out["occurs"] = f"L{line} {name} {lo}..{occurs_max}{dep}" + if redefines: + out["redefines"] = f"L{line} {name} REDEFINES {redefines.upper()}" + v = norm_value(value) + if v is not None: + out["value"] = f"L{line} {name} = {v}" + return out + + +def merge_item(facts: dict[str, set[str]], values: dict[str, str]) -> None: + for ch, v in values.items(): + facts.setdefault(ch, set()).add(v) + + +def call_value(verb: str, form: str, operand: Optional[str]) -> Optional[str]: + if not operand: + return None + op = operand.strip().strip("'\"").strip().upper() # CICS names are blank-padded literals + return f"{verb.upper()} '{op}'" if form == "literal" else f"{verb.upper()} {op}" + + +def cics_verb(verb: str) -> str: + """An EXEC CICS command's first word (`WEB OPEN` -> `WEB`, `START ATTACH` -> `START`).""" + return verb.split()[0].upper() if verb.strip() else "" + + +def cics_command(line: int, verb: str) -> str: + return f"L{line} {cics_verb(verb)}" + + +def edge_value(unit: str, verb: str, target: str) -> str: + return f"{unit} -> {'GO TO' if verb in ('GO_TO', 'GO TO') else 'PERFORM'} {target.upper()}" + + +def unit_name(program_index: int, program_id: Optional[str], name: str) -> str: + """#4206: a second or later program's units are `PROG:NAME`.""" + return f"{(program_id or '').upper()}:{name.upper()}" if program_index and name != MAIN_LINE else name.upper() + + +# ------------------------------------------------------------------------------ +# The answer key as a fact source (ground truth). +# ------------------------------------------------------------------------------ +def _key_blocks(prog: dict[str, Any]) -> list[tuple[Optional[str], dict[str, Any]]]: + return [(None, prog), *prog.get("siblings", {}).items()] + + +def key_doc(key: dict[str, Any]) -> dict[str, Any]: + """The answer key (tests/cobol_mainframe/answer_key/.json) as a fact document.""" + doc = new_doc("key", key.get("ref", "")[:12], key["corpus"], CHANNELS) + cics_ops: dict[str, list[dict[str, Any]]] = {} + for rel, entry in key.get("cics_resources", {}).items(): + cics_ops.setdefault(rel, []).extend(entry.get("operations", [])) + tasks = {rel: entry.get("operations", []) for rel, entry in key.get("cics_tasks", {}).items()} + sql = {rel: entry.get("accesses", []) for rel, entry in key.get("sql_access", {}).items()} + for rel, prog in key.get("programs", {}).items(): + f: dict[str, set[str]] = {ch: set() for ch in CHANNELS if ch not in COPYBOOK_CHANNELS} + for pid, block in _key_blocks(prog): + f["program_ids"].add((block.get("program_id") or pid or "").upper()) + units = list(block.get("units", [])) + if block.get("main_line"): + units.append({"name": MAIN_LINE, **block["main_line"]}) + for u in units: + name = f"{pid}:{u['name']}" if pid and u["name"] != MAIN_LINE else u["name"] + if u["name"] != MAIN_LINE: + name = name.upper() + f["units"].add(name) + if u.get("end") is not None: + f["unit_extents"].add(f"{name} L{u['line']}-{u['end']}") + for e in u.get("edges", []): + f["edges"].add(edge_value(name, e["verb"], e["target"])) + f["program_ids"].discard("") + for c in prog.get("calls", []): + v = call_value(c["verb"], c.get("form", "literal"), c.get("operand")) + if v: + f["calls"].add(v) + if c.get("target"): + f["call_targets"].add(c["target"].upper()) + if c["verb"].upper() in ("LINK", "XCTL") and c.get("line"): + f["cics_commands"].add(cics_command(c["line"], c["verb"])) + f["copybooks"] = {c["name"].upper() for c in prog.get("copybooks", [])} + for r in prog.get("records", []): + merge_item(f, item_values(r["line"], r["level"], r["name"], r.get("pic"), r.get("usage"), + r.get("occurs_min"), r.get("occurs_max"), r.get("occurs_depending_on"), + r.get("redefines"), r.get("value"))) # fmt: skip + f["sql_access"] = set(sql.get(rel, [])) + for op in cics_ops.get(rel, []): + f["cics_commands"].add(cics_command(op["line"], op["verb"])) + if op.get("kind") == "FILE" and op.get("name"): + f["cics_files"].add(f"{cics_verb(op['verb'])} {op['name'].upper()}") + for op in tasks.get(rel, []): + f["cics_commands"].add(cics_command(op["line"], op["verb"])) + add_file(doc, rel, f) + for rel, entry in key.get("copybook_layouts", {}).items(): + add_file(doc, rel, {"layouts": entry.get("units", [])}) + return doc diff --git a/tests/tools/referees/java/CobrixFacts.java b/tests/tools/referees/java/CobrixFacts.java new file mode 100644 index 0000000000..78e6dd9e28 --- /dev/null +++ b/tests/tools/referees/java/CobrixFacts.java @@ -0,0 +1,91 @@ +// Cobrix referee driver (#4377): parses each copybook named on stdin (one path per line) with +// Cobrix's CopybookParser and prints one JSON line per copybook with every field's offset/size. +// Compiled and run by tests/tools/referees/cobrix_adapter.py against a Cobrix build installed +// OUTSIDE this repository (see tests/tools/referees/README.md); nothing of Cobrix is vendored. +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import za.co.absa.cobrix.cobol.parser.Copybook; +import za.co.absa.cobrix.cobol.parser.CopybookParser; +import za.co.absa.cobrix.cobol.parser.ast.Group; +import za.co.absa.cobrix.cobol.parser.ast.Primitive; +import za.co.absa.cobrix.cobol.parser.ast.Statement; +import za.co.absa.cobrix.cobol.parser.policies.CommentPolicy; + +public final class CobrixFacts { + private static String q(String s) { + if (s == null) return "null"; + StringBuilder b = new StringBuilder("\""); + for (char c : s.toCharArray()) { + if (c == '"' || c == '\\') b.append('\\').append(c); + else if (c < 0x20) b.append(String.format("\\u%04x", (int) c)); + else b.append(c); + } + return b.append('"').toString(); + } + + private static String opt(scala.Option o) { + return o.isDefined() ? q(String.valueOf(o.get())) : "null"; + } + + private static void walk(Statement st, String root, StringBuilder out, boolean[] first) { + boolean group = st instanceof Group; + if (!first[0]) out.append(','); + first[0] = false; + out.append("{\"root\":").append(q(root)) + .append(",\"name\":").append(q(st.originalName())) + .append(",\"level\":").append(st.level()) + .append(",\"line\":").append(st.lineNumber()) + .append(",\"group\":").append(group) + .append(",\"offset\":").append(st.binaryProperties().offset()) + .append(",\"size\":").append(st.binaryProperties().actualSize()) + .append(",\"data_size\":").append(st.binaryProperties().dataSize()) + .append(",\"occurs\":").append(opt(st.occurs())) + .append(",\"occurs_to\":").append(opt(st.to())) + .append(",\"depending_on\":").append(opt(st.dependingOn())) + .append(",\"redefines\":").append(opt(st.redefines())) + .append(",\"filler\":").append(st.isFiller()); + if (!group) { + Primitive p = (Primitive) st; + out.append(",\"pic\":").append(opt(p.dataType().originalPic())) + .append(",\"type\":").append(q(p.dataType().getClass().getSimpleName())) + .append(",\"datatype\":").append(q(p.dataType().toString())); + } + out.append('}'); + if (group) { + scala.collection.mutable.ArrayBuffer kids = ((Group) st).children(); + for (int i = 0; i < kids.size(); i++) walk(kids.apply(i), root, out, first); + } + } + + public static void main(String[] args) throws Exception { + BufferedReader in = new BufferedReader(new InputStreamReader(System.in, StandardCharsets.UTF_8)); + String path; + while ((path = in.readLine()) != null) { + if (path.isBlank()) continue; + long t0 = System.nanoTime(); + StringBuilder out = new StringBuilder(); + try { + String text = new String(Files.readAllBytes(Paths.get(path)), StandardCharsets.ISO_8859_1); + Copybook cb = CopybookParser.parseSimple(text, false, false, new CommentPolicy(true, 6, 72), false, false); + StringBuilder fields = new StringBuilder(); + boolean[] first = {true}; + scala.collection.mutable.ArrayBuffer roots = cb.ast().children(); + for (int i = 0; i < roots.size(); i++) walk(roots.apply(i), roots.apply(i).originalName(), fields, first); + out.append("{\"path\":").append(q(path)).append(",\"ok\":true,\"seconds\":") + .append((System.nanoTime() - t0) / 1e9).append(",\"fields\":[").append(fields).append("]}"); + } catch (Throwable e) { + String msg = String.valueOf(e.getMessage()); + if (msg.length() > 300) msg = msg.substring(0, 300); + out.setLength(0); + out.append("{\"path\":").append(q(path)).append(",\"ok\":false,\"seconds\":") + .append((System.nanoTime() - t0) / 1e9).append(",\"error\":") + .append(q(e.getClass().getSimpleName() + ": " + msg)).append("}"); + } + System.out.println(out); + System.out.flush(); + } + } +} diff --git a/tests/tools/referees/mapa_adapter.py b/tests/tools/referees/mapa_adapter.py new file mode 100644 index 0000000000..e2be14f959 --- /dev/null +++ b/tests/tools/referees/mapa_adapter.py @@ -0,0 +1,155 @@ +r"""mapa referee (#4377): cschneid-the-elder/mapa's COBOL CallTree tool (MIT; ANTLR grammars for +IBM Z COBOL, CICS and DB2) as a referee-facts/1 document: PROGRAM-IDs, COPY / SQL INCLUDE members, +CALL / CICS LINK / XCTL targets, DB2 table access and EXEC CICS file-control commands. + + python tests/tools/referees/mapa_adapter.py --corpus --root \ + --key tests/cobol_mainframe/answer_key/.json --out facts//mapa.json + +Needs the environment variable MAPA_CALLTREE_JAR (mapa/cobol/CallTree.jar from a clone outside +this repository; README.md). The tool runs once per corpus over the key's programs, with every +directory of the corpus that holds a copybook on its copy list; mapa writes its CSV, a log and +temporary files into a scratch directory this adapter removes afterwards. + +mapa reports a resolved identifier's program, not the identifier, so it feeds `call_targets` but +not `calls`. Its per-member time is the run's wall time divided by the members (one JVM). +""" + +from __future__ import annotations + +import argparse +import csv +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +import facts as F + +JAR_ENV = "MAPA_CALLTREE_JAR" +COPY_EXTS = {".cpy", ".copy", ".dcl", ".cbl", ".cob", ".inc", ""} +_CICS_FILE = {"CICSREAD": "READ", "CICSWRITE": "WRITE", "CICSREWRITE": "REWRITE", "CICSDELETE": "DELETE", + "CICSSTARTBR": "STARTBR", "CICSREADNEXT": "READNEXT", "CICSREADPREV": "READPREV"} # fmt: skip +_CALL_TYPES = {"CALLBY": "CALL", "CICSLINKBY": "LINK", "CICSXCTLBY": "XCTL", "SQLCALLBY": "SQLCALL"} + + +def sql_access(kind: str) -> str: + k = kind.lower() + for word in ("insert", "update", "delete"): + if word in k: + return word + return "read" + + +def mapa_version(jar: Path) -> str: + repo = jar.resolve().parents[1] + out = subprocess.run(["git", "rev-parse", "--short=12", "HEAD"], cwd=repo, capture_output=True, text=True) + return out.stdout.strip() or jar.name + + +def parse_csv(text: str, root: Path) -> dict[str, dict[str, set[str]]]: + """mapa's record stream -> {absolute file path: channel facts}.""" + files: dict[str, str] = {} + prog_file: dict[str, str] = {} + out: dict[str, dict[str, set[str]]] = {} + + def facts_of(path: str) -> dict[str, set[str]]: + return out.setdefault(path, {"program_ids": set(), "copybooks": set(), "call_targets": set(), + "sql_access": set(), "cics_files": set()}) # fmt: skip + + rows = [r for r in csv.reader(text.splitlines()) if r] + for row in rows: # first pass: files and programs (a program's PGM record follows its INCLUDEs) + if row[0] == "FILE" and len(row) >= 3: + files[row[1]] = row[2] + facts_of(row[2]) + for row in rows: + if row[0] in ("PGM", "FUNCTION", "CLASS") and len(row) >= 4 and row[2] in files: + prog_file[row[1]] = files[row[2]] + facts_of(files[row[2]])["program_ids"].add(row[3].strip().upper()) + for row in rows: + kind = row[0] + if len(row) < 4: + continue + # the owner column is a file UUID for COPY / SQLINCLUDE and a program UUID otherwise + # (mapa's README says program for SQLINCLUDE; its output says file) + owner = files.get(row[2]) or prog_file.get(row[2]) + if owner is None: + continue + name = row[3].strip().upper() + if kind in ("COPY", "SQLINCLUDE"): + facts_of(owner)["copybooks"].add(name) + elif kind == "CALL" and len(row) >= 6: + facts_of(owner)["call_targets"].add(row[5].strip().upper()) + elif kind == "DB2TABLE" and len(row) >= 5: + facts_of(owner)["sql_access"].add(f"{sql_access(row[4])} {name}") + elif kind in _CICS_FILE: + facts_of(owner)["cics_files"].add(f"{_CICS_FILE[kind]} {name}") + return out + + +def copy_dirs(root: Path) -> list[Path]: + dirs = set() + for p in root.rglob("*"): + if p.is_file() and ".git" not in p.parts and p.suffix.lower() in COPY_EXTS: + dirs.add(p.parent) + return sorted(dirs) + + +def mapa_doc(corpus: str, root: Path, key: dict[str, Any]) -> dict[str, Any]: + jar = Path(os.environ.get(JAR_ENV) or "") + if not jar.is_file(): + raise SystemExit(f"{JAR_ENV} does not name mapa's CallTree.jar (see tests/tools/referees/README.md)") + rels = sorted(key.get("programs", {})) + channels = ["program_ids", "copybooks", "call_targets", "sql_access", "cics_files"] + doc = F.new_doc("mapa", mapa_version(jar), corpus, channels) + scratch = os.environ.get("REFEREES_CACHE") + if scratch: + Path(scratch).mkdir(parents=True, exist_ok=True) + work = Path(tempfile.mkdtemp(prefix="mapa-", dir=scratch)) + try: + (work / "files").write_text("\n".join(str((root / r).resolve()) for r in rels) + "\n", encoding="utf-8") + (work / "libs").write_text("\n".join(str(d.resolve()) for d in copy_dirs(root)) + "\n", encoding="utf-8") + t0 = time.perf_counter() + proc = subprocess.run( + ["java", "-jar", str(jar), "-fileList", "files", "-copyList", "libs", "-out", "out.csv"], + cwd=work, capture_output=True, text=True, + ) # fmt: skip + wall = time.perf_counter() - t0 + csv_path = work / "out.csv" + found = parse_csv(csv_path.read_text(encoding="utf-8"), root) if csv_path.is_file() else {} + err_tail = (proc.stderr or proc.stdout or "")[-300:] + finally: + shutil.rmtree(work, ignore_errors=True) + doc["wall_seconds"] = wall + per = wall / max(len(rels), 1) + for rel in rels: + facts = found.get(str((root / rel).resolve())) + if not facts or not facts["program_ids"]: + F.add_file(doc, rel, {}, status="fail", seconds=per, error=f"no PGM record (exit {proc.returncode}): {err_tail}") + else: + F.add_file(doc, rel, facts, seconds=per) + return doc + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--corpus", required=True) + ap.add_argument("--root", type=Path, required=True) + ap.add_argument("--key", type=Path, required=True) + ap.add_argument("--out", type=Path, required=True) + args = ap.parse_args() + key = json.loads(args.key.read_text(encoding="utf-8")) + F.dump(mapa_doc(args.corpus, args.root, key), args.out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tools/referees/score.py b/tests/tools/referees/score.py new file mode 100644 index 0000000000..a036e6c888 --- /dev/null +++ b/tests/tools/referees/score.py @@ -0,0 +1,288 @@ +r"""Scorecard for the COBOL / CICS referee panel (#4377). + +Reads referee-facts/1 documents (facts.py) -- the engine's, each referee's -- and the answer key, +and scores every source per fact channel against the key (ground truth) and against each other. + + python tests/tools/referees/score.py --facts [--md scorecard.md] [--json scorecard.json] \ + [--disagreements disagreements.json] + +`` holds one sub-directory per corpus (`//.json`); the key is read from +tests/cobol_mainframe/answer_key/.json unless `//key.json` exists. + +Scoring rules (pinned by tests/cobol_mainframe/test_referee_score.py): +- The universe of a channel is the key's files: its programs, or its keyed copybooks for + `layouts`. A source's facts on other files are ignored. +- P = correct / reported, R = correct / true, over (file, value) pairs. A file a source failed to + parse reports nothing, so its truth counts against R. `R(parsed)` is R over the files the source + parsed, which separates "could not parse" from "parsed, and got it wrong". +- A channel a source cannot produce at all (not in its `channels`) is n/a, not zero. +- `cics_commands` is scored over the verbs the key censuses in that corpus (file control, BMS + SEND/RECEIVE MAP, queues, LINK / XCTL, task control): a referee's ASKTIME is not a false positive. +- Agreement between two sources is |A & B| / |A | B| over the files both parsed. +""" + +from __future__ import annotations + +import argparse +import json +import random +import statistics +import sys +from pathlib import Path +from typing import Any, Optional + +HERE = Path(__file__).resolve().parent +REPO_ROOT = HERE.parents[2] +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +import facts as F + +KEY_DIR = REPO_ROOT / "tests" / "cobol_mainframe" / "answer_key" + + +def universe(key: dict[str, Any], channel: str) -> list[str]: + files = key["files"] + if channel in F.COPYBOOK_CHANNELS: + return sorted(r for r, e in files.items() if channel in e["facts"]) + return sorted(r for r, e in files.items() if "program_ids" in e["facts"]) + + +def _verb(value: str) -> str: + return value.split(" ", 1)[1] if " " in value else value + + +def file_values(doc: dict[str, Any], rel: str, channel: str, verbs: Optional[set[str]] = None) -> Optional[set[str]]: + """The source's values for one file and channel; None when it did not parse the file.""" + entry = doc["files"].get(rel) + if entry is None or entry["status"] == "fail": + return None + vals = set(entry["facts"].get(channel, [])) + if verbs is not None: + vals = {v for v in vals if _verb(v) in verbs} + return vals + + +def key_verbs(key: dict[str, Any]) -> set[str]: + return {_verb(v) for e in key["files"].values() for v in e["facts"].get("cics_commands", [])} + + +def score_channel(key: dict[str, Any], doc: dict[str, Any], channel: str) -> Optional[dict[str, int]]: + """tp / reported / true over the channel's universe, plus the parsed-only counterparts.""" + if channel not in doc["channels"]: + return None + verbs = key_verbs(key) if channel == "cics_commands" else None + out = {"tp": 0, "reported": 0, "true": 0, "true_parsed": 0, "files": 0, "files_parsed": 0} + for rel in universe(key, channel): + truth = file_values(key, rel, channel) or set() + got = file_values(doc, rel, channel, verbs) + out["files"] += 1 + out["true"] += len(truth) + if got is None: + continue + out["files_parsed"] += 1 + out["true_parsed"] += len(truth) + out["reported"] += len(got) + out["tp"] += len(truth & got) + return out + + +def agreement(key: dict[str, Any], a: dict[str, Any], b: dict[str, Any], channel: str) -> Optional[dict[str, int]]: + if channel not in a["channels"] or channel not in b["channels"]: + return None + verbs = key_verbs(key) if channel == "cics_commands" else None + both = union = 0 + for rel in universe(key, channel): + va, vb = file_values(a, rel, channel, verbs), file_values(b, rel, channel, verbs) + if va is None or vb is None: + continue + both += len(va & vb) + union += len(va | vb) + return {"both": both, "union": union} + + +def disagreements(key: dict[str, Any], a: dict[str, Any], b: dict[str, Any], channel: str) -> list[dict[str, Any]]: + """Every value exactly one of `a` / `b` reports on a file both parsed, with the key's verdict.""" + if channel not in a["channels"] or channel not in b["channels"]: + return [] + verbs = key_verbs(key) if channel == "cics_commands" else None + out = [] + for rel in universe(key, channel): + va, vb = file_values(a, rel, channel, verbs), file_values(b, rel, channel, verbs) + if va is None or vb is None: + continue + truth = file_values(key, rel, channel) or set() + for v in sorted(va ^ vb): + out.append({"file": rel, "value": v, "only": a["source"] if v in va else b["source"], "in_key": v in truth}) + return out + + +def _pct(n: int, d: int) -> str: + return "—" if d == 0 else f"{100.0 * n / d:.1f}%" + + +def _cell(s: Optional[dict[str, int]]) -> str: + if s is None: + return "n/a" + if s["true"] == 0 and s["reported"] == 0: + return "—" + cell = f"P {_pct(s['tp'], s['reported'])} · R {_pct(s['tp'], s['true'])}" + if s["files_parsed"] < s["files"]: + cell += f" (R parsed {_pct(s['tp'], s['true_parsed'])})" + return cell + + +def _sum(a: Optional[dict[str, int]], b: Optional[dict[str, int]]) -> Optional[dict[str, int]]: + if a is None: + return b + if b is None: + return a + return {k: a[k] + b[k] for k in a} + + +def load_corpora(facts_dir: Path) -> dict[str, dict[str, Any]]: + """corpus -> {"key": key doc, "sources": {name: doc}}.""" + out: dict[str, dict[str, Any]] = {} + for cdir in sorted(p for p in facts_dir.iterdir() if p.is_dir()): + kp = cdir / "key.json" + if kp.is_file(): + key = F.load(kp) + else: + raw = KEY_DIR / f"{cdir.name}.json" + if not raw.is_file(): + continue + key = F.key_doc(json.loads(raw.read_text(encoding="utf-8"))) + sources = {d["source"]: d for d in (F.load(p) for p in sorted(cdir.glob("*.json")) if p.name != "key.json")} + out[cdir.name] = {"key": key, "sources": sources} + return out + + +def source_order(corpora: dict[str, dict[str, Any]]) -> list[str]: + names = {s for c in corpora.values() for s in c["sources"]} + return (["engine"] if "engine" in names else []) + sorted(names - {"engine"}) + + +def parse_stats(corpora: dict[str, dict[str, Any]], source: str) -> dict[str, Any]: + counts = {"ok": 0, "partial": 0, "fail": 0, "missing": 0} + secs: list[float] = [] + for c in corpora.values(): + doc = c["sources"].get(source) + if doc is None: + continue + files: set[str] = set() + for ch in doc["channels"]: + files.update(universe(c["key"], ch)) + for rel in files: + entry = doc["files"].get(rel) + if entry is None: + counts["missing"] += 1 + continue + counts[entry["status"]] += 1 + if entry.get("seconds") is not None: + secs.append(entry["seconds"]) + return {**counts, "median_s": statistics.median(secs) if secs else None, "total_s": sum(secs) if secs else None} + + +def scorecard(corpora: dict[str, dict[str, Any]], seed: int = 4377) -> tuple[str, dict[str, Any], dict[str, Any]]: + sources = source_order(corpora) + data: dict[str, Any] = {"sources": {}, "channels": {}, "agreement": {}, "per_corpus": {}} + md = ["## Referee panel scorecard (#4377)", ""] + md.append("Corpora: " + ", ".join(f"`{n}` @ `{c['key']['version']}`" for n, c in corpora.items()) + ".") + md.append("") + md.append("### Parse success and speed (the key-covered members each source reads: programs and / or keyed copybooks)") + md.append("") + md.append("| source | version(s) | clean | partial | failed | not attempted | median s / member |") + md.append("|---|---|---|---|---|---|---|") + for s in sources: + st = parse_stats(corpora, s) + vers = sorted({c["sources"][s]["version"] for c in corpora.values() if s in c["sources"]}) + data["sources"][s] = {**st, "versions": vers} + med = "—" if st["median_s"] is None else f"{st['median_s']:.3f}" + md.append(f"| {s} | {', '.join(vers)} | {st['ok']} | {st['partial']} | {st['fail']} | {st['missing']} | {med} |") + md.append("") + md.append("Per corpus, clean / partial / failed:") + md.append("") + md.append("| source | " + " | ".join(corpora) + " |") + md.append("|---|" + "---|" * len(corpora)) + for s in sources: + cells = [] + for c in corpora.values(): + st = parse_stats({"one": c}, s) if s in c["sources"] else None + cells.append("—" if st is None else f"{st['ok']} / {st['partial']} / {st['fail']}") + md.append(f"| {s} | " + " | ".join(cells) + " |") + md.append("") + md.append("### Each source vs the answer keys (ground truth), all corpora pooled") + md.append("") + md.append("| channel | " + " | ".join(sources) + " |") + md.append("|---|" + "---|" * len(sources)) + for ch in F.CHANNELS: + row = [] + data["channels"][ch] = {} + for s in sources: + tot: Optional[dict[str, int]] = None + for name, c in corpora.items(): + if s in c["sources"]: + sc = score_channel(c["key"], c["sources"][s], ch) + data["per_corpus"].setdefault(name, {}).setdefault(ch, {})[s] = sc + tot = _sum(tot, sc) + data["channels"][ch][s] = tot + row.append(_cell(tot)) + md.append(f"| {ch} | " + " | ".join(row) + " |") + md.append("") + md.append("P = correct / reported, R = correct / true over (file, value) pairs; `R parsed` counts only " + "the files the source parsed. n/a = the source does not produce that channel.") # fmt: skip + if "engine" in sources and len(sources) > 1: + md.append("") + md.append("### Agreement with the engine (|A ∩ B| / |A ∪ B| on files both parsed)") + md.append("") + refs = [s for s in sources if s != "engine"] + md.append("| channel | " + " | ".join(refs) + " |") + md.append("|---|" + "---|" * len(refs)) + for ch in F.CHANNELS: + row = [] + for r in refs: + tot = None + for c in corpora.values(): + if r in c["sources"] and "engine" in c["sources"]: + tot = _sum(tot, agreement(c["key"], c["sources"]["engine"], c["sources"][r], ch)) + data["agreement"].setdefault(ch, {})[r] = tot + row.append("n/a" if tot is None else _pct(tot["both"], tot["union"])) + md.append(f"| {ch} | " + " | ".join(row) + " |") + dis: dict[str, Any] = {} + if "engine" in sources: + rng = random.Random(seed) + for r in [s for s in sources if s != "engine"]: + for ch in F.CHANNELS: + rows = [] + for name, c in corpora.items(): + if r in c["sources"] and "engine" in c["sources"]: + rows += [{"corpus": name, **d} for d in disagreements(c["key"], c["sources"]["engine"], c["sources"][r], ch)] # fmt: skip + if rows: + by = {} + for d in rows: + by.setdefault(f"{d['only']}-only, {'in key' if d['in_key'] else 'not in key'}", []).append(d) + dis.setdefault(r, {})[ch] = {pat: {"count": len(v), "sample": rng.sample(v, min(8, len(v)))} + for pat, v in sorted(by.items())} # fmt: skip + return "\n".join(md) + "\n", data, dis + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--facts", type=Path, required=True) + ap.add_argument("--md", type=Path) + ap.add_argument("--json", type=Path) + ap.add_argument("--disagreements", type=Path) + args = ap.parse_args() + md, data, dis = scorecard(load_corpora(args.facts)) + print(md) + if args.md: + args.md.write_text(md, encoding="utf-8") + if args.json: + args.json.write_text(json.dumps(data, indent=1, sort_keys=True) + "\n", encoding="utf-8") + if args.disagreements: + args.disagreements.write_text(json.dumps(dis, indent=1, sort_keys=True) + "\n", encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/tools/referees/treesitter_adapter.py b/tests/tools/referees/treesitter_adapter.py new file mode 100644 index 0000000000..530d371d92 --- /dev/null +++ b/tests/tools/referees/treesitter_adapter.py @@ -0,0 +1,321 @@ +r"""Spantree tree-sitter-cobol-enterprise referee (#4377): units, extents, PERFORM / GO TO edges, +CALL / LINK / XCTL, COPY, data items, typed EXEC CICS and EXEC SQL, as a referee-facts/1 document. + + python tests/tools/referees/treesitter_adapter.py --corpus --root \ + --key tests/cobol_mainframe/answer_key/.json --out facts//tree-sitter-cobol-enterprise.json + +Needs py-tree-sitter (>= 0.22) and the grammar compiled to a shared library named by the +environment variable TS_COBOL_ENTERPRISE_LIB (README.md: `tree-sitter generate` + `cc -shared`). + +The grammar does the parsing; this adapter only walks its tree. Two readings are the adapter's +own and are kept minimal: an identifier operand of LINK / XCTL / CALL / FILE(...) is resolved +through the single VALUE the grammar parsed on that data item (as the key and the engine do), and +an SQL statement's table names are read from the typed sql_* node's text. A member whose tree +has ERROR / MISSING nodes is `partial`: tree-sitter recovers, and the facts outside the damaged +region still count. +""" + +from __future__ import annotations + +import argparse +import ctypes +import json +import os +import re +import subprocess +import sys +import time +import warnings +from pathlib import Path +from typing import Any, Optional +from collections.abc import Iterator + +HERE = Path(__file__).resolve().parent +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +import facts as F + +LIB_ENV = "TS_COBOL_ENTERPRISE_LIB" +NAME_TYPES = ("WORD", "numeric_name") +HEADER_TYPES = ("paragraph", "section") +_SQL_TABLE = re.compile(r"\b(FROM|JOIN|INTO|UPDATE)\s+([A-Z0-9_#@$]+(?:\.[A-Z0-9_#@$]+)?)", re.I) +_SQL_ACCESS = {"sql_select": "read", "sql_declare_cursor": "read", "sql_insert": "insert", + "sql_update": "update", "sql_delete_sql": "delete"} # fmt: skip + + +def load_language() -> Any: + from tree_sitter import Language + + lib_path = os.environ.get(LIB_ENV) + if not lib_path: + raise SystemExit(f"{LIB_ENV} is not set (see tests/tools/referees/README.md)") + lib = ctypes.cdll.LoadLibrary(lib_path) + fn = lib.tree_sitter_cobol + fn.restype = ctypes.c_void_p + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + return Language(fn()) + + +def grammar_version(lib_path: str) -> str: + repo = os.environ.get("TS_COBOL_ENTERPRISE_REPO") + if repo: + out = subprocess.run(["git", "rev-parse", "--short=12", "HEAD"], cwd=repo, capture_output=True, text=True) + if out.stdout.strip(): + return out.stdout.strip() + return Path(lib_path).name + + +def text(node: Any) -> str: + return node.text.decode("latin-1") + + +def line(node: Any) -> int: + return int(node.start_point[0]) + 1 + + +def end_line(node: Any) -> int: + row, col = node.end_point + return int(row) + (1 if col else 0) + + +def walk(node: Any, types: tuple[str, ...], stop: tuple[str, ...] = ("program_definition",)) -> Iterator[Any]: + """Descendants of `node` of the given types, not entering a nested program.""" + for c in node.children: + if c.type in types: + yield c + if c.type not in stop: + yield from walk(c, types, stop) + + +def first_name(node: Any) -> Optional[str]: + for c in node.children: + if c.type in NAME_TYPES or c.type == "string_literal": + return text(c).strip("'\"").upper() + return None + + +def has_error(node: Any) -> bool: + return bool(node.has_error) + + +def _option(cmd: Any, name: str) -> Optional[Any]: + for opt in cmd.children: + if opt.type == "cics_option" and opt.children and text(opt.children[0]).upper() == name: + vals = [c for c in opt.children[1:] if c.type not in ("(", ")")] + return vals[0] if vals else None + return None + + +# The key censuses SEND / RECEIVE of a BMS MAP and GET / PUT of a CONTAINER; any other form of +# those verbs (SEND TEXT, RECEIVE into a terminal buffer, GET COUNTER) is named apart so that the +# verb filter in score.py leaves it out (likewise DELETE COUNTER) instead of scoring it as a false positive. +_QUALIFIED = {"SEND": "MAP", "RECEIVE": "MAP", "GET": "CONTAINER", "PUT": "CONTAINER", "DELETE": "FILE|DATASET"} + + +def cics_verb(cmd: Any) -> str: + if cmd.type == "cics_generic": + words = text(cmd).split() + verb = words[0].upper() if words else "" + else: + verb = cmd.type.removeprefix("cics_").split("_")[0].upper() + if verb in _QUALIFIED and not re.search(rf"\b({_QUALIFIED[verb]})\b", text(cmd), re.I): + return f"{verb}-{cmd.type.removeprefix('cics_').upper() if cmd.type != 'cics_generic' else 'OTHER'}" + return verb + + +class Program: + """One program_definition: its facts, read from the tree.""" + + def __init__(self, node: Any, index: int) -> None: + self.node = node + self.index = index + pid = next(walk(node, ("program_id_paragraph",)), None) + self.program_id = first_name(pid) if pid is not None else None + self.values: dict[str, str] = {} + + def data_items(self, facts: dict[str, set[str]]) -> None: + for dd in walk(self.node, ("data_description",)): + kids = {c.type: c for c in dd.children} + lvl = kids.get("level_number") + if lvl is None or not text(lvl).isdigit(): + continue + name = text(kids["entry_name"]).upper() if "entry_name" in kids else "FILLER" + pic = None + if "picture_clause" in kids: + ps = [c for c in kids["picture_clause"].children if c.type == "pic_string"] + pic = text(ps[0]) if ps else None + usage = None + if "usage_clause" in kids: + usage = re.sub(r"^USAGE\s+(IS\s+)?", "", text(kids["usage_clause"]).upper()).strip() + omin = omax = None + dep = None + if "occurs_clause" in kids: + o = text(kids["occurs_clause"]).upper() + nums = [int(n) for n in re.findall(r"\b(\d+)\b", o.split("DEPENDING")[0])] + if nums: + omin, omax = (nums[0], nums[1]) if " TO " in o and len(nums) > 1 else (nums[0], nums[0]) + m = re.search(r"DEPENDING\s+(?:ON\s+)?([A-Z0-9-]+)", o) + dep = m.group(1) if m else None + red = first_name(kids["redefines_clause"]) if "redefines_clause" in kids else None + value = None + if "value_clause" in kids: + vals = [c for c in kids["value_clause"].children if c.is_named] + value = text(vals[0]) if vals else re.sub(r"^VALUES?\s+(IS\s+|ARE\s+)?", "", text(kids["value_clause"]), flags=re.I) # fmt: skip + if value and value[:1] in "'\"" and name != "FILLER": + self.values[name] = value.strip("'\"").strip().upper() + F.merge_item(facts, F.item_values(line(dd), int(text(lvl)), name, pic, usage, omin, omax, dep, red, value)) + + def resolve(self, operand: Any) -> Optional[str]: + if operand is None: + return None + if operand.type == "string_literal": + return text(operand).strip("'\"").strip().upper() + return self.values.get(text(operand).upper()) + + def procedure(self, facts: dict[str, set[str]]) -> None: + proc = next(walk(self.node, ("procedure_division",)), None) + if proc is None: + return + items: list[tuple[str, Any]] = [] + + def flatten(n: Any) -> None: + for c in n.children: + if c.type in HEADER_TYPES: + items.append(("header", c)) + flatten(c) + elif c.type == "sentence" or (c.is_named and c.type.endswith("_statement")): + items.append(("code", c)) + + flatten(proc) + current = F.MAIN_LINE + start, last = line(proc), None + spans: list[tuple[str, int, Optional[int]]] = [] + for kind, n in items: + if kind == "header": + spans.append((current, start, last)) + current = F.unit_name(self.index, self.program_id, first_name(n) or "?") + facts["units"].add(current) + start, last = line(n), line(n) + continue + last = max(last or 0, end_line(n)) + for st in walk(n, ("perform_statement", "go_to_statement")): + if st.type == "perform_statement": + tgt = first_name(st) + if tgt: + facts["edges"].add(F.edge_value(current, "PERFORM", tgt)) + else: + for c in st.children: + if c.type in NAME_TYPES: + facts["edges"].add(F.edge_value(current, "GO_TO", text(c))) + elif c.type not in ("GO", "TO") and not c.is_named and text(c).upper() == "DEPENDING": + break + spans.append((current, start, last)) + for name, s, e in spans: + if e is not None and not (name == F.MAIN_LINE and e < s): + facts["unit_extents"].add(f"{name} L{s}-{e}") + + def calls_and_exec(self, facts: dict[str, set[str]]) -> None: + for st in walk(self.node, ("call_statement",)): + ops = [c for c in st.children if c.is_named] + if not ops: + continue + op = ops[0] + form = "literal" if op.type == "string_literal" else "identifier" + v = F.call_value("CALL", form, text(op)) + if v: + facts["calls"].add(v) + tgt = self.resolve(op) + if tgt: + facts["call_targets"].add(tgt) + for st in walk(self.node, ("copy_statement", "sql_include")): + name = first_name(st) + if name is None and st.type == "sql_include": + m = re.search(r"INCLUDE\s+([A-Z0-9#@$-]+)", text(st), re.I) + name = m.group(1).upper() if m else None + if name: + facts["copybooks"].add(name) + for ex in walk(self.node, ("exec_cics_statement",)): + cmds = [c for c in ex.children if c.is_named and c.type.startswith("cics_")] + if not cmds: + continue + cmd = cmds[0] + verb = cics_verb(cmd) + facts["cics_commands"].add(f"L{line(ex)} {verb}") + if verb in ("LINK", "XCTL"): + op = _option(cmd, "PROGRAM") + if op is not None: + form = "literal" if op.type == "string_literal" else "identifier" + v = F.call_value(verb, form, text(op)) + if v: + facts["calls"].add(v) + tgt = self.resolve(op) + if tgt: + facts["call_targets"].add(tgt) + if verb in F.CICS_FILE_VERBS: + ds = self.resolve(_option(cmd, "FILE") or _option(cmd, "DATASET")) + if ds: + facts["cics_files"].add(f"{verb} {ds}") + for ex in walk(self.node, ("exec_sql_statement",)): + for st in ex.children: + access = _SQL_ACCESS.get(st.type) + if access is None: + continue + for kw, table in _SQL_TABLE.findall(text(st)): + kw = kw.upper() + if (access == "read" and kw in ("FROM", "JOIN")) or (access == "insert" and kw == "INTO") or \ + (access == "update" and kw == "UPDATE") or (access == "delete" and kw == "FROM"): # fmt: skip + facts["sql_access"].add(f"{access} {table.upper()}") + + +def file_facts(parser: Any, path: Path) -> tuple[str, dict[str, set[str]], Optional[str]]: + tree = parser.parse(path.read_bytes()) + root = tree.root_node + facts: dict[str, set[str]] = {ch: set() for ch in F.CHANNELS if ch not in F.COPYBOOK_CHANNELS} + progs = list(walk(root, ("program_definition",), stop=())) + if not progs: + return "fail", {}, "no program_definition in the tree" + for i, node in enumerate(progs): + p = Program(node, i) + if p.program_id: + facts["program_ids"].add(p.program_id) + p.data_items(facts) + p.procedure(facts) + p.calls_and_exec(facts) + if not facts["program_ids"]: + return "fail", {}, "no PROGRAM-ID in the tree" + errors = sum(1 for _ in walk(root, ("ERROR",), stop=())) + (1 if root.has_error else 0) + return ("partial" if root.has_error else "ok"), facts, (f"{errors} ERROR/MISSING region(s)" if errors else None) + + +def treesitter_doc(corpus: str, root: Path, key: dict[str, Any]) -> dict[str, Any]: + from tree_sitter import Parser + + parser = Parser(load_language()) + channels = [ch for ch in F.CHANNELS if ch not in F.COPYBOOK_CHANNELS] + doc = F.new_doc("tree-sitter-cobol-enterprise", grammar_version(os.environ[LIB_ENV]), corpus, channels) + for rel in sorted(key.get("programs", {})): + t0 = time.perf_counter() + try: + status, facts, err = file_facts(parser, root / rel) + except Exception as e: + status, facts, err = "fail", {}, f"{type(e).__name__}: {e}" + F.add_file(doc, rel, facts, status=status, seconds=time.perf_counter() - t0, error=err) + return doc + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--corpus", required=True) + ap.add_argument("--root", type=Path, required=True) + ap.add_argument("--key", type=Path, required=True) + ap.add_argument("--out", type=Path, required=True) + args = ap.parse_args() + key = json.loads(args.key.read_text(encoding="utf-8")) + F.dump(treesitter_doc(args.corpus, args.root, key), args.out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From bed449a432ba8c9194c437a7f0397f67efad2f7f Mon Sep 17 00:00:00 2001 From: squid-protocol Date: Sun, 4 Oct 2026 20:48:56 -0400 Subject: [PATCH 2/2] test(cobol): referees -- mapa per-member fallback, FROM lists, Che4z extents (#4377) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01FNWHcUSn5p7eMxFTsB5rFo --- tests/tools/referees/mapa_adapter.py | 49 +++++++++++++++------- tests/tools/referees/treesitter_adapter.py | 13 +++--- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/tests/tools/referees/mapa_adapter.py b/tests/tools/referees/mapa_adapter.py index e2be14f959..e099fdaeb2 100644 --- a/tests/tools/referees/mapa_adapter.py +++ b/tests/tools/referees/mapa_adapter.py @@ -73,7 +73,10 @@ def facts_of(path: str) -> dict[str, set[str]]: for row in rows: if row[0] in ("PGM", "FUNCTION", "CLASS") and len(row) >= 4 and row[2] in files: prog_file[row[1]] = files[row[2]] - facts_of(files[row[2]])["program_ids"].add(row[3].strip().upper()) + pid = row[3].strip().strip("'\"").upper() # a literal PROGRAM-ID keeps its quotes in mapa's CSV + facts = facts_of(files[row[2]]) + if pid: + facts["program_ids"].add(pid) for row in rows: kind = row[0] if len(row) < 4: @@ -103,20 +106,15 @@ def copy_dirs(root: Path) -> list[Path]: return sorted(dirs) -def mapa_doc(corpus: str, root: Path, key: dict[str, Any]) -> dict[str, Any]: - jar = Path(os.environ.get(JAR_ENV) or "") - if not jar.is_file(): - raise SystemExit(f"{JAR_ENV} does not name mapa's CallTree.jar (see tests/tools/referees/README.md)") - rels = sorted(key.get("programs", {})) - channels = ["program_ids", "copybooks", "call_targets", "sql_access", "cics_files"] - doc = F.new_doc("mapa", mapa_version(jar), corpus, channels) +def run_calltree(jar: Path, paths: list[Path], libs: list[Path], root: Path) -> tuple[dict[str, dict[str, set[str]]], float, str]: + """One CallTree run over `paths`: (facts by absolute path, wall seconds, error tail).""" scratch = os.environ.get("REFEREES_CACHE") if scratch: Path(scratch).mkdir(parents=True, exist_ok=True) work = Path(tempfile.mkdtemp(prefix="mapa-", dir=scratch)) try: - (work / "files").write_text("\n".join(str((root / r).resolve()) for r in rels) + "\n", encoding="utf-8") - (work / "libs").write_text("\n".join(str(d.resolve()) for d in copy_dirs(root)) + "\n", encoding="utf-8") + (work / "files").write_text("\n".join(str(p.resolve()) for p in paths) + "\n", encoding="utf-8") + (work / "libs").write_text("\n".join(str(d.resolve()) for d in libs) + "\n", encoding="utf-8") t0 = time.perf_counter() proc = subprocess.run( ["java", "-jar", str(jar), "-fileList", "files", "-copyList", "libs", "-out", "out.csv"], @@ -125,17 +123,40 @@ def mapa_doc(corpus: str, root: Path, key: dict[str, Any]) -> dict[str, Any]: wall = time.perf_counter() - t0 csv_path = work / "out.csv" found = parse_csv(csv_path.read_text(encoding="utf-8"), root) if csv_path.is_file() else {} - err_tail = (proc.stderr or proc.stdout or "")[-300:] + tail = (proc.stderr or proc.stdout or "").strip().splitlines() + err = f"exit {proc.returncode}: " + " | ".join(ln.strip() for ln in tail[:2])[:240] if proc.returncode else "" finally: shutil.rmtree(work, ignore_errors=True) + return found, wall, err + + +def mapa_doc(corpus: str, root: Path, key: dict[str, Any]) -> dict[str, Any]: + jar = Path(os.environ.get(JAR_ENV) or "") + if not jar.is_file(): + raise SystemExit(f"{JAR_ENV} does not name mapa's CallTree.jar (see tests/tools/referees/README.md)") + rels = sorted(key.get("programs", {})) + channels = ["program_ids", "copybooks", "call_targets", "sql_access", "cics_files"] + doc = F.new_doc("mapa", mapa_version(jar), corpus, channels) + libs = copy_dirs(root) + found, wall, err = run_calltree(jar, [root / r for r in rels], libs, root) + per = {rel: wall / max(len(rels), 1) for rel in rels} + errors = dict.fromkeys(rels, err) + if err and not found: + # one member that crashes CallTree (an uncaught exception) aborts the whole run: + # fall back to one run per member so the others still count + found, wall = {}, 0.0 + for rel in rels: + f1, w1, e1 = run_calltree(jar, [root / rel], libs, root) + found.update(f1) + per[rel], errors[rel] = w1, e1 + wall += w1 doc["wall_seconds"] = wall - per = wall / max(len(rels), 1) for rel in rels: facts = found.get(str((root / rel).resolve())) if not facts or not facts["program_ids"]: - F.add_file(doc, rel, {}, status="fail", seconds=per, error=f"no PGM record (exit {proc.returncode}): {err_tail}") + F.add_file(doc, rel, {}, status="fail", seconds=per[rel], error=f"no PGM record ({errors[rel] or 'no error'})") else: - F.add_file(doc, rel, facts, seconds=per) + F.add_file(doc, rel, facts, seconds=per[rel]) return doc diff --git a/tests/tools/referees/treesitter_adapter.py b/tests/tools/referees/treesitter_adapter.py index 530d371d92..206a0a477e 100644 --- a/tests/tools/referees/treesitter_adapter.py +++ b/tests/tools/referees/treesitter_adapter.py @@ -39,7 +39,9 @@ LIB_ENV = "TS_COBOL_ENTERPRISE_LIB" NAME_TYPES = ("WORD", "numeric_name") HEADER_TYPES = ("paragraph", "section") -_SQL_TABLE = re.compile(r"\b(FROM|JOIN|INTO|UPDATE)\s+([A-Z0-9_#@$]+(?:\.[A-Z0-9_#@$]+)?)", re.I) +_TABLE = r"[A-Z0-9_#@$]+(?:\.[A-Z0-9_#@$]+)?" +# a FROM list may name several tables (`FROM POLICY, MOTOR`), each with an optional alias +_SQL_TABLE = re.compile(rf"\b(FROM|JOIN|INTO|UPDATE)\s+({_TABLE}(?:\s+(?!WHERE\b|ORDER\b|GROUP\b)[A-Z][A-Z0-9_]*)?(?:\s*,\s*{_TABLE}(?:\s+(?!WHERE\b)[A-Z][A-Z0-9_]*)?)*)", re.I) _SQL_ACCESS = {"sql_select": "read", "sql_declare_cursor": "read", "sql_insert": "insert", "sql_update": "update", "sql_delete_sql": "delete"} # fmt: skip @@ -262,11 +264,12 @@ def calls_and_exec(self, facts: dict[str, set[str]]) -> None: access = _SQL_ACCESS.get(st.type) if access is None: continue - for kw, table in _SQL_TABLE.findall(text(st)): + for kw, tables in _SQL_TABLE.findall(text(st)): kw = kw.upper() - if (access == "read" and kw in ("FROM", "JOIN")) or (access == "insert" and kw == "INTO") or \ - (access == "update" and kw == "UPDATE") or (access == "delete" and kw == "FROM"): # fmt: skip - facts["sql_access"].add(f"{access} {table.upper()}") + for table in (t.split()[0] for t in tables.split(",") if t.strip()): + if (access == "read" and kw in ("FROM", "JOIN")) or (access == "insert" and kw == "INTO") or \ + (access == "update" and kw == "UPDATE") or (access == "delete" and kw == "FROM"): # fmt: skip + facts["sql_access"].add(f"{access} {table.upper()}") def file_facts(parser: Any, path: Path) -> tuple[str, dict[str, set[str]], Optional[str]]: