test(cobol): referee panel -- independent COBOL/CICS parsers scored per fact channel (#4377)

tests/cobol_mainframe/test_referee_score.py

@@ -0,0 +1,164 @@
+"""The referee panel's scoring logic (#4377): tests/tools/referees/{facts,score}.py.
+
+No referee is run here -- the panel's tools (JVM / tree-sitter builds) live outside the repo and
+run on demand. These tests pin the arithmetic the scorecard reports.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools" / "referees"))
+
+import cobrix_adapter
+import facts as F
+import score as S
+
+
+def _key() -> dict:
+    raw = {
+        "corpus": "toy",
+        "ref": "0123456789abcdef",
+        "programs": {
+            "A.cbl": {
+                "program_id": "PROGA",
+                "units": [
+                    {"name": "P1", "kind": "paragraph", "line": 10, "end": 12,
+                     "edges": [{"verb": "PERFORM_THRU", "target": "P2", "thru": "P2-EXIT", "line": 11}]},
+                    {"name": "P2", "kind": "paragraph", "line": 14, "end": 15,
+                     "edges": [{"verb": "GO_TO", "target": "P1", "line": 15}]},
+                ],
+                "main_line": {"line": 8, "end": 9, "edges": [{"verb": "PERFORM", "target": "P1", "line": 9}]},
+                "siblings": {"PROGB": {"program_id": "PROGB", "units": [
+                    {"name": "MAINLINE", "kind": "section", "line": 30, "end": 31, "edges": []}]}},
+                "calls": [
+                    {"verb": "CALL", "form": "literal", "operand": "SUBX", "target": "SUBX", "line": 12},
+                    {"verb": "LINK", "form": "identifier", "operand": "WS-PGM", "target": "PROGC", "line": 13},
+                ],
+                "copybooks": [{"name": "CPY1"}],
+                "records": [
+                    {"line": 5, "level": 1, "name": "WS-A", "pic": "x(3)", "usage": None, "occurs_min": None,
+                     "occurs_max": None, "occurs_depending_on": None, "redefines": None, "value": "abc"},
+                    {"line": 6, "level": 5, "name": "WS-N", "pic": "S9(4)", "usage": "BINARY", "occurs_min": 1,
+                     "occurs_max": 5, "occurs_depending_on": "WS-K", "redefines": None, "value": None},
+                ],
+            },
+            "B.cbl": {"program_id": "PROGD", "units": [], "calls": [], "copybooks": [], "records": []},
+        },
+        "cics_resources": {"A.cbl": {"operations": [
+            {"verb": "READ", "kind": "FILE", "name": "ACCTS", "line": 20},
+            {"verb": "SEND", "kind": "MAP", "name": "M1", "line": 21},
+        ]}},
+        "cics_tasks": {"A.cbl": {"operations": [{"verb": "START ATTACH", "line": 22}]}},
+        "sql_access": {"B.cbl": {"accesses": ["read T1"]}},
+        "copybook_layouts": {"C.cpy": {"units": ["R/F1 @0+3", "R/F2 @3+2"]}},
+    }  # fmt: skip
+    return F.key_doc(raw)
+
+
+def _doc(source: str, files: dict, channels=F.CHANNELS) -> dict:
+    doc = F.new_doc(source, "v", "toy", channels)
+    for rel, (status, facts) in files.items():
+        F.add_file(doc, rel, facts, status=status, seconds=0.5)
+    return doc
+
+
+def test_key_doc_shapes() -> None:
+    k = _key()["files"]["A.cbl"]["facts"]
+    assert k["program_ids"] == ["PROGA", "PROGB"]
+    assert "PROGB:MAINLINE" in k["units"] and "P1" in k["units"]
+    assert "(procedure division) L8-9" in k["unit_extents"]
+    assert set(k["edges"]) == {"P1 -> PERFORM P2", "P2 -> GO TO P1", "(procedure division) -> PERFORM P1"}
+    assert set(k["calls"]) == {"CALL 'SUBX'", "LINK WS-PGM"}
+    assert set(k["call_targets"]) == {"SUBX", "PROGC"}
+    assert set(k["cics_commands"]) == {"L13 LINK", "L20 READ", "L21 SEND", "L22 START"}
+    assert k["cics_files"] == ["READ ACCTS"]
+    assert "L6 WS-N COMP" in k["usage"]  # BINARY folds to COMP
+    assert "L6 WS-N 1..5 DEPENDING ON WS-K" in k["occurs"]
+    assert "L5 WS-A X(3)" in k["pic"] and "L5 WS-A = ABC" in k["value"]
+    assert _key()["files"]["C.cpy"]["facts"] == {"layouts": ["R/F1 @0+3", "R/F2 @3+2"]}
+
+
+@pytest.mark.parametrize(
+    "raw,norm",
+    [("'  ab '", "AB"), ("+12", "12"), ("ZERO", "ZEROES"), ("spaces", "SPACES"), ("SPACE", "SPACES"), ("-3", "-3")],
+)
+def test_norm_value(raw: str, norm: str) -> None:
+    assert F.norm_value(raw) == norm
+
+
+def test_depending_on_qualifier_is_dropped() -> None:
+    assert F.item_values(4, 5, "V", occurs_min=0, occurs_max=9, depending_on="LEN OF GRP")["occurs"] == "L4 V 0..9 DEPENDING ON LEN"
+
+
+def test_call_value_strips_cics_padding() -> None:
+    assert F.call_value("LINK", "literal", "'INQCUST '") == "LINK 'INQCUST'"
+    assert F.call_value("CALL", "identifier", "ws-pgm") == "CALL WS-PGM"
+
+
+def test_universe_programs_and_copybooks() -> None:
+    key = _key()
+    assert S.universe(key, "units") == ["A.cbl", "B.cbl"]
+    assert S.universe(key, "layouts") == ["C.cpy"]
+
+
+def test_score_channel_counts_and_failed_files() -> None:
+    key = _key()
+    ref = _doc("ref", {
+        "A.cbl": ("ok", {"units": ["P1", "BOGUS"]}),
+        "B.cbl": ("fail", {}),
+    })  # fmt: skip
+    sc = S.score_channel(key, ref, "units")
+    assert sc == {"tp": 1, "reported": 2, "true": 3, "true_parsed": 3, "files": 2, "files_parsed": 1}
+    # B.cbl has no units, so its failure costs nothing; program_ids shows the cost of a failed file
+    pid = S.score_channel(key, _doc("ref", {"A.cbl": ("ok", {"program_ids": ["PROGA"]}), "B.cbl": ("fail", {})}), "program_ids")
+    assert (pid["tp"], pid["true"], pid["true_parsed"]) == (1, 3, 2)
+
+
+def test_channel_a_source_cannot_produce_is_na() -> None:
+    assert S.score_channel(_key(), _doc("ref", {}, channels=["units"]), "layouts") is None
+    assert S._cell(None) == "n/a"
+
+
+def test_cics_commands_scored_over_key_verbs() -> None:
+    key = _key()
+    ref = _doc("ref", {"A.cbl": ("ok", {"cics_commands": ["L20 READ", "L23 ASKTIME", "L24 SEND-SEND_TEXT"]})})
+    sc = S.score_channel(key, ref, "cics_commands")
+    assert (sc["tp"], sc["reported"]) == (1, 1)  # ASKTIME / SEND TEXT are not censused: not false positives
+
+
+def test_agreement_and_disagreements_only_on_files_both_parsed() -> None:
+    key = _key()
+    eng = _doc("engine", {"A.cbl": ("ok", {"units": ["P1", "P2"]}), "B.cbl": ("ok", {"units": ["X"]})})
+    ref = _doc("ref", {"A.cbl": ("partial", {"units": ["P1", "P9"]}), "B.cbl": ("fail", {})})
+    assert S.agreement(key, eng, ref, "units") == {"both": 1, "union": 3}
+    dis = S.disagreements(key, eng, ref, "units")
+    assert {(d["value"], d["only"], d["in_key"]) for d in dis} == {("P2", "engine", True), ("P9", "ref", False)}
+
+
+def test_scorecard_markdown() -> None:
+    key = _key()
+    corpora = {"toy": {"key": key, "sources": {
+        "engine": _doc("engine", {"A.cbl": ("ok", {"units": ["P1", "P2", "PROGB:MAINLINE"]}), "B.cbl": ("ok", {})}),
+        "ref": _doc("ref", {"A.cbl": ("ok", {"units": ["P1"]}), "B.cbl": ("fail", {})}, channels=["units"]),
+    }}}  # fmt: skip
+    md, data, dis = S.scorecard(corpora)
+    assert "| units | P 100.0% · R 100.0% | P 100.0% · R 33.3% (R parsed 33.3%) |" in md
+    assert "| layouts | P — · R 0.0% (R parsed —) | n/a |" in md  # a keyed copybook the engine never read
+    assert data["sources"]["ref"]["fail"] == 1 and data["sources"]["ref"]["ok"] == 1
+    assert dis["ref"]["units"]["engine-only, in key"]["count"] == 2
+
+
+def test_cobrix_layout_units_skip_overlays_and_fillers() -> None:
+    fields = [
+        {"root": "R", "name": "R", "level": 1, "group": True, "offset": 0, "size": 10},
+        {"root": "R", "name": "A", "level": 5, "group": False, "offset": 0, "size": 4, "pic": "X(4)"},
+        {"root": "R", "name": "B", "level": 5, "group": True, "offset": 0, "size": 4, "redefines": "A"},
+        {"root": "R", "name": "B1", "level": 10, "group": False, "offset": 0, "size": 4, "pic": "9(4)"},
+        {"root": "R", "name": "FILLER", "level": 5, "group": False, "offset": 4, "size": 2, "pic": "X(2)", "filler": True},
+        {"root": "R", "name": "C", "level": 5, "group": False, "offset": 6, "size": 4, "pic": "X(2)"},
+    ]
+    assert cobrix_adapter.layout_units(fields) == {"R/A @0+4", "R/C @6+4"}

tests/tools/referees/README.md

@@ -0,0 +1,97 @@
+# COBOL / CICS referee panel (#4377)
+
+Independent open-source COBOL parsers ("referees") scored against the engine and against the
+hand-verified answer keys, per fact channel. Test / validation tooling only: nothing here is
+imported by `gitgalaxy/`, and no referee's code is vendored. Every referee is installed outside
+the repository and invoked as an external, unmodified tool.
+
+| file | role |
+|---|---|
+| `facts.py` | the common `referee-facts/1` format, the canonical value shapes, and the answer key as a fact source |
+| `engine_adapter.py` | the engine: a scanned corpus's master DB through GalaxyIR (+ the unit graph `cobol_answer_key.py` reads) |
+| `cobrix_adapter.py` + `java/CobrixFacts.java` | Cobrix: copybook record layouts (offsets, sizes) |
+| `treesitter_adapter.py` | Spantree tree-sitter-cobol-enterprise: units, extents, edges, calls, COPY, data items, typed EXEC CICS / SQL |
+| `che4z_adapter.py` | Eclipse Che4z COBOL LSP's headless `analysis` CLI: units, extents, edges, COPY, data items |
+| `mapa_adapter.py` | mapa's CallTree: PROGRAM-IDs, COPY / INCLUDE, CALL / LINK / XCTL targets, DB2 tables, CICS file commands |
+| `estate_key.py` | estate-crucible's generated key (#4317) in the answer-key shape, so every adapter runs on it unchanged |
+| `score.py` | the scorecard: each source vs the key, agreement with the engine, disagreements for adjudication |
+
+The scoring rules are in `score.py`'s docstring and pinned by `tests/cobol_mainframe/test_referee_score.py`.
+
+## Installing the referees
+
+Put them anywhere outside the checkout (the examples use `$REF`). Record the versions you ran: the
+scorecard prints each source's version.
+
+**Cobrix** (Apache-2.0, AbsaOSS/cobrix; JDK 11+ and Maven):
+
+```sh
+git clone https://github.com/AbsaOSS/cobrix $REF/cobrix
+cd $REF/cobrix
+# the parent POM still targets Java 7, which current JDKs refuse
+mvn -q -pl cobol-parser -am -DskipTests -Dmaven.compiler.source=8 -Dmaven.compiler.target=8 package
+mvn -q -pl cobol-parser dependency:build-classpath -Dmdep.outputFile=$REF/cobrix.cp
+export COBRIX_CLASSPATH=$(cat $REF/cobrix.cp):$(ls $REF/cobrix/cobol-parser/target/cobol-parser_2.12-*-SNAPSHOT.jar | grep -v javadoc)
+```
+
+The adapter compiles `java/CobrixFacts.java` into `$REFEREES_CACHE` (default
+`~/.cache/gitgalaxy-referees`) on first use.
+
+**Spantree tree-sitter-cobol-enterprise** (MIT; node, a C compiler, py-tree-sitter >= 0.22):
+
+```sh
+git clone https://github.com/Spantree/tree-sitter-cobol-enterprise $REF/tree-sitter-cobol-enterprise
+(cd $REF && npm install tree-sitter-cli@0.25)            # src/parser.c is not committed upstream
+cd $REF/tree-sitter-cobol-enterprise && $REF/node_modules/.bin/tree-sitter generate   # ~5 min
+cc -O2 -fPIC -shared -I src src/parser.c src/scanner.c -o $REF/cobol_enterprise.so
+pip install tree-sitter
+export TS_COBOL_ENTERPRISE_LIB=$REF/cobol_enterprise.so TS_COBOL_ENTERPRISE_REPO=$REF/tree-sitter-cobol-enterprise
+```
+
+**Eclipse Che4z COBOL Language Support** (EPL-2.0; JDK 17+). The server jar ships in the VS Code
+extension; it is used unmodified, as an external process:
+
+```sh
+gh release download 2.5.1 -R eclipse-che4z/che-che4z-lsp-for-cobol -p 'cobol-language-support-2.5.1.vsix' -D $REF/che4z
+unzip -q $REF/che4z/cobol-language-support-2.5.1.vsix -d $REF/che4z/vsix
+export CHE4Z_SERVER_JAR=$REF/che4z/vsix/extension/server/jar/server.jar
+```
+
+Known limits of 2.5.1's `analysis` CLI (see the scorecard): it prints no result and exits 1 on any
+program containing a `CALL` statement, its `--ast` JSON writer overflows / runs out of memory on
+large CICS programs, and its serialised OCCURS clause carries no DEPENDING ON. Driving the server
+over LSP (`textDocument/documentSymbol`) would avoid all three; that is #4377's next phase.
+
+**mapa** (MIT, cschneid-the-elder/mapa; JDK 21). `cobol/CallTree.jar` is committed upstream:
+
+```sh
+git clone https://github.com/cschneid-the-elder/mapa $REF/mapa
+export MAPA_CALLTREE_JAR=$REF/mapa/cobol/CallTree.jar
+```
+
+## Running the panel
+
+```sh
+# 1. engine facts: scan with the checkout under test (cached per engine commit), then extract
+python tests/tools/mainframe_corpus.py scan <corpus>
+python tests/tools/referees/engine_adapter.py --db "$(python tests/tools/mainframe_corpus.py path <corpus> --db)" \
+    --corpus <corpus> --key tests/cobol_mainframe/answer_key/<corpus>.json --out $FACTS/<corpus>/engine.json
+
+# 2. each referee, on the same members (the key's programs / keyed copybooks)
+for a in cobrix treesitter che4z mapa; do
+  python tests/tools/referees/${a}_adapter.py --corpus <corpus> --root "$(python tests/tools/mainframe_corpus.py path <corpus>)" \
+      --key tests/cobol_mainframe/answer_key/<corpus>.json --out $FACTS/<corpus>/$a.json
+done
+
+# estate-crucible: convert its key once, scan with tests/tools/estate_crucible.py --scan-dir, then as above
+python tests/tools/referees/estate_key.py --crucible ../estate-crucible \
+    --out $WORK/estate-crucible.answer_key.json --facts $FACTS/estate-crucible/key.json
+
+# 3. the scorecard
+python tests/tools/referees/score.py --facts $FACTS --md scorecard.md --json scorecard.json \
+    --disagreements disagreements.json
+```
+
+`disagreements.json` groups every engine / referee difference by pattern
+(`engine-only, in key`, `<referee>-only, not in key`, ...) with a seeded sample of each, for
+adjudication against the source: engine wrong, referee wrong, definitional, or key wrong.