diff --git a/.gitignore b/.gitignore index f3f9646701..66fdd1aa53 100644 --- a/.gitignore +++ b/.gitignore @@ -112,6 +112,8 @@ museum_2/ # no ctags -- e.g. abap): hand-reviewed and committed like the ledger above, read by # tests/tools/tri_comparison_chart.py but never written by it. !docs/self_scan/manual_verification.json +# #3315: hand-labelled thin-wrapper verdicts -- source data the wrapper probe scores against. +!tests/tools/wrapper_labels.json # ...and the COBOL modernization answer key (#3210): hand-verified per-program truth, # committed and reviewed like the ledger above. A new key file dropped silently by # the *.json rule above is how its first PR shipped without its own data. diff --git a/tests/tools/test_wrapper_probe.py b/tests/tools/test_wrapper_probe.py new file mode 100644 index 0000000000..4ebb71a6ae --- /dev/null +++ b/tests/tools/test_wrapper_probe.py @@ -0,0 +1,198 @@ +""" +#3315 (epic #3313 step 1): the thin-wrapper probe's candidate, call-site and +resolution rules, each pinned on the smallest real shape that decided it. + + - fortran/wrf's `wrf_error_fatal` (`print *,string` then `stop`), called from + another file: the case the epic exists for. + - go/core's `func (cr *connReader) lock()`: a METHOD, reachable only as + `cr.lock()`. The step-0 census credited it 209 bare `lock(` calls that go to + a different, out-of-sample `lock`; the method rule makes that impossible. + - a qualified call (`obj.log_it(...)`) is a call on something else. + - two same-named definitions: a caller's own file wins; from a third file the + name is ambiguous and credits nothing. +""" + +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import wrapper_probe as wp # noqa: E402 + +from gitgalaxy.core.prism import Prism # noqa: E402 +from gitgalaxy.standards.gitgalaxy_config import LEXICAL_FAMILY_HEURISTICS # noqa: E402 +from gitgalaxy.standards.language_standards import LANGUAGE_DEFINITIONS # noqa: E402 + +LABELS_FILE = Path(__file__).resolve().parent / "wrapper_labels.json" + + +def _probe(tmp_path, lang, files, rules): + crucible = tmp_path / "data" + group = crucible / lang / "repo" + for rel, text in files.items(): + path = group / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + prism = Prism(LEXICAL_FAMILY_HEURISTICS, LANGUAGE_DEFINITIONS) + return wp.probe_group(lang, group, tuple(rules), 12, prism, crucible) + + +def _by_name(result): + return {(c["name"], c["rule"], c["file"]): c for c in result["candidates"].values()} + + +WRF_WRAPPER = """\ +subroutine wrf_error_fatal (string) + character (len=*) :: string + print *,string + stop +end subroutine wrf_error_fatal +""" + +WRF_CALLER = """\ +subroutine init_domain (n) + integer :: n + if (n < 0) then + call wrf_error_fatal('negative domain') + end if + if (n > 99) then + call wrf_error_fatal('too many domains') + end if + call wrf_error_fatal('unreachable in practice') +end subroutine init_domain +""" + + +def test_a_thin_wrapper_is_credited_with_its_cross_file_call_sites(tmp_path): + result = _probe( + tmp_path, + "fortran", + {"wrapper.F": WRF_WRAPPER, "caller.F": WRF_CALLER}, + ["debug_prints", "panics_and_aborts"], + ) + cands = _by_name(result) + for rule in ("debug_prints", "panics_and_aborts"): + wrapper = cands[("wrf_error_fatal", rule, "wrapper.F")] + assert wrapper["sites"] == 3, "three calls from another file" + assert (wrapper["branches"], wrapper["header_has_name"], wrapper["method"]) == (0, True, False) + + +GO_METHOD = """\ +package http + +func (cr *connReader) lock() { + cr.mu.Lock() +} +""" + +GO_CALLER = """\ +package runtime + +func park() { + lock(&sched.lock) + unlock(&sched.lock) +} +""" + + +def test_a_method_never_receives_unqualified_calls(tmp_path): + """The census's go/core false attribution: bare `lock(` calls reach some other + `lock`, never a method that can only be called as `cr.lock()`.""" + result = _probe(tmp_path, "go", {"server.go": GO_METHOD, "proc.go": GO_CALLER}, ["sync_locks"]) + lock = _by_name(result)[("lock", "sync_locks", "server.go")] + assert lock["method"] is True + assert lock["sites"] == 0 + + +PY_WRAPPER = """\ +def log_it(message): + print(message) +""" + +PY_CALLER = """\ +def handler(obj): + obj.log_it("a method on something else") + log_it("the module-level helper") +""" + + +def test_a_qualified_call_is_a_call_on_something_else(tmp_path): + result = _probe(tmp_path, "python", {"log.py": PY_WRAPPER, "handler.py": PY_CALLER}, ["debug_prints"]) + assert _by_name(result)[("log_it", "debug_prints", "log.py")]["sites"] == 1 + + +PY_OTHER_WRAPPER = """\ +def log_it(message): + print("other:", message) + + +def local_caller(): + log_it("resolves to this file's own log_it") +""" + +PY_THIRD = """\ +def elsewhere(): + log_it("which log_it? two are defined") +""" + + +def test_same_file_wins_and_an_ambiguous_name_credits_nothing(tmp_path): + result = _probe( + tmp_path, + "python", + {"a.py": PY_WRAPPER, "b.py": PY_OTHER_WRAPPER, "c.py": PY_THIRD}, + ["debug_prints"], + ) + cands = _by_name(result) + assert cands[("log_it", "debug_prints", "b.py")]["sites"] == 1, "b.py's own caller" + assert cands[("log_it", "debug_prints", "a.py")]["sites"] == 0 + assert result["unattributed_sites"] == {"debug_prints": 1}, "c.py's call is ambiguous" + + +def test_method_definitions_by_shape(): + assert wp.is_method_definition("go", "func (cr *connReader) lock() {", "lock") + assert not wp.is_method_definition("go", "func lock(l *mutex) {", "lock") + assert wp.is_method_definition("python", "def log(self, msg):", "log") + assert not wp.is_method_definition("python", "def log(msg):", "log") + assert wp.is_method_definition("lua", "function M.log(msg)", "log") + assert wp.is_method_definition("cpp", "void Logger::log(const char *m) {", "log") + + +def test_the_label_file_is_well_formed(): + """Every verdict is one of the four labels and says why. A label is read from + the body the probe prints, never set from a guess.""" + doc = json.loads(LABELS_FILE.read_text(encoding="utf-8")) + labels = doc["labels"] + assert len(labels) >= 60 + for key, entry in labels.items(): + group, file, name, rule = key.split("::") + assert group and file and name and rule in wp.DEFAULT_RULES, key + assert entry["label"] in wp.LABELS, key + assert entry["reason"].strip(), key + + +def _filter(doc, max_loc): + kept = {} + for key, entry in doc["labels"].items(): + ev = entry["evidence"] + if ev["sites"] and ev["loc"] <= max_loc and ev["header_has_name"] and ev["branches"] == 0: + kept[key] = entry + return kept + + +def test_the_recorded_filter_decision_reproduces_from_the_labels(): + """#3315's decision -- max_loc 8, aligned span, no branches -- re-scored from the + committed labels and their recorded evidence, so the claim in the PR and on the + epic cannot drift from the data it came from.""" + doc = json.loads(LABELS_FILE.read_text(encoding="utf-8")) + max_loc = doc["filter"]["recommended_max_loc"] + judged = {k: e for k, e in doc["labels"].items() if e["evidence"]["sites"] and e["evidence"]["loc"] <= max_loc} + kept = _filter(doc, max_loc) + + def sites(entries, label=None): + return sum(e["evidence"]["sites"] for e in entries.values() if label is None or e["label"] == label) + + assert sites(kept, "wrapper") == sites(judged, "wrapper"), "the filter drops a labelled wrapper's call sites" + strict = sum(1 for e in kept.values() if e["label"] == "wrapper") / len(kept) + assert strict >= 0.7 + assert sites(kept, "wrapper") / sites(kept) >= 0.95 diff --git a/tests/tools/wrapper_labels.json b/tests/tools/wrapper_labels.json new file mode 100644 index 0000000000..f1da0e0fe3 --- /dev/null +++ b/tests/tools/wrapper_labels.json @@ -0,0 +1,727 @@ +{ + "about": "Hand-labelled thin-wrapper candidates for tests/tools/wrapper_probe.py (#3315, epic #3313). Each key is a probe candidate (group::file::name::rule) from language-crucible v1.4.0: every candidate with >= 20 resolved call sites plus a seed-3313 random sample of 60 others, all at max_loc 12. wrapper = exists to perform the rule's behaviour, unconditionally; conditional = a check-and-fail helper (throws/prints only on a condition); incidental = the hit is a side detail of real work (including every case where the engine's function span is misaligned and the hit belongs to other code); test_helper = a helper in test code. Read from the body the probe prints; never set from a guess. `evidence` is the probe's measurement for that candidate at labelling time, so the filter decision below can be re-scored in CI without the corpus.", + "corpus": "language-crucible v1.4.0", + "filter": { + "decision": "#3315: max_loc 8 with the aligned+no_branches filter keeps 13 of 31 labelled candidates, 77% strict precision (98% of kept call sites are real wrappers'), and drops none of the labelled wrappers' call sites. max_loc 12 raises the kept set to 17 at 65% strict precision for one extra 1-site wrapper.", + "definition": "header_has_name (name in the span's first 3 lines) and branches == 0", + "name": "aligned+no_branches", + "recommended_max_loc": 8 + }, + "labels": { + "ada/alire::alr-commands.adb::Reportaise_Command_Failed::panics_and_aborts": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 5, + "method": false, + "sites": 6 + }, + "label": "wrapper", + "reason": "Pretty_Print then raise, unconditional" + }, + "ada/alire::alr-commands.adb::Reportaise_Wrong_Arguments::panics_and_aborts": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 5, + "method": false, + "sites": 9 + }, + "label": "wrapper", + "reason": "Pretty_Print then raise, unconditional: its name says report-and-raise" + }, + "c/cpython::dictobject.c::PyDict_ContainsString::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 11, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "error propagation: return -1 on a failed allocation" + }, + "c/cpython::object.c::PyObject_HashNotImplemented::panics_and_aborts": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 7, + "method": false, + "sites": 2 + }, + "label": "wrapper", + "reason": "PyErr_Format + return -1: always raises TypeError (CPython error convention)" + }, + "c/cpython::typeobject.c::update_subclasses::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 9, + "method": false, + "sites": 2 + }, + "label": "incidental", + "reason": "error propagation (return -1 when the callback fails), not a raise helper" + }, + "c/sqlite::lemon.c::ErrorMsg::debug_prints": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 8, + "method": false, + "sites": 49 + }, + "label": "wrapper", + "reason": "prints a formatted error to stderr unconditionally; its purpose is printing" + }, + "c/sqlite::lemon.c::memory_error::debug_prints": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 4, + "method": false, + "sites": 2 + }, + "label": "wrapper", + "reason": "prints 'Out of memory' then exit(1): unconditional print" + }, + "c/sqlite::lemon.c::memory_error::panics_and_aborts": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 4, + "method": false, + "sites": 2 + }, + "label": "wrapper", + "reason": "prints 'Out of memory' then exit(1): unconditional abort" + }, + "csharp/roslyn::CSharpCompilation.cs::CreatePointerTypeSymbol::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 9, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "constructor; throws only on a null argument" + }, + "csharp/roslyn::Workspace.cs::CheckAdditionalDocumentIsInSolution::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 9, + "method": false, + "sites": 2 + }, + "label": "conditional", + "reason": "check-and-throw helper" + }, + "csharp/roslyn::Workspace.cs::CheckAdditionalDocumentIsNotInSolution::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 9, + "method": false, + "sites": 2 + }, + "label": "conditional", + "reason": "check-and-throw helper: throws only if the document is present" + }, + "csharp/roslyn::Workspace.cs::CheckAnalyzerConfigDocumentIsInSolution::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 9, + "method": false, + "sites": 2 + }, + "label": "conditional", + "reason": "check-and-throw helper" + }, + "csharp/roslyn::Workspace.cs::CheckDocumentIsInSolution::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 9, + "method": false, + "sites": 4 + }, + "label": "conditional", + "reason": "check-and-throw helper" + }, + "csharp/roslyn::Workspace.cs::CheckProjectDoesNotHaveMetadataReference::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 7, + "method": false, + "sites": 1 + }, + "label": "conditional", + "reason": "check-and-throw helper" + }, + "csharp/roslyn::Workspace.cs::CheckProjectHasMetadataReference::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 7, + "method": false, + "sites": 1 + }, + "label": "conditional", + "reason": "check-and-throw helper" + }, + "csharp/roslyn::Workspace.cs::CheckProjectHasProjectReference::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 9, + "method": false, + "sites": 1 + }, + "label": "conditional", + "reason": "check-and-throw helper" + }, + "csharp/roslyn::Workspace.cs::CheckProjectIsInSolution::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 9, + "method": false, + "sites": 5 + }, + "label": "conditional", + "reason": "check-and-throw helper" + }, + "csharp/roslyn::Workspace.cs::CheckProjectIsNotInSolution::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 9, + "method": false, + "sites": 3 + }, + "label": "conditional", + "reason": "check-and-throw helper" + }, + "csharp/roslyn::Workspace.cs::CheckSolutionDoesNotHaveAnalyzerReference::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 7, + "method": false, + "sites": 1 + }, + "label": "conditional", + "reason": "check-and-throw helper" + }, + "csharp/roslyn::Workspace.cs::CheckSolutionIsEmpty::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 7, + "method": false, + "sites": 1 + }, + "label": "conditional", + "reason": "check-and-throw helper" + }, + "embedded_python/meow_turtle::boot.py::print_manifest::debug_prints": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 11, + "method": false, + "sites": 1 + }, + "label": "wrapper", + "reason": "boot diagnostics: its purpose is printing the manifest" + }, + "fortran/wrf::module_initialize_real.F::wrf_error_fatal::debug_prints": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 5, + "method": false, + "sites": 163 + }, + "label": "wrapper", + "reason": "`print *,string` then `stop`: unconditional print" + }, + "fortran/wrf::module_initialize_real.F::wrf_error_fatal::panics_and_aborts": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 5, + "method": false, + "sites": 163 + }, + "label": "wrapper", + "reason": "`print *,string` then `stop`: unconditional abort" + }, + "fortran/wrf::module_initialize_real.F::wrf_message::debug_prints": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 7, + "method": false, + "sites": 111 + }, + "label": "wrapper", + "reason": "`print *,trim(message)`: unconditional print" + }, + "go/core::main.go::handlePanic::panics_and_aborts": { + "evidence": { + "branches": 2, + "header_has_name": true, + "loc": 11, + "method": false, + "sites": 1 + }, + "label": "conditional", + "reason": "recover handler: re-panics or Fatalf only when a panic was caught" + }, + "go/core::proc.go::dumpgstatus::debug_prints": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 5, + "method": false, + "sites": 6 + }, + "label": "wrapper", + "reason": "runtime status dump: unconditional prints" + }, + "go/core::value.go::MakeMapWithSize::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 8, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "constructor; panics only on a non-map type" + }, + "groovy/fineract_gradle_plugin::org_apache_fineract_gradle_service_TemplateService.groovy::handleTemplateException::debug_prints": { + "evidence": { + "branches": 2, + "header_has_name": true, + "loc": 9, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "framework error-handler override; pw.print writes the template error to a writer" + }, + "groovy/spock_smoke_specs::GroovyCallChain.groovy::c::panics_and_aborts": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 3, + "method": false, + "sites": 1 + }, + "label": "test_helper", + "reason": "Spock smoke-spec fixture that throws unconditionally to build a call chain" + }, + "javascript/react::ReactFlightServer.js::error::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 10, + "method": false, + "sites": 10 + }, + "label": "incidental", + "reason": "`abort` is the string event name in removeEventListener('abort', ...)" + }, + "javascript/react::ReactFlightServer.js::serializeEval::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 10, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "dev-only guard, then the real work (returns '$E' + source)" + }, + "javascript/react::build.js::forbidFBJSImports::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 12, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "returns a plugin object; the throw is inside a nested method" + }, + "javascript/react::build.js::resolveId::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 7, + "method": false, + "sites": 1 + }, + "label": "conditional", + "reason": "import guard: throws only for forbidden fbjs imports" + }, + "lua/cosmopolitan::all.lua::round::debug_prints": { + "evidence": { + "branches": 0, + "header_has_name": false, + "loc": 4, + "method": false, + "sites": 3 + }, + "label": "incidental", + "reason": "span misaligned: the slice is a local-alias declaration line, not round's body" + }, + "lua/cosmopolitan::binarytrees.lua::CheckTree::panics_and_aborts": { + "evidence": { + "branches": 3, + "header_has_name": true, + "loc": 10, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "benchmark tree checker; errors only when out of CPU" + }, + "lua/cosmopolitan::binarytrees.lua::MakeTree::panics_and_aborts": { + "evidence": { + "branches": 3, + "header_has_name": true, + "loc": 12, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "benchmark tree builder; errors only when out of CPU" + }, + "lua/cosmopolitan::heavy.lua::aux::debug_prints": { + "evidence": { + "branches": 4, + "header_has_name": false, + "loc": 11, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "span misaligned: the slice is a neighbouring test function's code" + }, + "lua/cosmopolitan::locals.lua::foo::debug_prints": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 4, + "method": false, + "sites": 2 + }, + "label": "incidental", + "reason": "`print` used as a value (__close = print), not called" + }, + "lua/cosmopolitan::nextvar.lua::check::debug_prints": { + "evidence": { + "branches": 4, + "header_has_name": true, + "loc": 8, + "method": false, + "sites": 3 + }, + "label": "test_helper", + "reason": "Lua test-suite check: prints and asserts on mismatch" + }, + "lua/cosmopolitan::nextvar.lua::mp2::debug_prints": { + "evidence": { + "branches": 4, + "header_has_name": false, + "loc": 5, + "method": false, + "sites": 2 + }, + "label": "incidental", + "reason": "span misaligned: the print belongs to code above mp2" + }, + "lua/redis::sort.lua::show::debug_prints": { + "evidence": { + "branches": 2, + "header_has_name": true, + "loc": 10, + "method": false, + "sites": 4 + }, + "label": "wrapper", + "reason": "table printer: its purpose is writing to io" + }, + "matlab/eeglab::eeg_eval.m::update_datafield::debug_prints": { + "evidence": { + "branches": 1, + "header_has_name": false, + "loc": 8, + "method": false, + "sites": 3 + }, + "label": "incidental", + "reason": "span misaligned: the slice is mid-function code, not update_datafield's definition" + }, + "perl/spamassassin::Makefile.PL::macro_ref::debug_prints": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 5, + "method": false, + "sites": 10 + }, + "label": "incidental", + "reason": "sprintf formats a string; it prints nothing (a debug_prints rule quirk)" + }, + "python/cython::Nodes.py::call::debug_prints": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 5, + "method": false, + "sites": 2 + }, + "label": "incidental", + "reason": "inner tracing closure named `call`; a debug print only when retval is None" + }, + "python/cython::Nodes.py::check::debug_prints": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 7, + "method": false, + "sites": 2 + }, + "label": "incidental", + "reason": "decorator factory; the print is inside the inner closure, conditional" + }, + "python/numpy::_linalg.py::_assert_finite::panics_and_aborts": { + "evidence": { + "branches": 3, + "header_has_name": true, + "loc": 4, + "method": false, + "sites": 2 + }, + "label": "conditional", + "reason": "assert helper: raises only on non-finite input" + }, + "python/numpy::core.py::_extremum_fill_value::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 12, + "method": false, + "sites": 2 + }, + "label": "incidental", + "reason": "fill-value computation; the raise is in a nested helper" + }, + "python/numpy::core.py::_scalar_fill_value::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 7, + "method": false, + "sites": 2 + }, + "label": "incidental", + "reason": "dict lookup; raises only on KeyError" + }, + "python/numpy::crackfortran.py::_resolvetypedefpattern::debug_prints": { + "evidence": { + "branches": 5, + "header_has_name": true, + "loc": 9, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "stray debug print inside a regex-parsing helper" + }, + "python/numpy::crackfortran.py::myeval::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 6, + "method": false, + "sites": 5 + }, + "label": "incidental", + "reason": "eval helper; raises only on a non-numeric result" + }, + "python/numpy::genapi.py::skip_brackets::panics_and_aborts": { + "evidence": { + "branches": 6, + "header_has_name": true, + "loc": 10, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "bracket scanner; raises only on no match" + }, + "python/numpy::symbolic.py::as_integer::panics_and_aborts": { + "evidence": { + "branches": 3, + "header_has_name": true, + "loc": 9, + "method": false, + "sites": 3 + }, + "label": "incidental", + "reason": "type conversion; raises only when no branch matched" + }, + "python/numpy::symbolic.py::as_number::panics_and_aborts": { + "evidence": { + "branches": 5, + "header_has_name": true, + "loc": 11, + "method": false, + "sites": 21 + }, + "label": "incidental", + "reason": "type conversion; raises only when no branch matched" + }, + "python/numpy::symbolic.py::fromstring::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 10, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "parser entry point; raises only when the parse fails" + }, + "rust/bevy::bevy_ecs_query.rs::assert_all_exact_sizes_iterator_equal::debug_prints": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 11, + "method": false, + "sites": 7 + }, + "label": "test_helper", + "reason": "test assertion helper in bevy's query tests" + }, + "scala/kafka::Partition.scala::localLogWithEpochOrThrow::panics_and_aborts": { + "evidence": { + "branches": 6, + "header_has_name": true, + "loc": 12, + "method": false, + "sites": 4 + }, + "label": "conditional", + "reason": "`*OrThrow` accessor: throws only on the error branch" + }, + "scala/kafka::ReplicaManager.scala::getPartitionOrException::panics_and_aborts": { + "evidence": { + "branches": 6, + "header_has_name": true, + "loc": 11, + "method": false, + "sites": 13 + }, + "label": "conditional", + "reason": "`*OrException` accessor: throws only on the error branches" + }, + "scala/kafka::SocketServer.scala::parseRequestHeader::panics_and_aborts": { + "evidence": { + "branches": 6, + "header_has_name": true, + "loc": 10, + "method": false, + "sites": 1 + }, + "label": "incidental", + "reason": "request parsing; throws only on a disabled/unsupported API" + }, + "scala/kafka::SocketServer.scala::processException::panics_and_aborts": { + "evidence": { + "branches": 3, + "header_has_name": true, + "loc": 6, + "method": false, + "sites": 4 + }, + "label": "conditional", + "reason": "exception router: rethrows ControlThrowable, else logs" + }, + "solidity/openzeppelin::Governor.sol::_validateStateBitmap::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 7, + "method": false, + "sites": 4 + }, + "label": "conditional", + "reason": "state validator: reverts only when the state is not allowed" + }, + "typescript/playwright::dispatcher.ts::collect::panics_and_aborts": { + "evidence": { + "branches": 1, + "header_has_name": true, + "loc": 8, + "method": false, + "sites": 2 + }, + "label": "incidental", + "reason": "span misaligned: the `abort` hit is the enclosing method's controller.abort, not collect's" + }, + "typescript/vscode::lifecycle.ts::dispose::panics_and_aborts": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 10, + "method": false, + "sites": 4 + }, + "label": "incidental", + "reason": "span misaligned: the slice is the enclosing code's catch/rethrow" + }, + "zig/tigerbeetle::build.zig::build_ci_step::panics_and_aborts": { + "evidence": { + "branches": 0, + "header_has_name": true, + "loc": 12, + "method": false, + "sites": 9 + }, + "label": "incidental", + "reason": "build-step helper; @panic only on allocation failure" + }, + "zig/zig::InternPool.zig::unwrapCoercedFunc::panics_and_aborts": { + "evidence": { + "branches": 2, + "header_has_name": true, + "loc": 11, + "method": false, + "sites": 2 + }, + "label": "incidental", + "reason": "switch with `else => unreachable` for impossible tags" + }, + "zig/zig::Sema.zig::addDeclaredHereNote::panics_and_aborts": { + "evidence": { + "branches": 3, + "header_has_name": true, + "loc": 12, + "method": false, + "sites": 6 + }, + "label": "incidental", + "reason": "note builder; `unreachable` only for impossible types" + } + } +} diff --git a/tests/tools/wrapper_probe.py b/tests/tools/wrapper_probe.py new file mode 100644 index 0000000000..71ce229842 --- /dev/null +++ b/tests/tools/wrapper_probe.py @@ -0,0 +1,343 @@ +#!/usr/bin/env python3 +# ============================================================================== +# GitGalaxy +# Copyright (c) 2026 Joe Esquibel +# +# This source code is licensed under the PolyForm Noncommercial License 1.0.0. +# You may not use this file except in compliance with the License. +# A copy of the license can be found in the LICENSE file in the root directory +# of this project, or at https://polyformproject.org/licenses/noncommercial/1.0.0/ +# ============================================================================== +""" +Wrapper probe: which project-local THIN WRAPPERS hide a literal-vocabulary rule's +real use, and how many call sites go through them? (#3315, step 1 of epic #3313) + +A literal rule (`debug_prints`, `panics_and_aborts`, `memory_alloc`, ...) counts +calls to a primitive by name. A project that wraps the primitive in its own +helper -- fortran/wrf's `wrf_error_fatal` is `print *,string` then `stop`, with +166 call sites -- makes every one of those call sites invisible to the rule. +`docs/known_blind_spots.md` records the limitation; this probe measures it, so +the detection heuristic the epic builds is chosen from data, not guessed. + +MEASUREMENT ONLY. Nothing here changes what the engine records. + +USAGE + # the step-1 run: both rules, every crucible language, the LOC sweep + python tests/tools/wrapper_probe.py --json /tmp/wrappers.json + + # one language, with every candidate's body (what the label pass reads) + python tests/tools/wrapper_probe.py --langs fortran --detail + + # precision against the committed hand labels + python tests/tools/wrapper_probe.py --json /tmp/wrappers.json --score tests/tools/wrapper_labels.json + +DEFINITIONS (each is a decision #3315 measures, not a given) + candidate A non-synthetic extracted function of <= --max-loc LOC whose body + hits the rule anywhere other than on its own name. + call site An UNQUALIFIED `name(` occurrence inside another function's body, + after the engine's literal shield. The engine's own `calls_out_to` + gates it (the language's call-form rule saw a call), so a + same-spelled word that is not a call is never credited. `x.f(`, + `x::f(` and `x->f(` are method or namespace calls on something + else and are never credited. + resolution A call site credits a candidate only when the callee resolves to + it: a definition of that name in the CALLER'S OWN FILE first, else + exactly one definition in the repo group. Zero definitions (the + callee lives outside the corpus sample) or several (a name + collision) credit nothing -- that is the census's go/core `lock` + false attribution, made impossible by construction. + method A candidate defined as a method (Go receiver, Python `self`/`cls` + first parameter, a qualified header such as Lua `function M.f` or + C++ `X::f`) is only reachable through a qualifier, so it receives + no unqualified call sites at all. + +The crucible samples files per repository, so a wrapper defined outside the +sample (CPython's obmalloc.c, curl's curlx) is invisible here. Counts are a +floor for those, not an estimate. +""" + +from __future__ import annotations + +import argparse +import collections +import json +import random +import re +import sys +from pathlib import Path +from typing import Any, Optional + +REPO_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "tests" / "tools")) + +import rule_probe # noqa: E402 (corpus discovery, shared deliberately) + +from gitgalaxy.core.detector import StructuralExtractor # noqa: E402 +from gitgalaxy.core.prism import Prism # noqa: E402 +from gitgalaxy.standards.gitgalaxy_config import LEXICAL_FAMILY_HEURISTICS # noqa: E402 +from gitgalaxy.standards.language_standards import LANGUAGE_DEFINITIONS # noqa: E402 + +DEFAULT_RULES = ("debug_prints", "panics_and_aborts") +LOC_SWEEP = (3, 5, 8, 12) +LABELS = ("wrapper", "conditional", "incidental", "test_helper") +_IDENT = re.compile(r"[A-Za-z_$][\w$]*") + + +def candidate_key(group: str, file: str, name: str, rule: str) -> str: + """The stable identity of one candidate across runs (and in the label file).""" + return f"{group}::{file}::{name}::{rule}" + + +def is_method_definition(lang: str, header: str, name: str) -> bool: + """Whether the definition header makes `name` reachable only through a qualifier. + + `header` is the code from the function's start to the end of its first line. + Conservative: an unrecognised shape is NOT a method, so the rule can only + remove credit that a real qualifier would have removed anyway. + """ + before = header.split(name, 1)[0] + if re.search(r"(?:\.|::|->|:)\s*$", before): # Lua `M.f`/`M:f`, C++ `X::f`, Ruby `self.f` + return True + if lang == "go" and re.match(r"\s*func\s*\(", header): # receiver + return True + if lang == "python": + m = re.search(re.escape(name) + r"\s*\(\s*(\w+)", header) + return bool(m and m.group(1) in ("self", "cls")) + return False + + +def _unqualified_sites(body: str, name: str) -> int: + """Unqualified `name(` occurrences in an already literal-shielded body.""" + return sum(1 for _ in re.finditer(r"(?])" + re.escape(name) + r"\s*\(", body)) + + +def probe_group( + lang: str, group_dir: Path, rules: tuple[str, ...], max_loc: int, prism: Prism, crucible: Path +) -> dict[str, Any]: + """Candidates and their resolved call sites for one crucible repo group.""" + registry = LANGUAGE_DEFINITIONS[lang]["rules"] + patterns = {r: registry.get(r) for r in rules} + patterns = {r: p for r, p in patterns.items() if hasattr(p, "finditer")} + group = str(group_dir.relative_to(crucible)) + extractor = StructuralExtractor(lang, LANGUAGE_DEFINITIONS) + exts = rule_probe._extensions(lang) + + literal: collections.Counter[str] = collections.Counter() + definitions: dict[str, list[str]] = collections.defaultdict(list) # name -> files + candidates: dict[str, dict[str, Any]] = {} + callers: list[tuple[str, str, list[str]]] = [] # (file, shielded body, calls_out_to) + + for path in sorted(group_dir.rglob("*"), key=lambda p: p.parts): + if not path.is_file() or (exts and path.suffix.lower() not in exts and path.name.lower() not in exts): + continue + if path.stat().st_size > rule_probe.MAX_FILE_BYTES: + continue + rel = str(path.relative_to(group_dir)) + try: + code = prism.split_streams(path.read_text(encoding="utf-8", errors="ignore"), lang)["code_stream"] + functions = extractor.splice(code, "")["functions"] + except Exception: # a probe reports what it can read; one bad file is not a finding + continue + for rule, pattern in patterns.items(): + literal[rule] += sum(1 for _ in pattern.finditer(code)) + for func in functions: + name = func.get("name") or "" + if func.get("is_synthetic_slice") or not _IDENT.fullmatch(name): + continue + definitions[name].append(rel) + body = code[func["start_idx"] : func["end_idx"]] + callers.append((rel, extractor._apply_literal_shield(body, lang), list(func.get("calls_out_to") or []))) + if (func.get("loc") or 0) > max_loc: + continue + own = [m.span() for m in re.finditer(r"(? dict[str, Any]: + crucible = rule_probe.CRUCIBLE + prism = Prism(LEXICAL_FAMILY_HEURISTICS, LANGUAGE_DEFINITIONS) + groups = [] + for lang_dir in sorted(p for p in crucible.iterdir() if p.is_dir()): + lang = lang_dir.name + if langs and lang not in langs: + continue + definition = LANGUAGE_DEFINITIONS.get(lang) + if not definition or definition.get("invocation_model") == "positional": + continue # no invoke-by-name form, so no wrapper can be called by name + if not any(hasattr(definition["rules"].get(r), "finditer") for r in rules): + continue + for group_dir in sorted((p for p in lang_dir.iterdir() if p.is_dir()), key=lambda p: p.name): + result = probe_group(lang, group_dir, rules, max_loc, prism, crucible) + if result["candidates"]: + groups.append(result) + return {"crucible": str(crucible), "rules": list(rules), "max_loc": max_loc, "groups": groups} + + +def all_candidates(data: dict[str, Any]) -> dict[str, dict[str, Any]]: + return {k: c for g in data["groups"] for k, c in g["candidates"].items()} + + +# Candidate filters the label pass scores, each a predicate over one candidate. +FILTERS = { + "all": lambda c: True, + "aligned": lambda c: c.get("header_has_name", True), + "aligned+no_branches": lambda c: c.get("header_has_name", True) and c.get("branches", 0) == 0, +} + + +def score_filters(data: dict[str, Any], labels: dict[str, Any], max_loc: int) -> None: + """Labelled precision of each filter at `max_loc`: per candidate and site-weighted, + strict (`wrapper` only) and lenient (`wrapper` + `conditional`).""" + cands = all_candidates(data) + judged = {k: c for k, c in cands.items() if k in labels and c["sites"] and c["loc"] <= max_loc} + print(f"\nlabelled candidates at max_loc={max_loc}: {len(judged)}") + print( + f"{'filter':22} {'kept':>4} {'strict':>7} {'lenient':>8} {'sites kept':>10} {'strict/site':>11} {'recall(site)':>12}" + ) + wrapper_sites = sum(c["sites"] for k, c in judged.items() if labels[k]["label"] == "wrapper") + for fname, pred in FILTERS.items(): + kept = {k: c for k, c in judged.items() if pred(c)} + if not kept: + continue + lbl = [labels[k]["label"] for k in kept] + sites = sum(c["sites"] for c in kept.values()) + w_sites = sum(c["sites"] for k, c in kept.items() if labels[k]["label"] == "wrapper") + print( + f"{fname:22} {len(kept):4} {lbl.count('wrapper') / len(kept):7.0%} " + f"{(lbl.count('wrapper') + lbl.count('conditional')) / len(kept):8.0%} {sites:10} " + f"{w_sites / max(1, sites):11.0%} {w_sites / max(1, wrapper_sites):12.0%}" + ) + + +def summarize(data: dict[str, Any], labels: Optional[dict[str, Any]] = None) -> None: + cands = all_candidates(data) + print(f"{'rule':20} {'max_loc':>7} {'literal':>8} {'wrappers':>8} {'sites':>6} labelled precision") + for rule in data["rules"]: + lit = sum(g["literal"].get(rule, 0) for g in data["groups"]) + for n in LOC_SWEEP: + if n > data["max_loc"]: + continue + live = [c for c in cands.values() if c["rule"] == rule and c["loc"] <= n and c["sites"]] + sites = sum(c["sites"] for c in live) + prec = "" + if labels: + judged = [(labels[k]["label"], c["sites"]) for k, c in cands.items() if k in labels and c in live] + if judged: + strict = sum(1 for lbl, _ in judged if lbl == "wrapper") / len(judged) + site_w = sum(s for lbl, s in judged if lbl == "wrapper") / max(1, sum(s for _, s in judged)) + prec = f"strict {strict:.0%} of {len(judged)} labelled, {site_w:.0%} site-weighted" + print(f"{rule:20} {n:7} {lit:8} {len(live):8} {sites:6} {prec}") + + +def sample_for_labelling(data: dict[str, Any], seed: int, size: int, floor: int) -> list[str]: + """Every candidate with >= `floor` call sites, plus a seeded random sample of the rest.""" + live = {k: c for k, c in all_candidates(data).items() if c["sites"]} + heavy = sorted(k for k, c in live.items() if c["sites"] >= floor) + rest = sorted(k for k in live if k not in heavy) + random.Random(seed).shuffle(rest) + return heavy + rest[:size] + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--langs", help="comma-separated crucible languages (default: all)") + ap.add_argument("--rules", default=",".join(DEFAULT_RULES)) + ap.add_argument("--max-loc", type=int, default=max(LOC_SWEEP)) + ap.add_argument("--json", type=Path, help="write the full result here") + ap.add_argument("--load", type=Path, help="read a previous --json result instead of scanning") + ap.add_argument("--score", type=Path, help="report labelled precision from this label file") + ap.add_argument("--sample", type=int, metavar="N", help="print the label-pass sample (N random + every heavy)") + ap.add_argument("--detail", action="store_true", help="print every live candidate's body") + args = ap.parse_args() + + if args.load: + data = json.loads(args.load.read_text(encoding="utf-8")) + else: + langs = args.langs.split(",") if args.langs else None + data = run(langs, tuple(args.rules.split(",")), args.max_loc) + if args.json: + args.json.write_text(json.dumps(data, indent=1, sort_keys=True) + "\n", encoding="utf-8") + labels = json.loads(args.score.read_text(encoding="utf-8"))["labels"] if args.score else None + summarize(data, labels) + if labels: + for n in LOC_SWEEP: + if n <= data["max_loc"]: + score_filters(data, labels, n) + if args.sample is not None: + cands = all_candidates(data) + for key in sample_for_labelling(data, seed=3313, size=args.sample, floor=20): + c = cands[key] + print(f"\n=== {key} loc={c['loc']} sites={c['sites']} method={c['method']} hit={c['hit']!r}\n{c['body']}") + elif args.detail: + for key, c in sorted(all_candidates(data).items()): + if c["sites"]: + print(f"\n=== {key} loc={c['loc']} sites={c['sites']} hit={c['hit']!r}\n{c['body']}") + return 0 + + +if __name__ == "__main__": + sys.exit(main())