From abb66328d2a014edaad2c14291fae538e53068db Mon Sep 17 00:00:00 2001 From: squid-protocol Date: Sat, 5 Sep 2026 22:06:21 -0400 Subject: [PATCH] fix(core-engine): dependency_density is coupling per line, not per control-flow ratio (#2770) `dependency_density` divided by `max(int(coding_loc * control_flow_ratio), 1)`. `control_flow_ratio` is `branch / (branch + structural_boundaries)`, so it is 0 for ANY file with no branches -- config, constants modules, data classes, most yaml/dockerfile -- and the denominator then collapsed onto its own `max(..., 1)` floor, making the "density" report the raw import COUNT. Measured on keyword-rosetta (one 12-probe program in 46 languages): 4 of the 5 files in every language have branch == 0, so the column read 1.000 for python's b.py and 2.000 for embedded_python's a.py -- their import counts -- against 0.048 for yaml's main.yml, which has branches. A 40x spread on one program. Three failures stacked: 1. the discontinuity at branch == 0 (above), up to 20x on one added `if`; 2. `int()` truncated the product BEFORE the floor, so any product under 2.0 also landed on 1; 3. where a language's `structural_boundaries` rule never fires, control_flow_ratio is 1.0 and the denominator silently became coding_loc -- the right answer, reached by accident (makefile). A coupling measure has no reason to read a control-flow ratio at all. `analysis_lens.ENGINE_CONSTANTS` already states the doctrine this column was missed by: "every per-file density equation divides by max(coding_loc, EVIDENCE_MASS_FLOOR)" (#2655). `dependency_density` now does, matching the per-function treatment its sibling `func_internal_density` got in #2705 six lines above it. security_auditor.py carried the same `coding_loc * control_flow_ratio` idiom. `logic_loc` is kept there unchanged -- `log_logic_loc` is an input to the pre-trained model and redefining a trained input is a retrain, not a bug fix (the frozen `func_internal_density: 0.0` placeholder is the precedent) -- but its defect is now recorded so the retrain can price it. The dead half is removed: control_flow_ratio is bounded [0, 1] by construction, so `logic_loc <= coding_loc` always and `max(logic_loc, coding_loc, 1)` could never select `logic_loc`. Written as the `max(coding_loc, 1)` it always was; no behavior change. Tests: tests/tools_recorders/test_dependency_density_floor.py pins the same four properties its #2705 sibling pins (invariance below the floor, no cliff at it, the density regime above it, zero/empty safety) plus the one specific to this fix -- the column does not move with control_flow_ratio. 14 of its 16 cases fail against the old formula. Verification: full suite 7791 passed. Golden masters unchanged in BOTH modes (`pytest -m golden_crucible`, full_precision and zero_dependency) -- the column is sqlite-only and appears in neither fixture. ruff/mypy baseline-gated checks green; the ruff baseline is regenerated for pure line shifts in record_keeper.py (every (file, rule) count identical before and after). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_016BHBDqfybkLJqnQt9AoyTt --- gitgalaxy/recorders/record_keeper.py | 23 ++- gitgalaxy/security/security_auditor.py | 18 ++- tests/ruff_audit_baseline.json | 34 ++--- .../test_dependency_density_floor.py | 144 ++++++++++++++++++ 4 files changed, 196 insertions(+), 23 deletions(-) create mode 100644 tests/tools_recorders/test_dependency_density_floor.py diff --git a/gitgalaxy/recorders/record_keeper.py b/gitgalaxy/recorders/record_keeper.py index 490b713a91..0673b1dbfb 100644 --- a/gitgalaxy/recorders/record_keeper.py +++ b/gitgalaxy/recorders/record_keeper.py @@ -25,6 +25,9 @@ # #2705: per-function evidence-mass floor (see analysis_lens.ENGINE_CONSTANTS). FUNC_EVIDENCE_MASS_FLOOR = float(cast("int", ENGINE_CONSTANTS["FUNC_EVIDENCE_MASS_FLOOR"])) +# #2655: per-file evidence-mass floor, the denominator every per-file density +# equation divides by. #2770 brought dependency_density under it. +EVIDENCE_MASS_FLOOR = float(cast("int", ENGINE_CONSTANTS["EVIDENCE_MASS_FLOOR"])) class FolderStats(TypedDict): @@ -495,12 +498,22 @@ def record_mission( # avg_loc > 0 guard: no functions -> avg_comp 0 -> density 0. func_internal_density = avg_comp / max(avg_loc, FUNC_EVIDENCE_MASS_FLOOR) - logic_loc_denom = max( - int(file_data.get("coding_loc", 1) * tel.get("control_flow_ratio", 0.0)), - 1, - ) + # #2770: coupling per line of code, over the same evidence-mass floor + # every other per-file density uses (analysis_lens' EVIDENCE_MASS_FLOOR + # doctrine, #2655). The old denominator was + # `max(int(coding_loc * control_flow_ratio), 1)`, which failed three ways + # at once: (1) control_flow_ratio is 0 for any file with no branches -- + # config, constants modules, data classes, most yaml/dockerfile -- so the + # denominator collapsed onto the max(..., 1) floor and the column reported + # the raw import COUNT, up to 20x the value the same file got after adding + # one `if`; (2) the int() truncated the product before the floor, so + # anything under 2.0 also landed on 1; (3) where a language's + # structural_boundaries rule never fires, control_flow_ratio is 1.0 and the + # denominator silently became coding_loc -- the right answer, reached by + # accident. A coupling measure has no reason to read a control-flow ratio + # in the first place; `import / coding_loc` is what the name describes. import_count = len(file_data.get("raw_imports", [])) - dependency_density = import_count / float(logic_loc_denom) + dependency_density = import_count / max(float(file_data.get("coding_loc", 0) or 0), EVIDENCE_MASS_FLOOR) # #364: security_auditor.py writes "AI Threat Score" -- "AI Threat # Confidence" was never a real producer key here, just a dead primary diff --git a/gitgalaxy/security/security_auditor.py b/gitgalaxy/security/security_auditor.py index 8e97a17ec3..96d0d1c3dd 100644 --- a/gitgalaxy/security/security_auditor.py +++ b/gitgalaxy/security/security_auditor.py @@ -356,8 +356,24 @@ def _construct_feature_matrix(self, artifacts): # 1. Base Variables cfr = tel.get("control_flow_ratio", 0.0) coding_loc = artifact.get("coding_loc", 0) + + # #2770: `logic_loc` is `coding_loc * control_flow_ratio`, the same + # idiom record_keeper.py used for dependency_density until #2770 + # retired it there. It is kept HERE, unchanged, for one reason only: + # `log_logic_loc` below is an input to the pre-trained model, and + # redefining a trained input is a retrain, not a bug fix -- the same + # call the frozen `func_internal_density: 0.0` placeholder documents + # further down. Its known defect, recorded so the retrain can price + # it: control_flow_ratio is 0 for any branchless file, so logic_loc + # collapses onto the max(..., 1) floor and the feature reads 1 for a + # 500-line config as readily as for a 5-line one. logic_loc = max(int(round(coding_loc * cfr)), 1) - safe_denom = max(logic_loc, coding_loc, 1) + + # control_flow_ratio is bounded [0, 1] by construction (branch over + # branch + structural_boundaries), so logic_loc <= coding_loc always + # and the old `max(logic_loc, coding_loc, 1)` could never select + # logic_loc. Written as what it always computed. + safe_denom = max(coding_loc, 1) # #2691: per-function statistics describe real functions, not the # slicer's synthetic top-level buckets. diff --git a/tests/ruff_audit_baseline.json b/tests/ruff_audit_baseline.json index 213f921c2c..71f569dc39 100644 --- a/tests/ruff_audit_baseline.json +++ b/tests/ruff_audit_baseline.json @@ -23,28 +23,28 @@ "gitgalaxy/recorders/audit_recorder.py:358: PERF401": "Use `list.extend` to create a transformed list", "gitgalaxy/recorders/gpu_recorder.py:285: C414": "Unnecessary `list()` call within `sorted()`", "gitgalaxy/recorders/llm_recorder.py:1049: SIM102": "Use a single `if` statement instead of nested `if` statements", - "gitgalaxy/recorders/record_keeper.py:190: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:191: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:192: W291": "Trailing whitespace", "gitgalaxy/recorders/record_keeper.py:193: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:292: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:293: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:294: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:194: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:195: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:196: W291": "Trailing whitespace", "gitgalaxy/recorders/record_keeper.py:295: W291": "Trailing whitespace", "gitgalaxy/recorders/record_keeper.py:296: W291": "Trailing whitespace", "gitgalaxy/recorders/record_keeper.py:297: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:774: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:778: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:780: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:781: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:827: RUF005": "Consider iterable unpacking instead of concatenation", - "gitgalaxy/recorders/record_keeper.py:896: RUF005": "Consider iterable unpacking instead of concatenation", - "gitgalaxy/recorders/record_keeper.py:933: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:934: W291": "Trailing whitespace", - "gitgalaxy/recorders/record_keeper.py:965: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:298: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:299: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:300: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:787: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:791: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:793: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:794: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:840: RUF005": "Consider iterable unpacking instead of concatenation", + "gitgalaxy/recorders/record_keeper.py:909: RUF005": "Consider iterable unpacking instead of concatenation", + "gitgalaxy/recorders/record_keeper.py:946: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:947: W291": "Trailing whitespace", + "gitgalaxy/recorders/record_keeper.py:978: W291": "Trailing whitespace", "gitgalaxy/recorders/sbom_recorder.py:221: PERF401": "Use `list.extend` to create a transformed list", - "gitgalaxy/security/security_auditor.py:359: RUF046": "Value being cast to `int` is already an integer", - "gitgalaxy/security/security_auditor.py:441: PERF203": "`try`-`except` within a loop incurs performance overhead", + "gitgalaxy/security/security_auditor.py:370: RUF046": "Value being cast to `int` is already an integer", + "gitgalaxy/security/security_auditor.py:457: PERF203": "`try`-`except` within a loop incurs performance overhead", "gitgalaxy/security/security_lens.py:443: SIM102": "Use a single `if` statement instead of nested `if` statements", "gitgalaxy/standards/config_resolver.py:254: UP045": "Use `X | None` for type annotations", "gitgalaxy/standards/config_resolver.py:255: UP045": "Use `X | None` for type annotations", diff --git a/tests/tools_recorders/test_dependency_density_floor.py b/tests/tools_recorders/test_dependency_density_floor.py new file mode 100644 index 0000000000..c80db2ea77 --- /dev/null +++ b/tests/tools_recorders/test_dependency_density_floor.py @@ -0,0 +1,144 @@ +# ============================================================================== +# GitGalaxy +# Copyright (c) 2026 Joe Esquibel +# +# This source code is licensed under the PolyForm Noncommercial License 1.0.0. +# You may not use this file except in compliance with the License. +# A copy of the license can be found in the LICENSE file in the root directory +# of this project, or at https://polyformproject.org/licenses/noncommercial/1.0.0/ +# ============================================================================== +""" +dependency_density under the evidence-mass floor (#2770). + +The column used to divide by `max(int(coding_loc * control_flow_ratio), 1)`, which +failed three ways at once: + + * control_flow_ratio is `branch / (branch + structural_boundaries)`, so it is 0 + for ANY file with no branches -- config, constants modules, data classes, most + yaml/dockerfile. The denominator then collapsed onto its own max(..., 1) floor + and the "density" reported the raw import COUNT, up to 20x the value the same + file recorded after one `if` was added to it. + * the int() truncated the product BEFORE the floor, so any product under 2.0 also + landed on 1 (measured on keyword-rosetta: 4 of the 5 files in every one of the + 46 languages). + * where a language's structural_boundaries rule never fires, control_flow_ratio is + 1.0 and the denominator silently became coding_loc -- the right answer, reached + by accident. + +The replacement is coupling per line of code over EVIDENCE_MASS_FLOOR, the same +denominator every other per-file density equation uses (#2655, pinned by +tests/core_engine/test_uef_length_invariance.py). This module pins the same +four properties its per-function sibling pins in test_func_internal_density_floor.py, +plus the one specific to this fix: + + 1. INVARIANCE -- below the floor, an identical import count scores identically at + any file length. + 2. CONTINUITY -- the value at the floor equals the value just below it. + 3. PARITY -- at floor+1 and above, the column is import_count / coding_loc. + 4. INDEPENDENCE - the column does not move with control_flow_ratio (the regression). + 5. ZERO -- no imports records 0, and coding_loc 0 does not divide by zero. + +Driven through RecordKeeper.record_mission, the only place the column is computed. +""" + +import sqlite3 +from unittest.mock import patch + +import pytest + +from gitgalaxy.recorders.record_keeper import RecordKeeper +from gitgalaxy.standards.analysis_lens import ENGINE_CONSTANTS + +FLOOR = int(ENGINE_CONSTANTS["EVIDENCE_MASS_FLOOR"]) + + +@pytest.fixture +def keeper(): + schemas = {"RISK_SCHEMA": ["cognitive_load"], "SIGNAL_SCHEMA": ["branch", "io"]} + with patch("gitgalaxy.recorders.record_keeper.RECORDING_SCHEMAS", schemas): + return RecordKeeper() + + +def _file(coding_loc, imports=2, cfr=0.1): + return { + "path": "main.py", + "name": "main.py", + "lang_id": "python", + "directory_group": ".", + "lock_tier": 0, + "total_loc": coding_loc + 5, + "coding_loc": coding_loc, + "doc_loc": 1, + "file_impact": 1.0, + "raw_imports": [f"dep_{i}.py" for i in range(imports)], + "hit_vector": [3, 1], + "telemetry": {"control_flow_ratio": cfr, "network_metrics": {}, "domain_context": {}}, + "classes": [], + "functions": [], + } + + +def _density(keeper, tmp_path, parsed): + db = tmp_path / "out.db" + if db.exists(): + db.unlink() + keeper.record_mission(parsed, [], {}, {"target": "t", "git_audit": {}}, str(db)) + conn = sqlite3.connect(db) + try: + return conn.execute("SELECT dependency_density FROM file_data").fetchone()[0] + finally: + conn.close() + + +@pytest.mark.parametrize("coding_loc", [1, 5, 13, 25, FLOOR - 1]) +def test_identical_imports_score_identically_below_the_floor(keeper, tmp_path, coding_loc): + """Property 1: the rosetta shells are 4-21 coding lines; every one of them must + read the same density as the same import count at the floor.""" + at_floor = _density(keeper, tmp_path, [_file(FLOOR)]) + shorter = _density(keeper, tmp_path, [_file(coding_loc)]) + assert shorter == pytest.approx(at_floor) + # and that shared value is the rescaled import count, nothing else + assert shorter == pytest.approx(2 / FLOOR) + + +def test_no_cliff_at_the_floor(keeper, tmp_path): + """Property 2: floor and floor-1 agree; floor+1 is the first point on the curve.""" + below = _density(keeper, tmp_path, [_file(FLOOR - 1)]) + at = _density(keeper, tmp_path, [_file(FLOOR)]) + above = _density(keeper, tmp_path, [_file(FLOOR + 1)]) + assert below == pytest.approx(at) + assert above < at # the density regime resumes, monotonically + + +@pytest.mark.parametrize("coding_loc", [FLOOR + 1, 150, 500]) +def test_density_regime_is_imports_per_coding_loc(keeper, tmp_path, coding_loc): + """Property 3: at or above the floor the column is exactly import_count / coding_loc.""" + assert _density(keeper, tmp_path, [_file(coding_loc, imports=3)]) == pytest.approx(3 / coding_loc) + + +@pytest.mark.parametrize("cfr", [0.0, 0.05, 0.5, 1.0]) +def test_column_does_not_move_with_control_flow_ratio(keeper, tmp_path, cfr): + """Property 4, the #2770 regression itself. A measure of imports has no reason to + move when branch structure does. Under the old formula the cfr=0.0 case divided by + max(int(200 * 0.0), 1) == 1 and recorded 2.0 -- its raw import count, 400x the + cfr=0.5 case, for a file whose imports are identical.""" + assert _density(keeper, tmp_path, [_file(200, cfr=cfr)]) == pytest.approx(2 / 200) + + +def test_no_imports_records_zero(keeper, tmp_path): + """Property 5a.""" + assert _density(keeper, tmp_path, [_file(200, imports=0)]) == 0.0 + + +def test_empty_file_does_not_divide_by_zero(keeper, tmp_path): + """Property 5b: the floor absorbs coding_loc 0 (a file the prism found no code in), + which the old max(..., 1) guard was carrying.""" + assert _density(keeper, tmp_path, [_file(0)]) == pytest.approx(2 / FLOOR) + + +def test_floor_is_the_shared_file_level_constant(): + """The column must track EVIDENCE_MASS_FLOOR itself, not a private copy of 50 -- + the point of #2655 was that one constant governs every per-file density.""" + from gitgalaxy.recorders import record_keeper + + assert record_keeper.EVIDENCE_MASS_FLOOR == float(ENGINE_CONSTANTS["EVIDENCE_MASS_FLOOR"])