Persist file/repo composition archetypes to the DB + add fit z-scores

gitgalaxy/metrics/archetype_classifier.py

@@ -17,7 +17,7 @@
 
 import bisect
 import math
-from typing import Any, Optional
+from typing import Any
 
 from gitgalaxy.standards import analysis_lens
 
@@ -31,28 +31,37 @@ def _rank(value: float, ref: list) -> float:
     return bisect.bisect_right(ref, float(value)) / (len(ref) - 1)
 
 
-def _nearest(vec: list, centroids: dict) -> Optional[str]:
+def _nearest(vec: list, centroids: dict):
+    """Return (name, euclidean_distance) of the nearest centroid, or (None, None)."""
     best, best_d = None, None
     for name, c in centroids.items():
         if len(c) != len(vec):
             continue  # length guard (stale brain) -> unclassified
         d = sum((a - b) * (a - b) for a, b in zip(vec, c))
         if best_d is None or d < best_d:
             best_d, best = d, name
-    return best
+    return best, (math.sqrt(best_d) if best_d is not None else None)
 
 
-def classify_file(f: dict[str, Any]) -> Optional[str]:
-    """Assign a file its composition archetype (function-stoichiometry + structure
-    + graph role). Returns None if the brain is unavailable."""
+def _fit_z(dist, name, brain) -> float:
+    """Archetype-fit z-score: (distance - cluster mean) / cluster std. 0 if unknown."""
+    zp = (brain.get("z_score_params") or {}).get(name)
+    if not zp or dist is None:
+        return 0.0
+    return round((dist - zp["mean"]) / (zp["std"] or 1.0), 3)
+
+
+def classify_file(f: dict[str, Any]):
+    """Return (composition_archetype, fit_z_score) for a file, or (None, None) if the
+    brain is unavailable. Bucket labels (non-code) carry z=0.0."""
     b = analysis_lens.FILE_ARCHETYPE_BRAIN
     if not b:
-        return None
+        return None, None
     tel = f.get("telemetry", {}) or {}
     lang = str(f.get("lang_id", "")).lower()
     coding_loc = float(f.get("coding_loc", 0) or 0)
     if lang in set(b["noncode_languages"]) or coding_loc < b["min_coding_loc"]:
-        return b["noncode_bucket"]
+        return b["noncode_bucket"], 0.0
 
     mix = tel.get("function_archetype_mix", {}) or {}
     total = sum(mix.get(a, 0) for a in b["stoich_archetypes"])
@@ -72,7 +81,10 @@ def classify_file(f: dict[str, Any]) -> Optional[str]:
         "log_blast_radius": math.log1p(max(float(net.get("normalized_blast_radius", 0.0) or 0.0), 0.0)),
     }
     aux = [_rank(raw[fn], b["aux_quantiles"][fn]) for fn in b["aux_features"]]
-    return _nearest(stoich + aux, b["centroids"]) or b["noncode_bucket"]
+    name, dist = _nearest(stoich + aux, b["centroids"])
+    if name is None:
+        return b["noncode_bucket"], 0.0
+    return name, _fit_z(dist, name, b)
 
 
 def _gini(vals: list) -> float:
@@ -83,15 +95,16 @@ def _gini(vals: list) -> float:
     return sum((2 * (i + 1) - n - 1) * x for i, x in enumerate(xs)) / (n * s)
 
 
-def classify_repo(parsed_files: list[dict[str, Any]]) -> Optional[str]:
-    """Aggregate the scan's per-file composition archetypes + scale + coupling into a
-    repo archetype. Requires classify_file to have populated composition_file_archetype."""
+def classify_repo(parsed_files: list[dict[str, Any]]):
+    """Return (repo_archetype, fit_z_score) aggregated from the scan's per-file
+    composition archetypes + scale + coupling, or (None, None). Requires classify_file
+    to have populated composition_file_archetype. Micro bucket carries z=0.0."""
     b = analysis_lens.REPO_ARCHETYPE_BRAIN
     if not b:
-        return None
+        return None, None
     file_count = len(parsed_files)
     if file_count < b["min_files"]:
-        return b["micro_bucket"]
+        return b["micro_bucket"], 0.0
 
     comp_counts: dict[str, int] = {}
     noncode = 0
@@ -115,4 +128,7 @@ def classify_repo(parsed_files: list[dict[str, Any]]) -> Optional[str]:
     scale = [_rank(math.log1p(file_count), q["log_file_count"]), _rank(math.log1p(total_loc), q["log_total_loc"])]
     ncf = [noncode / file_count if file_count else 0.0]
     coup = [_rank(_gini(pageranks), q["pagerank_gini"])]
-    return _nearest(comp + scale + ncf + coup, b["centroids"])
+    name, dist = _nearest(comp + scale + ncf + coup, b["centroids"])
+    if name is None:
+        return None, None
+    return name, _fit_z(dist, name, b)

gitgalaxy/metrics/signal_processor.py

@@ -1339,10 +1339,12 @@ def get_avg(metric_name):
         # available. Assign each file its composition archetype (function stoichiometry
         # + structure + graph role), then aggregate into a repo archetype.
         for _f in parsed_files:
-            _fa = archetype_classifier.classify_file(_f)
+            _fa, _fz = archetype_classifier.classify_file(_f)
             if _fa is not None:
-                _f.setdefault("telemetry", {})["composition_file_archetype"] = _fa
-        repo_composition_archetype = archetype_classifier.classify_repo(parsed_files)
+                _tel = _f.setdefault("telemetry", {})
+                _tel["composition_file_archetype"] = _fa
+                _tel["composition_file_z"] = _fz
+        repo_composition_archetype, repo_composition_z = archetype_classifier.classify_repo(parsed_files)
         file_composition_distribution: dict[str, int] = {}
         for _f in parsed_files:
             _fa = (_f.get("telemetry", {}) or {}).get("composition_file_archetype")
@@ -1361,6 +1363,7 @@ def get_avg(metric_name):
                 "volatility_index": volatility_idx,
                 "Percent_Visible": round((1 - darkness_ratio) * 100, 1),
                 "repo_composition_archetype": repo_composition_archetype,
+                "repo_composition_z": repo_composition_z,
                 "file_composition_distribution": file_composition_distribution,
             },
             "repo_macro_species": repo_macro_data,

gitgalaxy/recorders/audit_recorder.py

@@ -262,6 +262,7 @@ def generate_report(
                     ),
                     "File Archetype": telemetry.get("local_archetype", "N/A"),
                     "Composition Archetype": telemetry.get("composition_file_archetype", "N/A"),
+                    "Composition Fit (Z-Score)": round(float(telemetry.get("composition_file_z", 0.0) or 0.0), 3),
                     "File Drift (Z-Score)": telemetry.get("local_drift", 0.0),
                     "File Fingerprint": (
                         {k: round(v, 3) for k, v in telemetry.get("local_fingerprint", {}).items()}
@@ -591,6 +592,9 @@ def generate_report(
         _inner = summary.get("summary", {})
         if _inner.get("repo_composition_archetype"):
             summary["Repository Composition Archetype"] = _inner["repo_composition_archetype"]
+            summary["Repository Composition Fit (Z-Score)"] = round(
+                float(_inner.get("repo_composition_z", 0.0) or 0.0), 3
+            )
         _fcd = _inner.get("file_composition_distribution", {})
         _fcd_total = sum(_fcd.values())
         if _fcd_total:

gitgalaxy/recorders/llm_recorder.py

@@ -418,7 +418,10 @@ def _build_markdown(
         # file-archetype mix that composes it.
         repo_comp = sum_data.get("repo_composition_archetype")
         if repo_comp:
-            lines.append(f"> **Composition Archetype:** `{repo_comp}` (from the repo's file-archetype mix)")
+            rz = sum_data.get("repo_composition_z", 0.0) or 0.0
+            lines.append(
+                f"> **Composition Archetype:** `{repo_comp}` (z {rz:+.2f}; from the repo's file-archetype mix)"
+            )
         fcd = sum_data.get("file_composition_distribution", {})
         fcd_total = sum(fcd.values())
         if fcd_total:
@@ -885,7 +888,8 @@ def _outbound(file_data):
                 lines.append(f"- **Archetype:** `{arch}` (Distance: {dist} IQR)")
                 comp_arch = tel.get("composition_file_archetype")
                 if comp_arch:
-                    lines.append(f"- **Composition Archetype:** `{comp_arch}`")
+                    cz = tel.get("composition_file_z", 0.0) or 0.0
+                    lines.append(f"- **Composition Archetype:** `{comp_arch}` (z {cz:+.2f})")
                 lines.append(
                     f"- **Magnitude:** {m} | **LOC:** {loc} | **CtrlFlow:** {round(tel.get('control_flow_ratio', 0.0) * 100, 1)}% | **Authorship Centralization:** {round(tel.get('author_distribution', 0.0), 1)}%"
                 )

gitgalaxy/recorders/record_keeper.py

@@ -305,6 +305,8 @@ def record_mission(
                 is_zero_dependency_mode INTEGER DEFAULT 0,
                 {", ".join(hit_cols)},
                 file_composition TEXT,
+                repo_composition_archetype TEXT,
+                repo_composition_z REAL,
                 UNIQUE(repo_name, commit_hash)
             )
         """)
@@ -322,6 +324,8 @@ def record_mission(
         # gitgalaxy#2985: repo_data's half of the hit_cols heal (see file_data's
         # below for why the INSERTs make this mandatory, not merely tidy).
         _ensure_columns(cursor, "repo_data", hit_cols)
+        # repo composition archetype (function-stoichiometry taxonomy) + its fit z-score
+        _ensure_columns(cursor, "repo_data", ["repo_composition_archetype TEXT", "repo_composition_z REAL"])
         # #2992: same heal for a pre-#2992 repo_data.
         _ensure_columns(cursor, "repo_data", ["network_edges_unrecorded INTEGER"])
 
@@ -396,6 +400,8 @@ def record_mission(
                 pct_z_above_15 REAL DEFAULT 0.0,
                 file_archetype TEXT,
                 file_fingerprint TEXT,
+                composition_file_archetype TEXT,
+                composition_file_z REAL,
                 ecosystem_baseline TEXT,
                 repo_z_score REAL,
                 ai_threat_score REAL,
@@ -433,6 +439,8 @@ def record_mission(
         # repo_data is healed further down, beside its own existing guarded
         # ALTER; class_data carries no hit columns.
         _ensure_columns(cursor, "file_data", hit_cols)
+        # composition archetype (function-stoichiometry taxonomy) + its fit z-score
+        _ensure_columns(cursor, "file_data", ["composition_file_archetype TEXT", "composition_file_z REAL"])
 
         cursor.execute("""
             CREATE TABLE IF NOT EXISTS class_data (
@@ -775,6 +783,8 @@ def record_mission(
 
             file_archetype = tel.get("archetype", "Unknown")
             file_fingerprint_str = json.dumps(tel.get("archetype_fingerprint", {}))
+            composition_file_archetype = tel.get("composition_file_archetype", "Unclassified")
+            composition_file_z = float(tel.get("composition_file_z", 0.0) or 0.0)
 
             agg_total_loc += file_data.get("total_loc", 0)
             agg_coding_loc += file_data.get("coding_loc", 0)
@@ -991,6 +1001,8 @@ def record_mission(
                 pct_z_above_15,
                 file_archetype,
                 file_fingerprint_str,
+                composition_file_archetype,
+                composition_file_z,
                 repo_macro,
                 repo_z,
                 ai_score,
@@ -1051,6 +1063,7 @@ def record_mission(
                     author, ai_threat_class, ai_threat_confidence,
                     func_z_max, func_z_mean, func_z_median, pct_z_above_5, pct_z_above_15,
                     file_archetype, file_fingerprint,
+                    composition_file_archetype, composition_file_z,
                     ecosystem_baseline, repo_z_score,
                     ai_threat_score, is_malware, has_credentials, binary_anomaly, obfuscation_flag,
                     token_mass, financial_read_cost, agentic_isolation_risk, requires_hitl, appsec_god_mode, hallucination_zone, silent_mutation_risk,
@@ -1186,6 +1199,9 @@ def record_mission(
 
         macro_info = summary.get("repo_macro_species", {})
         repo_composition_str = json.dumps(summary.get("composition", {}))
+        _repo_summary = summary.get("summary", {})
+        repo_comp_archetype = _repo_summary.get("repo_composition_archetype") or "Unclassified"
+        repo_comp_z = float(_repo_summary.get("repo_composition_z", 0.0) or 0.0)
 
         total_files = len(parsed_files)
         total_unparsable = len(unparsable_files)
@@ -1247,7 +1263,7 @@ def record_mission(
                 1 if session_meta.get("zero_dependency_mode") else 0,
             ]
             + agg_hits
-            + [repo_composition_str]
+            + [repo_composition_str, repo_comp_archetype, repo_comp_z]
         )
 
         repo_placeholders = ",".join(["?"] * len(repo_row_data))
@@ -1262,7 +1278,7 @@ def record_mission(
                 network_edges_unrecorded,
                 audit_shadow_apis, audit_binary_anomalies, audit_unknown_packages, is_zero_dependency_mode,
                 {", ".join([self.SHORT_KEY_MAP.get(h, h) for h in self.SIGNAL_SCHEMA])},
-                file_composition
+                file_composition, repo_composition_archetype, repo_composition_z
             ) VALUES ({repo_placeholders})
         """,  # noqa: S608 -- SHORT_KEY_MAP/SIGNAL_SCHEMA are internal constants, values go through repo_placeholders/`?`
             repo_row_data,

gitgalaxy/standards/archetype_brains/file_archetype_brain.json

@@ -1347,5 +1347,67 @@
   "html"
  ],
  "min_coding_loc": 8,
- "noncode_bucket": "Data / Markup / Trivial"
+ "noncode_bucket": "Data / Markup / Trivial",
+ "z_score_params": {
+  "Annotated Framework Methods Files": {
+   "mean": 1.14047,
+   "std": 0.24309
+  },
+  "Generic / Templated Code Files": {
+   "mean": 0.99437,
+   "std": 0.28142
+  },
+  "Declarative / Non-Code": {
+   "mean": 0.73449,
+   "std": 0.28824
+  },
+  "Compute Cores Files": {
+   "mean": 0.96302,
+   "std": 0.27757
+  },
+  "Interface Declarations Files": {
+   "mean": 1.00586,
+   "std": 0.32891
+  },
+  "I/O & Config Routines Files": {
+   "mean": 0.86008,
+   "std": 0.28214
+  },
+  "Parameter Forwarders Files": {
+   "mean": 1.11351,
+   "std": 0.26412
+  },
+  "C Struct Operations Files": {
+   "mean": 0.99733,
+   "std": 0.31658
+  },
+  "Tests & Verification Files": {
+   "mean": 0.89246,
+   "std": 0.28056
+  },
+  "State Mutators Files": {
+   "mean": 1.13795,
+   "std": 0.27682
+  },
+  "Defensive Guards Files": {
+   "mean": 1.15937,
+   "std": 0.24528
+  },
+  "Large Core Modules": {
+   "mean": 1.29647,
+   "std": 0.26896
+  },
+  "Encapsulated Accessors Files": {
+   "mean": 1.19217,
+   "std": 0.21126
+  },
+  "Many-Argument Workhorses Files": {
+   "mean": 1.09469,
+   "std": 0.2469
+  },
+  "Callbacks & Closures Files": {
+   "mean": 1.04457,
+   "std": 0.29171
+  }
+ }
 }

gitgalaxy/standards/archetype_brains/repo_archetype_brain.json

@@ -482,5 +482,35 @@
    0.32529
   ]
  },
- "micro_bucket": "Micro Repo (<30 files)"
+ "micro_bucket": "Micro Repo (<30 files)",
+ "z_score_params": {
+  "Guard/Validation-Heavy": {
+   "mean": 0.67488,
+   "std": 0.20802
+  },
+  "Small Flat Repo": {
+   "mean": 0.61657,
+   "std": 0.1712
+  },
+  "Hub-Coupled Monorepo": {
+   "mean": 0.46715,
+   "std": 0.15065
+  },
+  "Typed Library": {
+   "mean": 0.49024,
+   "std": 0.13481
+  },
+  "Hub-Coupled App": {
+   "mean": 0.4981,
+   "std": 0.14391
+  },
+  "Mainframe / COBOL & Config": {
+   "mean": 0.4951,
+   "std": 0.15935
+  },
+  "Flat Modular Platform": {
+   "mean": 0.55504,
+   "std": 0.14283
+  }
+ }
 }

tests/dead_key_audit.py

@@ -109,6 +109,9 @@
     "log_file_count": "repo-archetype-brain aux-quantile key (JSON-written, archetype_classifier.py)",
     "log_total_loc": "repo-archetype-brain aux-quantile key (JSON-written, archetype_classifier.py)",
     "pagerank_gini": "repo-archetype-brain aux-quantile key (JSON-written, archetype_classifier.py)",
+    "z_score_params": "archetype-brain key: per-cluster distance stats (JSON-written, archetype_classifier.py)",
+    "mean": "archetype-brain z_score_params sub-key (JSON-written, archetype_classifier.py._fit_z)",
+    "std": "archetype-brain z_score_params sub-key (JSON-written, archetype_classifier.py._fit_z)",
     # --- External YAML/env config ---
     "galaxyscope": "top-level section name in a user's .galaxyscope.yml project config file",
     "GITGALAXY_LICENSE_KEY": "environment variable (os.environ.get), not a repo-produced dict",