From c7ae9b62d7af22483040d32701de68a880112b17 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 14 Jun 2026 01:48:12 -0700 Subject: [PATCH] feat(scoring): surface upstream scoring constants gittensory does not model (staleness visibility) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes scoring drift VISIBLE — the safe half of the scoring-currency work (the ranking-changing time-decay implementation is deferred for owner review with a before/after diff, per decision). The upstream constants parse is `knownOnly` (it keeps only constants we already encode), which silently HID upstream ADDITIONS — so when gittensor introduced new scoring dimensions (e.g. the TIME_DECAY_* constants) gittensory drifted behind with no signal. New findUnmodeledUpstreamConstants() detects constants upstream defines that we don't model; refreshScoringModelSnapshot now records them on the snapshot payload (unmodeledUpstreamConstants) AND raises a warning ("Upstream gittensor defines N scoring constant(s) gittensory does not yet model: …. Scoring may be behind upstream."), which surfaces on /v1/scoring/model, /v1/upstream/status, and the operator dashboard. Detection only — changes no score. (The live upstream-contract test already runs on a schedule via the existing upstream-contract.yml workflow, so that half was already covered.) Tests: findUnmodeledUpstreamConstants flags TIME_DECAY_* (not modeled) and not SRC_TOK_SATURATION_SCALE (modeled); refresh raises the staleness warning + records the payload field. 97% branch. Part of #525. --- src/scoring/model.ts | 23 ++++++++++++++++++++++- test/unit/scoring.test.ts | 26 +++++++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/scoring/model.ts b/src/scoring/model.ts index 87ef55c83c..5db3c7b238 100644 --- a/src/scoring/model.ts +++ b/src/scoring/model.ts @@ -57,8 +57,15 @@ export async function refreshScoringModelSnapshot(env: Env): Promise 0) { + warnings.push( + `Upstream gittensor defines ${unmodeled.length} scoring constant(s) gittensory does not yet model: ${unmodeled.slice(0, 12).join(", ")}${unmodeled.length > 12 ? ", …" : ""}. Scoring may be behind upstream.`, + ); + } } else { sourceKind = "fallback"; warnings.push(`Scoring constants fetch failed: ${constantsResult.error}`); @@ -104,6 +111,20 @@ export function parsePythonNumberConstants(source: string, options: { knownOnly? return constants; } +/** + * Numeric constant names upstream gittensor defines that gittensory's scoring engine does NOT model. + * The normal parse is `knownOnly` (it keeps only constants we already encode), which silently hides + * upstream ADDITIONS — e.g. a newly-introduced time-decay constant. Surfacing these makes scoring + * staleness visible: if upstream adds a scoring dimension, an operator sees it instead of the gate + * silently drifting behind. Detection only — it does not change any score. + */ +export function findUnmodeledUpstreamConstants(source: string): string[] { + const all = parsePythonNumberConstants(source, { knownOnly: false }); + return Object.keys(all) + .filter((name) => !SCORING_CONSTANT_NAMES.has(name)) + .sort(); +} + export function detectActiveModel(constants: Record): ScoringModelSnapshotRecord["activeModel"] { if (hasSaturationConstants(constants)) return "pending_saturation_model"; if (hasDensityConstants(constants)) { diff --git a/test/unit/scoring.test.ts b/test/unit/scoring.test.ts index 47bd0a924a..914450165e 100644 --- a/test/unit/scoring.test.ts +++ b/test/unit/scoring.test.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { getLatestScoringModelSnapshot } from "../../src/db/repositories"; -import { detectActiveModel, parsePythonNumberConstants, refreshScoringModelSnapshot } from "../../src/scoring/model"; +import { detectActiveModel, findUnmodeledUpstreamConstants, parsePythonNumberConstants, refreshScoringModelSnapshot } from "../../src/scoring/model"; import { buildScorePreview, makeScorePreviewRecord } from "../../src/scoring/preview"; import type { ScorePreviewInput } from "../../src/scoring/preview"; import type { RepositoryRecord, ScoringModelSnapshotRecord } from "../../src/types"; @@ -118,6 +118,30 @@ MAX_CODE_DENSITY_MULTIPLIER = 1.15 expect(refreshed.warnings.join(" ")).toMatch(/recognized active-model indicator/i); }); + it("flags upstream scoring constants gittensory does not model (staleness visibility)", () => { + // SRC_TOK_SATURATION_SCALE is modeled; the TIME_DECAY_* constants are NOT — so they surface as unmodeled. + const unmodeled = findUnmodeledUpstreamConstants( + "SRC_TOK_SATURATION_SCALE = 58.0\nTIME_DECAY_GRACE_PERIOD_HOURS = 12\nTIME_DECAY_SIGMOID_MIDPOINT = 10\n", + ); + expect(unmodeled).toEqual(["TIME_DECAY_GRACE_PERIOD_HOURS", "TIME_DECAY_SIGMOID_MIDPOINT"]); + expect(unmodeled).not.toContain("SRC_TOK_SATURATION_SCALE"); + }); + + it("warns on the snapshot when upstream defines an unmodeled scoring dimension", async () => { + const env = createTestEnv(); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("constants.py")) return new Response("SRC_TOK_SATURATION_SCALE = 58.0\nTIME_DECAY_GRACE_PERIOD_HOURS = 12\n"); + if (url.includes("programming_languages.json")) return Response.json({ TypeScript: 1 }); + return new Response("not found", { status: 404 }); + }); + + const refreshed = await refreshScoringModelSnapshot(env); + + expect(refreshed.warnings.join(" ")).toMatch(/does not yet model.*TIME_DECAY_GRACE_PERIOD_HOURS/); + expect(refreshed.payload.constants).toMatchObject({ unmodeledUpstreamConstants: ["TIME_DECAY_GRACE_PERIOD_HOURS"] }); + }); + it("uses saturation math as the active private preview model", () => { const saturationSnapshot: ScoringModelSnapshotRecord = { ...snapshot,