Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion src/scoring/model.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,15 @@ export async function refreshScoringModelSnapshot(env: Env): Promise<ScoringMode
const parsed = parsePythonNumberConstants(constantsResult.value);
constants = { ...constants, ...parsed };
activeModelConstants = parsed;
constantsPayload = { parsedConstantCount: Object.keys(parsed).length, sourceBytes: constantsResult.value.length };
const unmodeled = findUnmodeledUpstreamConstants(constantsResult.value);
constantsPayload = { parsedConstantCount: Object.keys(parsed).length, sourceBytes: constantsResult.value.length, unmodeledUpstreamConstants: unmodeled };
warnings.push(...activeModelWarnings(parsed));
// Make staleness visible: upstream defines scoring constants gittensory does not yet model.
if (unmodeled.length > 0) {
warnings.push(
`Upstream gittensor defines ${unmodeled.length} scoring constant(s) gittensory does not yet model: ${unmodeled.slice(0, 12).join(", ")}${unmodeled.length > 12 ? ", …" : ""}. Scoring may be behind upstream.`,
);
}
} else {
sourceKind = "fallback";
warnings.push(`Scoring constants fetch failed: ${constantsResult.error}`);
Expand Down Expand Up @@ -104,6 +111,20 @@ export function parsePythonNumberConstants(source: string, options: { knownOnly?
return constants;
}

/**
* Numeric constant names upstream gittensor defines that gittensory's scoring engine does NOT model.
* The normal parse is `knownOnly` (it keeps only constants we already encode), which silently hides
* upstream ADDITIONS — e.g. a newly-introduced time-decay constant. Surfacing these makes scoring
* staleness visible: if upstream adds a scoring dimension, an operator sees it instead of the gate
* silently drifting behind. Detection only — it does not change any score.
*/
export function findUnmodeledUpstreamConstants(source: string): string[] {
const all = parsePythonNumberConstants(source, { knownOnly: false });
return Object.keys(all)
.filter((name) => !SCORING_CONSTANT_NAMES.has(name))
.sort();
}

export function detectActiveModel(constants: Record<string, number>): ScoringModelSnapshotRecord["activeModel"] {
if (hasSaturationConstants(constants)) return "pending_saturation_model";
if (hasDensityConstants(constants)) {
Expand Down
26 changes: 25 additions & 1 deletion test/unit/scoring.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { getLatestScoringModelSnapshot } from "../../src/db/repositories";
import { detectActiveModel, parsePythonNumberConstants, refreshScoringModelSnapshot } from "../../src/scoring/model";
import { detectActiveModel, findUnmodeledUpstreamConstants, parsePythonNumberConstants, refreshScoringModelSnapshot } from "../../src/scoring/model";
import { buildScorePreview, makeScorePreviewRecord } from "../../src/scoring/preview";
import type { ScorePreviewInput } from "../../src/scoring/preview";
import type { RepositoryRecord, ScoringModelSnapshotRecord } from "../../src/types";
Expand Down Expand Up @@ -118,6 +118,30 @@ MAX_CODE_DENSITY_MULTIPLIER = 1.15
expect(refreshed.warnings.join(" ")).toMatch(/recognized active-model indicator/i);
});

it("flags upstream scoring constants gittensory does not model (staleness visibility)", () => {
// SRC_TOK_SATURATION_SCALE is modeled; the TIME_DECAY_* constants are NOT — so they surface as unmodeled.
const unmodeled = findUnmodeledUpstreamConstants(
"SRC_TOK_SATURATION_SCALE = 58.0\nTIME_DECAY_GRACE_PERIOD_HOURS = 12\nTIME_DECAY_SIGMOID_MIDPOINT = 10\n",
);
expect(unmodeled).toEqual(["TIME_DECAY_GRACE_PERIOD_HOURS", "TIME_DECAY_SIGMOID_MIDPOINT"]);
expect(unmodeled).not.toContain("SRC_TOK_SATURATION_SCALE");
});

it("warns on the snapshot when upstream defines an unmodeled scoring dimension", async () => {
const env = createTestEnv();
vi.stubGlobal("fetch", async (input: RequestInfo | URL) => {
const url = input.toString();
if (url.includes("constants.py")) return new Response("SRC_TOK_SATURATION_SCALE = 58.0\nTIME_DECAY_GRACE_PERIOD_HOURS = 12\n");
if (url.includes("programming_languages.json")) return Response.json({ TypeScript: 1 });
return new Response("not found", { status: 404 });
});

const refreshed = await refreshScoringModelSnapshot(env);

expect(refreshed.warnings.join(" ")).toMatch(/does not yet model.*TIME_DECAY_GRACE_PERIOD_HOURS/);
expect(refreshed.payload.constants).toMatchObject({ unmodeledUpstreamConstants: ["TIME_DECAY_GRACE_PERIOD_HOURS"] });
});

it("uses saturation math as the active private preview model", () => {
const saturationSnapshot: ScoringModelSnapshotRecord = {
...snapshot,
Expand Down