Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion src/api/routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -320,7 +320,7 @@ import { isRagEnabled } from "../review/rag-wire";
import { getPublicStats, isPublicStatsEnabled, resolvePublicStatsManifestOverride } from "../review/public-stats";
import { loadPublicAccuracyTrend } from "../services/public-accuracy-trend";
import { loadCalibrationTrend } from "../services/rule-calibration-trend";
import { isSatisfactionFloorAutotuneEnabled, runSatisfactionFloorLoosening } from "../services/satisfaction-floor-loosening-run";
import { isSatisfactionFloorAutotuneEnabled, loadSatisfactionFloorStatus, runSatisfactionFloorLoosening } from "../services/satisfaction-floor-loosening-run";
import { loadPublicReuseRateTrend } from "../services/public-reuse-rate-trend";
import { loadPublicReviewVolumeTrend } from "../services/public-review-volume-trend";
import { buildMaintainerQualityDashboard, isMaintainerQualityDataStale } from "../services/maintainer-quality-dashboard";
Expand Down Expand Up @@ -4826,6 +4826,12 @@ export function createApp() {
return c.json(result);
});

// Operator visibility for the loosening loop (#8161): flag state, shipped vs live floor, the stored
// override row, and the applied-loosening history with both split verdicts. Deliberately NOT flag-gated
// (unlike the trigger above): an operator must be able to see a lingering override row while the flag is
// off. Same INTERNAL_JOB_TOKEN gate via the /v1/internal/* middleware; aggregate numbers/verdicts only.
app.get("/v1/internal/calibration/satisfaction-floor", async (c) => c.json(await loadSatisfactionFloorStatus(c.env)));

app.post("/v1/internal/jobs/refresh-registry", async (c) => {
const message: JobMessage = { type: "refresh-registry", requestedBy: "api" };
await c.env.JOBS.send(message);
Expand Down
101 changes: 101 additions & 0 deletions src/services/satisfaction-floor-loosening-run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -172,3 +172,104 @@ export async function loadSatisfactionFloorRecState(env: Env, nowMs: number = Da

return { flagEnabled, proposal, lastAppliedAt };
}

// ── Operator visibility (#8161) ────────────────────────────────────────────────────────────────────────

export type SatisfactionFloorAppliedEntry = {
at: string;
currentFloor: number | null;
proposedFloor: number | null;
visibleCases: number | null;
heldOutCases: number | null;
visibleVerdict: string | null;
heldOutVerdict: string | null;
};

export type SatisfactionFloorStatus = {
flagEnabled: boolean;
shippedFloor: number;
/** The floor the live assessment actually uses right now: the validated override when the flag is on,
* else the shipped constant. */
liveFloor: number;
/** The RAW stored override row (validated), reported even when the flag is off — an operator looking at
* this surface needs to see a lingering row that would take effect the moment the flag flips. Null when
* no row exists or the stored value fails the loosening-only bounds. */
storedOverride: number | null;
applied: SatisfactionFloorAppliedEntry[];
};

const SATISFACTION_FLOOR_STATUS_HISTORY_LIMIT = 25;

function numberOrNull(value: unknown): number | null {
return typeof value === "number" && Number.isFinite(value) ? value : null;
}

function verdictOrNull(value: unknown): string | null {
const verdict = (value as { verdict?: unknown } | undefined)?.verdict;
return typeof verdict === "string" ? verdict : null;
}

/**
* The operator status read (#8161): flag state, shipped vs live floor, the stored override row (validated,
* shown regardless of flag state — see the type's own doc), and the applied-loosening history projected
* from the calibration.satisfaction_floor_loosened audit events (#8121's evidence trail), newest first.
* Aggregate numbers and verdicts only — no corpus content of any kind. Fail-safe: a read error degrades the
* affected section (empty history / null override) rather than throwing the operator endpoint.
*/
export async function loadSatisfactionFloorStatus(env: Env): Promise<SatisfactionFloorStatus> {
const flagEnabled = isSatisfactionFloorAutotuneEnabled(env);

let storedOverride: number | null = null;
try {
const row = await env.DB.prepare("SELECT value FROM system_flags WHERE key = ?")
.bind(SATISFACTION_FLOOR_OVERRIDE_FLAG_KEY)
.first<{ value: string }>();
if (row) {
const parsed = Number(row.value);
if (Number.isFinite(parsed) && parsed < LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR && parsed >= SATISFACTION_FLOOR_HARD_MINIMUM) {
storedOverride = parsed;
}
}
} catch {
storedOverride = null;
}

const applied: SatisfactionFloorAppliedEntry[] = [];
try {
const rows = await env.DB.prepare(
"SELECT created_at, metadata_json FROM audit_events WHERE event_type = ? ORDER BY created_at DESC LIMIT ?",
)
.bind(SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE, SATISFACTION_FLOOR_STATUS_HISTORY_LIMIT)
.all<{ created_at: string; metadata_json: string }>();
/* v8 ignore next -- .all() over a live D1/TestD1 always yields a defined results array, never undefined;
* the ?? [] guards a future driver-shape change, mirroring loadOrbDayRows' identical note. */
for (const row of rows.results ?? []) {
let proposal: Record<string, unknown> = {};
try {
const metadata = JSON.parse(row.metadata_json) as { proposal?: Record<string, unknown> };
proposal = metadata.proposal && typeof metadata.proposal === "object" ? metadata.proposal : {};
} catch {
/* corrupt row -- keep the entry with nulls rather than hiding that an apply happened */
}
applied.push({
at: row.created_at,
currentFloor: numberOrNull(proposal.currentFloor),
proposedFloor: numberOrNull(proposal.proposedFloor),
visibleCases: numberOrNull(proposal.visibleCases),
heldOutCases: numberOrNull(proposal.heldOutCases),
visibleVerdict: verdictOrNull(proposal.visible),
heldOutVerdict: verdictOrNull(proposal.heldOut),
});
}
} catch {
/* degrade to an empty history -- the endpoint must not throw on a read blip */
}

return {
flagEnabled,
shippedFloor: LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR,
liveFloor: flagEnabled && storedOverride !== null ? storedOverride : LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR,
storedOverride,
applied,
};
}
137 changes: 137 additions & 0 deletions test/unit/satisfaction-floor-status.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
import { describe, expect, it } from "vitest";
import {
loadSatisfactionFloorStatus,
SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE,
SATISFACTION_FLOOR_OVERRIDE_FLAG_KEY,
} from "../../src/services/satisfaction-floor-loosening-run";
import { SATISFACTION_FLOOR_HARD_MINIMUM } from "../../src/services/satisfaction-floor-loosening";
import { LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR } from "../../src/services/linked-issue-satisfaction";
import { recordAuditEvent } from "../../src/db/repositories";
import { createApp } from "../../src/api/routes";
import { createTestEnv } from "../helpers/d1";

// Operator visibility for the loosening loop (#8161): the status read + its internal route. The loop's own
// behavior lives in satisfaction-floor-loosening-run.test.ts — this file pins the reporting surface.

const enabledEnv = () => createTestEnv({ SATISFACTION_FLOOR_AUTOTUNE_ENABLED: "true" as never });

async function setOverrideRow(env: Env, value: string) {
await env.DB.prepare("INSERT INTO system_flags (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP) ON CONFLICT(key) DO UPDATE SET value = excluded.value")
.bind(SATISFACTION_FLOOR_OVERRIDE_FLAG_KEY, value)
.run();
}

describe("loadSatisfactionFloorStatus (#8161)", () => {
it("reports the shipped defaults on a fresh deployment: flag off, no override, live floor = shipped, empty history", async () => {
const status = await loadSatisfactionFloorStatus(createTestEnv());
expect(status).toEqual({
flagEnabled: false,
shippedFloor: LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR,
liveFloor: LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR,
storedOverride: null,
applied: [],
});
});

it("shows a lingering override row even while the flag is OFF, but liveFloor only follows it when ON", async () => {
const offEnv = createTestEnv();
await setOverrideRow(offEnv, "0.4");
const offStatus = await loadSatisfactionFloorStatus(offEnv);
expect(offStatus.storedOverride).toBe(0.4);
expect(offStatus.flagEnabled).toBe(false);
expect(offStatus.liveFloor).toBe(LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR); // flag off ⇒ shipped floor rules

const onEnv = enabledEnv();
await setOverrideRow(onEnv, "0.4");
const onStatus = await loadSatisfactionFloorStatus(onEnv);
expect(onStatus.liveFloor).toBe(0.4);
});

it("rejects out-of-bounds or unparseable stored values from BOTH storedOverride and liveFloor", async () => {
for (const bad of ["0.9", String(SATISFACTION_FLOOR_HARD_MINIMUM - 0.05), "junk", String(LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR)]) {
const env = enabledEnv();
await setOverrideRow(env, bad);
const status = await loadSatisfactionFloorStatus(env);
expect(status.storedOverride).toBeNull();
expect(status.liveFloor).toBe(LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR);
}
});

it("projects the applied history newest-first with both split verdicts, and a corrupt row degrades to nulls instead of vanishing", async () => {
const env = enabledEnv();
await recordAuditEvent(env, {
eventType: SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE,
actor: "loopover",
targetKey: "linked_issue_scope_mismatch",
outcome: "completed",
metadata: {
proposal: {
currentFloor: 0.5,
proposedFloor: 0.45,
visibleCases: 24,
heldOutCases: 7,
visible: { verdict: "improved" },
heldOut: { verdict: "unchanged" },
},
},
createdAt: "2026-07-20T00:00:00.000Z",
});
await env.DB.prepare(
"INSERT INTO audit_events (id, event_type, actor, target_key, outcome, detail, metadata_json, created_at) VALUES ('corrupt', ?, 'loopover', 'x', 'completed', '', 'not-json', '2026-07-21T00:00:00.000Z')",
)
.bind(SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE)
.run();

// A parseable row whose proposal is not an object degrades the same way as unparseable JSON.
await env.DB.prepare(
"INSERT INTO audit_events (id, event_type, actor, target_key, outcome, detail, metadata_json, created_at) VALUES ('nonobject', ?, 'loopover', 'x', 'completed', '', '{\"proposal\": 5}', '2026-07-22T00:00:00.000Z')",
)
.bind(SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE)
.run();

const status = await loadSatisfactionFloorStatus(env);
expect(status.applied).toHaveLength(3);
expect(status.applied[1]!.proposedFloor).toBeNull(); // the unparseable-JSON row
expect(status.applied[0]).toEqual({
at: "2026-07-22T00:00:00.000Z",
currentFloor: null,
proposedFloor: null,
visibleCases: null,
heldOutCases: null,
visibleVerdict: null,
heldOutVerdict: null,
});
expect(status.applied[2]).toEqual({
at: "2026-07-20T00:00:00.000Z",
currentFloor: 0.5,
proposedFloor: 0.45,
visibleCases: 24,
heldOutCases: 7,
visibleVerdict: "improved",
heldOutVerdict: "unchanged",
});
});

it("fails safe to defaults + empty history on a DB error", async () => {
const env = enabledEnv();
env.DB = { prepare: () => { throw new Error("boom"); } } as never;
const status = await loadSatisfactionFloorStatus(env);
expect(status.storedOverride).toBeNull();
expect(status.applied).toEqual([]);
expect(status.liveFloor).toBe(LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR);
});
});

describe("GET /v1/internal/calibration/satisfaction-floor (#8161)", () => {
it("401s without the internal token, and is NOT flag-gated: 200s with the flag off (visibility must survive a flag flip)", async () => {
const app = createApp();
const env = createTestEnv();
expect((await app.request("/v1/internal/calibration/satisfaction-floor", {}, env)).status).toBe(401);
const res = await app.request("/v1/internal/calibration/satisfaction-floor", { headers: { authorization: `Bearer ${env.INTERNAL_JOB_TOKEN}` } }, env);
expect(res.status).toBe(200);
const body = (await res.json()) as { flagEnabled: boolean; liveFloor: number; applied: unknown[] };
expect(body.flagEnabled).toBe(false);
expect(body.liveFloor).toBe(LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR);
expect(JSON.stringify(body)).not.toMatch(/reward|payout|trust|wallet|hotkey|issueText|modelResponse/i);
});
});