diff --git a/apps/gittensory-ui/src/lib/selfhost-env-reference.ts b/apps/gittensory-ui/src/lib/selfhost-env-reference.ts index f4199e26cd..179ae5dce9 100644 --- a/apps/gittensory-ui/src/lib/selfhost-env-reference.ts +++ b/apps/gittensory-ui/src/lib/selfhost-env-reference.ts @@ -65,6 +65,18 @@ export const SELFHOST_ENV_REFERENCE_ROWS: SelfHostEnvReferenceRow[] = [ name: "CLAUDE_AI_TIMEOUT_MS", firstReference: "src/selfhost/ai.ts", }, + { + name: "CLOUDFLARE_D1_MONITOR_ACCOUNT_ID", + firstReference: "src/selfhost/d1-size-probe.ts", + }, + { + name: "CLOUDFLARE_D1_MONITOR_API_TOKEN", + firstReference: "src/selfhost/d1-size-probe.ts", + }, + { + name: "CLOUDFLARE_D1_MONITOR_DATABASE_ID", + firstReference: "src/selfhost/d1-size-probe.ts", + }, { name: "CODEX_AI_EFFORT", firstReference: "src/selfhost/ai.ts", @@ -409,6 +421,9 @@ export const SELFHOST_ENV_REFERENCE_MARKDOWN = [ "| `CLAUDE_AI_EFFORT` | `src/selfhost/ai.ts` |", "| `CLAUDE_AI_MODEL` | `src/selfhost/ai.ts` |", "| `CLAUDE_AI_TIMEOUT_MS` | `src/selfhost/ai.ts` |", + "| `CLOUDFLARE_D1_MONITOR_ACCOUNT_ID` | `src/selfhost/d1-size-probe.ts` |", + "| `CLOUDFLARE_D1_MONITOR_API_TOKEN` | `src/selfhost/d1-size-probe.ts` |", + "| `CLOUDFLARE_D1_MONITOR_DATABASE_ID` | `src/selfhost/d1-size-probe.ts` |", "| `CODEX_AI_EFFORT` | `src/selfhost/ai.ts` |", "| `CODEX_AI_FIRST_OUTPUT_TIMEOUT_MS` | `src/selfhost/ai.ts` |", "| `CODEX_AI_MODEL` | `src/selfhost/ai.ts` |", diff --git a/grafana/dashboards/gittensory.json b/grafana/dashboards/gittensory.json index 4e0a82713f..807f276896 100644 --- a/grafana/dashboards/gittensory.json +++ b/grafana/dashboards/gittensory.json @@ -3182,6 +3182,159 @@ "refId": "A" } ] + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 222 }, + "id": 200, + "title": "Cloudflare D1 (Central Cloud, #3810)", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 7000000000 }, + { "color": "red", "value": 9000000000 } + ] + }, + "unit": "decbytes" + } + }, + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 223 }, + "id": 201, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "D1 Database Size (-1 = probe disabled/no sample yet)", + "description": "Cloudflare D1 file size for the monitored database, from the opt-in Management API probe (src/selfhost/d1-size-probe.ts, CLOUDFLARE_D1_MONITOR_* env vars). -1 means the probe is disabled or has never completed a successful sample. D1's known per-database cap is ~10GB (#3810).", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_d1_database_size_bytes", + "legendFormat": "size", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 10 }, + { "color": "red", "value": 50 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 223 }, + "id": 202, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "signal_snapshots Rows per Dedup Key (-1 = unavailable)", + "description": "signal_snapshots row count divided by its distinct (signal_type, target_key) count, scoped to the latest-only-dedup signal types dedupeSignalSnapshots (src/db/retention.ts) converges to ~1 row per key. A climbing value means the daily dedup job has stopped running or its allowlist regressed -- the 2026-07-06 incident ratio was ~157 (342243 rows / 2183 keys).", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_signal_snapshots_rows_per_key", + "legendFormat": "rows/key", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "unit": "short" + } + }, + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 223 }, + "id": 203, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "title": "D1 Probe Errors (total)", + "description": "Cloudflare Management API probe failures (src/selfhost/d1-size-probe.ts), labeled by part (database_info/table_row_count). Climbing while the size/row-count panels stay flat means the PROBE is broken (bad/expired token, wrong account or database id), not that the database stopped growing.", + "type": "stat", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "sum(gittensory_d1_probe_errors_total) or vector(0)", + "legendFormat": "errors", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "lineWidth": 2, "fillOpacity": 10 }, + "unit": "short" + } + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 227 }, + "id": 204, + "options": { + "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "D1 Database Size & Monitored Table Row Counts", + "description": "Trend view of the same D1 size/row-count probe as the stat panels above -- database bytes on the left axis scale, monitored-table row counts (RETENTION_POLICY's tables, src/db/retention.ts) labeled by table.", + "type": "timeseries", + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_d1_database_size_bytes", + "legendFormat": "database bytes", + "refId": "A" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "gittensory_d1_table_row_count", + "legendFormat": "{{table}} rows", + "refId": "B" + } + ] } ], "refresh": "30s", @@ -3206,5 +3359,5 @@ "timezone": "browser", "title": "Gittensory Self-Host", "uid": "gittensory-selfhost", - "version": 9 + "version": 10 } diff --git a/prometheus/rules/alerts.yml b/prometheus/rules/alerts.yml index 5f3d732445..b5b6f5f447 100644 --- a/prometheus/rules/alerts.yml +++ b/prometheus/rules/alerts.yml @@ -552,3 +552,66 @@ groups: summary: "gittensory host clock skew is CRITICAL ({{ $value | printf \"%.0f\" }}s) -- GitHub App auth is likely failing" description: "Clock skew has exceeded 120s (sustained 2m), well past the point GitHub App JWT auth (\"Bad credentials\") is expected to start failing fleet-wide." runbook: "Same as GittensoryClockSkewWarning, but treat as urgent: fix NTP sync immediately (chronyc sources, chronyc makestep, restart chrony if every source stays at Reach: 0). Check for github_app_jwt_rejected logs to confirm auth impact." + + # ── Cloudflare D1 (central cloud) size + signal_snapshots dedup regression (#3810) ──── + # gittensory_d1_* metrics come from the OPT-IN Cloudflare Management API probe (src/selfhost/ + # d1-size-probe.ts, CLOUDFLARE_D1_MONITOR_* env vars) -- absent/disabled reads -1 on every gauge below, + # comfortably under every threshold here, so these rules never fire on an install that hasn't configured + # the probe (most self-host installs run their own SQLite/Postgres backend and have nothing to monitor). + - name: gittensory-d1-storage + rules: + - alert: GittensoryD1DatabaseSizeWarning + # 7e9 bytes is ~70% of D1's known ~10GB per-database cap that was hit on 2026-07-06 (#3810), which + # caused real D1 writes (including an Orb relay registration handshake) to start failing/timing out. + expr: gittensory_d1_database_size_bytes > 7000000000 + for: 30m + labels: + severity: warning + annotations: + summary: "Cloudflare D1 database size is approaching its cap" + description: "The monitored D1 database is {{ $value | humanize }}B, over 70% of the ~10GB per-database cap (sustained 30m)." + runbook: "Check the D1 panel's per-table row counts (Cloudflare D1 (Central Cloud) row) and confirm the daily signal_snapshots dedup job (dedupeSignalSnapshots, wired into prune-retention) is actually running. See #3810 for the 2026-07-06 incident this guards against." + + - alert: GittensoryD1DatabaseSizeCritical + # 9e9 bytes is ~90% of the ~10GB cap -- D1 writes started failing fleet-wide the last time the + # database actually hit the cap, so this is meant to fire well before that happens again. + expr: gittensory_d1_database_size_bytes > 9000000000 + for: 10m + labels: + severity: critical + annotations: + summary: "Cloudflare D1 database size is CRITICAL -- writes may start failing soon" + description: "The monitored D1 database is {{ $value | humanize }}B, over 90% of the ~10GB per-database cap (sustained 10m). D1 writes failed fleet-wide the last time this cap was hit (2026-07-06)." + runbook: "Immediate: verify the signal_snapshots dedup job is running (POST /v1/internal/retention/preview, or check audit_events for its record) and identify + trim/archive any other unbounded table from the row-count panel. Contact Cloudflare to raise the account storage limit if cleanup alone doesn't recover enough headroom." + + - alert: GittensorySignalSnapshotsDedupRegression + # gittensory_signal_snapshots_rows_per_key is rows-per-distinct-key scoped ONLY to the four + # latest-only-dedup signal types dedupeSignalSnapshots (src/db/retention.ts) actually converges to + # ~1 row per key -- NOT the whole signal_snapshots table, which intentionally keeps bounded + # multi-row history for other signal types (queue-health, contributor-decision-pack, ...). Healthy + # steady-state stays a small multiple of 1 (rows can accumulate for up to a day between the daily + # dedup run); 10 is a wide margin above that, while the actual 2026-07-06 incident ratio (342243 + # rows / 2183 keys) was ~157 -- so a value anywhere near double digits means the dedup job has + # stopped running, started erroring, or its allowlist regressed. + expr: gittensory_signal_snapshots_rows_per_key > 10 + for: 30m + labels: + severity: warning + annotations: + summary: "signal_snapshots dedup-by-key ratio is climbing" + description: "signal_snapshots has {{ $value | printf \"%.1f\" }} rows per distinct dedup key (sustained 30m) -- the daily dedupeSignalSnapshots job may not be running." + runbook: "Confirm the prune-retention cron (03:00 UTC daily) is completing (audit_events around that time) and that dedupeSignalSnapshots isn't throwing. See #3810 and src/db/retention.ts." + + - alert: GittensoryD1ProbeFailing + # The probe itself (a Cloudflare Management API call) can fail independently of the database it + # monitors -- an expired/rotated API token, a wrong account/database id, or a Cloudflare API outage. + # Without this, an operator would see a flat/stale D1 panel and could mistake "the probe broke" for + # "the database stopped growing". + expr: increase(gittensory_d1_probe_errors_total[1h]) > 3 + for: 15m + labels: + severity: warning + annotations: + summary: "the Cloudflare D1 size/row-count probe is failing" + description: "{{ $value | printf \"%.0f\" }} D1 Management API probe failure(s) over the last 1h (sustained 15m, part={{ $labels.part }}). The size/row-count gauges below may be stale." + runbook: "Check CLOUDFLARE_D1_MONITOR_API_TOKEN is still valid and CLOUDFLARE_D1_MONITOR_ACCOUNT_ID/DATABASE_ID are correct. Tail logs for level=error event=d1_size_probe_error." diff --git a/src/db/retention.ts b/src/db/retention.ts index ea904d9dc0..399b28102b 100644 --- a/src/db/retention.ts +++ b/src/db/retention.ts @@ -94,7 +94,12 @@ export async function pruneExpiredRecords( export type SignalSnapshotDedupeResult = { signalType: string; deleted: number }; -const LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES = [ +/** Exported so the D1 size/row-count observability probe (#3810, src/selfhost/d1-size-probe.ts) can scope its + * signal_snapshots "rows per dedup key" ratio to exactly the population this dedup job converges to ~1 row + * per key -- NOT the whole table, which intentionally keeps bounded multi-row history for other signal + * types (queue-health, contributor-decision-pack, ...). Single source of truth: if this list changes, the + * probe's ratio scope changes with it automatically. */ +export const LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES = [ "repo-culture-profile", "repo-doc-refresh-attempt", "repo-focus-manifest", diff --git a/src/env.d.ts b/src/env.d.ts index 2b9c07fcbc..38c67fbfed 100644 --- a/src/env.d.ts +++ b/src/env.d.ts @@ -379,6 +379,19 @@ declare global { * flagged (see the same-author guard in buildCollisionReport). Default OFF — unset/false leaves every * PullRequestRecord's changedFiles unset, byte-identical to today. See src/signals/engine.ts prItem. */ GITTENSORY_OPEN_PR_FILE_COLLISION?: string; + /** D1 size/row-count observability probe (#3810): the Cloudflare account id that owns the D1 database to + * monitor. Presence of this AND the two vars below IS the enablement switch (see isD1SizeProbeEnabled, + * src/selfhost/d1-size-probe.ts) -- unset/blank ⇒ the probe never runs, byte-identical to today. Most + * self-host operators run their own SQLite/Postgres backend and have no Cloudflare D1 to watch; this is + * for whichever deployment owns a real D1 worth monitoring (including gittensory's own central cloud + * database, the one that hit its ~10GB cap on 2026-07-06). */ + CLOUDFLARE_D1_MONITOR_ACCOUNT_ID?: string; + /** The D1 database id (uuid) to monitor. See CLOUDFLARE_D1_MONITOR_ACCOUNT_ID. */ + CLOUDFLARE_D1_MONITOR_DATABASE_ID?: string; + /** A Cloudflare API token with read access to D1 for the account above (a scoped, read-only custom + * token is sufficient — this probe never writes). A secret — never commit a real value. See + * CLOUDFLARE_D1_MONITOR_ACCOUNT_ID. */ + CLOUDFLARE_D1_MONITOR_API_TOKEN?: string; } } diff --git a/src/selfhost/d1-size-probe.ts b/src/selfhost/d1-size-probe.ts new file mode 100644 index 0000000000..46c244078f --- /dev/null +++ b/src/selfhost/d1-size-probe.ts @@ -0,0 +1,248 @@ +// Cloudflare D1 size + row-count observability probe (central-cloud storage, #3810). The Cloudflare D1 +// database backing gittensory's shared cloud gittensory-api/Orb deployment hit its ~10GB account storage +// cap on 2026-07-06 -- see src/db/retention.ts's dedupeSignalSnapshots for the write-side root-cause fix +// (signal_snapshots was accumulating hundreds of superseded rows per key). D1's own query surface has no way +// to report the database's FILE size as a metric from inside a query -- that figure only exists via the +// Cloudflare Management API (`GET .../d1/database/{id}` -> `file_size`), and per-table row counts need an +// actual `COUNT(*)` run through that same account-scoped HTTP API rather than a local binding: self-host +// runs its own SQLite/Postgres backend (see d1-adapter.ts / pg-adapter.ts), so there is no `env.DB` binding +// anywhere that actually points at the central cloud database this module is built to watch. +// +// OPT-IN, CREDENTIAL-GATED, NEW INTEGRATION: grepping this repo before writing this file found ZERO existing +// Cloudflare Management API usage anywhere (no wrangler binding covers it), so every call here is a plain +// authenticated HTTPS request -- the same shape as this repo's existing external JSON-API clients (see +// src/gittensor/api.ts's fetchJson: hard fetch timeout, throw on non-OK). Presence of all three +// CLOUDFLARE_D1_MONITOR_* env vars IS the enablement switch, the same convention as isOrbBrokerMode's +// ORB_ENROLLMENT_SECRET-presence check (src/orb/broker-client.ts) -- most self-host operators run their own +// SQLite/Postgres backend and have nothing to monitor here; this exists for whichever deployment owns a real +// Cloudflare D1 worth watching (including gittensory's own central cloud database). Wired into the self-host +// process's OWN boot-time interval registrations in server.ts (mirroring the Orb relay registration retry +// timer), NOT the Cloudflare Worker `scheduled()` cron: that cron's job registry is shared with the hosted +// cloud Worker's ephemeral, multi-isolate request lifecycle, which cannot reliably carry an in-memory sample +// from a scheduled tick through to a later /metrics scrape the way one long-running self-host process can +// (self-host's /metrics is served from the SAME process that runs this timer -- see GET /metrics in +// server.ts). + +import { LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES, RETENTION_POLICY } from "../db/retention"; +import { errorMessage } from "../utils/json"; +import { incr } from "./metrics"; +import type { VectorSample } from "./metrics"; + +export interface D1SizeProbeEnv { + CLOUDFLARE_D1_MONITOR_ACCOUNT_ID?: string | undefined; + CLOUDFLARE_D1_MONITOR_DATABASE_ID?: string | undefined; + CLOUDFLARE_D1_MONITOR_API_TOKEN?: string | undefined; +} + +// The same high-volume, unbounded-growth-risk tables RETENTION_POLICY already age-prunes (src/db/ +// retention.ts) -- reused directly rather than re-listed so the two never drift apart. +const DEFAULT_MONITORED_TABLES: readonly string[] = RETENTION_POLICY.map((rule) => rule.table); + +export interface D1SizeProbeConfig { + accountId: string; + databaseId: string; + apiToken: string; + /** Tables to report a row count for. Always {@link DEFAULT_MONITORED_TABLES} in production; overridable + * only so tests can exercise the fan-out without mocking a fetch per real monitored table. */ + tables: readonly string[]; +} + +/** Every identifier passed here comes only from the hardcoded {@link DEFAULT_MONITORED_TABLES} (never user + * input); validated defensively anyway, mirroring retention.ts's own SAFE_IDENTIFIER check. */ +const SAFE_TABLE_NAME = /^[a-z_]+$/; + +/** signal_snapshots' own dedup key, mirroring dedupeSignalSnapshots' partition (src/db/retention.ts). */ +const SIGNAL_SNAPSHOTS_DEDUP_KEY_SQL = "signal_type || ':' || target_key"; + +const CLOUDFLARE_API_BASE = "https://api.cloudflare.com/client/v4"; +/** Hard cap on a single Cloudflare Management API request, mirroring src/gittensor/api.ts's + * GITTENSOR_FETCH_TIMEOUT_MS -- a slow/half-open Cloudflare API call must never hang the self-host + * process's probe timer indefinitely. */ +const D1_PROBE_FETCH_TIMEOUT_MS = 10_000; + +/** Reads all three CLOUDFLARE_D1_MONITOR_* vars; returns null (probe disabled) unless every one is a + * non-empty string. Config presence IS the enablement switch -- there is no separate boolean flag. */ +export function resolveD1SizeProbeConfig(env: D1SizeProbeEnv): D1SizeProbeConfig | null { + const accountId = env.CLOUDFLARE_D1_MONITOR_ACCOUNT_ID; + const databaseId = env.CLOUDFLARE_D1_MONITOR_DATABASE_ID; + const apiToken = env.CLOUDFLARE_D1_MONITOR_API_TOKEN; + if (!accountId || !databaseId || !apiToken) return null; + return { accountId, databaseId, apiToken, tables: DEFAULT_MONITORED_TABLES }; +} + +export function isD1SizeProbeEnabled(env: D1SizeProbeEnv): boolean { + return resolveD1SizeProbeConfig(env) !== null; +} + +interface CloudflareApiError { + code: number; + message: string; +} +interface CloudflareApiEnvelope { + success: boolean; + errors?: CloudflareApiError[]; + result: T; +} + +async function cloudflareApiRequest(config: D1SizeProbeConfig, path: string, init: RequestInit | undefined, fetchImpl: typeof fetch): Promise { + const response = await fetchImpl(`${CLOUDFLARE_API_BASE}/accounts/${config.accountId}${path}`, { + ...init, + headers: { + authorization: `Bearer ${config.apiToken}`, + "content-type": "application/json", + ...(init?.headers ?? {}), + }, + signal: AbortSignal.timeout(D1_PROBE_FETCH_TIMEOUT_MS), + }); + let body: CloudflareApiEnvelope | null = null; + try { + body = (await response.json()) as CloudflareApiEnvelope; + } catch { + body = null; + } + if (!response.ok || !body?.success) { + const message = body?.errors?.map((e) => `${e.code}: ${e.message}`).join("; ") || `HTTP ${response.status}`; + throw new Error(`Cloudflare D1 API request failed (${path}): ${message}`); + } + return body.result; +} + +export interface D1DatabaseInfo { + fileSizeBytes: number; + numTables: number; +} + +/** `GET /accounts/{account}/d1/database/{database}` -- the only place D1's own file size is exposed. */ +export async function fetchD1DatabaseInfo(config: D1SizeProbeConfig, fetchImpl: typeof fetch = fetch): Promise { + const result = await cloudflareApiRequest<{ file_size?: number; num_tables?: number }>(config, `/d1/database/${config.databaseId}`, { method: "GET" }, fetchImpl); + return { fileSizeBytes: Number(result.file_size ?? 0), numTables: Number(result.num_tables ?? 0) }; +} + +export interface D1TableRowCount { + table: string; + rowCount: number; + /** Present ONLY for "signal_snapshots" -- the row count and distinct (signal_type, target_key) count + * scoped to JUST {@link LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES} (dedupeSignalSnapshots' own partition). + * Absent for every other monitored table, which has no per-key dedup invariant to measure. Modeled as one + * optional nested object (not two independently-nullable fields) because the two numbers are only ever + * meaningful, or only ever absent, TOGETHER -- there is no state where one exists without the other. */ + dedup?: { rowCount: number; distinctKeyCount: number }; +} + +function rowCountQuery(table: string): { sql: string; params: string[] } { + if (table === "signal_snapshots") { + // Numbered placeholders reused across both IN(...) clauses need only ONE bound value per index -- + // mirrors dedupeSignalSnapshots' own `?1` reuse in src/db/retention.ts. + const placeholders = LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES.map((_, index) => `?${index + 1}`).join(", "); + return { + sql: + `SELECT COUNT(*) AS total, ` + + `(SELECT COUNT(*) FROM signal_snapshots WHERE signal_type IN (${placeholders})) AS dedup_total, ` + + `(SELECT COUNT(DISTINCT ${SIGNAL_SNAPSHOTS_DEDUP_KEY_SQL}) FROM signal_snapshots WHERE signal_type IN (${placeholders})) AS dedup_distinct_keys ` + + `FROM signal_snapshots`, + params: [...LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES], + }; + } + return { sql: `SELECT COUNT(*) AS total FROM ${table}`, params: [] }; +} + +/** `POST /accounts/{account}/d1/database/{database}/query` for one monitored table's row count (plus, + * for "signal_snapshots" only, the dedup-scoped row/distinct-key counts). Throws on an unsafe table name + * (defense in depth -- see {@link SAFE_TABLE_NAME}) or a failed/malformed API response. */ +export async function fetchD1TableRowCount(config: D1SizeProbeConfig, table: string, fetchImpl: typeof fetch = fetch): Promise { + if (!SAFE_TABLE_NAME.test(table)) throw new Error(`Unsafe D1 monitored table identifier: ${table}`); + const { sql, params } = rowCountQuery(table); + const [queryResult] = await cloudflareApiRequest<{ results?: Record[] }[]>( + config, + `/d1/database/${config.databaseId}/query`, + { method: "POST", body: JSON.stringify({ sql, params }) }, + fetchImpl, + ); + const row = queryResult?.results?.[0] ?? {}; + return { + table, + rowCount: Number(row.total ?? 0), + ...(table === "signal_snapshots" + ? { dedup: { rowCount: Number(row.dedup_total ?? 0), distinctKeyCount: Number(row.dedup_distinct_keys ?? 0) } } + : {}), + }; +} + +interface D1ProbeSample { + fileSizeBytes: number; + tableRowCounts: D1TableRowCount[]; +} + +let lastSample: D1ProbeSample | null = null; + +function logD1ProbeError(part: "database_info" | "table_row_count", error: unknown, table?: string): void { + incr("gittensory_d1_probe_errors_total", { part }); + console.error(JSON.stringify({ level: "error", event: "d1_size_probe_error", part, ...(table ? { table } : {}), message: errorMessage(error).slice(0, 200) })); +} + +/** + * Refresh the D1 size/row-count sample (called on a slow self-host timer, see server.ts). No-op when + * {@link resolveD1SizeProbeConfig} returns null (probe disabled/unconfigured). + * + * Size and each monitored table's row count are fetched independently and a failure in one never blanks the + * other: a failed fetch keeps its PREVIOUS reading (recorded via `gittensory_d1_probe_errors_total`) instead + * of resetting to -1 or dropping out of the row-count vector, so a transient Cloudflare API hiccup reads on + * the dashboard as "stale" rather than a false "suddenly zero" or a gap. + */ +export async function runD1SizeProbe(env: D1SizeProbeEnv, fetchImpl: typeof fetch = fetch): Promise { + const config = resolveD1SizeProbeConfig(env); + if (!config) return; + + const [freshInfo, freshRowCounts] = await Promise.all([ + fetchD1DatabaseInfo(config, fetchImpl).catch((error: unknown) => { + logD1ProbeError("database_info", error); + return null; + }), + Promise.all( + config.tables.map((table) => + fetchD1TableRowCount(config, table, fetchImpl).catch((error: unknown) => { + logD1ProbeError("table_row_count", error, table); + return null; + }), + ), + ).then((rows) => rows.filter((row): row is D1TableRowCount => row !== null)), + ]); + + const tableRowCountsByTable = new Map((lastSample?.tableRowCounts ?? []).map((row) => [row.table, row])); + for (const row of freshRowCounts) tableRowCountsByTable.set(row.table, row); + + lastSample = { + fileSizeBytes: freshInfo?.fileSizeBytes ?? lastSample?.fileSizeBytes ?? -1, + tableRowCounts: [...tableRowCountsByTable.values()], + }; +} + +/** -1 sentinel (matching gittensory_host_load_avg1_per_core's convention): distinguishes "probe disabled or + * has never completed a successful sample" from a genuine 0-byte reading. */ +export function d1DatabaseSizeBytesSample(): number { + return lastSample?.fileSizeBytes ?? -1; +} + +/** One series per monitored table with a successful sample so far. Empty (not absent) when the probe is + * disabled or has never completed -- see metrics.ts's gaugeVector: "no data" on the dashboard, not a + * missing metric name. */ +export function d1TableRowCountSamples(): VectorSample[] { + return (lastSample?.tableRowCounts ?? []).map((row) => ({ labels: { table: row.table }, value: row.rowCount })); +} + +/** + * signal_snapshots rows per distinct dedup key, scoped to {@link LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES} only + * (see rowCountQuery) -- should stay a small multiple of 1 once dedupeSignalSnapshots (src/db/retention.ts) + * runs on its daily cadence; a climbing value means the dedup job has stopped running or its allowlist + * regressed. -1 when unavailable (probe disabled, never sampled yet, or no dedup-scoped rows exist). + */ +export function d1SignalSnapshotsRowsPerKeySample(): number { + const dedup = lastSample?.tableRowCounts.find((r) => r.table === "signal_snapshots")?.dedup; + if (!dedup || dedup.distinctKeyCount === 0) return -1; + return dedup.rowCount / dedup.distinctKeyCount; +} + +/** Test-only: reset the module-level sample between tests. */ +export function resetD1SizeProbeForTest(): void { + lastSample = null; +} diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts index a52623387a..260b9f685e 100644 --- a/src/selfhost/metrics.ts +++ b/src/selfhost/metrics.ts @@ -134,6 +134,10 @@ const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [ ["gittensory_pr_state_cache_total", { help: "Pull-request state cache outcomes.", type: "counter" }], ["gittensory_ci_state_cache_total", { help: "CI-state snapshot cache outcomes.", type: "counter" }], ["gittensory_ops_anomaly_total", { help: "Ops anomaly scan detections (review burst / review failure burst), by repo and kind.", type: "counter" }], + ["gittensory_d1_database_size_bytes", { help: "Cloudflare D1 database file size in bytes, from the opt-in Management API size/row-count probe (#3810); -1 when the probe is disabled or has never completed a successful sample.", type: "gauge" }], + ["gittensory_d1_table_row_count", { help: "Row count for a monitored D1 table, from the same probe as gittensory_d1_database_size_bytes, labeled by table.", type: "gauge" }], + ["gittensory_signal_snapshots_rows_per_key", { help: "signal_snapshots row count divided by its distinct (signal_type, target_key) count, scoped to the latest-only-dedup signal types dedupeSignalSnapshots converges to ~1 row per key; -1 when the probe is disabled or has never completed a successful sample.", type: "gauge" }], + ["gittensory_d1_probe_errors_total", { help: "D1 size/row-count Management API probe failures, by part (database_info/table_row_count).", type: "counter" }], ]; const metricMeta = new Map(DEFAULT_METRIC_META); diff --git a/src/server.ts b/src/server.ts index 99005fc6ed..44198a8f90 100644 --- a/src/server.ts +++ b/src/server.ts @@ -52,6 +52,7 @@ import { type ReadinessProbe, } from "./selfhost/health"; import { clockSkewSecondsSample } from "./selfhost/clock-skew"; +import { d1DatabaseSizeBytesSample, d1SignalSnapshotsRowsPerKeySample, d1TableRowCountSamples, isD1SizeProbeEnabled, runD1SizeProbe } from "./selfhost/d1-size-probe"; import { gauge, gaugeVector, incr, observe, renderMetrics, setSelfHostedMetricsMode } from "./selfhost/metrics"; import { runSelfHostMigrations } from "./selfhost/migrate"; import { createPgAdapter, tuneGithubRateLimitObservationsAutovacuum } from "./selfhost/pg-adapter"; @@ -673,6 +674,13 @@ async function main(): Promise { // from "no signal on this platform" (see host-pressure.ts). gauge("gittensory_host_load_avg1_per_core", async () => (await maintenancePressure()).hostLoadAvg1PerCore ?? -1); gauge("gittensory_clock_skew_seconds", () => clockSkewSecondsSample()); + // D1 size/row-count observability probe (#3810): opt-in Cloudflare Management API poll for the shared + // cloud D1's file size and monitored-table row counts. Always registered (byte-identical -1/empty samples + // when the probe is disabled or has never completed) so the metric names/HELP/TYPE lines are present on + // the very first scrape, matching the seeded-counter convention below. + gauge("gittensory_d1_database_size_bytes", () => d1DatabaseSizeBytesSample()); + gaugeVector("gittensory_d1_table_row_count", () => d1TableRowCountSamples()); + gauge("gittensory_signal_snapshots_rows_per_key", () => d1SignalSnapshotsRowsPerKeySample()); // Backlog-vs-fresh-intake fairness lanes (#selfhost-lane-observability, see queue-fairness.ts): the SAME // `foreground_lane` classification the claim-time fairness mechanism itself consults, so an operator can see // whether a stuck-looking queue is actually a real, unresolved PR-review backlog (high backlog-convergence @@ -713,6 +721,9 @@ async function main(): Promise { // first scrape (keeping the metric consistently labeled — never mix labeled and unlabeled samples). for (const status of ["2xx", "3xx", "4xx", "5xx"]) incr("gittensory_http_requests_total", { status }, 0); + // Same seeding for the D1 probe's error counter (#3810) -- byte-identical to 0 whether or not the probe is + // even enabled, so its stat panel reads "0" rather than "No data" before any failure has ever occurred. + for (const part of ["database_info", "table_row_count"]) incr("gittensory_d1_probe_errors_total", { part }, 0); const ctx = { waitUntil: (p: Promise) => @@ -999,6 +1010,24 @@ async function main(): Promise { ); /* v8 ignore stop */ + // D1 size/row-count observability probe (#3810): a no-op everywhere until an operator sets all three + // CLOUDFLARE_D1_MONITOR_* vars (see isD1SizeProbeEnabled) -- most self-host installs run their own + // SQLite/Postgres backend and have no Cloudflare D1 to watch. 15-minute cadence: the underlying figures + // (a multi-GB database's file size, monitored-table row counts) move slowly, so this stays well clear of + // Cloudflare Management API rate limits even across a large monitored-table list. + const d1ProbeEnv = { + CLOUDFLARE_D1_MONITOR_ACCOUNT_ID: process.env.CLOUDFLARE_D1_MONITOR_ACCOUNT_ID, + CLOUDFLARE_D1_MONITOR_DATABASE_ID: process.env.CLOUDFLARE_D1_MONITOR_DATABASE_ID, + CLOUDFLARE_D1_MONITOR_API_TOKEN: process.env.CLOUDFLARE_D1_MONITOR_API_TOKEN, + }; + if (isD1SizeProbeEnabled(d1ProbeEnv)) { + /* v8 ignore start -- self-host entrypoint timer; probe logic itself is unit-tested in d1-size-probe.test.ts. */ + const runD1Probe = () => runD1SizeProbe(d1ProbeEnv).catch((error) => captureError(error, { kind: "d1_size_probe" })); + void runD1Probe(); + setInterval(runD1Probe, 900_000); + /* v8 ignore stop */ + } + // Pull-mode relay drain (#secure-relay): when ORB_RELAY_MODE=pull, the engine DRAINS its events from the Orb on a // timer instead of exposing an inbound endpoint — the right fit behind NAT/tailnet. Acks the previous batch so the // Orb deletes delivered events; best-effort (a failed tick retries next interval). Each event enqueues into the diff --git a/test/unit/selfhost-d1-size-probe.test.ts b/test/unit/selfhost-d1-size-probe.test.ts new file mode 100644 index 0000000000..e0ed728216 --- /dev/null +++ b/test/unit/selfhost-d1-size-probe.test.ts @@ -0,0 +1,334 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES } from "../../src/db/retention"; +import { + d1DatabaseSizeBytesSample, + d1SignalSnapshotsRowsPerKeySample, + d1TableRowCountSamples, + fetchD1DatabaseInfo, + fetchD1TableRowCount, + isD1SizeProbeEnabled, + resetD1SizeProbeForTest, + resolveD1SizeProbeConfig, + runD1SizeProbe, + type D1SizeProbeConfig, + type D1SizeProbeEnv, +} from "../../src/selfhost/d1-size-probe"; +import { renderMetrics, resetMetrics, gauge, gaugeVector, counterValue } from "../../src/selfhost/metrics"; + +afterEach(() => { + resetD1SizeProbeForTest(); + resetMetrics(); +}); + +const FULL_ENV: D1SizeProbeEnv = { + CLOUDFLARE_D1_MONITOR_ACCOUNT_ID: "acct-1", + CLOUDFLARE_D1_MONITOR_DATABASE_ID: "db-1", + CLOUDFLARE_D1_MONITOR_API_TOKEN: "token-1", +}; + +function envelope(result: T): string { + return JSON.stringify({ success: true, errors: [], result }); +} + +function requestUrl(input: RequestInfo | URL): string { + if (typeof input === "string") return input; + if (input instanceof URL) return input.toString(); + return input.url; +} + +/** A fetch stub that answers the database-info GET and the per-table query POST generically -- the query + * handler inspects the outer table name (the LAST "FROM ") so it works for any monitored table + * without per-test enumeration, and special-cases signal_snapshots' extra dedup columns. */ +function mockFetch(opts: { + databaseInfo?: () => Response; + rowsForTable?: (table: string) => { total: number; dedupTotal?: number; dedupDistinctKeys?: number } | null; + calls?: { url: string; method: string }[]; +}): typeof fetch { + return (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = requestUrl(input); + const method = init?.method ?? "GET"; + opts.calls?.push({ url, method }); + if (method === "POST") { + const body = JSON.parse(String(init?.body ?? "{}")) as { sql: string }; + const table = body.sql.match(/FROM (\w+)\s*$/)?.[1] ?? "unknown"; + const row = opts.rowsForTable?.(table) ?? null; + if (row === null) return new Response(envelope([{ results: [], success: true, meta: {} }]), { status: 500 }); + const resultRow: Record = { total: row.total }; + if (row.dedupTotal !== undefined) resultRow.dedup_total = row.dedupTotal; + if (row.dedupDistinctKeys !== undefined) resultRow.dedup_distinct_keys = row.dedupDistinctKeys; + return new Response(envelope([{ results: [resultRow], success: true, meta: {} }])); + } + return opts.databaseInfo ? opts.databaseInfo() : new Response(envelope({}), { status: 500 }); + }) as typeof fetch; +} + +function okDatabaseInfo(fileSize: number, numTables = 12): () => Response { + return () => new Response(envelope({ file_size: fileSize, num_tables: numTables })); +} + +describe("resolveD1SizeProbeConfig / isD1SizeProbeEnabled", () => { + it("resolves a config when all three vars are present", () => { + const config = resolveD1SizeProbeConfig(FULL_ENV); + expect(config).not.toBeNull(); + expect(config?.accountId).toBe("acct-1"); + expect(config?.databaseId).toBe("db-1"); + expect(config?.apiToken).toBe("token-1"); + expect(config?.tables.length).toBeGreaterThan(0); + expect(isD1SizeProbeEnabled(FULL_ENV)).toBe(true); + }); + + it("returns null (disabled) when the account id is missing", () => { + expect(resolveD1SizeProbeConfig({ ...FULL_ENV, CLOUDFLARE_D1_MONITOR_ACCOUNT_ID: undefined })).toBeNull(); + }); + + it("returns null (disabled) when the database id is missing", () => { + expect(resolveD1SizeProbeConfig({ ...FULL_ENV, CLOUDFLARE_D1_MONITOR_DATABASE_ID: undefined })).toBeNull(); + }); + + it("returns null (disabled) when the api token is missing", () => { + expect(resolveD1SizeProbeConfig({ ...FULL_ENV, CLOUDFLARE_D1_MONITOR_API_TOKEN: "" })).toBeNull(); + }); + + it("isD1SizeProbeEnabled is false with no config at all", () => { + expect(isD1SizeProbeEnabled({})).toBe(false); + }); +}); + +describe("fetchD1DatabaseInfo", () => { + const config: D1SizeProbeConfig = { accountId: "a", databaseId: "d", apiToken: "t", tables: [] }; + + it("parses file_size and num_tables from a successful response", async () => { + const info = await fetchD1DatabaseInfo(config, mockFetch({ databaseInfo: okDatabaseInfo(3_890_057_216, 9) })); + expect(info).toEqual({ fileSizeBytes: 3_890_057_216, numTables: 9 }); + }); + + it("defaults missing fields to 0", async () => { + const info = await fetchD1DatabaseInfo( + config, + mockFetch({ databaseInfo: () => new Response(envelope({})) }), + ); + expect(info).toEqual({ fileSizeBytes: 0, numTables: 0 }); + }); + + it("throws with the Cloudflare error message on a non-OK response", async () => { + const fetchImpl: typeof fetch = async () => + new Response(JSON.stringify({ success: false, errors: [{ code: 7003, message: "Could not route to account" }], result: null }), { status: 403 }); + await expect(fetchD1DatabaseInfo(config, fetchImpl)).rejects.toThrow(/7003: Could not route to account/); + }); + + it("throws a generic HTTP-status error when the body isn't valid JSON", async () => { + const fetchImpl: typeof fetch = async () => new Response("gateway error", { status: 502 }); + await expect(fetchD1DatabaseInfo(config, fetchImpl)).rejects.toThrow(/HTTP 502/); + }); + + it("throws when the envelope reports success:false even with a 200 status", async () => { + const fetchImpl: typeof fetch = async () => new Response(JSON.stringify({ success: false, errors: [], result: null })); + await expect(fetchD1DatabaseInfo(config, fetchImpl)).rejects.toThrow(/HTTP 200/); + }); +}); + +describe("fetchD1TableRowCount", () => { + const config: D1SizeProbeConfig = { accountId: "a", databaseId: "d", apiToken: "t", tables: [] }; + + it("rejects an unsafe table identifier before making any request", async () => { + let called = false; + const fetchImpl: typeof fetch = async () => { + called = true; + return new Response(envelope([{ results: [{ total: 1 }], success: true, meta: {} }])); + }; + await expect(fetchD1TableRowCount(config, "bad; DROP TABLE x", fetchImpl)).rejects.toThrow(/Unsafe D1 monitored table identifier/); + expect(called).toBe(false); + }); + + it("reports a plain row count for a non-signal_snapshots table with no dedup field at all", async () => { + const row = await fetchD1TableRowCount(config, "audit_events", mockFetch({ rowsForTable: () => ({ total: 4242 }) })); + expect(row).toEqual({ table: "audit_events", rowCount: 4242 }); + expect(row.dedup).toBeUndefined(); + }); + + it("reports dedup-scoped row/distinct-key counts for signal_snapshots, binding LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES once each via numbered placeholders", async () => { + const calls: { url: string; method: string }[] = []; + let capturedParams: string[] = []; + const fetchImpl: typeof fetch = async (input, init) => { + calls.push({ url: requestUrl(input), method: init?.method ?? "GET" }); + const body = JSON.parse(String(init?.body ?? "{}")) as { sql: string; params: string[] }; + capturedParams = body.params; + expect(body.sql).toContain("signal_type IN (?1, ?2, ?3, ?4)"); + return new Response(envelope([{ results: [{ total: 20225, dedup_total: 107, dedup_distinct_keys: 36 }], success: true, meta: {} }])); + }; + const row = await fetchD1TableRowCount(config, "signal_snapshots", fetchImpl); + expect(row).toEqual({ table: "signal_snapshots", rowCount: 20225, dedup: { rowCount: 107, distinctKeyCount: 36 } }); + // Each dedup type bound exactly once, matching the SQL's numbered-placeholder reuse (?1 used twice needs + // only one bound value, mirroring dedupeSignalSnapshots' own ?1 reuse in src/db/retention.ts). + expect(capturedParams).toEqual([...LATEST_ONLY_SIGNAL_SNAPSHOT_TYPES]); + expect(calls).toHaveLength(1); + }); + + it("defaults missing result fields to 0", async () => { + const fetchImpl: typeof fetch = async () => new Response(envelope([{ results: [{}], success: true, meta: {} }])); + const row = await fetchD1TableRowCount(config, "signal_snapshots", fetchImpl); + expect(row).toEqual({ table: "signal_snapshots", rowCount: 0, dedup: { rowCount: 0, distinctKeyCount: 0 } }); + }); + + it("defaults to an empty row when the query result is missing entirely", async () => { + const fetchImpl: typeof fetch = async () => new Response(envelope([{ results: [], success: true, meta: {} }])); + const row = await fetchD1TableRowCount(config, "audit_events", fetchImpl); + expect(row).toEqual({ table: "audit_events", rowCount: 0 }); + expect(row.dedup).toBeUndefined(); + }); +}); + +describe("runD1SizeProbe", () => { + it("is a no-op when the probe is not configured (no fetch call at all)", async () => { + const calls: { url: string; method: string }[] = []; + await runD1SizeProbe({}, mockFetch({ calls })); + expect(calls).toHaveLength(0); + expect(d1DatabaseSizeBytesSample()).toBe(-1); + expect(d1TableRowCountSamples()).toEqual([]); + }); + + it("populates size and every monitored table's row count on a full success", async () => { + await runD1SizeProbe( + FULL_ENV, + mockFetch({ + databaseInfo: okDatabaseInfo(3_890_057_216), + rowsForTable: (table) => (table === "signal_snapshots" ? { total: 20225, dedupTotal: 107, dedupDistinctKeys: 36 } : { total: 500 }), + }), + ); + expect(d1DatabaseSizeBytesSample()).toBe(3_890_057_216); + const samples = d1TableRowCountSamples(); + expect(samples.length).toBeGreaterThanOrEqual(2); + expect(samples).toContainEqual({ labels: { table: "signal_snapshots" }, value: 20225 }); + expect(samples).toContainEqual({ labels: { table: "audit_events" }, value: 500 }); + // 107 / 36 ~= 2.972... + expect(d1SignalSnapshotsRowsPerKeySample()).toBeCloseTo(107 / 36, 6); + }); + + it("keeps the previous sample when a later tick fails entirely, and records the failure", async () => { + await runD1SizeProbe(FULL_ENV, mockFetch({ databaseInfo: okDatabaseInfo(1_000_000), rowsForTable: () => ({ total: 10 }) })); + expect(d1DatabaseSizeBytesSample()).toBe(1_000_000); + + const failingFetch: typeof fetch = async () => new Response(envelope(null), { status: 500 }); + await runD1SizeProbe(FULL_ENV, failingFetch); + + expect(d1DatabaseSizeBytesSample()).toBe(1_000_000); // stale, not reset to -1 + expect(d1TableRowCountSamples().length).toBeGreaterThan(0); // stale rows kept, not blanked + expect(counterValue("gittensory_d1_probe_errors_total", { part: "database_info" })).toBeGreaterThan(0); + expect(counterValue("gittensory_d1_probe_errors_total", { part: "table_row_count" })).toBeGreaterThan(0); + }); + + it("reads -1 on the very first tick when everything fails (no previous sample to fall back to)", async () => { + const failingFetch: typeof fetch = async () => new Response(envelope(null), { status: 500 }); + await runD1SizeProbe(FULL_ENV, failingFetch); + expect(d1DatabaseSizeBytesSample()).toBe(-1); + expect(d1TableRowCountSamples()).toEqual([]); + }); + + it("isolates a single failing table: other tables still update and the failed one keeps its stale value", async () => { + await runD1SizeProbe( + FULL_ENV, + mockFetch({ + databaseInfo: okDatabaseInfo(1_000_000), + rowsForTable: (table) => (table === "signal_snapshots" ? { total: 100, dedupTotal: 10, dedupDistinctKeys: 5 } : { total: 1 }), + }), + ); + const before = d1TableRowCountSamples().find((s) => s.labels.table === "audit_events"); + expect(before?.value).toBe(1); + + // Second tick: audit_events now errors, everything else (including signal_snapshots) succeeds with new values. + await runD1SizeProbe( + FULL_ENV, + mockFetch({ + databaseInfo: okDatabaseInfo(2_000_000), + rowsForTable: (table) => { + if (table === "audit_events") return null; // triggers a 500 in the mock -> throws + if (table === "signal_snapshots") return { total: 100, dedupTotal: 20, dedupDistinctKeys: 5 }; + return { total: 2 }; + }, + }), + ); + + expect(d1DatabaseSizeBytesSample()).toBe(2_000_000); // unrelated size fetch still updates + const samples = d1TableRowCountSamples(); + expect(samples).toContainEqual({ labels: { table: "audit_events" }, value: 1 }); // stale, kept from tick 1 + expect(samples).toContainEqual({ labels: { table: "signal_snapshots" }, value: 100 }); // freshly updated + expect(d1SignalSnapshotsRowsPerKeySample()).toBeCloseTo(20 / 5, 6); // ratio reflects the FRESH sample + expect(counterValue("gittensory_d1_probe_errors_total", { part: "table_row_count" })).toBeGreaterThan(0); + }); + + it("keeps size at its previous value when only the database-info fetch fails but tables succeed", async () => { + await runD1SizeProbe(FULL_ENV, mockFetch({ databaseInfo: okDatabaseInfo(5_000_000), rowsForTable: () => ({ total: 1 }) })); + expect(d1DatabaseSizeBytesSample()).toBe(5_000_000); + + await runD1SizeProbe( + FULL_ENV, + mockFetch({ databaseInfo: () => new Response(envelope(null), { status: 500 }), rowsForTable: () => ({ total: 2 }) }), + ); + expect(d1DatabaseSizeBytesSample()).toBe(5_000_000); // stale, size fetch failed this tick + expect(d1TableRowCountSamples().every((s) => s.value === 2)).toBe(true); // tables still refreshed + }); +}); + +describe("d1SignalSnapshotsRowsPerKeySample", () => { + it("is -1 before any sample has been taken", () => { + expect(d1SignalSnapshotsRowsPerKeySample()).toBe(-1); + }); + + it("is -1 when signal_snapshots has never been successfully sampled (its fetch keeps failing)", async () => { + await runD1SizeProbe( + FULL_ENV, + mockFetch({ databaseInfo: okDatabaseInfo(1), rowsForTable: (table) => (table === "signal_snapshots" ? null : { total: 1 }) }), + ); + // signal_snapshots' own fetch failed (mockFetch returns a 500 for a null row) so no dedup sample was ever + // recorded for it, even though every OTHER monitored table succeeded. + expect(d1SignalSnapshotsRowsPerKeySample()).toBe(-1); + expect(d1TableRowCountSamples().some((s) => s.labels.table === "signal_snapshots")).toBe(false); + }); + + it("is -1 when the distinct-key count is 0 (division-by-zero guard)", async () => { + await runD1SizeProbe( + FULL_ENV, + mockFetch({ + databaseInfo: okDatabaseInfo(1), + rowsForTable: (table) => (table === "signal_snapshots" ? { total: 0, dedupTotal: 0, dedupDistinctKeys: 0 } : { total: 1 }), + }), + ); + expect(d1SignalSnapshotsRowsPerKeySample()).toBe(-1); + }); +}); + +describe("D1 metrics end-to-end via renderMetrics()", () => { + it("renders gauges and vector series with the registered HELP/TYPE metadata after a successful probe", async () => { + gauge("gittensory_d1_database_size_bytes", () => d1DatabaseSizeBytesSample()); + gaugeVector("gittensory_d1_table_row_count", () => d1TableRowCountSamples()); + gauge("gittensory_signal_snapshots_rows_per_key", () => d1SignalSnapshotsRowsPerKeySample()); + + await runD1SizeProbe( + FULL_ENV, + mockFetch({ + databaseInfo: okDatabaseInfo(3_890_057_216), + rowsForTable: (table) => (table === "signal_snapshots" ? { total: 107, dedupTotal: 107, dedupDistinctKeys: 36 } : { total: 9 }), + }), + ); + + const out = await renderMetrics(); + expect(out).toContain("# TYPE gittensory_d1_database_size_bytes gauge"); + expect(out).toContain("gittensory_d1_database_size_bytes 3890057216"); + expect(out).toContain("# TYPE gittensory_d1_table_row_count gauge"); + expect(out).toContain('gittensory_d1_table_row_count{table="signal_snapshots"} 107'); + expect(out).toContain("# TYPE gittensory_signal_snapshots_rows_per_key gauge"); + expect(out).toMatch(/gittensory_signal_snapshots_rows_per_key 2\.9\d+/); + }); + + it("renders -1 sentinels and an empty vector before the probe ever runs", async () => { + gauge("gittensory_d1_database_size_bytes", () => d1DatabaseSizeBytesSample()); + gaugeVector("gittensory_d1_table_row_count", () => d1TableRowCountSamples()); + gauge("gittensory_signal_snapshots_rows_per_key", () => d1SignalSnapshotsRowsPerKeySample()); + + const out = await renderMetrics(); + expect(out).toContain("gittensory_d1_database_size_bytes -1"); + expect(out).toContain("gittensory_signal_snapshots_rows_per_key -1"); + expect(out).toContain("# TYPE gittensory_d1_table_row_count gauge"); + expect(out).not.toContain('gittensory_d1_table_row_count{table='); + }); +});