+ --profile backup
diff --git a/docker-compose.yml b/docker-compose.yml
index 8ab7100efb..4005e7fdbe 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -15,6 +15,8 @@
# --profile pgbouncer PgBouncer connection pooler in front of Postgres
# --profile qdrant Qdrant vector database for RAG
# --profile ollama Local Ollama AI backend
+# --profile gpu GPU metrics exporter (nvidia-smi -> Prometheus -> Grafana), requires the NVIDIA
+# Container Toolkit; combine with --profile ollama on a GPU host
# --profile visual-review Headless Chromium (browserless) for before/after PR screenshot capture
# --profile rees Review-enrichment service (REES) for heavier PR analysis, in-network only
# --profile litestream Continuous SQLite backup to S3/B2/R2 via Litestream
@@ -369,6 +371,30 @@ services:
limits:
memory: "${OLLAMA_MEM_LIMIT:-20g}"
+ # ── GPU metrics exporter (--profile gpu, #4367) ─────────────────────────────
+ # Shells to `nvidia-smi` via the NVIDIA Container Toolkit -- DCGM is built for datacenter fleets and is
+ # the wrong fit for a single consumer/workstation-class card like the one this deployment targets. The
+ # device reservation below only takes effect once this profile is actually activated, so a host with no
+ # GPU (and no --profile gpu) never evaluates it.
+ gpu-exporter:
+ image: utkuozdemir/nvidia_gpu_exporter:1.3.2
+ restart: unless-stopped
+ <<: *default-logging
+ profiles: ["gpu"]
+ environment:
+ # The exporter only shells out to nvidia-smi -- it never runs CUDA workloads itself, so the
+ # lightest capability tier is sufficient (matches the upstream project's own recommended config).
+ NVIDIA_DRIVER_CAPABILITIES: utility
+ deploy:
+ resources:
+ reservations:
+ devices:
+ - driver: nvidia
+ count: all
+ capabilities: [gpu]
+ expose:
+ - "9835"
+
# ── Browserless / visual review (--profile visual-review) ──────────────────
# Headless Chromium for automated before/after PR screenshot capture (src/review/visual/**).
# Set BOTH of these in .env (browserless always requires a token; there is no way around setting
diff --git a/grafana/dashboards/gpu-metrics.json b/grafana/dashboards/gpu-metrics.json
new file mode 100644
index 0000000000..36685d17b7
--- /dev/null
+++ b/grafana/dashboards/gpu-metrics.json
@@ -0,0 +1,196 @@
+{
+ "__inputs": [],
+ "__requires": [
+ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" },
+ { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }
+ ],
+ "annotations": { "list": [] },
+ "editable": false,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 1,
+ "id": null,
+ "links": [],
+ "panels": [
+ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "title": "GPU Summary", "type": "row" },
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "fieldConfig": {
+ "defaults": {
+ "color": { "mode": "thresholds" },
+ "mappings": [],
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 }] },
+ "unit": "percentunit"
+ }
+ },
+ "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
+ "id": 2,
+ "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
+ "title": "GPU Utilization",
+ "type": "stat",
+ "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_utilization_gpu_ratio", "legendFormat": "util" }]
+ },
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "fieldConfig": {
+ "defaults": {
+ "color": { "mode": "thresholds" },
+ "mappings": [],
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+ "unit": "bytes"
+ }
+ },
+ "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
+ "id": 3,
+ "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
+ "title": "VRAM Used",
+ "description": "See the VRAM Used vs Total trend panel below for headroom against the card's total capacity.",
+ "type": "stat",
+ "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_memory_used_bytes", "legendFormat": "used" }]
+ },
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "fieldConfig": {
+ "defaults": {
+ "color": { "mode": "thresholds" },
+ "mappings": [],
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 85 }] },
+ "unit": "celsius"
+ }
+ },
+ "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
+ "id": 4,
+ "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
+ "title": "GPU Temperature",
+ "type": "stat",
+ "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_temperature_gpu", "legendFormat": "temp" }]
+ },
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "fieldConfig": {
+ "defaults": {
+ "color": { "mode": "thresholds" },
+ "mappings": [],
+ "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+ "unit": "watt"
+ }
+ },
+ "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
+ "id": 5,
+ "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
+ "title": "Power Draw",
+ "type": "stat",
+ "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_power_draw_watts", "legendFormat": "power" }]
+ },
+
+ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 10, "title": "AI Request Performance", "type": "row" },
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "reqps" } },
+ "gridPos": { "h": 8, "w": 8, "x": 0, "y": 6 },
+ "id": 11,
+ "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+ "title": "AI Request Rate",
+ "description": "Request rate by provider and request kind (embedding vs review), derived from the new request-duration histogram's observation count.",
+ "type": "timeseries",
+ "targets": [
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "expr": "sum by (provider, request_kind) (rate(gittensory_ai_provider_request_duration_seconds_count[5m]))",
+ "legendFormat": "{{provider}} / {{request_kind}}",
+ "refId": "A"
+ }
+ ]
+ },
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "s" } },
+ "gridPos": { "h": 8, "w": 8, "x": 8, "y": 6 },
+ "id": 12,
+ "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+ "title": "AI Request Latency (p50 / p95 / p99)",
+ "type": "timeseries",
+ "targets": [
+ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(gittensory_ai_provider_request_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A" },
+ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(gittensory_ai_provider_request_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B" },
+ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(gittensory_ai_provider_request_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C" }
+ ]
+ },
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "reqps" } },
+ "gridPos": { "h": 8, "w": 8, "x": 16, "y": 6 },
+ "id": 13,
+ "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+ "title": "AI Request Error Rate",
+ "description": "Excludes expected embedding-routing fallbacks (those never increment this counter -- see gittensory_ai_provider_request_errors_total's HELP text).",
+ "type": "timeseries",
+ "targets": [
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "expr": "sum by (provider, request_kind) (rate(gittensory_ai_provider_request_errors_total[5m]))",
+ "legendFormat": "{{provider}} / {{request_kind}}",
+ "refId": "A"
+ }
+ ]
+ },
+
+ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 20, "title": "GPU Trends", "type": "row" },
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "percentunit", "max": 1, "min": 0 } },
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
+ "id": 21,
+ "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+ "title": "GPU Utilization Over Time",
+ "type": "timeseries",
+ "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_utilization_gpu_ratio", "legendFormat": "util", "refId": "A" }]
+ },
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "bytes" } },
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
+ "id": 22,
+ "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+ "title": "VRAM Used vs Total",
+ "type": "timeseries",
+ "targets": [
+ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_memory_used_bytes", "legendFormat": "used", "refId": "A" },
+ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_memory_total_bytes", "legendFormat": "total", "refId": "B" }
+ ]
+ },
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "celsius" } },
+ "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 },
+ "id": 23,
+ "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+ "title": "GPU Temperature Over Time",
+ "type": "timeseries",
+ "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_temperature_gpu", "legendFormat": "temp", "refId": "A" }]
+ },
+ {
+ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+ "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "watt" } },
+ "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 },
+ "id": 24,
+ "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+ "title": "Power Draw Over Time",
+ "type": "timeseries",
+ "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_power_draw_watts", "legendFormat": "power", "refId": "A" }]
+ }
+ ],
+ "refresh": "30s",
+ "schemaVersion": 38,
+ "tags": ["gittensory", "gpu"],
+ "templating": {
+ "list": [
+ { "current": {}, "hide": 0, "includeAll": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "refresh": 1, "type": "datasource" }
+ ]
+ },
+ "time": { "from": "now-6h", "to": "now" },
+ "timepicker": {},
+ "timezone": "browser",
+ "title": "Gittensory — GPU Metrics",
+ "uid": "gittensory-gpu",
+ "version": 1
+}
diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml
index b2715e8e2f..44b330c3f3 100644
--- a/prometheus/prometheus.yml
+++ b/prometheus/prometheus.yml
@@ -30,6 +30,13 @@ scrape_configs:
- targets: ["postgres-exporter:9187"]
scrape_interval: 30s
+ # GPU utilization/VRAM/temp/power from nvidia_gpu_exporter (#4367). The exporter starts only with
+ # --profile gpu; on a non-GPU install this target just shows as absent/down, no alert fires on it.
+ - job_name: gpu
+ static_configs:
+ - targets: ["gpu-exporter:9835"]
+ scrape_interval: 15s
+
# Backup freshness from the read-only backup-exporter sidecar. The metrics exist only when the backup
# profile is active, which keeps backup alerts opt-in with the backup feature.
- job_name: gittensory-backup
diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts
index 699ffc8a90..1d6cfaba42 100644
--- a/src/selfhost/ai.ts
+++ b/src/selfhost/ai.ts
@@ -9,7 +9,7 @@
import type { AiContentBlock, CombineStrategy, OnMerge } from "../services/ai-review";
import { isConfiguredSelfHostProvider, resolveConfiguredProviderNames } from "./ai-config";
export { assertNoLegacySharedAiEnv } from "./ai-config";
-import { incr } from "./metrics";
+import { incr, observe } from "./metrics";
import { withReviewSpan } from "./tracing";
import { delimiter } from "node:path";
@@ -1096,12 +1096,18 @@ async function runProviderWithOtel(
`circuit_open: provider "${provider.name}" is in cooldown after ${AI_PROVIDER_FAILURE_THRESHOLD} consecutive failures — skipping this attempt`,
);
}
+ const requestKindLabel = requestKind(options);
+ const startedAtMs = Date.now();
try {
const result = await withReviewSpan(
"selfhost.ai.provider",
- { "ai.provider": provider.name, "ai.model": model || "default", "ai.request_kind": requestKind(options) },
+ { "ai.provider": provider.name, "ai.model": model || "default", "ai.request_kind": requestKindLabel },
() => provider.ai.run(model, options),
);
+ observe("gittensory_ai_provider_request_duration_seconds", (Date.now() - startedAtMs) / 1000, {
+ provider: provider.name,
+ request_kind: requestKindLabel,
+ });
aiProviderCircuits.delete(provider.name);
if (result.usage) {
return {
@@ -1115,8 +1121,13 @@ async function runProviderWithOtel(
}
return result;
} catch (error) {
+ observe("gittensory_ai_provider_request_duration_seconds", (Date.now() - startedAtMs) / 1000, {
+ provider: provider.name,
+ request_kind: requestKindLabel,
+ });
if (isExpectedEmbeddingRoutingError(options, error)) throw error;
incr("gittensory_ai_provider_failures_total", { provider: provider.name });
+ incr("gittensory_ai_provider_request_errors_total", { provider: provider.name, request_kind: requestKindLabel });
// Re-read the map here rather than reusing the `circuit` captured above: that read happened BEFORE the
// `await` on the real provider call, so under concurrent same-provider calls it can be stale by the time
// this catch runs, and computing `failures` from it would clobber a sibling call's write (lost-update race)
diff --git a/src/selfhost/metrics.ts b/src/selfhost/metrics.ts
index 260b9f685e..00e31cc138 100644
--- a/src/selfhost/metrics.ts
+++ b/src/selfhost/metrics.ts
@@ -111,6 +111,8 @@ const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [
["gittensory_ai_total_tokens_total", { help: "AI provider total tokens observed.", type: "counter" }],
["gittensory_ai_provider_circuit_open_total", { help: "AI provider circuit-open events.", type: "counter" }],
["gittensory_ai_provider_failures_total", { help: "AI provider failures by provider.", type: "counter" }],
+ ["gittensory_ai_provider_request_duration_seconds", { help: "AI provider request duration in seconds, by provider and request kind.", type: "histogram" }],
+ ["gittensory_ai_provider_request_errors_total", { help: "AI provider request errors, by provider and request kind (excludes expected embedding-routing fallbacks).", type: "counter" }],
["gittensory_ai_review_cache_hit_total", { help: "AI review cache hits.", type: "counter" }],
["gittensory_ai_review_cache_miss_total", { help: "AI review cache misses.", type: "counter" }],
["gittensory_ai_review_cache_write_error_total", { help: "AI review cache write errors.", type: "counter" }],
diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts
index 661ec61810..54013d99f4 100644
--- a/test/unit/selfhost-ai.test.ts
+++ b/test/unit/selfhost-ai.test.ts
@@ -437,6 +437,42 @@ describe("createChainAi (fallback)", () => {
});
});
+describe("AI provider request duration/error metrics (#4367)", () => {
+ it("records a duration observation and no error on a successful call", async () => {
+ const provider = { name: "gpu-metrics-ok-provider", ai: { run: async () => ({ response: "ok" }) } };
+ await createChainAi([provider]).run("m", { prompt: "review this" });
+ const metrics = await renderMetrics();
+ expect(metrics).toContain('gittensory_ai_provider_request_duration_seconds_count{provider="gpu-metrics-ok-provider",request_kind="review"} 1');
+ expect(metrics).not.toContain('gittensory_ai_provider_request_errors_total{provider="gpu-metrics-ok-provider"');
+ });
+
+ it("labels an embedding call's duration by request_kind=embedding", async () => {
+ const provider = { name: "gpu-metrics-embed-provider", ai: { run: async () => ({ response: "ok" }) } };
+ await createChainAi([provider]).run("m", { text: ["chunk one", "chunk two"] });
+ const metrics = await renderMetrics();
+ expect(metrics).toContain('gittensory_ai_provider_request_duration_seconds_count{provider="gpu-metrics-embed-provider",request_kind="embedding"} 1');
+ });
+
+ it("records duration AND increments the error counter on a real failure", async () => {
+ const provider = { name: "gpu-metrics-fail-provider", ai: { run: async () => { throw new Error("boom"); } } };
+ await expect(createChainAi([provider]).run("m", { prompt: "review this" })).rejects.toThrow(/boom/);
+ const metrics = await renderMetrics();
+ expect(metrics).toContain('gittensory_ai_provider_request_duration_seconds_count{provider="gpu-metrics-fail-provider",request_kind="review"} 1');
+ expect(metrics).toContain('gittensory_ai_provider_request_errors_total{provider="gpu-metrics-fail-provider",request_kind="review"} 1');
+ });
+
+ it("records duration but NOT the error counter for an expected embedding-routing fallback (matches gittensory_ai_provider_failures_total's exemption)", async () => {
+ const provider = {
+ name: "gpu-metrics-routing-provider",
+ ai: { run: async () => { throw new Error("claude_code_no_embed"); } },
+ };
+ await expect(createChainAi([provider]).run("m", { text: ["chunk"] })).rejects.toThrow();
+ const metrics = await renderMetrics();
+ expect(metrics).toContain('gittensory_ai_provider_request_duration_seconds_count{provider="gpu-metrics-routing-provider",request_kind="embedding"} 1');
+ expect(metrics).not.toContain('gittensory_ai_provider_request_errors_total{provider="gpu-metrics-routing-provider"');
+ });
+});
+
describe("per-provider circuit breaker (#2540 — skip fast during a sustained outage)", () => {
afterEach(() => {
vi.useRealTimers();
|