feat(selfhost): GPU + AI-provider observability (exporter, Prometheus, dashboard)

apps/gittensory-ui/src/routes/docs.self-hosting-operations.tsx

@@ -307,6 +307,21 @@ DISCORD_REPO_WEBHOOKS={"owner/repoA":"https://discord.com/api/webhooks/...","own
                 the file.
               </td>
             </tr>
+            <tr>
+              <td className="py-2 pr-4 align-top">
+                + <code>--profile gpu</code>
+              </td>
+              <td className="py-2 pr-4 align-top text-muted-foreground">Near-zero</td>
+              <td className="py-2 pr-4 align-top text-muted-foreground">
+                Tens of MiB — a single Go binary shelling out to <code>nvidia-smi</code>
+              </td>
+              <td className="py-2 align-top text-muted-foreground">
+                Estimated. Adds <code>gpu-exporter</code> (nvidia_gpu_exporter) feeding the{" "}
+                <code>gpu</code> Prometheus job and the GPU metrics Grafana dashboard — requires the
+                NVIDIA Container Toolkit on the host; a device reservation only takes effect once
+                this profile is activated, so a non-GPU host is unaffected either way.
+              </td>
+            </tr>
             <tr>
               <td className="py-2 pr-4 align-top">
                 + <code>--profile backup</code>

docker-compose.yml

@@ -15,6 +15,8 @@
 #   --profile pgbouncer     PgBouncer connection pooler in front of Postgres
 #   --profile qdrant        Qdrant vector database for RAG
 #   --profile ollama        Local Ollama AI backend
+#   --profile gpu           GPU metrics exporter (nvidia-smi -> Prometheus -> Grafana), requires the NVIDIA
+#                           Container Toolkit; combine with --profile ollama on a GPU host
 #   --profile visual-review Headless Chromium (browserless) for before/after PR screenshot capture
 #   --profile rees          Review-enrichment service (REES) for heavier PR analysis, in-network only
 #   --profile litestream    Continuous SQLite backup to S3/B2/R2 via Litestream
@@ -369,6 +371,30 @@ services:
         limits:
           memory: "${OLLAMA_MEM_LIMIT:-20g}"
 
+  # ── GPU metrics exporter (--profile gpu, #4367) ─────────────────────────────
+  # Shells to `nvidia-smi` via the NVIDIA Container Toolkit -- DCGM is built for datacenter fleets and is
+  # the wrong fit for a single consumer/workstation-class card like the one this deployment targets. The
+  # device reservation below only takes effect once this profile is actually activated, so a host with no
+  # GPU (and no --profile gpu) never evaluates it.
+  gpu-exporter:
+    image: utkuozdemir/nvidia_gpu_exporter:1.3.2
+    restart: unless-stopped
+    <<: *default-logging
+    profiles: ["gpu"]
+    environment:
+      # The exporter only shells out to nvidia-smi -- it never runs CUDA workloads itself, so the
+      # lightest capability tier is sufficient (matches the upstream project's own recommended config).
+      NVIDIA_DRIVER_CAPABILITIES: utility
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: all
+              capabilities: [gpu]
+    expose:
+      - "9835"
+
   # ── Browserless / visual review (--profile visual-review) ──────────────────
   # Headless Chromium for automated before/after PR screenshot capture (src/review/visual/**).
   # Set BOTH of these in .env (browserless always requires a token; there is no way around setting

grafana/dashboards/gpu-metrics.json

@@ -0,0 +1,196 @@
+{
+  "__inputs": [],
+  "__requires": [
+    { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" },
+    { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }
+  ],
+  "annotations": { "list": [] },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "id": null,
+  "links": [],
+  "panels": [
+    { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "title": "GPU Summary", "type": "row" },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "mappings": [],
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 }] },
+          "unit": "percentunit"
+        }
+      },
+      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 1 },
+      "id": 2,
+      "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
+      "title": "GPU Utilization",
+      "type": "stat",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_utilization_gpu_ratio", "legendFormat": "util" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "mappings": [],
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+          "unit": "bytes"
+        }
+      },
+      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 1 },
+      "id": 3,
+      "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
+      "title": "VRAM Used",
+      "description": "See the VRAM Used vs Total trend panel below for headroom against the card's total capacity.",
+      "type": "stat",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_memory_used_bytes", "legendFormat": "used" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "mappings": [],
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 75 }, { "color": "red", "value": 85 }] },
+          "unit": "celsius"
+        }
+      },
+      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 1 },
+      "id": 4,
+      "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
+      "title": "GPU Temperature",
+      "type": "stat",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_temperature_gpu", "legendFormat": "temp" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "thresholds" },
+          "mappings": [],
+          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
+          "unit": "watt"
+        }
+      },
+      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 1 },
+      "id": 5,
+      "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" },
+      "title": "Power Draw",
+      "type": "stat",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_power_draw_watts", "legendFormat": "power" }]
+    },
+
+    { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 10, "title": "AI Request Performance", "type": "row" },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "reqps" } },
+      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 6 },
+      "id": 11,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "AI Request Rate",
+      "description": "Request rate by provider and request kind (embedding vs review), derived from the new request-duration histogram's observation count.",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "sum by (provider, request_kind) (rate(gittensory_ai_provider_request_duration_seconds_count[5m]))",
+          "legendFormat": "{{provider}} / {{request_kind}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "s" } },
+      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 6 },
+      "id": 12,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "AI Request Latency (p50 / p95 / p99)",
+      "type": "timeseries",
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(gittensory_ai_provider_request_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A" },
+        { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(gittensory_ai_provider_request_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B" },
+        { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(gittensory_ai_provider_request_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C" }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "reqps" } },
+      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 6 },
+      "id": 13,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "AI Request Error Rate",
+      "description": "Excludes expected embedding-routing fallbacks (those never increment this counter -- see gittensory_ai_provider_request_errors_total's HELP text).",
+      "type": "timeseries",
+      "targets": [
+        {
+          "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+          "expr": "sum by (provider, request_kind) (rate(gittensory_ai_provider_request_errors_total[5m]))",
+          "legendFormat": "{{provider}} / {{request_kind}}",
+          "refId": "A"
+        }
+      ]
+    },
+
+    { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 20, "title": "GPU Trends", "type": "row" },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "percentunit", "max": 1, "min": 0 } },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 },
+      "id": 21,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "GPU Utilization Over Time",
+      "type": "timeseries",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_utilization_gpu_ratio", "legendFormat": "util", "refId": "A" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "bytes" } },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 },
+      "id": 22,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "VRAM Used vs Total",
+      "type": "timeseries",
+      "targets": [
+        { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_memory_used_bytes", "legendFormat": "used", "refId": "A" },
+        { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_memory_total_bytes", "legendFormat": "total", "refId": "B" }
+      ]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "celsius" } },
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 },
+      "id": 23,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "GPU Temperature Over Time",
+      "type": "timeseries",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_temperature_gpu", "legendFormat": "temp", "refId": "A" }]
+    },
+    {
+      "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
+      "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "watt" } },
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 },
+      "id": 24,
+      "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } },
+      "title": "Power Draw Over Time",
+      "type": "timeseries",
+      "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "nvidia_smi_power_draw_watts", "legendFormat": "power", "refId": "A" }]
+    }
+  ],
+  "refresh": "30s",
+  "schemaVersion": 38,
+  "tags": ["gittensory", "gpu"],
+  "templating": {
+    "list": [
+      { "current": {}, "hide": 0, "includeAll": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "refresh": 1, "type": "datasource" }
+    ]
+  },
+  "time": { "from": "now-6h", "to": "now" },
+  "timepicker": {},
+  "timezone": "browser",
+  "title": "Gittensory — GPU Metrics",
+  "uid": "gittensory-gpu",
+  "version": 1
+}

prometheus/prometheus.yml

@@ -30,6 +30,13 @@ scrape_configs:
       - targets: ["postgres-exporter:9187"]
     scrape_interval: 30s
 
+  # GPU utilization/VRAM/temp/power from nvidia_gpu_exporter (#4367). The exporter starts only with
+  # --profile gpu; on a non-GPU install this target just shows as absent/down, no alert fires on it.
+  - job_name: gpu
+    static_configs:
+      - targets: ["gpu-exporter:9835"]
+    scrape_interval: 15s
+
   # Backup freshness from the read-only backup-exporter sidecar. The metrics exist only when the backup
   # profile is active, which keeps backup alerts opt-in with the backup feature.
   - job_name: gittensory-backup

src/selfhost/ai.ts

@@ -9,7 +9,7 @@
 import type { AiContentBlock, CombineStrategy, OnMerge } from "../services/ai-review";
 import { isConfiguredSelfHostProvider, resolveConfiguredProviderNames } from "./ai-config";
 export { assertNoLegacySharedAiEnv } from "./ai-config";
-import { incr } from "./metrics";
+import { incr, observe } from "./metrics";
 import { withReviewSpan } from "./tracing";
 import { delimiter } from "node:path";
 
@@ -1096,12 +1096,18 @@ async function runProviderWithOtel(
       `circuit_open: provider "${provider.name}" is in cooldown after ${AI_PROVIDER_FAILURE_THRESHOLD} consecutive failures — skipping this attempt`,
     );
   }
+  const requestKindLabel = requestKind(options);
+  const startedAtMs = Date.now();
   try {
     const result = await withReviewSpan(
       "selfhost.ai.provider",
-      { "ai.provider": provider.name, "ai.model": model || "default", "ai.request_kind": requestKind(options) },
+      { "ai.provider": provider.name, "ai.model": model || "default", "ai.request_kind": requestKindLabel },
       () => provider.ai.run(model, options),
     );
+    observe("gittensory_ai_provider_request_duration_seconds", (Date.now() - startedAtMs) / 1000, {
+      provider: provider.name,
+      request_kind: requestKindLabel,
+    });
     aiProviderCircuits.delete(provider.name);
     if (result.usage) {
       return {
@@ -1115,8 +1121,13 @@ async function runProviderWithOtel(
     }
     return result;
   } catch (error) {
+    observe("gittensory_ai_provider_request_duration_seconds", (Date.now() - startedAtMs) / 1000, {
+      provider: provider.name,
+      request_kind: requestKindLabel,
+    });
     if (isExpectedEmbeddingRoutingError(options, error)) throw error;
     incr("gittensory_ai_provider_failures_total", { provider: provider.name });
+    incr("gittensory_ai_provider_request_errors_total", { provider: provider.name, request_kind: requestKindLabel });
     // Re-read the map here rather than reusing the `circuit` captured above: that read happened BEFORE the
     // `await` on the real provider call, so under concurrent same-provider calls it can be stale by the time
     // this catch runs, and computing `failures` from it would clobber a sibling call's write (lost-update race)

src/selfhost/metrics.ts

@@ -111,6 +111,8 @@ const DEFAULT_METRIC_META: readonly (readonly [string, MetricMeta])[] = [
   ["gittensory_ai_total_tokens_total", { help: "AI provider total tokens observed.", type: "counter" }],
   ["gittensory_ai_provider_circuit_open_total", { help: "AI provider circuit-open events.", type: "counter" }],
   ["gittensory_ai_provider_failures_total", { help: "AI provider failures by provider.", type: "counter" }],
+  ["gittensory_ai_provider_request_duration_seconds", { help: "AI provider request duration in seconds, by provider and request kind.", type: "histogram" }],
+  ["gittensory_ai_provider_request_errors_total", { help: "AI provider request errors, by provider and request kind (excludes expected embedding-routing fallbacks).", type: "counter" }],
   ["gittensory_ai_review_cache_hit_total", { help: "AI review cache hits.", type: "counter" }],
   ["gittensory_ai_review_cache_miss_total", { help: "AI review cache misses.", type: "counter" }],
   ["gittensory_ai_review_cache_write_error_total", { help: "AI review cache write errors.", type: "counter" }],