diff --git a/grafana/dashboards/rees-metrics.json b/grafana/dashboards/rees-metrics.json new file mode 100644 index 0000000000..6937b29fd4 --- /dev/null +++ b/grafana/dashboards/rees-metrics.json @@ -0,0 +1,101 @@ +{ + "__inputs": [], + "__requires": [ + { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" }, + { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } + ], + "annotations": { "list": [] }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "description": "REES (review-enrichment service, --profile rees, #5367) request outcomes and per-analyzer health โ€” request rate/latency by outcome status, plus which of the 53 registered analyzers are actually running, timing out, or degrading. Absent/empty unless the optional rees profile is active.", + "panels": [ + { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "title": "Request outcomes (/v1/enrich)", "type": "row" }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "reqps" } }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, + "id": 2, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "title": "Request rate by outcome status", + "description": "status โˆˆ ok/empty/http_error/unauthorized/service_not_configured/bad_request.", + "type": "timeseries", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (status) (rate(rees_enrich_requests_total[5m]))", "legendFormat": "{{status}}", "refId": "A" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "s" } }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 }, + "id": 3, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "title": "Request latency (p50 / p95 / p99)", + "type": "timeseries", + "targets": [ + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.50, sum by (le) (rate(rees_enrich_request_duration_seconds_bucket[5m])))", "legendFormat": "p50", "refId": "A" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.95, sum by (le) (rate(rees_enrich_request_duration_seconds_bucket[5m])))", "legendFormat": "p95", "refId": "B" }, + { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "histogram_quantile(0.99, sum by (le) (rate(rees_enrich_request_duration_seconds_bucket[5m])))", "legendFormat": "p99", "refId": "C" } + ] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "reqps" } }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 }, + "id": 4, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "title": "Non-ok outcome rate by status", + "description": "Excludes the healthy 'ok' status -- a non-zero, sustained rate here means enrichment is silently degrading (the engine still proceeds without a brief; see enrichment-wire.ts's fail-safe design).", + "type": "timeseries", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (status) (rate(rees_enrich_requests_total{status!=\"ok\"}[5m]))", "legendFormat": "{{status}}", "refId": "A" }] + }, + + { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, "id": 10, "title": "Analyzer health (53 registered analyzers)", "type": "row" }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "reqps" } }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 10 }, + "id": 11, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "title": "Top analyzers by run rate", + "type": "timeseries", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "topk(10, sum by (analyzer) (rate(rees_analyzer_runs_total[5m])))", "legendFormat": "{{analyzer}}", "refId": "A" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "reqps" } }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 10 }, + "id": 12, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "title": "Analyzer outcomes by status", + "description": "status โˆˆ ok/degraded/timeout/capped/skipped, summed across every analyzer.", + "type": "timeseries", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "sum by (status) (rate(rees_analyzer_runs_total[5m]))", "legendFormat": "{{status}}", "refId": "A" }] + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "lineWidth": 2, "fillOpacity": 10 }, "unit": "s" } }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 10 }, + "id": 13, + "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi", "sort": "desc" } }, + "title": "Slowest analyzers (p95 duration)", + "description": "Only analyzers that actually ran (ok/degraded/timeout) contribute a duration sample -- a skipped/capped analyzer never invoked the analyzer function.", + "type": "timeseries", + "targets": [{ "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "expr": "topk(10, histogram_quantile(0.95, sum by (le, analyzer) (rate(rees_analyzer_duration_seconds_bucket[5m]))))", "legendFormat": "{{analyzer}}", "refId": "A" }] + } + ], + "refresh": "30s", + "schemaVersion": 38, + "tags": ["gittensory", "rees", "observability"], + "templating": { + "list": [ + { "current": {}, "hide": 0, "includeAll": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "refresh": 1, "type": "datasource" } + ] + }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "Gittensory โ€” REES (review-enrichment)", + "uid": "gittensory-rees", + "version": 1 +} diff --git a/grafana/dashboards/resource-hub.json b/grafana/dashboards/resource-hub.json index ffec2fc46c..c20c3eb4a8 100644 --- a/grafana/dashboards/resource-hub.json +++ b/grafana/dashboards/resource-hub.json @@ -66,7 +66,7 @@ "gridPos": { "h": 13, "w": 12, "x": 12, "y": 0 }, "options": { "mode": "markdown", - "content": "## ๐Ÿ“Š Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** โ€” live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** โ€” gittensory's own review activity + reviewed-PR log.\n- **[AI usage](/d/gittensory-ai-usage)** โ€” durable cross-provider ai_usage_events (filterable by provider/feature/model), live Prometheus counters, and Claude Code's own OTEL session telemetry, all in one place.\n- **[Gittensory (infra)](/d/gittensory)** โ€” queue, jobs, HTTP, GitHub API cache/rate limits.\n- **[GPU metrics](/d/gittensory-gpu)** โ€” utilization/VRAM for a self-hosted Ollama GPU box.\n- **[Infra health](/d/gittensory-infra-health)** โ€” host CPU/mem/disk/network (node-exporter), per-container resource usage (cAdvisor), Redis, Qdrant, and whether the observability stack itself is up.\n\n## ๐Ÿ“ˆ Metrics & logs\n- **Prometheus** โ€” [targets](http://localhost:9090/targets) ยท [graph](http://localhost:9090)\n- **Alertmanager** โ€” [alerts](http://localhost:9093)\n- **Loki** โ€” query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n- **Sentry** โ€” release/source-map enriched errors. Edit the dashboard link if your project URL differs.\n\n## ๐Ÿฉบ Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` โ†’ 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | **[AI usage](/d/gittensory-ai-usage)** dashboard |\n\n## ๐Ÿ“š Docs\n- [Maintainer self-hosting](https://gittensory.aethereal.dev/docs/maintainer-self-hosting) โ€” setup, configuration, AI, REES, RAG, operations, and troubleshooting." + "content": "## ๐Ÿ“Š Dashboards\n- **[Upstream PRs & issues (GitHub)](/d/gittensory-github)** โ€” live, accurate census + open-PR triage (GitHub API).\n- **[Reviews & PRs (maintainer)](/d/gittensory-maintainer)** โ€” gittensory's own review activity + reviewed-PR log.\n- **[AI usage](/d/gittensory-ai-usage)** โ€” durable cross-provider ai_usage_events (filterable by provider/feature/model), live Prometheus counters, and Claude Code's own OTEL session telemetry, all in one place.\n- **[Gittensory (infra)](/d/gittensory)** โ€” queue, jobs, HTTP, GitHub API cache/rate limits.\n- **[GPU metrics](/d/gittensory-gpu)** โ€” utilization/VRAM for a self-hosted Ollama GPU box.\n- **[Infra health](/d/gittensory-infra-health)** โ€” host CPU/mem/disk/network (node-exporter), per-container resource usage (cAdvisor), Redis, Qdrant, and whether the observability stack itself is up.\n- **[REES (review-enrichment)](/d/gittensory-rees)** โ€” request outcomes/latency and per-analyzer run/timeout/degrade rates for the optional `--profile rees` service.\n\n## ๐Ÿ“ˆ Metrics & logs\n- **Prometheus** โ€” [targets](http://localhost:9090/targets) ยท [graph](http://localhost:9090)\n- **Alertmanager** โ€” [alerts](http://localhost:9093)\n- **Loki** โ€” query in [Explore](/explore) (pick the *Loki* datasource), e.g. `{compose_service=\"gittensory\"}`\n- **Sentry** โ€” release/source-map enriched errors. Edit the dashboard link if your project URL differs.\n\n## ๐Ÿฉบ Quick health checks\n| What | Where |\n|---|---|\n| App serving | `GET /ready` โ†’ 200 |\n| AI wired | boot log `selfhost_ai_provider` |\n| Embeds wired | boot log `selfhost_embed_provider` |\n| Vectors wired | boot log `selfhost_vectorize` |\n| Token spend | **[AI usage](/d/gittensory-ai-usage)** dashboard |\n\n## ๐Ÿ“š Docs\n- [Maintainer self-hosting](https://gittensory.aethereal.dev/docs/maintainer-self-hosting) โ€” setup, configuration, AI, REES, RAG, operations, and troubleshooting." } } ] diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index ed3913a350..a664e2af09 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -83,6 +83,13 @@ scrape_configs: - targets: ["qdrant:6333"] scrape_interval: 30s + # REES (review-enrichment service) request + per-analyzer metrics (--profile rees, #5367). Absent/down + # unless the optional REES analyzer service is active โ€” same "other profile" gating as postgres/gpu/qdrant. + - job_name: rees + static_configs: + - targets: ["rees:8080"] + scrape_interval: 15s + # Self-monitoring: the observability stack watching itself (--profile observability, #5366). Every # component here ships its own native Prometheus /metrics โ€” previously scraped by nothing, so a stack # component silently degrading (e.g. Loki dropping ingested lines, Alertmanager failing to notify) had diff --git a/test/unit/selfhost-grafana-rees-dashboard.test.ts b/test/unit/selfhost-grafana-rees-dashboard.test.ts new file mode 100644 index 0000000000..530022e5bc --- /dev/null +++ b/test/unit/selfhost-grafana-rees-dashboard.test.ts @@ -0,0 +1,73 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +type DashboardTarget = { expr?: string; legendFormat?: string }; +type DashboardPanel = { + id?: number; + title?: string; + type?: string; + description?: string; + datasource?: { type?: string; uid?: string }; + targets?: DashboardTarget[]; +}; +type Dashboard = { + uid: string; + title: string; + tags: string[]; + panels: DashboardPanel[]; +}; + +const dashboardPath = join(process.cwd(), "grafana/dashboards/rees-metrics.json"); + +function readDashboard(): Dashboard { + return JSON.parse(readFileSync(dashboardPath, "utf8")) as Dashboard; +} + +function allTargets(dashboard = readDashboard()): DashboardTarget[] { + return dashboard.panels.flatMap((panel) => panel.targets ?? []); +} + +describe("Gittensory โ€” REES (review-enrichment) dashboard (#5367)", () => { + it("declares the expected uid/title/tags", () => { + const dashboard = readDashboard(); + expect(dashboard.uid).toBe("gittensory-rees"); + expect(dashboard.title).toBe("Gittensory โ€” REES (review-enrichment)"); + expect(dashboard.tags).toEqual(["gittensory", "rees", "observability"]); + }); + + it("every panel target uses the Prometheus datasource variable, never a hardcoded uid", () => { + const dashboard = readDashboard(); + for (const panel of dashboard.panels) { + if (panel.type === "row") continue; + expect(panel.datasource?.type, panel.title).toBe("prometheus"); + expect(panel.datasource?.uid, panel.title).toBe("${DS_PROMETHEUS}"); + for (const target of panel.targets ?? []) { + expect(target.expr, panel.title).toBeTruthy(); + } + } + }); + + it("uses the real metric names REES's own metrics.ts registers (rees_enrich_requests_total/rees_enrich_request_duration_seconds/rees_analyzer_runs_total/rees_analyzer_duration_seconds)", () => { + const targets = allTargets(); + expect(targets.some((t) => t.expr?.includes("rees_enrich_requests_total"))).toBe(true); + expect(targets.some((t) => t.expr?.includes("rees_enrich_request_duration_seconds_bucket"))).toBe(true); + expect(targets.some((t) => t.expr?.includes("rees_analyzer_runs_total"))).toBe(true); + expect(targets.some((t) => t.expr?.includes("rees_analyzer_duration_seconds_bucket"))).toBe(true); + }); + + it("excludes the healthy 'ok' status from the non-ok outcome-rate panel", () => { + const target = allTargets().find((t) => t.expr?.includes("rees_enrich_requests_total{status")); + expect(target?.expr).toContain('status!="ok"'); + }); + + it("bounds every panel that breaks down BY analyzer with topk, so a future analyzer influx can't unbound the panel legend", () => { + // Deliberately excludes the "outcomes by status" panel: it groups by the fixed 5-value AnalyzerStatus + // enum, not by analyzer name, so it can never grow unbounded and needs no topk. + const perAnalyzerTargets = allTargets().filter((t) => t.expr?.includes("by (analyzer)") || t.expr?.includes("by (le, analyzer)")); + expect(perAnalyzerTargets.length).toBeGreaterThan(0); + for (const target of perAnalyzerTargets) { + expect(target.expr).toContain("topk("); + } + }); +}); diff --git a/test/unit/selfhost-observability-config.test.ts b/test/unit/selfhost-observability-config.test.ts index 0829bc5902..d4d578fd43 100644 --- a/test/unit/selfhost-observability-config.test.ts +++ b/test/unit/selfhost-observability-config.test.ts @@ -253,4 +253,22 @@ describe("self-host observability trace config", () => { // forwarded Claude Code OTLP data) -- both are scraped, but they carry structurally different metrics. expect(record(collector.exporters).prometheus.endpoint).toBe("0.0.0.0:8889"); }); + + it("scrapes REES's own /metrics, gated on the optional rees profile (#5367)", () => { + const compose = record(readYaml("docker-compose.yml")); + const rees = record(record(compose.services).rees); + const prometheus = record(readYaml("prometheus/prometheus.yml")); + const scrapeConfigs = prometheus.scrape_configs as Array>; + + expect(rees.profiles).toEqual(["rees"]); + expect(rees.expose).toEqual(["8080"]); + expect(scrapeConfigs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + job_name: "rees", + static_configs: [{ targets: ["rees:8080"] }], + }), + ]), + ); + }); });