Skip to content

docs(ops): add runbooks for rate-limit, cache, Qdrant, Orb, and AI provider recovery #2638

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JSONbored merged 2 commits into from
Jul 2, 2026
Merged

docs(ops): add runbooks for rate-limit, cache, Qdrant, Orb, and AI provider recovery #2638

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6d0b065
docs(ops): add runbooks for rate-limit, cache, Qdrant, Orb, and AI pr…
JSONbored Jul 2, 2026
cc404f7
docs(nav): nest self-hosting under Maintainers, fix gate review nits
JSONbored Jul 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 95 additions & 47 deletions apps/gittensory-ui/src/components/site/docs-nav.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,12 @@ import { Link, useRouterState } from "@tanstack/react-router";
import { cn } from "@/lib/utils";

type DocsItem = { to: string; label: string };
type DocsGroup = { title: string; items: DocsItem[] };
type DocsSubgroup = { title: string; items: DocsItem[] };
// A group is either a flat list (`items`) or a nested category/sub-category/step hierarchy
// (`subgroups`) — never both. Self-hosting is deliberately nested UNDER "Maintainers" (a maintainer
// concern: running your own instance) rather than sitting as its own top-level sibling category, and
// its own pages are grouped into sub-categories instead of one long flat list.
type DocsGroup = { title: string } & ({ items: DocsItem[] } | { subgroups: DocsSubgroup[] });

export const docsNav: DocsGroup[] = [
{
Expand All @@ -21,28 +26,48 @@ export const docsNav: DocsGroup[] = [
},
{
title: "Maintainers",
items: [
{ to: "/docs/maintainer-workflow", label: "Maintainer workflow" },
{ to: "/docs/github-app", label: "GitHub App" },
{ to: "/docs/maintainer-install-trust", label: "Maintainer install & trust" },
],
},
{
title: "Self-hosting",
items: [
{ to: "/docs/maintainer-self-hosting", label: "Overview" },
{ to: "/docs/self-hosting-quickstart", label: "Quickstart" },
{ to: "/docs/self-hosting-configuration", label: "Configuration" },
{ to: "/docs/self-hosting-github-app", label: "GitHub App & Orb" },
{ to: "/docs/self-hosting-ai-providers", label: "AI providers" },
{ to: "/docs/self-hosting-rees", label: "REES enrichment" },
{ to: "/docs/self-hosting-rees-analyzers", label: "REES analyzers" },
{ to: "/docs/self-hosting-rag", label: "RAG indexing" },
{ to: "/docs/self-hosting-operations", label: "Operations" },
{ to: "/docs/self-hosting-backup-scaling", label: "Backup & scaling" },
{ to: "/docs/self-hosting-releases", label: "Releases & images" },
{ to: "/docs/self-hosting-security", label: "Security" },
{ to: "/docs/self-hosting-troubleshooting", label: "Troubleshooting" },
subgroups: [
{
title: "Hosted app",
items: [
{ to: "/docs/maintainer-workflow", label: "Maintainer workflow" },
{ to: "/docs/github-app", label: "GitHub App" },
{ to: "/docs/maintainer-install-trust", label: "Maintainer install & trust" },
],
},
{
title: "Self-hosting: setup",
items: [
{ to: "/docs/maintainer-self-hosting", label: "Overview" },
{ to: "/docs/self-hosting-quickstart", label: "Quickstart" },
{ to: "/docs/self-hosting-configuration", label: "Configuration" },
],
},
{
title: "Self-hosting: integrations",
items: [
{ to: "/docs/self-hosting-github-app", label: "GitHub App & Orb" },
{ to: "/docs/self-hosting-ai-providers", label: "AI providers" },
{ to: "/docs/self-hosting-rees", label: "REES enrichment" },
{ to: "/docs/self-hosting-rees-analyzers", label: "REES analyzers" },
{ to: "/docs/self-hosting-rag", label: "RAG indexing" },
],
},
{
title: "Self-hosting: operations",
items: [
{ to: "/docs/self-hosting-operations", label: "Operations" },
{ to: "/docs/self-hosting-backup-scaling", label: "Backup & scaling" },
{ to: "/docs/self-hosting-troubleshooting", label: "Troubleshooting" },
],
},
{
title: "Self-hosting: release & security",
items: [
{ to: "/docs/self-hosting-releases", label: "Releases & images" },
{ to: "/docs/self-hosting-security", label: "Security" },
],
},
],
},
{
Expand All @@ -64,6 +89,38 @@ export const docsNav: DocsGroup[] = [
},
];

function groupItems(group: DocsGroup): DocsItem[] {
return "items" in group ? group.items : group.subgroups.flatMap((sub) => sub.items);
}

function DocsItemList({ items, pathname }: { items: DocsItem[]; pathname: string }) {
return (
<ul className="space-y-0.5">
{items.map((it) => {
const active = pathname === it.to;
return (
<li key={it.to}>
<Link
to={it.to as "/docs"}
className={cn(
"relative block rounded-token px-3 py-1.5 text-token-sm transition-colors",
active
? "bg-mint/10 text-mint"
: "text-foreground/75 hover:bg-accent/50 hover:text-foreground",
)}
>
{active && (
<span className="absolute left-0 top-1/2 h-4 w-px -translate-y-1/2 bg-mint" />
)}
{it.label}
</Link>
</li>
);
})}
</ul>
);
}

export function DocsNav() {
const pathname = useRouterState({ select: (s) => s.location.pathname });
return (
Expand All @@ -73,29 +130,20 @@ export function DocsNav() {
<div className="mb-2 font-mono text-token-2xs uppercase tracking-wider text-muted-foreground">
{group.title}
</div>
<ul className="space-y-0.5">
{group.items.map((it) => {
const active = pathname === it.to;
return (
<li key={it.to}>
<Link
to={it.to as "/docs"}
className={cn(
"relative block rounded-token px-3 py-1.5 text-token-sm transition-colors",
active
? "bg-mint/10 text-mint"
: "text-foreground/75 hover:bg-accent/50 hover:text-foreground",
)}
>
{active && (
<span className="absolute left-0 top-1/2 h-4 w-px -translate-y-1/2 bg-mint" />
)}
{it.label}
</Link>
</li>
);
})}
</ul>
{"items" in group ? (
<DocsItemList items={group.items} pathname={pathname} />
) : (
<div className="space-y-4">
{group.subgroups.map((sub) => (
<div key={sub.title}>
<div className="mb-1 pl-3 text-token-2xs font-medium text-foreground/50">
{sub.title}
</div>
<DocsItemList items={sub.items} pathname={pathname} />
</div>
))}
</div>
)}
</div>
))}
</nav>
Expand All @@ -104,7 +152,7 @@ export function DocsNav() {

export function DocsPrevNext() {
const pathname = useRouterState({ select: (s) => s.location.pathname });
const flat = docsNav.flatMap((g) => g.items);
const flat = docsNav.flatMap(groupItems);
const idx = flat.findIndex((i) => i.to === pathname);
const prev = idx > 0 ? flat[idx - 1] : null;
const next = idx >= 0 && idx < flat.length - 1 ? flat[idx + 1] : null;
Expand Down
149 changes: 147 additions & 2 deletions apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ export const Route = createFileRoute("/docs/self-hosting-troubleshooting")({
{
name: "description",
content:
"Troubleshoot self-hosted Gittensory reviews: webhook delivery, AI unavailable, REES silent, RAG empty, queue stuck, and readiness failures.",
"Troubleshoot self-hosted Gittensory reviews: webhook delivery, AI unavailable, REES silent, RAG empty, queue stuck, GitHub rate limits, Qdrant, Orb, AI provider circuit breakers, and readiness failures.",
},
{ property: "og:title", content: "Self-host troubleshooting — Gittensory docs" },
{
property: "og:description",
content:
"Troubleshoot self-hosted Gittensory reviews: webhook delivery, AI unavailable, REES silent, RAG empty, queue stuck, and readiness failures.",
"Troubleshoot self-hosted Gittensory reviews: webhook delivery, AI unavailable, REES silent, RAG empty, queue stuck, GitHub rate limits, Qdrant, Orb, AI provider circuit breakers, and readiness failures.",
},
{ property: "og:url", content: "/docs/self-hosting-troubleshooting" },
],
Expand Down Expand Up @@ -121,6 +121,151 @@ rees_analyzer_config_invalid`}
docker compose logs gittensory | grep selfhost_job_dead`}
/>

<h2>GitHub rate-limit responses or admission deferrals</h2>
<p>
Two independent signals cover this:{" "}
<code>gittensory_github_rest_rate_limit_responses_total</code> counts actual 403/429
responses from GitHub, and the{" "}
<code>gittensory_jobs_rate_limit_admission_deferred_total</code> /{" "}
<code>gittensory_jobs_rate_limit_budget_deferred_total</code> /{" "}
<code>gittensory_jobs_rate_limited_by_type_total</code> counters track jobs the queue itself
held back <em>before</em> making a request, to avoid tripping a limit. All three job-side
counters carry the same three labels — <code>kind</code> (<code>webhook</code> or{" "}
<code>background</code>), <code>key_scope</code> (<code>installation</code>,{" "}
<code>public</code>, <code>global</code>, or <code>other</code>), and <code>job_type</code>{" "}
(the queue job's type, e.g. <code>agent-regate-pr</code>) — so you can break a spike down to
exactly which token pool and which job type is under pressure.
</p>
<p>
A short burst of deferrals is expected and self-resolving: the queue is deliberately trading
a few seconds of delay to avoid a real 429. Treat it as a real problem only once it&apos;s
<strong> sustained</strong> — which is exactly what{" "}
<code>GittensoryGitHubRateLimitResponses</code> (real 403/429s observed) and{" "}
<code>GittensoryQueueRateLimitDeferralsHigh</code> (a sustained deferral rate, not a blip)
are tuned to alert on, rather than firing on every brief admission hold.
</p>
<CodeBlock
lang="promql"
code={`# Deferrals broken down by token pool and job type over the last 10m
sum by (key_scope, job_type) (rate(gittensory_jobs_rate_limit_admission_deferred_total[10m]))

# Is one key_scope (e.g. a single installation token) the bottleneck?
topk(5, sum by (key_scope) (rate(gittensory_jobs_rate_limit_budget_deferred_total[10m])))

# Real rate-limit responses from GitHub itself (not just internal deferrals)
sum(rate(gittensory_github_rest_rate_limit_responses_total[10m]))`}
/>
<p>
If a single <code>key_scope=installation</code> pool is consistently the bottleneck, the fix
is usually spreading load across more installation tokens (fewer repos per installation) or
raising the GitHub App&apos;s own rate-limit tier, not code changes here.
</p>

<h2>Low GitHub response-cache hit rate</h2>
<p>
<code>gittensory_github_response_cache_total</code> (REST) and{" "}
<code>gittensory_github_graphql_cache_total</code> (GraphQL) both carry a{" "}
<code>result</code> label — <code>hit</code>, <code>miss</code>, <code>set</code>,{" "}
<code>coalesced</code>, <code>bypassed</code>, or <code>error</code> — and a{" "}
<code>class</code> label identifying the endpoint family. A healthy cache should show most
traffic as <code>hit</code> for endpoints that are read repeatedly in one review/maintenance
pass (PR reads, check-run lookups); a low hit rate on those specific classes, not the
overall average, is the useful signal.
</p>
<CodeBlock
lang="promql"
code={`# REST hit rate by endpoint class over the last 15m
sum by (class) (rate(gittensory_github_response_cache_total{result="hit"}[15m]))
/
sum by (class) (rate(gittensory_github_response_cache_total[15m]))

# GraphQL hit rate — same shape, separate metric
sum by (class) (rate(gittensory_github_graphql_cache_total{result="hit"}[15m]))
/
sum by (class) (rate(gittensory_github_graphql_cache_total[15m]))`}
/>

<h2>Qdrant / vector-store errors</h2>
<p>
<code>gittensory_qdrant_errors_total</code> carries an <code>op</code> label (
<code>upsert</code>, <code>query</code>, or <code>delete</code>) so you can tell whether
indexing or retrieval is failing. <code>GittensoryQdrantErrorRateHigh</code> fires on a
sustained error ratio, not an isolated blip.
</p>
<ul>
<li>
Confirm <code>QDRANT_URL</code> (e.g. <code>http://qdrant:6333</code>) is reachable from
the app container and the <code>qdrant</code> Compose profile is running.
</li>
<li>
If Qdrant requires auth, confirm <code>QDRANT_API_KEY</code> is set and matches the Qdrant
deployment&apos;s configuration.
</li>
<li>
A dimension-mismatch error means the existing <code>gittensory</code> collection (the
fixed collection name self-host always uses) was created with a different embedding model
than the one currently configured (<code>AI_EMBED_MODEL</code>). Recreating it — delete
the collection and let the next index run recreate it at the current width — is the fix,
but it temporarily removes ALL indexed RAG context for every repo until re-indexing
completes, so treat it as a deliberate, disruptive step, not a routine one.
</li>
</ul>
<CodeBlock
lang="bash"
code={`curl "$QDRANT_URL/collections/gittensory"
docker compose --profile qdrant ps qdrant

# Only after confirming a dimension mismatch is the actual cause:
curl -X DELETE "$QDRANT_URL/collections/gittensory"`}
/>

<h2>Orb export or relay problems</h2>
<p>
For brokered self-host deployments, <code>gittensory_orb_events_exported_total</code> and{" "}
<code>gittensory_orb_export_errors_total</code> track the hourly outcome-export loop;{" "}
<code>GittensoryOrbExportErrorRateHigh</code> fires on a sustained error ratio there. The
pull-mode relay loop (for installations receiving events outbound from Orb) reports through{" "}
<code>gittensory_orb_relay_drains_total</code> (<code>result=events</code> when it drained
something, <code>result=empty</code> otherwise) and{" "}
<code>gittensory_orb_webhook_total</code> (<code>event</code> + <code>result</code> labels)
for what happened to each relayed event once enqueued locally.
</p>
<p>
If exports are failing but the relay itself looks healthy, the export loop&apos;s Sentry
cron monitor (see <Link to="/docs/self-hosting-operations">Self-host operations</Link>) is
the fastest way to confirm whether the loop is even running, before digging into the error
counters.
</p>

<h2>AI provider circuit breaker keeps opening</h2>
<p>
Each AI provider (self-host <code>AI_PROVIDER</code> entries) has its own circuit breaker:
after 3 consecutive failures it stops attempting real calls to that provider for 60 seconds,
recorded as <code>gittensory_ai_provider_circuit_open_total{'{provider="..."}'}</code>{" "}
(skipped calls) alongside{" "}
<code>gittensory_ai_provider_failures_total{'{provider="..."}'}</code> (real failures). It
self-heals automatically — there is no manual reset — but it will reopen immediately if the
underlying problem is still there.
</p>
<ul>
<li>
Search logs for <code>circuit_open: provider "..."</code> to confirm which provider
tripped, and <code>selfhost_ai_provider_failed_in_chain</code> for the real error each
failed attempt hit before the breaker opened.
</li>
<li>
A provider that keeps re-tripping after its cooldown almost always means a persistent
problem, not a transient blip: an expired/invalid API key, a CLI binary missing from the
image (see <code>selfhost_ai_cli_missing</code> at boot), or the endpoint being genuinely
unreachable from the container.
</li>
<li>
<code>GittensoryAiProviderCircuitOpen</code> fires on any circuit-open event in a
15-minute window — a single trip during a real but brief outage is expected; a rule that
keeps firing across multiple windows points at the persistent case above.
</li>
</ul>

<h2>Grafana traces error or show no data</h2>
<p>
The trace path is app or smoke process → OTEL collector → Tempo → Grafana. Tempo is only
Expand Down
42 changes: 42 additions & 0 deletions test/unit/docs-selfhost-troubleshooting-metric-names.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";

// Drift guard (#1943 gate review finding): the self-hosting troubleshooting runbooks reference exact
// Prometheus metric names and alert names. If a metric is ever renamed/removed in src/, or an alert is
// renamed/removed in prometheus/rules/alerts.yml, this test fails instead of the docs silently going stale
// — mirrors the same source-of-truth-diff approach as scripts/check-openapi-settings-parity.mjs (#2556).

const DOC_PATH = "apps/gittensory-ui/src/routes/docs.self-hosting-troubleshooting.tsx";
const doc = readFileSync(DOC_PATH, "utf8");

// The exact source files that emit every gittensory_*_total metric referenced in the runbooks, per an
// audit against the real incr()/gauge()/observe() call sites (src/selfhost/metrics.ts's API).
const METRIC_SOURCE_FILES = [
"src/github/client.ts",
"src/github/graphql-cache.ts",
"src/selfhost/queue-common.ts",
"src/selfhost/sqlite-queue.ts",
"src/selfhost/pg-queue.ts",
"src/selfhost/qdrant-vectorize.ts",
"src/selfhost/orb-collector.ts",
"src/selfhost/monitored-work.ts",
"src/selfhost/ai.ts",
];
const metricSource = METRIC_SOURCE_FILES.map((path) => readFileSync(path, "utf8")).join("\n");
const alertsSource = readFileSync("prometheus/rules/alerts.yml", "utf8");

describe("self-hosting-troubleshooting doc: metric/alert names match source (#1943)", () => {
it("every gittensory_..._total metric name referenced in the doc is actually emitted by the code", () => {
const names = [...new Set([...doc.matchAll(/gittensory_[a-z0-9_]+_total/g)].map((m) => m[0]))];
expect(names.length).toBeGreaterThan(5); // sanity: the extraction found the runbooks' real content
const missing = names.filter((name) => !metricSource.includes(name));
expect(missing).toEqual([]);
});

it("every GittensoryXxx alert name referenced in the doc exists in prometheus/rules/alerts.yml", () => {
const names = [...new Set([...doc.matchAll(/Gittensory[A-Za-z]+/g)].map((m) => m[0]))];
expect(names.length).toBeGreaterThan(2);
const missing = names.filter((name) => !alertsSource.includes(`alert: ${name}`));
expect(missing).toEqual([]);
});
});
Loading